- refactor regex handling for the different usecases to allow providing the preferred method

- introduce new `exec` and `execAll` variants which support named groups
- introduce `replaceAll` in addition to `replace`
- update test cases
- cleanup code

__tests__/regexUtils.test.ts

@@ -0,0 +1,57 @@
+import { transformStringToValue, validateRegex } from '../src/pr-collector/regexUtils'
+import { Regex } from '../src/pr-collector/types'
+
+jest.setTimeout(180000)
+
+it('Replace into target', async () => {
+  const regex: Regex = {
+    pattern: '.*(\\[Feature\\]|\\[Issue\\]).*',
+    target: '$1',
+  }
+  const validatedRegex = validateRegex(regex)
+  expect(validateRegex).not.toBeNull()
+  expect(transformStringToValue("[Feature] TEST", validatedRegex!!)).toStrictEqual(`[Feature]`)
+})
+
+it('Replace all into target', async () => {
+  const regex: Regex = {
+    pattern: '.*(\\[Feature\\]|\\[Issue\\]).*',
+    method: 'replaceAll',
+    target: '$1',
+  }
+  const validatedRegex = validateRegex(regex)
+  expect(validateRegex).not.toBeNull()
+  expect(transformStringToValue("[Feature] TEST", validatedRegex!!)).toStrictEqual(`[Feature]`)
+})
+
+it('Match without target', async () => {
+  const regex: Regex = {
+    pattern: '\\[Feature\\]|\\[Issue\\]',
+    method: 'match'
+  }
+  const validatedRegex = validateRegex(regex)
+  expect(validateRegex).not.toBeNull()
+  expect(transformStringToValue("[Feature] TEST", validatedRegex!!)).toStrictEqual(`[Feature]`)
+})
+
+it('Match into target', async () => {
+  const regex: Regex = {
+    pattern: '(?<label>\\[Feature\\]|\\[Issue\\])',
+    method: 'match',
+    target: '$1',
+  }
+  const validatedRegex = validateRegex(regex)
+  expect(validateRegex).not.toBeNull()
+  expect(transformStringToValue("[Feature] TEST", validatedRegex!!)).toStrictEqual(`[Feature]`)
+})
+
+it('Match into named group', async () => {
+  const regex: Regex = {
+    pattern: '(?<label>\\[Feature\\]|\\[Issue\\])',
+    method: 'match',
+    target: 'label',
+  }
+  const validatedRegex = validateRegex(regex)
+  expect(validateRegex).not.toBeNull()
+  expect(transformStringToValue("[Feature] TEST", validatedRegex!!)).toStrictEqual(`[Feature]`)
+})

__tests__/tags.test.ts

@@ -1,4 +1,5 @@
-import { validateTransformer } from '../src/pr-collector/regexUtils'
+import { TagResolver } from '../src/configuration'
+import { validateRegex } from '../src/pr-collector/regexUtils'
 import {filterTags, prepareAndSortTags, TagInfo, transformTags} from '../src/pr-collector/tags'
 
 jest.setTimeout(180000)
@@ -100,14 +101,16 @@ it('Should filter tags correctly using the regex', async () => {
     {name: '20.0.2', commit: ''}
   ]
 
-  const tagResolver = {
+  const tagResolver: TagResolver = {
     method: 'non-existing-method',
     filter: {
       pattern: 'api-(.+)',
+      method: 'match',
       flags: 'gu'
     }
   }
-  const filtered = filterTags(tags, tagResolver)
+  const filter = validateRegex(tagResolver.filter)
+  const filtered = filterTags(tags, filter)
     .map(function (tag) {
       return tag.name
     })
@@ -131,14 +134,16 @@ it('Should filter tags correctly using the regex (inverse)', async () => {
     {name: '20.0.2', commit: ''}
   ]
 
-  const tagResolver = {
+  const tagResolver: TagResolver = {
     method: 'non-existing-method',
     filter: {
       pattern: '^(?!\\w+-)(.+)',
+      method: 'match',
       flags: 'gu'
     }
   }
-  const filtered = filterTags(tags, tagResolver)
+  const filter = validateRegex(tagResolver.filter)
+  const filtered = filterTags(tags, filter)
     .map(function (tag) {
       return tag.name
     })
@@ -160,7 +165,7 @@ it('Should transform tags correctly using the regex', async () => {
     {name: '20.0.2', commit: ''}
   ]
 
-  const tagResolver = {
+  const tagResolver: TagResolver = {
     method: 'non-existing-method',
     transformer: {
       pattern: '(api\-)?(.+)',
@@ -168,7 +173,7 @@ it('Should transform tags correctly using the regex', async () => {
     }
   }
 
-  const transformer = validateTransformer(tagResolver.transformer)
+  const transformer = validateRegex(tagResolver.transformer)
   if(transformer != null) {
     const transformed = transformTags(tags, transformer)
       .map(function (tag) {

src/configuration.ts

@@ -1,4 +1,4 @@
-import {Extractor, PullConfiguration, Regex, Rule, Sort, Transformer} from './pr-collector/types'
+import {Extractor, PullConfiguration, Regex, Rule, Sort} from './pr-collector/types'
 
 export interface Configuration extends PullConfiguration {
   max_tags_to_fetch: number
@@ -14,7 +14,7 @@ export interface Configuration extends PullConfiguration {
   label_extractor: Extractor[]
   duplicate_filter?: Extractor // extract an identifier from a PR used to detect duplicates, will keep the last match (depends on `sort`)
   reference?: Extractor // extracts a reference from a PR, used to establish parent child relations. This will remove the child from the main PR list.
-  transformers: Transformer[]
+  transformers: Regex[]
   tag_resolver: TagResolver
   base_branches: string[]
   custom_placeholders?: Placeholder[]
@@ -51,13 +51,13 @@ export type Property =
 export interface TagResolver {
   method: string // semver, sort
   filter?: Regex // the regex to filter the tags, prior to sorting
-  transformer?: Transformer // transforms the tag name using the regex, run after the filter
+  transformer?: Regex // transforms the tag name using the regex, run after the filter
 }
 
 export interface Placeholder {
   name: string // the name of the new placeholder
   source: string // the src placeholder which will be used to apply the transformer on
-  transformer: Transformer // the transformer to use to transform the original placeholder into the custom placheolder
+  transformer: Regex // the transformer to use to transform the original placeholder into the custom placheolder
 }
 
 export const DefaultConfiguration: Configuration = {

src/pr-collector/regexUtils.ts

@@ -1,25 +1,17 @@
 import * as core from '@actions/core'
-import {Extractor, Property, Regex, RegexTransformer, Transformer} from './types'
+import {Extractor, Property, Regex, RegexTransformer} from './types'
 
-export function validateTransformer(transformer?: Regex): RegexTransformer | null {
-  if (transformer === undefined) {
+export function validateRegex(regex?: Regex): RegexTransformer | null {
+  if (regex === undefined) {
     return null
   }
   try {
-    let target = undefined
-    if (transformer.hasOwnProperty('target')) {
-      target = (transformer as Transformer).target
-    }
-
+    const target = regex.target
+    const method = regex.method
+    const onEmpty = regex.on_empty
     let onProperty = undefined
-    let method = undefined
-    let onEmpty = undefined
-    if (transformer.hasOwnProperty('method')) {
-      method = (transformer as Extractor).method
-      onEmpty = (transformer as Extractor).on_empty
-      onProperty = (transformer as Extractor).on_property
-    } else if (transformer.hasOwnProperty('on_property')) {
-      onProperty = (transformer as Extractor).on_property
+    if (regex.hasOwnProperty('on_property')) {
+      onProperty = (regex as Extractor).on_property
     }
     // legacy handling, transform single value input to array
     if (!Array.isArray(onProperty)) {
@@ -28,9 +20,9 @@ export function validateTransformer(transformer?: Regex): RegexTransformer | nul
       }
     }
 
-    return buildRegex(transformer, target, onProperty, method, onEmpty)
+    return buildRegex(regex, target, onProperty, method, onEmpty)
   } catch (e) {
-    core.warning(`⚠️ Failed to validate transformer: ${transformer.pattern}`)
+    core.warning(`⚠️ Failed to validate transformer: ${regex.pattern}`)
     return null
   }
 }
@@ -42,7 +34,7 @@ export function buildRegex(
   regex: Regex,
   target: string | undefined,
   onProperty?: Property[] | undefined,
-  method?: 'replace' | 'match' | undefined,
+  method?: 'replace' | 'replaceAll' | 'match' | 'exec' | 'execAll' | undefined,
   onEmpty?: string | undefined
 ): RegexTransformer | null {
   try {
@@ -58,3 +50,95 @@ export function buildRegex(
     return null
   }
 }
+
+// eslint-disable-next-line no-undef
+export function applyCaptureGroup(value: RegExpMatchArray, target: string): string | null {
+  const groups = value['groups']
+  if (groups) {
+    const matched = groups[target]
+    if (matched) {
+      // if we had a perfect group match return that.
+      return matched
+    }
+  }
+
+  if (target.startsWith('$') && !target.startsWith('$$')) {
+    // if we start with $ offer support for matching index based capture groups
+    const index = Number(target.substring(1))
+    if (!isNaN(index) && index < value.length) {
+      return value[index]
+    }
+  }
+
+  return null
+}
+
+export function transformStringToValues(value: string, extractor: RegexTransformer): string[] | null {
+  if (extractor.pattern == null) {
+    return null
+  }
+
+  if (extractor.method === 'exec' || extractor.method === 'execAll') {
+    // eslint-disable-next-line no-undef
+    let matches: RegExpMatchArray | null
+    const result: Set<string> = new Set()
+    // match regex to all occurrences in the string if we run `execAll`
+    // otherwise just do the first match with exec
+    do {
+      matches = extractor.pattern.exec(value)
+      if (matches) {
+        if (extractor.target) {
+          const matchedGroup = applyCaptureGroup(matches, extractor.target)
+          if (matchedGroup) {
+            result.add(matchedGroup)
+          }
+        } else {
+          for (const match of matches) {
+            result.add(match)
+          }
+        }
+      }
+    } while (matches && extractor.method === 'execAll')
+    if (result.size > 0) {
+      return [...result]
+    }
+  } else if (extractor.method === 'match') {
+    const matches = value.match(extractor.pattern)
+    if (matches !== null && matches.length > 0) {
+      if (extractor.target) {
+        const matchedGroup = applyCaptureGroup(matches, extractor.target)
+        if (matchedGroup) {
+          return [matchedGroup]
+        }
+      }
+      return matches.map(match => match || '')
+    }
+  } else if (extractor.method === 'replaceAll') {
+    const match = value.replaceAll(extractor.pattern, extractor.target)
+    if (match !== '') {
+      return [match]
+    }
+  } else {
+    const match = value.replace(extractor.pattern, extractor.target)
+    if (match !== '') {
+      return [match]
+    }
+  }
+  if (extractor.onEmpty !== undefined) {
+    return [extractor.onEmpty]
+  }
+  return null
+}
+
+export function transformStringToOptionalValue(value: string, extractor: RegexTransformer): string | null {
+  const result = transformStringToValues(value, extractor)
+  if (result != null && result.length > 0) {
+    return result[0]
+  } else {
+    return null
+  }
+}
+
+export function transformStringToValue(value: string, extractor: RegexTransformer): string {
+  return transformStringToOptionalValue(value, extractor) || ''
+}

src/pr-collector/tags.ts

@@ -2,10 +2,10 @@ import * as core from '@actions/core'
 import * as github from '@actions/github'
 import * as semver from 'semver'
 import {SemVer} from 'semver'
-import {RegexTransformer, TagResolver, Transformer} from './types'
+import {Regex, RegexTransformer, TagResolver} from './types'
 import {createCommandManager} from './gitHelper'
 import moment from 'moment'
-import {validateTransformer} from './regexUtils'
+import {transformStringToOptionalValue, transformStringToValue, validateRegex} from './regexUtils'
 import {BaseRepository} from '../repositories/BaseRepository'
 
 export interface TagResult {
@@ -88,15 +88,17 @@ export class Tags {
     let tags: TagInfo[] = []
 
     if (!toTag || !fromTag) {
+      const filterRegex = validateRegex(tagResolver.filter)
+
       // filter out tags not matching the specified filter
       const filteredTags = filterTags(
         // retrieve the tags from the API
         await this.getTags(owner, repo, maxTagsToFetch),
-        tagResolver
+        filterRegex
       )
 
       // check if a transformer, legacy handling, transform single value input to array
-      let tagTransfomers: Transformer[] | undefined = undefined
+      let tagTransfomers: Regex[] | undefined = undefined
       if (tagResolver.transformer !== undefined) {
         if (!Array.isArray(tagResolver.transformer)) {
           tagTransfomers = [tagResolver.transformer]
@@ -109,7 +111,7 @@ export class Tags {
       let transformedTags: TagInfo[] = filteredTags
       if (tagTransfomers !== undefined && tagTransfomers.length > 0) {
         for (const transformer of tagTransfomers) {
-          const tagTransformer = validateTransformer(transformer)
+          const tagTransformer = validateRegex(transformer)
           if (tagTransformer != null) {
             core.debug(`ℹ️ Using configured tagTransformer (${transformer.pattern})`)
             transformedTags = transformTags(transformedTags, tagTransformer)
@@ -196,11 +198,9 @@ export class Tags {
  * Uses the provided filter (if available) to filter out any tags not currently relevant.
  * https://github.com/mikepenz/release-changelog-builder-action/issues/566
  */
-export function filterTags(tags: TagInfo[], tagResolver: TagResolver): TagInfo[] {
-  const filter = tagResolver.filter
-  if (filter !== undefined) {
-    const regex = new RegExp(filter.pattern.replace('\\\\', '\\'), filter.flags ?? 'gu')
-    const filteredTags = tags.filter(tag => tag.name.match(regex) !== null)
+export function filterTags(tags: TagInfo[], filterRegex: RegexTransformer | null): TagInfo[] {
+  if (filterRegex !== null) {
+    const filteredTags = tags.filter(tag => transformStringToOptionalValue(tag.name, filterRegex) !== null)
     core.debug(`ℹ️ Filtered tags count: ${filteredTags.length}, original count: ${tags.length}`)
     return filteredTags
   } else {
@@ -214,7 +214,7 @@ export function filterTags(tags: TagInfo[], tagResolver: TagResolver): TagInfo[]
 export function transformTags(tags: TagInfo[], transformer: RegexTransformer): TagInfo[] {
   return tags.map(function (tag) {
     if (transformer.pattern) {
-      const transformedName = tag.name.replace(transformer.pattern, transformer.target)
+      const transformedName = transformStringToValue(tag.name, transformer)
       core.debug(`ℹ️ Transformed ${tag.name} to ${transformedName}`)
       return {
         tmp: tag.name, // remember the original name

src/pr-collector/types.ts

@@ -36,28 +36,25 @@ export interface Sort {
 export interface TagResolver {
   method: string // semver, sort
   filter?: Regex // the regex to filter the tags, prior to sorting
-  transformer?: Transformer | Transformer[] // transforms the tag name using the regex, run after the filter
+  transformer?: Regex | Regex[] // transforms the tag name using the regex, run after the filter
 }
 
 export interface Regex {
   pattern: string // the regex pattern to match
   flags?: string // the regex flag to use for RegExp
-}
-
-export interface Transformer extends Regex {
   target?: string // the target string to transform the source string using the regex to
+  method?: 'replace' | 'replaceAll' | 'match' | 'exec' | 'execAll' | undefined // the method to use to extract the value, `match` will not use the `target` property
+  on_empty?: string | undefined // in case the regex results in an empty string, this value is gonna be used instead (only for label_extractor currently)
 }
 
-export interface Extractor extends Transformer {
+export interface Extractor extends Regex {
   on_property?: Property[] | Property | undefined // retrieve the property to extract the value from
-  method?: 'replace' | 'match' | undefined // the method to use to extract the value, `match` will not use the `target` property
-  on_empty?: string | undefined // in case the regex results in an empty string, this value is gonna be used instead (only for label_extractor currently)
 }
 
 export interface RegexTransformer {
   pattern: RegExp | null
   target: string
   onProperty?: Property[]
-  method?: 'replace' | 'match'
+  method?: 'replace' | 'replaceAll' | 'match' | 'exec' | 'execAll'
   onEmpty?: string
 }

src/regexUtils.ts

@@ -1,13 +1,13 @@
 import * as core from '@actions/core'
 import {RegexTransformer, Rule} from './pr-collector/types'
 import {PullRequestInfo, retrieveProperty} from './pr-collector/pullRequests'
-import {validateTransformer} from './pr-collector/regexUtils'
+import {validateRegex} from './pr-collector/regexUtils'
 
 /**
  * Checks if any of the rules match the given PR
  */
 export function matchesRules(rules: Rule[], pr: PullRequestInfo, exhaustive: Boolean): boolean {
-  const transformers: RegexTransformer[] = rules.map(rule => validateTransformer(rule)).filter(t => t !== null) as RegexTransformer[]
+  const transformers: RegexTransformer[] = rules.map(rule => validateRegex(rule)).filter(t => t !== null) as RegexTransformer[]
   if (exhaustive) {
     return transformers.every(transformer => {
       return matches(pr, transformer, 'rule')

src/transform.ts

@@ -10,8 +10,8 @@ import {
   sortPullRequests
 } from './pr-collector/pullRequests'
 import {DiffInfo} from './pr-collector/commits'
-import {validateTransformer} from './pr-collector/regexUtils'
-import {RegexTransformer, Transformer} from './pr-collector/types'
+import {transformStringToOptionalValue, transformStringToValues, validateRegex} from './pr-collector/regexUtils'
+import {Regex, RegexTransformer} from './pr-collector/types'
 import {ReleaseNotesOptions} from './releaseNotesBuilder'
 import {matchesRules} from './regexUtils'
 
@@ -40,7 +40,7 @@ export function buildChangelog(diffInfo: DiffInfo, origPrs: PullRequestInfo[], o
 
   // establish parent child PR relations
   if (config.reference !== undefined) {
-    const reference = validateTransformer(config.reference)
+    const reference = validateRegex(config.reference)
     if (reference !== null) {
       core.info(`ℹ️ Identifying PR references using \`reference\``)
 
@@ -77,7 +77,7 @@ export function buildChangelog(diffInfo: DiffInfo, origPrs: PullRequestInfo[], o
 
   // drop duplicate pull requests
   if (config.duplicate_filter !== undefined) {
-    const extractor = validateTransformer(config.duplicate_filter)
+    const extractor = validateRegex(config.duplicate_filter)
     if (extractor !== null) {
       core.info(`ℹ️ Remove duplicated pull requests using \`duplicate_filter\``)
 
@@ -461,11 +461,12 @@ function handlePlaceholder(
   const phs = placeholders.get(key)
   if (phs) {
     for (const placeholder of phs) {
-      const transformer = validateTransformer(placeholder.transformer)
+      const transformer = validateRegex(placeholder.transformer)
       if (transformer?.pattern) {
-        const extractedValue = value.replace(transformer.pattern, transformer.target)
+        const extractedValue = transformStringToOptionalValue(value, transformer)
         // note: `.replace` will return the full string again if there was no match
-        if (extractedValue && (extractedValue !== value || (extractedValue === value && value.match(transformer.pattern)))) {
+        // note: This is mostly backwards compatiblity
+        if (extractedValue && ((transformer.method && transformer.method !== 'replace') || extractedValue !== value)) {
           if (placeholderPrMap) {
             createOrSet(placeholderPrMap, placeholder.name, extractedValue)
           }
@@ -475,7 +476,7 @@ function handlePlaceholder(
           )
 
           if (core.isDebug()) {
-            core.debug(`    Custom Placeholder successfully matched data - ${extractValues} (${placeholder.name})`)
+            core.debug(`    Custom Placeholder successfully matched data - ${extractedValue} (${placeholder.name})`)
           }
         } else if (core.isDebug() && extractedValue === value) {
           core.debug(`    Custom Placeholder did result in the full original value returned. Skipping. (${placeholder.name})`)
@@ -580,11 +581,11 @@ function transform(filled: string, transformers: RegexTransformer[]): string {
   return transformed
 }
 
-function validateTransformers(specifiedTransformers: Transformer[]): RegexTransformer[] {
+function validateTransformers(specifiedTransformers: Regex[]): RegexTransformer[] {
   const transformers = specifiedTransformers
   return transformers
     .map(transformer => {
-      return validateTransformer(transformer)
+      return validateRegex(transformer)
     })
     .filter(transformer => transformer?.pattern != null)
     .map(transformer => {
@@ -619,20 +620,10 @@ function extractValuesFromString(value: string, extractor: RegexTransformer): st
   if (extractor.pattern == null) {
     return null
   }
-
-  if (extractor.method === 'match') {
-    const lables = value.match(extractor.pattern)
-    if (lables !== null && lables.length > 0) {
-      return lables.map(label => label?.toLocaleLowerCase('en') || '')
-    }
+  const transformed = transformStringToValues(value, extractor)
+  if (transformed) {
+    return transformed.map(val => val?.toLocaleLowerCase('en') || '')
   } else {
-    const label = value.replace(extractor.pattern, extractor.target)
-    if (label !== '') {
-      return [label.toLocaleLowerCase('en')]
-    }
+    return null
   }
-  if (extractor.onEmpty !== undefined) {
-    return [extractor.onEmpty.toLocaleLowerCase('en')]
-  }
-  return null
 }