- introduce new configuration duplicate_filter allowing to drop duplicated elements from the changelog

- restructure transform.ts to simplify code
  - introduce new test cases to verify new deduplication feature

README.md

@@ -326,7 +326,7 @@ $ npm run build && npm run package
 $ npm test
 
 # Verify lint is happy
-$ npm run lint -- --fixnpm run lint -- --fix
+$ npm run lint -- --fix
 ```
 
 It's suggested to export the token to your path before running the tests so that API calls can be done to GitHub.

__tests__/transform.test.ts

@@ -1,11 +1,11 @@
 import {buildChangelog} from '../src/transform'
 import {PullRequestInfo} from '../src/pullRequests'
 import moment from 'moment'
-import {DefaultConfiguration} from '../src/configuration'
+import { DefaultConfiguration, Configuration } from '../src/configuration';
 
 jest.setTimeout(180000)
 
-const configuration = DefaultConfiguration
+const configuration = Object.assign({}, DefaultConfiguration)
 configuration.categories = [
   {
     title: '## 🚀 Features',
@@ -197,14 +197,14 @@ const pullRequestsWithLabels: PullRequestInfo[] = []
 pullRequestsWithLabels.push(
   {
     number: 1,
-    title: '[AB-1234] - this is a PR 1 title message',
+    title: '[ABC-1234] - this is a PR 1 title message',
     htmlURL: '',
     baseBranch: '',
     mergedAt: moment(),
     mergeCommitSha: 'sha1',
     author: 'Mike',
     repoName: 'test-repo',
-    labels: new Set<string>().add('Feature'),
+    labels: new Set<string>().add('feature'),
     milestone: '',
     body: 'no magic body for this matter',
     assignees: [],
@@ -212,14 +212,14 @@ pullRequestsWithLabels.push(
   },
   {
     number: 2,
-    title: '[AB-4321] - this is a PR 2 title message',
+    title: '[ABC-4321] - this is a PR 2 title message',
     htmlURL: '',
     baseBranch: '',
     mergedAt: moment(),
     mergeCommitSha: 'sha1',
     author: 'Mike',
     repoName: 'test-repo',
-    labels: new Set<string>().add('Issue'),
+    labels: new Set<string>().add('issue').add('fix'),
     milestone: '',
     body: 'no magic body for this matter',
     assignees: [],
@@ -227,14 +227,14 @@ pullRequestsWithLabels.push(
   },
   {
     number: 3,
-    title: '[AB-1234321] - this is a PR 3 title message',
+    title: '[ABC-1234] - this is a PR 3 title message',
     htmlURL: '',
     baseBranch: '',
-    mergedAt: moment(),
+    mergedAt: moment().add(1, 'days'),
     mergeCommitSha: 'sha1',
     author: 'Mike',
     repoName: 'test-repo',
-    labels: new Set<string>().add('Issue').add('Feature'),
+    labels: new Set<string>().add('issue').add('feature').add('fix'),
     milestone: '',
     body: 'no magic body for this matter',
     assignees: [],
@@ -258,26 +258,26 @@ pullRequestsWithLabels.push(
 )
 
 it('Match multiple labels exhaustive for category', async () => {
-  const customConfig = DefaultConfiguration
+  const customConfig = Object.assign({}, DefaultConfiguration)
   customConfig.categories = [
     {
       title: '## 🚀 Features and 🐛 Issues',
-      labels: ['[Feature]', '[Issue]'],
+      labels: ['Feature', 'Issue'],
       exhaustive: true
     },
     {
       title: '## 🚀 Features',
-      labels: ['[Feature]', '[Feature2]'],
+      labels: ['Feature', 'Feature2'],
       exhaustive: true
     },
     {
       title: '## 🐛 Fixes',
-      labels: ['[Issue]', '[Issue2]'],
+      labels: ['Issue', 'Issue2'],
       exhaustive: true
     }
   ]
 
-  const resultChangelog = buildChangelog(mergedPullRequests, {
+  const resultChangelog = buildChangelog(pullRequestsWithLabels, {
     owner: 'mikepenz',
     repo: 'test-repo',
     fromTag: '1.0.0',
@@ -288,6 +288,53 @@ it('Match multiple labels exhaustive for category', async () => {
   })
 
   expect(resultChangelog).toStrictEqual(
-    `## 🚀 Features and 🐛 Issues\n\n- [Issue][Feature][AB-1234321] - this is a PR 3 title message\n   - PR: #3\n\n`
+    `## 🚀 Features and 🐛 Issues\n\n- [ABC-1234] - this is a PR 3 title message\n   - PR: #3\n\n`
   )
 })
+
+it('Deduplicate duplicated PRs', async () => {
+  const customConfig = Object.assign({}, DefaultConfiguration)
+  customConfig.duplicate_filter = {
+    pattern: '\\[ABC-....\\]',
+    on_property: 'title',
+    method: 'match'
+  }
+
+  const resultChangelog = buildChangelog(pullRequestsWithLabels, {
+    owner: 'mikepenz',
+    repo: 'test-repo',
+    fromTag: '1.0.0',
+    toTag: '2.0.0',
+    failOnError: false,
+    commitMode: false,
+    configuration: customConfig
+  })
+
+  expect(resultChangelog).toStrictEqual(
+    `## 🚀 Features\n\n- [ABC-1234] - this is a PR 3 title message\n   - PR: #3\n\n## 🐛 Fixes\n\n- [ABC-1234] - this is a PR 3 title message\n   - PR: #3\n- [ABC-4321] - this is a PR 2 title message\n   - PR: #2\n\n`
+  )
+})
+
+it('Deduplicate duplicated PRs DESC', async () => {
+  const customConfig = Object.assign({}, DefaultConfiguration)
+  customConfig.sort = "DESC"
+  customConfig.duplicate_filter = {
+    pattern: '\\[ABC-....\\]',
+    on_property: 'title',
+    method: 'match'
+  }
+
+  const resultChangelog = buildChangelog(pullRequestsWithLabels, {
+    owner: 'mikepenz',
+    repo: 'test-repo',
+    fromTag: '1.0.0',
+    toTag: '2.0.0',
+    failOnError: false,
+    commitMode: false,
+    configuration: customConfig
+  })
+
+  expect(resultChangelog).toStrictEqual(
+    `## 🚀 Features\n\n- [ABC-1234] - this is a PR 1 title message\n   - PR: #1\n\n## 🐛 Fixes\n\n- [ABC-4321] - this is a PR 2 title message\n   - PR: #2\n\n`
+  )
+})

dist/index.js

@@ -169,6 +169,7 @@ exports.DefaultConfiguration = {
     ],
     ignore_labels: ['ignore'],
     label_extractor: [],
+    duplicate_filter: undefined,
     transformers: [],
     tag_resolver: {
         // defines the logic on how to resolve the previous tag, only relevant if `fromTag` is not specified
@@ -1135,36 +1136,38 @@ function buildChangelog(prs, options) {
     const sortAsc = sort.toUpperCase() === 'ASC';
     prs = (0, pullRequests_1.sortPullRequests)(prs, sortAsc);
     core.info(`ℹ️ Sorted all pull requests ascending: ${sort}`);
+    // drop duplicate pull requests
+    if (config.duplicate_filter !== undefined) {
+        const extractor = validateTransformer(config.duplicate_filter);
+        if (extractor != null) {
+            core.info(`ℹ️ Remove duplicated pull requests using \`duplicate_filter\``);
+            const deduplicatedMap = new Map();
+            for (const pr of prs) {
+                const extracted = extractValues(pr, extractor, 'dupliate_filter');
+                if (extracted !== null && extracted.length > 0) {
+                    deduplicatedMap.set(extracted[0], pr);
+                }
+                else {
+                    core.debug(`ℹ️ PR (${pr.number}) did not resolve a ID using the \`duplicate_filter\``);
+                }
+            }
+            const deduplicatedPRs = Array.from(deduplicatedMap.values());
+            const removedElements = prs.length - deduplicatedPRs.length;
+            core.info(`ℹ️ Removed ${removedElements} pull requests during deduplication`);
+            prs = deduplicatedPRs;
+        }
+        else {
+            core.warning(`⚠️ Configured \`duplicate_filter\` invalid.`);
+        }
+    }
     // extract additional labels from the commit message
     const labelExtractors = validateTransformers(config.label_extractor);
     for (const extractor of labelExtractors) {
-        if (extractor.pattern != null) {
+        for (const pr of prs) {
-            for (const pr of prs) {
+            const extracted = extractValues(pr, extractor, 'label_extractor');
-                let onValue;
+            if (extracted !== null) {
-                if (extractor.onProperty !== undefined) {
+                for (const label of extracted) {
-                    let value = pr[extractor.onProperty];
+                    pr.labels.add(label);
-                    if (value === undefined) {
-                        core.warning(`⚠️ the provided property '${extractor.onProperty}' for \`label_extractor\` is not valid`);
-                        value = pr['body'];
-                    }
-                    onValue = value;
-                }
-                else {
-                    onValue = pr.body;
-                }
-                if (extractor.method === 'match') {
-                    const lables = onValue.match(extractor.pattern);
-                    if (lables !== null) {
-                        for (const label of lables) {
-                            pr.labels.add(label.toLocaleLowerCase());
-                        }
-                    }
-                }
-                else {
-                    const label = onValue.replace(extractor.pattern, extractor.target);
-                    if (label !== '') {
-                        pr.labels.add(label.toLocaleLowerCase());
-                    }
                 }
             }
         }
@@ -1306,30 +1309,66 @@ function validateTransformers(specifiedTransformers) {
     const transformers = specifiedTransformers || configuration_1.DefaultConfiguration.transformers;
     return transformers
         .map(transformer => {
-        var _a;
+        return validateTransformer(transformer);
-        try {
-            let onProperty = undefined;
-            let method = undefined;
-            if (transformer.hasOwnProperty('on_property')) {
-                onProperty = transformer.on_property;
-                method = transformer.method;
-            }
-            return {
-                pattern: new RegExp(transformer.pattern.replace('\\\\', '\\'), (_a = transformer.flags) !== null && _a !== void 0 ? _a : 'gu'),
-                target: transformer.target || '',
-                onProperty,
-                method
-            };
-        }
-        catch (e) {
-            core.warning(`⚠️ Bad replacer regex: ${transformer.pattern}`);
-            return {
-                pattern: null,
-                target: ''
-            };
-        }
     })
-        .filter(transformer => transformer.pattern != null);
+        .filter(transformer => (transformer === null || transformer === void 0 ? void 0 : transformer.pattern) != null)
+        .map(transformer => {
+        return transformer;
+    });
+}
+function validateTransformer(transformer) {
+    var _a;
+    if (transformer === undefined) {
+        return null;
+    }
+    try {
+        let onProperty = undefined;
+        let method = undefined;
+        if (transformer.hasOwnProperty('on_property')) {
+            onProperty = transformer.on_property;
+            method = transformer.method;
+        }
+        return {
+            pattern: new RegExp(transformer.pattern.replace('\\\\', '\\'), (_a = transformer.flags) !== null && _a !== void 0 ? _a : 'gu'),
+            target: transformer.target || '',
+            onProperty,
+            method
+        };
+    }
+    catch (e) {
+        core.warning(`⚠️ Bad replacer regex: ${transformer.pattern}`);
+        return null;
+    }
+}
+function extractValues(pr, extractor, extractor_usecase) {
+    if (extractor.pattern == null) {
+        return null;
+    }
+    let onValue;
+    if (extractor.onProperty !== undefined) {
+        let value = pr[extractor.onProperty];
+        if (value === undefined) {
+            core.warning(`⚠️ the provided property '${extractor.onProperty}' for \`${extractor_usecase}\` is not valid`);
+            value = pr['body'];
+        }
+        onValue = value;
+    }
+    else {
+        onValue = pr.body;
+    }
+    if (extractor.method === 'match') {
+        const lables = onValue.match(extractor.pattern);
+        if (lables !== null) {
+            return lables.map(label => label.toLocaleLowerCase());
+        }
+    }
+    else {
+        const label = onValue.replace(extractor.pattern, extractor.target);
+        if (label !== '') {
+            return [label.toLocaleLowerCase()];
+        }
+    }
+    return null;
 }
 
 

src/configuration.ts

@@ -3,13 +3,14 @@ export interface Configuration {
   max_pull_requests: number
   max_back_track_time_days: number
   exclude_merge_branches: string[]
-  sort: string
+  sort: string // "ASC" or "DESC"
   template: string
   pr_template: string
   empty_template: string
   categories: Category[]
   ignore_labels: string[]
   label_extractor: Extractor[]
+  duplicate_filter?: Extractor // extract an identifier from a PR used to detect duplicates, will keep the last match (depends on `sort`)
   transformers: Transformer[]
   tag_resolver: TagResolver
   base_branches: string[]
@@ -61,6 +62,7 @@ export const DefaultConfiguration: Configuration = {
   ], // the categories to support for the ordering
   ignore_labels: ['ignore'], // list of lables being ignored from the changelog
   label_extractor: [], // extracts additional labels from the commit message given a regex
+  duplicate_filter: undefined, // extract an identifier from a PR used to detect duplicates, will keep the last match (depends on `sort`)
   transformers: [], // transformers to apply on the PR description according to the `pr_template`
   tag_resolver: {
     // defines the logic on how to resolve the previous tag, only relevant if `fromTag` is not specified

src/transform.ts

@@ -19,37 +19,42 @@ export function buildChangelog(
   prs = sortPullRequests(prs, sortAsc)
   core.info(`ℹ️ Sorted all pull requests ascending: ${sort}`)
 
+  // drop duplicate pull requests
+  if (config.duplicate_filter !== undefined) {
+    const extractor = validateTransformer(config.duplicate_filter)
+    if (extractor != null) {
+      core.info(`ℹ️ Remove duplicated pull requests using \`duplicate_filter\``)
+
+      const deduplicatedMap = new Map<string, PullRequestInfo>()
+      for (const pr of prs) {
+        const extracted = extractValues(pr, extractor, 'dupliate_filter')
+        if (extracted !== null && extracted.length > 0) {
+          deduplicatedMap.set(extracted[0], pr)
+        } else {
+          core.debug(
+            `ℹ️ PR (${pr.number}) did not resolve a ID using the \`duplicate_filter\``
+          )
+        }
+      }
+      const deduplicatedPRs = Array.from(deduplicatedMap.values())
+      const removedElements = prs.length - deduplicatedPRs.length
+      core.info(
+        `ℹ️ Removed ${removedElements} pull requests during deduplication`
+      )
+      prs = deduplicatedPRs
+    } else {
+      core.warning(`⚠️ Configured \`duplicate_filter\` invalid.`)
+    }
+  }
+
   // extract additional labels from the commit message
   const labelExtractors = validateTransformers(config.label_extractor)
   for (const extractor of labelExtractors) {
-    if (extractor.pattern != null) {
+    for (const pr of prs) {
-      for (const pr of prs) {
+      const extracted = extractValues(pr, extractor, 'label_extractor')
-        let onValue
+      if (extracted !== null) {
-        if (extractor.onProperty !== undefined) {
+        for (const label of extracted) {
-          let value: string = pr[extractor.onProperty]
+          pr.labels.add(label)
-          if (value === undefined) {
-            core.warning(
-              `⚠️ the provided property '${extractor.onProperty}' for \`label_extractor\` is not valid`
-            )
-            value = pr['body']
-          }
-          onValue = value
-        } else {
-          onValue = pr.body
-        }
-
-        if (extractor.method === 'match') {
-          const lables = onValue.match(extractor.pattern)
-          if (lables !== null) {
-            for (const label of lables) {
-              pr.labels.add(label.toLocaleLowerCase())
-            }
-          }
-        } else {
-          const label = onValue.replace(extractor.pattern, extractor.target)
-          if (label !== '') {
-            pr.labels.add(label.toLocaleLowerCase())
-          }
         }
       }
     }
@@ -276,32 +281,78 @@ function validateTransformers(
     specifiedTransformers || DefaultConfiguration.transformers
   return transformers
     .map(transformer => {
-      try {
+      return validateTransformer(transformer)
-        let onProperty = undefined
-        let method = undefined
-        if (transformer.hasOwnProperty('on_property')) {
-          onProperty = (transformer as Extractor).on_property
-          method = (transformer as Extractor).method
-        }
-
-        return {
-          pattern: new RegExp(
-            transformer.pattern.replace('\\\\', '\\'),
-            transformer.flags ?? 'gu'
-          ),
-          target: transformer.target || '',
-          onProperty,
-          method
-        }
-      } catch (e) {
-        core.warning(`⚠️ Bad replacer regex: ${transformer.pattern}`)
-        return {
-          pattern: null,
-          target: ''
-        }
-      }
     })
-    .filter(transformer => transformer.pattern != null)
+    .filter(transformer => transformer?.pattern != null)
+    .map(transformer => {
+      return transformer as RegexTransformer
+    })
+}
+
+function validateTransformer(
+  transformer?: Transformer
+): RegexTransformer | null {
+  if (transformer === undefined) {
+    return null
+  }
+  try {
+    let onProperty = undefined
+    let method = undefined
+    if (transformer.hasOwnProperty('on_property')) {
+      onProperty = (transformer as Extractor).on_property
+      method = (transformer as Extractor).method
+    }
+
+    return {
+      pattern: new RegExp(
+        transformer.pattern.replace('\\\\', '\\'),
+        transformer.flags ?? 'gu'
+      ),
+      target: transformer.target || '',
+      onProperty,
+      method
+    }
+  } catch (e) {
+    core.warning(`⚠️ Bad replacer regex: ${transformer.pattern}`)
+    return null
+  }
+}
+
+function extractValues(
+  pr: PullRequestInfo,
+  extractor: RegexTransformer,
+  extractor_usecase: string
+): string[] | null {
+  if (extractor.pattern == null) {
+    return null
+  }
+
+  let onValue
+  if (extractor.onProperty !== undefined) {
+    let value: string = pr[extractor.onProperty]
+    if (value === undefined) {
+      core.warning(
+        `⚠️ the provided property '${extractor.onProperty}' for \`${extractor_usecase}\` is not valid`
+      )
+      value = pr['body']
+    }
+    onValue = value
+  } else {
+    onValue = pr.body
+  }
+
+  if (extractor.method === 'match') {
+    const lables = onValue.match(extractor.pattern)
+    if (lables !== null) {
+      return lables.map(label => label.toLocaleLowerCase())
+    }
+  } else {
+    const label = onValue.replace(extractor.pattern, extractor.target)
+    if (label !== '') {
+      return [label.toLocaleLowerCase()]
+    }
+  }
+  return null
 }
 
 interface RegexTransformer {