Merge pull request #12 from mikepenz/feature/enhanced_performance_configs

At max read back configurations

.github/workflows/test.yml

@@ -41,8 +41,8 @@ jobs:
           configuration: "configuration_complex.json"
           owner: "mikepenz"
           repo: "release-changelog-builder-action"
-          fromTag: "0.0.2"
+          fromTag: "v0.0.1"
-          toTag: "0.0.3"
+          toTag: "v0.0.3"
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Echo Complex

README.md

@@ -59,11 +59,18 @@ By default the action will look for a file called `configuration.json` within th
             "pattern": "[\\-\\*] (\\[(...|TEST|CI|SKIP)\\])( )?(.+?)\n(.+?[\\-\\*] )(.+)",
             "target": "- $4\n  - $6"
         }
+    ],
+    "max_tags_to_fetch": 200,
+    "max_pull_requests": 200,
+    "max_back_track_time_days": 90,
+    "exclude_merge_branches": [
+        "Owner/qa"
     ]
 }
 ```
 
 Any section of the configruation can be ommited, to have defaults apply
+Defaults for the configuraiton can be found in the [configuration.ts](https://github.com/mikepenz/release-changelog-builder-action/blob/develop/src/configuration.ts)
 
 
 ## Advanced workflow specification

__tests__/main.test.ts

@@ -30,7 +30,7 @@ it('Should be true', async () => {
     owner: 'mikepenz',
     repo: 'release-changelog-builder-action',
     fromTag: null,
-    toTag: '0.0.3',
+    toTag: 'v0.0.3',
     configuration: configuration
   })
 

configuration_complex.json

@@ -26,5 +26,11 @@
             "pattern": "[\\-\\*] (\\[(...|TEST|CI|SKIP)\\])( )?(.+?)\n(.+?[\\-\\*] )(.+)",
             "target": "- $4\n  - $6"
         }
+    ],
+    "max_tags_to_fetch": 200,
+    "max_pull_requests": 200,
+    "max_back_track_time_days": 90,
+    "exclude_merge_branches": [
+        "Owner/qa"
     ]
 }

dist/index.js

@@ -118,12 +118,16 @@ exports.Commits = Commits;
 Object.defineProperty(exports, "__esModule", ({ value: true }));
 exports.DefaultConfiguration = void 0;
 exports.DefaultConfiguration = {
+    max_tags_to_fetch: 200,
+    max_pull_requests: 200,
+    max_back_track_time_days: 90,
+    exclude_merge_branches: [],
     sort: 'ASC',
     template: '${{CHANGELOG}}',
     pr_template: '- ${{TITLE}}\n   - PR: #${{NUMBER}}',
     empty_template: '- no changes',
     categories: [],
-    transformers: []
+    transformers: [] // transformers to apply on the PR description according to the `pr_template`
 };
 
 
@@ -468,8 +472,7 @@ class PullRequests {
             }
         });
     }
-    getBetweenDates(owner, repo, fromDate, toDate // eslint-disable-line @typescript-eslint/no-unused-vars
+    getBetweenDates(owner, repo, fromDate, toDate, maxPullRequests) {
-    ) {
         var e_1, _a;
         return __awaiter(this, void 0, void 0, function* () {
             const mergedPRs = [];
@@ -499,7 +502,11 @@ class PullRequests {
                         });
                     }
                     const firstPR = prs[0];
-                    if (firstPR.merged_at && fromDate.isAfter(moment_1.default(firstPR.merged_at))) {
+                    if ((firstPR.merged_at && fromDate.isAfter(moment_1.default(firstPR.merged_at))) ||
+                        mergedPRs.length >= maxPullRequests) {
+                        if (mergedPRs.length >= maxPullRequests) {
+                            core.info(`Reached 'maxPullRequests' count ${maxPullRequests}`);
+                        }
                         // bail out early to not keep iterating on PRs super old
                         return sortPullRequests(mergedPRs, true);
                     }
@@ -515,10 +522,22 @@ class PullRequests {
             return sortPullRequests(mergedPRs, true);
         });
     }
-    filterCommits(commits) {
+    filterCommits(commits, excludeMergeBranches) {
         const prRegex = /Merge pull request #(\d+)/;
         const filteredCommits = [];
         for (const commit of commits) {
+            if (excludeMergeBranches) {
+                let matched = false;
+                for (const excludeMergeBranch of excludeMergeBranches) {
+                    if (commit.summary.includes(excludeMergeBranch)) {
+                        matched = true;
+                        break;
+                    }
+                }
+                if (matched) {
+                    continue;
+                }
+            }
             const match = commit.summary.match(prRegex);
             if (!match) {
                 continue;
@@ -615,7 +634,9 @@ class ReleaseNotes {
             if (!this.options.fromTag) {
                 core.debug(`fromTag undefined, trying to resolve via API`);
                 const tagsApi = new tags_1.Tags(octokit);
-                const previousTag = yield tagsApi.findPredecessorTag(owner, repo, toTag);
+                const previousTag = yield tagsApi.findPredecessorTag(owner, repo, toTag, configuration.max_tags_to_fetch
+                    ? configuration.max_tags_to_fetch
+                    : configuration_1.DefaultConfiguration.max_tags_to_fetch);
                 if (previousTag == null) {
                     core.error(`Unable to retrieve previous tag given ${toTag}`);
                     return configuration.empty_template
@@ -637,7 +658,7 @@ class ReleaseNotes {
     }
     getMergedPullRequests(octokit) {
         return __awaiter(this, void 0, void 0, function* () {
-            const { owner, repo, fromTag, toTag } = this.options;
+            const { owner, repo, fromTag, toTag, configuration } = this.options;
             core.info(`Comparing ${owner}/${repo} - ${fromTag}...${toTag}`);
             const commitsApi = new commits_1.Commits(octokit);
             const commits = yield commitsApi.getDiff(owner, repo, fromTag, toTag);
@@ -646,13 +667,26 @@ class ReleaseNotes {
             }
             const firstCommit = commits[0];
             const lastCommit = commits[commits.length - 1];
-            const fromDate = firstCommit.date;
+            let fromDate = firstCommit.date;
             const toDate = lastCommit.date;
-            core.info(`Fetching PRs between dates ${fromDate.toISOString()} ${toDate.toISOString()} for ${owner}/${repo}`);
+            const maxDays = configuration.max_back_track_time_days
+                ? configuration.max_back_track_time_days
+                : configuration_1.DefaultConfiguration.max_back_track_time_days;
+            const maxFromDate = toDate.clone().subtract(maxDays, 'days');
+            if (maxFromDate.isAfter(fromDate)) {
+                core.info(`Adjusted 'fromDate' to go max ${maxDays} back`);
+                fromDate = maxFromDate;
+            }
+            core.info(`Fetching PRs between dates ${fromDate.toISOString()} to ${toDate.toISOString()} for ${owner}/${repo}`);
             const pullRequestsApi = new pullRequests_1.PullRequests(octokit);
-            const pullRequests = yield pullRequestsApi.getBetweenDates(owner, repo, fromDate, toDate);
+            const pullRequests = yield pullRequestsApi.getBetweenDates(owner, repo, fromDate, toDate, configuration.max_pull_requests
-            core.info(`Found ${pullRequests.length} merged PRs for ${owner}/${repo}`);
+                ? configuration.max_pull_requests
-            const prCommits = pullRequestsApi.filterCommits(commits);
+                : configuration_1.DefaultConfiguration.max_pull_requests);
+            core.info(`Retrieved ${pullRequests.length} merged PRs for ${owner}/${repo}`);
+            const prCommits = pullRequestsApi.filterCommits(commits, configuration.exclude_merge_branches
+                ? configuration.exclude_merge_branches
+                : configuration_1.DefaultConfiguration.exclude_merge_branches);
+            core.info(`Retrieved ${prCommits.length} PR merge commits for ${owner}/${repo}`);
             const filteredPullRequests = [];
             const pullRequestsByNumber = {};
             for (const pr of pullRequests) {
@@ -667,7 +701,6 @@ class ReleaseNotes {
                     filteredPullRequests.push(pullRequestsByNumber[commit.prNumber]);
                 }
                 else if (fromDate.toISOString() === toDate.toISOString()) {
-                    core.info(`${prRef} not in date range, fetching explicitly`);
                     const pullRequest = yield pullRequestsApi.getSingle(owner, repo, commit.prNumber);
                     if (pullRequest) {
                         filteredPullRequests.push(pullRequest);
@@ -677,7 +710,7 @@ class ReleaseNotes {
                     }
                 }
                 else {
-                    core.info(`${prRef} not in date range, likely a merge commit from a fork-to-fork PR`);
+                    core.info(`${prRef} not in date range, excluding from changelog`);
                 }
             }
             return filteredPullRequests;
@@ -736,7 +769,7 @@ class Tags {
     constructor(octokit) {
         this.octokit = octokit;
     }
-    getTags(owner, repo) {
+    getTags(owner, repo, maxTagsToFetch) {
         var e_1, _a;
         return __awaiter(this, void 0, void 0, function* () {
             const tagsInfo = [];
@@ -746,7 +779,6 @@ class Tags {
                 direction: 'desc',
                 per_page: 100
             });
-            const max = 200;
             try {
                 for (var _b = __asyncValues(this.octokit.paginate.iterator(options)), _c; _c = yield _b.next(), !_c.done;) {
                     const response = _c.value;
@@ -757,8 +789,8 @@ class Tags {
                             commit: tag.commit.sha
                         });
                     }
-                    // for performance only fetch newest 200 tags!!
+                    // for performance only fetch newest maxTagsToFetch tags!!
-                    if (tagsInfo.length >= max) {
+                    if (tagsInfo.length >= maxTagsToFetch) {
                         break;
                     }
                 }
@@ -770,13 +802,13 @@ class Tags {
                 }
                 finally { if (e_1) throw e_1.error; }
             }
-            core.info(`Found ${tagsInfo.length} (fetching max: ${max}) tags from the GitHub API for ${owner}/${repo}`);
+            core.info(`Found ${tagsInfo.length} (fetching max: ${maxTagsToFetch}) tags from the GitHub API for ${owner}/${repo}`);
             return tagsInfo;
         });
     }
-    findPredecessorTag(owner, repo, tag) {
+    findPredecessorTag(owner, repo, tag, maxTagsToFetch) {
         return __awaiter(this, void 0, void 0, function* () {
-            const tags = this.sortTags(yield this.getTags(owner, repo));
+            const tags = this.sortTags(yield this.getTags(owner, repo, maxTagsToFetch));
             const length = tags.length;
             for (let i = 0; i < length; i++) {
                 if (tags[i].name.toLowerCase() === tag.toLowerCase()) {
@@ -926,7 +958,7 @@ function fillTemplate(pr, template) {
     transformed = transformed.replace('${{NUMBER}}', pr.number.toString());
     transformed = transformed.replace('${{TITLE}}', pr.title);
     transformed = transformed.replace('${{URL}}', pr.htmlURL);
-    transformed = transformed.replace('${{MERGED_AT}}', pr.mergedAt.toString());
+    transformed = transformed.replace('${{MERGED_AT}}', pr.mergedAt.toISOString());
     transformed = transformed.replace('${{AUTHOR}}', pr.author);
     transformed = transformed.replace('${{BODY}}', pr.body);
     return transformed;

src/configuration.ts

@@ -1,4 +1,8 @@
 export interface Configuration {
+  max_tags_to_fetch: number
+  max_pull_requests: number
+  max_back_track_time_days: number
+  exclude_merge_branches: string[]
   sort: string
   template: string
   pr_template: string
@@ -18,10 +22,14 @@ export interface Transformer {
 }
 
 export const DefaultConfiguration: Configuration = {
-  sort: 'ASC',
+  max_tags_to_fetch: 200, // the amount of tags to fetch from the github API
-  template: '${{CHANGELOG}}',
+  max_pull_requests: 200, // the amount of pull requests to process
-  pr_template: '- ${{TITLE}}\n   - PR: #${{NUMBER}}',
+  max_back_track_time_days: 90, // allow max of 90 days to check up on pull requests
-  empty_template: '- no changes',
+  exclude_merge_branches: [], // branches to exclude from counting as PRs (e.g. YourOrg/qa, YourOrg/main)
-  categories: [],
+  sort: 'ASC', // sorting order for filling the changelog (ASC or DESC) supported
-  transformers: []
+  template: '${{CHANGELOG}}', // the global template to host the changelog
+  pr_template: '- ${{TITLE}}\n   - PR: #${{NUMBER}}', // the per PR template to pick
+  empty_template: '- no changes', // the template to use if no pull requests are found
+  categories: [], // the categories to support for the ordering
+  transformers: [] // transformers to apply on the PR description according to the `pr_template`
 }

src/pullRequests.ts

@@ -52,7 +52,8 @@ export class PullRequests {
     owner: string,
     repo: string,
     fromDate: moment.Moment,
-    toDate: moment.Moment // eslint-disable-line @typescript-eslint/no-unused-vars
+    toDate: moment.Moment,
+    maxPullRequests: number
   ): Promise<PullRequestInfo[]> {
     const mergedPRs: PullRequestInfo[] = []
     const options = this.octokit.pulls.list.endpoint.merge({
@@ -83,7 +84,14 @@ export class PullRequests {
       }
 
       const firstPR = prs[0]
-      if (firstPR.merged_at && fromDate.isAfter(moment(firstPR.merged_at))) {
+      if (
+        (firstPR.merged_at && fromDate.isAfter(moment(firstPR.merged_at))) ||
+        mergedPRs.length >= maxPullRequests
+      ) {
+        if (mergedPRs.length >= maxPullRequests) {
+          core.info(`Reached 'maxPullRequests' count ${maxPullRequests}`)
+        }
+
         // bail out early to not keep iterating on PRs super old
         return sortPullRequests(mergedPRs, true)
       }
@@ -92,11 +100,27 @@ export class PullRequests {
     return sortPullRequests(mergedPRs, true)
   }
 
-  filterCommits(commits: CommitInfo[]): CommitInfo[] {
+  filterCommits(
+    commits: CommitInfo[],
+    excludeMergeBranches: string[]
+  ): CommitInfo[] {
     const prRegex = /Merge pull request #(\d+)/
     const filteredCommits = []
 
     for (const commit of commits) {
+      if (excludeMergeBranches) {
+        let matched = false
+        for (const excludeMergeBranch of excludeMergeBranches) {
+          if (commit.summary.includes(excludeMergeBranch)) {
+            matched = true
+            break
+          }
+        }
+        if (matched) {
+          continue
+        }
+      }
+
       const match = commit.summary.match(prRegex)
       if (!match) {
         continue

src/releaseNotes.ts

@@ -28,7 +28,14 @@ export class ReleaseNotes {
       core.debug(`fromTag undefined, trying to resolve via API`)
       const tagsApi = new Tags(octokit)
 
-      const previousTag = await tagsApi.findPredecessorTag(owner, repo, toTag)
+      const previousTag = await tagsApi.findPredecessorTag(
+        owner,
+        repo,
+        toTag,
+        configuration.max_tags_to_fetch
+          ? configuration.max_tags_to_fetch
+          : DefaultConfiguration.max_tags_to_fetch
+      )
       if (previousTag == null) {
         core.error(`Unable to retrieve previous tag given ${toTag}`)
         return configuration.empty_template
@@ -57,7 +64,7 @@ export class ReleaseNotes {
   private async getMergedPullRequests(
     octokit: Octokit
   ): Promise<PullRequestInfo[]> {
-    const {owner, repo, fromTag, toTag} = this.options
+    const {owner, repo, fromTag, toTag, configuration} = this.options
     core.info(`Comparing ${owner}/${repo} - ${fromTag}...${toTag}`)
 
     const commitsApi = new Commits(octokit)
@@ -69,11 +76,20 @@ export class ReleaseNotes {
 
     const firstCommit = commits[0]
     const lastCommit = commits[commits.length - 1]
-    const fromDate = firstCommit.date
+    let fromDate = firstCommit.date
     const toDate = lastCommit.date
 
+    const maxDays = configuration.max_back_track_time_days
+      ? configuration.max_back_track_time_days
+      : DefaultConfiguration.max_back_track_time_days
+    const maxFromDate = toDate.clone().subtract(maxDays, 'days')
+    if (maxFromDate.isAfter(fromDate)) {
+      core.info(`Adjusted 'fromDate' to go max ${maxDays} back`)
+      fromDate = maxFromDate
+    }
+
     core.info(
-      `Fetching PRs between dates ${fromDate.toISOString()} ${toDate.toISOString()} for ${owner}/${repo}`
+      `Fetching PRs between dates ${fromDate.toISOString()} to ${toDate.toISOString()} for ${owner}/${repo}`
     )
 
     const pullRequestsApi = new PullRequests(octokit)
@@ -81,12 +97,27 @@ export class ReleaseNotes {
       owner,
       repo,
       fromDate,
-      toDate
+      toDate,
+      configuration.max_pull_requests
+        ? configuration.max_pull_requests
+        : DefaultConfiguration.max_pull_requests
     )
 
-    core.info(`Found ${pullRequests.length} merged PRs for ${owner}/${repo}`)
+    core.info(
+      `Retrieved ${pullRequests.length} merged PRs for ${owner}/${repo}`
+    )
+
+    const prCommits = pullRequestsApi.filterCommits(
+      commits,
+      configuration.exclude_merge_branches
+        ? configuration.exclude_merge_branches
+        : DefaultConfiguration.exclude_merge_branches
+    )
+
+    core.info(
+      `Retrieved ${prCommits.length} PR merge commits for ${owner}/${repo}`
+    )
 
-    const prCommits = pullRequestsApi.filterCommits(commits)
     const filteredPullRequests = []
     const pullRequestsByNumber: {[key: number]: PullRequestInfo} = {}
 
@@ -104,7 +135,6 @@ export class ReleaseNotes {
       if (pullRequestsByNumber[commit.prNumber]) {
         filteredPullRequests.push(pullRequestsByNumber[commit.prNumber])
       } else if (fromDate.toISOString() === toDate.toISOString()) {
-        core.info(`${prRef} not in date range, fetching explicitly`)
         const pullRequest = await pullRequestsApi.getSingle(
           owner,
           repo,
@@ -117,9 +147,7 @@ export class ReleaseNotes {
           core.warning(`${prRef} not found! Commit text: ${commit.summary}`)
         }
       } else {
-        core.info(
+        core.info(`${prRef} not in date range, excluding from changelog`)
-          `${prRef} not in date range, likely a merge commit from a fork-to-fork PR`
-        )
       }
     }
 

src/tags.ts

@@ -9,7 +9,11 @@ export interface TagInfo {
 export class Tags {
   constructor(private octokit: Octokit) {}
 
-  async getTags(owner: string, repo: string): Promise<TagInfo[]> {
+  async getTags(
+    owner: string,
+    repo: string,
+    maxTagsToFetch: number
+  ): Promise<TagInfo[]> {
     const tagsInfo: TagInfo[] = []
     const options = this.octokit.repos.listTags.endpoint.merge({
       owner,
@@ -18,7 +22,6 @@ export class Tags {
       per_page: 100
     })
 
-    const max = 200
     for await (const response of this.octokit.paginate.iterator(options)) {
       type TagsListData = RestEndpointMethodTypes['repos']['listTags']['response']['data']
       const tags: TagsListData = response.data as TagsListData
@@ -30,14 +33,14 @@ export class Tags {
         })
       }
 
-      // for performance only fetch newest 200 tags!!
+      // for performance only fetch newest maxTagsToFetch tags!!
-      if (tagsInfo.length >= max) {
+      if (tagsInfo.length >= maxTagsToFetch) {
         break
       }
     }
 
     core.info(
-      `Found ${tagsInfo.length} (fetching max: ${max}) tags from the GitHub API for ${owner}/${repo}`
+      `Found ${tagsInfo.length} (fetching max: ${maxTagsToFetch}) tags from the GitHub API for ${owner}/${repo}`
     )
     return tagsInfo
   }
@@ -45,9 +48,10 @@ export class Tags {
   async findPredecessorTag(
     owner: string,
     repo: string,
-    tag: string
+    tag: string,
+    maxTagsToFetch: number
   ): Promise<TagInfo | null> {
-    const tags = this.sortTags(await this.getTags(owner, repo))
+    const tags = this.sortTags(await this.getTags(owner, repo, maxTagsToFetch))
 
     const length = tags.length
     for (let i = 0; i < length; i++) {

src/transform.ts

@@ -105,7 +105,7 @@ function fillTemplate(pr: PullRequestInfo, template: string): string {
   transformed = transformed.replace('${{NUMBER}}', pr.number.toString())
   transformed = transformed.replace('${{TITLE}}', pr.title)
   transformed = transformed.replace('${{URL}}', pr.htmlURL)
-  transformed = transformed.replace('${{MERGED_AT}}', pr.mergedAt.toString())
+  transformed = transformed.replace('${{MERGED_AT}}', pr.mergedAt.toISOString())
   transformed = transformed.replace('${{AUTHOR}}', pr.author)
   transformed = transformed.replace('${{BODY}}', pr.body)
   return transformed