Merge pull request #1224 from mikepenz/feature/adjusted_caching_mechanism

Refactor caching logic to solve data length limitations

README.md

@@ -91,7 +91,7 @@ A full set list of possible output values for this action.
 | `outputs.changes`           | Total count of changes in this release (lines).                                                                           |
 | `outputs.commits`           | Count of commits which have been added in this release.                                                                   |
 | `outputs.categorized`       | The categorized pull requests used to build the changelog as serialized JSON.                                             |
-| `outputs.cache`             | The cache for the current fetched data. Can be provided to another action step.                                                           |
+| `outputs.cache`             | The file pointing to the cache for the current fetched data. Can be provided to another action step.                      |
 
 ## Full Sample 🖥️
 
@@ -338,7 +338,9 @@ For advanced use cases additional settings can be provided to the action
 | `fetchReleaseInformation` | Will enable fetching additional release information from tags. Default: false                                                                                               |
 | `fetchReviews`            | Will enable fetching the reviews on of the PR. Default: false                                                                                                               |
 | `commitMode`              | Special configuration for projects which work without PRs. Uses commit messages as changelog. This mode looses access to information only available for PRs. Default: false |
-| `cache`                  | Cached data from a previous run. Will skip fetching PR information |
+| `exportCache`             | Will enable exporting the fetched PR information to a cache, which can be re-used by later runs. Default: false                                                             |
+| `exportOnly`              | When enabled, will result in only exporting the cache, without genearting a changelog. Default: false (Requires `exportCache` to be enabled)                                |
+| `cache`                   | The file path to write/read the cache to/from.                                                                                                                              |
 
 > **Warning**: `${{ secrets.GITHUB_TOKEN }}` only grants rights to the current repository, for other repositories please use a PAT (Personal Access Token).
 
@@ -598,6 +600,7 @@ it('Test custom changelog builder', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null,  // path to the cache
     configuration  // configuration
   )
 

__tests__/releaseNotesBuilder.test.ts

@@ -23,6 +23,7 @@ it('Should match generated changelog (unspecified fromTag)', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -56,6 +57,7 @@ it('Should match generated changelog (unspecified tags)', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -84,6 +86,7 @@ it('Should use empty placeholder', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -112,6 +115,7 @@ it('Should fill empty placeholders', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -142,6 +146,7 @@ it('Should fill `template` placeholders', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -173,6 +178,7 @@ it('Should fill `template` placeholders, ignore', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -203,6 +209,7 @@ it('Uncategorized category', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -233,6 +240,7 @@ it('Verify commit based changelog', async () => {
     true, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -263,6 +271,7 @@ it('Verify commit based changelog, with emoji categorisation', async () => {
     true, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration
   )
 
@@ -293,6 +302,7 @@ it('Verify default inclusion of open PRs', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration // configuration
   )
 
@@ -323,6 +333,7 @@ it('Verify custom categorisation of open PRs', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration // configuration
   )
 
@@ -353,6 +364,7 @@ it('Verify reviewers who approved are fetched and also release information', asy
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration // configuration
   )
 
@@ -384,6 +396,7 @@ it('Fetch release information', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration // configuration
   )
 
@@ -413,6 +426,7 @@ it('Fetch release information for non existing tag / release', async () => {
     false, // enable commitMode
     false, // enable exportCache
     false, // enable exportOnly
+    null, // path to the cache
     configuration // configuration
   )
 

action.yml

@@ -43,12 +43,6 @@ inputs:
   commitMode:
     description: 'Enables a `light` commit based mode. This mode generates changelogs based on the commits. Please note that this is not officially supported, and lacks a lot of features only possible with PRs.'
     default: "false"
-  exportCache:
-    description: 'If enabled, the action will export the collected data to the cache. This is disabled by default. Can be passed to a follow up run via `cache`.'
-    default: "false"
-  exportOnly:
-    description: 'If enabled, the action will only collect the data and terminate afterwards. Data can then be consumed by steps afterwards. This requires `exportCache` to be enabled'
-    default: "false"
   outputFile:
     description: 'If defined, the changelog will get written to this file. (relative to the checkout dir)'
   token:
@@ -56,6 +50,12 @@ inputs:
     default: ${{ github.token }}
   baseUrl:
     description: 'Defines the base url for GitHub Enterprise authentication, uses `https://api.github.com` by default'
+  exportCache:
+    description: 'If enabled, the action will export the collected data to the cache. This is disabled by default. Can be passed to a follow up run via `cache`.'
+    default: "false"
+  exportOnly:
+    description: 'If enabled, the action will only collect the data and terminate afterwards. Data can then be consumed by steps afterwards. This requires `exportCache` to be enabled'
+    default: "false"
   cache:
     description: 'Provide the cache of a previous run. Allows to re-use collected information multiple times to generate different release notes. Requires `exportCache` to be enabled for the previous run.'
 outputs:

dist/index.js

@@ -148,7 +148,8 @@ function run() {
             const commitMode = core.getInput('commitMode') === 'true';
             const exportCache = core.getInput('exportCache') === 'true';
             const exportOnly = core.getInput('exportOnly') === 'true';
-            const result = yield new releaseNotesBuilder_1.ReleaseNotesBuilder(baseUrl, token, repositoryPath, owner, repo, fromTag, toTag, includeOpen, failOnError, ignorePreReleases, fetchViaCommits, fetchReviewers, fetchReleaseInformation, fetchReviews, commitMode, exportCache, exportOnly, configuration).build();
+            const cache = core.getInput('cache');
+            const result = yield new releaseNotesBuilder_1.ReleaseNotesBuilder(baseUrl, token, repositoryPath, owner, repo, fromTag, toTag, includeOpen, failOnError, ignorePreReleases, fetchViaCommits, fetchReviewers, fetchReleaseInformation, fetchReviews, commitMode, exportCache, exportOnly, cache, configuration).build();
             core.setOutput('changelog', result);
             // write the result in changelog to file if possible
             const outputFile = core.getInput('outputFile');
@@ -1842,7 +1843,7 @@ const transform_1 = __nccwpck_require__(1644);
 const prCollector_1 = __nccwpck_require__(2267);
 const utils_2 = __nccwpck_require__(9613);
 class ReleaseNotesBuilder {
-    constructor(baseUrl, token, repositoryPath, owner, repo, fromTag, toTag, includeOpen = false, failOnError, ignorePreReleases, fetchViaCommits = false, fetchReviewers = false, fetchReleaseInformation = false, fetchReviews = false, commitMode = false, exportCache = false, exportOnly = false, configuration) {
+    constructor(baseUrl, token, repositoryPath, owner, repo, fromTag, toTag, includeOpen = false, failOnError, ignorePreReleases, fetchViaCommits = false, fetchReviewers = false, fetchReleaseInformation = false, fetchReviews = false, commitMode = false, exportCache = false, exportOnly = false, cache = null, configuration) {
         this.baseUrl = baseUrl;
         this.token = token;
         this.repositoryPath = repositoryPath;
@@ -1860,11 +1861,19 @@ class ReleaseNotesBuilder {
         this.commitMode = commitMode;
         this.exportCache = exportCache;
         this.exportOnly = exportOnly;
+        this.cache = cache;
         this.configuration = configuration;
     }
     build() {
         return __awaiter(this, void 0, void 0, function* () {
-            const releaseNotesData = (0, utils_1.checkExportedData)();
+            let releaseNotesData;
+            try {
+                releaseNotesData = (0, utils_1.checkExportedData)(this.exportCache, this.cache);
+            }
+            catch (error) {
+                (0, utils_2.failOrError)(`${error}`, this.failOnError);
+                return null;
+            }
             if (releaseNotesData == null) {
                 if (!this.owner) {
                     (0, utils_2.failOrError)(`💥 Missing or couldn't resolve 'owner'`, this.failOnError);
@@ -1902,13 +1911,12 @@ class ReleaseNotesBuilder {
                 const diffInfo = prData.diffInfo;
                 this.setOutputs(options, diffInfo, mergedPullRequests);
                 if (this.exportCache) {
-                    const cache = {
+                    const cacheData = {
                         mergedPullRequests,
                         diffInfo,
                         options
                     };
-                    core.setOutput(`cache`, JSON.stringify(cache));
+                    (0, utils_1.writeCacheData)(cacheData, this.cache);
-                    //fs.writeFileSync(path.resolve('cache.json'), JSON.stringify(cache))
                     if (this.exportOnly) {
                         core.info(`ℹ️ Enabled 'exportOnly' will not generate changelog`);
                         core.endGroup();
@@ -2558,12 +2566,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.haveEveryElementsArr = exports.haveEveryElements = exports.haveCommonElementsArr = exports.haveCommonElements = exports.createOrSet = exports.writeOutput = exports.mergeConfiguration = exports.parseConfiguration = exports.resolveConfiguration = exports.checkExportedData = exports.retrieveRepositoryPath = void 0;
+exports.haveEveryElementsArr = exports.haveEveryElements = exports.haveCommonElementsArr = exports.haveCommonElements = exports.createOrSet = exports.writeOutput = exports.mergeConfiguration = exports.parseConfiguration = exports.resolveConfiguration = exports.checkExportedData = exports.writeCacheData = exports.retrieveRepositoryPath = void 0;
 const core = __importStar(__nccwpck_require__(2186));
 const fs = __importStar(__nccwpck_require__(7147));
 const path = __importStar(__nccwpck_require__(1017));
 const configuration_1 = __nccwpck_require__(5527);
 const moment_1 = __importDefault(__nccwpck_require__(9623));
+const process_1 = __nccwpck_require__(7282);
 /**
  * Resolves the repository path, relatively to the GITHUB_WORKSPACE
  */
@@ -2580,14 +2589,53 @@ function retrieveRepositoryPath(providedPath) {
     return repositoryPath;
 }
 exports.retrieveRepositoryPath = retrieveRepositoryPath;
+function writeCacheData(data, cacheOutput) {
+    let cacheFile;
+    if (cacheOutput && !cacheOutput.startsWith('{')) {
+        // legacy handling, originally we allowed cache as direct string.
+        // However, this can result in a "Argument list too long" exception for very long caches
+        cacheFile = cacheOutput;
+    }
+    else {
+        if (process_1.env.RUNNER_TEMP && !fs.existsSync(process_1.env.RUNNER_TEMP)) {
+            fs.mkdirSync(process_1.env.RUNNER_TEMP);
+        }
+        cacheFile = `${process_1.env.RUNNER_TEMP}/rcba-cache.json`;
+        core.debug(`Defined cacheFile as ${cacheFile}`);
+    }
+    try {
+        fs.writeFileSync(cacheFile, JSON.stringify(data));
+        core.setOutput(`cache`, cacheFile);
+    }
+    catch (error) {
+        core.warning(`Failed to write cache file. (${error})`);
+    }
+}
+exports.writeCacheData = writeCacheData;
 /**
  * Retrieves the exported information from a previous run of the `release-changelog-builder-action`.
  * If available, return a [ReleaseNotesData].
  */
-function checkExportedData() {
+function checkExportedData(exportCache, cacheInput) {
-    const rawCache = core.getInput(`cache`);
+    if (exportCache) {
-    if (rawCache) {
+        return null;
-        const cache = JSON.parse(rawCache);
+    }
+    if (cacheInput) {
+        // legacy handling, originally we allowed cache as direct string.
+        // However, this can result in a "Argument list too long" exception for very long caches
+        const legacyJsonCache = cacheInput.startsWith('{');
+        let cache;
+        if (legacyJsonCache) {
+            cache = JSON.parse(cacheInput);
+        }
+        else {
+            if (!fs.existsSync(cacheInput)) {
+                throw new Error(`💥 The provided cache file does not exist`);
+            }
+            else {
+                cache = JSON.parse(fs.readFileSync(cacheInput, 'utf8'));
+            }
+        }
         const diffInfo = cache.diffInfo;
         const mergedPullRequests = cache.mergedPullRequests;
         for (const pr of mergedPullRequests) {
@@ -27903,6 +27951,14 @@ module.exports = require("path");
 
 /***/ }),
 
+/***/ 7282:
+/***/ ((module) => {
+
+"use strict";
+module.exports = require("process");
+
+/***/ }),
+
 /***/ 5477:
 /***/ ((module) => {
 

src/main.ts

@@ -57,6 +57,7 @@ async function run(): Promise<void> {
     const commitMode = core.getInput('commitMode') === 'true'
     const exportCache = core.getInput('exportCache') === 'true'
     const exportOnly = core.getInput('exportOnly') === 'true'
+    const cache = core.getInput('cache')
 
     const result = await new ReleaseNotesBuilder(
       baseUrl,
@@ -76,6 +77,7 @@ async function run(): Promise<void> {
       commitMode,
       exportCache,
       exportOnly,
+      cache,
       configuration
     ).build()
 

src/releaseNotesBuilder.ts

@@ -1,12 +1,13 @@
 import * as core from '@actions/core'
 import {Configuration} from './configuration'
-import {checkExportedData} from './utils'
+import {checkExportedData, writeCacheData} from './utils'
 import {PullRequestData, buildChangelog} from './transform'
 import {PullRequestCollector} from './pr-collector/prCollector'
 import {failOrError} from './pr-collector/utils'
 import {TagInfo} from './pr-collector/tags'
 import {DiffInfo} from './pr-collector/commits'
 import {PullRequestInfo} from './pr-collector/pullRequests'
+import * as fs from 'fs'
 
 export interface ReleaseNotesOptions {
   owner: string // the owner of the repository
@@ -47,11 +48,18 @@ export class ReleaseNotesBuilder {
     private commitMode = false,
     private exportCache = false,
     private exportOnly = false,
+    private cache: string | null = null,
     private configuration: Configuration
   ) {}
 
   async build(): Promise<string | null> {
-    const releaseNotesData = checkExportedData()
+    let releaseNotesData: Data | null
+    try {
+      releaseNotesData = checkExportedData(this.exportCache, this.cache)
+    } catch (error) {
+      failOrError(`${error}`, this.failOnError)
+      return null
+    }
     if (releaseNotesData == null) {
       if (!this.owner) {
         failOrError(`💥 Missing or couldn't resolve 'owner'`, this.failOnError)
@@ -109,13 +117,13 @@ export class ReleaseNotesBuilder {
       this.setOutputs(options, diffInfo, mergedPullRequests)
 
       if (this.exportCache) {
-        const cache = {
+        const cacheData = {
           mergedPullRequests,
           diffInfo,
           options
         }
-        core.setOutput(`cache`, JSON.stringify(cache))
+
-        //fs.writeFileSync(path.resolve('cache.json'), JSON.stringify(cache))
+        writeCacheData(cacheData, this.cache)
 
         if (this.exportOnly) {
           core.info(`ℹ️ Enabled 'exportOnly' will not generate changelog`)

src/utils.ts

@@ -6,6 +6,7 @@ import moment from 'moment'
 import {DiffInfo} from './pr-collector/commits'
 import {PullRequestInfo} from './pr-collector/pullRequests'
 import {Data, ReleaseNotesOptions} from './releaseNotesBuilder'
+import {env} from 'process'
 /**
  * Resolves the repository path, relatively to the GITHUB_WORKSPACE
  */
@@ -23,15 +24,53 @@ export function retrieveRepositoryPath(providedPath: string): string {
   return repositoryPath
 }
 
+export function writeCacheData(data: Data, cacheOutput: string | null): void {
+  let cacheFile: string
+
+  if (cacheOutput && !cacheOutput.startsWith('{')) {
+    // legacy handling, originally we allowed cache as direct string.
+    // However, this can result in a "Argument list too long" exception for very long caches
+    cacheFile = cacheOutput
+  } else {
+    if (env.RUNNER_TEMP && !fs.existsSync(env.RUNNER_TEMP)) {
+      fs.mkdirSync(env.RUNNER_TEMP)
+    }
+    cacheFile = `${env.RUNNER_TEMP}/rcba-cache.json`
+    core.debug(`Defined cacheFile as ${cacheFile}`)
+  }
+
+  try {
+    fs.writeFileSync(cacheFile, JSON.stringify(data))
+    core.setOutput(`cache`, cacheFile)
+  } catch (error) {
+    core.warning(`Failed to write cache file. (${error})`)
+  }
+}
+
 /**
  * Retrieves the exported information from a previous run of the `release-changelog-builder-action`.
  * If available, return a [ReleaseNotesData].
  */
-export function checkExportedData(): Data | null {
+export function checkExportedData(exportCache: boolean, cacheInput: string | null): Data | null {
-  const rawCache = core.getInput(`cache`)
+  if (exportCache) {
+    return null
+  }
+  if (cacheInput) {
+    // legacy handling, originally we allowed cache as direct string.
+    // However, this can result in a "Argument list too long" exception for very long caches
+    const legacyJsonCache = cacheInput.startsWith('{')
+
+    let cache: Data
+    if (legacyJsonCache) {
+      cache = JSON.parse(cacheInput)
+    } else {
+      if (!fs.existsSync(cacheInput)) {
+        throw new Error(`💥 The provided cache file does not exist`)
+      } else {
+        cache = JSON.parse(fs.readFileSync(cacheInput, 'utf8'))
+      }
+    }
 
-  if (rawCache) {
-    const cache: Data = JSON.parse(rawCache)
     const diffInfo: DiffInfo = cache.diffInfo
     const mergedPullRequests: PullRequestInfo[] = cache.mergedPullRequests