Detect cached folders from multiple directories (#735)

* Add project-dir * Fix find lock file * Remove package-dir input * format & resolve conflicts * Add unit tests * build dist * Apply change request fixes * handle non-dir cache-dependency-path * bump cache version * run checks * Handle globs in cacheDependencyPath * refactor, introduce `cacheDependencyPathToProjectsDirectories` it is necessary for the next PR related yarn optimization * Changes requests * Apply fixes * review fixes * add e2e * Add unique * review updates * review updates second stage * Review fixes 3 * imporve e2e tests
2025-10-31 16:14:00 +08:00 · 2023-06-21 17:52:17 +02:00
parent 698d50532e
commit 8170e22e8f
16 changed files with 3445 additions and 1496 deletions
--- a/src/cache-restore.ts
+++ b/src/cache-restore.ts
@@ -6,14 +6,14 @@ import fs from 'fs';

 import {State} from './constants';
 import {
-  getCacheDirectoryPath,
+  getCacheDirectories,
  getPackageManagerInfo,
  PackageManagerInfo
 } from './cache-utils';

 export const restoreCache = async (
  packageManager: string,
-  cacheDependencyPath?: string
+  cacheDependencyPath: string
 ) => {
  const packageManagerInfo = await getPackageManagerInfo(packageManager);
  if (!packageManagerInfo) {
@@ -21,10 +21,11 @@ export const restoreCache = async (
  }
  const platform = process.env.RUNNER_OS;

-  const cachePath = await getCacheDirectoryPath(
+  const cachePaths = await getCacheDirectories(
    packageManagerInfo,
-    packageManager
+    cacheDependencyPath
  );
+  core.saveState(State.CachePaths, cachePaths);
  const lockFilePath = cacheDependencyPath
    ? cacheDependencyPath
    : findLockFile(packageManagerInfo);
@@ -41,7 +42,7 @@ export const restoreCache = async (

  core.saveState(State.CachePrimaryKey, primaryKey);

-  const cacheKey = await cache.restoreCache([cachePath], primaryKey);
+  const cacheKey = await cache.restoreCache(cachePaths, primaryKey);
  core.setOutput('cache-hit', Boolean(cacheKey));

  if (!cacheKey) {
@@ -56,6 +57,7 @@ export const restoreCache = async (
 const findLockFile = (packageManager: PackageManagerInfo) => {
  const lockFiles = packageManager.lockFilePatterns;
  const workspace = process.env.GITHUB_WORKSPACE!;
+
  const rootContent = fs.readdirSync(workspace);

  const lockFile = lockFiles.find(item => rootContent.includes(item));
--- a/src/cache-save.ts
+++ b/src/cache-save.ts
@@ -1,8 +1,7 @@
 import * as core from '@actions/core';
 import * as cache from '@actions/cache';
-import fs from 'fs';
 import {State} from './constants';
-import {getCacheDirectoryPath, getPackageManagerInfo} from './cache-utils';
+import {getPackageManagerInfo} from './cache-utils';

 // Catch and log any unhandled exceptions.  These exceptions can leak out of the uploadChunk method in
 // @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to
@@ -24,6 +23,7 @@ export async function run() {
 const cachePackages = async (packageManager: string) => {
  const state = core.getState(State.CacheMatchedKey);
  const primaryKey = core.getState(State.CachePrimaryKey);
+  const cachePaths = JSON.parse(core.getState(State.CachePaths) || '[]');

  const packageManagerInfo = await getPackageManagerInfo(packageManager);
  if (!packageManagerInfo) {
@@ -31,14 +31,12 @@ const cachePackages = async (packageManager: string) => {
    return;
  }

-  const cachePath = await getCacheDirectoryPath(
-    packageManagerInfo,
-    packageManager
-  );
-
-  if (!fs.existsSync(cachePath)) {
+  if (cachePaths.length === 0) {
+    // TODO: core.getInput has a bug - it can return undefined despite its definition (tests only?)
+    //       export declare function getInput(name: string, options?: InputOptions): string;
+    const cacheDependencyPath = core.getInput('cache-dependency-path') || '';
    throw new Error(
-      `Cache folder path is retrieved for ${packageManager} but doesn't exist on disk: ${cachePath}`
+      `Cache folder paths are not retrieved for ${packageManager} with cache-dependency-path = ${cacheDependencyPath}`
    );
  }

@@ -49,7 +47,7 @@ const cachePackages = async (packageManager: string) => {
    return;
  }

-  const cacheId = await cache.saveCache([cachePath], primaryKey);
+  const cacheId = await cache.saveCache(cachePaths, primaryKey);
  if (cacheId == -1) {
    return;
  }
--- a/src/cache-utils.ts
+++ b/src/cache-utils.ts
@@ -1,40 +1,79 @@
 import * as core from '@actions/core';
 import * as exec from '@actions/exec';
 import * as cache from '@actions/cache';
-
-type SupportedPackageManagers = {
-  [prop: string]: PackageManagerInfo;
-};
+import * as glob from '@actions/glob';
+import path from 'path';
+import fs from 'fs';
+import {unique} from './util';

 export interface PackageManagerInfo {
+  name: string;
  lockFilePatterns: Array<string>;
-  getCacheFolderCommand: string;
+  getCacheFolderPath: (projectDir?: string) => Promise<string>;
 }

+interface SupportedPackageManagers {
+  npm: PackageManagerInfo;
+  pnpm: PackageManagerInfo;
+  yarn: PackageManagerInfo;
+}
 export const supportedPackageManagers: SupportedPackageManagers = {
  npm: {
+    name: 'npm',
    lockFilePatterns: ['package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock'],
-    getCacheFolderCommand: 'npm config get cache'
+    getCacheFolderPath: () =>
+      getCommandOutputNotEmpty(
+        'npm config get cache',
+        'Could not get npm cache folder path'
+      )
  },
  pnpm: {
+    name: 'pnpm',
    lockFilePatterns: ['pnpm-lock.yaml'],
-    getCacheFolderCommand: 'pnpm store path --silent'
+    getCacheFolderPath: () =>
+      getCommandOutputNotEmpty(
+        'pnpm store path --silent',
+        'Could not get pnpm cache folder path'
+      )
  },
-  yarn1: {
+  yarn: {
+    name: 'yarn',
    lockFilePatterns: ['yarn.lock'],
-    getCacheFolderCommand: 'yarn cache dir'
-  },
-  yarn2: {
-    lockFilePatterns: ['yarn.lock'],
-    getCacheFolderCommand: 'yarn config get cacheFolder'
+    getCacheFolderPath: async projectDir => {
+      const yarnVersion = await getCommandOutputNotEmpty(
+        `yarn --version`,
+        'Could not retrieve version of yarn',
+        projectDir
+      );
+
+      core.debug(
+        `Consumed yarn version is ${yarnVersion} (working dir: "${
+          projectDir || ''
+        }")`
+      );
+
+      const stdOut = yarnVersion.startsWith('1.')
+        ? await getCommandOutput('yarn cache dir', projectDir)
+        : await getCommandOutput('yarn config get cacheFolder', projectDir);
+
+      if (!stdOut) {
+        throw new Error(
+          `Could not get yarn cache folder path for ${projectDir}`
+        );
+      }
+      return stdOut;
+    }
  }
 };

-export const getCommandOutput = async (toolCommand: string) => {
+export const getCommandOutput = async (
+  toolCommand: string,
+  cwd?: string
+): Promise<string> => {
  let {stdout, stderr, exitCode} = await exec.getExecOutput(
    toolCommand,
    undefined,
-    {ignoreReturnCode: true}
+    {ignoreReturnCode: true, ...(cwd && {cwd})}
  );

  if (exitCode) {
@@ -47,16 +86,15 @@ export const getCommandOutput = async (toolCommand: string) => {
  return stdout.trim();
 };

-const getPackageManagerVersion = async (
-  packageManager: string,
-  command: string
-) => {
-  const stdOut = await getCommandOutput(`${packageManager} ${command}`);
-
+export const getCommandOutputNotEmpty = async (
+  toolCommand: string,
+  error: string,
+  cwd?: string
+): Promise<string> => {
+  const stdOut = getCommandOutput(toolCommand, cwd);
  if (!stdOut) {
-    throw new Error(`Could not retrieve version of ${packageManager}`);
+    throw new Error(error);
  }
-
  return stdOut;
 };

@@ -66,35 +104,102 @@ export const getPackageManagerInfo = async (packageManager: string) => {
  } else if (packageManager === 'pnpm') {
    return supportedPackageManagers.pnpm;
  } else if (packageManager === 'yarn') {
-    const yarnVersion = await getPackageManagerVersion('yarn', '--version');
-
-    core.debug(`Consumed yarn version is ${yarnVersion}`);
-
-    if (yarnVersion.startsWith('1.')) {
-      return supportedPackageManagers.yarn1;
-    } else {
-      return supportedPackageManagers.yarn2;
-    }
+    return supportedPackageManagers.yarn;
  } else {
    return null;
  }
 };

-export const getCacheDirectoryPath = async (
+/**
+ * Expands (converts) the string input `cache-dependency-path` to list of directories that
+ * may be project roots
+ * @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
+ *                              expected to be the result of `core.getInput('cache-dependency-path')`
+ * @return list of directories and possible
+ */
+const getProjectDirectoriesFromCacheDependencyPath = async (
+  cacheDependencyPath: string
+): Promise<string[]> => {
+  const globber = await glob.create(cacheDependencyPath);
+  const cacheDependenciesPaths = await globber.glob();
+
+  const existingDirectories: string[] = cacheDependenciesPaths
+    .map(path.dirname)
+    .filter(unique())
+    .filter(directory => fs.lstatSync(directory).isDirectory());
+
+  if (!existingDirectories.length)
+    core.warning(
+      `No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`
+    );
+
+  return existingDirectories;
+};
+
+/**
+ * Finds the cache directories configured for the repo if cache-dependency-path is not empty
+ * @param packageManagerInfo - an object having getCacheFolderPath method specific to given PM
+ * @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
+ *                              expected to be the result of `core.getInput('cache-dependency-path')`
+ * @return list of files on which the cache depends
+ */
+const getCacheDirectoriesFromCacheDependencyPath = async (
  packageManagerInfo: PackageManagerInfo,
-  packageManager: string
-) => {
-  const stdOut = await getCommandOutput(
-    packageManagerInfo.getCacheFolderCommand
+  cacheDependencyPath: string
+): Promise<string[]> => {
+  const projectDirectories = await getProjectDirectoriesFromCacheDependencyPath(
+    cacheDependencyPath
  );
+  const cacheFoldersPaths = await Promise.all(
+    projectDirectories.map(async projectDirectory => {
+      const cacheFolderPath =
+        packageManagerInfo.getCacheFolderPath(projectDirectory);
+      core.debug(
+        `${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`
+      );
+      return cacheFolderPath;
+    })
+  );
+  // uniq in order to do not cache the same directories twice
+  return cacheFoldersPaths.filter(unique());
+};

-  if (!stdOut) {
-    throw new Error(`Could not get cache folder path for ${packageManager}`);
+/**
+ * Finds the cache directories configured for the repo ignoring cache-dependency-path
+ * @param packageManagerInfo - an object having getCacheFolderPath method specific to given PM
+ * @return list of files on which the cache depends
+ */
+const getCacheDirectoriesForRootProject = async (
+  packageManagerInfo: PackageManagerInfo
+): Promise<string[]> => {
+  const cacheFolderPath = await packageManagerInfo.getCacheFolderPath();
+  core.debug(
+    `${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the root directory`
+  );
+  return [cacheFolderPath];
+};
+
+/**
+ * A function to find the cache directories configured for the repo
+ * currently it handles only the case of PM=yarn && cacheDependencyPath is not empty
+ * @param packageManagerInfo - an object having getCacheFolderPath method specific to given PM
+ * @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
+ *                              expected to be the result of `core.getInput('cache-dependency-path')`
+ * @return list of files on which the cache depends
+ */
+export const getCacheDirectories = async (
+  packageManagerInfo: PackageManagerInfo,
+  cacheDependencyPath: string
+): Promise<string[]> => {
+  // For yarn, if cacheDependencyPath is set, ask information about cache folders in each project
+  // folder satisfied by cacheDependencyPath https://github.com/actions/setup-node/issues/488
+  if (packageManagerInfo.name === 'yarn' && cacheDependencyPath) {
+    return getCacheDirectoriesFromCacheDependencyPath(
+      packageManagerInfo,
+      cacheDependencyPath
+    );
  }
-
-  core.debug(`${packageManager} path is ${stdOut}`);
-
-  return stdOut.trim();
+  return getCacheDirectoriesForRootProject(packageManagerInfo);
 };

 export function isGhes(): boolean {
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -6,7 +6,8 @@ export enum LockType {

 export enum State {
  CachePrimaryKey = 'CACHE_KEY',
-  CacheMatchedKey = 'CACHE_RESULT'
+  CacheMatchedKey = 'CACHE_RESULT',
+  CachePaths = 'CACHE_PATHS'
 }

 export enum Outputs {
--- a/src/util.ts
+++ b/src/util.ts
@@ -61,3 +61,12 @@ async function getToolVersion(tool: string, options: string[]) {
    return '';
  }
 }
+
+export const unique = () => {
+  const encountered = new Set();
+  return (value: unknown): boolean => {
+    if (encountered.has(value)) return false;
+    encountered.add(value);
+    return true;
+  };
+};