Use manifests for external cache to avoid overfetching

README.md

@@ -53,13 +53,13 @@ Default [`${{ github.token }}`][5].
 ### Simple configuration
 
 ```yaml
-- uses: p0deje/setup-bazel@0.1.0
+- uses: p0deje/setup-bazel@0.2.0
 ```
 
 ### Additional `.bazelrc` options
 
 ```yaml
-- uses: p0deje/setup-bazel@0.1.0
+- uses: p0deje/setup-bazel@0.2.0
   with:
     bazelrc: |
       build --show_timestamps
@@ -68,7 +68,7 @@ Default [`${{ github.token }}`][5].
 ### Full caching enabled
 
 ```yaml
-- uses: p0deje/setup-bazel@0.1.0
+- uses: p0deje/setup-bazel@0.2.0
   with:
     bazelisk-cache: true
     disk-cache: true
@@ -79,7 +79,7 @@ Default [`${{ github.token }}`][5].
 ### Separate disk cache between workflows
 
 ```yaml
-- uses: p0deje/setup-bazel@0.1.0
+- uses: p0deje/setup-bazel@0.2.0
   with:
     disk-cache: ${{ github.workflow }}}
 ```
@@ -87,9 +87,10 @@ Default [`${{ github.token }}`][5].
 ### Cache external repository based on different files
 
 ```yaml
-- uses: p0deje/setup-bazel@0.1.0
+- uses: p0deje/setup-bazel@0.2.0
   with:
     external-cache: |
+      manifest:
         npm: package-lock.json
 ```
 

action.yml

@@ -29,10 +29,6 @@ inputs:
     description: Cache repositories based on WORKSPACE
     required: false
     default: false
-  token:
-    description: GitHub token to use for external cache
-    required: false
-    default: ${{ github.token }}
 
 runs:
   using: node16

config.js

@@ -2,6 +2,7 @@ const fs = require('fs')
 const os = require('os')
 const yaml = require('yaml')
 const core = require('@actions/core')
+const github = require('@actions/github')
 
 const cacheVersion = core.getInput('cache-version')
 const externalCacheConfig = yaml.parse(core.getInput('external-cache'))
@@ -57,29 +58,38 @@ if (googleCredentials.length > 0 && !googleCredentialsSaved) {
 const bazelExternal = core.toPosixPath(`${bazelOutputBase}/external`)
 const externalCache = {}
 if (externalCacheConfig) {
+  const { workflow, job } = github.context
+  const manifestName = externalCacheConfig.name ||
+    `${workflow.toLowerCase().replaceAll(/[ /]/g, '-')}-${job}`
+
   externalCache.enabled = true
   externalCache.minSize = 10 // MB
   externalCache.baseCacheKey = `${baseCacheKey}-external-`
-  externalCache.regexp = `^${baseCacheKey}-external-(?<name>.+)-[a-z0-9]+$`
+  externalCache.manifest = {
+    files: [
+      'WORKSPACE.bazel',
+      'WORKSPACE'
+    ],
+    name: `external-${manifestName}-manifest`,
+    path: `${os.tmpdir()}/external-cache-manifest.txt`
+  }
   externalCache.default = {
     files: [
       'WORKSPACE.bazel',
       'WORKSPACE'
-    ]
-  }
-  externalCache.name = (name) => {
-    return `external-${name}`
-  }
-  externalCache.paths = (name) => {
+    ],
+    name: (name) => { return `external-${name}` },
+    paths: (name) => {
       return [
         `${bazelExternal}/@${name}.marker`,
         `${bazelExternal}/${name}`
       ]
     }
+  }
 
-  for (const name in externalCacheConfig) {
+  for (const name in externalCacheConfig.manifest) {
     externalCache[name] = {
-      files: Array(externalCacheConfig[name]).flat()
+      files: Array(externalCacheConfig.manifest[name]).flat()
     }
   }
 }
@@ -118,5 +128,4 @@ module.exports = {
     name: 'repository',
     paths: [bazelRepository]
   },
-  token: core.getInput('token')
 }

dist/main/index.js

@@ -8,6 +8,7 @@ const fs = __nccwpck_require__(7147)
 const os = __nccwpck_require__(2037)
 const yaml = __nccwpck_require__(4083)
 const core = __nccwpck_require__(2186)
+const github = __nccwpck_require__(5438)
 
 const cacheVersion = core.getInput('cache-version')
 const externalCacheConfig = yaml.parse(core.getInput('external-cache'))
@@ -63,29 +64,38 @@ if (googleCredentials.length > 0 && !googleCredentialsSaved) {
 const bazelExternal = core.toPosixPath(`${bazelOutputBase}/external`)
 const externalCache = {}
 if (externalCacheConfig) {
+  const { workflow, job } = github.context
+  const manifestName = externalCacheConfig.name ||
+    `${workflow.toLowerCase().replaceAll(/[ /]/g, '-')}-${job}`
+
   externalCache.enabled = true
   externalCache.minSize = 10 // MB
   externalCache.baseCacheKey = `${baseCacheKey}-external-`
-  externalCache.regexp = `^${baseCacheKey}-external-(?<name>.+)-[a-z0-9]+$`
+  externalCache.manifest = {
+    files: [
+      'WORKSPACE.bazel',
+      'WORKSPACE'
+    ],
+    name: `external-${manifestName}-manifest`,
+    path: `${os.tmpdir()}/external-cache-manifest.txt`
+  }
   externalCache.default = {
     files: [
       'WORKSPACE.bazel',
       'WORKSPACE'
-    ]
-  }
-  externalCache.name = (name) => {
-    return `external-${name}`
-  }
-  externalCache.paths = (name) => {
+    ],
+    name: (name) => { return `external-${name}` },
+    paths: (name) => {
       return [
         `${bazelExternal}/@${name}.marker`,
         `${bazelExternal}/${name}`
       ]
     }
+  }
 
-  for (const name in externalCacheConfig) {
+  for (const name in externalCacheConfig.manifest) {
     externalCache[name] = {
-      files: Array(externalCacheConfig[name]).flat()
+      files: Array(externalCacheConfig.manifest[name]).flat()
     }
   }
 }
@@ -124,7 +134,6 @@ module.exports = {
     name: 'repository',
     paths: [bazelRepository]
   },
-  token: core.getInput('token')
 }
 
 
@@ -66132,14 +66141,6 @@ module.exports = require("net");
 
 /***/ }),
 
-/***/ 9397:
-/***/ ((module) => {
-
-"use strict";
-module.exports = require("node:timers/promises");
-
-/***/ }),
-
 /***/ 2037:
 /***/ ((module) => {
 
@@ -74616,10 +74617,8 @@ var __webpack_exports__ = {};
 // This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk.
 (() => {
 const fs = __nccwpck_require__(7147)
-const { setTimeout } = __nccwpck_require__(9397)
 const core = __nccwpck_require__(2186)
 const cache = __nccwpck_require__(7799)
-const github = __nccwpck_require__(5438)
 const glob = __nccwpck_require__(8090)
 const config = __nccwpck_require__(5532)
 
@@ -74661,34 +74660,27 @@ async function restoreExternalCaches (cacheConfig) {
     return
   }
 
-  const repo = github.context.repo
-  const octokit = github.getOctokit(config.token)
-  const { data: { actions_caches: caches } } = await octokit.rest.actions.getActionsCacheList({
-    owner: repo.owner,
-    repo: repo.repo,
-    key: cacheConfig.baseCacheKey,
-    per_page: 100
+  // First fetch the manifest of external caches used.
+  const path = cacheConfig.manifest.path
+  await restoreCache({
+    enabled: true,
+    files: cacheConfig.manifest.files,
+    name: cacheConfig.manifest.name,
+    paths: [path]
   })
 
-  const names = new Set([])
-  const regexp = new RegExp(cacheConfig.regexp)
-  for (const cache of caches) {
-    core.debug(`Cache key is ${cache.key}`)
-
-    const match = cache.key.match(regexp)
-    if (match) {
-      names.add(match.groups.name)
-    }
-  }
-
-  for (const name of names) {
+  // Now restore all external caches defined in manifest
+  if (fs.existsSync(path)) {
+    const manifest = fs.readFileSync(path, { encoding: 'utf8' })
+    for (const name of manifest.split('\n').filter(s => s)) {
       await restoreCache({
         enabled: true,
         files: cacheConfig[name]?.files || cacheConfig.default.files,
-      name: cacheConfig.name(name),
-      paths: cacheConfig.paths(name)
+        name: cacheConfig.default.name(name),
+        paths: cacheConfig.default.paths(name)
       })
     }
+  }
 }
 
 async function restoreCache (cacheConfig) {
@@ -74706,12 +74698,10 @@ async function restoreCache (cacheConfig) {
 
   console.log(`Attempting to restore ${name} cache from ${key}`)
 
-  const restoredKey = await setTimeout(1000, async function() {
-    return await cache.restoreCache(
+  const restoredKey = await cache.restoreCache(
     paths, key, [restoreKey],
     { segmentTimeoutInMs: 300000 } // 5 minutes
   )
-  }())
 
   if (restoredKey) {
     console.log(`Successfully restored cache from ${restoredKey}`)

index.js

@@ -1,8 +1,6 @@
 const fs = require('fs')
-const { setTimeout } = require('node:timers/promises')
 const core = require('@actions/core')
 const cache = require('@actions/cache')
-const github = require('@actions/github')
 const glob = require('@actions/glob')
 const config = require('./config')
 
@@ -44,34 +42,27 @@ async function restoreExternalCaches (cacheConfig) {
     return
   }
 
-  const repo = github.context.repo
-  const octokit = github.getOctokit(config.token)
-  const { data: { actions_caches: caches } } = await octokit.rest.actions.getActionsCacheList({
-    owner: repo.owner,
-    repo: repo.repo,
-    key: cacheConfig.baseCacheKey,
-    per_page: 100
+  // First fetch the manifest of external caches used.
+  const path = cacheConfig.manifest.path
+  await restoreCache({
+    enabled: true,
+    files: cacheConfig.manifest.files,
+    name: cacheConfig.manifest.name,
+    paths: [path]
   })
 
-  const names = new Set([])
-  const regexp = new RegExp(cacheConfig.regexp)
-  for (const cache of caches) {
-    core.debug(`Cache key is ${cache.key}`)
-
-    const match = cache.key.match(regexp)
-    if (match) {
-      names.add(match.groups.name)
-    }
-  }
-
-  for (const name of names) {
+  // Now restore all external caches defined in manifest
+  if (fs.existsSync(path)) {
+    const manifest = fs.readFileSync(path, { encoding: 'utf8' })
+    for (const name of manifest.split('\n').filter(s => s)) {
       await restoreCache({
         enabled: true,
         files: cacheConfig[name]?.files || cacheConfig.default.files,
-      name: cacheConfig.name(name),
-      paths: cacheConfig.paths(name)
+        name: cacheConfig.default.name(name),
+        paths: cacheConfig.default.paths(name)
       })
     }
+  }
 }
 
 async function restoreCache (cacheConfig) {
@@ -89,12 +80,10 @@ async function restoreCache (cacheConfig) {
 
   console.log(`Attempting to restore ${name} cache from ${key}`)
 
-  const restoredKey = await setTimeout(1000, async function() {
-    return await cache.restoreCache(
+  const restoredKey = await cache.restoreCache(
     paths, key, [restoreKey],
     { segmentTimeoutInMs: 300000 } // 5 minutes
   )
-  }())
 
   if (restoredKey) {
     console.log(`Successfully restored cache from ${restoredKey}`)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "setup-bazel",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Install and configure Bazel for GitHub Actions",
   "main": "index.js",
   "engines" : {

post.js

@@ -1,3 +1,4 @@
+const fs = require('fs')
 const path = require('path')
 const cache = require('@actions/cache')
 const core = require('@actions/core')
@@ -26,6 +27,7 @@ async function saveExternalCaches (cacheConfig) {
     { implicitDescendants: false }
   )
   const externalPaths = await globber.glob()
+  const savedCaches = []
 
   for (const externalPath of externalPaths) {
     const size = await getFolderSize(externalPath)
@@ -37,11 +39,23 @@ async function saveExternalCaches (cacheConfig) {
       await saveCache({
         enabled: true,
         files: cacheConfig[name]?.files || cacheConfig.default.files,
-        name: cacheConfig.name(name),
-        paths: cacheConfig.paths(name)
+        name: cacheConfig.default.name(name),
+        paths: cacheConfig.default.paths(name)
       })
+      savedCaches.push(name)
     }
   }
+
+  if (savedCaches.length > 0) {
+    const path = cacheConfig.manifest.path
+    fs.writeFileSync(path, savedCaches.join('\n'))
+    await saveCache({
+      enabled: true,
+      files: cacheConfig.manifest.files,
+      name: cacheConfig.manifest.name,
+      paths: [path]
+    })
+  }
 }
 
 async function saveCache (cacheConfig) {