bazel-contrib/setup-bazel
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Use manifests for external cache to avoid overfetching

Browse source
This commit is contained in:
Alex Rodionov 2023-01-11 13:40:42 -08:00
parent 74154140df
commit 3c0903ad5b
10 changed files with 3105 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

13
README.md
View file

@ -53,13 +53,13 @@ Default [`${{ github.token }}`][5].
### Simple configuration
```yaml
- uses: p0deje/setup-bazel@0.1.0
- uses: p0deje/setup-bazel@0.2.0
```
### Additional `.bazelrc` options
```yaml
- uses: p0deje/setup-bazel@0.1.0
- uses: p0deje/setup-bazel@0.2.0
with:
bazelrc: |
build --show_timestamps
@ -68,7 +68,7 @@ Default [`${{ github.token }}`][5].
### Full caching enabled
```yaml
- uses: p0deje/setup-bazel@0.1.0
- uses: p0deje/setup-bazel@0.2.0
with:
bazelisk-cache: true
disk-cache: true
@ -79,7 +79,7 @@ Default [`${{ github.token }}`][5].
### Separate disk cache between workflows
```yaml
- uses: p0deje/setup-bazel@0.1.0
- uses: p0deje/setup-bazel@0.2.0
with:
disk-cache: ${{ github.workflow }}}
```
@ -87,10 +87,11 @@ Default [`${{ github.token }}`][5].
### Cache external repository based on different files
```yaml
- uses: p0deje/setup-bazel@0.1.0
- uses: p0deje/setup-bazel@0.2.0
with:
external-cache: |
npm: package-lock.json
manifest:
npm: package-lock.json
```
[1]: https://github.com/bazelbuild/bazelisk

4
action.yml
View file

@ -29,10 +29,6 @@ inputs:
description: Cache repositories based on WORKSPACE
required: false
default: false
token:
description: GitHub token to use for external cache
required: false
default: ${{ github.token }}
runs:
using: node16

37
config.js
View file

@ -2,6 +2,7 @@ const fs = require('fs')
const os = require('os')
const yaml = require('yaml')
const core = require('@actions/core')
const github = require('@actions/github')
const cacheVersion = core.getInput('cache-version')
const externalCacheConfig = yaml.parse(core.getInput('external-cache'))
@ -57,29 +58,38 @@ if (googleCredentials.length > 0 && !googleCredentialsSaved) {
const bazelExternal = core.toPosixPath(`${bazelOutputBase}/external`)
const externalCache = {}
if (externalCacheConfig) {
const { workflow, job } = github.context
const manifestName = externalCacheConfig.name ||
`${workflow.toLowerCase().replaceAll(/[ /]/g, '-')}-${job}`
externalCache.enabled = true
externalCache.minSize = 10 // MB
externalCache.baseCacheKey = `${baseCacheKey}-external-`
externalCache.regexp = `^${baseCacheKey}-external-(?<name>.+)-[a-z0-9]+$`
externalCache.manifest = {
files: [
'WORKSPACE.bazel',
'WORKSPACE'
],
name: `external-${manifestName}-manifest`,
path: `${os.tmpdir()}/external-cache-manifest.txt`
}
externalCache.default = {
files: [
'WORKSPACE.bazel',
'WORKSPACE'
]
}
externalCache.name = (name) => {
return `external-${name}`
}
externalCache.paths = (name) => {
return [
`${bazelExternal}/@${name}.marker`,
`${bazelExternal}/${name}`
]
],
name: (name) => { return `external-${name}` },
paths: (name) => {
return [
`${bazelExternal}/@${name}.marker`,
`${bazelExternal}/${name}`
]
}
}
for (const name in externalCacheConfig) {
for (const name in externalCacheConfig.manifest) {
externalCache[name] = {
files: Array(externalCacheConfig[name]).flat()
files: Array(externalCacheConfig.manifest[name]).flat()
}
}
}
@ -118,5 +128,4 @@ module.exports = {
name: 'repository',
paths: [bazelRepository]
},
token: core.getInput('token')
}

98
dist/main/index.js vendored
View file

@ -8,6 +8,7 @@ const fs = __nccwpck_require__(7147)
const os = __nccwpck_require__(2037)
const yaml = __nccwpck_require__(4083)
const core = __nccwpck_require__(2186)
const github = __nccwpck_require__(5438)
const cacheVersion = core.getInput('cache-version')
const externalCacheConfig = yaml.parse(core.getInput('external-cache'))
@ -63,29 +64,38 @@ if (googleCredentials.length > 0 && !googleCredentialsSaved) {
const bazelExternal = core.toPosixPath(`${bazelOutputBase}/external`)
const externalCache = {}
if (externalCacheConfig) {
const { workflow, job } = github.context
const manifestName = externalCacheConfig.name ||
`${workflow.toLowerCase().replaceAll(/[ /]/g, '-')}-${job}`
externalCache.enabled = true
externalCache.minSize = 10 // MB
externalCache.baseCacheKey = `${baseCacheKey}-external-`
externalCache.regexp = `^${baseCacheKey}-external-(?<name>.+)-[a-z0-9]+$`
externalCache.manifest = {
files: [
'WORKSPACE.bazel',
'WORKSPACE'
],
name: `external-${manifestName}-manifest`,
path: `${os.tmpdir()}/external-cache-manifest.txt`
}
externalCache.default = {
files: [
'WORKSPACE.bazel',
'WORKSPACE'
]
}
externalCache.name = (name) => {
return `external-${name}`
}
externalCache.paths = (name) => {
return [
`${bazelExternal}/@${name}.marker`,
`${bazelExternal}/${name}`
]
],
name: (name) => { return `external-${name}` },
paths: (name) => {
return [
`${bazelExternal}/@${name}.marker`,
`${bazelExternal}/${name}`
]
}
}
for (const name in externalCacheConfig) {
for (const name in externalCacheConfig.manifest) {
externalCache[name] = {
files: Array(externalCacheConfig[name]).flat()
files: Array(externalCacheConfig.manifest[name]).flat()
}
}
}
@ -124,7 +134,6 @@ module.exports = {
name: 'repository',
paths: [bazelRepository]
},
token: core.getInput('token')
}
@ -66132,14 +66141,6 @@ module.exports = require("net");
/***/ }),
/***/ 9397:
/***/ ((module) => {
"use strict";
module.exports = require("node:timers/promises");
/***/ }),
/***/ 2037:
/***/ ((module) => {
@ -74616,10 +74617,8 @@ var __webpack_exports__ = {};
// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk.
(() => {
const fs = __nccwpck_require__(7147)
const { setTimeout } = __nccwpck_require__(9397)
const core = __nccwpck_require__(2186)
const cache = __nccwpck_require__(7799)
const github = __nccwpck_require__(5438)
const glob = __nccwpck_require__(8090)
const config = __nccwpck_require__(5532)
@ -74661,34 +74660,27 @@ async function restoreExternalCaches (cacheConfig) {
return
}
const repo = github.context.repo
const octokit = github.getOctokit(config.token)
const { data: { actions_caches: caches } } = await octokit.rest.actions.getActionsCacheList({
owner: repo.owner,
repo: repo.repo,
key: cacheConfig.baseCacheKey,
per_page: 100
// First fetch the manifest of external caches used.
const path = cacheConfig.manifest.path
await restoreCache({
enabled: true,
files: cacheConfig.manifest.files,
name: cacheConfig.manifest.name,
paths: [path]
})
const names = new Set([])
const regexp = new RegExp(cacheConfig.regexp)
for (const cache of caches) {
core.debug(`Cache key is ${cache.key}`)
const match = cache.key.match(regexp)
if (match) {
names.add(match.groups.name)
// Now restore all external caches defined in manifest
if (fs.existsSync(path)) {
const manifest = fs.readFileSync(path, { encoding: 'utf8' })
for (const name of manifest.split('\n').filter(s => s)) {
await restoreCache({
enabled: true,
files: cacheConfig[name]?.files || cacheConfig.default.files,
name: cacheConfig.default.name(name),
paths: cacheConfig.default.paths(name)
})
}
}
for (const name of names) {
await restoreCache({
enabled: true,
files: cacheConfig[name]?.files || cacheConfig.default.files,
name: cacheConfig.name(name),
paths: cacheConfig.paths(name)
})
}
}
async function restoreCache (cacheConfig) {
@ -74706,12 +74698,10 @@ async function restoreCache (cacheConfig) {
console.log(`Attempting to restore ${name} cache from ${key}`)
const restoredKey = await setTimeout(1000, async function() {
return await cache.restoreCache(
paths, key, [restoreKey],
{ segmentTimeoutInMs: 300000 } // 5 minutes
)
}())
const restoredKey = await cache.restoreCache(
paths, key, [restoreKey],
{ segmentTimeoutInMs: 300000 } // 5 minutes
)
if (restoredKey) {
console.log(`Successfully restored cache from ${restoredKey}`)

2
dist/main/index.js.map vendored
View file

File diff suppressed because one or more lines are too long

3007
dist/post/index.js vendored
View file

File diff suppressed because one or more lines are too long

2
dist/post/index.js.map vendored
View file

File diff suppressed because one or more lines are too long

53
index.js
View file

@ -1,8 +1,6 @@
const fs = require('fs')
const { setTimeout } = require('node:timers/promises')
const core = require('@actions/core')
const cache = require('@actions/cache')
const github = require('@actions/github')
const glob = require('@actions/glob')
const config = require('./config')
@ -44,34 +42,27 @@ async function restoreExternalCaches (cacheConfig) {
return
}
const repo = github.context.repo
const octokit = github.getOctokit(config.token)
const { data: { actions_caches: caches } } = await octokit.rest.actions.getActionsCacheList({
owner: repo.owner,
repo: repo.repo,
key: cacheConfig.baseCacheKey,
per_page: 100
// First fetch the manifest of external caches used.
const path = cacheConfig.manifest.path
await restoreCache({
enabled: true,
files: cacheConfig.manifest.files,
name: cacheConfig.manifest.name,
paths: [path]
})
const names = new Set([])
const regexp = new RegExp(cacheConfig.regexp)
for (const cache of caches) {
core.debug(`Cache key is ${cache.key}`)
const match = cache.key.match(regexp)
if (match) {
names.add(match.groups.name)
// Now restore all external caches defined in manifest
if (fs.existsSync(path)) {
const manifest = fs.readFileSync(path, { encoding: 'utf8' })
for (const name of manifest.split('\n').filter(s => s)) {
await restoreCache({
enabled: true,
files: cacheConfig[name]?.files || cacheConfig.default.files,
name: cacheConfig.default.name(name),
paths: cacheConfig.default.paths(name)
})
}
}
for (const name of names) {
await restoreCache({
enabled: true,
files: cacheConfig[name]?.files || cacheConfig.default.files,
name: cacheConfig.name(name),
paths: cacheConfig.paths(name)
})
}
}
async function restoreCache (cacheConfig) {
@ -89,12 +80,10 @@ async function restoreCache (cacheConfig) {
console.log(`Attempting to restore ${name} cache from ${key}`)
const restoredKey = await setTimeout(1000, async function() {
return await cache.restoreCache(
paths, key, [restoreKey],
{ segmentTimeoutInMs: 300000 } // 5 minutes
)
}())
const restoredKey = await cache.restoreCache(
paths, key, [restoreKey],
{ segmentTimeoutInMs: 300000 } // 5 minutes
)
if (restoredKey) {
console.log(`Successfully restored cache from ${restoredKey}`)

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "setup-bazel",
"version": "0.1.0",
"version": "0.2.0",
"description": "Install and configure Bazel for GitHub Actions",
"main": "index.js",
"engines" : {

18
post.js
View file

@ -1,3 +1,4 @@
const fs = require('fs')
const path = require('path')
const cache = require('@actions/cache')
const core = require('@actions/core')
@ -26,6 +27,7 @@ async function saveExternalCaches (cacheConfig) {
{ implicitDescendants: false }
)
const externalPaths = await globber.glob()
const savedCaches = []
for (const externalPath of externalPaths) {
const size = await getFolderSize(externalPath)
@ -37,11 +39,23 @@ async function saveExternalCaches (cacheConfig) {
await saveCache({
enabled: true,
files: cacheConfig[name]?.files || cacheConfig.default.files,
name: cacheConfig.name(name),
paths: cacheConfig.paths(name)
name: cacheConfig.default.name(name),
paths: cacheConfig.default.paths(name)
})
savedCaches.push(name)
}
}
if (savedCaches.length > 0) {
const path = cacheConfig.manifest.path
fs.writeFileSync(path, savedCaches.join('\n'))
await saveCache({
enabled: true,
files: cacheConfig.manifest.files,
name: cacheConfig.manifest.name,
paths: [path]
})
}
}
async function saveCache (cacheConfig) {