13 files+1201−1820/13 viewed
| 1 | + | # Keeps, on main, the cache entries that only other workflows' runs would | |
| 2 | + | # otherwise make. A run restores from its own ref, then its pull request's | |
| 3 | + | # base branch, then main (docs.g1t.sh/guides/actions/#the-cache): a release | |
| 4 | + | # tag's run never sees another tag's entries, and before this every release | |
| 5 | + | # built its Rust from the start (desktop: 10 of 14 Linux and 12 of 13 | |
| 6 | + | # Windows builds found nothing). So what a release restores is saved here. | |
| 7 | + | # | |
| 8 | + | # look looks every key up without downloading anything; a | |
| 9 | + | # lookup counts as a use, so a warm entry is never taken | |
| 10 | + | # for an unused one, and an entry past its 28 days is | |
| 11 | + | # simply made again | |
| 12 | + | # npm npm's downloads for the whole lockfile, which every npm | |
| 13 | + | # install in every workflow starts from | |
| 14 | + | # runner the runner release's build, for each target | |
| 15 | + | # desktop-linux the desktop release's Linux build | |
| 16 | + | # desktop-windows its Windows cross build, cargo-xwin, the MSVC CRT and | |
| 17 | + | # apt's downloads for it | |
| 18 | + | # | |
| 19 | + | # A job runs only when its key is missing (a new lockfile, a new toolchain, | |
| 20 | + | # or an expired entry), and builds what the release builds. Nothing waits for | |
| 21 | + | # this workflow, and a failed build only leaves the cache as it was. CI | |
| 22 | + | # keeps the rest of main's caches (ci.yml); CONTRIBUTING.md ("How g1t's own | |
| 23 | + | # CI caches") has the whole scheme, and the paths and keys here must stay | |
| 24 | + | # the same as the steps that restore them (repository_workflows.rs checks). | |
| 25 | + | name: Caches | |
| 26 | + | ||
| 27 | + | on: | |
| 28 | + | push: | |
| 29 | + | branches: [main] | |
| 30 | + | workflow_dispatch: | |
| 31 | + | ||
| 32 | + | permissions: | |
| 33 | + | contents: read | |
| 34 | + | ||
| 35 | + | # A build half done saves nothing, so one is never cut off; the next run | |
| 36 | + | # waits, and finds the key saved. | |
| 37 | + | concurrency: | |
| 38 | + | group: caches-${{ github.ref }} | |
| 39 | + | cancel-in-progress: false | |
| 40 | + | ||
| 41 | + | env: | |
| 42 | + | CARGO_TERM_COLOR: never | |
| 43 | + | ||
| 44 | + | jobs: | |
| 45 | + | look: | |
| 46 | + | name: What is missing | |
| 47 | + | runs-on: ubuntu-latest | |
| 48 | + | timeout-minutes: 10 | |
| 49 | + | outputs: | |
| 50 | + | npm: ${{ steps.npm.outputs.cache-hit != 'true' }} | |
| 51 | + | runner: ${{ steps.missing.outputs.runner }} | |
| 52 | + | desktop-linux: ${{ steps.desktop-linux.outputs.cache-hit != 'true' }} | |
| 53 | + | desktop-windows: ${{ steps.desktop-windows.outputs.cache-hit != 'true' || steps.xwin.outputs.cache-hit != 'true' || steps.apt-windows.outputs.cache-hit != 'true' }} | |
| 54 | + | steps: | |
| 55 | + | - uses: actions/checkout@v5 | |
| 56 | + | - name: Cache keys | |
| 57 | + | id: keys | |
| 58 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 59 | + | - id: npm | |
| 60 | + | uses: actions/cache/restore@v4 | |
| 61 | + | with: | |
| 62 | + | path: ~/.npm | |
| 63 | + | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 64 | + | lookup-only: true | |
| 65 | + | - id: zig | |
| 66 | + | uses: actions/cache/restore@v4 | |
| 67 | + | with: | |
| 68 | + | path: | | |
| 69 | + | ~/.local/lib/python3*/site-packages/ziglang | |
| 70 | + | ~/.local/bin | |
| 71 | + | ~/.cargo/bin/cargo-zigbuild | |
| 72 | + | key: zig-0.13.0-zigbuild-${{ runner.os }}-1 | |
| 73 | + | lookup-only: true | |
| 74 | + | - id: runner-linux-x64 | |
| 75 | + | uses: actions/cache/restore@v4 | |
| 76 | + | with: | |
| 77 | + | path: | | |
| 78 | + | target | |
| 79 | + | !target/runner-release | |
| 80 | + | !target/**/incremental | |
| 81 | + | key: runner-release-${{ runner.os }}-linux-x64-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 82 | + | lookup-only: true | |
| 83 | + | - id: runner-linux-arm64 | |
| 84 | + | uses: actions/cache/restore@v4 | |
| 85 | + | with: | |
| 86 | + | path: | | |
| 87 | + | target | |
| 88 | + | !target/runner-release | |
| 89 | + | !target/**/incremental | |
| 90 | + | key: runner-release-${{ runner.os }}-linux-arm64-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 91 | + | lookup-only: true | |
| 92 | + | - id: runner-windows-x64 | |
| 93 | + | uses: actions/cache/restore@v4 | |
| 94 | + | with: | |
| 95 | + | path: | | |
| 96 | + | target | |
| 97 | + | !target/runner-release | |
| 98 | + | !target/**/incremental | |
| 99 | + | key: runner-release-${{ runner.os }}-windows-x64-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 100 | + | lookup-only: true | |
| 101 | + | - id: desktop-linux | |
| 102 | + | uses: actions/cache/restore@v4 | |
| 103 | + | with: | |
| 104 | + | path: | | |
| 105 | + | ~/.cargo/registry/index | |
| 106 | + | ~/.cargo/registry/cache | |
| 107 | + | apps/desktop/src-tauri/target/release | |
| 108 | + | !apps/desktop/src-tauri/target/release/bundle | |
| 109 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 110 | + | key: desktop-release-linux-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 111 | + | lookup-only: true | |
| 112 | + | - id: desktop-windows | |
| 113 | + | uses: actions/cache/restore@v4 | |
| 114 | + | with: | |
| 115 | + | path: | | |
| 116 | + | ~/.cargo/registry/index | |
| 117 | + | ~/.cargo/registry/cache | |
| 118 | + | apps/desktop/src-tauri/target/release | |
| 119 | + | apps/desktop/src-tauri/target/x86_64-pc-windows-msvc/release | |
| 120 | + | !apps/desktop/src-tauri/target/**/bundle | |
| 121 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 122 | + | key: desktop-release-windows-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 123 | + | lookup-only: true | |
| 124 | + | - id: xwin | |
| 125 | + | uses: actions/cache/restore@v4 | |
| 126 | + | with: | |
| 127 | + | path: | | |
| 128 | + | ~/.cargo/bin/cargo-xwin | |
| 129 | + | ~/.cache/cargo-xwin | |
| 130 | + | key: xwin-${{ runner.os }}-0.23.1 | |
| 131 | + | lookup-only: true | |
| 132 | + | - id: apt-windows | |
| 133 | + | uses: actions/cache/restore@v4 | |
| 134 | + | with: | |
| 135 | + | path: ~/.cache/apt/tauri-windows | |
| 136 | + | key: apt-tauri-windows-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 137 | + | lookup-only: true | |
| 138 | + | # The runner's targets to build: those whose build is missing, or all | |
| 139 | + | # of them when zig is (each saves it, the first to finish wins). | |
| 140 | + | - name: The runner's targets to build | |
| 141 | + | id: missing | |
| 142 | + | env: | |
| 143 | + | ZIG: ${{ steps.zig.outputs.cache-hit }} | |
| 144 | + | LINUX_X64: ${{ steps.runner-linux-x64.outputs.cache-hit }} | |
| 145 | + | LINUX_ARM64: ${{ steps.runner-linux-arm64.outputs.cache-hit }} | |
| 146 | + | WINDOWS_X64: ${{ steps.runner-windows-x64.outputs.cache-hit }} | |
| 147 | + | # Each with its Rust triple, as scripts/runner-release.mjs names it. | |
| 148 | + | run: | | |
| 149 | + | targets=() | |
| 150 | + | [ "$ZIG" = true ] && [ "$LINUX_X64" = true ] || targets+=('{"target":"linux-x64","triple":"x86_64-unknown-linux-musl"}') | |
| 151 | + | [ "$ZIG" = true ] && [ "$LINUX_ARM64" = true ] || targets+=('{"target":"linux-arm64","triple":"aarch64-unknown-linux-musl"}') | |
| 152 | + | [ "$ZIG" = true ] && [ "$WINDOWS_X64" = true ] || targets+=('{"target":"windows-x64","triple":"x86_64-pc-windows-gnu"}') | |
| 153 | + | list="[$(IFS=,; echo "${targets[*]}")]" | |
| 154 | + | echo "runner=$list" >> "$GITHUB_OUTPUT" | |
| 155 | + | echo "The runner's targets to build: $list" | |
| 156 | + | ||
| 157 | + | # npm's downloads for the whole lockfile, saved once for each lockfile. | |
| 158 | + | # Before, every job saved its own part under one key, and the first to | |
| 159 | + | # finish (Wrangler alone, 73 MB) stood in for the rest: each stage then | |
| 160 | + | # downloaded hundreds of MB through the runner's egress, 50 MB tarballs | |
| 161 | + | # among them. Starts from the last lockfile's, with scripts off. | |
| 162 | + | npm: | |
| 163 | + | name: npm's downloads | |
| 164 | + | needs: look | |
| 165 | + | if: ${{ needs.look.outputs.npm == 'true' }} | |
| 166 | + | runs-on: ubuntu-latest | |
| 167 | + | timeout-minutes: 20 | |
| 168 | + | steps: | |
| 169 | + | - uses: actions/checkout@v5 | |
| 170 | + | - name: Start from the last lockfile's | |
| 171 | + | uses: actions/cache/restore@v4 | |
| 172 | + | with: | |
| 173 | + | path: ~/.npm | |
| 174 | + | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 175 | + | restore-keys: npm-full-${{ runner.os }}- | |
| 176 | + | # A failed download leaves the cache as it was, and the run green. | |
| 177 | + | - name: Download every package | |
| 178 | + | id: download | |
| 179 | + | continue-on-error: true | |
| 180 | + | run: node scripts/deploy/npm-ci.mjs --prefer-offline --ignore-scripts --no-audit --no-fund | |
| 181 | + | - name: Save them | |
| 182 | + | if: ${{ steps.download.outcome == 'success' }} | |
| 183 | + | uses: actions/cache/save@v4 | |
| 184 | + | with: | |
| 185 | + | path: ~/.npm | |
| 186 | + | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 187 | + | ||
| 188 | + | # The steps of runner-release.yml's binaries job that fill its cache. | |
| 189 | + | runner: | |
| 190 | + | name: The runner's build for ${{ matrix.target }} | |
| 191 | + | needs: look | |
| 192 | + | if: ${{ needs.look.outputs.runner != '[]' }} | |
| 193 | + | runs-on: ubuntu-latest | |
| 194 | + | timeout-minutes: 60 | |
| 195 | + | strategy: | |
| 196 | + | fail-fast: false | |
| 197 | + | matrix: | |
| 198 | + | include: ${{ fromJSON(needs.look.outputs.runner) }} | |
| 199 | + | env: | |
| 200 | + | G1T_RUNNER_RELEASE_KEY: ${{ vars.RUNNER_RELEASE_PUBLIC_KEY }} | |
| 201 | + | steps: | |
| 202 | + | - uses: actions/checkout@v5 | |
| 203 | + | - name: Cache zig and cargo-zigbuild | |
| 204 | + | id: zig | |
| 205 | + | uses: actions/cache@v4 | |
| 206 | + | with: | |
| 207 | + | path: | | |
| 208 | + | ~/.local/lib/python3*/site-packages/ziglang | |
| 209 | + | ~/.local/bin | |
| 210 | + | ~/.cargo/bin/cargo-zigbuild | |
| 211 | + | key: zig-0.13.0-zigbuild-${{ runner.os }}-1 | |
| 212 | + | - name: Install zig and cargo-zigbuild | |
| 213 | + | if: steps.zig.outputs.cache-hit != 'true' | |
| 214 | + | run: | | |
| 215 | + | pip install --user --break-system-packages ziglang==0.13.0 | |
| 216 | + | cargo install --locked cargo-zigbuild | |
| 217 | + | - name: zig on the PATH | |
| 218 | + | run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| 219 | + | - name: Cache keys | |
| 220 | + | id: keys | |
| 221 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 222 | + | - name: Cache crates | |
| 223 | + | uses: actions/cache/restore@v4 | |
| 224 | + | with: | |
| 225 | + | path: | | |
| 226 | + | ~/.cargo/registry/index | |
| 227 | + | ~/.cargo/registry/cache | |
| 228 | + | key: cargo-crates-${{ runner.os }}-${{ steps.keys.outputs.cargo }} | |
| 229 | + | restore-keys: cargo-crates-${{ runner.os }}- | |
| 230 | + | - name: Cache the runner's build | |
| 231 | + | id: runner-build | |
| 232 | + | uses: actions/cache@v4 | |
| 233 | + | with: | |
| 234 | + | path: | | |
| 235 | + | target | |
| 236 | + | !target/runner-release | |
| 237 | + | !target/**/incremental | |
| 238 | + | key: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 239 | + | restore-keys: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}- | |
| 240 | + | - name: sccache | |
| 241 | + | run: bash scripts/sccache.sh install | |
| 242 | + | - name: Build | |
| 243 | + | run: node scripts/runner-release.mjs build --targets ${{ matrix.target }} | |
| 244 | + | - name: Only this build's files are saved | |
| 245 | + | if: ${{ steps.runner-build.outputs.cache-hit != 'true' }} | |
| 246 | + | run: node scripts/deploy/ci-cache.mjs prune target/release target/${{ matrix.triple }}/release -- zigbuild --release --locked --package g1t-runner --target ${{ matrix.triple }} | |
| 247 | + | - name: sccache's hits and misses | |
| 248 | + | if: ${{ always() }} | |
| 249 | + | run: bash scripts/sccache.sh stats | |
| 250 | + | ||
| 251 | + | # desktop-release.yml's Linux build, without its installers: `tauri | |
| 252 | + | # build --no-bundle` compiles exactly what `tauri build` does, and needs | |
| 253 | + | # no signing key. | |
| 254 | + | desktop-linux: | |
| 255 | + | name: The desktop app's Linux build | |
| 256 | + | needs: look | |
| 257 | + | if: ${{ needs.look.outputs.desktop-linux == 'true' }} | |
| 258 | + | # The larger machine: linking Tauri on the small one dies with a bus error. | |
| 259 | + | runs-on: g1t-4core | |
| 260 | + | timeout-minutes: 60 | |
| 261 | + | steps: | |
| 262 | + | - uses: actions/checkout@v5 | |
| 263 | + | - name: Cache keys | |
| 264 | + | id: keys | |
| 265 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 266 | + | # Saved by CI's desktop job; only restored here. | |
| 267 | + | - name: Cache apt's downloads | |
| 268 | + | uses: actions/cache/restore@v4 | |
| 269 | + | with: | |
| 270 | + | path: ~/.cache/apt/tauri | |
| 271 | + | key: apt-tauri-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 272 | + | - name: What Tauri needs on Linux | |
| 273 | + | run: bash scripts/apt-install.sh tauri | |
| 274 | + | - name: Cache the release build | |
| 275 | + | uses: actions/cache@v4 | |
| 276 | + | with: | |
| 277 | + | path: | | |
| 278 | + | ~/.cargo/registry/index | |
| 279 | + | ~/.cargo/registry/cache | |
| 280 | + | apps/desktop/src-tauri/target/release | |
| 281 | + | !apps/desktop/src-tauri/target/release/bundle | |
| 282 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 283 | + | key: desktop-release-linux-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 284 | + | restore-keys: desktop-release-linux-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 285 | + | - name: Cache npm's downloads | |
| 286 | + | uses: actions/cache/restore@v4 | |
| 287 | + | with: | |
| 288 | + | path: ~/.npm | |
| 289 | + | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 290 | + | restore-keys: npm-full-${{ runner.os }}- | |
| 291 | + | - name: Install | |
| 292 | + | run: node scripts/deploy/npm-ci.mjs --prefer-offline --no-audit --no-fund | |
| 293 | + | - name: Build | |
| 294 | + | working-directory: apps/desktop | |
| 295 | + | run: npx tauri build --ci --no-bundle | |
| 296 | + | ||
| 297 | + | # desktop-release.yml's Windows cross build, without its installer. | |
| 298 | + | desktop-windows: | |
| 299 | + | name: The desktop app's Windows build | |
| 300 | + | needs: look | |
| 301 | + | if: ${{ needs.look.outputs.desktop-windows == 'true' && vars.DESKTOP_WINDOWS_FROM_LINUX == 'true' && vars.DESKTOP_WINDOWS_RUNNER != 'true' }} | |
| 302 | + | runs-on: g1t-4core | |
| 303 | + | timeout-minutes: 60 | |
| 304 | + | steps: | |
| 305 | + | - uses: actions/checkout@v5 | |
| 306 | + | - name: Cache keys | |
| 307 | + | id: keys | |
| 308 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 309 | + | - name: Cache apt's downloads | |
| 310 | + | uses: actions/cache@v4 | |
| 311 | + | with: | |
| 312 | + | path: ~/.cache/apt/tauri-windows | |
| 313 | + | key: apt-tauri-windows-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 314 | + | - name: What the cross build needs | |
| 315 | + | run: bash scripts/desktop-cross-tools.sh | |
| 316 | + | - name: Cache cargo-xwin and the MSVC CRT | |
| 317 | + | id: xwin | |
| 318 | + | uses: actions/cache@v4 | |
| 319 | + | with: | |
| 320 | + | path: | | |
| 321 | + | ~/.cargo/bin/cargo-xwin | |
| 322 | + | ~/.cache/cargo-xwin | |
| 323 | + | key: xwin-${{ runner.os }}-0.23.1 | |
| 324 | + | - name: cargo-xwin | |
| 325 | + | if: ${{ steps.xwin.outputs.cache-hit != 'true' }} | |
| 326 | + | run: cargo install --locked cargo-xwin@0.23.1 | |
| 327 | + | - name: Cache the release build | |
| 328 | + | uses: actions/cache@v4 | |
| 329 | + | with: | |
| 330 | + | path: | | |
| 331 | + | ~/.cargo/registry/index | |
| 332 | + | ~/.cargo/registry/cache | |
| 333 | + | apps/desktop/src-tauri/target/release | |
| 334 | + | apps/desktop/src-tauri/target/x86_64-pc-windows-msvc/release | |
| 335 | + | !apps/desktop/src-tauri/target/**/bundle | |
| 336 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 337 | + | key: desktop-release-windows-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 338 | + | restore-keys: desktop-release-windows-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 339 | + | - name: Cache npm's downloads | |
| 340 | + | uses: actions/cache/restore@v4 | |
| 341 | + | with: | |
| 342 | + | path: ~/.npm | |
| 343 | + | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 344 | + | restore-keys: npm-full-${{ runner.os }}- | |
| 345 | + | - name: Install | |
| 346 | + | run: node scripts/deploy/npm-ci.mjs --prefer-offline --no-audit --no-fund | |
| 347 | + | - name: Build | |
| 348 | + | working-directory: apps/desktop | |
| 349 | + | run: | | |
| 350 | + | rustup target add x86_64-pc-windows-msvc | |
| 351 | + | npx tauri build --ci --runner cargo-xwin --target x86_64-pc-windows-msvc --no-bundle | |
| 352 | + | # What the CRT and SDK came to, for the cache's 2 GB an entry may hold. | |
| 353 | + | - name: How large the MSVC CRT is | |
| 354 | + | run: du -sh ~/.cache/cargo-xwin || true |
| 8 | 8 | # deploy and ops scripts, and the deploy manifest | |
| 9 | 9 | # build the site, sudo and the docs build as they deploy | |
| 10 | 10 | # | |
| 11 | − | # On a push to main only `rust` runs, to keep main's caches current: a pull | |
| 12 | − | # request's run restores from main's cache, never from another pull | |
| 13 | − | # request's, so without it each pull request would start from nothing. | |
| 11 | + | # On a push to main `rust` and `desktop` run, keeping main's caches current: | |
| 12 | + | # a pull request's run restores from main's cache, never from another pull | |
| 13 | + | # request's, so without them each pull request would start from nothing. | |
| 14 | + | # What each cache holds, its key and which job saves it: CONTRIBUTING.md, | |
| 15 | + | # "How g1t's own CI caches". The Caches workflow (caches.yml) keeps the | |
| 16 | + | # ones only releases build. | |
| 14 | 17 | name: CI | |
| 15 | 18 | ||
| 16 | 19 | on: | |
| ⋯ | |||
| 37 | 40 | name: Rust | |
| 38 | 41 | runs-on: g1t-4core | |
| 39 | 42 | timeout-minutes: 45 | |
| 43 | + | env: | |
| 44 | + | # Line tables, not full debug info: a panic still names its file and | |
| 45 | + | # line, and the test build is a fraction of the size to link, keep | |
| 46 | + | # and restore (at full debug info its cache entry reached 1.7 GB of | |
| 47 | + | # the 2 GB an entry may hold). | |
| 48 | + | CARGO_PROFILE_DEV_DEBUG: line-tables-only | |
| 40 | 49 | steps: | |
| 41 | 50 | - uses: actions/checkout@v5 | |
| 51 | + | # rust: the toolchain; cargo: Cargo.lock's crates.io and git packages, | |
| 52 | + | # not the workspace's own versions (scripts/deploy/ci-cache.mjs). | |
| 53 | + | - name: Cache keys | |
| 54 | + | id: keys | |
| 55 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 56 | + | # Every crate the lockfile names, for every target (the fetch below): | |
| 57 | + | # saved by this job alone, and only restored by Deploy and the | |
| 58 | + | # releases, which each download part of it. | |
| 42 | 59 | - name: Cache crates | |
| 43 | 60 | uses: actions/cache@v4 | |
| 44 | 61 | with: | |
| 45 | − | path: ~/.cargo/registry/cache | |
| 46 | − | key: cargo-crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| 62 | + | path: | | |
| 63 | + | ~/.cargo/registry/index | |
| 64 | + | ~/.cargo/registry/cache | |
| 65 | + | key: cargo-crates-${{ runner.os }}-${{ steps.keys.outputs.cargo }} | |
| 47 | 66 | restore-keys: cargo-crates-${{ runner.os }}- | |
| 67 | + | - name: Every crate the lockfile names | |
| 68 | + | run: cargo fetch --locked | |
| 69 | + | # The compiled dependencies and workspace crates. Saved when the | |
| 70 | + | # dependencies or the toolchain change; otherwise the last entry for | |
| 71 | + | # this toolchain is a start, and only this build's files go into the | |
| 72 | + | # new one (the prune step below). | |
| 48 | 73 | - name: Cache the test build | |
| 74 | + | id: test-build | |
| 49 | 75 | uses: actions/cache@v4 | |
| 50 | 76 | with: | |
| 51 | 77 | path: | | |
| 52 | 78 | target/debug | |
| 53 | 79 | !target/debug/incremental | |
| 54 | − | key: cargo-test-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'services/runner/base.json') }} | |
| 55 | − | restore-keys: cargo-test-${{ runner.os }}- | |
| 56 | − | # The workspace's library crates, which Cargo compiles again on every | |
| 57 | − | # checkout, come back from the repository's Actions cache when their | |
| 58 | − | # inputs did not change (scripts/sccache.sh). Test harnesses and | |
| 59 | − | # Workers' own crates are linked, and still compiled. | |
| 80 | + | key: cargo-test-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 81 | + | restore-keys: cargo-test-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 82 | + | # The workspace's library crates that Cargo compiles again come back | |
| 83 | + | # from the repository's Actions cache when their inputs did not change | |
| 84 | + | # (scripts/sccache.sh). Test harnesses and Workers' own crates are | |
| 85 | + | # linked, and still compiled. | |
| 60 | 86 | - name: sccache | |
| 61 | 87 | run: bash scripts/sccache.sh install | |
| 62 | 88 | - name: Tests | |
| 63 | 89 | run: cargo test --workspace --locked --quiet | |
| 90 | + | # A restored target keeps the old versions of whatever changed beside | |
| 91 | + | # the new ones; only what this build used goes into the new entry. | |
| 92 | + | - name: Only this build's files are saved | |
| 93 | + | if: ${{ steps.test-build.outputs.cache-hit != 'true' }} | |
| 94 | + | run: node scripts/deploy/ci-cache.mjs prune target/debug -- test --workspace --locked --no-run | |
| 64 | 95 | - name: sccache's hits and misses | |
| 65 | 96 | if: ${{ always() }} | |
| 66 | 97 | run: bash scripts/sccache.sh stats | |
| ⋯ | |||
| 127 | 158 | timeout-minutes: 30 | |
| 128 | 159 | steps: | |
| 129 | 160 | - uses: actions/checkout@v5 | |
| 161 | + | - name: Cache keys | |
| 162 | + | id: keys | |
| 163 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 164 | + | # apt's lists and downloads (scripts/apt-install.sh): 110 s of every | |
| 165 | + | # run before, mostly fetching 335 MB. Saved here, on main; the desktop | |
| 166 | + | # release's Linux build restores the same entry. | |
| 167 | + | - name: Cache apt's downloads | |
| 168 | + | uses: actions/cache@v4 | |
| 169 | + | with: | |
| 170 | + | path: ~/.cache/apt/tauri | |
| 171 | + | key: apt-tauri-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 130 | 172 | - name: What Tauri needs on Linux | |
| 131 | − | # Tried three times: the mirror has answered with a mismatched package before. | |
| 132 | − | run: | | |
| 133 | − | ok=0 | |
| 134 | − | for try in 1 2 3; do | |
| 135 | − | if sudo apt-get update && sudo apt-get install -y libwebkit2gtk-4.1-dev build-essential libxdo-dev libssl-dev libayatana-appindicator3-dev librsvg2-dev; then ok=1; break; fi | |
| 136 | − | sudo apt-get clean; sleep 15 | |
| 137 | − | done | |
| 138 | − | [ "$ok" = 1 ] | |
| 139 | − | - uses: actions/cache@v4 | |
| 173 | + | run: bash scripts/apt-install.sh tauri | |
| 174 | + | - name: Cache the desktop app's build | |
| 175 | + | id: desktop-build | |
| 176 | + | uses: actions/cache@v4 | |
| 140 | 177 | with: | |
| 141 | 178 | path: | | |
| 142 | − | ~/.cargo/registry | |
| 143 | − | apps/desktop/src-tauri/target | |
| 144 | − | key: desktop-check-${{ hashFiles('apps/desktop/src-tauri/Cargo.lock') }} | |
| 179 | + | ~/.cargo/registry/index | |
| 180 | + | ~/.cargo/registry/cache | |
| 181 | + | apps/desktop/src-tauri/target/debug | |
| 182 | + | !apps/desktop/src-tauri/target/debug/incremental | |
| 183 | + | key: desktop-check-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 184 | + | restore-keys: desktop-check-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 145 | 185 | # Outside the workspace on purpose (Cargo.toml says why): checked and | |
| 146 | 186 | # tested here, built into installers by desktop-release.yml. | |
| 147 | 187 | - name: Check and test | |
| 148 | 188 | run: cargo test --locked --manifest-path apps/desktop/src-tauri/Cargo.toml --quiet | |
| 189 | + | - name: Only this build's files are saved | |
| 190 | + | if: ${{ steps.desktop-build.outputs.cache-hit != 'true' }} | |
| 191 | + | run: node scripts/deploy/ci-cache.mjs prune apps/desktop/src-tauri/target/debug -- test --locked --manifest-path apps/desktop/src-tauri/Cargo.toml --no-run | |
| 149 | 192 | - name: The release scripts' tests | |
| 150 | 193 | run: node --test scripts/desktop-release.test.mjs scripts/mobile-release.test.mjs scripts/cli-release.test.mjs | |
| 72 | 72 | timeout-minutes: 20 | |
| 73 | 73 | steps: | |
| 74 | 74 | - uses: actions/checkout@v5 | |
| 75 | − | # npm's downloads for the whole lockfile, which the npm-cache job | |
| 76 | − | # keeps: an install reads them first and asks the registry only for | |
| 75 | + | # npm's downloads for the whole lockfile, which the Caches workflow | |
| 76 | + | # keeps (caches.yml): an install reads them first and asks the registry only for | |
| 77 | 77 | # what changed, so a dropped connection to it rarely stops the job. | |
| 78 | 78 | # Only restored: a job that installs part of the lockfile never | |
| 79 | 79 | # saves, or its part would stand in for the whole under that key. | |
| ⋯ | |||
| 89 | 89 | run: node scripts/deploy.mjs manifest --check | |
| 90 | 90 | - name: The deploy tool's tests | |
| 91 | 91 | run: npm run test:deploy | |
| 92 | − | ||
| 93 | − | # npm's downloads for the whole lockfile, saved once for each lockfile | |
| 94 | − | # on main, which every install of this and the other workflows starts | |
| 95 | − | # from. Before, every job saved its own part under one key, and the | |
| 96 | − | # first to finish (Wrangler alone, 73 MB) stood in for the rest: each | |
| 97 | − | # stage then downloaded hundreds of MB through the runner's egress, | |
| 98 | − | # 50 MB tarballs among them. Nothing waits for this job, and a failed | |
| 99 | − | # download only leaves the cache as it was. | |
| 100 | − | npm-cache: | |
| 101 | − | name: npm's downloads | |
| 102 | − | runs-on: ubuntu-latest | |
| 103 | − | timeout-minutes: 20 | |
| 104 | − | steps: | |
| 105 | − | - uses: actions/checkout@v5 | |
| 106 | − | - name: Look for this lockfile's downloads | |
| 107 | − | id: saved | |
| 108 | − | uses: actions/cache/restore@v4 | |
| 109 | − | with: | |
| 110 | − | path: ~/.npm | |
| 111 | − | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 112 | − | lookup-only: true | |
| 113 | − | - name: Start from the last lockfile's | |
| 114 | − | if: ${{ steps.saved.outputs.cache-hit != 'true' }} | |
| 115 | − | uses: actions/cache/restore@v4 | |
| 116 | − | with: | |
| 117 | − | path: ~/.npm | |
| 118 | − | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 119 | − | restore-keys: npm-full-${{ runner.os }}- | |
| 120 | − | - name: Download every package | |
| 121 | − | id: download | |
| 122 | − | if: ${{ steps.saved.outputs.cache-hit != 'true' }} | |
| 123 | − | continue-on-error: true | |
| 124 | − | run: node scripts/deploy/npm-ci.mjs --prefer-offline --ignore-scripts --no-audit --no-fund | |
| 125 | − | - name: Save them | |
| 126 | − | if: ${{ steps.saved.outputs.cache-hit != 'true' && steps.download.outcome == 'success' }} | |
| 127 | − | uses: actions/cache/save@v4 | |
| 128 | − | with: | |
| 129 | − | path: ~/.npm | |
| 130 | − | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 131 | 92 | ||
| 132 | 93 | # Runs beside check: nothing deploys until both have succeeded. | |
| 133 | 94 | plan: | |
| ⋯ | |||
| 233 | 194 | with: | |
| 234 | 195 | path: ~/.cache/worker-build | |
| 235 | 196 | key: worker-build-tools-${{ runner.os }}-${{ hashFiles('scripts/build-rust-worker.mjs') }} | |
| 197 | + | # rust: the toolchain; cargo: Cargo.lock's crates.io and git packages, | |
| 198 | + | # not the workspace's own versions (scripts/deploy/ci-cache.mjs). | |
| 199 | + | - name: Cache keys | |
| 200 | + | id: keys | |
| 201 | + | if: ${{ matrix.rust || matrix.image }} | |
| 202 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 203 | + | # Every crate the lockfile names, as CI's Rust job saves them on | |
| 204 | + | # main. Only restored: a stage downloads the crates of its own units, | |
| 205 | + | # and saving those would stand in for the whole under that key. | |
| 236 | 206 | - name: Cache crates | |
| 237 | − | if: ${{ matrix.rust }} | |
| 238 | − | uses: actions/cache@v4 | |
| 207 | + | if: ${{ matrix.rust || matrix.image }} | |
| 208 | + | uses: actions/cache/restore@v4 | |
| 239 | 209 | with: | |
| 240 | − | path: ~/.cargo/registry/cache | |
| 241 | − | key: cargo-crates-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} | |
| 210 | + | path: | | |
| 211 | + | ~/.cargo/registry/index | |
| 212 | + | ~/.cargo/registry/cache | |
| 213 | + | key: cargo-crates-${{ runner.os }}-${{ steps.keys.outputs.cargo }} | |
| 242 | 214 | restore-keys: cargo-crates-${{ runner.os }}- | |
| 243 | 215 | # The compiled dependencies of this job's units, for wasm32 and the | |
| 244 | 216 | # build scripts and proc macros they run. Cargo calls rustc again for | |
| 245 | 217 | # the workspace's own crates whatever is cached (a checkout's sources | |
| 246 | 218 | # are newer); sccache, below, answers those calls. So an entry is | |
| 247 | − | # saved only when the dependencies change: a new Cargo.lock, or a new | |
| 248 | − | # base image (base.json names its Rust). Otherwise the nearest earlier | |
| 249 | − | # entry, of any group, is a start. | |
| 219 | + | # saved only when the dependencies or the toolchain change, not when | |
| 220 | + | # a release bumps a workspace crate's version; otherwise this group's | |
| 221 | + | # last entry for the toolchain is a start, else any group's. | |
| 250 | 222 | - name: Cache the Cargo target | |
| 251 | 223 | if: ${{ matrix.rust }} | |
| 252 | 224 | uses: actions/cache@v4 | |
| ⋯ | |||
| 256 | 228 | target/wasm32-unknown-unknown/release | |
| 257 | 229 | !target/**/incremental | |
| 258 | 230 | !target/**/*.wasm | |
| 259 | − | key: cargo-target-${{ runner.os }}-${{ matrix.group }}-${{ hashFiles('Cargo.lock', 'services/runner/base.json') }} | |
| 231 | + | key: cargo-workers-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ matrix.group }}-${{ steps.keys.outputs.cargo }} | |
| 260 | 232 | restore-keys: | | |
| 261 | − | cargo-target-${{ runner.os }}-${{ matrix.group }}- | |
| 262 | − | cargo-target-${{ runner.os }}- | |
| 233 | + | cargo-workers-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ matrix.group }}- | |
| 234 | + | cargo-workers-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 263 | 235 | # The runner's image: its binary, built natively for musl (the base | |
| 264 | 236 | # has musl-gcc; the target is added here), with its Cargo target kept | |
| 265 | − | # between runs. The image itself is made in the registry, without | |
| 266 | − | # Docker: the base's layers and one with the binary | |
| 267 | − | # (scripts/deploy/registry.mjs). Building it with the job's Docker | |
| 268 | − | # Engine, only if that fails, pulls the 4.5 GB base onto the job's | |
| 269 | − | # disk. | |
| 237 | + | # between runs: the musl target's, and target/release, where Cargo | |
| 238 | + | # puts the build scripts and proc macros it runs on this machine | |
| 239 | + | # (without it, 87 crates were compiled again on every deploy). The | |
| 240 | + | # image itself is made in the registry, without Docker: the base's | |
| 241 | + | # layers and one with the binary (scripts/deploy/registry.mjs). | |
| 242 | + | # Building it with the job's Docker Engine, only if that fails, pulls | |
| 243 | + | # the 4.5 GB base onto the job's disk. | |
| 270 | 244 | - name: Rust for the runner | |
| 271 | 245 | if: ${{ matrix.image }} | |
| 272 | 246 | run: rustup target add x86_64-unknown-linux-musl | |
| ⋯ | |||
| 275 | 249 | uses: actions/cache@v4 | |
| 276 | 250 | with: | |
| 277 | 251 | path: | | |
| 278 | − | ~/.cargo/registry/cache | |
| 252 | + | target/release | |
| 279 | 253 | target/x86_64-unknown-linux-musl/release | |
| 280 | 254 | !target/**/incremental | |
| 281 | − | key: runner-musl-${{ runner.os }}-${{ hashFiles('Cargo.lock', 'services/runner/base.json') }} | |
| 282 | − | restore-keys: runner-musl-${{ runner.os }}- | |
| 255 | + | key: runner-musl-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 256 | + | restore-keys: runner-musl-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 283 | 257 | # Every rustc call that makes a library (the workspace's crates, | |
| 284 | 258 | # which Cargo compiles again on every checkout, and any dependency | |
| 285 | 259 | # not restored above) is looked up by its inputs in the repository's | |
| 75 | 75 | timeout-minutes: 60 # the most g1t's own machines run a job | |
| 76 | 76 | steps: | |
| 77 | 77 | - uses: actions/checkout@v5 | |
| 78 | + | # rust: the toolchain; desktop: the app's Cargo.lock's crates.io and | |
| 79 | + | # git packages, not its own version (scripts/deploy/ci-cache.mjs). | |
| 80 | + | - name: Cache keys | |
| 81 | + | id: keys | |
| 82 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 83 | + | # apt's lists and downloads, as CI's desktop job saves them on main. | |
| 84 | + | - name: Cache apt's downloads | |
| 85 | + | uses: actions/cache/restore@v4 | |
| 86 | + | with: | |
| 87 | + | path: ~/.cache/apt/tauri | |
| 88 | + | key: apt-tauri-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 78 | 89 | - name: What Tauri needs on Linux | |
| 79 | − | # Tried three times: the mirror has answered with a mismatched package before. | |
| 80 | − | run: | | |
| 81 | − | ok=0 | |
| 82 | − | for try in 1 2 3; do | |
| 83 | − | if sudo apt-get update && sudo apt-get install -y libwebkit2gtk-4.1-dev build-essential curl wget file libxdo-dev libssl-dev libayatana-appindicator3-dev librsvg2-dev; then ok=1; break; fi | |
| 84 | − | sudo apt-get clean; sleep 15 | |
| 85 | − | done | |
| 86 | − | [ "$ok" = 1 ] | |
| 87 | − | - uses: actions/cache@v4 | |
| 90 | + | run: bash scripts/apt-install.sh tauri | |
| 91 | + | # The app's release build. A tag's run restores only from its own tag | |
| 92 | + | # and from main, never from the last release's, so the Caches | |
| 93 | + | # workflow (caches.yml) builds and saves this on main whenever the | |
| 94 | + | # key is new. Only restored here. | |
| 95 | + | - name: Cache the release build | |
| 96 | + | uses: actions/cache/restore@v4 | |
| 88 | 97 | with: | |
| 89 | 98 | path: | | |
| 90 | − | ~/.cargo/registry | |
| 91 | − | apps/desktop/src-tauri/target | |
| 92 | − | key: desktop-release-linux-${{ hashFiles('apps/desktop/src-tauri/Cargo.lock') }} | |
| 93 | − | # npm's downloads, kept between runs: an install reads them first | |
| 94 | − | # and asks the registry only for what changed, so a dropped | |
| 95 | − | # connection to it rarely stops the job. | |
| 99 | + | ~/.cargo/registry/index | |
| 100 | + | ~/.cargo/registry/cache | |
| 101 | + | apps/desktop/src-tauri/target/release | |
| 102 | + | !apps/desktop/src-tauri/target/release/bundle | |
| 103 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 104 | + | key: desktop-release-linux-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 105 | + | restore-keys: desktop-release-linux-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 106 | + | # npm's downloads for the whole lockfile, which the Caches workflow | |
| 107 | + | # keeps on main: an install reads them first and asks the registry | |
| 108 | + | # only for what changed. Only restored: a tag's entries are read by | |
| 109 | + | # nothing else. | |
| 96 | 110 | - name: Cache npm's downloads | |
| 97 | − | uses: actions/cache@v4 | |
| 111 | + | uses: actions/cache/restore@v4 | |
| 98 | 112 | with: | |
| 99 | 113 | path: ~/.npm | |
| 100 | 114 | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| ⋯ | |||
| 165 | 179 | timeout-minutes: 60 # the most g1t's own machines run a job | |
| 166 | 180 | steps: | |
| 167 | 181 | - uses: actions/checkout@v5 | |
| 182 | + | - name: Cache keys | |
| 183 | + | id: keys | |
| 184 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 185 | + | # apt's lists and downloads, which the Caches workflow keeps on main. | |
| 186 | + | - name: Cache apt's downloads | |
| 187 | + | uses: actions/cache/restore@v4 | |
| 188 | + | with: | |
| 189 | + | path: ~/.cache/apt/tauri-windows | |
| 190 | + | key: apt-tauri-windows-${{ runner.os }}-${{ hashFiles('scripts/apt-install.sh') }} | |
| 168 | 191 | - name: What the cross build needs | |
| 169 | 192 | # The tray's Linux library too: Tauri's bundler looks for it on the | |
| 170 | 193 | # machine it runs on, whatever the target, and stops without it. | |
| 171 | 194 | run: | | |
| 172 | − | ok=0 | |
| 173 | − | for try in 1 2 3; do | |
| 174 | − | if sudo apt-get update && sudo apt-get install -y build-essential curl wget file libssl-dev libayatana-appindicator3-dev nsis clang lld llvm; then ok=1; break; fi | |
| 175 | − | sudo apt-get clean; sleep 15 | |
| 176 | − | done | |
| 177 | − | [ "$ok" = 1 ] | |
| 178 | − | # cargo-xwin compiles C (ring's, among others) with clang-cl and | |
| 179 | − | # links with lld-link and llvm-lib. Each is a mode of a tool | |
| 180 | − | # Debian does install, chosen by the name it is run as: clang-cl | |
| 181 | − | # is clang, lld-link is lld, and llvm-lib is llvm-ar. Debian ships | |
| 182 | − | # none of the three names, so each is linked to its tool. | |
| 183 | − | for pair in clang-cl:clang lld-link:lld llvm-lib:llvm-ar; do | |
| 184 | − | name="${pair%%:*}" tool="${pair##*:}" | |
| 185 | − | command -v "$name" >/dev/null && continue | |
| 186 | − | found="$(command -v "$tool" || ls /usr/lib/llvm-*/bin/"$tool" 2>/dev/null | sort -V | tail -1)" | |
| 187 | − | [ -n "$found" ] || { echo "::error::$tool is not installed, so there is no $name"; exit 1; } | |
| 188 | − | sudo ln -sf "$(readlink -f "$found")" "/usr/local/bin/$name" | |
| 189 | − | echo "$name: $(readlink -f "$found")" | |
| 190 | − | done | |
| 191 | − | # Tauri's installer script includes Win\RestartManager.nsh, which | |
| 192 | − | # NSIS ships from 3.10 on; Debian's is 3.08. The header stands | |
| 193 | − | # alone (defines and System calls), so it is added from NSIS 3.11's | |
| 194 | − | # source, checked against its SHA-256. | |
| 195 | − | rm_nsh=/usr/share/nsis/Include/Win/RestartManager.nsh | |
| 196 | − | if [ ! -f "$rm_nsh" ]; then | |
| 197 | − | curl -fsSL --retry 3 -o /tmp/RestartManager.nsh https://raw.githubusercontent.com/kichik/nsis/v311/Include/Win/RestartManager.nsh | |
| 198 | − | echo "67149978412da9b283e2e1abe48a360093df9caf6363373e2cabe463978b7c0e /tmp/RestartManager.nsh" | sha256sum -c - | |
| 199 | − | sudo install -m 0644 /tmp/RestartManager.nsh "$rm_nsh" | |
| 200 | − | fi | |
| 201 | − | cargo install --locked cargo-xwin | |
| 202 | − | - uses: actions/cache@v4 | |
| 195 | + | bash scripts/desktop-cross-tools.sh | |
| 196 | + | # cargo-xwin, pinned, and the MSVC CRT and Windows SDK it downloads on | |
| 197 | + | # first use: 88 s compiling it and 59 s downloading them in every | |
| 198 | + | # release before. The Caches workflow keeps them on main. | |
| 199 | + | - name: Cache cargo-xwin and the MSVC CRT | |
| 200 | + | id: xwin | |
| 201 | + | uses: actions/cache/restore@v4 | |
| 202 | + | with: | |
| 203 | + | path: | | |
| 204 | + | ~/.cargo/bin/cargo-xwin | |
| 205 | + | ~/.cache/cargo-xwin | |
| 206 | + | key: xwin-${{ runner.os }}-0.23.1 | |
| 207 | + | - name: cargo-xwin | |
| 208 | + | if: ${{ steps.xwin.outputs.cache-hit != 'true' }} | |
| 209 | + | run: cargo install --locked cargo-xwin@0.23.1 | |
| 210 | + | # The app's Windows build, which the Caches workflow keeps on main. | |
| 211 | + | - name: Cache the release build | |
| 212 | + | uses: actions/cache/restore@v4 | |
| 203 | 213 | with: | |
| 204 | 214 | path: | | |
| 205 | − | ~/.cargo/registry | |
| 206 | − | apps/desktop/src-tauri/target | |
| 207 | − | key: desktop-release-windows-${{ hashFiles('apps/desktop/src-tauri/Cargo.lock') }} | |
| 208 | − | # npm's downloads, kept between runs: an install reads them first | |
| 209 | − | # and asks the registry only for what changed, so a dropped | |
| 210 | − | # connection to it rarely stops the job. | |
| 215 | + | ~/.cargo/registry/index | |
| 216 | + | ~/.cargo/registry/cache | |
| 217 | + | apps/desktop/src-tauri/target/release | |
| 218 | + | apps/desktop/src-tauri/target/x86_64-pc-windows-msvc/release | |
| 219 | + | !apps/desktop/src-tauri/target/**/bundle | |
| 220 | + | !apps/desktop/src-tauri/target/**/incremental | |
| 221 | + | key: desktop-release-windows-${{ runner.os }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.desktop }} | |
| 222 | + | restore-keys: desktop-release-windows-${{ runner.os }}-${{ steps.keys.outputs.rust }}- | |
| 223 | + | # npm's downloads for the whole lockfile, which the Caches workflow | |
| 224 | + | # keeps on main: an install reads them first and asks the registry | |
| 225 | + | # only for what changed. Only restored: a tag's entries are read by | |
| 226 | + | # nothing else. | |
| 211 | 227 | - name: Cache npm's downloads | |
| 212 | − | uses: actions/cache@v4 | |
| 228 | + | uses: actions/cache/restore@v4 | |
| 213 | 229 | with: | |
| 214 | 230 | path: ~/.npm | |
| 215 | 231 | key: npm-full-${{ runner.os }}-${{ hashFiles('package-lock.json') }} | |
| 45 | 45 | fail-fast: true | |
| 46 | 46 | matrix: | |
| 47 | 47 | target: [linux-x64, linux-arm64, windows-x64] | |
| 48 | + | env: | |
| 49 | + | G1T_RUNNER_RELEASE_KEY: ${{ vars.RUNNER_RELEASE_PUBLIC_KEY }} | |
| 48 | 50 | steps: | |
| 49 | 51 | - uses: actions/checkout@v5 | |
| 50 | 52 | - name: The tag names this version | |
| ⋯ | |||
| 52 | 54 | run: | | |
| 53 | 55 | version="$(sed -n 's/^version = "\(.*\)"/\1/p' crates/runner/Cargo.toml | head -1)" | |
| 54 | 56 | [ "runner-v$version" = "${GITHUB_REF_NAME}" ] || { echo "::error::The tag is ${GITHUB_REF_NAME}, but crates/runner is $version"; exit 1; } | |
| 55 | − | # zig and cargo-zigbuild, kept between releases: installing them took | |
| 56 | − | # four minutes of every one. Change the key to take newer versions. | |
| 57 | + | # zig and cargo-zigbuild, kept on main by the Caches workflow | |
| 58 | + | # (caches.yml): installing them took four minutes of every release. | |
| 59 | + | # Change the key there and here to take newer versions. | |
| 57 | 60 | - name: Cache zig and cargo-zigbuild | |
| 58 | 61 | id: zig | |
| 59 | − | uses: actions/cache@v4 | |
| 62 | + | uses: actions/cache/restore@v4 | |
| 60 | 63 | with: | |
| 61 | 64 | path: | | |
| 62 | 65 | ~/.local/lib/python3*/site-packages/ziglang | |
| ⋯ | |||
| 71 | 74 | cargo install --locked cargo-zigbuild | |
| 72 | 75 | - name: zig on the PATH | |
| 73 | 76 | run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| 74 | − | # Each target's build, kept between releases; a release after a | |
| 75 | − | # Cargo.lock change starts from the last one's rather than from nothing. | |
| 76 | − | - uses: actions/cache@v4 | |
| 77 | + | # rust: the toolchain; cargo: Cargo.lock's crates.io and git packages, | |
| 78 | + | # not the workspace's own versions (scripts/deploy/ci-cache.mjs). | |
| 79 | + | - name: Cache keys | |
| 80 | + | id: keys | |
| 81 | + | run: node scripts/deploy/ci-cache.mjs keys | |
| 82 | + | # Every crate the lockfile names, as CI's Rust job saves them on main. | |
| 83 | + | - name: Cache crates | |
| 84 | + | uses: actions/cache/restore@v4 | |
| 77 | 85 | with: | |
| 78 | 86 | path: | | |
| 79 | − | ~/.cargo/registry | |
| 87 | + | ~/.cargo/registry/index | |
| 88 | + | ~/.cargo/registry/cache | |
| 89 | + | key: cargo-crates-${{ runner.os }}-${{ steps.keys.outputs.cargo }} | |
| 90 | + | restore-keys: cargo-crates-${{ runner.os }}- | |
| 91 | + | # Each target's build. A tag's run restores only from its own tag and | |
| 92 | + | # from main, never from the last release's tag, so the Caches workflow | |
| 93 | + | # (caches.yml) builds and saves this on main whenever the key is new; | |
| 94 | + | # a release finds it whole, compiles the workspace's crates that | |
| 95 | + | # changed, and saves nothing. | |
| 96 | + | - name: Cache the runner's build | |
| 97 | + | uses: actions/cache/restore@v4 | |
| 98 | + | with: | |
| 99 | + | path: | | |
| 80 | 100 | target | |
| 81 | − | key: runner-release-${{ matrix.target }}-${{ hashFiles('Cargo.lock') }} | |
| 82 | − | restore-keys: runner-release-${{ matrix.target }}- | |
| 101 | + | !target/runner-release | |
| 102 | + | !target/**/incremental | |
| 103 | + | key: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}-${{ steps.keys.outputs.cargo }} | |
| 104 | + | restore-keys: runner-release-${{ runner.os }}-${{ matrix.target }}-${{ steps.keys.outputs.rust }}- | |
| 83 | 105 | # Crates whose inputs did not change come back from the Actions cache | |
| 84 | 106 | # instead of being compiled (scripts/sccache.sh); without it, the job | |
| 85 | 107 | # builds as before. | |
| 86 | 108 | - name: sccache | |
| 87 | 109 | run: bash scripts/sccache.sh install | |
| 88 | − | # The cache above brings back the last release's target/runner-release | |
| 89 | − | # too: left there, it would go out with this one's, and the jobs after | |
| 90 | − | # would find two of each file. | |
| 91 | − | - name: Only this release's files | |
| 92 | − | run: rm -rf target/runner-release | |
| 93 | 110 | - name: Build Linux and Windows | |
| 94 | 111 | # The macOS targets need Apple's SDK to link (the Security and | |
| 95 | 112 | # CoreFoundation frameworks), which zig does not carry, so they are | |
| 96 | 113 | # left out until this builds in the rust-cross/cargo-zigbuild image | |
| 97 | 114 | # as the CLI's release does (scripts/cli-release.mjs). A Mac runner | |
| 98 | 115 | # keeps the version it has, since the manifest names no file for it. | |
| 99 | − | env: | |
| 100 | − | G1T_RUNNER_RELEASE_KEY: ${{ vars.RUNNER_RELEASE_PUBLIC_KEY }} | |
| 101 | 116 | run: node scripts/runner-release.mjs build --targets ${{ matrix.target }} | |
| 102 | 117 | - name: sccache's hits | |
| 103 | 118 | if: always() | |
| 164 | 164 | `/-/notify`, notify's phone push) deploys with the site and services. | |
| 165 | 165 | The guide is [The phone app](https://docs.g1t.sh/guides/mobile/). | |
| 166 | 166 | ||
| 167 | + | ## How g1t's own CI caches | |
| 168 | + | ||
| 169 | + | g1t's workflows cache with `actions/cache` against g1t's own cache | |
| 170 | + | ([the Actions guide](https://docs.g1t.sh/guides/actions/#the-cache) has its | |
| 171 | + | rules). Three of them shape everything here: a run restores from its own | |
| 172 | + | ref, its pull request's base, then `main`, so a release tag never reads | |
| 173 | + | another tag's entries; a key is written once; and a repository holds | |
| 174 | + | 10 GiB, past which the entries restored longest ago go. So: | |
| 175 | + | ||
| 176 | + | - **`main` saves; pull requests and tags restore.** CI's Rust and desktop | |
| 177 | + | jobs keep their caches on each push to `main`, and the Caches workflow | |
| 178 | + | (`.g1t/workflows/caches.yml`) builds and saves what only releases use, | |
| 179 | + | when its key is missing. Release workflows only restore. | |
| 180 | + | - **Rust caches are keyed by the toolchain and the dependencies**, from | |
| 181 | + | `node scripts/deploy/ci-cache.mjs keys`: `rustc -V`, and a hash of the | |
| 182 | + | packages `Cargo.lock` takes from crates.io and git. A release bumping a | |
| 183 | + | workspace crate's version changes neither. A restore key never crosses | |
| 184 | + | toolchains. | |
| 185 | + | - **A Cargo target restored from an older entry is pruned before it is | |
| 186 | + | saved** (`ci-cache.mjs prune`), to what this build used. | |
| 187 | + | - **A partial install never saves a whole-install key.** npm's downloads | |
| 188 | + | and the crates are saved only by jobs that fetch all of them. | |
| 189 | + | - **One set of paths per entry.** An entry's version is the hash of its | |
| 190 | + | paths, so every step that names it, in every workflow, names the same | |
| 191 | + | ones. `crates/actions/tests/repository_workflows.rs` holds all of this. | |
| 192 | + | ||
| 193 | + | | Entry | Key | Saved by | Restored by | | |
| 194 | + | | --- | --- | --- | --- | | |
| 195 | + | | npm's downloads | `npm-full-<os>-<package-lock.json>` | Caches (`npm`) | every npm install | | |
| 196 | + | | Crates and their index | `cargo-crates-<os>-<deps>` | CI Rust (`cargo fetch` of the whole lockfile) | Deploy, the runner release | | |
| 197 | + | | The test build | `cargo-test-<os>-<rust>-<deps>` | CI Rust | CI Rust on pull requests | | |
| 198 | + | | Workers' builds | `cargo-workers-<os>-<rust>-<group>-<deps>` | Deploy | Deploy | | |
| 199 | + | | The runner's image build | `runner-musl-<os>-<rust>-<deps>` | Deploy | Deploy | | |
| 200 | + | | worker-build and its tools | `worker-build-…`, `worker-build-tools-…` | Deploy | Deploy | | |
| 201 | + | | Compiled crates | sccache's, one per crate | every Rust job | every Rust job | | |
| 202 | + | | The desktop app's check | `desktop-check-<os>-<rust>-<desktop deps>` | CI desktop | CI desktop on pull requests | | |
| 203 | + | | apt for Tauri | `apt-tauri-<os>-<scripts/apt-install.sh>` | CI desktop | the desktop release | | |
| 204 | + | | The runner release | `runner-release-<os>-<target>-<rust>-<deps>`, zig | Caches (`runner`) | the runner release | | |
| 205 | + | | The desktop release | `desktop-release-linux-…`, `desktop-release-windows-…`, `xwin-…`, `apt-tauri-windows-…` | Caches (`desktop-linux`, `desktop-windows`) | the desktop release | | |
| 206 | + | ||
| 207 | + | To see whether a change helped, read a run's cache steps: `Cache restored | |
| 208 | + | from key` names the entry (the key itself is a hit; another is a start), | |
| 209 | + | `Cache Size` and the step's time say what it cost, and the post step says | |
| 210 | + | whether it saved. `sccache's hits and misses` is on each Rust job's summary. | |
| 211 | + | ||
| 167 | 212 | ## Speed | |
| 168 | 213 | ||
| 169 | 214 | Pages are a few rounds of service calls, and each round costs a trip to |
| 793 | 793 | | One entry | Up to 2 GiB, compressed. A larger one is not saved, and the job goes on. | | |
| 794 | 794 | | A repository's entries | Up to 10 GiB together. Saving past it removes the entries restored longest ago. | | |
| 795 | 795 | | A free workspace | Its cache and [artifacts](#artifacts), in every repository, up to 2 GiB together. Past it, a save is refused (the job goes on) until old entries expire or artifacts are deleted; on the g1t plan there is no such cap. | | |
| 796 | − | | How long | Until it has not been restored for 7 days, and at most 28 days after it was saved. | | |
| 797 | − | | Which branch | An entry belongs to the branch, tag or pull request whose run saved it. A run restores from its own, then from the branch its pull request merges into, then from the default branch. | | |
| 798 | − | | Keys | Written once on each branch: saving under a key that exists there does nothing. On each branch in turn, a restore finds its `key` exactly, else the newest entry whose key starts with one of its `restore-keys`. | | |
| 796 | + | | How long | Until it has not been restored for 7 days, and at most 28 days after it was saved. A lookup (`lookup-only: true`) counts as a restore. | | |
| 797 | + | | Which branch | An entry belongs to the branch, tag or pull request whose run saved it. A run restores from its own, then from the branch its pull request merges into, then from the default branch. A tag's run reads its own tag's entries and the default branch's, never another tag's. | | |
| 798 | + | | Keys | Written once on each branch: saving under a key that exists there does nothing, and the step says the key is already cached. On each branch in turn, a restore finds its `key` exactly, else the newest entry whose key starts with its `key`, else the newest whose key starts with each of its `restore-keys` in order. | | |
| 799 | 799 | | `path` | Files and folders; globs, `**` included; `~/` is the home folder; a line starting with `!` leaves matching paths out. The same key saved for other paths is another entry. | | |
| 800 | 800 | | Compression | zstd. | | |
| 801 | 801 | ||
| 802 | 802 | So a feature branch can read what `main` saved, but `main` never reads | |
| 803 | 803 | what a feature branch saved, and a pull request from outside the | |
| 804 | 804 | repository saves where nothing else ever reads it: nobody can plant an | |
| 805 | − | entry that the default branch's builds restore. Actions that cache through the | |
| 805 | + | entry that the default branch's builds restore. For the same reason, a | |
| 806 | + | release built from a tag finds only what the default branch saved: have | |
| 807 | + | a workflow on the default branch save what your releases restore. Actions that cache through the | |
| 806 | 808 | toolkit, such as `actions/setup-node` with `cache: npm`, follow the same | |
| 807 | 809 | rules. | |
| 808 | 810 | ||
| ⋯ | |||
| 888 | 890 | it, from its paths and compression, as on GitHub: an entry `setup-node` | |
| 889 | 891 | saved is restored by `actions/cache` with the same key and paths, and | |
| 890 | 892 | the other way round. | |
| 891 | − | - An entry the toolkit sends whole, which it does below 128 MB, is not | |
| 892 | − | saved when it is over 100 MB, the most g1t takes in one request. The | |
| 893 | − | step warns and the job goes on. | |
| 893 | + | - Against any server but github.com, the toolkit uses the cache's older | |
| 894 | + | protocol, which sends an entry in 32 MB chunks, so an entry up to the | |
| 895 | + | cache's 2 GiB is saved. A client of the newer protocol that sends an | |
| 896 | + | entry whole in one request is held to 100 MB there; past it, the step | |
| 897 | + | warns and the job goes on. | |
| 894 | 898 | - The toolkit's artifact library refuses to run against any server but | |
| 895 | 899 | github.com, so an action that uploads artifacts with it directly fails | |
| 896 | 900 | with its own message. `actions/upload-artifact`, | |
| 187 | 187 | assert_eq!(concurrency.cancel_in_progress, json!(false)); | |
| 188 | 188 | assert_eq!( | |
| 189 | 189 | deploy.job_order(), | |
| 190 | − | [ | |
| 191 | − | "check", | |
| 192 | − | "npm-cache", | |
| 193 | − | "plan", | |
| 194 | − | "migrate", | |
| 195 | − | "core", | |
| 196 | − | "edge", | |
| 197 | − | "front", | |
| 198 | − | "smoke" | |
| 199 | − | ] | |
| 200 | − | ); | |
| 201 | − | // Nothing waits for npm's downloads to be saved. | |
| 202 | − | assert!( | |
| 203 | − | deploy | |
| 204 | − | .jobs | |
| 205 | − | .iter() | |
| 206 | − | .all(|job| !job.needs.iter().any(|n| n == "npm-cache")) | |
| 190 | + | ["check", "plan", "migrate", "core", "edge", "front", "smoke"] | |
| 207 | 191 | ); | |
| 208 | 192 | // The stages share their steps (a YAML alias), and read the token only | |
| 209 | 193 | // where they deploy. | |
| ⋯ | |||
| 274 | 258 | } | |
| 275 | 259 | } | |
| 276 | 260 | } | |
| 277 | − | // main's whole cache is kept by the deploy workflow, beside its check. | |
| 278 | − | assert!( | |
| 279 | − | saving.contains(&"deploy.yml npm-cache".to_owned()), | |
| 280 | − | "{saving:?}" | |
| 261 | + | // main's whole cache is kept by the Caches workflow. | |
| 262 | + | assert!(saving.contains(&"caches.yml npm".to_owned()), "{saving:?}"); | |
| 263 | + | } | |
| 264 | + | ||
| 265 | + | /// Every `actions/cache` step of every workflow: (file, job, step). | |
| 266 | + | fn cache_steps() -> Vec<(String, workflow::Job, workflow::Step)> { | |
| 267 | + | let mut out = Vec::new(); | |
| 268 | + | let mut names: Vec<String> = std::fs::read_dir(workflows_dir()) | |
| 269 | + | .unwrap() | |
| 270 | + | .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) | |
| 271 | + | .filter(|n| n.ends_with(".yml")) | |
| 272 | + | .collect(); | |
| 273 | + | names.sort(); | |
| 274 | + | for name in names { | |
| 275 | + | for job in read(&name).jobs { | |
| 276 | + | for step in &job.steps { | |
| 277 | + | if step | |
| 278 | + | .uses | |
| 279 | + | .as_deref() | |
| 280 | + | .is_some_and(|u| u.starts_with("actions/cache")) | |
| 281 | + | { | |
| 282 | + | out.push((name.clone(), job.clone(), step.clone())); | |
| 283 | + | } | |
| 284 | + | } | |
| 285 | + | } | |
| 286 | + | } | |
| 287 | + | out | |
| 288 | + | } | |
| 289 | + | ||
| 290 | + | /// What a cache step's key starts with before its first expression: the | |
| 291 | + | /// entry it is, whatever its inputs (`cargo-test-`, `apt-tauri-windows-`). | |
| 292 | + | fn family(key: &str) -> String { | |
| 293 | + | key.split("${{").next().unwrap_or_default().to_owned() | |
| 294 | + | } | |
| 295 | + | ||
| 296 | + | fn saves(step: &workflow::Step) -> bool { | |
| 297 | + | let uses = step.uses.as_deref().unwrap_or_default(); | |
| 298 | + | (uses.starts_with("actions/cache@") || uses.starts_with("actions/cache/save@")) | |
| 299 | + | && step.raw["with"]["lookup-only"] != json!(true) | |
| 300 | + | } | |
| 301 | + | ||
| 302 | + | fn paths(step: &workflow::Step) -> Vec<String> { | |
| 303 | + | let text = step.raw["with"]["path"].as_str().unwrap_or_default(); | |
| 304 | + | text.lines() | |
| 305 | + | .map(str::trim) | |
| 306 | + | .filter(|l| !l.is_empty()) | |
| 307 | + | .map(str::to_owned) | |
| 308 | + | .collect() | |
| 309 | + | } | |
| 310 | + | ||
| 311 | + | #[test] | |
| 312 | + | fn each_cache_has_one_set_of_paths_wherever_it_is_used() { | |
| 313 | + | // An entry's version is the hash of its paths: a step that names the | |
| 314 | + | // same key with other paths finds nothing, silently. So every step of | |
| 315 | + | // an entry, in every workflow, names the same paths. | |
| 316 | + | let mut seen: std::collections::BTreeMap<String, (Vec<String>, String)> = Default::default(); | |
| 317 | + | for (file, job, step) in cache_steps() { | |
| 318 | + | let key = step.raw["with"]["key"].as_str().unwrap_or_default(); | |
| 319 | + | let family = family(key); | |
| 320 | + | assert!(!family.is_empty(), "{file} {}: a key starts with its name", job.id); | |
| 321 | + | let here = format!("{file} {}", job.id); | |
| 322 | + | match seen.get(&family) { | |
| 323 | + | Some((first, at)) => assert_eq!( | |
| 324 | + | &paths(&step), | |
| 325 | + | first, | |
| 326 | + | "{here} names other paths for {family} than {at}" | |
| 327 | + | ), | |
| 328 | + | None => { | |
| 329 | + | seen.insert(family, (paths(&step), here)); | |
| 330 | + | } | |
| 331 | + | } | |
| 332 | + | } | |
| 333 | + | for family in [ | |
| 334 | + | "npm-full-", | |
| 335 | + | "cargo-crates-", | |
| 336 | + | "cargo-test-", | |
| 337 | + | "cargo-workers-", | |
| 338 | + | "runner-musl-", | |
| 339 | + | "runner-release-", | |
| 340 | + | "desktop-check-", | |
| 341 | + | "desktop-release-linux-", | |
| 342 | + | "desktop-release-windows-", | |
| 343 | + | "apt-tauri-", | |
| 344 | + | "apt-tauri-windows-", | |
| 345 | + | "xwin-", | |
| 346 | + | "zig-0.13.0-zigbuild-", | |
| 347 | + | ] { | |
| 348 | + | assert!(seen.contains_key(family), "{family}: {:?}", seen.keys()); | |
| 349 | + | } | |
| 350 | + | } | |
| 351 | + | ||
| 352 | + | #[test] | |
| 353 | + | fn main_saves_every_cache_that_pull_requests_and_releases_restore() { | |
| 354 | + | // A run restores from its own ref, its pull request's base and main; | |
| 355 | + | // a release tag's run never reads another tag's entries. So every | |
| 356 | + | // entry any step restores is saved by a workflow that runs on pushes | |
| 357 | + | // to main, and the release workflows only restore. | |
| 358 | + | let on_main = ["ci.yml", "caches.yml", "deploy.yml"]; | |
| 359 | + | for file in on_main { | |
| 360 | + | let push = read(file).trigger("push").cloned().unwrap(); | |
| 361 | + | assert!(push.branches.allows("main"), "{file}"); | |
| 362 | + | } | |
| 363 | + | let releases = [ | |
| 364 | + | "cli-release.yml", | |
| 365 | + | "desktop-release.yml", | |
| 366 | + | "mobile-build.yml", | |
| 367 | + | "mobile-release.yml", | |
| 368 | + | "runner-release.yml", | |
| 369 | + | ]; | |
| 370 | + | let steps = cache_steps(); | |
| 371 | + | let saved_on_main: std::collections::BTreeSet<String> = steps | |
| 372 | + | .iter() | |
| 373 | + | .filter(|(file, _, step)| on_main.contains(&file.as_str()) && saves(step)) | |
| 374 | + | .map(|(_, _, step)| family(step.raw["with"]["key"].as_str().unwrap_or_default())) | |
| 375 | + | .collect(); | |
| 376 | + | for (file, job, step) in &steps { | |
| 377 | + | let family = family(step.raw["with"]["key"].as_str().unwrap_or_default()); | |
| 378 | + | // The conformance run restores what its own first job saved. | |
| 379 | + | if family == "conformance-" { | |
| 380 | + | continue; | |
| 381 | + | } | |
| 382 | + | assert!( | |
| 383 | + | saved_on_main.contains(&family), | |
| 384 | + | "{file} {}: nothing on main saves {family}", | |
| 385 | + | job.id | |
| 386 | + | ); | |
| 387 | + | if releases.contains(&file.as_str()) { | |
| 388 | + | assert!(!saves(step), "{file} {}: a release saves {family}, which only its own tag would read", job.id); | |
| 389 | + | } | |
| 390 | + | } | |
| 391 | + | } | |
| 392 | + | ||
| 393 | + | #[test] | |
| 394 | + | fn rust_caches_are_keyed_by_the_toolchain_and_the_dependencies() { | |
| 395 | + | // Not by Cargo.lock as a whole: 25 of its 31 changes in a week were a | |
| 396 | + | // release's version bump, and each saved every Rust cache again. | |
| 397 | + | for (file, job, step) in cache_steps() { | |
| 398 | + | let key = step.raw["with"]["key"].as_str().unwrap_or_default(); | |
| 399 | + | assert!( | |
| 400 | + | !key.contains("Cargo.lock") && !key.contains("base.json"), | |
| 401 | + | "{file} {}: {key}", | |
| 402 | + | job.id | |
| 403 | + | ); | |
| 404 | + | let target = paths(&step).iter().any(|p| p.contains("target")); | |
| 405 | + | if target { | |
| 406 | + | assert!(key.contains("steps.keys.outputs.rust"), "{file} {}: {key}", job.id); | |
| 407 | + | // A restore key never reaches another toolchain's entries. | |
| 408 | + | for prefix in step.raw["with"]["restore-keys"] | |
| 409 | + | .as_str() | |
| 410 | + | .unwrap_or_default() | |
| 411 | + | .lines() | |
| 412 | + | .filter(|l| !l.trim().is_empty()) | |
| 413 | + | { | |
| 414 | + | assert!(prefix.contains("steps.keys.outputs.rust"), "{file} {}: {prefix}", job.id); | |
| 415 | + | } | |
| 416 | + | } | |
| 417 | + | if key.contains("steps.keys.outputs.") { | |
| 418 | + | let at = job.steps.iter().position(|s| s == &step).unwrap(); | |
| 419 | + | let keys = job | |
| 420 | + | .steps | |
| 421 | + | .iter() | |
| 422 | + | .position(|s| s.id.as_deref() == Some("keys")) | |
| 423 | + | .unwrap_or_else(|| panic!("{file} {}: no keys step", job.id)); | |
| 424 | + | assert!(keys < at, "{file} {}", job.id); | |
| 425 | + | assert_eq!( | |
| 426 | + | job.steps[keys].run.as_deref(), | |
| 427 | + | Some("node scripts/deploy/ci-cache.mjs keys") | |
| 428 | + | ); | |
| 429 | + | } | |
| 430 | + | } | |
| 431 | + | } | |
| 432 | + | ||
| 433 | + | #[test] | |
| 434 | + | fn a_cargo_target_restored_from_an_older_entry_is_pruned_before_it_is_saved() { | |
| 435 | + | // Otherwise the old versions of what changed ride along forever: CI's | |
| 436 | + | // test build grew from 671 MB to 1,687 MB in a day. | |
| 437 | + | for (file, id, cache) in [ | |
| 438 | + | ("ci.yml", "rust", "test-build"), | |
| 439 | + | ("ci.yml", "desktop", "desktop-build"), | |
| 440 | + | ("caches.yml", "runner", "runner-build"), | |
| 441 | + | ] { | |
| 442 | + | let workflow = read(file); | |
| 443 | + | let job = workflow.jobs.iter().find(|j| j.id == id).unwrap(); | |
| 444 | + | let at = job | |
| 445 | + | .steps | |
| 446 | + | .iter() | |
| 447 | + | .position(|s| s.id.as_deref() == Some(cache)) | |
| 448 | + | .unwrap(); | |
| 449 | + | assert!(saves(&job.steps[at]), "{file} {id}"); | |
| 450 | + | let prune = job | |
| 451 | + | .steps | |
| 452 | + | .iter() | |
| 453 | + | .position(|s| { | |
| 454 | + | s.run | |
| 455 | + | .as_deref() | |
| 456 | + | .is_some_and(|r| r.starts_with("node scripts/deploy/ci-cache.mjs prune ")) | |
| 457 | + | }) | |
| 458 | + | .unwrap_or_else(|| panic!("{file} {id}: no prune")); | |
| 459 | + | assert!(at < prune, "{file} {id}"); | |
| 460 | + | assert_eq!( | |
| 461 | + | job.steps[prune].condition.as_deref().map(str::trim), | |
| 462 | + | Some(format!("${{{{ steps.{cache}.outputs.cache-hit != 'true' }}}}").as_str()), | |
| 463 | + | "{file} {id}" | |
| 464 | + | ); | |
| 465 | + | } | |
| 466 | + | let ci = read("ci.yml"); | |
| 467 | + | let rust = ci.jobs.iter().find(|j| j.id == "rust").unwrap(); | |
| 468 | + | assert_eq!( | |
| 469 | + | rust.raw["env"]["CARGO_PROFILE_DEV_DEBUG"], | |
| 470 | + | json!("line-tables-only") | |
| 281 | 471 | ); | |
| 282 | 472 | } | |
| 283 | 473 | ||
| 284 | 474 | #[test] | |
| 475 | + | fn the_caches_workflow_builds_only_what_is_missing() { | |
| 476 | + | let caches = read("caches.yml"); | |
| 477 | + | let push = caches.trigger("push").unwrap(); | |
| 478 | + | assert!(push.branches.allows("main")); | |
| 479 | + | assert!(!push.branches.allows("feature")); | |
| 480 | + | // Never cut off halfway: a build half done saves nothing. | |
| 481 | + | assert_eq!( | |
| 482 | + | caches.concurrency.as_ref().unwrap().cancel_in_progress, | |
| 483 | + | json!(false) | |
| 484 | + | ); | |
| 485 | + | let look = caches.jobs.iter().find(|j| j.id == "look").unwrap(); | |
| 486 | + | // Looked up, not downloaded. | |
| 487 | + | for step in &look.steps { | |
| 488 | + | if step.uses.as_deref().is_some_and(|u| u.starts_with("actions/cache")) { | |
| 489 | + | assert_eq!(step.uses.as_deref(), Some("actions/cache/restore@v4")); | |
| 490 | + | assert_eq!(step.raw["with"]["lookup-only"], json!(true), "{:?}", step.id); | |
| 491 | + | } | |
| 492 | + | } | |
| 493 | + | for job in &caches.jobs { | |
| 494 | + | if job.id != "look" { | |
| 495 | + | assert_eq!(job.needs, ["look"], "{}", job.id); | |
| 496 | + | assert!( | |
| 497 | + | job.condition | |
| 498 | + | .as_deref() | |
| 499 | + | .is_some_and(|c| c.contains("needs.look.outputs.")), | |
| 500 | + | "{}", | |
| 501 | + | job.id | |
| 502 | + | ); | |
| 503 | + | } | |
| 504 | + | } | |
| 505 | + | } | |
| 506 | + | ||
| 507 | + | #[test] | |
| 285 | 508 | fn deploy_stages_follow_one_another() { | |
| 286 | 509 | let deploy = read("deploy.yml"); | |
| 287 | 510 | let all = plan_outputs( | |
| 1 | + | #!/usr/bin/env bash | |
| 2 | + | # Installs a named set of Debian packages, from the job's cache when it has | |
| 3 | + | # them. CI's desktop job spent 110 s of its 175 s here on every run: 21 s | |
| 4 | + | # on `apt-get update`, 35 s downloading 335 MB, the rest unpacking. | |
| 5 | + | # | |
| 6 | + | # bash scripts/apt-install.sh tauri # Tauri's build on Linux | |
| 7 | + | # bash scripts/apt-install.sh tauri-windows # its Windows cross build | |
| 8 | + | # | |
| 9 | + | # apt's package lists and downloaded .debs are kept in ~/.cache/apt/<set>, | |
| 10 | + | # which the workflow restores and saves with actions/cache under | |
| 11 | + | # apt-<os>-<set>-<hash of this file>, so changing a set here starts a new | |
| 12 | + | # entry. With them restored, the install runs offline (--no-download); when | |
| 13 | + | # that fails (a list out of date for a package the image already has newer, | |
| 14 | + | # say) or nothing was restored, it updates the lists and downloads as | |
| 15 | + | # before, tried three times, since the mirror has answered with a | |
| 16 | + | # mismatched package before. An entry lives at most 28 days, so the lists | |
| 17 | + | # are never older than that. | |
| 18 | + | set -euo pipefail | |
| 19 | + | ||
| 20 | + | set_name="${1:-}" | |
| 21 | + | case "$set_name" in | |
| 22 | + | tauri) | |
| 23 | + | packages=(libwebkit2gtk-4.1-dev build-essential curl wget file libxdo-dev libssl-dev libayatana-appindicator3-dev librsvg2-dev) | |
| 24 | + | ;; | |
| 25 | + | tauri-windows) | |
| 26 | + | packages=(build-essential curl wget file libssl-dev libayatana-appindicator3-dev nsis clang lld llvm) | |
| 27 | + | ;; | |
| 28 | + | *) | |
| 29 | + | echo "usage: bash scripts/apt-install.sh tauri|tauri-windows" >&2 | |
| 30 | + | exit 2 | |
| 31 | + | ;; | |
| 32 | + | esac | |
| 33 | + | ||
| 34 | + | dir="$HOME/.cache/apt/$set_name" | |
| 35 | + | mkdir -p "$dir/archives/partial" "$dir/lists/partial" | |
| 36 | + | apt=(sudo apt-get -o "Dir::Cache::Archives=$dir/archives" -o "Dir::State::Lists=$dir/lists" -y) | |
| 37 | + | ||
| 38 | + | # The cache's files go back into it as this user, and apt leaves its | |
| 39 | + | # partial folders readable by root alone. | |
| 40 | + | mine() { sudo chown -R "$(id -u):$(id -g)" "$dir"; } | |
| 41 | + | trap mine EXIT | |
| 42 | + | ||
| 43 | + | if compgen -G "$dir/lists/*_Packages*" >/dev/null && "${apt[@]}" install --no-download "${packages[@]}"; then | |
| 44 | + | echo "Installed ${#packages[@]} packages from the cache." | |
| 45 | + | exit 0 | |
| 46 | + | fi | |
| 47 | + | ||
| 48 | + | for try in 1 2 3; do | |
| 49 | + | if "${apt[@]}" update && "${apt[@]}" install "${packages[@]}"; then | |
| 50 | + | # Only what this install used is kept. | |
| 51 | + | "${apt[@]}" autoclean >/dev/null 2>&1 || true | |
| 52 | + | exit 0 | |
| 53 | + | fi | |
| 54 | + | sudo rm -rf "$dir/archives/partial/"* | |
| 55 | + | sleep 15 | |
| 56 | + | done | |
| 57 | + | exit 1 |
| 1 | + | // The keys of the Rust caches in g1t's own workflows, and the pruning of a | |
| 2 | + | // Cargo target before it is saved. CONTRIBUTING.md ("How g1t's own CI | |
| 3 | + | // caches") has the whole scheme. | |
| 4 | + | // | |
| 5 | + | // node scripts/deploy/ci-cache.mjs keys | |
| 6 | + | // node scripts/deploy/ci-cache.mjs prune <profile dir>... -- <cargo args> | |
| 7 | + | // | |
| 8 | + | // keys writes to the step's outputs (GITHUB_OUTPUT): | |
| 9 | + | // | |
| 10 | + | // rust rustc's version, such as 1.99.0. Everything Cargo compiles | |
| 11 | + | // depends on it, so a new toolchain starts a new line of entries | |
| 12 | + | // rather than restoring the old one's, whose files it would | |
| 13 | + | // only carry along unused. | |
| 14 | + | // cargo a hash of the packages Cargo.lock takes from crates.io and git, | |
| 15 | + | // leaving out the workspace's own: a release bumps a crate's | |
| 16 | + | // version, which changes Cargo.lock and nothing that is cached. | |
| 17 | + | // Keyed by the whole file, 25 of the 31 changes to Cargo.lock in | |
| 18 | + | // the week before this were version bumps, and each saved every | |
| 19 | + | // Rust cache again (the test build's 0.7 to 1.7 GB each time). | |
| 20 | + | // desktop the same for apps/desktop/src-tauri/Cargo.lock. | |
| 21 | + | // | |
| 22 | + | // prune runs Cargo again with --message-format=json once the build is done | |
| 23 | + | // (everything is fresh, so it only lists what it built) and deletes, from | |
| 24 | + | // each profile dir's deps/, build/ and .fingerprint/, the files and folders | |
| 25 | + | // of units this build did not use. A target restored from an older entry | |
| 26 | + | // (a restore key) keeps the old versions of whatever changed beside the new | |
| 27 | + | // ones, and saving it carried them on forever: CI's test build grew from | |
| 28 | + | // 671 MB to 1,687 MB in a day, two thirds of the way to the 2 GB an entry | |
| 29 | + | // may hold, and its restore from 15 s to a minute. Anything unexpected | |
| 30 | + | // (Cargo fails, lists nothing) prunes nothing. | |
| 31 | + | ||
| 32 | + | import { spawnSync } from "node:child_process"; | |
| 33 | + | import { createHash } from "node:crypto"; | |
| 34 | + | import { appendFileSync, existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs"; | |
| 35 | + | import { basename, join } from "node:path"; | |
| 36 | + | import { fileURLToPath } from "node:url"; | |
| 37 | + | ||
| 38 | + | const ROOT = fileURLToPath(new URL("../..", import.meta.url)); | |
| 39 | + | ||
| 40 | + | /** The `[[package]]` blocks of a Cargo.lock that come from a registry or git: not the workspace's own. */ | |
| 41 | + | export function externalPackages(lock) { | |
| 42 | + | return lock | |
| 43 | + | .split(/^\[\[package\]\]\s*$/m) | |
| 44 | + | .slice(1) | |
| 45 | + | .map((block) => block.trim()) | |
| 46 | + | .filter((block) => /^source = /m.test(block)) | |
| 47 | + | .sort(); | |
| 48 | + | } | |
| 49 | + | ||
| 50 | + | /** 16 hex characters of the SHA-256 of a Cargo.lock's external packages. */ | |
| 51 | + | export function depsHash(lock) { | |
| 52 | + | return createHash("sha256").update(externalPackages(lock).join("\n\n")).digest("hex").slice(0, 16); | |
| 53 | + | } | |
| 54 | + | ||
| 55 | + | /** `1.99.0` from `rustc 1.99.0 (90b35a623 2026-09-30)`; null if it is not that. */ | |
| 56 | + | export function rustVersion(text) { | |
| 57 | + | return text.match(/^rustc (\S+)/)?.[1] ?? null; | |
| 58 | + | } | |
| 59 | + | ||
| 60 | + | function keys() { | |
| 61 | + | const rustc = spawnSync("rustc", ["-V"], { encoding: "utf8" }); | |
| 62 | + | const rust = rustVersion(rustc.stdout ?? ""); | |
| 63 | + | if (!rust) throw new Error(`rustc -V said ${JSON.stringify(rustc.stdout ?? rustc.error?.message)}`); | |
| 64 | + | const outputs = { rust }; | |
| 65 | + | for (const [name, path] of [ | |
| 66 | + | ["cargo", "Cargo.lock"], | |
| 67 | + | ["desktop", "apps/desktop/src-tauri/Cargo.lock"], | |
| 68 | + | ]) { | |
| 69 | + | const file = join(ROOT, path); | |
| 70 | + | if (existsSync(file)) outputs[name] = depsHash(readFileSync(file, "utf8")); | |
| 71 | + | } | |
| 72 | + | const lines = Object.entries(outputs).map(([k, v]) => `${k}=${v}`); | |
| 73 | + | console.log(lines.join("\n")); | |
| 74 | + | if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); | |
| 75 | + | } | |
| 76 | + | ||
| 77 | + | /** The 16-hex-character hash Cargo puts on a unit's files and folders (`libserde-0123456789abcdef.rlib`), or null. */ | |
| 78 | + | export function unitHash(name) { | |
| 79 | + | return name.match(/-([0-9a-f]{16})(?:\.[^/]*)?$/)?.[1] ?? null; | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | /** The hashes of every unit in Cargo's JSON messages: its files, and a build script's out dir. */ | |
| 83 | + | export function usedHashes(messages) { | |
| 84 | + | const used = new Set(); | |
| 85 | + | for (const line of messages.split("\n")) { | |
| 86 | + | if (!line.startsWith("{")) continue; | |
| 87 | + | let m; | |
| 88 | + | try { | |
| 89 | + | m = JSON.parse(line); | |
| 90 | + | } catch { | |
| 91 | + | continue; | |
| 92 | + | } | |
| 93 | + | const paths = [...(m.filenames ?? []), ...(m.executable ? [m.executable] : [])]; | |
| 94 | + | if (m.reason === "build-script-executed" && m.out_dir) paths.push(m.out_dir.replace(/[\\/]out$/, "")); | |
| 95 | + | for (const path of paths) { | |
| 96 | + | // A unit's own folder (build/foo-<hash>/build-script-build) or file (deps/libfoo-<hash>.rlib). | |
| 97 | + | for (const part of path.split(/[\\/]/).slice(-2)) { | |
| 98 | + | const hash = unitHash(part); | |
| 99 | + | if (hash) used.add(hash); | |
| 100 | + | } | |
| 101 | + | } | |
| 102 | + | } | |
| 103 | + | return used; | |
| 104 | + | } | |
| 105 | + | ||
| 106 | + | /** What to delete from a profile dir: deps/ files and build/, .fingerprint/ folders of units not in `used`. */ | |
| 107 | + | export function stale(dir, used) { | |
| 108 | + | const out = []; | |
| 109 | + | for (const sub of ["deps", "build", ".fingerprint"]) { | |
| 110 | + | const at = join(dir, sub); | |
| 111 | + | if (!existsSync(at)) continue; | |
| 112 | + | for (const name of readdirSync(at)) { | |
| 113 | + | const hash = unitHash(name); | |
| 114 | + | if (hash && !used.has(hash)) out.push(join(at, name)); | |
| 115 | + | } | |
| 116 | + | } | |
| 117 | + | return out; | |
| 118 | + | } | |
| 119 | + | ||
| 120 | + | function bytes(path) { | |
| 121 | + | const s = statSync(path, { throwIfNoEntry: false }); | |
| 122 | + | if (!s) return 0; | |
| 123 | + | if (!s.isDirectory()) return s.size; | |
| 124 | + | return readdirSync(path).reduce((sum, name) => sum + bytes(join(path, name)), 0); | |
| 125 | + | } | |
| 126 | + | ||
| 127 | + | function prune(args) { | |
| 128 | + | const split = args.indexOf("--"); | |
| 129 | + | const dirs = split < 0 ? [] : args.slice(0, split); | |
| 130 | + | const cargo = split < 0 ? [] : args.slice(split + 1); | |
| 131 | + | if (!dirs.length || !cargo.length) throw new Error("usage: ci-cache.mjs prune <profile dir>... -- <cargo args>"); | |
| 132 | + | const listed = spawnSync("cargo", [...cargo, "--message-format=json"], { | |
| 133 | + | cwd: ROOT, | |
| 134 | + | encoding: "utf8", | |
| 135 | + | maxBuffer: 1 << 30, | |
| 136 | + | stdio: ["ignore", "pipe", "inherit"], | |
| 137 | + | }); | |
| 138 | + | const used = listed.status === 0 ? usedHashes(listed.stdout) : new Set(); | |
| 139 | + | if (!used.size) { | |
| 140 | + | console.log(`::warning title=Cargo cache::cargo ${cargo.join(" ")} did not list what it built, so nothing is pruned`); | |
| 141 | + | return; | |
| 142 | + | } | |
| 143 | + | for (const dir of dirs) { | |
| 144 | + | const before = bytes(join(ROOT, dir)); | |
| 145 | + | const gone = stale(join(ROOT, dir), used); | |
| 146 | + | for (const path of gone) rmSync(path, { recursive: true, force: true }); | |
| 147 | + | const after = bytes(join(ROOT, dir)); | |
| 148 | + | const mb = (n) => `${Math.round(n / 1_048_576)} MB`; | |
| 149 | + | console.log(`${dir}: ${gone.length} files and folders of units this build did not use, ${mb(before - after)}; ${mb(after)} left`); | |
| 150 | + | } | |
| 151 | + | } | |
| 152 | + | ||
| 153 | + | if (process.argv[1] && basename(process.argv[1]) === "ci-cache.mjs") { | |
| 154 | + | const [command, ...rest] = process.argv.slice(2); | |
| 155 | + | if (command === "keys") keys(); | |
| 156 | + | else if (command === "prune") prune(rest); | |
| 157 | + | else { | |
| 158 | + | console.error("usage: node scripts/deploy/ci-cache.mjs keys | prune <profile dir>... -- <cargo args>"); | |
| 159 | + | process.exit(2); | |
| 160 | + | } | |
| 161 | + | } |
| 1 | + | import { test } from "node:test"; | |
| 2 | + | import assert from "node:assert/strict"; | |
| 3 | + | import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; | |
| 4 | + | import { tmpdir } from "node:os"; | |
| 5 | + | import { join } from "node:path"; | |
| 6 | + | import { depsHash, externalPackages, rustVersion, stale, unitHash, usedHashes } from "./ci-cache.mjs"; | |
| 7 | + | ||
| 8 | + | const LOCK = (runner, serde) => `# This file is automatically @generated by Cargo. | |
| 9 | + | version = 4 | |
| 10 | + | ||
| 11 | + | [[package]] | |
| 12 | + | name = "g1t-runner" | |
| 13 | + | version = "${runner}" | |
| 14 | + | dependencies = [ | |
| 15 | + | "serde", | |
| 16 | + | ] | |
| 17 | + | ||
| 18 | + | [[package]] | |
| 19 | + | name = "serde" | |
| 20 | + | version = "${serde}" | |
| 21 | + | source = "registry+https://github.com/rust-lang/crates.io-index" | |
| 22 | + | checksum = "abc" | |
| 23 | + | `; | |
| 24 | + | ||
| 25 | + | test("a release's version bump keeps the Rust caches' key, a new dependency changes it", () => { | |
| 26 | + | assert.deepEqual(externalPackages(LOCK("0.3.3", "1.0.200")).length, 1); | |
| 27 | + | assert.equal(depsHash(LOCK("0.3.3", "1.0.200")), depsHash(LOCK("0.3.4", "1.0.200"))); | |
| 28 | + | assert.notEqual(depsHash(LOCK("0.3.3", "1.0.200")), depsHash(LOCK("0.3.3", "1.0.201"))); | |
| 29 | + | assert.match(depsHash(LOCK("0.3.3", "1.0.200")), /^[0-9a-f]{16}$/); | |
| 30 | + | }); | |
| 31 | + | ||
| 32 | + | test("the toolchain is read from rustc -V", () => { | |
| 33 | + | assert.equal(rustVersion("rustc 1.99.0 (90b35a623 2026-09-30)\n"), "1.99.0"); | |
| 34 | + | assert.equal(rustVersion("rustc 1.100.0-nightly (abc 2026-10-01)"), "1.100.0-nightly"); | |
| 35 | + | assert.equal(rustVersion(""), null); | |
| 36 | + | }); | |
| 37 | + | ||
| 38 | + | test("a unit's hash is read from its files and folders", () => { | |
| 39 | + | assert.equal(unitHash("libserde-0123456789abcdef.rlib"), "0123456789abcdef"); | |
| 40 | + | assert.equal(unitHash("serde-0123456789abcdef"), "0123456789abcdef"); | |
| 41 | + | assert.equal(unitHash("g1t_actions-0123456789abcdef.d"), "0123456789abcdef"); | |
| 42 | + | assert.equal(unitHash("repository_workflows-0123456789abcdef"), "0123456789abcdef"); | |
| 43 | + | assert.equal(unitHash("g1t"), null); | |
| 44 | + | assert.equal(unitHash("libg1t.rlib"), null); | |
| 45 | + | }); | |
| 46 | + | ||
| 47 | + | test("only the units the build did not use are pruned", () => { | |
| 48 | + | const dir = mkdtempSync(join(tmpdir(), "ci-cache-")); | |
| 49 | + | try { | |
| 50 | + | const [used, old, script, run] = ["1111111111111111", "2222222222222222", "3333333333333333", "4444444444444444"]; | |
| 51 | + | for (const sub of ["deps", "build", ".fingerprint"]) mkdirSync(join(dir, sub), { recursive: true }); | |
| 52 | + | for (const f of [`libserde-${used}.rlib`, `libserde-${used}.rmeta`, `serde-${used}.d`, `libserde-${old}.rlib`, `serde-${old}.d`, `tests-${used}`]) { | |
| 53 | + | writeFileSync(join(dir, "deps", f), "x"); | |
| 54 | + | } | |
| 55 | + | for (const d of [`serde-${script}`, `serde-${run}`, `serde-${old}`]) mkdirSync(join(dir, "build", d)); | |
| 56 | + | for (const d of [`serde-${used}`, `serde-${script}`, `serde-${run}`, `serde-${old}`]) mkdirSync(join(dir, ".fingerprint", d)); | |
| 57 | + | const messages = [ | |
| 58 | + | " Compiling serde", | |
| 59 | + | JSON.stringify({ reason: "compiler-artifact", filenames: [join(dir, "build", `serde-${script}`, "build-script-build")], executable: null }), | |
| 60 | + | JSON.stringify({ reason: "build-script-executed", out_dir: join(dir, "build", `serde-${run}`, "out") }), | |
| 61 | + | JSON.stringify({ reason: "compiler-artifact", filenames: [join(dir, "deps", `libserde-${used}.rlib`), join(dir, "deps", `libserde-${used}.rmeta`)] }), | |
| 62 | + | JSON.stringify({ reason: "compiler-artifact", filenames: [], executable: join(dir, "deps", `tests-${used}`) }), | |
| 63 | + | JSON.stringify({ reason: "build-finished", success: true }), | |
| 64 | + | ].join("\n"); | |
| 65 | + | const hashes = usedHashes(messages); | |
| 66 | + | assert.deepEqual([...hashes].sort(), [used, script, run].sort()); | |
| 67 | + | const gone = stale(dir, hashes).map((p) => p.slice(dir.length + 1).replaceAll("\\", "/")).sort(); | |
| 68 | + | assert.deepEqual(gone, [`.fingerprint/serde-${old}`, `build/serde-${old}`, `deps/libserde-${old}.rlib`, `deps/serde-${old}.d`]); | |
| 69 | + | // Nothing listed: nothing would be kept, so the caller prunes nothing. | |
| 70 | + | assert.equal(usedHashes("error: could not compile").size, 0); | |
| 71 | + | } finally { | |
| 72 | + | rmSync(dir, { recursive: true, force: true }); | |
| 73 | + | } | |
| 74 | + | }); |
| 1 | + | #!/usr/bin/env bash | |
| 2 | + | # What the desktop app's Windows cross build needs on g1t's Linux machines, | |
| 3 | + | # besides cargo-xwin: Debian's packages (scripts/apt-install.sh, from the | |
| 4 | + | # job's cache when it has them), the names cargo-xwin runs LLVM's tools by, | |
| 5 | + | # and an NSIS header Debian lacks. desktop-release.yml's Windows build and | |
| 6 | + | # caches.yml, which keeps its cache warm on main, both run it. | |
| 7 | + | set -euo pipefail | |
| 8 | + | ||
| 9 | + | bash "$(dirname "$0")/apt-install.sh" tauri-windows | |
| 10 | + | # cargo-xwin compiles C (ring's, among others) with clang-cl and | |
| 11 | + | # links with lld-link and llvm-lib. Each is a mode of a tool | |
| 12 | + | # Debian does install, chosen by the name it is run as: clang-cl | |
| 13 | + | # is clang, lld-link is lld, and llvm-lib is llvm-ar. Debian ships | |
| 14 | + | # none of the three names, so each is linked to its tool. | |
| 15 | + | for pair in clang-cl:clang lld-link:lld llvm-lib:llvm-ar; do | |
| 16 | + | name="${pair%%:*}" tool="${pair##*:}" | |
| 17 | + | command -v "$name" >/dev/null && continue | |
| 18 | + | found="$(command -v "$tool" || ls /usr/lib/llvm-*/bin/"$tool" 2>/dev/null | sort -V | tail -1)" | |
| 19 | + | [ -n "$found" ] || { echo "::error::$tool is not installed, so there is no $name"; exit 1; } | |
| 20 | + | sudo ln -sf "$(readlink -f "$found")" "/usr/local/bin/$name" | |
| 21 | + | echo "$name: $(readlink -f "$found")" | |
| 22 | + | done | |
| 23 | + | # Tauri's installer script includes Win\RestartManager.nsh, which | |
| 24 | + | # NSIS ships from 3.10 on; Debian's is 3.08. The header stands | |
| 25 | + | # alone (defines and System calls), so it is added from NSIS 3.11's | |
| 26 | + | # source, checked against its SHA-256. | |
| 27 | + | rm_nsh=/usr/share/nsis/Include/Win/RestartManager.nsh | |
| 28 | + | if [ ! -f "$rm_nsh" ]; then | |
| 29 | + | curl -fsSL --retry 3 -o /tmp/RestartManager.nsh https://raw.githubusercontent.com/kichik/nsis/v311/Include/Win/RestartManager.nsh | |
| 30 | + | echo "67149978412da9b283e2e1abe48a360093df9caf6363373e2cabe463978b7c0e /tmp/RestartManager.nsh" | sha256sum -c - | |
| 31 | + | sudo install -m 0644 /tmp/RestartManager.nsh "$rm_nsh" | |
| 32 | + | fi |
| 13 | 13 | //! for other paths is another entry. | |
| 14 | 14 | //! - A key is written once in its scope and version. In each scope in | |
| 15 | 15 | //! turn, a restore finds its key exactly, else the newest entry whose key | |
| 16 | − | //! starts with one of its restore keys. | |
| 16 | + | //! starts with its key, else with each of its restore keys in order | |
| 17 | + | //! (`prefixes`), as GitHub's cache does. | |
| 17 | 18 | //! - An entry is at most `CACHE_MAX_ENTRY_BYTES`. A repository's entries | |
| 18 | 19 | //! hold at most `CACHE_REPO_QUOTA_BYTES` together: saving past it evicts | |
| 19 | 20 | //! the entries restored longest ago. | |
| ⋯ | |||
| 103 | 104 | (gb_months * CACHE_MICROS_PER_GB_MONTH as f64).ceil() as i64 | |
| 104 | 105 | } | |
| 105 | 106 | ||
| 107 | + | /// The prefixes a restore tries in each scope once its key has no exact | |
| 108 | + | /// match, in order: the key itself, then its restore keys. Blank ones and | |
| 109 | + | /// repeats are dropped. | |
| 110 | + | pub(crate) fn prefixes<'a>(key: &'a str, restore: &'a [String]) -> Vec<&'a str> { | |
| 111 | + | let mut out: Vec<&str> = Vec::new(); | |
| 112 | + | for prefix in std::iter::once(key).chain(restore.iter().map(String::as_str)).map(str::trim) { | |
| 113 | + | if !prefix.is_empty() && !out.contains(&prefix) { | |
| 114 | + | out.push(prefix); | |
| 115 | + | } | |
| 116 | + | } | |
| 117 | + | out | |
| 118 | + | } | |
| 119 | + | ||
| 106 | 120 | /// Where a job's cache entries are found and saved: its repository, and in | |
| 107 | 121 | /// it the refs it restores from, in order, and the one it saves to. | |
| 108 | 122 | pub(crate) struct CacheScope { | |
| ⋯ | |||
| 175 | 189 | if found.is_some() { | |
| 176 | 190 | break; | |
| 177 | 191 | } | |
| 178 | − | for prefix in a.restore.iter().map(|p| p.trim()).filter(|p| !p.is_empty()) { | |
| 192 | + | for prefix in prefixes(&a.key, &a.restore) { | |
| 179 | 193 | found = self | |
| 180 | 194 | .db | |
| 181 | 195 | .prepare( | |
| ⋯ | |||
| 549 | 563 | } | |
| 550 | 564 | ||
| 551 | 565 | #[test] | |
| 566 | + | fn a_restore_tries_its_key_as_a_prefix_before_its_restore_keys() { | |
| 567 | + | let restore = vec!["cargo-Linux-1.99.0-".to_owned(), " ".to_owned(), "cargo-".to_owned(), "cargo-".to_owned()]; | |
| 568 | + | assert_eq!(prefixes("cargo-Linux-1.99.0-abc", &restore), ["cargo-Linux-1.99.0-abc", "cargo-Linux-1.99.0-", "cargo-"]); | |
| 569 | + | assert_eq!(prefixes("k", &[]), ["k"]); | |
| 570 | + | } | |
| 571 | + | ||
| 572 | + | #[test] | |
| 552 | 573 | fn keys_are_checked() { | |
| 553 | 574 | assert!(valid_key("cargo-Linux-abc123")); | |
| 554 | 575 | assert!(!valid_key("")); | |