Skip to content
Sign up

flagon-io/g1t

public
0
180 linesCodeBlameRaw

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Merge the runner sandbox: g1t-runner sandbox up makes a person's own computer a self-hosted runner that nothing a job runs can get out of. Linux jobs run in a Linux VM of the runner's own (a WSL 2 distribution with no host mounts, interop or Docker socket on Windows; vfkit on macOS; Sysbox, or QEMU and KVM, on Linux), each in a fresh container under Sysbox inside it; on Windows 11 Pro and up, Windows jobs each run in a Windows Sandbox discarded after the job. Runners report their isolation, and the Runners page shows it.1#!/usr/bin/env node
2// The sandbox's images: what `g1t-runner sandbox up` runs on a person's own
3// computer (crates/runner/src/selfhosted/launcher/), built from
4// deploy/sandbox/Dockerfile for each runner release and listed, with their
5// SHA-256, in the release's signed manifest (scripts/runner-release.mjs).
6//
7// node scripts/sandbox-image.mjs build --arch x64 --runner target/runner-release/0.3.0/g1t-runner-linux-x64 \
8// [--kind wsl|vm|all] [--out DIR] [--version 0.3.0]
9// node scripts/sandbox-image.mjs section --dir DIR [--runner-image g1t.sh/flagon-io/g1t-runner@sha256:…]
10//
11// What it makes, in --out (default target/runner-release/<version>/):
12//
13// g1t-sandbox-wsl-<arch>.tar.gz a root filesystem for `wsl --import`
14// g1t-sandbox-vm-<arch>-kernel a VM's kernel (uncompressed on arm64,
15// g1t-sandbox-vm-<arch>-initrd as Apple's Virtualization framework
16// g1t-sandbox-vm-<arch>-disk.raw.gz wants), initrd and ext4 system disk
17//
18// `section` prints the manifest's `sandbox` section for the files in a
19// folder; runner-release.mjs `build` and `sign` take it from sandbox.json
20// there. Building needs Docker with buildx (and QEMU's emulators for the
21// other architecture); the VM's disk is made with mke2fs -d inside a
22// container, so nothing needs root on the machine.
23
24import { spawnSync } from "node:child_process";
25import { createHash } from "node:crypto";
26import { copyFileSync, createReadStream, createWriteStream, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
27import { tmpdir } from "node:os";
28import { dirname, join, resolve } from "node:path";
29import { pipeline } from "node:stream/promises";
30import { fileURLToPath } from "node:url";
31import { createGzip } from "node:zlib";
32
33const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
34
35/** The files of a sandbox for an architecture, by the key the manifest and the launcher use. */
36export function sandboxFiles(arch) {
37 return {
38 [`wsl-${arch}`]: `g1t-sandbox-wsl-${arch}.tar.gz`,
39 [`vm-${arch}-kernel`]: `g1t-sandbox-vm-${arch}-kernel`,
40 [`vm-${arch}-initrd`]: `g1t-sandbox-vm-${arch}-initrd`,
41 [`vm-${arch}-disk`]: `g1t-sandbox-vm-${arch}-disk.raw.gz`,
42 };
43}
44
45export const ARCHES = ["x64", "arm64"];
46const PLATFORM = { x64: "linux/amd64", arm64: "linux/arm64" };
47
48const sha256File = async (path) => {
49 const hash = createHash("sha256");
50 await pipeline(createReadStream(path), hash);
51 return hash.digest("hex");
52};
53
54/** The manifest's `sandbox` section for the sandbox files in `dir`. */
55export async function section(dir, runnerImage = process.env.RUNNER_SANDBOX_IMAGE || null) {
56 const files = {};
57 for (const arch of ARCHES) {
58 for (const [key, name] of Object.entries(sandboxFiles(arch))) {
59 const path = join(dir, name);
60 if (existsSync(path)) files[key] = { name, sha256: await sha256File(path), size: statSync(path).size };
61 }
62 }
63 return { runner_image: runnerImage, files };
64}
65
66function run(command, args, options = {}) {
67 const done = spawnSync(command, args, { stdio: "inherit", cwd: ROOT, ...options });
68 if (done.status !== 0) throw new Error(`${command} ${args.join(" ")} failed`);
69}
70
71function output(command, args) {
72 const done = spawnSync(command, args, { cwd: ROOT, encoding: "utf8" });
73 if (done.status !== 0) throw new Error(`${command} ${args.join(" ")} failed: ${done.stderr}`);
74 return done.stdout.trim();
75}
76
77function version() {
78 const toml = readFileSync(join(ROOT, "crates/runner/Cargo.toml"), "utf8");
79 return /^version\s*=\s*"([^"]+)"/m.exec(toml)[1];
80}
81
82/** A build context: deploy/sandbox and the runner's Linux binary. */
83function context(runner) {
84 const dir = mkdtempSync(join(tmpdir(), "g1t-sandbox-"));
85 for (const name of ["Dockerfile", "wsl.conf", "g1t-sandbox.service", "vm-network.network"]) {
86 copyFileSync(join(ROOT, "deploy/sandbox", name), join(dir, name));
87 }
88 copyFileSync(runner, join(dir, "g1t-runner"));
89 return dir;
90}
91
92/** Builds a target and exports its file system as a tar. */
93function exportTarget(ctx, target, arch, v, tar) {
94 const tag = `g1t-sandbox-build:${target}-${arch}`;
95 run("docker", ["buildx", "build", "--platform", PLATFORM[arch], "--target", target, "--build-arg", `IMAGE_VERSION=${v}`, "-t", tag, "--load", ctx]);
96 const id = output("docker", ["create", "--platform", PLATFORM[arch], tag]);
97 try {
98 run("docker", ["export", "-o", tar, id]);
99 } finally {
100 run("docker", ["rm", id], { stdio: "ignore" });
101 }
102}
103
104async function gzip(from, to) {
105 await pipeline(createReadStream(from), createGzip({ level: 9 }), createWriteStream(to));
106 rmSync(from);
107}
108
109async function buildWsl(ctx, arch, v, out) {
110 const tar = join(out, `g1t-sandbox-wsl-${arch}.tar`);
111 exportTarget(ctx, "wsl", arch, v, tar);
112 await gzip(tar, join(out, sandboxFiles(arch)[`wsl-${arch}`]));
113}
114
115async function buildVm(ctx, arch, v, out) {
116 const tar = join(out, `g1t-sandbox-vm-${arch}.tar`);
117 exportTarget(ctx, "vm", arch, v, tar);
118 const names = sandboxFiles(arch);
119 // Kernel and initrd out of /boot, and the rest made into an ext4 disk,
120 // in a container with e2fsprogs, as root there and nowhere else.
121 const script = [
122 "set -e",
123 "apt-get update -qq >/dev/null && apt-get install -y -qq e2fsprogs >/dev/null",
124 "mkdir /work/root && tar -C /work/root -xpf /out/vm.tar",
125 `cp /work/root/boot/vmlinuz-* /out/${names[`vm-${arch}-kernel`]}`,
126 `cp /work/root/boot/initrd.img-* /out/${names[`vm-${arch}-initrd`]}`,
127 // Apple's Virtualization framework boots an uncompressed arm64 Image.
128 arch === "arm64" ? `if gzip -t /out/${names[`vm-${arch}-kernel`]} 2>/dev/null; then mv /out/${names[`vm-${arch}-kernel`]} /out/k.gz && gzip -dc /out/k.gz > /out/${names[`vm-${arch}-kernel`]} && rm /out/k.gz; fi` : "true",
129 "mke2fs -q -t ext4 -L g1t-system -d /work/root /out/disk.raw 3G",
130 "chmod 0644 /out/*",
131 ].join(" && ");
132 copyFileSync(tar, join(out, "vm.tar"));
133 rmSync(tar);
134 run("docker", ["run", "--rm", "--tmpfs", "/work:size=6g", "-v", `${resolve(out)}:/out`, "debian:bookworm-slim", "sh", "-c", script]);
135 rmSync(join(out, "vm.tar"));
136 await gzip(join(out, "disk.raw"), join(out, names[`vm-${arch}-disk`]));
137}
138
139export async function build({ arch, runner, kind = "all", out, v = version() }) {
140 if (!PLATFORM[arch]) throw new Error(`--arch is one of ${ARCHES.join(", ")}`);
141 if (!runner || !existsSync(runner)) throw new Error("--runner names the runner's static Linux build for that architecture");
142 mkdirSync(out, { recursive: true });
143 const ctx = context(runner);
144 try {
145 if (kind === "wsl" || kind === "all") await buildWsl(ctx, arch, v, out);
146 if (kind === "vm" || kind === "all") await buildVm(ctx, arch, v, out);
147 } finally {
148 rmSync(ctx, { recursive: true, force: true });
149 }
150}
151
152if (process.argv[1]?.replaceAll("\\", "/").endsWith("scripts/sandbox-image.mjs")) {
153 const [command, ...rest] = process.argv.slice(2);
154 const option = (name) => {
155 const at = rest.indexOf(`--${name}`);
156 return at >= 0 ? rest[at + 1] : undefined;
157 };
158 try {
159 switch (command) {
160 case "build": {
161 const v = option("version") ?? version();
162 await build({ arch: option("arch") ?? "x64", runner: option("runner"), kind: option("kind") ?? "all", out: option("out") ?? join(ROOT, "target", "runner-release", v), v });
163 break;
164 }
165 case "section": {
166 const dir = option("dir") ?? join(ROOT, "target", "runner-release", version());
167 const s = await section(dir, option("runner-image") ?? process.env.RUNNER_SANDBOX_IMAGE ?? null);
168 writeFileSync(join(dir, "sandbox.json"), `${JSON.stringify(s, null, 2)}\n`);
169 console.log(JSON.stringify(s, null, 2));
170 break;
171 }
172 default:
173 console.error("usage: node scripts/sandbox-image.mjs build|section");
174 process.exit(2);
175 }
176 } catch (error) {
177 console.error(String(error.message ?? error));
178 process.exit(1);
179 }
180}

This file's history is long; its oldest lines are credited to the oldest commit read.