Sign in

flagon-io/g1t

public

g1t/crates/runner/src/deploy.rs

667 lines23,982 bytesCodeBlame

Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.

Deployments: a preview for every pull request, production on g1t.page1//! Builds one commit of a repository and hands the result to Cloudflare, as
2//! a preview of a pull request or as the repository's production.
3//!
4//! The sandbox never holds a Cloudflare credential that could touch anything
5//! else. The deployments service opens an upload for exactly the files
6//! this build produced and gives back a key that can only upload those;
7//! the sandbox uploads them with it, and sends the Worker's code to the
8//! service, which puts the app in place.
9//!
10//! What gets built:
11//!
12//! - A Workers project (a `wrangler.jsonc`, `wrangler.json` or
13//! `wrangler.toml`): bundled by `wrangler deploy --dry-run`, with its
14//! static assets, compatibility settings and `vars`. Other bindings (D1,
15//! KV, R2, Durable Objects…) are not provisioned yet; the deployment says
16//! which were left out.
17//! - Anything else: a static site. Its `build` script runs, and the first
18//! of `dist`, `build`, `out`, `public`, `_site` or `.output/public` that
19//! exists is served, or the repository itself if it has an `index.html`.
20//!
21//! Configuration comes from the environment:
22//!
23//! - `G1T_API`, `DEPLOY_ID`, `DEPLOY_TOKEN`: where and how to report.
24//! - `GIT_REMOTE`, `GIT_COMMIT`, `G1T_USER`, `G1T_TOKEN`: what to check out.
25//! - `BUILD_COMMAND`, `OUTPUT_DIR`: the repository's own choices, if any.
26//! - `BUILD_ENV`: a JSON object of variables the build runs with.
27
28use std::collections::BTreeMap;
29use std::path::{Path, PathBuf};
30use std::time::Instant;
31
32use anyhow::{Context, Result, bail};
33use base64::Engine;
34use base64::engine::general_purpose::STANDARD;
35use serde::{Deserialize, Serialize};
36use serde_json::{Value, json};
37use sha2::{Digest, Sha256};
38
39use crate::checks::{redact, run_command};
40use crate::{WORKDIR, auth_option, env, git};
41
42/// Where `wrangler deploy --dry-run` writes the bundle.
43const BUNDLE_DIR: &str = "/work/g1t-bundle";
44/// Cloudflare's limits on a Worker's static assets.
45const MAX_FILES: usize = 20_000;
46const MAX_FILE_BYTES: u64 = 25 * 1024 * 1024;
47/// How much of the build's output is kept for the deployment's log.
48const MAX_LOG_CHARS: usize = 20_000;
49/// Directories a static build usually writes to, in the order they are tried.
50const OUTPUT_DIRS: [&str; 6] = ["dist", "build", "out", "public", "_site", ".output/public"];
51/// Bindings a Workers project may declare that are not provisioned yet.
52const UNSUPPORTED_BINDINGS: [&str; 10] = [
53 "kv_namespaces",
54 "d1_databases",
55 "r2_buckets",
56 "durable_objects",
57 "services",
58 "queues",
59 "vectorize",
60 "hyperdrive",
61 "ai",
62 "workflows",
63];
64
65struct Reporter {
66 base: String,
67 token: String,
68}
69
70impl Reporter {
71 fn send(&self, step: &str, mut body: Value) -> Result<Value> {
72 body["token"] = self.token.clone().into();
73 let response = ureq::post(&format!("{}/{step}", self.base))
74 .send_json(body)
75 .with_context(|| format!("could not report `{step}` to g1t"))?;
76 Ok(response.into_json().unwrap_or(Value::Null))
77 }
78}
79
80/// The build's log, kept to its end.
81#[derive(Default)]
82struct Log {
83 text: String,
84}
85
86impl Log {
87 fn line(&mut self, line: &str) {
88 self.text.push_str(line);
89 self.text.push('\n');
90 }
91
92 fn tail(&self) -> String {
93 let length = self.text.chars().count();
94 if length <= MAX_LOG_CHARS {
95 return self.text.clone();
96 }
97 let kept: String = self.text.chars().skip(length - MAX_LOG_CHARS).collect();
98 format!("… (earlier output not shown)\n{kept}")
99 }
100}
101
102/// Runs a command in the checkout, logging it; fails if it fails.
103fn step(log: &mut Log, command: &str, secrets: &[String]) -> Result<()> {
104 log.line(&format!("$ {command}"));
105 let result = run_command(command, Path::new(WORKDIR), secrets);
106 if !result.output_text().is_empty() {
107 log.line(result.output_text());
108 }
109 if !result.passed {
110 bail!("`{command}` failed");
111 }
112 Ok(())
113}
114
115/// A Workers project's settings, from whichever config file it has.
116#[derive(Debug, Default, Deserialize)]
117struct WranglerConfig {
118 main: Option<String>,
119 compatibility_date: Option<String>,
120 #[serde(default)]
121 compatibility_flags: Vec<String>,
122 assets: Option<AssetsConfig>,
123 #[serde(default)]
124 vars: BTreeMap<String, Value>,
125 #[serde(flatten)]
126 rest: BTreeMap<String, Value>,
127}
128
129#[derive(Debug, Default, Deserialize)]
130struct AssetsConfig {
131 directory: Option<String>,
132 binding: Option<String>,
133 html_handling: Option<String>,
134 not_found_handling: Option<String>,
135}
136
137/// JSON with comments and trailing commas, as `wrangler.jsonc` allows.
138fn strip_jsonc(text: &str) -> String {
139 let mut out = String::with_capacity(text.len());
140 let mut chars = text.chars().peekable();
141 let mut in_string = false;
142 while let Some(c) = chars.next() {
143 if in_string {
144 out.push(c);
145 if c == '\\' {
146 if let Some(next) = chars.next() {
147 out.push(next);
148 }
149 } else if c == '"' {
150 in_string = false;
151 }
152 continue;
153 }
154 match (c, chars.peek()) {
155 ('"', _) => {
156 in_string = true;
157 out.push(c);
158 }
159 ('/', Some('/')) => {
160 for c in chars.by_ref() {
161 if c == '\n' {
162 out.push('\n');
163 break;
164 }
165 }
166 }
167 ('/', Some('*')) => {
168 chars.next();
169 let mut last = ' ';
170 for c in chars.by_ref() {
171 if last == '*' && c == '/' {
172 break;
173 }
174 last = c;
175 }
176 }
177 _ => out.push(c),
178 }
179 }
180 // Trailing commas before a closing bracket.
181 let mut cleaned = String::with_capacity(out.len());
182 let chars: Vec<char> = out.chars().collect();
183 let mut in_string = false;
184 let mut i = 0;
185 while i < chars.len() {
186 let c = chars[i];
187 if c == '"' && (i == 0 || chars[i - 1] != '\\') {
188 in_string = !in_string;
189 }
190 if c == ',' && !in_string {
191 let next = chars[i + 1..].iter().find(|c| !c.is_whitespace());
192 if matches!(next, Some('}') | Some(']')) {
193 i += 1;
194 continue;
195 }
196 }
197 cleaned.push(c);
198 i += 1;
199 }
200 cleaned
201}
202
203fn read_wrangler(dir: &Path) -> Result<Option<WranglerConfig>> {
204 for name in ["wrangler.jsonc", "wrangler.json"] {
205 let path = dir.join(name);
206 if path.exists() {
207 let text = std::fs::read_to_string(&path)?;
208 return Ok(Some(
209 serde_json::from_str(&strip_jsonc(&text)).with_context(|| format!("could not read {name}"))?,
210 ));
211 }
212 }
213 let path = dir.join("wrangler.toml");
214 if path.exists() {
215 let text = std::fs::read_to_string(&path)?;
216 return Ok(Some(toml::from_str(&text).context("could not read wrangler.toml")?));
217 }
218 Ok(None)
219}
220
221/// How to install the project's dependencies, judged by its lockfile.
222fn install_command(dir: &Path) -> Option<&'static str> {
223 if !dir.join("package.json").exists() {
224 return None;
225 }
226 Some(if dir.join("pnpm-lock.yaml").exists() {
227 "corepack enable && pnpm install --frozen-lockfile"
228 } else if dir.join("yarn.lock").exists() {
229 "corepack enable && yarn install"
230 } else if dir.join("bun.lockb").exists() || dir.join("bun.lock").exists() {
231 "npx --yes bun install"
232 } else if dir.join("package-lock.json").exists() {
233 "npm ci"
234 } else {
235 "npm install"
236 })
237}
238
239fn has_build_script(dir: &Path) -> bool {
240 std::fs::read_to_string(dir.join("package.json"))
241 .ok()
242 .and_then(|text| serde_json::from_str::<Value>(&text).ok())
243 .is_some_and(|package| package["scripts"]["build"].is_string())
244}
245
246/// One file of the site, as Cloudflare's asset upload names it.
247struct Asset {
248 path: String,
249 hash: String,
250 size: u64,
251 file: PathBuf,
252}
253
254fn content_type(path: &str) -> &'static str {
255 let extension = path.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
256 match extension.as_str() {
257 "html" | "htm" => "text/html",
258 "css" => "text/css",
259 "js" | "mjs" => "application/javascript",
260 "json" | "map" => "application/json",
261 "svg" => "image/svg+xml",
262 "png" => "image/png",
263 "jpg" | "jpeg" => "image/jpeg",
264 "gif" => "image/gif",
265 "webp" => "image/webp",
266 "avif" => "image/avif",
267 "ico" => "image/x-icon",
268 "woff" => "font/woff",
269 "woff2" => "font/woff2",
270 "ttf" => "font/ttf",
271 "txt" => "text/plain",
272 "xml" => "application/xml",
273 "wasm" => "application/wasm",
274 "pdf" => "application/pdf",
275 "mp4" => "video/mp4",
276 "webm" => "video/webm",
277 _ => "application/octet-stream",
278 }
279}
280
281/// Every file under `root`, but for what never belongs in a site.
282fn collect(root: &Path, dir: &Path, skip_project: bool, out: &mut Vec<Asset>) -> Result<()> {
283 for entry in std::fs::read_dir(dir)? {
284 let entry = entry?;
285 let name = entry.file_name().to_string_lossy().into_owned();
286 let path = entry.path();
287 let kind = entry.file_type()?;
288 if name == ".git" || (skip_project && (name == "node_modules" || name.starts_with(".g1t"))) {
289 continue;
290 }
291 if kind.is_dir() {
292 collect(root, &path, skip_project, out)?;
293 continue;
294 }
295 if !kind.is_file() || name == "_headers" || name == "_redirects" {
296 continue;
297 }
298 let size = entry.metadata()?.len();
299 let relative = path
300 .strip_prefix(root)?
301 .to_string_lossy()
302 .replace('\\', "/");
303 if size > MAX_FILE_BYTES {
304 bail!("{relative} is larger than Cloudflare's 25 MiB limit for one file");
305 }
306 let bytes = std::fs::read(&path)?;
307 let digest = hex::encode(Sha256::digest(&bytes));
308 out.push(Asset {
309 path: format!("/{relative}"),
310 hash: digest[..32].to_owned(),
311 size,
312 file: path,
313 });
314 if out.len() > MAX_FILES {
315 bail!("the site has more than {MAX_FILES} files, Cloudflare's limit");
316 }
317 }
318 Ok(())
319}
320
321#[derive(Deserialize)]
322#[serde(rename_all = "camelCase")]
323struct UploadSession {
324 jwt: String,
325 #[serde(default)]
326 buckets: Vec<Vec<String>>,
327 upload_url: String,
328}
329
330/// Sends one bucket of files with the upload's key. The last bucket's
331/// answer carries the key that completes the upload.
332fn upload_bucket(session: &UploadSession, bucket: &[String], by_hash: &BTreeMap<&str, &Asset>) -> Result<Option<String>> {
333 let boundary = format!("g1t-{}", hex::encode(Sha256::digest(bucket.join(",").as_bytes()))[..24].to_owned());
334 let mut body: Vec<u8> = Vec::new();
335 for hash in bucket {
336 let asset = by_hash
337 .get(hash.as_str())
338 .with_context(|| format!("Cloudflare asked for a file this build does not have ({hash})"))?;
339 let bytes = std::fs::read(&asset.file)?;
340 body.extend_from_slice(
341 format!(
342 "--{boundary}\r\nContent-Disposition: form-data; name=\"{hash}\"; filename=\"{hash}\"\r\nContent-Type: {}\r\n\r\n",
343 content_type(&asset.path)
344 )
345 .as_bytes(),
346 );
347 body.extend_from_slice(STANDARD.encode(bytes).as_bytes());
348 body.extend_from_slice(b"\r\n");
349 }
350 body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
351 let response = ureq::post(&session.upload_url)
352 .set("authorization", &format!("Bearer {}", session.jwt))
353 .set("content-type", &format!("multipart/form-data; boundary={boundary}"))
354 .send_bytes(&body);
355 let response = match response {
356 Ok(response) => response,
357 Err(ureq::Error::Status(code, response)) => {
358 bail!("Cloudflare refused the upload ({code}): {}", response.into_string().unwrap_or_default())
359 }
360 Err(error) => bail!("could not upload to Cloudflare: {error}"),
361 };
362 let answer: Value = response.into_json().unwrap_or(Value::Null);
363 Ok(answer["result"]["jwt"].as_str().map(str::to_owned))
364}
365
366#[derive(Serialize)]
367#[serde(rename_all = "camelCase")]
368struct Module {
369 name: String,
370 content_base64: String,
371 content_type: String,
372}
373
374/// The bundle `wrangler deploy --dry-run` wrote, main module first.
375fn bundle_modules(main: &str) -> Result<(String, Vec<Module>)> {
376 let stem = Path::new(main)
377 .file_stem()
378 .map(|stem| stem.to_string_lossy().into_owned())
379 .unwrap_or_else(|| "index".to_owned());
380 let mut modules = Vec::new();
381 let mut files = Vec::new();
382 collect_files(Path::new(BUNDLE_DIR), &mut files)?;
383 for file in files {
384 let name = file
385 .strip_prefix(BUNDLE_DIR)?
386 .to_string_lossy()
387 .replace('\\', "/");
388 let kind = match name.rsplit('.').next().unwrap_or("") {
389 "js" | "mjs" => "application/javascript+module",
390 "wasm" => "application/wasm",
391 "map" | "md" => continue,
392 _ => "text/plain",
393 };
394 modules.push(Module {
395 content_base64: STANDARD.encode(std::fs::read(&file)?),
396 content_type: kind.to_owned(),
397 name,
398 });
399 }
400 let main_name = modules
401 .iter()
402 .map(|module| module.name.clone())
403 .find(|name| *name == format!("{stem}.js") || *name == format!("{stem}.mjs"))
404 .or_else(|| {
405 modules
406 .iter()
407 .find(|module| module.content_type == "application/javascript+module")
408 .map(|module| module.name.clone())
409 })
410 .context("wrangler wrote no JavaScript module")?;
411 Ok((main_name, modules))
412}
413
414fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
415 for entry in std::fs::read_dir(dir)? {
416 let entry = entry?;
417 if entry.file_type()?.is_dir() {
418 collect_files(&entry.path(), out)?;
419 } else {
420 out.push(entry.path());
421 }
422 }
423 Ok(())
424}
425
426/// What was built: the Worker's code and settings, and where its site is.
427struct Built {
428 worker: Value,
429 assets_dir: Option<PathBuf>,
430 warnings: Vec<String>,
431}
432
433fn build(log: &mut Log, secrets: &[String]) -> Result<Built> {
434 let dir = Path::new(WORKDIR);
435 let config = read_wrangler(dir)?;
436 let custom_build = std::env::var("BUILD_COMMAND").ok().filter(|c| !c.trim().is_empty());
437 if let Some(install) = install_command(dir) {
438 step(log, install, secrets)?;
439 }
440 let mut warnings = Vec::new();
441 match config {
442 Some(config) => {
443 if let Some(command) = &custom_build {
444 step(log, command, secrets)?;
445 }
446 for binding in UNSUPPORTED_BINDINGS {
447 if config.rest.get(binding).is_some_and(|value| !value.is_null()) {
448 warnings.push(format!(
449 "`{binding}` is not provisioned on g1t.page yet, so the app runs without it."
450 ));
451 }
452 }
453 let mut worker = json!({
454 "compatibilityDate": config.compatibility_date.clone().unwrap_or_else(|| "2026-09-26".to_owned()),
455 "compatibilityFlags": config.compatibility_flags,
456 "vars": config.vars,
457 });
458 if let Some(main) = &config.main {
459 // `--dry-run` runs the project's own build and bundles it,
460 // without deploying anywhere.
461 step(
462 log,
463 &format!("npx --yes wrangler@4 deploy --dry-run --outdir {BUNDLE_DIR}"),
464 secrets,
465 )?;
466 let (main_module, modules) = bundle_modules(main)?;
467 worker["mainModule"] = main_module.into();
468 worker["modules"] = serde_json::to_value(modules)?;
469 }
470 let assets = config.assets.unwrap_or_default();
471 let assets_dir = assets.directory.as_ref().map(|directory| dir.join(directory));
472 worker["assetsBinding"] = assets.binding.into();
473 worker["htmlHandling"] = assets.html_handling.into();
474 worker["notFoundHandling"] = assets.not_found_handling.into();
475 Ok(Built {
476 worker,
477 assets_dir,
478 warnings,
479 })
480 }
481 None => {
482 if let Some(command) = &custom_build {
483 step(log, command, secrets)?;
484 } else if has_build_script(dir) {
485 step(log, "npm run build", secrets)?;
486 }
487 let chosen = std::env::var("OUTPUT_DIR").ok().filter(|d| !d.trim().is_empty());
488 let assets_dir = match chosen {
489 Some(chosen) => {
490 let path = dir.join(chosen.trim_matches('/'));
491 if !path.is_dir() {
492 bail!("the output directory `{chosen}` does not exist after the build");
493 }
494 path
495 }
496 None => OUTPUT_DIRS
497 .iter()
498 .map(|name| dir.join(name))
499 .find(|path| path.join("index.html").exists() || (path.is_dir() && path != &dir.join("public")))
500 .or_else(|| dir.join("index.html").exists().then(|| dir.to_path_buf()))
501 .context(
502 "found nothing to serve: no Workers config, no index.html, and none of dist, build, out, public, _site or .output/public",
503 )?,
504 };
505 let spa = !assets_dir.join("404.html").exists();
506 Ok(Built {
507 worker: json!({
508 "compatibilityDate": "2026-09-26",
509 "compatibilityFlags": [],
510 "vars": {},
511 "notFoundHandling": if spa { "single-page-application" } else { "404-page" },
512 }),
513 assets_dir: Some(assets_dir),
514 warnings,
515 })
516 }
517 }
518}
519
520fn check_out(secrets: &[String]) -> Result<()> {
521 let remote = env("GIT_REMOTE")?;
522 let commit = env("GIT_COMMIT")?;
523 let auth = auth_option(&env("G1T_USER")?, &env("G1T_TOKEN")?);
524 std::fs::create_dir_all("/work")?;
525 let cloned = git(Path::new("/work"), &["-c", &auth, "clone", "--quiet", &remote, WORKDIR]).and_then(|_| {
526 git(
527 Path::new(WORKDIR),
528 &["-c", "advice.detachedHead=false", "checkout", "--quiet", &commit],
529 )
530 });
531 if let Err(error) = cloned {
532 bail!("{}", redact(&format!("{error:#}"), secrets));
533 }
534 Ok(())
535}
536
537fn deploy(reporter: &Reporter, log: &mut Log, secrets: &[String]) -> Result<Value> {
538 check_out(secrets).context("the commit could not be checked out")?;
539 // What the repository's settings ask the build to run with.
540 if let Ok(vars) = std::env::var("BUILD_ENV")
541 && let Ok(Value::Object(vars)) = serde_json::from_str::<Value>(&vars)
542 {
543 for (name, value) in vars {
544 if let Some(value) = value.as_str() {
545 // SAFETY: single-threaded; set before any command runs.
546 unsafe { std::env::set_var(name, value) };
547 }
548 }
549 }
550 let built = build(log, secrets)?;
551 let mut finish = json!({
552 "worker": built.worker,
553 "warnings": built.warnings,
554 });
555 if let Some(dir) = &built.assets_dir {
556 let skip_project = dir == Path::new(WORKDIR);
557 let mut assets = Vec::new();
558 collect(dir, dir, skip_project, &mut assets)?;
559 if assets.is_empty() {
560 bail!("the site to serve is empty");
561 }
562 log.line(&format!("Uploading {} files.", assets.len()));
563 for special in ["_headers", "_redirects"] {
564 if let Ok(text) = std::fs::read_to_string(dir.join(special)) {
565 finish["worker"][special] = text.into();
566 }
567 }
568 let manifest: BTreeMap<&str, Value> = assets
569 .iter()
570 .map(|asset| (asset.path.as_str(), json!({ "hash": asset.hash, "size": asset.size })))
571 .collect();
572 let answer = reporter.send("session", json!({ "manifest": manifest }))?;
573 if answer["ok"] == false {
574 bail!("{}", answer["error"]["message"].as_str().unwrap_or("g1t refused the upload"));
575 }
576 let session: UploadSession =
577 serde_json::from_value(answer["value"].clone()).context("g1t's answer to the upload was not understood")?;
578 let by_hash: BTreeMap<&str, &Asset> = assets.iter().map(|asset| (asset.hash.as_str(), asset)).collect();
579 let mut completion = session.jwt.clone();
580 for bucket in &session.buckets {
581 if let Some(jwt) = upload_bucket(&session, bucket, &by_hash)? {
582 completion = jwt;
583 }
584 }
585 finish["completionJwt"] = completion.into();
586 }
587 Ok(finish)
588}
589
590pub fn main() -> i32 {
591 let reporter = match (env("G1T_API"), env("DEPLOY_ID"), env("DEPLOY_TOKEN")) {
592 (Ok(api), Ok(id), Ok(token)) => Reporter {
593 base: format!("{api}/deployments/jobs/{id}"),
594 token,
595 },
596 _ => {
597 eprintln!("g1t-runner: G1T_API, DEPLOY_ID and DEPLOY_TOKEN must be set");
598 return 2;
599 }
600 };
601 let secrets: Vec<String> = ["G1T_TOKEN", "DEPLOY_TOKEN"]
602 .iter()
603 .filter_map(|name| std::env::var(name).ok())
604 .filter(|secret| !secret.is_empty())
605 .collect();
606 if let Err(error) = reporter.send("started", json!({})) {
607 eprintln!("g1t-runner: {error:#}");
608 return 1;
609 }
610 let started = Instant::now();
611 let mut log = Log::default();
612 let outcome = deploy(&reporter, &mut log, &secrets);
613 let seconds = started.elapsed().as_secs();
614 let sent = match outcome {
615 Ok(mut finish) => {
616 finish["log"] = redact(&log.tail(), &secrets).into();
617 finish["buildSeconds"] = seconds.into();
618 reporter.send("finish", finish)
619 }
620 Err(error) => {
621 let message = redact(&format!("{error:#}"), &secrets);
622 log.line(&format!("The build failed: {message}"));
623 reporter.send(
624 "fail",
625 json!({ "message": message, "log": redact(&log.tail(), &secrets), "buildSeconds": seconds }),
626 )
627 }
628 };
629 match sent {
630 Ok(_) => 0,
631 Err(error) => {
632 eprintln!("g1t-runner: {error:#}");
633 1
634 }
635 }
636}
637
638#[cfg(test)]
639mod tests {
640 use super::*;
641
642 #[test]
643 fn jsonc_comments_and_trailing_commas_are_dropped() {
644 let text = r#"{
645 // a comment
646 "main": "src/index.ts", /* another */
647 "vars": { "URL": "https://x.dev//not-a-comment", },
648 }"#;
649 let config: WranglerConfig = serde_json::from_str(&strip_jsonc(text)).unwrap();
650 assert_eq!(config.main.as_deref(), Some("src/index.ts"));
651 assert_eq!(config.vars["URL"], "https://x.dev//not-a-comment");
652 }
653
654 #[test]
655 fn unsupported_bindings_are_noticed() {
656 let config: WranglerConfig =
657 serde_json::from_str(r#"{ "main": "a.js", "d1_databases": [{ "binding": "DB" }] }"#).unwrap();
658 assert!(config.rest.contains_key("d1_databases"));
659 }
660
661 #[test]
662 fn files_are_typed_by_extension() {
663 assert_eq!(content_type("/index.HTML"), "text/html");
664 assert_eq!(content_type("/a/b.woff2"), "font/woff2");
665 assert_eq!(content_type("/LICENSE"), "application/octet-stream");
666 }
667}