Sign in

flagon-io/g1t

public

Commit

A repository has its own sidebar, as settings do

In a repository the sidebar slides to its menu: the way back to mission control, the repository, then Code, Work, Automate and its settings. The workspace's usage and settings stay at the bottom under its name. Actions: artifacts (upload-artifact, download-artifact) and the cache (actions/cache, cache/restore, cache/save) work, kept in Workers KV in chunks with expiry until R2 is enabled; a run's artifacts download at /repos/{owner}/{repo}/actions/runs/{run}/artifacts/{name}. A g1t agent's pull request runs its workflows when marked ready, and each head runs each workflow once.

syntaqxcommitted Parent6731c5dBrowse files
11 files+748−830/11 viewed
+257−0
1+//! Artifacts and the cache of GitHub Actions jobs, kept in Workers KV in
2+//! chunks, with KV's own expiry: artifacts for 14 days with their run,
3+//! cache entries for 7 days with their repository.
4+//!
5+//! A sandbox reaches these with its job's token, at
6+//! `/actions/jobs/{job}/artifacts[/{name}]` and `/actions/jobs/{job}/cache`.
7+//! People download an artifact at
8+//! `/repos/{owner}/{repo}/actions/runs/{run}/artifacts/{name}`.
9+
10+use serde::{Deserialize, Serialize};
11+use serde_json::{Value, json};
12+use worker::kv::KvStore;
13+use worker::{Env, Request, Response, Result};
14+
15+use g1t_contracts::{FailureCode, Outcome};
16+
17+use crate::operations::Services;
18+
19+/// KV's largest value is 25 MiB; chunks stay under it.
20+const CHUNK: usize = 20 * 1024 * 1024;
21+/// The largest artifact or cache entry, kept within a Worker's memory.
22+const MAX_BYTES: usize = 60 * 1024 * 1024;
23+const ARTIFACT_TTL: u64 = 14 * 24 * 60 * 60;
24+const CACHE_TTL: u64 = 7 * 24 * 60 * 60;
25+
26+#[derive(Serialize, Deserialize)]
27+struct Meta {
28+ size: usize,
29+ chunks: usize,
30+ /// Milliseconds since the epoch, to find the newest cache entry.
31+ at: u64,
32+ /// The name or key it was saved under.
33+ name: String,
34+}
35+
36+fn store(env: &Env) -> Result<KvStore> {
37+ env.kv("BLOBS")
38+}
39+
40+async fn put(kv: &KvStore, base: &str, name: &str, bytes: &[u8], ttl: u64) -> Result<()> {
41+ let chunks: Vec<&[u8]> = if bytes.is_empty() { vec![&[][..]] } else { bytes.chunks(CHUNK).collect() };
42+ for (index, chunk) in chunks.iter().enumerate() {
43+ kv.put_bytes(&format!("{base}#{index}"), chunk)?.expiration_ttl(ttl).execute().await?;
44+ }
45+ let meta = Meta { size: bytes.len(), chunks: chunks.len(), at: g1t_kit::now_ms(), name: name.to_owned() };
46+ // The metadata travels with the key in listings, so the newest entry
47+ // can be found without reading each.
48+ kv.put(base, serde_json::to_string(&meta)?)?
49+ .metadata(&meta)?
50+ .expiration_ttl(ttl)
51+ .execute()
52+ .await?;
53+ Ok(())
54+}
55+
56+async fn get(kv: &KvStore, base: &str) -> Result<Option<Vec<u8>>> {
57+ let Some(meta) = kv.get(base).json::<Meta>().await? else { return Ok(None) };
58+ let mut out = Vec::with_capacity(meta.size);
59+ for index in 0..meta.chunks {
60+ match kv.get(&format!("{base}#{index}")).bytes().await? {
61+ Some(bytes) => out.extend_from_slice(&bytes),
62+ // A chunk that expired first: the whole entry is gone.
63+ None => return Ok(None),
64+ }
65+ }
66+ Ok(Some(out))
67+}
68+
69+/// The metadata of every entry under a prefix, newest first.
70+async fn list(kv: &KvStore, prefix: &str) -> Result<Vec<(String, Meta)>> {
71+ let mut found = Vec::new();
72+ let mut cursor: Option<String> = None;
73+ loop {
74+ let mut listing = kv.list().prefix(prefix.to_owned());
75+ if let Some(cursor) = cursor.take() {
76+ listing = listing.cursor(cursor);
77+ }
78+ let page = listing.execute().await?;
79+ for key in page.keys {
80+ if key.name.contains('#') {
81+ continue;
82+ }
83+ if let Some(meta) = key.metadata.and_then(|m| serde_json::from_value::<Meta>(m).ok()) {
84+ found.push((key.name, meta));
85+ }
86+ }
87+ if page.list_complete || page.cursor.is_none() {
88+ break;
89+ }
90+ cursor = page.cursor;
91+ }
92+ found.sort_by_key(|entry| std::cmp::Reverse(entry.1.at));
93+ Ok(found)
94+}
95+
96+fn error(status: u16, message: &str) -> Result<Response> {
97+ Ok(Response::from_json(&json!({ "error": { "message": message } }))?.with_status(status))
98+}
99+
100+fn valid_name(name: &str) -> bool {
101+ !name.is_empty() && name.len() <= 200 && !name.starts_with('.') && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ' '))
102+}
103+
104+fn decode(text: &str) -> String {
105+ let bytes = text.as_bytes();
106+ let mut out = Vec::with_capacity(bytes.len());
107+ let mut i = 0;
108+ while i < bytes.len() {
109+ match bytes[i] {
110+ b'%' if i + 2 < bytes.len() => {
111+ match u8::from_str_radix(std::str::from_utf8(&bytes[i + 1..i + 3]).unwrap_or("zz"), 16) {
112+ Ok(byte) => {
113+ out.push(byte);
114+ i += 3;
115+ }
116+ Err(_) => {
117+ out.push(b'%');
118+ i += 1;
119+ }
120+ }
121+ }
122+ b'+' => {
123+ out.push(b' ');
124+ i += 1;
125+ }
126+ byte => {
127+ out.push(byte);
128+ i += 1;
129+ }
130+ }
131+ }
132+ String::from_utf8_lossy(&out).into_owned()
133+}
134+
135+fn query(request: &Request, name: &str) -> Option<String> {
136+ let url = request.url().ok()?;
137+ url.query_pairs().find(|(key, _)| key == name).map(|(_, value)| value.into_owned())
138+}
139+
140+/// A sandbox storing or fetching an artifact or cache entry. `rest` is
141+/// the path after `/actions/jobs/`.
142+pub async fn for_job(mut request: Request, env: &Env, services: &Services, method: &str, rest: &str) -> Result<Response> {
143+ let (job, what) = rest.split_once('/').unwrap_or((rest, ""));
144+ let token = request
145+ .headers()
146+ .get("authorization")?
147+ .and_then(|h| h.strip_prefix("Bearer ").map(str::to_owned))
148+ .unwrap_or_default();
149+ let owner: Outcome<Value> = g1t_kit::call(&services.actions, "job_auth", &json!({ "job": job, "token": token })).await?;
150+ let owner = match owner {
151+ Outcome::Ok(owner) => owner,
152+ Outcome::Fail(_) => return error(401, "That job is not running, or the token is not its."),
153+ };
154+ let run = owner["run"].as_str().unwrap_or_default().to_owned();
155+ let repo = owner["repoId"].as_str().unwrap_or_default().to_owned();
156+ let kv = store(env)?;
157+ match (method, what) {
158+ ("GET", "artifacts") => {
159+ let listed: Vec<Value> = list(&kv, &format!("a/{run}/"))
160+ .await?
161+ .into_iter()
162+ .map(|(_, meta)| json!({ "name": meta.name, "size": meta.size }))
163+ .collect();
164+ Response::from_json(&listed)
165+ }
166+ (_, what) if what.starts_with("artifacts/") => {
167+ let name = decode(&what["artifacts/".len()..]);
168+ if !valid_name(&name) {
169+ return error(400, "That is not an artifact name.");
170+ }
171+ let base = format!("a/{run}/{name}");
172+ if method == "PUT" {
173+ let bytes = request.bytes().await?;
174+ if bytes.len() > MAX_BYTES {
175+ return error(413, "Artifacts are at most 60 MB.");
176+ }
177+ put(&kv, &base, &name, &bytes, ARTIFACT_TTL).await?;
178+ return Response::from_json(&json!({ "name": name, "size": bytes.len() }));
179+ }
180+ match get(&kv, &base).await? {
181+ Some(bytes) => Response::from_bytes(bytes),
182+ None => error(404, "No such artifact."),
183+ }
184+ }
185+ (_, "cache") => {
186+ let key = query(&request, "key").unwrap_or_default();
187+ if key.is_empty() || key.len() > 400 {
188+ return error(400, "A cache key is 1 to 400 characters.");
189+ }
190+ if method == "PUT" {
191+ let base = format!("c/{repo}/{key}");
192+ // A key is written once, as on GitHub.
193+ if kv.get(&base).text().await?.is_some() {
194+ return Response::from_json(&json!({ "saved": false, "reason": "That key is already cached." }));
195+ }
196+ let bytes = request.bytes().await?;
197+ if bytes.len() > MAX_BYTES {
198+ return error(413, "Cache entries are at most 60 MB.");
199+ }
200+ put(&kv, &base, &key, &bytes, CACHE_TTL).await?;
201+ return Response::from_json(&json!({ "saved": true }));
202+ }
203+ // The exact key, else the newest entry under each restore key.
204+ let exact = format!("c/{repo}/{key}");
205+ if let Some(bytes) = get(&kv, &exact).await? {
206+ let mut response = Response::from_bytes(bytes)?;
207+ response.headers_mut().set("x-g1t-key", &key)?;
208+ return Ok(response);
209+ }
210+ for prefix in query(&request, "restore").unwrap_or_default().lines().map(str::trim).filter(|p| !p.is_empty()) {
211+ if let Some((base, meta)) = list(&kv, &format!("c/{repo}/{prefix}")).await?.into_iter().next()
212+ && let Some(bytes) = get(&kv, &base).await?
213+ {
214+ let mut response = Response::from_bytes(bytes)?;
215+ response.headers_mut().set("x-g1t-key", &meta.name)?;
216+ return Ok(response);
217+ }
218+ }
219+ error(404, "Nothing cached under those keys.")
220+ }
221+ _ => error(404, "No such endpoint."),
222+ }
223+}
224+
225+/// Someone who can see the run downloading one of its artifacts.
226+pub async fn download(env: &Env, services: &Services, viewer: &g1t_contracts::Viewer, owner: &str, repo: &str, run: &str, name: &str) -> Result<Response> {
227+ let seen: Outcome<Value> = g1t_kit::call(
228+ &services.actions,
229+ "run",
230+ &json!({ "repo": { "namespace": owner, "name": repo }, "viewer": viewer, "id": run }),
231+ )
232+ .await?;
233+ if let Outcome::Fail(refused) = seen {
234+ let status = if refused.code == FailureCode::NotFound { 404 } else { 403 };
235+ return error(status, &refused.message);
236+ }
237+ let name = decode(name);
238+ match get(&store(env)?, &format!("a/{run}/{name}")).await? {
239+ Some(bytes) => {
240+ let mut response = Response::from_bytes(bytes)?;
241+ let headers = response.headers_mut();
242+ headers.set("content-type", "application/gzip")?;
243+ headers.set("content-disposition", &format!("attachment; filename=\"{}.tar.gz\"", name.replace('"', "")))?;
244+ Ok(response)
245+ }
246+ None => error(404, "No such artifact, or it has expired."),
247+ }
248+}
249+
250+/// A run's artifacts, for its page.
251+pub async fn of_run(env: &Env, run: &str) -> Result<Vec<Value>> {
252+ Ok(list(&store(env)?, &format!("a/{run}/"))
253+ .await?
254+ .into_iter()
255+ .map(|(_, meta)| json!({ "name": meta.name, "size": meta.size, "at": meta.at }))
256+ .collect())
257+}
+33−0
44 //! operations (see [`operations::Op`]), which call the services that own
55 //! the data. This Worker holds none.
66
7+mod blobs;
78 mod mcp;
89 mod oauth;
910 mod openapi;
346347 return receive_hook(&mut request, &services, id).await;
347348 }
348349
350+ // A sandbox's artifacts and cache, with its job's token, which is not a
351+ // g1t token either.
352+ if !on_mcp
353+ && let Some(rest) = path.strip_prefix("/actions/jobs/")
354+ && (rest.contains("/artifacts") || rest.ends_with("/cache"))
355+ {
356+ let rest = rest.to_owned();
357+ return blobs::for_job(request, env, &services, method, &rest).await;
358+ }
359+
349360 let viewer = match authenticate(&request, &services).await? {
350361 Ok(viewer) => viewer,
351362 Err(refused) => return Ok(refused),
378389 match (method, path.trim_end_matches('/')) {
379390 ("GET", "") => return Response::from_json(&index()),
380391 ("GET", "/openapi.json") => return Response::from_json(&openapi::document()),
392+ // A run's artifacts: listed, or one downloaded.
393+ ("GET", path) if path.starts_with("/repos/") && path.contains("/actions/runs/") && path.contains("/artifacts") => {
394+ let parts: Vec<&str> = path.trim_start_matches("/repos/").split('/').collect();
395+ if let [owner, repo, "actions", "runs", run, "artifacts", rest @ ..] = parts.as_slice() {
396+ return match rest {
397+ [] => {
398+ let seen: Outcome<Value> = g1t_kit::call(
399+ &services.actions,
400+ "run",
401+ &json!({ "repo": { "namespace": owner, "name": repo }, "viewer": viewer, "id": run }),
402+ )
403+ .await?;
404+ match seen {
405+ Outcome::Ok(_) => Response::from_json(&blobs::of_run(env, run).await?),
406+ Outcome::Fail(refused) => failure(&refused),
407+ }
408+ }
409+ [name] => blobs::download(env, &services, &viewer, owner, repo, run, name).await,
410+ _ => fail(FailureCode::NotFound, "No such endpoint."),
411+ };
412+ }
413+ }
381414 ("POST", "/device/code") => return device_code(&mut request, &services).await,
382415 ("POST", "/device/token") => return device_token(&mut request, &services).await,
383416 // Where a pull request lives, for a tool that knows only its fork.
+3−0
2424 { "binding": "AUTOMATIONS", "service": "g1t-automations" },
2525 { "binding": "ACTIONS", "service": "g1t-actions" }
2626 ],
27+ // GitHub Actions artifacts and cache, in chunks, with KV's own expiry.
28+ // (Moves to R2 once R2 is enabled on the account.)
29+ "kv_namespaces": [{ "binding": "BLOBS", "id": "16a4232cb746418db53782aa068be693" }],
2730 "observability": { "enabled": true }
2831 }
+95−58
300300 );
301301 }
302302
303+type ActiveRepo = NonNullable<ShellData["repo"]>;
304+
305+/**
306+ * A repository's own menu, which the sidebar slides to while you are in
307+ * it, as it does for settings: everything about the repository and nothing
308+ * else, with the way back to everything.
309+ */
310+function RepoMenu({ repo, isPrivate, open }: { repo: ActiveRepo; isPrivate: boolean; open: boolean }) {
311+ const base = `/${repo.namespace}/${repo.name}`;
312+ return (
313+ <nav aria-label={`${repo.namespace}/${repo.name}`} inert={!open} className={PANEL}>
314+ <Link
315+ to="/"
316+ className="group mt-3 flex h-8 items-center gap-2 rounded-md px-2 text-[0.8125rem] text-muted transition-colors hover:bg-raised/60 hover:text-fg"
317+ >
318+ <ArrowLeft size={15} className="text-faint transition-transform group-hover:-translate-x-0.5 group-hover:text-muted" />
319+ Mission control
320+ </Link>
321+ <Link
322+ to={base}
323+ className="mt-3 flex items-center gap-2 rounded-md px-2 py-1.5 transition-colors hover:bg-raised/60"
324+ >
325+ <span className="flex size-6 shrink-0 items-center justify-center rounded-md bg-raised text-muted ring-1 ring-line">
326+ {isPrivate ? <Lock size={13} /> : <BookMarked size={13} />}
327+ </span>
328+ <span className="min-w-0 truncate font-mono text-[0.8125rem]">
329+ <span className="text-faint">{repo.namespace}/</span>
330+ <span className="font-semibold text-fg">{repo.name}</span>
331+ </span>
332+ </Link>
333+ <SidebarGroup title="Code">
334+ <SidebarLink to={base} end icon={<Code2 size={15} />} also={`${base}/tree`}>
335+ Code
336+ </SidebarLink>
337+ <SidebarLink to={`${base}/commits`} also={`${base}/commit`} icon={<History size={15} />}>
338+ Commits
339+ </SidebarLink>
340+ </SidebarGroup>
341+ <SidebarGroup title="Work">
342+ <SidebarLink to={`${base}/issues`} icon={<CircleDot size={15} />} count={repo.issues}>
343+ Issues
344+ </SidebarLink>
345+ <SidebarLink to={`${base}/pulls`} also={`${base}/pull`} icon={<GitPullRequest size={15} />} count={repo.pulls}>
346+ Pull requests
347+ </SidebarLink>
348+ {repo.member && (
349+ <SidebarLink to={`${base}/plans`} icon={<ListTree size={15} />}>
350+ Plan
351+ </SidebarLink>
352+ )}
353+ <SidebarLink to={`${base}/queue`} icon={<Layers size={15} />}>
354+ Merge queue
355+ </SidebarLink>
356+ </SidebarGroup>
357+ <SidebarGroup title="Automate">
358+ <SidebarLink to={`${base}/actions`} icon={<PlayCircle size={15} />}>
359+ Actions
360+ </SidebarLink>
361+ <SidebarLink to={`${base}/automations`} icon={<Zap size={15} />}>
362+ Automations
363+ </SidebarLink>
364+ </SidebarGroup>
365+ {repo.member && (
366+ <SidebarGroup title="Repository">
367+ <SidebarLink to={`${base}/settings`} icon={<Settings size={15} />}>
368+ Settings
369+ </SidebarLink>
370+ </SidebarGroup>
371+ )}
372+ </nav>
373+ );
374+}
375+
303376 /** Your own settings, as the sidebar shows them on the settings page. */
304377 function AccountSettingsMenu({ open }: { open: boolean }) {
305378 const { hash } = useLocation();
344417 // the sidebar over.
345418 const inSettings = ws != null && SETTINGS_PAGE.exec(going ?? pathname)?.[1]?.toLowerCase() === ws.slug;
346419 const inAccount = (going ?? pathname) === "/settings";
347− const away = inSettings || inAccount;
348− // Which settings sit on the far side of the track. Kept while sliding back,
349− // so they do not vanish on the way out.
350− const side = useRef<"workspace" | "account">("workspace");
420+ const active = shell.repo;
421+ // In a repository, or on the way into one, its own menu takes the sidebar.
422+ const target = going ?? pathname;
423+ const repoPath = /^\/([^/]+)\/([^/-][^/]*)(\/|$)/.exec(target);
424+ const reserved = new Set(["settings", "explore", "search", "new", "workspaces", "login", "logout", "register", "verify", "forgot", "reset", "device", "oauth"]);
425+ const inRepo = repoPath != null && !reserved.has(repoPath[1]) && repoPath[2] !== "-";
426+ const away = inSettings || inAccount || inRepo;
427+ // What sits on the far side of the track. Kept while sliding back, so it
428+ // does not vanish on the way out.
429+ const side = useRef<"workspace" | "account" | "repo">("workspace");
351430 if (inAccount) side.current = "account";
352431 else if (inSettings) side.current = "workspace";
353− const active = shell.repo;
354− const repoBase = active ? `/${active.namespace}/${active.name}` : null;
432+ else if (inRepo) side.current = "repo";
433+ // The repository last shown, kept for the slide back.
434+ const shown = useRef(active);
435+ if (active) shown.current = active;
355436 // The repository being looked at is listed even when it is someone else's.
356437 const listed =
357438 active && !shell.repos.some((repo) => repo.namespace === active.namespace && repo.name === active.name)
410491 <p className="px-2 py-1 text-xs text-faint">None yet.</p>
411492 )}
412493 {listed.map((repo) => {
413− const open =
414− active && repo.namespace === active.namespace && repo.name === active.name;
415494 const base = `/${repo.namespace}/${repo.name}`;
416495 return (
417496 <div key={base}>
418497 <SidebarLink
419498 to={base}
420− end={!open}
421499 icon={repo.isPrivate ? <Lock size={15} /> : <BookMarked size={15} />}
422500 >
423501 <span className="font-mono text-[0.8125rem]">
427505 {repo.name}
428506 </span>
429507 </SidebarLink>
430− {open && repoBase && (
431− <div className="my-0.5 ml-4 space-y-px border-l border-line pl-2">
432− <SidebarLink to={repoBase} end icon={<Code2 size={14} />} also={`${repoBase}/tree`}>
433− Code
434− </SidebarLink>
435− <SidebarLink
436− to={`${repoBase}/issues`}
437− icon={<CircleDot size={14} />}
438− count={active.issues}
439− >
440− Issues
441− </SidebarLink>
442− <SidebarLink
443− to={`${repoBase}/pulls`}
444− also={`${repoBase}/pull`}
445− icon={<GitPullRequest size={14} />}
446− count={active.pulls}
447− >
448− Pull requests
449− </SidebarLink>
450− <SidebarLink to={`${repoBase}/actions`} icon={<PlayCircle size={14} />}>
451− Actions
452− </SidebarLink>
453− <SidebarLink to={`${repoBase}/queue`} icon={<Layers size={14} />}>
454− Merge queue
455− </SidebarLink>
456− <SidebarLink
457− to={`${repoBase}/commits`}
458− also={`${repoBase}/commit`}
459− icon={<History size={14} />}
460− >
461− Commits
462− </SidebarLink>
463− {active.member && (
464− <SidebarLink to={`${repoBase}/plans`} icon={<ListTree size={14} />}>
465− Plan
466− </SidebarLink>
467− )}
468− <SidebarLink to={`${repoBase}/automations`} icon={<Zap size={14} />}>
469− Automations
470− </SidebarLink>
471− {active.member && (
472− <SidebarLink to={`${repoBase}/settings`} icon={<Settings size={14} />}>
473− Settings
474− </SidebarLink>
475− )}
476− </div>
477− )}
478508 </div>
479509 );
480510 })}
481511 </SidebarGroup>
482512 </nav>
483− {side.current === "account" || !ws ? (
513+ {side.current === "repo" && shown.current ? (
514+ <RepoMenu
515+ repo={shown.current}
516+ isPrivate={shell.repos.some((repo) => repo.namespace === shown.current!.namespace && repo.name === shown.current!.name && repo.isPrivate)}
517+ open={inRepo}
518+ />
519+ ) : side.current === "account" || !ws ? (
484520 <AccountSettingsMenu open={inAccount} />
485521 ) : (
486522 <SettingsMenu slug={ws.slug} owner={ws.role === "owner"} open={inSettings} />
492528 <div className="space-y-2 p-2">
493529 {ws && (
494530 <div className="space-y-px">
531+ <p className="px-2 pb-1 text-[0.6875rem] font-medium uppercase tracking-wider text-faint">{ws.slug}</p>
495532 <SidebarLink to={`/${ws.slug}/-/usage`} icon={<BarChart3 size={15} />}>
496533 Usage
497534 </SidebarLink>
+7−0
7979 pub fn wants_type(&self, action: Option<&str>) -> bool {
8080 let Some(action) = action else { return true };
8181 if self.types.is_empty() {
82+ // A g1t agent's pull request has no code until it is marked
83+ // ready, so that is when its default runs start, as `opened`
84+ // would on GitHub.
85+ if action == "ready_for_review" && self.event.starts_with("pull_request") && self.event != "pull_request_review" {
86+ return true;
87+ }
8288 let defaults = default_types(&self.event);
8389 return defaults.is_empty() || defaults.contains(&action);
8490 }
539545 assert!(pr.wants_type(Some("opened")));
540546 assert!(pr.wants_type(Some("synchronize")));
541547 assert!(!pr.wants_type(Some("closed")));
548+ assert!(pr.wants_type(Some("ready_for_review")));
542549 let typed = parse("on:\n pull_request:\n types: [closed]\njobs:\n a:\n runs-on: ubuntu-latest\n steps: [{ run: 'true' }]").unwrap();
543550 assert!(typed.trigger("pull_request").unwrap().wants_type(Some("closed")));
544551 assert!(!typed.trigger("pull_request").unwrap().wants_type(Some("opened")));
+295−0
1+//! Artifacts and the cache: `actions/upload-artifact`,
2+//! `actions/download-artifact` and `actions/cache`, done natively against
3+//! g1t, which keeps them in R2. Artifacts belong to the run; cache entries
4+//! to the repository, found by exact key or by the newest under a
5+//! `restore-keys` prefix.
6+
7+use std::collections::BTreeMap;
8+use std::io::Read;
9+use std::path::Path;
10+use std::process::Command;
11+use std::time::Duration;
12+
13+use super::process::{self, Commands, Ended};
14+use super::{Job, Post, PostRun};
15+
16+/// The largest upload g1t takes, as the platform limits a request.
17+const MAX_UPLOAD: u64 = 60 * 1024 * 1024;
18+
19+fn lines(text: &str) -> Vec<String> {
20+ text.lines().map(str::trim).filter(|line| !line.is_empty() && !line.starts_with('#')).map(str::to_owned).collect()
21+}
22+
23+fn quote(text: &str) -> String {
24+ format!("'{}'", text.replace('\'', "'\\''"))
25+}
26+
27+fn safe_name(name: &str) -> bool {
28+ !name.is_empty() && name.len() <= 200 && name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ' ')) && !name.starts_with('.')
29+}
30+
31+impl Job {
32+ fn url(&self, rest: &str) -> String {
33+ format!("{}/actions/jobs/{}/{rest}", self.log.api.base, self.log.api.job)
34+ }
35+
36+ fn auth(&self) -> String {
37+ format!("Bearer {}", self.log.api.token)
38+ }
39+
40+ /// Runs a shell line, logging its output; whether it succeeded.
41+ fn shell(&mut self, script: &str) -> bool {
42+ let mut command = Command::new("bash");
43+ command.args(["-c", script]).current_dir(&self.workspace);
44+ let mut commands = Commands::default();
45+ matches!(process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
46+ }
47+
48+ fn upload(&mut self, rest: &str, file: &Path) -> Result<u64, String> {
49+ let size = std::fs::metadata(file).map_err(|e| e.to_string())?.len();
50+ if size > MAX_UPLOAD {
51+ return Err(format!("it is {} MB, more than g1t takes at once ({} MB)", size / 1_048_576, MAX_UPLOAD / 1_048_576));
52+ }
53+ let bytes = std::fs::read(file).map_err(|e| e.to_string())?;
54+ ureq::put(&self.url(rest))
55+ .set("authorization", &self.auth())
56+ .set("content-type", "application/gzip")
57+ .timeout(Duration::from_secs(600))
58+ .send_bytes(&bytes)
59+ .map_err(|e| e.to_string())?;
60+ Ok(size)
61+ }
62+
63+ /// Downloads into `file`; `Ok(None)` when there is nothing there.
64+ fn download(&mut self, rest: &str, file: &Path) -> Result<Option<String>, String> {
65+ let response = match ureq::get(&self.url(rest)).set("authorization", &self.auth()).timeout(Duration::from_secs(600)).call() {
66+ Ok(response) => response,
67+ Err(ureq::Error::Status(404, _)) => return Ok(None),
68+ Err(error) => return Err(error.to_string()),
69+ };
70+ let matched = response.header("x-g1t-key").map(str::to_owned).unwrap_or_default();
71+ let mut bytes = Vec::new();
72+ response.into_reader().take(MAX_UPLOAD * 2).read_to_end(&mut bytes).map_err(|e| e.to_string())?;
73+ std::fs::write(file, bytes).map_err(|e| e.to_string())?;
74+ Ok(Some(matched))
75+ }
76+
77+ /// `actions/upload-artifact`.
78+ pub(crate) fn upload_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
79+ let name = with.get("name").filter(|n| !n.is_empty()).cloned().unwrap_or_else(|| "artifact".into());
80+ if !safe_name(&name) {
81+ self.log.line(&format!("##[error]`{name}` is not an artifact name g1t takes: letters, digits, spaces, `-`, `_` and `.`."));
82+ return (false, BTreeMap::new());
83+ }
84+ let paths = lines(with.get("path").map(String::as_str).unwrap_or_default());
85+ let missing = with.get("if-no-files-found").map(String::as_str).unwrap_or("warn").to_owned();
86+ let archive = self.temp.join(format!("artifact-{name}.tgz"));
87+ // As on GitHub: one folder uploads its contents; otherwise paths
88+ // are kept relative to the workspace.
89+ let single_dir = paths.len() == 1 && self.workspace.join(&paths[0]).is_dir();
90+ let script = if single_dir {
91+ format!("tar -czf {} -C {} .", quote(&archive.display().to_string()), quote(&paths[0]))
92+ } else {
93+ let patterns: Vec<String> = paths.iter().filter(|p| !p.starts_with('!')).map(|p| p.replace('\'', "")).collect();
94+ format!(
95+ "shopt -s globstar nullglob dotglob; files=( {} ); if [ ${{#files[@]}} -eq 0 ]; then exit 3; fi; tar -czf {} -- \"${{files[@]}}\"",
96+ patterns.join(" "),
97+ quote(&archive.display().to_string())
98+ )
99+ };
100+ let mut command = Command::new("bash");
101+ command.args(["-c", &script]).current_dir(&self.workspace);
102+ let mut commands = Commands::default();
103+ match process::run(command, Duration::from_secs(1800), &mut self.log, &mut commands) {
104+ Ok(Ended::Exited(0)) => {}
105+ Ok(Ended::Exited(3)) => {
106+ let message = format!("No files were found at {}.", paths.join(", "));
107+ return match missing.as_str() {
108+ "error" => {
109+ self.log.line(&format!("##[error]{message}"));
110+ (false, BTreeMap::new())
111+ }
112+ "ignore" => (true, BTreeMap::new()),
113+ _ => {
114+ self.log.line(&format!("##[warning]{message} Nothing was uploaded."));
115+ (true, BTreeMap::new())
116+ }
117+ };
118+ }
119+ _ => {
120+ self.log.line("##[error]The files could not be packed.");
121+ return (false, BTreeMap::new());
122+ }
123+ }
124+ match self.upload(&format!("artifacts/{name}"), &archive) {
125+ Ok(size) => {
126+ self.log.line(&format!("Uploaded artifact {name} ({} KB). It is kept with the run for 14 days.", size.div_ceil(1024)));
127+ let mut outputs = BTreeMap::new();
128+ outputs.insert("artifact-id".into(), name.clone());
129+ (true, outputs)
130+ }
131+ Err(error) => {
132+ self.log.line(&format!("##[error]The artifact could not be uploaded: {error}"));
133+ (false, BTreeMap::new())
134+ }
135+ }
136+ }
137+
138+ /// `actions/download-artifact`: one by name, or every artifact of the
139+ /// run, each into a folder of its name.
140+ pub(crate) fn download_artifact(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
141+ let dest = with.get("path").filter(|p| !p.is_empty()).map_or(self.workspace.clone(), |p| self.workspace.join(p));
142+ let names: Vec<String> = match with.get("name").filter(|n| !n.is_empty()) {
143+ Some(name) => vec![name.clone()],
144+ None => {
145+ let listed = ureq::get(&self.url("artifacts")).set("authorization", &self.auth()).call().ok().and_then(|r| r.into_json::<Vec<serde_json::Value>>().ok()).unwrap_or_default();
146+ listed.iter().filter_map(|a| a["name"].as_str().map(str::to_owned)).collect()
147+ }
148+ };
149+ let merge = with.get("merge-multiple").is_some_and(|m| m == "true");
150+ let single = with.get("name").is_some_and(|n| !n.is_empty());
151+ for name in &names {
152+ let archive = self.temp.join(format!("download-{name}.tgz"));
153+ match self.download(&format!("artifacts/{name}"), &archive) {
154+ Ok(Some(_)) => {}
155+ Ok(None) => {
156+ self.log.line(&format!("##[error]This run has no artifact called {name}."));
157+ return (false, BTreeMap::new());
158+ }
159+ Err(error) => {
160+ self.log.line(&format!("##[error]The artifact {name} could not be downloaded: {error}"));
161+ return (false, BTreeMap::new());
162+ }
163+ }
164+ let target = if single || merge { dest.clone() } else { dest.join(name) };
165+ let _ = std::fs::create_dir_all(&target);
166+ if !self.shell(&format!("tar -xzf {} -C {}", quote(&archive.display().to_string()), quote(&target.display().to_string()))) {
167+ return (false, BTreeMap::new());
168+ }
169+ self.log.line(&format!("Downloaded artifact {name} into {}", target.display()));
170+ }
171+ let mut outputs = BTreeMap::new();
172+ outputs.insert("download-path".into(), dest.display().to_string());
173+ (true, outputs)
174+ }
175+
176+ fn cache_paths(&self, with: &BTreeMap<String, String>) -> Vec<String> {
177+ let home = self.base_env_value("HOME").unwrap_or_else(|| "/home/node".into());
178+ lines(with.get("path").map(String::as_str).unwrap_or_default())
179+ .into_iter()
180+ .map(|p| {
181+ let p = if let Some(rest) = p.strip_prefix("~/") { format!("{home}/{rest}") } else { p };
182+ if p.starts_with('/') { p } else { self.workspace.join(p).display().to_string() }
183+ })
184+ .collect()
185+ }
186+
187+ /// `actions/cache` and `actions/cache/restore`: restores what it can,
188+ /// and for `actions/cache`, saves at the end of the job on a miss.
189+ pub(crate) fn cache(&mut self, with: &BTreeMap<String, String>, save_after: bool, title: &str) -> (bool, BTreeMap<String, String>) {
190+ let key = with.get("key").cloned().unwrap_or_default();
191+ if key.is_empty() {
192+ self.log.line("##[error]The cache needs a `key`.");
193+ return (false, BTreeMap::new());
194+ }
195+ let paths = self.cache_paths(with);
196+ let restore = lines(with.get("restore-keys").map(String::as_str).unwrap_or_default());
197+ let query = format!(
198+ "cache?key={}&restore={}",
199+ urlencode(&key),
200+ urlencode(&restore.join("\n"))
201+ );
202+ let archive = self.temp.join("cache-restore.tgz");
203+ let mut outputs = BTreeMap::new();
204+ let exact = match self.download(&query, &archive) {
205+ Ok(Some(matched)) => {
206+ let lookup_only = with.get("lookup-only").is_some_and(|v| v == "true");
207+ if !lookup_only && !self.shell(&format!("tar -xzPf {}", quote(&archive.display().to_string()))) {
208+ self.log.line("##[warning]The cache was found but could not be unpacked.");
209+ }
210+ self.log.line(&format!("Cache restored from key: {matched}"));
211+ outputs.insert("cache-matched-key".into(), matched.clone());
212+ matched == key
213+ }
214+ Ok(None) => {
215+ self.log.line(&format!("Cache not found for input keys: {}", std::iter::once(key.clone()).chain(restore).collect::<Vec<_>>().join(", ")));
216+ if with.get("fail-on-cache-miss").is_some_and(|v| v == "true") {
217+ self.log.line("##[error]The cache missed, and `fail-on-cache-miss` is set.");
218+ return (false, outputs);
219+ }
220+ false
221+ }
222+ Err(error) => {
223+ self.log.line(&format!("##[warning]The cache could not be read: {error}"));
224+ false
225+ }
226+ };
227+ outputs.insert("cache-hit".into(), exact.to_string());
228+ outputs.insert("cache-primary-key".into(), key.clone());
229+ if save_after && !exact {
230+ self.posts.push(Post {
231+ name: format!("Post {title}"),
232+ condition: "success()".into(),
233+ env: BTreeMap::new(),
234+ run: PostRun::CacheSave { key, paths },
235+ });
236+ }
237+ (true, outputs)
238+ }
239+
240+ /// Saves paths under a key, unless the key is taken.
241+ pub(crate) fn cache_save(&mut self, key: &str, paths: &[String]) -> bool {
242+ if paths.is_empty() {
243+ self.log.line("##[warning]Nothing to cache: no `path`.");
244+ return true;
245+ }
246+ let archive = self.temp.join("cache-save.tgz");
247+ let list: Vec<String> = paths.iter().filter(|p| Path::new(p).exists()).map(|p| quote(p)).collect();
248+ if list.is_empty() {
249+ self.log.line("##[warning]None of the cache's paths exist; nothing was saved.");
250+ return true;
251+ }
252+ if !self.shell(&format!("tar -czPf {} {}", quote(&archive.display().to_string()), list.join(" "))) {
253+ self.log.line("##[warning]The cache could not be packed; nothing was saved.");
254+ return true;
255+ }
256+ match self.upload(&format!("cache?key={}", urlencode(key)), &archive) {
257+ Ok(size) => self.log.line(&format!("Cache saved with key: {key} ({} KB)", size.div_ceil(1024))),
258+ // A cache that cannot be saved does not fail the job, as on GitHub.
259+ Err(error) => self.log.line(&format!("##[warning]The cache could not be saved: {error}")),
260+ }
261+ true
262+ }
263+
264+ /// `actions/cache/save`.
265+ pub(crate) fn cache_save_now(&mut self, with: &BTreeMap<String, String>) -> (bool, BTreeMap<String, String>) {
266+ let key = with.get("key").cloned().unwrap_or_default();
267+ let paths = self.cache_paths(with);
268+ (self.cache_save(&key, &paths), BTreeMap::new())
269+ }
270+}
271+
272+fn urlencode(text: &str) -> String {
273+ let mut out = String::new();
274+ for byte in text.bytes() {
275+ if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
276+ out.push(byte as char);
277+ } else {
278+ out.push_str(&format!("%{byte:02X}"));
279+ }
280+ }
281+ out
282+}
283+
284+#[cfg(test)]
285+mod tests {
286+ use super::*;
287+
288+ #[test]
289+ fn keys_are_encoded_and_names_checked() {
290+ assert_eq!(urlencode("Linux-node-abc/1 2"), "Linux-node-abc%2F1%202");
291+ assert!(safe_name("coverage report"));
292+ assert!(!safe_name("../etc"));
293+ assert_eq!(lines("dist/\n\n# note\n coverage \n"), ["dist/", "coverage"]);
294+ }
295+}
+17−4
88 //! and `ACTIONS_TOKEN`, the job and its own token. Everything else, the
99 //! job's definition, its contexts and its secrets, is fetched with them.
1010
11+mod blobs;
1112 mod files;
1213 mod process;
1314 mod report;
4243 pub(crate) env: BTreeMap<String, String>,
4344 }
4445
45−/// An action's `post` step, run when the job's steps are done.
46+/// A step run when the job's steps are done: an action's `post`, or
47+/// saving the cache.
4648 pub(crate) struct Post {
4749 pub(crate) name: String,
48− pub(crate) action_dir: PathBuf,
49− pub(crate) script: String,
5050 pub(crate) condition: String,
5151 pub(crate) env: BTreeMap<String, String>,
52+ pub(crate) run: PostRun,
5253 }
5354
55+pub(crate) enum PostRun {
56+ Node { action_dir: PathBuf, script: String },
57+ CacheSave { key: String, paths: Vec<String> },
58+}
59+
5460 pub(crate) struct Job {
5561 pub(crate) log: Log,
5662 pub(crate) spec: Value,
94100 }
95101
96102 impl Job {
103+ pub(crate) fn base_env_value(&self, name: &str) -> Option<String> {
104+ self.base_env.get(name).cloned()
105+ }
106+
97107 fn status(&self) -> Status {
98108 if self.failed { Status::Failure } else { Status::Success }
99109 }
534544 }
535545 job.log.step(number);
536546 job.log.step_state(number, &post.name, "in_progress", None);
537− let ok = job.run_node(&post.action_dir, &post.script, &post.env);
547+ let ok = match &post.run {
548+ PostRun::Node { action_dir, script } => job.run_node(action_dir, script, &post.env),
549+ PostRun::CacheSave { key, paths } => job.cache_save(key, paths),
550+ };
538551 job.log.step_state(number, &post.name, "completed", Some(if ok { "success" } else { "failure" }));
539552 if !ok {
540553 job.failed = true;
+7−11
1515
1616 use super::files::StepFiles;
1717 use super::process::{self, Commands, Ended};
18−use super::{Frame, Job, Post};
18+use super::{Frame, Job, Post, PostRun};
1919
2020 const ACTIONS_DIR: &str = "/home/runner/_actions";
2121
214214 let lower = name.to_ascii_lowercase();
215215 match lower.as_str() {
216216 "actions/checkout" => return self.checkout(with),
217− "actions/upload-artifact" => {
218− self.log.line("##[warning]Artifacts are not kept on g1t yet: nothing was uploaded, and the job goes on.");
219− return (true, BTreeMap::new());
220− }
221− "actions/download-artifact" => {
222− self.log.line("##[error]Artifacts are not kept on g1t yet, so there is nothing to download.");
223− return (false, BTreeMap::new());
224− }
217+ "actions/upload-artifact" => return self.upload_artifact(with),
218+ "actions/download-artifact" => return self.download_artifact(with),
219+ "actions/cache" => return self.cache(with, true, title),
220+ "actions/cache/restore" => return self.cache(with, false, title),
221+ "actions/cache/save" => return self.cache_save_now(with),
225222 _ => {}
226223 }
227224 let source = if let Some(local) = name.strip_prefix("./") {
330327 }
331328 self.posts.push(Post {
332329 name: format!("Post {title}"),
333− action_dir: dir.clone(),
334− script: post,
335330 condition: condition_of("post-if"),
336331 env: post_env,
332+ run: PostRun::Node { action_dir: dir.clone(), script: post },
337333 });
338334 }
339335 return (ok, outputs);
+1−0
176176 "set_setting" => reply(&service.set_setting(args(body)?).await?),
177177 "delete_setting" => reply(&service.delete_setting(args(body)?).await?),
178178 "job_spec" => reply(&service.job_spec(args(body)?).await?),
179+ "job_auth" => reply(&service.job_auth(args(body)?).await?),
179180 "job_report" => reply(&service.job_report(args(body)?).await?),
180181 _ => Response::error("Unknown method", 404),
181182 }
+7−0
945945 })
946946 }
947947
948+ /// `job_auth`: which run and repository a running job's token is for,
949+ /// so the API can keep its artifacts and cache.
950+ pub async fn job_auth(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
951+ let job = check!(self.job_for_token(&a).await?);
952+ Ok(Outcome::Ok(json!({ "run": job.run_id, "repoId": job.repo_id })))
953+ }
954+
948955 /// `job_spec`: everything the sandbox needs to run the job.
949956 pub async fn job_spec(&self, a: JobCallArgs) -> Result<Outcome<Value>> {
950957 let job = check!(self.job_for_token(&a).await?);
+26−10
206206 subject.paths = Some(pull.files.iter().map(|f| f.path.clone()).collect());
207207 return Ok(Some(subject));
208208 }
209− // A merged pull request's run is on the commit it landed as.
210− let sha = match (action, data["commit"].as_str()) {
211− (Some("closed"), Some(commit)) => commit.to_owned(),
212− _ => match &pull.head_commit {
213− Some(head) => head.clone(),
214− None => return Ok(None),
215− },
209+ // A merged pull request's run is on the commit it landed as,
210+ // in the repository; otherwise on its head, where that is.
211+ let landed = match (action, data["commit"].as_str()) {
212+ (Some("closed"), Some(commit)) => Some(commit.to_owned()),
213+ _ => None,
214+ };
215+ let sha = match (&landed, data["commit"].as_str(), &pull.head_commit) {
216+ (Some(commit), _, _) => commit.clone(),
217+ (None, Some(commit), _) => commit.to_owned(),
218+ (None, None, Some(head)) => head.clone(),
219+ (None, None, None) => return Ok(None),
220+ };
221+ let source = match landed {
222+ Some(_) => path.clone(),
223+ None => pull.fork.clone().unwrap_or_else(|| path.clone()),
216224 };
217− let source = pull.fork.clone().unwrap_or_else(|| path.clone());
218225 Some(Subject {
219− source: if data["commit"].is_string() { path.clone() } else { source },
226+ source,
220227 source_ref: Some(sha.clone()),
221228 git_ref: format!("refs/pull/{}/merge", pull.number),
222229 sha,
281288 continue;
282289 };
283290 let read = self.read_workflows(&subject.source, &ws, subject.source_ref.as_deref()).await?;
284− self.start_matching(&repo, &ws, read, &mut subject, event_name, action, &event.id, event.actor.as_deref(), &sender)
291+ // A pull request's head runs each workflow once, however many
292+ // events say it is there (marked ready, and pushed).
293+ let key = match subject.pull {
294+ Some(number) if event_name.starts_with("pull_request") && event_name != "pull_request_review" => {
295+ let phase = if action == Some("closed") { "closed" } else { "open" };
296+ format!("{event_name}:{number}:{}:{phase}", subject.sha)
297+ }
298+ _ => event.id.clone(),
299+ };
300+ self.start_matching(&repo, &ws, read, &mut subject, event_name, action, &key, event.actor.as_deref(), &sender)
285301 .await?;
286302 }
287303 Ok(())