Skip to content
Sign up

flagon-io/g1t

public
0

Commit

Merge remote-tracking branch 'origin/claude/peaceful-franklin-7qtkwx'

syntaqxcommitted Parents618de27a4c727bBrowse files
12 files+638−160/12 viewed
+1−0
924924 version = "0.1.0"
925925 dependencies = [
926926 "base64 0.22.1",
927+ "futures-util",
927928 "g1t-actions",
928929 "g1t-contracts",
929930 "g1t-kit",
+27−0
728728 }
729729 // A sandbox running a GitHub Actions job: fetching the job, and
730730 // reporting how it goes. The job's own token is the credential.
731+ // A sandbox downloading an action's archive that g1t keeps
732+ // (actions' registry.rs). The job's own token is the credential.
733+ ("POST", path) if path.starts_with("/actions/jobs/") && path.ends_with("/archive") => {
734+ let job = path.trim_start_matches("/actions/jobs/").trim_end_matches("/archive").to_owned();
735+ let body = json_body(&mut request).await;
736+ let kept: Outcome<Value> = g1t_kit::call(
737+ &services.actions,
738+ "job_action_archive",
739+ &json!({ "job": job, "token": body["token"], "report": body["report"] }),
740+ )
741+ .await?;
742+ let kept = match kept {
743+ Outcome::Ok(kept) => kept,
744+ Outcome::Fail(refused) => return failure(&refused),
745+ };
746+ let bucket = env.bucket("ACTIONS_CACHE")?;
747+ let object = kept["object"].as_str().unwrap_or_default();
748+ let Some(stored) = bucket.get(object).execute().await? else {
749+ return fail(FailureCode::NotFound, "The archive is gone.");
750+ };
751+ let Some(body) = stored.body() else {
752+ return fail(FailureCode::NotFound, "The archive is gone.");
753+ };
754+ let mut response = Response::from_body(body.response_body()?)?;
755+ response.headers_mut().set("content-type", "application/gzip")?;
756+ return Ok(response);
757+ }
731758 ("POST", path) if path.starts_with("/actions/jobs/") => {
732759 let rest = path.trim_start_matches("/actions/jobs/");
733760 // `/action`: where to fetch another repository's action from (on
+26−1
133133 | On g1t and public | Your workflows use it, from any workspace. |
134134 | On g1t, private, in your workspace, with **Access** set to *Accessible from repositories in* the workspace | Your private repositories' workflows use it. A job reads it with a read-only token for that repository alone, which ends with the job. |
135135 | On g1t, private, and not shared that way | The step or job fails, and says why. A private repository's actions are never used by a public repository's workflows, whose logs anyone can read, nor from another workspace. |
136−| Not on g1t, or private in a workspace you cannot see | An action is fetched from GitHub, as before; a reusable workflow is read from a public repository on GitHub. |
136+| Not on g1t, or private in a workspace you cannot see | An action comes from GitHub, at the commit its ref names there (see [actions from GitHub](#actions-from-github)); a reusable workflow is read from a public repository on GitHub. |
137137
138138 `ref` is a branch, a tag or a commit. A reusable workflow may be under
139139 `.g1t/workflows/` or `.github/workflows/`; a `.github/workflows/` path
147147 `PUT /repos/{owner}/{repo}/actions/permissions/access` with
148148 `{"access_level": "organization"}` (`none` to stop).
149149
150+### Actions from GitHub
151+
152+g1t keeps the actions your workflows use from GitHub, so a job downloads
153+them from g1t rather than from GitHub:
154+
155+1. Before the first step, the job downloads every action its steps name,
156+ as GitHub's runner does in **Set up job**, and fails there if one can't
157+ be had. Its log names each one's commit:
158+ `Download action repository 'actions/setup-node@v4' (SHA:…) from g1t`.
159+2. g1t reads which commit the ref (`v4`, a branch, or a commit) names on
160+ GitHub, a tag before a branch of the same name, and looks again at
161+ most every 5 minutes, so a tag that moves is followed within minutes.
162+3. The first job to use an action at a commit has g1t fetch that commit's
163+ archive from GitHub and keep it. Every later job, in any workspace,
164+ downloads the same bytes from g1t.
165+
166+While GitHub doesn't answer, a ref reads as the commit it last named, and
167+the job says so in a warning; an action g1t keeps still downloads. An
168+action whose archive is larger than 32 MB, or from a private repository
169+on GitHub, is downloaded from GitHub by the job itself. An archive no job
170+has used for 30 days is deleted, and fetched again when next used.
171+Kept archives are g1t's and count toward no workspace's storage.
172+
173+Actions that a composite action uses are downloaded when it runs.
174+
150175 ### Secrets for a called workflow
151176
152177 A called workflow gets only the secrets its caller passes, plus
+3−1
750750 job.docker_hosted = false;
751751 }
752752 }
753− let containers_started = job.start_containers();
753+ // Actions first, then containers, as GitHub's "Set up job" and
754+ // "Initialize containers".
755+ let containers_started = job.prepare_actions(&steps) && job.start_containers();
754756 job.log.flush();
755757
756758 let mut frame = Frame::default();
+22−2
8585 }
8686
8787 /// Where to fetch another repository's action from: `{"source": "g1t",
88− /// "url", "ref", "token"}` or `{"source": "github"}`. Err holds why g1t
88+ /// "url", "ref", "token"}`, or `{"source": "github", "sha", "archive",
89+ /// "note"}`: the commit GitHub's ref names, and whether g1t keeps its
90+ /// archive (`action_archive`). Err holds why g1t
8991 /// refused (`true`: the repository is private and may not be used
9092 /// here), or that it could not be asked (`false`).
9193 pub(crate) fn action(&self, repository: &str, git_ref: &str) -> std::result::Result<Value, (bool, String)> {
94+ // The first job to use an action at a commit waits while g1t
95+ // fetches and keeps its archive.
9296 let sent = ureq::post(&format!("{}/actions/jobs/{}/action", self.base, self.job))
93− .timeout(Duration::from_secs(30))
97+ .timeout(Duration::from_secs(120))
9498 .send_json(json!({ "token": self.token, "report": { "repository": repository, "ref": git_ref } }));
9599 match sent {
96100 Ok(response) => response.into_json().map_err(|error| (false, error.to_string())),
103107 }
104108 }
105109
110+ /// Downloads the archive g1t keeps of `repository` at `sha` to `to`.
111+ pub(crate) fn action_archive(&self, repository: &str, sha: &str, to: &std::path::Path) -> std::result::Result<u64, String> {
112+ let response = ureq::post(&format!("{}/actions/jobs/{}/archive", self.base, self.job))
113+ .timeout(Duration::from_secs(300))
114+ .send_json(json!({ "token": self.token, "report": { "repository": repository, "sha": sha } }))
115+ .map_err(|error| match error {
116+ ureq::Error::Status(code, response) => {
117+ let body: Value = response.into_json().unwrap_or(Value::Null);
118+ body["error"]["message"].as_str().map_or_else(|| format!("g1t answered {code}"), str::to_owned)
119+ }
120+ other => other.to_string(),
121+ })?;
122+ let mut file = std::fs::File::create(to).map_err(|error| error.to_string())?;
123+ std::io::copy(&mut response.into_reader(), &mut file).map_err(|error| error.to_string())
124+ }
125+
106126 /// An API with no address, which sends nothing: what the tests use, so
107127 /// a step's run is not held up by reports that go nowhere (each one
108128 /// tried three times, with waits between, over a second and a half).
+147−10
11 //! `uses:` steps: `actions/checkout` done natively against g1t, actions
2−//! fetched from another repository on g1t (or else GitHub) and run as they are (JavaScript, composite and
3−//! Docker), `docker://` images, and a few of GitHub's own whose services
4−//! g1t does not have yet.
2+//! fetched from another repository on g1t (or else GitHub, at the commit
3+//! g1t resolved its ref to, from the archive g1t keeps of it when it has
4+//! one) and run as they are (JavaScript, composite and Docker),
5+//! `docker://` images, and a few of GitHub's own whose services g1t does
6+//! not have yet. Every action a job's steps name is downloaded before the
7+//! first step runs, as GitHub's runner does.
58
69 use std::collections::BTreeMap;
710 use std::path::{Path, PathBuf};
3336 !part.is_empty() && part.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/')) && !part.contains("..")
3437 }
3538
39+fn is_sha(text: &str) -> bool {
40+ text.len() == 40 && text.chars().all(|c| c.is_ascii_hexdigit())
41+}
42+
43+/// Whether the runner does `name` (lowercased, without its ref) itself
44+/// rather than fetching it.
45+fn native(name: &str, docker_hosted: bool) -> bool {
46+ matches!(
47+ name,
48+ "actions/checkout"
49+ | "actions/upload-artifact"
50+ | "actions/upload-artifact/merge"
51+ | "actions/download-artifact"
52+ | "actions/cache"
53+ | "actions/cache/restore"
54+ | "actions/cache/save"
55+ ) || (docker_hosted && name == "docker/setup-buildx-action")
56+}
57+
58+/// The repository and ref of an action `uses` fetches from another
59+/// repository: (owner, repo, ref). None for one that is local, a Docker
60+/// image, done natively, or not a name at all.
61+fn remote_action(uses: &str, docker_hosted: bool) -> Option<(String, String, String)> {
62+ let uses = uses.trim();
63+ if uses.starts_with("./") || uses.starts_with("docker://") {
64+ return None;
65+ }
66+ let (name, git_ref) = uses.split_once('@')?;
67+ if git_ref.is_empty() || native(&name.to_ascii_lowercase(), docker_hosted) {
68+ return None;
69+ }
70+ let mut parts = name.splitn(3, '/');
71+ let (owner, repo) = (parts.next()?, parts.next()?);
72+ (!owner.is_empty() && !repo.is_empty()).then(|| (owner.to_owned(), repo.to_owned(), git_ref.to_owned()))
73+}
74+
3675 impl Job {
3776 /// Runs a git command, logging it; the credential header is never logged.
3877 fn git(&mut self, dir: &Path, args: &[&str], auth: Option<&str>) -> bool {
192231 }
193232 match self.log.api.action(&format!("{owner}/{repo}"), git_ref) {
194233 Ok(found) if found["source"] == "g1t" => self.fetch_g1t_action(&on_g1t, owner, repo, git_ref, &found),
195− Ok(_) => self.fetch_action(owner, repo, git_ref),
234+ Ok(found) => {
235+ if let Some(note) = found["note"].as_str() {
236+ self.log.line(&format!("##[warning]{note}"));
237+ }
238+ let sha = found["sha"].as_str().filter(|sha| is_sha(sha)).map(str::to_owned);
239+ if let Some(sha) = &sha
240+ && found["archive"] == true
241+ && let Some(dir) = self.fetch_archived_action(owner, repo, git_ref, sha)
242+ {
243+ return Some(dir);
244+ }
245+ self.fetch_action(owner, repo, git_ref, sha.as_deref())
246+ }
196247 Err((true, why)) => {
197248 self.log.line(&format!("##[error]{why}"));
198249 None
199250 }
200251 Err((false, why)) => {
201252 self.log.line(&format!("##[warning]g1t could not say where {owner}/{repo} is ({why}); fetching it from GitHub."));
202− self.fetch_action(owner, repo, git_ref)
253+ self.fetch_action(owner, repo, git_ref, None)
203254 }
204255 }
205256 }
235286 Some(dir.to_path_buf())
236287 }
237288
238− /// Fetches an action from GitHub, once per job.
239− fn fetch_action(&mut self, owner: &str, repo: &str, git_ref: &str) -> Option<PathBuf> {
289+ /// Downloads a GitHub action's archive that g1t keeps, at the commit
290+ /// g1t resolved its ref to. None when it could not, and the job fetches
291+ /// it from GitHub instead.
292+ fn fetch_archived_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: &str) -> Option<PathBuf> {
240293 let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
294+ self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha}) from g1t"));
295+ let archive = self.temp.join(format!("action{}.tar.gz", super::rand_id()));
296+ let downloaded = self.log.api.action_archive(&format!("{owner}/{repo}"), sha, &archive);
297+ let unpacked = match downloaded {
298+ Ok(_) => {
299+ let _ = std::fs::remove_dir_all(&dir);
300+ let _ = std::fs::create_dir_all(&dir);
301+ let mut command = Command::new("tar");
302+ command.arg("-xzf").arg(&archive).arg("-C").arg(&dir).arg("--strip-components=1");
303+ let mut commands = Commands::default();
304+ matches!(process::run(command, Duration::from_secs(300).min(self.remaining_time()), &mut self.log, &mut commands), Ok(Ended::Exited(0)))
305+ }
306+ Err(why) => {
307+ self.log.line(&format!("g1t could not send it ({why}); fetching it from GitHub."));
308+ false
309+ }
310+ };
311+ let _ = std::fs::remove_file(&archive);
312+ if !unpacked {
313+ let _ = std::fs::remove_dir_all(&dir);
314+ return None;
315+ }
316+ let _ = std::fs::write(dir.join(".g1t-fetched"), "");
317+ Some(dir)
318+ }
319+
320+ /// Fetches an action from GitHub, once per job: at `sha` when g1t
321+ /// resolved its ref to one, so it is the commit g1t named.
322+ fn fetch_action(&mut self, owner: &str, repo: &str, git_ref: &str, sha: Option<&str>) -> Option<PathBuf> {
323+ let dir = super::paths::under_home(ACTIONS_DIR).join(owner).join(repo).join(git_ref);
241324 if dir.join(".g1t-fetched").exists() {
242325 return Some(dir);
243326 }
245328 self.log.line(&format!("##[error]`{owner}/{repo}@{git_ref}` is not a name g1t can fetch."));
246329 return None;
247330 }
248− self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}'"));
331+ match sha {
332+ Some(sha) => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}' (SHA:{sha})")),
333+ None => self.log.line(&format!("Download action repository '{owner}/{repo}@{git_ref}'")),
334+ }
249335 let _ = std::fs::create_dir_all(&dir);
250− let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{git_ref}");
336+ let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{}", sha.unwrap_or(git_ref));
251337 let script = format!("set -o pipefail; curl -fsSL --retry 3 '{url}' | tar -xz -C '{}' --strip-components=1", dir.display());
252338 let mut command = Command::new("bash");
253339 command.args(["-c", &script]);
265351 }
266352 }
267353
354+ /// Downloads every action the job's steps fetch from other
355+ /// repositories, before the first step runs, as GitHub's runner does in
356+ /// "Set up job". False when one could not be: the job fails before
357+ /// anything runs. Actions a composite action uses are fetched as it
358+ /// runs.
359+ pub(crate) fn prepare_actions(&mut self, steps: &[Map<String, Value>]) -> bool {
360+ let mut wanted: Vec<(String, String, String)> = Vec::new();
361+ for step in steps {
362+ if let Some(found) = step.get("uses").and_then(Value::as_str).and_then(|uses| remote_action(uses, self.docker_hosted))
363+ && !wanted.contains(&found)
364+ {
365+ wanted.push(found);
366+ }
367+ }
368+ if wanted.is_empty() {
369+ return true;
370+ }
371+ self.log.line("Prepare all required actions");
372+ let mut ready = true;
373+ for (owner, repo, git_ref) in &wanted {
374+ if self.fetch_remote_action(owner, repo, git_ref).is_none() {
375+ ready = false;
376+ }
377+ }
378+ if !ready {
379+ self.log.line("##[error]Not every action this job uses could be downloaded, so no step was run.");
380+ }
381+ ready
382+ }
383+
268384 /// Runs a JavaScript file of an action with Node.
269385 pub(crate) fn run_node(&mut self, action_dir: &Path, script: &str, env: &BTreeMap<String, String>) -> bool {
270386 let id = format!("node{}", super::rand_id());
337453 }
338454 let (name, git_ref) = uses.split_once('@').unwrap_or((uses, ""));
339455 let lower = name.to_ascii_lowercase();
340− if lower == "docker/setup-buildx-action" && self.docker_hosted {
456+ if lower == "docker/setup-buildx-action" && native(&lower, self.docker_hosted) {
341457 return self.setup_buildx(with);
342458 }
343459 match lower.as_str() {
575691 (ok, outputs)
576692 }
577693 }
694+
695+#[cfg(test)]
696+mod tests {
697+ use super::*;
698+
699+ #[test]
700+ fn the_actions_fetched_before_the_first_step_are_other_repositories() {
701+ let found = |uses: &str| remote_action(uses, true);
702+ assert_eq!(found("actions/setup-node@v4"), Some(("actions".into(), "setup-node".into(), "v4".into())));
703+ assert_eq!(found(" github/codeql-action/init@v3 "), Some(("github".into(), "codeql-action".into(), "v3".into())));
704+ // Done by the runner itself, local, an image, or no version.
705+ assert_eq!(found("actions/checkout@v4"), None);
706+ assert_eq!(found("Actions/Cache/Restore@v4"), None);
707+ assert_eq!(found("docker/setup-buildx-action@v3"), None);
708+ assert!(remote_action("docker/setup-buildx-action@v3", false).is_some());
709+ assert_eq!(found("./.g1t/actions/setup"), None);
710+ assert_eq!(found("docker://alpine:3"), None);
711+ assert_eq!(found("actions/setup-node"), None);
712+ assert_eq!(found("setup-node@v4"), None);
713+ }
714+}
+1−0
1515 g1t-secrets.workspace = true
1616 serde.workspace = true
1717 base64 = "0.22"
18+futures-util = { version = "0.3", default-features = false, features = ["alloc"] }
1819 hex = "0.4"
1920 serde_json = { workspace = true, features = ["preserve_order"] }
2021 worker.workspace = true
+22−0
1+-- Actions from GitHub, kept on g1t (src/registry.rs): the commit each
2+-- ref last named, and the archives of commits jobs have used, in R2.
3+-- Repositories are lowercased, as GitHub reads them.
4+CREATE TABLE IF NOT EXISTS action_refs (
5+ repository TEXT NOT NULL,
6+ ref TEXT NOT NULL,
7+ sha TEXT NOT NULL,
8+ resolved_at TEXT NOT NULL,
9+ PRIMARY KEY (repository, ref)
10+);
11+
12+CREATE TABLE IF NOT EXISTS action_archives (
13+ repository TEXT NOT NULL,
14+ sha TEXT NOT NULL,
15+ object TEXT NOT NULL,
16+ size INTEGER NOT NULL,
17+ stored_at TEXT NOT NULL,
18+ used_at TEXT NOT NULL,
19+ PRIMARY KEY (repository, sha)
20+);
21+
22+CREATE INDEX IF NOT EXISTS action_archives_used ON action_archives (used_at);
+5−0
1414 //! environments' protection rules, holds jobs at those rules until they
1515 //! pass, and lets pull requests' runs from outside wait for approval.
1616 //! - [`cache`] lists `actions/cache` entries, which the API keeps in R2.
17+//! - [`reach`] finds the actions and reusable workflows of other
18+//! repositories on g1t, and [`registry`] resolves those from GitHub to
19+//! commits and keeps their archives, for jobs to download from g1t.
1720 //! - [`runners`] keeps self-hosted runners, hands them jobs (and agent
1821 //! work from the runner service) when they ask, and hears back.
1922 //!
2932 mod plan;
3033 mod protection;
3134 mod reach;
35+mod registry;
3236 mod rename;
3337 pub mod runtime;
3438 mod runners;
249253 "job_spec" => reply(&service.job_spec(args(body)?).await?),
250254 "job_auth" => reply(&service.job_auth(args(body)?).await?),
251255 "job_action" => reply(&service.job_action(args(body)?).await?),
256+ "job_action_archive" => reply(&service.job_action_archive(args(body)?).await?),
252257 "check_runs" => reply(&service.check_runs(args(body)?).await?),
253258 "job_report" => reply(&service.job_report(args(body)?).await?),
254259 // actions/cache, through the API with the job's token.
+6−0
23402340 {
23412341 worker::console_error!("actions: the cache's sweep failed: {error}");
23422342 }
2343+ // And the archives of GitHub's actions no job uses any more.
2344+ if (now_ms / 60_000) % 60 == 47
2345+ && let Err(error) = self.sweep_action_archives(now_ms).await
2346+ {
2347+ worker::console_error!("actions: the action archives' sweep failed: {error}");
2348+ }
23432349 // And artifacts past their time, and the toolkit's abandoned parts.
23442350 if (now_ms / 60_000) % 60 == 37 {
23452351 if let Err(error) = self.sweep_artifacts(now_ms).await {
+4−2
102102 /// `report.repository` (`owner/repo`) and `report.ref`. Answers
103103 /// `{"source": "g1t", "url", "ref", "token"}` (a read-only token for
104104 /// that repository, ending with the job, when it is private),
105− /// `{"source": "github"}`, or a refusal saying why it may not be used.
105+ /// `{"source": "github", "sha", "archive", "note"}` (the commit its ref
106+ /// names, and whether g1t keeps its archive; see registry.rs), or a
107+ /// refusal saying why it may not be used.
106108 pub async fn job_action(&self, a: g1t_contracts::actions::JobCallArgs) -> Result<Outcome<Value>> {
107109 let job = crate::check!(self.job_for_token(&a).await?);
108110 let Some(run) = self.run_row(&job.run_id).await? else {
117119 return Ok(fail(FailureCode::NotFound, "The repository is gone."));
118120 };
119121 let target = match crate::check!(self.find_used(&caller, &ws, &used).await?) {
120− Found::GitHub => return Ok(Outcome::Ok(json!({ "source": "github" }))),
122+ Found::GitHub => return Ok(Outcome::Ok(self.github_action(&used).await?)),
121123 Found::G1t(target) => target,
122124 };
123125 let full_name = format!("{}/{}", target.namespace, target.name);
+374−0
1+//! Actions from GitHub, kept on g1t.
2+//!
3+//! An action `uses:` names that g1t does not host (reach.rs) comes from
4+//! GitHub. Before a job fetches it, g1t resolves its ref to a commit, as
5+//! GitHub's runner does, from the repository's advertisement
6+//! (`info/refs`, which no API rate limit applies to), and keeps that
7+//! commit's archive in R2. Every later job that uses the action at that
8+//! commit downloads it from g1t: faster than GitHub, the same bytes
9+//! whatever happens to the tag, and there while GitHub is not.
10+//!
11+//! - A ref is looked up again once `REF_FRESH_MS` has passed, so a moved
12+//! tag (`v4` moving to a new release) is followed within minutes. When
13+//! GitHub does not answer, the commit the ref last named is used, and
14+//! the job says so.
15+//! - A commit's archive is the one GitHub serves for it, which never
16+//! changes. One larger than `ARCHIVE_MAX_BYTES` is not kept: the job
17+//! fetches it from GitHub itself.
18+//! - An archive no job has used for `ARCHIVE_UNUSED_DAYS` is deleted by
19+//! the hourly sweep. Archives are g1t's own, shared by every workspace,
20+//! and charged to none.
21+
22+use futures_util::StreamExt;
23+use g1t_contracts::time::rfc3339;
24+use g1t_kit::now_ms;
25+use serde::Deserialize;
26+use serde_json::{Value, json};
27+use worker::Result;
28+
29+use crate::Actions;
30+use crate::reach::UsesRef;
31+
32+/// The largest archive kept: the Worker holds it whole while saving it.
33+pub(crate) const ARCHIVE_MAX_BYTES: usize = 32 * 1024 * 1024;
34+/// How long a ref's commit is trusted before GitHub is asked again.
35+pub(crate) const REF_FRESH_MS: u64 = 5 * 60 * 1000;
36+/// An archive unused this long is deleted.
37+pub(crate) const ARCHIVE_UNUSED_DAYS: u64 = 30;
38+/// An advertisement larger than this is not read: a ref is never worth it.
39+const ADVERTISEMENT_MAX_BYTES: usize = 8 * 1024 * 1024;
40+const DAY_MS: u64 = 24 * 60 * 60 * 1000;
41+
42+/// Whether `text` is a whole commit id.
43+pub(crate) fn is_sha(text: &str) -> bool {
44+ text.len() == 40 && text.chars().all(|c| c.is_ascii_hexdigit())
45+}
46+
47+/// The refs a smart HTTP advertisement (`info/refs?service=git-upload-pack`)
48+/// lists, as (commit, name). Peeled tags keep their `^{}`.
49+pub(crate) fn parse_advertisement(bytes: &[u8]) -> Vec<(String, String)> {
50+ let mut refs = Vec::new();
51+ let mut at = 0;
52+ while at + 4 <= bytes.len() {
53+ let Some(length) = std::str::from_utf8(&bytes[at..at + 4]).ok().and_then(|hex| usize::from_str_radix(hex, 16).ok()) else {
54+ break;
55+ };
56+ // A flush or delimiter packet.
57+ if length < 4 {
58+ at += 4;
59+ continue;
60+ }
61+ if at + length > bytes.len() {
62+ break;
63+ }
64+ let line = String::from_utf8_lossy(&bytes[at + 4..at + length]);
65+ at += length;
66+ let line = line.trim_end_matches('\n');
67+ if line.starts_with('#') {
68+ continue;
69+ }
70+ // The first ref carries the capabilities after a NUL.
71+ let line = line.split('\0').next().unwrap_or_default();
72+ if let Some((sha, name)) = line.split_once(' ')
73+ && is_sha(sha)
74+ {
75+ refs.push((sha.to_ascii_lowercase(), name.to_owned()));
76+ }
77+ }
78+ refs
79+}
80+
81+/// The commit `wanted` names among `refs`, as git reads a short name: a
82+/// commit id as it is, else a tag, else a branch (`refs/…` names one
83+/// exactly). A tag's commit is its peeled one.
84+pub(crate) fn pick(refs: &[(String, String)], wanted: &str) -> Option<String> {
85+ if is_sha(wanted) {
86+ return Some(wanted.to_ascii_lowercase());
87+ }
88+ let names: Vec<String> = if wanted.starts_with("refs/") {
89+ vec![wanted.to_owned()]
90+ } else {
91+ vec![format!("refs/tags/{wanted}"), format!("refs/heads/{wanted}")]
92+ };
93+ names.iter().find_map(|name| {
94+ let peeled = format!("{name}^{{}}");
95+ refs.iter()
96+ .find(|(_, listed)| *listed == peeled)
97+ .or_else(|| refs.iter().find(|(_, listed)| listed == name))
98+ .map(|(sha, _)| sha.clone())
99+ })
100+}
101+
102+/// Where a commit's archive is kept in R2.
103+pub(crate) fn archive_object(repository: &str, sha: &str) -> String {
104+ format!("action-archives/{}/{sha}.tar.gz", repository.to_ascii_lowercase())
105+}
106+
107+/// A GET to GitHub, whose body is read up to `max` bytes. None when it
108+/// does not answer 200, or sends more.
109+async fn fetch_capped(url: &str, max: usize) -> Option<Vec<u8>> {
110+ let headers = worker::Headers::new();
111+ // GitHub answers a smart HTTP advertisement to git, which it knows by
112+ // its agent.
113+ headers.set("user-agent", "git/2.47.0 (g1t)").ok()?;
114+ let mut init = worker::RequestInit::new();
115+ init.with_method(worker::Method::Get).with_headers(headers);
116+ let request = worker::Request::new_with_init(url, &init).ok()?;
117+ let mut response = worker::Fetch::Request(request).send().await.ok()?;
118+ if response.status_code() != 200 {
119+ return None;
120+ }
121+ let mut stream = response.stream().ok()?;
122+ let mut body = Vec::new();
123+ while let Some(chunk) = stream.next().await {
124+ body.extend_from_slice(&chunk.ok()?);
125+ if body.len() > max {
126+ return None;
127+ }
128+ }
129+ Some(body)
130+}
131+
132+#[derive(Deserialize)]
133+struct RefRow {
134+ sha: String,
135+ resolved_at: String,
136+}
137+
138+impl Actions {
139+ /// Where a job gets `used`, an action g1t does not host:
140+ /// `{"source": "github", "sha", "archive", "note"}`. `sha` is the commit
141+ /// its ref names (absent when GitHub does not know it, or never
142+ /// answered), `archive` whether g1t keeps that commit's archive for the
143+ /// job to download, and `note` anything the job should say about it.
144+ pub(crate) async fn github_action(&self, used: &UsesRef) -> Result<Value> {
145+ let repository = used.full_name().to_ascii_lowercase();
146+ let now = now_ms();
147+ let (sha, note) = self.resolve_github(&repository, &used.git_ref, now).await?;
148+ let Some(sha) = sha else {
149+ return Ok(json!({ "source": "github", "note": note }));
150+ };
151+ let archive = self.keep_archive(&repository, &sha, now).await?;
152+ Ok(json!({ "source": "github", "sha": sha, "archive": archive, "note": note }))
153+ }
154+
155+ /// The commit `git_ref` names in GitHub's `repository`, and a note when
156+ /// it is the one it last named because GitHub did not answer.
157+ async fn resolve_github(&self, repository: &str, git_ref: &str, now: u64) -> Result<(Option<String>, Option<String>)> {
158+ if is_sha(git_ref) {
159+ return Ok((Some(git_ref.to_ascii_lowercase()), None));
160+ }
161+ let known = self
162+ .db
163+ .prepare("SELECT sha, resolved_at FROM action_refs WHERE repository = ? AND ref = ?")
164+ .bind(&[repository.into(), git_ref.into()])?
165+ .first::<RefRow>(None)
166+ .await?;
167+ if let Some(row) = &known
168+ && row.resolved_at >= rfc3339(now.saturating_sub(REF_FRESH_MS))
169+ {
170+ return Ok((Some(row.sha.clone()), None));
171+ }
172+ let url = format!("https://github.com/{repository}.git/info/refs?service=git-upload-pack");
173+ match fetch_capped(&url, ADVERTISEMENT_MAX_BYTES).await {
174+ Some(advertisement) => {
175+ let Some(sha) = pick(&parse_advertisement(&advertisement), git_ref) else {
176+ return Ok((None, None));
177+ };
178+ self.db
179+ .prepare(
180+ "INSERT INTO action_refs (repository, ref, sha, resolved_at) VALUES (?1, ?2, ?3, ?4)
181+ ON CONFLICT (repository, ref) DO UPDATE SET sha = ?3, resolved_at = ?4",
182+ )
183+ .bind(&[repository.into(), git_ref.into(), sha.as_str().into(), rfc3339(now).into()])?
184+ .run()
185+ .await?;
186+ Ok((Some(sha), None))
187+ }
188+ None => Ok(match known {
189+ Some(row) => {
190+ let note = format!(
191+ "GitHub did not say which commit {repository}@{git_ref} is; using {}, which it named on {}.",
192+ &row.sha,
193+ row.resolved_at.get(..10).unwrap_or(&row.resolved_at)
194+ );
195+ (Some(row.sha), Some(note))
196+ }
197+ None => (None, None),
198+ }),
199+ }
200+ }
201+
202+ /// Whether g1t keeps the archive of `repository` at `sha`, fetching it
203+ /// from GitHub the first time.
204+ async fn keep_archive(&self, repository: &str, sha: &str, now: u64) -> Result<bool> {
205+ let Some(bucket) = &self.cache else { return Ok(false) };
206+ #[derive(Deserialize)]
207+ struct Kept {
208+ used_at: String,
209+ }
210+ let kept = self
211+ .db
212+ .prepare("SELECT used_at FROM action_archives WHERE repository = ? AND sha = ?")
213+ .bind(&[repository.into(), sha.into()])?
214+ .first::<Kept>(None)
215+ .await?;
216+ if let Some(kept) = kept {
217+ // Marked used at most once a day: the sweep needs no more.
218+ if kept.used_at < rfc3339(now.saturating_sub(DAY_MS)) {
219+ self.db
220+ .prepare("UPDATE action_archives SET used_at = ? WHERE repository = ? AND sha = ?")
221+ .bind(&[rfc3339(now).into(), repository.into(), sha.into()])?
222+ .run()
223+ .await?;
224+ }
225+ return Ok(true);
226+ }
227+ let url = format!("https://codeload.github.com/{repository}/tar.gz/{sha}");
228+ let Some(archive) = fetch_capped(&url, ARCHIVE_MAX_BYTES).await else {
229+ return Ok(false);
230+ };
231+ let object = archive_object(repository, sha);
232+ let size = archive.len();
233+ if let Err(error) = bucket.put(&object, archive).execute().await {
234+ worker::console_error!("actions: the archive of {repository}@{sha} was not kept: {error}");
235+ return Ok(false);
236+ }
237+ let at = rfc3339(now);
238+ self.db
239+ .prepare(
240+ "INSERT INTO action_archives (repository, sha, object, size, stored_at, used_at) VALUES (?1, ?2, ?3, ?4, ?5, ?5)
241+ ON CONFLICT (repository, sha) DO UPDATE SET used_at = ?5",
242+ )
243+ .bind(&[repository.into(), sha.into(), object.into(), (size as f64).into(), at.into()])?
244+ .run()
245+ .await?;
246+ Ok(true)
247+ }
248+
249+ /// `job_action_archive`: a running job downloads an action's archive
250+ /// g1t keeps, by `report.repository` and `report.sha`. Answers the R2
251+ /// object to send, `{"object", "size"}`. Any job may: they are public
252+ /// repositories' code.
253+ pub async fn job_action_archive(&self, a: g1t_contracts::actions::JobCallArgs) -> Result<g1t_contracts::Outcome<Value>> {
254+ crate::check!(self.job_for_token(&a).await?);
255+ let repository = a.report["repository"].as_str().unwrap_or_default().to_ascii_lowercase();
256+ let sha = a.report["sha"].as_str().unwrap_or_default().to_ascii_lowercase();
257+ if !is_sha(&sha) || crate::reach::parse_uses(&format!("{repository}@{sha}")).is_none() {
258+ return Ok(crate::fail(g1t_contracts::FailureCode::Invalid, "Name the action's repository as owner/repo, and its commit."));
259+ }
260+ #[derive(Deserialize)]
261+ struct Kept {
262+ object: String,
263+ size: f64,
264+ }
265+ let kept = self
266+ .db
267+ .prepare("SELECT object, size FROM action_archives WHERE repository = ? AND sha = ?")
268+ .bind(&[repository.as_str().into(), sha.as_str().into()])?
269+ .first::<Kept>(None)
270+ .await?;
271+ Ok(match kept {
272+ Some(kept) => g1t_contracts::Outcome::Ok(json!({ "object": kept.object, "size": kept.size })),
273+ None => crate::fail(g1t_contracts::FailureCode::NotFound, format!("g1t does not keep {repository}@{sha}.")),
274+ })
275+ }
276+
277+ /// Hourly: archives no job has used for `ARCHIVE_UNUSED_DAYS`, and refs
278+ /// not looked up as long.
279+ pub async fn sweep_action_archives(&self, now: u64) -> Result<()> {
280+ let before = rfc3339(now.saturating_sub(ARCHIVE_UNUSED_DAYS * DAY_MS));
281+ #[derive(Deserialize)]
282+ struct Gone {
283+ repository: String,
284+ sha: String,
285+ object: String,
286+ }
287+ let gone = self
288+ .db
289+ .prepare("SELECT repository, sha, object FROM action_archives WHERE used_at < ? LIMIT 100")
290+ .bind(&[before.as_str().into()])?
291+ .all()
292+ .await?
293+ .results::<Gone>()?;
294+ if let Some(bucket) = &self.cache
295+ && !gone.is_empty()
296+ {
297+ bucket.delete_multiple(gone.iter().map(|g| g.object.as_str()).collect()).await?;
298+ for archive in &gone {
299+ self.db
300+ .prepare("DELETE FROM action_archives WHERE repository = ? AND sha = ?")
301+ .bind(&[archive.repository.as_str().into(), archive.sha.as_str().into()])?
302+ .run()
303+ .await?;
304+ }
305+ }
306+ self.db.prepare("DELETE FROM action_refs WHERE resolved_at < ?").bind(&[before.into()])?.run().await?;
307+ Ok(())
308+ }
309+}
310+
311+#[cfg(test)]
312+mod tests {
313+ use super::*;
314+
315+ fn pkt(payload: &str) -> Vec<u8> {
316+ let mut out = format!("{:04x}", payload.len() + 4).into_bytes();
317+ out.extend_from_slice(payload.as_bytes());
318+ out
319+ }
320+
321+ const MAIN: &str = "1111111111111111111111111111111111111111";
322+ const V4_TAG: &str = "2222222222222222222222222222222222222222";
323+ const V4_COMMIT: &str = "3333333333333333333333333333333333333333";
324+ const V1_BRANCH: &str = "4444444444444444444444444444444444444444";
325+ const V1_TAG: &str = "5555555555555555555555555555555555555555";
326+
327+ fn advertisement() -> Vec<u8> {
328+ let mut bytes = pkt("# service=git-upload-pack\n");
329+ bytes.extend_from_slice(b"0000");
330+ bytes.extend(pkt(&format!("{MAIN} HEAD\0multi_ack thin-pack symref=HEAD:refs/heads/main\n")));
331+ bytes.extend(pkt(&format!("{MAIN} refs/heads/main\n")));
332+ bytes.extend(pkt(&format!("{V1_BRANCH} refs/heads/v1\n")));
333+ bytes.extend(pkt(&format!("{V1_TAG} refs/tags/v1\n")));
334+ bytes.extend(pkt(&format!("{V4_TAG} refs/tags/v4\n")));
335+ bytes.extend(pkt(&format!("{V4_COMMIT} refs/tags/v4^{{}}\n")));
336+ bytes.extend_from_slice(b"0000");
337+ bytes
338+ }
339+
340+ #[test]
341+ fn an_advertisement_lists_its_refs_capabilities_aside() {
342+ let refs = parse_advertisement(&advertisement());
343+ assert_eq!(refs.len(), 6);
344+ assert_eq!(refs[0], (MAIN.to_owned(), "HEAD".to_owned()));
345+ assert_eq!(refs[5], (V4_COMMIT.to_owned(), "refs/tags/v4^{}".to_owned()));
346+ // Cut short: what came whole is kept.
347+ let whole = advertisement();
348+ assert_eq!(parse_advertisement(&whole[..whole.len() - 30]).len(), 5);
349+ assert!(parse_advertisement(b"not git").is_empty());
350+ }
351+
352+ #[test]
353+ fn a_ref_is_read_as_git_reads_a_short_name() {
354+ let refs = parse_advertisement(&advertisement());
355+ // A tag's commit, not the tag object.
356+ assert_eq!(pick(&refs, "v4").as_deref(), Some(V4_COMMIT));
357+ // A tag before a branch of the same name.
358+ assert_eq!(pick(&refs, "v1").as_deref(), Some(V1_TAG));
359+ assert_eq!(pick(&refs, "refs/heads/v1").as_deref(), Some(V1_BRANCH));
360+ assert_eq!(pick(&refs, "main").as_deref(), Some(MAIN));
361+ // A commit is itself, known or not.
362+ let other = "ABCDEF0123456789ABCDEF0123456789ABCDEF01";
363+ assert_eq!(pick(&refs, other).as_deref(), Some("abcdef0123456789abcdef0123456789abcdef01"));
364+ assert_eq!(pick(&refs, "v9"), None);
365+ assert_eq!(pick(&refs, "abc123"), None);
366+ }
367+
368+ #[test]
369+ fn archives_are_kept_by_repository_and_commit() {
370+ assert_eq!(archive_object("Actions/Checkout", V4_COMMIT), format!("action-archives/actions/checkout/{V4_COMMIT}.tar.gz"));
371+ assert!(is_sha(MAIN));
372+ assert!(!is_sha("v4"));
373+ }
374+}