Sign in

flagon-io/g1t

public

Commit

A public import copies every branch and tag, so an imported library keeps its releases

An import without a credential fetched only the source's HEAD: an import of php-fig/log came in as one branch, and Composer had no versions to offer. It now copies every branch and tag (annotated tags kept) through the mirror's copy with an anonymous endpoint, and publishes a git.push for each ref, default branch first. Composer versions appearing and going are in the audit log.

syntaqxcommitted Parent5ab6b8aBrowse files
5 files+107−740/5 viewed
+2−1
132132 2. Choose where its code comes from:
133133 - **Start empty**: a new repository on g1t.
134134 - **Import code**: copy a public repository from GitHub or any git host
135− into a new one on g1t.
135+ into a new one on g1t, with every branch and tag (up to 40 MB of
136+ history).
136137 - **Import from GitHub**: import, mirror or move repositories you can
137138 reach on GitHub, private ones too, with every branch and tag and,
138139 if you like, their issues. See [GitHub](/guides/github/).
+9−2
1818 use base64::Engine;
1919 use base64::engine::general_purpose::STANDARD;
2020 use g1t_contracts::User;
21+use g1t_contracts::audit::AuditActor;
2122 use g1t_contracts::events::PackageEvent;
2223 use g1t_contracts::new_id;
2324 use g1t_contracts::repos::{
403404 }
404405 };
405406
406− let caller = Caller { actor: None };
407+ // Versions follow git, so g1t records them: what made them is the
408+ // push, already in the log as `git.push`.
409+ let caller = Caller { actor: Some(AuditActor::system()) };
407410 let wanted = wanted_versions(&refs, &repo.default_branch);
408411 let stored = self.db.versions(&package.id, 1000).await?;
409412 let mut manifests: HashMap<String, Option<Value>> = HashMap::new();
445448 if current.is_none() {
446449 let event = PackageEvent { version: Some(version.version.clone()), digest: Some(commit.clone()), ..self.event_of(&package) };
447450 self.announce("package.published", &package, event, &caller).await;
451+ self.audit(&caller, "package.publish", &package, Some(&format!("{name}@{}", version.version)), None).await;
448452 }
449453 }
450454 let kept: HashSet<&str> = wanted.iter().map(|(v, ..)| v.version.as_str()).collect();
452456 self.db.delete_version(&row.id).await?;
453457 let event = PackageEvent { version: Some(row.version.clone()), digest: Some(row.digest.clone()), ..self.event_of(&package) };
454458 self.announce("package.version_deleted", &package, event, &caller).await;
459+ self.audit(&caller, "package.delete_version", &package, Some(&format!("{}@{}", package.name, row.version)), None).await;
455460 changed = true;
456461 }
457462 if let Some(commit) = &default {
503508 async fn drop_composer(&self, package: &PackageRow) -> Result<()> {
504509 self.db.delete_package(&package.id).await?;
505510 self.db.measure(&package.workspace).await?;
506− self.announce("package.deleted", package, self.event_of(package), &Caller { actor: None }).await;
511+ let caller = Caller { actor: Some(AuditActor::system()) };
512+ self.announce("package.deleted", package, self.event_of(package), &caller).await;
513+ self.audit(&caller, "package.delete", package, Some(&package.name), None).await;
507514 Ok(())
508515 }
509516
+5−37
11 //! Importing a repository from another git host.
22 //!
3−//! g1t fetches the default branch the way a git client would, over smart
4−//! HTTP, and pushes the pack it receives into a new repository unchanged.
5−//! Only public repositories reachable over https can be imported, and only
6−//! their default branch.
3+//! This finds a public repository over smart HTTP and its default branch, so
4+//! an address that does not work is refused before anything is made; the
5+//! copy itself, every branch and tag, is mirror.rs's (`Endpoint::anonymous`).
6+//! Only public repositories reachable over https are imported this way.
77
8−use futures_util::StreamExt;
98 use worker::js_sys::Uint8Array;
109 use worker::{Fetch, Headers, Method, Request, RequestInit, Result, Url};
1110
12−use crate::land::{read_pkt_lines, unpack_sideband};
11+use crate::land::read_pkt_lines;
1312
14−/// The largest pack that is imported. A Worker holds the pack in memory
15−/// twice while relaying it.
16−const MAX_PACK_BYTES: usize = 40 * 1024 * 1024;
1713 const HEADS: &str = "refs/heads/";
1814 /// Some hosts only speak the smart protocol to something that says it is git.
1915 const USER_AGENT: &str = "git/2.45.0 (g1t import)";
122118 }
123119 let bytes = response.bytes().await?;
124120 Ok(parse_remote(&bytes).ok_or_else(|| "That repository is empty.".to_owned()))
125−}
126−
127−/// Fetches a pack holding everything reachable from `head`.
128−pub async fn fetch(url: &str, head: &str) -> Result<std::result::Result<Vec<u8>, String>> {
129− let mut body = format!("{:04x}want {head} side-band-64k\n", head.len() + 24).into_bytes();
130− body.extend_from_slice(b"00000009done\n");
131− let request = request(Method::Post, &format!("{url}/git-upload-pack"), Some(body))?;
132− let mut response = Fetch::Request(request).send().await?;
133− if response.status_code() != 200 {
134− return Ok(Err(
135− "The other host refused to send the repository.".to_owned()
136− ));
137− }
138− // Read in pieces, so a repository that is too large is noticed before
139− // it has all been held in memory.
140− let mut received = Vec::new();
141− let mut stream = response.stream()?;
142− while let Some(chunk) = stream.next().await {
143− received.extend_from_slice(&chunk?);
144− if received.len() > MAX_PACK_BYTES {
145− return Ok(Err(format!(
146− "That repository is larger than {} MB, the most that can be imported. Push it with git instead.",
147− MAX_PACK_BYTES / 1024 / 1024
148− )));
149− }
150− }
151− Ok(unpack_sideband(&received)
152− .map_err(|_| "The other host did not send a usable pack.".to_owned()))
153121 }
154122
155123 #[cfg(test)]
+20−33
511511 Ok(remote) => remote,
512512 Err(reason) => return Ok(Outcome::fail(FailureCode::Invalid, reason)),
513513 };
514− let pack = match import::fetch(&url, &remote.head).await? {
515− Ok(pack) => pack,
516− Err(reason) => return Ok(Outcome::fail(FailureCode::Invalid, reason)),
517− };
518− imported = Some((remote, pack));
514+ imported = Some((remote, url));
519515 }
520516 let now = now_ms();
521517 let repo = Repo {
560556 name: repo.name.clone(),
561557 })
562558 .await?;
563− let mut pushed = None;
564− if let Some((remote, pack)) = imported {
559+ // Every branch and tag the import made, announced as pushes.
560+ let mut pushed: Vec<(String, String)> = Vec::new();
561+ // A public repository, read with no credential: every branch and
562+ // tag is copied too, the default branch the one its HEAD names.
563+ if let Some((_, url)) = imported {
565564 let access = self
566565 .store
567566 .open(&store_key(&repo))
568567 .await?
569568 .access(Scope::Write)
570569 .await?;
571− let stored =
572− land::push_pack(&access, &repo.default_branch, None, &remote.head, pack).await?;
570+ let target = mirror::Endpoint::bearer(&access.remote, &access.token);
571+ let copied = mirror::copy(&mirror::Endpoint::anonymous(&url), &target, mirror::Prune::Yes).await?;
573572 self.refs_moved(&repo.id).await;
574− if let Err(reason) = stored {
575− self.registry.remove(&repo.id).await?;
576− return Ok(Outcome::fail(
577− FailureCode::Invalid,
578− format!("The repository could not be stored: {reason}"),
579− ));
573+ match copied {
574+ Ok(copied) => pushed = mirror::import_pushes(&copied.updated, &repo.default_branch),
575+ Err(reason) => {
576+ self.registry.remove(&repo.id).await?;
577+ return Ok(Outcome::fail(
578+ FailureCode::Invalid,
579+ format!("The repository could not be stored: {reason}"),
580+ ));
581+ }
580582 }
581− pushed = Some(remote.head);
582583 }
583584 if let Some((source, _)) = credentialed {
584585 let access = self
591592 let copied = mirror::copy(&source, &target, mirror::Prune::Yes).await?;
592593 self.refs_moved(&repo.id).await;
593594 match copied {
594− Ok(copied) => {
595− let branch = format!("refs/heads/{}", repo.default_branch);
596− pushed = copied
597− .updated
598− .into_iter()
599− .find(|(name, _, _)| *name == branch)
600− .map(|(_, _, new)| new);
601− }
595+ Ok(copied) => pushed = mirror::import_pushes(&copied.updated, &repo.default_branch),
602596 Err(reason) => {
603597 self.registry.remove(&repo.id).await?;
604598 return Ok(Outcome::fail(
621615 },
622616 })
623617 .await?;
624− if let Some(head) = pushed {
625− self.publish_push(
626− &repo,
627− &format!("refs/heads/{}", repo.default_branch),
628− None,
629− &head,
630− None,
631− )
632− .await?;
618+ for (git_ref, head) in &pushed {
619+ self.publish_push(&repo, git_ref, None, head, None).await?;
633620 }
634621 Ok(Outcome::Ok(repo))
635622 }
+71−1
5151 }
5252 }
5353
54+ /// A public repository anywhere, read with no credential: importing
55+ /// one copies every branch and tag, as with a credential.
56+ pub fn anonymous(url: &str) -> Self {
57+ Endpoint {
58+ url: url.trim_end_matches('/').to_owned(),
59+ authorization: String::new(),
60+ }
61+ }
62+
5463 /// A GitHub repository, with an installation access token. GitHub takes
5564 /// one as the password of the user `x-access-token`. The token is used
5665 /// as given: its length and shape are GitHub's to change.
259268 }
260269 let headers = Headers::new();
261270 headers.set("user-agent", USER_AGENT)?;
262− headers.set("authorization", &endpoint.authorization)?;
271+ if !endpoint.authorization.is_empty() {
272+ headers.set("authorization", &endpoint.authorization)?;
273+ }
263274 let mut init = RequestInit::new();
264275 if let Some((service, body)) = body {
265276 headers.set("content-type", &format!("application/x-{service}-request"))?;
371382 Ok(Ok(copied))
372383 }
373384
385+/// The pushes an import announces, one for each ref it made: the default
386+/// branch first (what follows a repository, such as its Composer package,
387+/// starts from it), then the other branches, then the tags.
388+pub fn import_pushes(updated: &[(String, Option<String>, String)], default_branch: &str) -> Vec<(String, String)> {
389+ let default = format!("refs/heads/{default_branch}");
390+ let rank = |name: &str| {
391+ if name == default {
392+ 0
393+ } else if name.starts_with("refs/heads/") {
394+ 1
395+ } else {
396+ 2
397+ }
398+ };
399+ let mut pushes: Vec<(String, String)> = updated
400+ .iter()
401+ .filter(|(name, _, new)| (name.starts_with("refs/heads/") || name.starts_with("refs/tags/")) && new != ZERO_ID)
402+ .map(|(name, _, new)| (name.clone(), new.clone()))
403+ .collect();
404+ pushes.sort_by(|a, b| rank(&a.0).cmp(&rank(&b.0)).then_with(|| a.0.cmp(&b.0)));
405+ pushes
406+}
407+
374408 impl<S: GitStore> Repos<S> {
375409 /// `mirror`: a mirror catching up with the host it mirrors, or a
376410 /// repository pushing its refs out to one. Each branch moved on g1t is
473507 }
474508
475509 #[test]
510+ fn a_public_import_copies_every_branch_and_tag_annotated_ones_whole() {
511+ // An empty repository being filled from a public source: every
512+ // branch and tag is made, an annotated tag as its tag object (so it
513+ // stays annotated), and the source's pull request refs and peeled
514+ // lines are left out.
515+ let bytes = [
516+ pkt_line("# service=git-upload-pack\n"),
517+ b"0000".to_vec(),
518+ pkt_line(&format!("{A} HEAD\0multi_ack symref=HEAD:refs/heads/master\n")),
519+ pkt_line(&format!("{A} refs/heads/master\n")),
520+ pkt_line(&format!("{B} refs/heads/next\n")),
521+ pkt_line(&format!("{C} refs/pull/7/head\n")),
522+ pkt_line(&format!("{C} refs/tags/1.0.0\n")),
523+ pkt_line(&format!("{A} refs/tags/1.0.0^{{}}\n")),
524+ pkt_line(&format!("{B} refs/tags/2.0.0\n")),
525+ b"0000".to_vec(),
526+ ]
527+ .concat();
528+ let source = parse_advertisement(&bytes);
529+ assert_eq!(source.default_branch().as_deref(), Some("master"), "HEAD stays the default branch");
530+ let commands = plan(&source, &Advertised::default(), Prune::Yes);
531+ let names: Vec<(&str, &str)> = commands.iter().map(|(name, _, new)| (name.as_str(), new.as_str())).collect();
532+ assert_eq!(
533+ names,
534+ [("refs/heads/master", A), ("refs/heads/next", B), ("refs/tags/1.0.0", C), ("refs/tags/2.0.0", B)]
535+ );
536+ let pushes = import_pushes(&commands, "master");
537+ let order: Vec<&str> = pushes.iter().map(|(name, _)| name.as_str()).collect();
538+ assert_eq!(order, ["refs/heads/master", "refs/heads/next", "refs/tags/1.0.0", "refs/tags/2.0.0"]);
539+ let pushes = import_pushes(&commands, "next");
540+ assert_eq!(pushes[0].0, "refs/heads/next", "the default branch is announced first");
541+ assert_eq!(Endpoint::anonymous("https://github.com/php-fig/log.git/").url, "https://github.com/php-fig/log.git");
542+ assert!(Endpoint::anonymous("https://x").authorization.is_empty(), "no credential is sent");
543+ }
544+
545+ #[test]
476546 fn a_mirror_prunes_and_a_push_out_does_not() {
477547 let source = advertised(&[("refs/heads/main", A), ("refs/tags/v1", B)]);
478548 let target = advertised(&[("refs/heads/main", B), ("refs/heads/old", C)]);