Sign in

pulls/pr_01m47d15m3e54sn21z27rpy5n9

public

pr_01m47d15m3e54sn21z27rpy5n9/services/repos/src/import.rs

224 lines8,265 bytesCodeBlame
1//! Importing a repository from another git host.
2//!
3//! g1t fetches the default branch the way a git client would, over smart
4//! HTTP, and pushes the pack it receives into a new repository unchanged.
5//! Only public repositories reachable over https can be imported, and only
6//! their default branch.
7
8use futures_util::StreamExt;
9use worker::js_sys::Uint8Array;
10use worker::{Fetch, Headers, Method, Request, RequestInit, Result, Url};
11
12use crate::land::{read_pkt_lines, unpack_sideband};
13
14/// The largest pack that is imported. A Worker holds the pack in memory
15/// twice while relaying it.
16const MAX_PACK_BYTES: usize = 40 * 1024 * 1024;
17const HEADS: &str = "refs/heads/";
18/// Some hosts only speak the smart protocol to something that says it is git.
19const USER_AGENT: &str = "git/2.45.0 (g1t import)";
20
21/// What the other host says its default branch is and where it points.
22#[derive(Debug, PartialEq, Eq)]
23pub struct Remote {
24 pub branch: String,
25 pub head: String,
26}
27
28/// The address to import from, tidied, or `None` if it is not one g1t will
29/// fetch: it must be https, with no credentials in it.
30pub fn clean_url(url: &str) -> Option<String> {
31 let parsed = Url::parse(url.trim()).ok()?;
32 let plain = parsed.scheme() == "https"
33 && parsed.username().is_empty()
34 && parsed.password().is_none()
35 && parsed.host_str().is_some()
36 && parsed.query().is_none();
37 if !plain {
38 return None;
39 }
40 let host = parsed.host_str()?;
41 let path = parsed.path().trim_end_matches('/');
42 (path.len() > 1).then(|| format!("https://{host}{path}"))
43}
44
45/// The default branch and its head, from a ref advertisement.
46fn parse_remote(bytes: &[u8]) -> Option<Remote> {
47 let (lines, _) = read_pkt_lines(bytes);
48 let mut head = None;
49 let mut branch = None;
50 let mut branches = Vec::new();
51 for line in lines {
52 let mut parts = line.splitn(2, |byte| *byte == 0);
53 let reference = std::str::from_utf8(parts.next()?).ok()?.trim_end();
54 // The first ref carries the capabilities, one of which names the
55 // branch HEAD points to.
56 if let Some(capabilities) = parts
57 .next()
58 .and_then(|bytes| std::str::from_utf8(bytes).ok())
59 {
60 branch = capabilities
61 .split(' ')
62 .find_map(|capability| capability.trim().strip_prefix("symref=HEAD:refs/heads/"))
63 .map(str::to_owned);
64 }
65 let Some((hash, name)) = reference.split_once(' ') else {
66 continue;
67 };
68 if name == "HEAD" {
69 head = Some(hash.to_owned());
70 } else if let Some(name) = name.strip_prefix(HEADS) {
71 branches.push((name.to_owned(), hash.to_owned()));
72 }
73 }
74 // Without a symref, the branch HEAD agrees with; failing that, main.
75 let branch = branch.or_else(|| {
76 let head = head.as_deref()?;
77 branches
78 .iter()
79 .find(|(_, hash)| hash == head)
80 .map(|(name, _)| name.clone())
81 })?;
82 let head = branches
83 .iter()
84 .find(|(name, _)| *name == branch)
85 .map(|(_, hash)| hash.clone())
86 .or(head)?;
87 Some(Remote { branch, head })
88}
89
90fn request(method: Method, url: &str, body: Option<Vec<u8>>) -> Result<Request> {
91 let headers = Headers::new();
92 headers.set("user-agent", USER_AGENT)?;
93 if body.is_some() {
94 headers.set("content-type", "application/x-git-upload-pack-request")?;
95 headers.set("accept", "application/x-git-upload-pack-result")?;
96 }
97 let mut init = RequestInit::new();
98 init.with_method(method).with_headers(headers);
99 if let Some(body) = body {
100 init.with_body(Some(Uint8Array::from(body.as_slice()).into()));
101 }
102 Request::new_with_init(url, &init)
103}
104
105/// Asks the other host what it has. `Err` in the inner result is a reason
106/// to show the person importing.
107pub async fn discover(url: &str) -> Result<std::result::Result<Remote, String>> {
108 let request = request(
109 Method::Get,
110 &format!("{url}/info/refs?service=git-upload-pack"),
111 None,
112 )?;
113 let mut response = match Fetch::Request(request).send().await {
114 Ok(response) => response,
115 Err(_) => return Ok(Err("That address could not be reached.".to_owned())),
116 };
117 if response.status_code() != 200 {
118 return Ok(Err(
119 "No public git repository was found at that address. Private repositories cannot be imported."
120 .to_owned(),
121 ));
122 }
123 let bytes = response.bytes().await?;
124 Ok(parse_remote(&bytes).ok_or_else(|| "That repository is empty.".to_owned()))
125}
126
127/// Fetches a pack holding everything reachable from `head`.
128pub async fn fetch(url: &str, head: &str) -> Result<std::result::Result<Vec<u8>, String>> {
129 let mut body = format!("{:04x}want {head} side-band-64k\n", head.len() + 24).into_bytes();
130 body.extend_from_slice(b"00000009done\n");
131 let request = request(Method::Post, &format!("{url}/git-upload-pack"), Some(body))?;
132 let mut response = Fetch::Request(request).send().await?;
133 if response.status_code() != 200 {
134 return Ok(Err(
135 "The other host refused to send the repository.".to_owned()
136 ));
137 }
138 // Read in pieces, so a repository that is too large is noticed before
139 // it has all been held in memory.
140 let mut received = Vec::new();
141 let mut stream = response.stream()?;
142 while let Some(chunk) = stream.next().await {
143 received.extend_from_slice(&chunk?);
144 if received.len() > MAX_PACK_BYTES {
145 return Ok(Err(format!(
146 "That repository is larger than {} MB, the most that can be imported. Push it with git instead.",
147 MAX_PACK_BYTES / 1024 / 1024
148 )));
149 }
150 }
151 Ok(unpack_sideband(&received)
152 .map_err(|_| "The other host did not send a usable pack.".to_owned()))
153}
154
155#[cfg(test)]
156mod tests {
157 use super::*;
158
159 fn pkt(payload: &str) -> Vec<u8> {
160 format!("{:04x}{payload}", payload.len() + 4).into_bytes()
161 }
162
163 #[test]
164 fn only_plain_https_addresses_are_fetched() {
165 assert_eq!(
166 clean_url(" https://github.com/syntaqx/hello/ ").as_deref(),
167 Some("https://github.com/syntaqx/hello")
168 );
169 assert_eq!(
170 clean_url("https://github.com/syntaqx/hello.git").as_deref(),
171 Some("https://github.com/syntaqx/hello.git")
172 );
173 for bad in [
174 "http://github.com/a/b",
175 "git@github.com:a/b.git",
176 "https://user:secret@github.com/a/b",
177 "https://github.com",
178 "https://github.com/a/b?x=1",
179 "not a url",
180 ] {
181 assert_eq!(clean_url(bad), None, "{bad}");
182 }
183 }
184
185 #[test]
186 fn the_default_branch_comes_from_the_symref() {
187 let trunk = "c71546fcd893ef8b0f57388b65e620d759705dda";
188 let other = "4807077b296e6edbf410d55e72749d3e1170c291";
189 let advertisement = [
190 pkt("# service=git-upload-pack\n"),
191 b"0000".to_vec(),
192 pkt(&format!(
193 "{trunk} HEAD\0multi_ack side-band-64k symref=HEAD:refs/heads/trunk agent=git/x\n"
194 )),
195 pkt(&format!("{other} refs/heads/feature\n")),
196 pkt(&format!("{trunk} refs/heads/trunk\n")),
197 b"0000".to_vec(),
198 ]
199 .concat();
200 assert_eq!(
201 parse_remote(&advertisement),
202 Some(Remote {
203 branch: "trunk".to_owned(),
204 head: trunk.to_owned()
205 })
206 );
207 }
208
209 #[test]
210 fn without_a_symref_the_branch_head_points_to_is_used() {
211 let main = "c71546fcd893ef8b0f57388b65e620d759705dda";
212 let advertisement = [
213 pkt("# service=git-upload-pack\n"),
214 b"0000".to_vec(),
215 pkt(&format!("{main} HEAD\0side-band-64k\n")),
216 pkt(&format!("{main} refs/heads/main\n")),
217 b"0000".to_vec(),
218 ]
219 .concat();
220 assert_eq!(parse_remote(&advertisement).unwrap().branch, "main");
221 let empty = [pkt("# service=git-upload-pack\n"), b"00000000".to_vec()].concat();
222 assert_eq!(parse_remote(&empty), None);
223 }
224}