Skip to content
Sign up

flagon-io/g1t

public
0
1,611 linesCodeBlameRaw
1//! Git over HTTPS: the smart HTTP remote at `/<namespace>/<repo>.git`,
2//! proxied to the git store with a short-lived token.
3
4use std::cell::RefCell;
5use std::rc::Rc;
6
7use futures_util::StreamExt;
8use g1t_contracts::identity::GitCredentialsArgs;
9use g1t_contracts::repos::{GitAccess, GitService, RepoPath};
10use g1t_contracts::{FailureCode, Outcome, Viewer};
11use worker::js_sys::Uint8Array;
12use worker::wasm_bindgen::JsValue;
13use worker::{Fetch, Fetcher, Headers, Method, Request, RequestInit, Response, Result, Url};
14
15use crate::meters;
16use crate::pack_limits::{PackSizer, Violation};
17use crate::push_checks::{Checks, Verdict};
18use crate::resilience::{self, Busy, Failure};
19
20const ENDPOINTS: [&str; 3] = ["info/refs", "git-upload-pack", "git-receive-pack"];
21const FORWARDED_HEADERS: [&str; 5] = [
22 "accept",
23 "content-encoding",
24 "content-type",
25 "git-protocol",
26 "user-agent",
27];
28
29/// A git request, parsed from its URL.
30pub struct GitRequest {
31 pub path: RepoPath,
32 pub endpoint: &'static str,
33 pub service: GitService,
34}
35
36impl GitRequest {
37 /// The same request for the repository of the same name under
38 /// `namespace`: where a workspace alias leads.
39 pub fn under(&self, namespace: &str) -> GitRequest {
40 GitRequest {
41 path: RepoPath {
42 namespace: namespace.to_owned(),
43 name: self.path.name.clone(),
44 },
45 endpoint: self.endpoint,
46 service: self.service,
47 }
48 }
49}
50
51/// Parses `/<namespace>/<name>[.git]/<endpoint>`, or returns `None` if the
52/// request is not git's.
53pub fn parse(url: &Url) -> Option<GitRequest> {
54 let path = url.path().strip_prefix('/')?;
55 let endpoint = ENDPOINTS
56 .into_iter()
57 .find(|endpoint| path.ends_with(&format!("/{endpoint}")))?;
58 let repo = &path[..path.len() - endpoint.len() - 1];
59 let (namespace, name) = repo.split_once('/')?;
60 let name = name.strip_suffix(".git").unwrap_or(name);
61 if namespace.is_empty() || name.is_empty() || name.contains('/') {
62 return None;
63 }
64 let service = if endpoint == "info/refs" {
65 url.query_pairs()
66 .find(|(key, _)| key == "service")
67 .map(|(_, value)| value.into_owned())?
68 } else {
69 endpoint.to_owned()
70 };
71 let service = match service.as_str() {
72 "git-upload-pack" => GitService::UploadPack,
73 "git-receive-pack" => GitService::ReceivePack,
74 _ => return None,
75 };
76 Some(GitRequest {
77 path: RepoPath {
78 namespace: namespace.to_owned(),
79 name: name.to_owned(),
80 },
81 endpoint,
82 service,
83 })
84}
85
86/// How long each step of a git request took, sent back to git as a
87/// `Server-Timing` header so that a slow clone shows where its time went.
88/// Step names and whole milliseconds only. The Workers clock moves only
89/// while a request waits on something, so each step is the time spent
90/// waiting on the database, another service or the git store. A note
91/// says how a step went without a duration: `refs;desc=hit-colo`. A part
92/// is one of several things a step did at once, with its own duration: a
93/// push's `checks` step is its `read`, `rules` and `scan` parts side by side.
94pub struct Timing {
95 started: u64,
96 last: u64,
97 steps: Vec<(&'static str, u64)>,
98 parts: Vec<(&'static str, u64)>,
99 notes: Vec<(&'static str, &'static str)>,
100}
101
102impl Timing {
103 pub fn start() -> Self {
104 let now = g1t_kit::now_ms();
105 Self {
106 started: now,
107 last: now,
108 steps: Vec::new(),
109 parts: Vec::new(),
110 notes: Vec::new(),
111 }
112 }
113
114 /// Says how long `part` of the step under way took. Parts overlap, so
115 /// they are listed after the steps and are not part of the total.
116 pub fn part(&mut self, part: &'static str, ms: u64) {
117 self.parts.push((part, ms));
118 }
119
120 /// Says how `name` went: where an answer or a credential came from.
121 pub fn note(&mut self, name: &'static str, description: &'static str) {
122 self.notes.push((name, description));
123 }
124
125 /// Ends a step, named `step`, that began when the last one ended.
126 pub fn mark(&mut self, step: &'static str) {
127 let now = g1t_kit::now_ms();
128 self.steps.push((step, now.saturating_sub(self.last)));
129 self.last = now;
130 }
131
132 /// `response`, with how long each step took.
133 pub fn apply(&self, response: Response) -> Result<Response> {
134 let total = g1t_kit::now_ms().saturating_sub(self.started);
135 let headers = response.headers().clone();
136 headers.set("server-timing", &server_timing(&self.steps, &self.parts, &self.notes, total))?;
137 Ok(response.with_headers(headers))
138 }
139}
140
141/// A `Server-Timing` value: each step with its duration, then each part,
142/// the notes, and the total.
143fn server_timing(steps: &[(&str, u64)], parts: &[(&str, u64)], notes: &[(&str, &str)], total: u64) -> String {
144 steps
145 .iter()
146 .chain(parts)
147 .map(|(step, ms)| format!("{step};dur={ms}"))
148 .chain(notes.iter().map(|(name, description)| format!("{name};desc={description}")))
149 .chain(std::iter::once(format!("total;dur={total}")))
150 .collect::<Vec<_>>()
151 .join(", ")
152}
153
154/// The user named by an HTTP Basic `Authorization` header, as git sends it.
155pub async fn viewer(request: &Request, identity: &Fetcher) -> Result<Viewer> {
156 let Some(header) = request.headers().get("authorization")? else {
157 return Ok(None);
158 };
159 let Some((scheme, encoded)) = header.split_once(' ') else {
160 return Ok(None);
161 };
162 if !scheme.eq_ignore_ascii_case("basic") {
163 return Ok(None);
164 }
165 let Some(decoded) = decode_base64(encoded.trim()) else {
166 return Ok(None);
167 };
168 let Some((username, secret)) = decoded.split_once(':') else {
169 return Ok(None);
170 };
171 g1t_kit::call(
172 identity,
173 "user_for_git_credentials",
174 &GitCredentialsArgs {
175 username: username.to_owned(),
176 secret: secret.to_owned(),
177 },
178 )
179 .await
180}
181
182/// Standard base64 to a UTF-8 string, or `None` if either step fails.
183fn decode_base64(input: &str) -> Option<String> {
184 let mut bytes = Vec::with_capacity(input.len() * 3 / 4);
185 let mut buffer = 0u32;
186 let mut bits = 0;
187 for byte in input.bytes().filter(|byte| *byte != b'=') {
188 let value = match byte {
189 b'A'..=b'Z' => byte - b'A',
190 b'a'..=b'z' => byte - b'a' + 26,
191 b'0'..=b'9' => byte - b'0' + 52,
192 b'+' => 62,
193 b'/' => 63,
194 _ => return None,
195 };
196 buffer = (buffer << 6) | u32::from(value);
197 bits += 6;
198 if bits >= 8 {
199 bits -= 8;
200 bytes.push((buffer >> bits) as u8);
201 }
202 }
203 String::from_utf8(bytes).ok()
204}
205
206/// The response for a refused git request. Anonymous callers are asked to
207/// authenticate, which is what makes git prompt for credentials.
208pub fn refuse<T>(outcome: Outcome<T>) -> Result<Response> {
209 let Outcome::Fail(failure) = outcome else {
210 return Response::error("Not found", 404);
211 };
212 let mut response = Response::error(failure.message, failure.code.http_status())?;
213 if failure.code == FailureCode::Unauthenticated {
214 response
215 .headers_mut()
216 .set("www-authenticate", "Basic realm=\"g1t\"")?;
217 }
218 Ok(response)
219}
220
221/// `url` with its first path segment, the workspace, replaced by `slug`.
222pub fn with_namespace(url: &Url, slug: &str) -> Option<String> {
223 let rest = url.path().strip_prefix('/')?.split_once('/')?.1;
224 let mut moved = url.clone();
225 moved.set_path(&format!("/{slug}/{rest}"));
226 Some(moved.to_string())
227}
228
229/// Where a git request for a renamed workspace's old address should go
230/// now, if its first segment is an old slug that still redirects.
231pub async fn renamed(url: &Url, identity: &Fetcher) -> Result<Option<String>> {
232 let Some(old) = url.path().strip_prefix('/').and_then(|path| path.split('/').next()) else {
233 return Ok(None);
234 };
235 let current: Option<String> = g1t_kit::call(
236 identity,
237 "resolve_slug",
238 &g1t_contracts::identity::SlugArgs {
239 slug: old.to_owned(),
240 },
241 )
242 .await?;
243 Ok(current.and_then(|slug| with_namespace(url, &slug)))
244}
245
246/// The request under the workspace its first segment is an alias of
247/// (identity's aliases.rs: `g1t` for `flagon-io`), if it is one. Answered
248/// in place rather than redirected: a push does not follow a redirect.
249pub async fn aliased(git: &GitRequest, identity: &Fetcher) -> Result<Option<GitRequest>> {
250 let slug: Option<String> = g1t_kit::call(
251 identity,
252 "resolve_alias",
253 &g1t_contracts::identity::SlugArgs {
254 slug: git.path.namespace.clone(),
255 },
256 )
257 .await?;
258 Ok(slug.map(|slug| git.under(&slug)))
259}
260
261/// `url` with its repository, the first two path segments, replaced by
262/// `to`: where a request for a transferred repository's old path goes.
263/// Keeps whether the old address ended in `.git`.
264pub fn transferred(url: &Url, to: &RepoPath) -> Option<String> {
265 let path = url.path().strip_prefix('/')?;
266 let mut segments = path.splitn(3, '/');
267 let (_, name, rest) = (segments.next()?, segments.next()?, segments.next()?);
268 let suffix = if name.ends_with(".git") { ".git" } else { "" };
269 let mut moved = url.clone();
270 moved.set_path(&format!("/{}/{}{suffix}/{rest}", to.namespace, to.name));
271 Some(moved.to_string())
272}
273
274/// A permanent redirect: 301 for git's first request for refs, which it
275/// follows and then uses the new address for the rest; 308 for the
276/// others, so a POST stays a POST.
277pub fn moved(location: &str, get: bool) -> Result<Response> {
278 let mut response = Response::empty()?.with_status(if get { 301 } else { 308 });
279 response.headers_mut().set("location", location)?;
280 Ok(response)
281}
282
283const ZERO_ID: &str = "0000000000000000000000000000000000000000";
284const HEADS: &str = "refs/heads/";
285const TAGS: &str = "refs/tags/";
286
287/// One ref a push asks to change.
288struct Command {
289 old: String,
290 new: String,
291 name: String,
292}
293
294/// The commands at the start of a receive-pack request, and the
295/// capabilities the client sent with the first of them.
296fn commands(body: &[u8]) -> (Vec<Command>, String) {
297 let mut commands = Vec::new();
298 let mut capabilities = String::new();
299 let mut position = 0;
300 // Commands are pkt-lines; a flush packet ends them and the pack follows.
301 while let Some(length) = body
302 .get(position..position + 4)
303 .and_then(|hex| std::str::from_utf8(hex).ok())
304 .and_then(|hex| usize::from_str_radix(hex, 16).ok())
305 {
306 if length < 4 || position + length > body.len() {
307 break;
308 }
309 let line = &body[position + 4..position + length];
310 position += length;
311 // `<old> <new> <ref>`, and on the first command a NUL then capabilities.
312 let mut halves = line.splitn(2, |byte| *byte == 0);
313 let command = halves.next().unwrap_or_default();
314 if let Some(rest) = halves.next() {
315 capabilities = String::from_utf8_lossy(rest).trim().to_owned();
316 }
317 let Ok(command) = std::str::from_utf8(command) else {
318 continue;
319 };
320 let mut parts = command.trim_end().splitn(3, ' ');
321 if let (Some(old), Some(new), Some(name)) = (parts.next(), parts.next(), parts.next()) {
322 commands.push(Command {
323 old: old.to_owned(),
324 new: new.to_owned(),
325 name: name.to_owned(),
326 });
327 }
328 }
329 (commands, capabilities)
330}
331
332/// The bytes of the pack a receive-pack request carries: everything after
333/// the flush packet that ends its commands. Zero for a push that only
334/// deletes refs.
335pub(crate) fn pack_bytes(body: &[u8]) -> u64 {
336 let mut position = 0;
337 while let Some(length) = body
338 .get(position..position + 4)
339 .and_then(|hex| std::str::from_utf8(hex).ok())
340 .and_then(|hex| usize::from_str_radix(hex, 16).ok())
341 {
342 if length == 0 {
343 return (body.len() - position - 4) as u64;
344 }
345 if length < 4 || position + length > body.len() {
346 break;
347 }
348 position += length;
349 }
350 0
351}
352
353fn pkt_line(payload: &[u8]) -> Vec<u8> {
354 let mut line = format!("{:04x}", payload.len() + 4).into_bytes();
355 line.extend_from_slice(payload);
356 line
357}
358
359/// The branches and tags a push asks to change, as rules see them: the
360/// full ref, where it pointed (`None`: it is created) and where it will
361/// (`None`: it is deleted).
362pub(crate) fn ref_updates(body: &[u8]) -> Vec<(String, Option<String>, Option<String>)> {
363 commands(body)
364 .0
365 .into_iter()
366 .filter(|command| command.name.starts_with(HEADS) || command.name.starts_with(TAGS))
367 .map(|Command { old, new, name }| (name, (old != ZERO_ID).then_some(old), (new != ZERO_ID).then_some(new)))
368 .collect()
369}
370
371/// A report-status answer, as git expects it.
372pub(crate) fn report_response(report: Vec<u8>) -> Result<Response> {
373 let headers = Headers::new();
374 headers.set("content-type", "application/x-git-receive-pack-result")?;
375 headers.set("cache-control", "no-cache")?;
376 Ok(Response::from_bytes(report)?.with_headers(headers))
377}
378
379/// What git is told when a push would change a protected branch: every ref
380/// in it is declined, with the reason against the protected one, so that
381/// git prints it beside the branch. `None` if the push leaves the branch
382/// alone, or creates it in a repository that does not have it yet. Only
383/// where rulesets cannot be read (an installation without the work
384/// service); rulesets decide everywhere else (rules.rs).
385pub(crate) fn refusal(body: &[u8], protected: &str) -> Option<Vec<u8>> {
386 let (commands, capabilities) = commands(body);
387 let reference = format!("{HEADS}{protected}");
388 if !commands
389 .iter()
390 .any(|command| command.name == reference && command.old != ZERO_ID)
391 {
392 return None;
393 }
394 let mut report = pkt_line(b"unpack ok\n");
395 for command in &commands {
396 let reason = if command.name == reference {
397 format!("{protected} is protected: push a branch and open a pull request")
398 } else {
399 format!("not pushed, because the same push would change {protected}")
400 };
401 report.extend(pkt_line(
402 format!("ng {} {reason}\n", command.name).as_bytes(),
403 ));
404 }
405 report.extend_from_slice(b"0000");
406 Some(framed(report, &capabilities, &[]))
407}
408
409/// A report-status as git expects it: inside channel 1 when the client
410/// asked for side-band, after `messages` on channel 2, which git prints as
411/// `remote:` lines. Without side-band the messages cannot be shown.
412fn framed(report: Vec<u8>, capabilities: &str, messages: &[String]) -> Vec<u8> {
413 let sideband = capabilities
414 .split(' ')
415 .any(|capability| capability.starts_with("side-band"));
416 if !sideband {
417 return report;
418 }
419 let mut body = Vec::new();
420 for message in messages {
421 let mut packet = vec![2u8];
422 packet.extend_from_slice(message.as_bytes());
423 packet.push(b'\n');
424 body.extend(pkt_line(&packet));
425 }
426 // side-band (not -64k) packets carry at most 1000 bytes.
427 for chunk in report.chunks(990) {
428 let mut packet = vec![1u8];
429 packet.extend_from_slice(chunk);
430 body.extend(pkt_line(&packet));
431 }
432 body.extend_from_slice(b"0000");
433 body
434}
435
436/// Declines every ref in a push with `reason`, explaining why in
437/// `messages`: what push protection answers when a push adds a secret.
438pub fn declined(body: &[u8], reason: &str, messages: &[String]) -> Result<Response> {
439 let (commands, capabilities) = commands(body);
440 let mut report = pkt_line(b"unpack ok\n");
441 for command in &commands {
442 report.extend(pkt_line(format!("ng {} {reason}\n", command.name).as_bytes()));
443 }
444 report.extend_from_slice(b"0000");
445 let headers = Headers::new();
446 headers.set("content-type", "application/x-git-receive-pack-result")?;
447 headers.set("cache-control", "no-cache")?;
448 Ok(Response::from_bytes(framed(report, &capabilities, messages))?.with_headers(headers))
449}
450
451/// A branch or tag a push asks to move.
452#[derive(Debug, PartialEq, Eq)]
453pub struct Pushed {
454 /// The full ref: `refs/heads/main`, `refs/tags/v1`.
455 pub git_ref: String,
456 /// Where it pointed before; `None` for a new ref.
457 pub before: Option<String>,
458 pub after: String,
459}
460
461impl Pushed {
462 pub fn branch(&self) -> Option<&str> {
463 self.git_ref.strip_prefix(HEADS)
464 }
465}
466
467/// The branches and tags a push asks to move, read from the commands at the
468/// start of a receive-pack request. Deletions and other refs are left out.
469fn pushed_branches(body: &[u8]) -> Vec<Pushed> {
470 commands(body)
471 .0
472 .into_iter()
473 .filter(|command| command.new != ZERO_ID)
474 .filter(|command| command.name.starts_with(HEADS) || command.name.starts_with(TAGS))
475 .map(|Command { old, new, name }| Pushed {
476 git_ref: name,
477 before: (old != ZERO_ID).then_some(old),
478 after: new,
479 })
480 .collect()
481}
482
483/// The git store's answer, and what the request asked it to change.
484pub struct Forwarded {
485 pub response: Response,
486 /// For a push: the branches and tags it asks to move, and the commits
487 /// to move them to. Whether each moved is for the caller to confirm.
488 pub pushed: Vec<Pushed>,
489 /// For a push: the size of the pack it sent, for the storage meter.
490 pub pack_bytes: u64,
491 /// The bytes sent to the store.
492 pub sent: u64,
493 /// Whether the answer is the store's own (not g1t's, for a store that
494 /// was busy).
495 pub from_store: bool,
496 /// For a push: whether it was too large to scan for secrets first and
497 /// was streamed to the store unscanned (`LargePushes::Unscanned`). Its
498 /// `git.push` events say so, and security scans it after it lands.
499 pub unscanned: bool,
500}
501
502/// What became of a git request.
503pub enum Push {
504 Forwarded(Forwarded),
505 /// A push the rules of its branches or tags refuse (rules.rs), answered
506 /// here without reaching the store.
507 Refused(Response),
508 /// A push that adds a secret nobody allowed, answered the same way.
509 Blocked(Response),
510 /// A push the store could not hold: an object or the repository too
511 /// large, or too large to check. With the reason, for the audit log.
512 Declined(Response, String),
513}
514
515/// What a push may bring, checked as it arrives (pack_limits.rs).
516#[derive(Clone, Copy, Debug)]
517pub struct PushLimits {
518 /// The largest object the store holds.
519 pub max_object: u64,
520 /// What the repository holds now, as g1t counts it.
521 pub held: u64,
522 /// The most a repository may hold.
523 pub repo_limit: u64,
524 /// The largest push that is read whole and scanned for secrets.
525 pub scan_cap: usize,
526 /// What happens to a larger one.
527 pub large: LargePushes,
528}
529
530impl Default for PushLimits {
531 fn default() -> Self {
532 PushLimits {
533 max_object: crate::pack_limits::MAX_OBJECT_BYTES,
534 held: 0,
535 repo_limit: crate::pack_limits::DEFAULT_REPO_LIMIT_BYTES,
536 scan_cap: crate::secret_scan::MAX_SCANNED_PUSH,
537 large: LargePushes::Refuse,
538 }
539 }
540}
541
542/// What happens to a push larger than [`PushLimits::scan_cap`]: set by
543/// `LARGE_PUSHES`.
544#[derive(Clone, Copy, Debug, PartialEq, Eq)]
545pub enum LargePushes {
546 /// Declined (the default): push protection cannot read it, so it does
547 /// not let it in.
548 Refuse,
549 /// Streamed to the store without a scan for secrets; the size limits are
550 /// still checked as it passes.
551 Unscanned,
552}
553
554impl LargePushes {
555 pub fn from_var(value: Option<&str>) -> Self {
556 match value.map(str::trim) {
557 Some("unscanned") => LargePushes::Unscanned,
558 _ => LargePushes::Refuse,
559 }
560 }
561}
562
563/// A push the store could not hold.
564#[derive(Clone, Debug, PartialEq, Eq)]
565pub enum SizeViolation {
566 Object { size: u64 },
567 Repository { held: u64, incoming: u64, limit: u64 },
568 Unscannable { size: u64, cap: usize },
569}
570
571/// Why a push is declined for its size, as git shows it: the `ng` reason,
572/// and the lines printed as `remote:`.
573pub fn size_refusal(violation: &SizeViolation) -> (String, Vec<String>) {
574 use crate::pack_limits::{MAX_OBJECT_BYTES, PLATFORM_BODY_LIMIT_BYTES, megabytes};
575 match violation {
576 SizeViolation::Object { size } => (
577 format!("a file of {} is over the {} limit", megabytes(*size), megabytes(MAX_OBJECT_BYTES)),
578 vec![
579 format!("g1t stores files of up to {} each; this push has one of {}.", megabytes(MAX_OBJECT_BYTES), megabytes(*size)),
580 "Take it out of the commits (git rm --cached, then amend or rebase), and keep large".to_owned(),
581 "files elsewhere: https://docs.g1t.sh/guides/git/#size-limits. Nothing was pushed.".to_owned(),
582 ],
583 ),
584 SizeViolation::Repository { held, incoming, limit } => (
585 "the repository would be over its size limit".to_owned(),
586 vec![
587 format!(
588 "This repository holds about {} and the push adds {}, past the {} a repository may hold.",
589 megabytes(*held),
590 megabytes(*incoming),
591 megabytes(*limit)
592 ),
593 "Delete what you no longer need, or split it: https://docs.g1t.sh/guides/git/#size-limits.".to_owned(),
594 "Nothing was pushed.".to_owned(),
595 ],
596 ),
597 SizeViolation::Unscannable { size, cap } => (
598 "the push is too large to check for secrets".to_owned(),
599 vec![
600 format!(
601 "g1t checks every push for secrets and reads up to {} at once; this one is {}.",
602 megabytes(*cap as u64),
603 megabytes(*size)
604 ),
605 "Push in parts, oldest commits first, then push as usual:".to_owned(),
606 " git rev-list --reverse HEAD | awk 'NR % 500 == 0' | xargs -I{} git push origin {}:refs/heads/main".to_owned(),
607 format!("A push over {} is refused by the network before it reaches g1t (HTTP 413).", megabytes(PLATFORM_BODY_LIMIT_BYTES)),
608 "See https://docs.g1t.sh/guides/git/#size-limits. Nothing was pushed.".to_owned(),
609 ],
610 ),
611 }
612}
613
614/// Feeds the next chunk of a push to the size check; the first violation.
615fn check_size(sizer: &mut Option<PackSizer>, chunk: &[u8], limits: &PushLimits) -> Option<SizeViolation> {
616 let walker = sizer.as_mut()?;
617 match walker.feed(chunk) {
618 Ok(()) => {}
619 Err(Violation::ObjectTooLarge { size }) => return Some(SizeViolation::Object { size }),
620 Err(Violation::Malformed(why)) => {
621 // Not for g1t to judge: the store will say.
622 worker::console_error!("push not checked for size: {why}");
623 *sizer = None;
624 return None;
625 }
626 }
627 let incoming = walker.pack_bytes();
628 crate::pack_limits::over_repo_limit(limits.held, incoming, limits.repo_limit).then_some(SizeViolation::Repository {
629 held: limits.held,
630 incoming,
631 limit: limits.repo_limit,
632 })
633}
634
635/// A request body that streams from `stream`, for `fetch`.
636pub(crate) fn stream_body<S>(stream: S) -> Result<JsValue>
637where
638 S: futures_util::TryStream + 'static,
639 S::Ok: Into<Vec<u8>>,
640 S::Error: Into<worker::Error>,
641{
642 let response: worker::web_sys::Response = Response::from_stream(stream)?.into();
643 Ok(response.body().map_or(JsValue::NULL, Into::into))
644}
645
646/// What git is told when the store is busy: 429 or 503, with when to try
647/// again (resilience.rs).
648pub fn busy_response(busy: Busy) -> Result<Response> {
649 let response = Response::error(busy.message(), busy.status())?;
650 response.headers().set("retry-after", &busy.retry_after.to_string())?;
651 Ok(response)
652}
653
654/// The rest of a request's body, read and thrown away so that git hears
655/// the answer; how many bytes it was.
656async fn drain(stream: &mut worker::ByteStream) -> Result<u64> {
657 let mut size = 0;
658 while let Some(chunk) = stream.next().await {
659 size += chunk?.len() as u64;
660 }
661 Ok(size)
662}
663
664/// One packet of a pkt-line stream: data, or a flush (`0000`), delimiter
665/// (`0001`) or response-end (`0002`) packet, kept as its four bytes.
666#[derive(Debug, PartialEq, Eq)]
667enum Packet {
668 Data(Vec<u8>),
669 Special([u8; 4]),
670}
671
672/// The packets in `bytes`, or `None` if it is not a whole pkt-line stream.
673fn packets(bytes: &[u8]) -> Option<Vec<Packet>> {
674 let mut out = Vec::new();
675 let mut position = 0;
676 while position < bytes.len() {
677 let header = bytes.get(position..position + 4)?;
678 let length = usize::from_str_radix(std::str::from_utf8(header).ok()?, 16).ok()?;
679 if length < 4 {
680 out.push(Packet::Special(header.try_into().ok()?));
681 position += 4;
682 continue;
683 }
684 out.push(Packet::Data(bytes.get(position + 4..position + length)?.to_vec()));
685 position += length;
686 }
687 Some(out)
688}
689
690fn encode(packets: &[Packet]) -> Vec<u8> {
691 let mut out = Vec::new();
692 for packet in packets {
693 match packet {
694 Packet::Data(data) => out.extend(pkt_line(data)),
695 Packet::Special(bytes) => out.extend_from_slice(bytes),
696 }
697 }
698 out
699}
700
701/// A ref advertisement (`info/refs` for upload-pack) or a protocol v2
702/// `ls-refs` answer with `HEAD` pointing at `branch`, the repository's
703/// default branch as g1t keeps it, so a clone checks it out. The git store
704/// holds the HEAD it was created with; g1t can change the default branch
705/// since. `None` when there is nothing to change: no `HEAD` line, `HEAD`
706/// already names `branch`, or `branch` is not advertised.
707pub fn with_head(body: &[u8], branch: &str) -> Option<Vec<u8>> {
708 let mut packets = packets(body)?;
709 let target = format!("{HEADS}{branch}");
710 let oid = packets.iter().find_map(|packet| {
711 let Packet::Data(data) = packet else { return None };
712 let line = data.split(|byte| *byte == 0).next()?;
713 let line = std::str::from_utf8(line).ok()?.trim_end();
714 let (oid, name) = line.split_once(' ')?;
715 // v2 lines may carry attributes after the name.
716 let name = name.split(' ').next()?;
717 (name == target).then(|| oid.to_owned())
718 })?;
719 let mut changed = false;
720 for packet in &mut packets {
721 let Packet::Data(data) = packet else { continue };
722 let text = String::from_utf8_lossy(data).into_owned();
723 let Some((_, rest)) = text.split_once(' ') else { continue };
724 if !(rest.starts_with("HEAD\0") || rest.starts_with("HEAD\n") || rest.starts_with("HEAD ") || rest == "HEAD") {
725 continue;
726 }
727 let mut line = format!("{oid} {rest}");
728 // v0: `symref=HEAD:refs/heads/<old>` among the capabilities.
729 // v2: `symref-target:refs/heads/<old>` after the name.
730 for marker in ["symref=HEAD:", "symref-target:"] {
731 if let Some(at) = line.find(marker) {
732 let start = at + marker.len();
733 let end = line[start..]
734 .find([' ', '\n', '\0'])
735 .map_or(line.len(), |offset| start + offset);
736 line.replace_range(start..end, &target);
737 }
738 }
739 changed = line != text;
740 if changed {
741 *data = line.into_bytes();
742 }
743 break;
744 }
745 changed.then(|| encode(&packets))
746}
747
748/// The capability that asks git to send a push's pack whole: every delta's
749/// base inside it, none left for the receiving end to find (a "thin" pack).
750const NO_THIN: &[u8] = b"no-thin";
751
752/// The receive-pack ref advertisement (`info/refs?service=git-receive-pack`,
753/// protocol v0 or v1) with `no-thin` among its capabilities, so that git
754/// sends a pack whose deltas have their bases in it (git's `send-pack`
755/// turns thin packs off when the server says `no-thin`). Push protection
756/// and the rules then read every object from the pack, and none from the
757/// store (secret_scan.rs `supply_bases`). The capabilities follow the NUL
758/// on the first ref line, or on the `capabilities^{}` line of an empty
759/// repository. `None` when there is nothing to change or it cannot be
760/// changed safely: `no-thin` is there already, or the answer is not a whole
761/// pkt-line advertisement in that shape. Never for upload-pack.
762pub fn with_no_thin(body: &[u8]) -> Option<Vec<u8>> {
763 let mut packets = packets(body)?;
764 // Past the `# service=` line and its flush, and v1's `version 1`: the
765 // first ref line, which carries the capabilities.
766 let line = packets.iter_mut().find_map(|packet| match packet {
767 Packet::Data(data) if !data.starts_with(b"# service=") && !data.starts_with(b"version ") => Some(data),
768 _ => None,
769 })?;
770 let nul = line.iter().position(|byte| *byte == 0)?;
771 // `<oid> <ref>` before the NUL: 40 (SHA-1) or 64 (SHA-256) hex digits.
772 let (oid, name) = std::str::from_utf8(&line[..nul]).ok()?.split_once(' ')?;
773 if !matches!(oid.len(), 40 | 64) || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) || name.is_empty() {
774 return None;
775 }
776 let end = if line.ends_with(b"\n") { line.len() - 1 } else { line.len() };
777 let capabilities = &line[nul + 1..end];
778 if capabilities.split(|byte| *byte == b' ').any(|capability| capability == NO_THIN) {
779 return None;
780 }
781 let mut added = Vec::with_capacity(NO_THIN.len() + 1);
782 if !capabilities.is_empty() && !capabilities.ends_with(b" ") {
783 added.push(b' ');
784 }
785 added.extend_from_slice(NO_THIN);
786 // A pkt-line holds at most 65516 bytes of data.
787 if line.len() + added.len() > 65516 {
788 return None;
789 }
790 line.splice(end..end, added);
791 Some(encode(&packets))
792}
793
794/// Whether a request to the git store is one whose answer names `HEAD`:
795/// the ref advertisement for a fetch, or a protocol v2 `ls-refs`.
796fn names_head(git: &GitRequest, body: Option<&[u8]>) -> bool {
797 if git.service != GitService::UploadPack {
798 return false;
799 }
800 match body {
801 None => git.endpoint == "info/refs",
802 Some(body) => {
803 git.endpoint == "git-upload-pack"
804 && body.windows(b"command=ls-refs".len()).any(|window| window == b"command=ls-refs")
805 }
806 }
807}
808
809/// Whether a request is a protocol v2 `fetch` still negotiating: it sends
810/// `have` lines and no `done`, so the answer may be acknowledgments only.
811fn negotiating(body: &[u8]) -> bool {
812 let Some(packets) = packets(body) else { return false };
813 let lines: Vec<&[u8]> = packets
814 .iter()
815 .filter_map(|packet| match packet {
816 Packet::Data(data) => Some(data.strip_suffix(b"\n").unwrap_or(data)),
817 Packet::Special(_) => None,
818 })
819 .collect();
820 lines.contains(&b"command=fetch".as_slice())
821 && lines.iter().any(|line| line.starts_with(b"have "))
822 && !lines.contains(&b"done".as_slice())
823}
824
825/// What to do with the start of a store's answer to a negotiating fetch.
826#[derive(Debug, PartialEq, Eq)]
827enum Acknowledged {
828 /// Not enough of it yet to tell.
829 NeedMore,
830 /// Send it on as it is.
831 Whole,
832 /// Acknowledgments without `ready`, followed by more sections: the
833 /// store's answer to keep is these first bytes, ended by a flush.
834 CutAt(usize),
835}
836
837/// How much of the answer to keep. The git store answers a fetch whose
838/// `have`s it does not know with `acknowledgments`, `NAK`, then a pack
839/// anyway; git refuses that ("expected no other sections to be sent after
840/// no 'ready'"), since a server that is not ready must end the response
841/// there and let the client negotiate again. Lines may be `sideband-all`
842/// framed (band 1, `\x01`).
843fn acknowledged(head: &[u8]) -> Acknowledged {
844 let mut position = 0;
845 let mut first = true;
846 loop {
847 let Some(header) = head.get(position..position + 4) else { return Acknowledged::NeedMore };
848 let Some(length) = std::str::from_utf8(header).ok().and_then(|hex| usize::from_str_radix(hex, 16).ok()) else {
849 return Acknowledged::Whole;
850 };
851 if length < 4 {
852 // The acknowledgments section's end: a delimiter means more
853 // sections follow, which only `ready` allows.
854 return match (first, header) {
855 (false, b"0001") => Acknowledged::CutAt(position),
856 _ => Acknowledged::Whole,
857 };
858 }
859 let Some(payload) = head.get(position + 4..position + length) else { return Acknowledged::NeedMore };
860 let line = payload.strip_prefix(b"\x01").unwrap_or(payload);
861 let line = line.strip_suffix(b"\n").unwrap_or(line);
862 if first && line != b"acknowledgments" {
863 return Acknowledged::Whole;
864 }
865 if line == b"ready" {
866 return Acknowledged::Whole;
867 }
868 first = false;
869 position += length;
870 }
871}
872
873/// The answer to a negotiating fetch, with the sections the store sent
874/// after acknowledgments without `ready` left off (see [`acknowledged`]).
875/// Reads only the start of the answer; the rest streams through.
876async fn without_early_pack(mut response: Response) -> Result<Response> {
877 const LOOK: usize = 64 * 1024;
878 let headers = response.headers().clone();
879 headers.delete("content-length")?;
880 let mut stream = response.stream()?;
881 let mut head = Vec::new();
882 loop {
883 match acknowledged(&head) {
884 Acknowledged::CutAt(at) => {
885 head.truncate(at);
886 // A flush ends the acknowledgments and the answer. No
887 // response-end packet: git's HTTP transport adds its own
888 // and refuses one from the server.
889 head.extend_from_slice(b"0000");
890 return Ok(Response::from_bytes(head)?.with_headers(headers));
891 }
892 Acknowledged::Whole => break,
893 Acknowledged::NeedMore if head.len() >= LOOK => break,
894 Acknowledged::NeedMore => match stream.next().await {
895 Some(chunk) => head.extend_from_slice(&chunk?),
896 None => break,
897 },
898 }
899 }
900 let rest = futures_util::stream::once(async move { Ok::<Vec<u8>, worker::Error>(head) }).chain(stream);
901 Ok(Response::from_stream(rest)?.with_headers(headers))
902}
903
904/// Sends the request on to the git store and returns its response as is,
905/// unless it is a push `checks` refuse (the workflow gate, the rules, push
906/// protection: push_checks.rs), or one the store could not hold (`limits`,
907/// pack_limits.rs). `checks` is given as much of the push as was read,
908/// whether that is all of it, and whether to scan it for secrets. A
909/// fetch's ref listing has its `HEAD` pointed at `default_branch` (see
910/// [`with_head`]). A POST's body is `read` when the caller has read it
911/// already.
912///
913/// A push is read as it arrives: up to `limits.scan_cap` is kept, to be
914/// scanned and sent on whole; past it, the push is declined, or streamed
915/// to the store unscanned (`LargePushes`), never held. Reads the store
916/// fails for a moment (429, 5xx) are tried again with backoff; a push never
917/// is. A store still busy after that is answered 429 or 503 with
918/// `Retry-After`.
919#[allow(clippy::too_many_arguments)]
920pub async fn forward(
921 mut request: Request,
922 read: Option<Vec<u8>>,
923 git: &GitRequest,
924 access: &GitAccess,
925 checks: impl AsyncFnOnce(&[u8], bool, bool) -> Result<Checks>,
926 default_branch: Option<&str>,
927 limits: PushLimits,
928 timing: &mut Timing,
929) -> Result<Push> {
930 let headers = Headers::new();
931 headers.set("authorization", &format!("Bearer {}", access.token))?;
932 for name in FORWARDED_HEADERS {
933 if let Some(value) = request.headers().get(name)? {
934 headers.set(name, &value)?;
935 }
936 }
937 let query = request.url()?.query().map(|query| format!("?{query}")).unwrap_or_default();
938 let url = format!("{}/{}{query}", access.remote, git.endpoint);
939 let method = request.method();
940 // Its own health and breaker: the fallback store's apart from Artifacts'.
941 let namespace = crate::store::health_namespace(&access.remote);
942
943 if method == Method::Post && git.endpoint == "git-receive-pack" {
944 return push(request, &url, headers, checks, limits, &namespace, timing).await;
945 }
946
947 // A read: the ref advertisement, `ls-refs`, or a fetch of objects.
948 let body = match (&method, read) {
949 (Method::Post, Some(body)) => Some(body),
950 (Method::Post, None) => Some(request.bytes().await?),
951 _ => None,
952 };
953 let lists_head = match &body {
954 None => method == Method::Get && names_head(git, None),
955 Some(body) => names_head(git, Some(body)),
956 };
957 let sent = body.as_ref().map_or(0, |body| body.len() as u64);
958 let mut attempt = 0;
959 let mut response = loop {
960 let mut init = RequestInit::new();
961 init.with_method(method.clone()).with_headers(headers.clone());
962 if let Some(body) = &body {
963 init.with_body(Some(Uint8Array::from(body.as_slice()).into()));
964 }
965 let started = g1t_kit::now_ms();
966 let answered = Fetch::Request(Request::new_with_init(&url, &init)?).send().await;
967 let ms = g1t_kit::now_ms().saturating_sub(started);
968 let failure = match &answered {
969 Ok(response) => resilience::classify_status(response.status_code()),
970 Err(_) => Some(Failure::Transient),
971 };
972 let outcome = match failure {
973 None => meters::Outcome::Ok,
974 Some(Failure::RateLimited) => meters::Outcome::RateLimited,
975 Some(_) => meters::Outcome::Failed,
976 };
977 meters::record_health(&namespace, outcome, ms);
978 match (answered, failure) {
979 (Ok(response), None) => break response,
980 (answered, Some(failure)) if resilience::retry(failure, attempt) => {
981 drop(answered);
982 let wait = resilience::backoff_ms(failure, attempt, worker::js_sys::Math::random());
983 worker::Delay::from(std::time::Duration::from_millis(wait)).await;
984 attempt += 1;
985 }
986 (_, Some(failure)) => {
987 let busy = Busy { rate_limited: failure == Failure::RateLimited, retry_after: 5, read_only: false };
988 return Ok(Push::Forwarded(Forwarded {
989 response: busy_response(busy)?,
990 pushed: Vec::new(),
991 pack_bytes: 0,
992 sent,
993 from_store: false,
994 unscanned: false,
995 }));
996 }
997 (Err(error), None) => return Err(error),
998 }
999 };
1000 if let (true, Some(branch)) = (lists_head, default_branch)
1001 && response.status_code() == 200
1002 {
1003 let headers = response.headers().clone();
1004 headers.delete("content-length")?;
1005 let body = response.bytes().await?;
1006 let body = with_head(&body, branch).unwrap_or(body);
1007 response = Response::from_bytes(body)?.with_headers(headers);
1008 }
1009 // A push's ref advertisement asks for a pack without outside bases.
1010 if method == Method::Get
1011 && git.service == GitService::ReceivePack
1012 && git.endpoint == "info/refs"
1013 && response.status_code() == 200
1014 {
1015 let headers = response.headers().clone();
1016 headers.delete("content-length")?;
1017 let body = response.bytes().await?;
1018 let body = with_no_thin(&body).unwrap_or(body);
1019 response = Response::from_bytes(body)?.with_headers(headers);
1020 }
1021 if git.endpoint == "git-upload-pack"
1022 && response.status_code() == 200
1023 && body.as_deref().is_some_and(negotiating)
1024 {
1025 response = without_early_pack(response).await?;
1026 }
1027 Ok(Push::Forwarded(Forwarded {
1028 response,
1029 pushed: Vec::new(),
1030 pack_bytes: 0,
1031 sent,
1032 from_store: true,
1033 unscanned: false,
1034 }))
1035}
1036
1037/// A receive-pack request; see [`forward`]. Its steps: `recv` (the push
1038/// read), `checks`, `upload` (the store's answer).
1039#[allow(clippy::too_many_arguments)]
1040async fn push(
1041 mut request: Request,
1042 url: &str,
1043 headers: Headers,
1044 checks: impl AsyncFnOnce(&[u8], bool, bool) -> Result<Checks>,
1045 limits: PushLimits,
1046 namespace: &str,
1047 timing: &mut Timing,
1048) -> Result<Push> {
1049 let mut stream = request.stream()?;
1050 let mut head: Vec<u8> = Vec::new();
1051 let mut sizer = Some(PackSizer::new(limits.max_object));
1052 let mut violation = None;
1053 let mut ended = false;
1054 while head.len() <= limits.scan_cap {
1055 match stream.next().await {
1056 Some(chunk) => {
1057 let chunk = chunk?;
1058 if violation.is_none() {
1059 violation = check_size(&mut sizer, &chunk, &limits);
1060 }
1061 head.extend_from_slice(&chunk);
1062 }
1063 None => {
1064 ended = true;
1065 break;
1066 }
1067 }
1068 }
1069 timing.mark("recv");
1070 // The workflow gate and the rules of the branches and tags it changes,
1071 // with push protection alongside for a push read whole that is within
1072 // the size limits (push_checks.rs). What the rules refuse is refused
1073 // whatever else is wrong with it.
1074 let checked = checks(&head, ended, ended && violation.is_none()).await?;
1075 for (part, ms) in checked.spans {
1076 timing.part(part, ms);
1077 }
1078 // Whether the pack came thin: g1t asks for whole ones (`no-thin`, see
1079 // [`with_no_thin`]), so a thin one is a client that ignored it, and its
1080 // bases were read from the store (`read`).
1081 if let Some(bases) = checked.bases {
1082 timing.note("thin", if bases.thin() { "yes" } else { "no" });
1083 if bases.thin() {
1084 let agent = request.headers().get("user-agent").ok().flatten().unwrap_or_default();
1085 worker::console_warn!(
1086 "a thin push from {agent}: {} bases outside the pack, {} asked of the store, {} left unresolved",
1087 bases.missing,
1088 bases.asked,
1089 bases.left
1090 );
1091 }
1092 }
1093 timing.mark("checks");
1094 let blocked = match checked.verdict {
1095 Verdict::Refused(response) => {
1096 if !ended {
1097 drain(&mut stream).await?;
1098 }
1099 return Ok(Push::Refused(response));
1100 }
1101 Verdict::Blocked(response) => Some(response),
1102 Verdict::Clear => None,
1103 };
1104 if !ended && limits.large == LargePushes::Refuse && violation.is_none() {
1105 let size = head.len() as u64 + drain(&mut stream).await?;
1106 violation = Some(SizeViolation::Unscannable { size, cap: limits.scan_cap });
1107 ended = true;
1108 }
1109 if let Some(violation) = violation {
1110 if !ended {
1111 drain(&mut stream).await?;
1112 }
1113 let (reason, messages) = size_refusal(&violation);
1114 return Ok(Push::Declined(declined(&head, &reason, &messages)?, reason));
1115 }
1116 let pushed = pushed_branches(&head);
1117 let mut init = RequestInit::new();
1118 init.with_method(Method::Post).with_headers(headers);
1119 let started = g1t_kit::now_ms();
1120 let (answered, pack_bytes, sent, unscanned) = if ended {
1121 if let Some(response) = blocked {
1122 return Ok(Push::Blocked(response));
1123 }
1124 let pack = pack_bytes(&head);
1125 let sent = head.len() as u64;
1126 init.with_body(Some(Uint8Array::from(head.as_slice()).into()));
1127 drop(head);
1128 (Fetch::Request(Request::new_with_init(url, &init)?).send().await, pack, sent, false)
1129 } else {
1130 // Larger than can be scanned, and let through unscanned: streamed,
1131 // with the size limits checked as it passes. A violation ends the
1132 // stream before the pack does, so the store refuses it whole.
1133 worker::console_warn!("a push of more than {} bytes goes to the store unscanned", limits.scan_cap);
1134 let commands = head.iter().take(64 * 1024).copied().collect::<Vec<u8>>();
1135 let found: Rc<RefCell<Option<SizeViolation>>> = Rc::default();
1136 let walked = Rc::new(RefCell::new((sizer, 0u64)));
1137 let rest = {
1138 let found = found.clone();
1139 let walked = walked.clone();
1140 stream.map(move |chunk| {
1141 let chunk = chunk?;
1142 let mut walked = walked.borrow_mut();
1143 walked.1 += chunk.len() as u64;
1144 if let Some(violation) = check_size(&mut walked.0, &chunk, &limits) {
1145 *found.borrow_mut() = Some(violation);
1146 return Err(worker::Error::RustError("push over the size limit".into()));
1147 }
1148 Ok(chunk)
1149 })
1150 };
1151 let first = head.len() as u64;
1152 let body = futures_util::stream::once(async move { Ok::<Vec<u8>, worker::Error>(head) }).chain(rest);
1153 init.with_body(Some(stream_body(body)?));
1154 let answered = Fetch::Request(Request::new_with_init(url, &init)?).send().await;
1155 if let Some(violation) = found.borrow_mut().take() {
1156 let (reason, messages) = size_refusal(&violation);
1157 return Ok(Push::Declined(declined(&commands, &reason, &messages)?, reason));
1158 }
1159 let walked = walked.borrow();
1160 let pack = walked.0.as_ref().map_or_else(|| pack_bytes(&commands), PackSizer::pack_bytes);
1161 (answered, pack, first + walked.1, true)
1162 };
1163 let ms = g1t_kit::now_ms().saturating_sub(started);
1164 timing.mark("upload");
1165 let failure = match &answered {
1166 Ok(response) => resilience::classify_status(response.status_code()),
1167 Err(_) => Some(Failure::Transient),
1168 };
1169 meters::record_health(
1170 namespace,
1171 match failure {
1172 None => meters::Outcome::Ok,
1173 Some(Failure::RateLimited) => meters::Outcome::RateLimited,
1174 Some(_) => meters::Outcome::Failed,
1175 },
1176 ms,
1177 );
1178 // A push is never tried again: the store may have taken it.
1179 let response = match (answered, failure) {
1180 (Ok(response), None) => response,
1181 (Ok(response), Some(Failure::RateLimited)) => {
1182 drop(response);
1183 busy_response(Busy { rate_limited: true, retry_after: 5, read_only: false })?
1184 }
1185 (Ok(response), Some(_)) => response,
1186 (Err(error), _) => {
1187 worker::console_error!("a push did not reach the store: {error}");
1188 busy_response(Busy { rate_limited: false, retry_after: 5, read_only: false })?
1189 }
1190 };
1191 Ok(Push::Forwarded(Forwarded {
1192 response,
1193 pushed,
1194 pack_bytes,
1195 sent,
1196 from_store: true,
1197 unscanned,
1198 }))
1199}
1200
1201#[cfg(test)]
1202mod tests {
1203 use super::{Acknowledged, GitService, Pushed, RepoPath, Url, ZERO_ID, acknowledged, framed, negotiating, pack_bytes, parse, pushed_branches, refusal, server_timing, transferred, with_head, with_namespace, with_no_thin};
1204
1205 #[test]
1206 fn server_timing_names_each_step_and_the_total() {
1207 assert_eq!(
1208 server_timing(&[("repo", 12), ("token", 0), ("store", 140)], &[], &[], 153),
1209 "repo;dur=12, token;dur=0, store;dur=140, total;dur=153"
1210 );
1211 assert_eq!(server_timing(&[], &[], &[], 3), "total;dur=3");
1212 // A push's checks run side by side: their parts come after the steps.
1213 assert_eq!(
1214 server_timing(&[("recv", 30), ("checks", 120), ("upload", 300)], &[("read", 40), ("rules", 110), ("scan", 90)], &[], 450),
1215 "recv;dur=30, checks;dur=120, upload;dur=300, read;dur=40, rules;dur=110, scan;dur=90, total;dur=450"
1216 );
1217 assert_eq!(
1218 server_timing(&[("repo", 1), ("cache", 2)], &[], &[("refs", "hit-colo")], 4),
1219 "repo;dur=1, cache;dur=2, refs;desc=hit-colo, total;dur=4"
1220 );
1221 }
1222
1223 #[test]
1224 fn a_renamed_repository_redirects_to_its_new_name() {
1225 // A rename keeps the old path in the same table as a transfer, so
1226 // the old remote is sent to the new name the same way.
1227 let to = RepoPath {
1228 namespace: "acme".into(),
1229 name: "booster".into(),
1230 };
1231 let url = Url::parse("https://g1t.sh/acme/rocket.git/info/refs?service=git-upload-pack").unwrap();
1232 assert_eq!(
1233 transferred(&url, &to).as_deref(),
1234 Some("https://g1t.sh/acme/booster.git/info/refs?service=git-upload-pack")
1235 );
1236 }
1237
1238 #[test]
1239 fn head_follows_the_default_branch_in_a_v0_advertisement() {
1240 let main = "1111111111111111111111111111111111111111";
1241 let trunk = "2222222222222222222222222222222222222222";
1242 let body = [
1243 pkt("# service=git-upload-pack\n"),
1244 b"0000".to_vec(),
1245 pkt(&format!("{main} HEAD\0multi_ack symref=HEAD:refs/heads/main agent=git/2\n")),
1246 pkt(&format!("{main} refs/heads/main\n")),
1247 pkt(&format!("{trunk} refs/heads/trunk\n")),
1248 b"0000".to_vec(),
1249 ]
1250 .concat();
1251 let changed = String::from_utf8(with_head(&body, "trunk").unwrap()).unwrap();
1252 assert!(changed.contains(&format!("{trunk} HEAD\0multi_ack symref=HEAD:refs/heads/trunk agent=git/2\n")));
1253 assert!(changed.contains(&format!("{main} refs/heads/main\n")));
1254 assert!(changed.starts_with("001e# service=git-upload-pack\n0000"));
1255 // Already right, or a branch it does not have: left alone.
1256 assert!(with_head(&body, "main").is_none());
1257 assert!(with_head(&body, "gone").is_none());
1258 }
1259
1260 #[test]
1261 fn a_push_advertisement_asks_for_a_pack_without_outside_bases() {
1262 let main = "1111111111111111111111111111111111111111";
1263 let topic = "2222222222222222222222222222222222222222";
1264 let body = [
1265 pkt("# service=git-receive-pack\n"),
1266 b"0000".to_vec(),
1267 pkt(&format!("{main} refs/heads/main\0report-status delete-refs side-band-64k quiet ofs-delta agent=git/2.45\n")),
1268 pkt(&format!("{topic} refs/heads/topic\n")),
1269 b"0000".to_vec(),
1270 ]
1271 .concat();
1272 let changed = with_no_thin(&body).unwrap();
1273 let expected = [
1274 pkt("# service=git-receive-pack\n"),
1275 b"0000".to_vec(),
1276 pkt(&format!("{main} refs/heads/main\0report-status delete-refs side-band-64k quiet ofs-delta agent=git/2.45 no-thin\n")),
1277 pkt(&format!("{topic} refs/heads/topic\n")),
1278 b"0000".to_vec(),
1279 ]
1280 .concat();
1281 assert_eq!(String::from_utf8(changed.clone()).unwrap(), String::from_utf8(expected).unwrap());
1282 // The length of the line that grew is its new one: the whole parses.
1283 assert!(super::packets(&changed).is_some());
1284 // Said once: an answer that has it already is left alone.
1285 assert!(with_no_thin(&changed).is_none());
1286
1287 // Protocol v1 begins with `version 1`.
1288 let v1 = [
1289 pkt("# service=git-receive-pack\n"),
1290 b"0000".to_vec(),
1291 pkt("version 1\n"),
1292 pkt(&format!("{main} refs/heads/main\0report-status ofs-delta\n")),
1293 b"0000".to_vec(),
1294 ]
1295 .concat();
1296 let changed = String::from_utf8(with_no_thin(&v1).unwrap()).unwrap();
1297 assert!(changed.contains(&String::from_utf8(pkt(&format!("{main} refs/heads/main\0report-status ofs-delta no-thin\n"))).unwrap()));
1298 assert!(changed.contains("000eversion 1\n"));
1299 }
1300
1301 #[test]
1302 fn an_empty_repository_advertisement_asks_for_a_whole_pack_too() {
1303 let zero = "0000000000000000000000000000000000000000";
1304 let body = [
1305 pkt("# service=git-receive-pack\n"),
1306 b"0000".to_vec(),
1307 pkt(&format!("{zero} capabilities^{{}}\0report-status delete-refs ofs-delta\n")),
1308 b"0000".to_vec(),
1309 ]
1310 .concat();
1311 let changed = String::from_utf8(with_no_thin(&body).unwrap()).unwrap();
1312 assert!(changed.contains(&String::from_utf8(pkt(&format!("{zero} capabilities^{{}}\0report-status delete-refs ofs-delta no-thin\n"))).unwrap()));
1313 // No capabilities at all, and no newline: still one list.
1314 let bare = [pkt("# service=git-receive-pack\n"), b"0000".to_vec(), pkt(&format!("{zero} capabilities^{{}}\0")), b"0000".to_vec()].concat();
1315 let changed = String::from_utf8(with_no_thin(&bare).unwrap()).unwrap();
1316 assert!(changed.contains(&String::from_utf8(pkt(&format!("{zero} capabilities^{{}}\0no-thin"))).unwrap()));
1317 }
1318
1319 #[test]
1320 fn an_advertisement_that_cannot_be_read_goes_through_untouched() {
1321 let main = "1111111111111111111111111111111111111111";
1322 // Not pkt-lines; a length past the end; an error page.
1323 assert!(with_no_thin(b"not a git answer").is_none());
1324 assert!(with_no_thin(b"00ff1111").is_none());
1325 assert!(with_no_thin(b"<html>503 Service Unavailable</html>").is_none());
1326 assert!(with_no_thin(b"").is_none());
1327 // A first ref line without capabilities, or without an object id.
1328 let without = [pkt("# service=git-receive-pack\n"), b"0000".to_vec(), pkt(&format!("{main} refs/heads/main\n")), b"0000".to_vec()].concat();
1329 assert!(with_no_thin(&without).is_none());
1330 let unnamed = [pkt("# service=git-receive-pack\n"), b"0000".to_vec(), pkt("nothing here\0report-status\n"), b"0000".to_vec()].concat();
1331 assert!(with_no_thin(&unnamed).is_none());
1332 // `no-thin` inside another capability's value is not `no-thin`.
1333 let lookalike = [pkt("# service=git-receive-pack\n"), b"0000".to_vec(), pkt(&format!("{main} refs/heads/main\0agent=no-thin-ish\n")), b"0000".to_vec()].concat();
1334 assert!(String::from_utf8(with_no_thin(&lookalike).unwrap()).unwrap().contains("agent=no-thin-ish no-thin\n"));
1335 }
1336
1337 #[test]
1338 fn head_follows_the_default_branch_in_a_v2_listing() {
1339 let main = "1111111111111111111111111111111111111111";
1340 let trunk = "2222222222222222222222222222222222222222";
1341 let body = [
1342 pkt(&format!("{main} HEAD symref-target:refs/heads/main\n")),
1343 pkt(&format!("{main} refs/heads/main\n")),
1344 pkt(&format!("{trunk} refs/heads/trunk\n")),
1345 b"0000".to_vec(),
1346 ]
1347 .concat();
1348 let changed = String::from_utf8(with_head(&body, "trunk").unwrap()).unwrap();
1349 assert!(changed.starts_with(&String::from_utf8(pkt(&format!("{trunk} HEAD symref-target:refs/heads/trunk\n"))).unwrap()));
1350 assert!(changed.ends_with("0000"));
1351 // Without symrefs asked for, only the commit changes.
1352 let plain = [pkt(&format!("{main} HEAD\n")), pkt(&format!("{trunk} refs/heads/trunk\n")), b"0000".to_vec()].concat();
1353 let changed = String::from_utf8(with_head(&plain, "trunk").unwrap()).unwrap();
1354 assert!(changed.starts_with(&format!("0032{trunk} HEAD\n")));
1355 }
1356
1357 #[test]
1358 fn a_push_is_measured_by_the_pack_after_its_commands() {
1359 let old = "c71546fcd893ef8b0f57388b65e620d759705dda";
1360 let new = "4807077b296e6edbf410d55e72749d3e1170c291";
1361 let pack = b"PACK\0\0\0\x02\0\0\0\0rest-of-pack";
1362 let body = [
1363 pkt(&format!("{old} {new} refs/heads/main\0 report-status\n")),
1364 b"0000".to_vec(),
1365 pack.to_vec(),
1366 ]
1367 .concat();
1368 assert_eq!(pack_bytes(&body), pack.len() as u64);
1369 // Only deletions: no pack.
1370 let body = [pkt(&format!("{old} {ZERO_ID} refs/heads/gone\n")), b"0000".to_vec()].concat();
1371 assert_eq!(pack_bytes(&body), 0);
1372 assert_eq!(pack_bytes(b"garbage"), 0);
1373 }
1374
1375 #[test]
1376 fn a_transferred_repository_keeps_the_rest_of_the_address() {
1377 let to = RepoPath {
1378 namespace: "flagon-io".into(),
1379 name: "g1t".into(),
1380 };
1381 let url = Url::parse("https://g1t.sh/syntaqx/g1t.git/info/refs?service=git-receive-pack").unwrap();
1382 assert_eq!(
1383 transferred(&url, &to).as_deref(),
1384 Some("https://g1t.sh/flagon-io/g1t.git/info/refs?service=git-receive-pack")
1385 );
1386 let url = Url::parse("https://g1t.sh/syntaqx/g1t/git-upload-pack").unwrap();
1387 assert_eq!(
1388 transferred(&url, &to).as_deref(),
1389 Some("https://g1t.sh/flagon-io/g1t/git-upload-pack")
1390 );
1391 }
1392
1393 #[test]
1394 fn an_alias_is_answered_as_its_workspaces_repository() {
1395 for (address, service) in [
1396 ("https://g1t.sh/g1t/g1t.git/info/refs?service=git-upload-pack", GitService::UploadPack),
1397 ("https://g1t.sh/g1t/g1t.git/git-receive-pack", GitService::ReceivePack),
1398 ("https://g1t.sh/g1t/g1t/git-upload-pack", GitService::UploadPack),
1399 ] {
1400 let git = parse(&Url::parse(address).unwrap()).unwrap();
1401 assert_eq!(git.path.namespace, "g1t", "{address}");
1402 let canonical = git.under("flagon-io");
1403 assert_eq!(
1404 canonical.path,
1405 RepoPath {
1406 namespace: "flagon-io".into(),
1407 name: "g1t".into(),
1408 },
1409 "{address}"
1410 );
1411 assert_eq!(canonical.service, service);
1412 assert_eq!(canonical.endpoint, git.endpoint);
1413 }
1414 }
1415
1416 #[test]
1417 fn a_renamed_workspace_keeps_the_rest_of_the_address() {
1418 let url = worker::Url::parse(
1419 "https://g1t.sh/acme/rocket.git/info/refs?service=git-upload-pack",
1420 )
1421 .unwrap();
1422 assert_eq!(
1423 with_namespace(&url, "acme-inc").as_deref(),
1424 Some("https://g1t.sh/acme-inc/rocket.git/info/refs?service=git-upload-pack")
1425 );
1426 let bare = worker::Url::parse("https://g1t.sh/acme").unwrap();
1427 assert_eq!(with_namespace(&bare, "acme-inc"), None);
1428 }
1429
1430 fn pkt(payload: &str) -> Vec<u8> {
1431 format!("{:04x}{payload}", payload.len() + 4).into_bytes()
1432 }
1433
1434 fn joined(parts: &[&[u8]]) -> Vec<u8> {
1435 parts.concat()
1436 }
1437
1438 #[test]
1439 fn a_fetch_with_haves_and_no_done_is_negotiating() {
1440 let request = |lines: &[&str]| {
1441 let mut body = joined(&[&pkt("command=fetch\n"), &pkt("object-format=sha1\n"), b"0001"]);
1442 for line in lines {
1443 body.extend(pkt(&format!("{line}\n")));
1444 }
1445 body.extend(b"0000");
1446 body
1447 };
1448 let want = "want 8407eba58b925619274d012258c2b474a5dbf012";
1449 let have = "have 55cd670a89a80df4fa9d9f0244c44fbd2ed1db8b";
1450 assert!(negotiating(&request(&["deepen 1", want, have])));
1451 assert!(!negotiating(&request(&[want, have, "done"])));
1452 assert!(!negotiating(&request(&[want, "done"])));
1453 let ls_refs = joined(&[&pkt("command=ls-refs\n"), b"0001", &pkt("have nothing\n"), b"0000"]);
1454 assert!(!negotiating(&ls_refs));
1455 }
1456
1457 #[test]
1458 fn acknowledgments_without_ready_end_the_answer() {
1459 // What the store sent a shallow fetch whose only `have` it did not
1460 // know, sideband-all framed: a NAK, then a pack anyway.
1461 let answer = joined(&[
1462 &pkt("\x01acknowledgments\n"),
1463 &pkt("\x01NAK\n"),
1464 b"0001",
1465 &pkt("\x01shallow-info\n"),
1466 &pkt("\x01shallow 8407eba58b925619274d012258c2b474a5dbf012\n"),
1467 b"0001",
1468 &pkt("\x01packfile\n"),
1469 ]);
1470 let cut = joined(&[&pkt("\x01acknowledgments\n"), &pkt("\x01NAK\n")]).len();
1471 assert_eq!(acknowledged(&answer), Acknowledged::CutAt(cut));
1472 // Not yet at the section's end.
1473 assert_eq!(acknowledged(&answer[..cut - 2]), Acknowledged::NeedMore);
1474 assert_eq!(acknowledged(&answer[..cut]), Acknowledged::NeedMore);
1475 // Without sideband framing too.
1476 let plain = joined(&[&pkt("acknowledgments\n"), &pkt("ACK abc\n"), b"0001", &pkt("packfile\n")]);
1477 assert!(matches!(acknowledged(&plain), Acknowledged::CutAt(_)));
1478 }
1479
1480 #[test]
1481 fn a_ready_store_or_a_plain_pack_streams_through() {
1482 let ready = joined(&[
1483 &pkt("\x01acknowledgments\n"),
1484 &pkt("\x01ACK bab14ff1b6d9c4918100098009747d776759a967\n"),
1485 &pkt("\x01ready\n"),
1486 b"0001",
1487 &pkt("\x01packfile\n"),
1488 ]);
1489 assert_eq!(acknowledged(&ready), Acknowledged::Whole);
1490 // Acknowledgments only, ended by a flush: already right.
1491 let only = joined(&[&pkt("acknowledgments\n"), &pkt("NAK\n"), b"0000"]);
1492 assert_eq!(acknowledged(&only), Acknowledged::Whole);
1493 let pack = joined(&[&pkt("\x01packfile\n"), b"0000"]);
1494 assert_eq!(acknowledged(&pack), Acknowledged::Whole);
1495 assert_eq!(acknowledged(b"00"), Acknowledged::NeedMore);
1496 }
1497
1498 #[test]
1499 fn pushed_branches_are_read_from_the_commands() {
1500 let old = "c71546fcd893ef8b0f57388b65e620d759705dda";
1501 let new = "4807077b296e6edbf410d55e72749d3e1170c291";
1502 let body = [
1503 pkt(&format!(
1504 "{old} {new} refs/heads/main\0 report-status side-band-64k\n"
1505 )),
1506 pkt(&format!("{ZERO_ID} {new} refs/heads/feature/x\n")),
1507 pkt(&format!("{old} {ZERO_ID} refs/heads/gone\n")),
1508 pkt(&format!("{ZERO_ID} {new} refs/tags/v1\n")),
1509 b"0000".to_vec(),
1510 b"PACK\0\0\0\x02\0\0\0\0".to_vec(),
1511 ]
1512 .concat();
1513 assert_eq!(
1514 pushed_branches(&body),
1515 [
1516 Pushed {
1517 git_ref: "refs/heads/main".to_owned(),
1518 before: Some(old.to_owned()),
1519 after: new.to_owned()
1520 },
1521 Pushed {
1522 git_ref: "refs/heads/feature/x".to_owned(),
1523 before: None,
1524 after: new.to_owned()
1525 },
1526 Pushed {
1527 git_ref: "refs/tags/v1".to_owned(),
1528 before: None,
1529 after: new.to_owned()
1530 },
1531 ]
1532 );
1533 }
1534
1535 #[test]
1536 fn a_push_to_a_protected_branch_is_declined_with_the_reason() {
1537 let old = "c71546fcd893ef8b0f57388b65e620d759705dda";
1538 let new = "4807077b296e6edbf410d55e72749d3e1170c291";
1539 let body = [
1540 pkt(&format!("{old} {new} refs/heads/main\0 report-status\n")),
1541 pkt(&format!("{ZERO_ID} {new} refs/heads/feature\n")),
1542 b"0000".to_vec(),
1543 ]
1544 .concat();
1545 let report = String::from_utf8(refusal(&body, "main").unwrap()).unwrap();
1546 assert!(report.starts_with("000eunpack ok\n"));
1547 assert!(report.contains("ng refs/heads/main main is protected"));
1548 assert!(report.contains("ng refs/heads/feature not pushed"));
1549 assert!(report.ends_with("0000"));
1550 }
1551
1552 #[test]
1553 fn the_report_is_framed_for_a_client_that_asked_for_side_band() {
1554 let old = "c71546fcd893ef8b0f57388b65e620d759705dda";
1555 let body = [
1556 pkt(&format!(
1557 "{old} {ZERO_ID} refs/heads/main\0 report-status side-band-64k\n"
1558 )),
1559 b"0000".to_vec(),
1560 ]
1561 .concat();
1562 let report = refusal(&body, "main").unwrap();
1563 // A length, then channel 1, then the report itself.
1564 assert_eq!(report[4], 1);
1565 assert_eq!(&report[5..18], b"000eunpack ok");
1566 assert!(report.ends_with(b"00000000"));
1567 }
1568
1569 #[test]
1570 fn other_branches_and_a_first_push_are_let_through() {
1571 let old = "c71546fcd893ef8b0f57388b65e620d759705dda";
1572 let new = "4807077b296e6edbf410d55e72749d3e1170c291";
1573 let feature = [
1574 pkt(&format!("{old} {new} refs/heads/feature\0 report-status\n")),
1575 b"0000".to_vec(),
1576 ]
1577 .concat();
1578 assert!(refusal(&feature, "main").is_none());
1579 // An empty repository has to be able to receive its first commits.
1580 let first = [
1581 pkt(&format!(
1582 "{ZERO_ID} {new} refs/heads/main\0 report-status\n"
1583 )),
1584 b"0000".to_vec(),
1585 ]
1586 .concat();
1587 assert!(refusal(&first, "main").is_none());
1588 }
1589
1590 #[test]
1591 fn a_blocked_push_explains_itself_on_the_progress_channel() {
1592 let report = b"000eunpack ok\n0000".to_vec();
1593 let messages = vec!["g1t found a secret in this push, so nothing was pushed.".to_owned()];
1594 let body = framed(report.clone(), "report-status side-band-64k", &messages);
1595 // Channel 2 first, which git prints as `remote:` lines.
1596 assert_eq!(body[4], 2);
1597 assert!(String::from_utf8_lossy(&body).contains("so nothing was pushed.\n"));
1598 let at = body.windows(5).position(|w| w == b"000eu").unwrap();
1599 assert_eq!(body[at - 1], 1);
1600 assert!(body.ends_with(b"0000"));
1601 // A client without side-band gets the bare report.
1602 assert_eq!(framed(report.clone(), "report-status", &messages), report);
1603 }
1604
1605 #[test]
1606 fn a_fetch_request_names_no_branches() {
1607 assert!(
1608 pushed_branches(b"0032want c71546fcd893ef8b0f57388b65e620d759705dda\n0000").is_empty()
1609 );
1610 }
1611}