| 1 | //! Landing a pull request: moving a repository's branch forward to a commit |
| 2 | //! from a fork, or from another of its own branches. |
| 3 | //! |
| 4 | //! The Artifacts binding cannot write, so this speaks git's smart HTTP |
| 5 | //! protocol directly. It asks the fork for a pack holding exactly the |
| 6 | //! objects the target is missing, then pushes that pack to the target |
| 7 | //! unchanged. No object is parsed or rebuilt along the way. When landing, |
| 8 | //! the pack streams from one to the other as it arrives |
| 9 | //! ([`fast_forward`]): it is never held whole, however large the pull |
| 10 | //! request (pack_limits.rs `Sideband`). |
| 11 | |
| 12 | use std::cell::RefCell; |
| 13 | use std::rc::Rc; |
| 14 | |
| 15 | use futures_util::StreamExt; |
| 16 | use g1t_contracts::repos::GitAccess; |
| 17 | use worker::js_sys::Uint8Array; |
| 18 | use worker::wasm_bindgen::JsValue; |
| 19 | use worker::{Error, Fetch, Headers, Method, Request, RequestInit, Response, Result}; |
| 20 | |
| 21 | use crate::meters; |
| 22 | use crate::pack_limits::Sideband; |
| 23 | |
| 24 | const ZERO_ID: &str = "0000000000000000000000000000000000000000"; |
| 25 | const FLUSH: &[u8] = b"0000"; |
| 26 | |
| 27 | /// Bind publication to the immutable commit approved by the caller. The |
| 28 | /// returned object ID, never a second branch lookup, is used for the push. |
| 29 | pub(crate) fn checked_commit(observed: &str, expected: Option<&str>) -> Option<String> { |
| 30 | expected.is_none_or(|expected| expected == observed).then(|| observed.to_owned()) |
| 31 | } |
| 32 | /// Side-band channels: pack data, and fatal errors. |
| 33 | const PACK_BAND: u8 = 1; |
| 34 | const ERROR_BAND: u8 = 3; |
| 35 | |
| 36 | /// A pack with no objects: what a push that only points a ref at a commit |
| 37 | /// the repository already has sends. |
| 38 | pub const EMPTY_PACK: &[u8] = &[ |
| 39 | b'P', b'A', b'C', b'K', 0, 0, 0, 2, 0, 0, 0, 0, 0x02, 0x9d, 0x08, 0x82, 0x3b, 0xd8, 0xa8, |
| 40 | 0xea, 0xb5, 0x10, 0xad, 0x6a, 0xc7, 0x5c, 0x82, 0x3c, 0xfd, 0x3e, 0xd3, 0x1e, |
| 41 | ]; |
| 42 | |
| 43 | fn pkt_line(payload: &str) -> Vec<u8> { |
| 44 | format!("{:04x}{payload}", payload.len() + 4).into_bytes() |
| 45 | } |
| 46 | |
| 47 | /// The meter for g1t's own request to `service` (meters.rs). |
| 48 | fn meter(service: &str) -> &'static str { |
| 49 | if service == "git-receive-pack" { "internal.git.receive_pack" } else { "internal.git.fetch" } |
| 50 | } |
| 51 | |
| 52 | /// Sends `body` to `service` on the store, answered in full or an error. |
| 53 | async fn send(access: &GitAccess, service: &str, body: JsValue, sent: u64) -> Result<Response> { |
| 54 | let headers = Headers::new(); |
| 55 | headers.set("authorization", &format!("Bearer {}", access.token))?; |
| 56 | headers.set("content-type", &format!("application/x-{service}-request"))?; |
| 57 | headers.set("accept", &format!("application/x-{service}-result"))?; |
| 58 | let mut init = RequestInit::new(); |
| 59 | init.with_method(Method::Post).with_headers(headers).with_body(Some(body)); |
| 60 | let request = Request::new_with_init(&format!("{}/{service}", access.remote), &init)?; |
| 61 | let mut response = Fetch::Request(request).send().await?; |
| 62 | meters::record_remote(meter(service), &access.remote, sent, 0); |
| 63 | if response.status_code() != 200 { |
| 64 | let text = response.text().await.unwrap_or_default(); |
| 65 | return Err(Error::RustError(format!("{service} returned {}: {text}", response.status_code()))); |
| 66 | } |
| 67 | Ok(response) |
| 68 | } |
| 69 | |
| 70 | async fn post(access: &GitAccess, service: &str, body: Vec<u8>) -> Result<Vec<u8>> { |
| 71 | let sent = body.len() as u64; |
| 72 | let mut response = send(access, service, Uint8Array::from(body.as_slice()).into(), sent).await?; |
| 73 | let bytes = response.bytes().await?; |
| 74 | if let Some(key) = crate::store::key_from_remote(&access.remote) { |
| 75 | meters::record_bytes(meter(service), &key, 0, bytes.len() as u64); |
| 76 | } |
| 77 | Ok(bytes) |
| 78 | } |
| 79 | |
| 80 | /// The payloads of the pkt-lines in `bytes`, and the offset where they stop. |
| 81 | pub(crate) fn read_pkt_lines(bytes: &[u8]) -> (Vec<&[u8]>, usize) { |
| 82 | let mut lines = Vec::new(); |
| 83 | let mut position = 0; |
| 84 | while position + 4 <= bytes.len() { |
| 85 | let Some(length) = std::str::from_utf8(&bytes[position..position + 4]) |
| 86 | .ok() |
| 87 | .and_then(|hex| usize::from_str_radix(hex, 16).ok()) |
| 88 | else { |
| 89 | break; |
| 90 | }; |
| 91 | if length < 4 { |
| 92 | // Flush, delimiter and response-end packets carry no payload. |
| 93 | position += 4; |
| 94 | continue; |
| 95 | } |
| 96 | let end = (position + length).min(bytes.len()); |
| 97 | lines.push(&bytes[position + 4..end]); |
| 98 | position = end; |
| 99 | } |
| 100 | (lines, position) |
| 101 | } |
| 102 | |
| 103 | /// The upload-pack request for everything reachable from `want` that is |
| 104 | /// not reachable from `have`. |
| 105 | fn want_request(want: &str, have: Option<&str>) -> Vec<u8> { |
| 106 | // Side-band framing puts the pack in its own channel, so its exact bytes |
| 107 | // can be recovered. Without it the response ends in a stray flush packet |
| 108 | // that a receiver rejects as junk after the pack. |
| 109 | let mut body = pkt_line(&format!("want {want} side-band-64k\n")); |
| 110 | body.extend_from_slice(FLUSH); |
| 111 | if let Some(have) = have { |
| 112 | body.extend(pkt_line(&format!("have {have}\n"))); |
| 113 | } |
| 114 | body.extend(pkt_line("done\n")); |
| 115 | body |
| 116 | } |
| 117 | |
| 118 | /// A pack holding everything reachable from `want` that is not reachable |
| 119 | /// from `have`. |
| 120 | pub(crate) async fn fetch_pack(source: &GitAccess, want: &str, have: Option<&str>) -> Result<Vec<u8>> { |
| 121 | let response = post(source, "git-upload-pack", want_request(want, have)).await?; |
| 122 | unpack_sideband(&response) |
| 123 | } |
| 124 | |
| 125 | /// The pack in an upload-pack response that used side-band framing. |
| 126 | pub(crate) fn unpack_sideband(response: &[u8]) -> Result<Vec<u8>> { |
| 127 | let (lines, _) = read_pkt_lines(response); |
| 128 | let mut pack = Vec::new(); |
| 129 | for line in lines { |
| 130 | match line.first() { |
| 131 | Some(&PACK_BAND) => pack.extend_from_slice(&line[1..]), |
| 132 | Some(&ERROR_BAND) => { |
| 133 | return Err(Error::RustError(format!( |
| 134 | "the source refused the fetch: {}", |
| 135 | String::from_utf8_lossy(&line[1..]) |
| 136 | ))); |
| 137 | } |
| 138 | // ACK and NAK lines, and progress messages. |
| 139 | _ => {} |
| 140 | } |
| 141 | } |
| 142 | if !pack.starts_with(b"PACK") { |
| 143 | return Err(Error::RustError(format!( |
| 144 | "the source did not send a pack: {}", |
| 145 | String::from_utf8_lossy(response) |
| 146 | ))); |
| 147 | } |
| 148 | Ok(pack) |
| 149 | } |
| 150 | |
| 151 | /// Updates `branch` on the target from `old` to `new`, sending `pack`. |
| 152 | /// `Err(reason)` in the inner result means git refused the update, for |
| 153 | /// example because the branch is no longer at `old`. |
| 154 | pub(crate) async fn push_pack( |
| 155 | target: &GitAccess, |
| 156 | branch: &str, |
| 157 | old: Option<&str>, |
| 158 | new: &str, |
| 159 | pack: Vec<u8>, |
| 160 | ) -> Result<std::result::Result<(), String>> { |
| 161 | push_ref(target, &format!("refs/heads/{branch}"), old, new, Some(pack)).await |
| 162 | } |
| 163 | |
| 164 | /// Removes `branch` from the target, if it is still at `old`. |
| 165 | pub(crate) async fn delete_ref( |
| 166 | target: &GitAccess, |
| 167 | branch: &str, |
| 168 | old: &str, |
| 169 | ) -> Result<std::result::Result<(), String>> { |
| 170 | push_ref(target, &format!("refs/heads/{branch}"), Some(old), ZERO_ID, None).await |
| 171 | } |
| 172 | |
| 173 | /// The commands at the start of a receive-pack request for one ref. |
| 174 | fn command(reference: &str, old: Option<&str>, new: &str, sends_pack: bool) -> Vec<u8> { |
| 175 | let capabilities = if sends_pack { "report-status" } else { "report-status delete-refs" }; |
| 176 | let mut body = pkt_line(&format!("{} {new} {reference}\0 {capabilities}\n", old.unwrap_or(ZERO_ID))); |
| 177 | body.extend_from_slice(FLUSH); |
| 178 | body |
| 179 | } |
| 180 | |
| 181 | /// Whether a report-status says `reference` moved. |
| 182 | fn reported(report: &[u8], reference: &str, sent_pack: bool) -> std::result::Result<(), String> { |
| 183 | let (lines, _) = read_pkt_lines(report); |
| 184 | let lines: Vec<String> = lines |
| 185 | .into_iter() |
| 186 | .map(|line| String::from_utf8_lossy(line).trim_end().to_owned()) |
| 187 | .collect(); |
| 188 | // With nothing to unpack a server may not say so. |
| 189 | let unpacked = !sent_pack || lines.iter().any(|line| line == "unpack ok"); |
| 190 | let updated = lines.iter().any(|line| *line == format!("ok {reference}")); |
| 191 | if unpacked && updated { Ok(()) } else { Err(lines.join("; ")) } |
| 192 | } |
| 193 | |
| 194 | /// Moves one ref (`refs/heads/main`, `refs/pull/<id>/head`) on the target |
| 195 | /// from `old` to `new`; a deletion sends no pack. |
| 196 | pub(crate) async fn push_ref( |
| 197 | target: &GitAccess, |
| 198 | reference: &str, |
| 199 | old: Option<&str>, |
| 200 | new: &str, |
| 201 | pack: Option<Vec<u8>>, |
| 202 | ) -> Result<std::result::Result<(), String>> { |
| 203 | let sends_pack = pack.is_some(); |
| 204 | let mut body = command(reference, old, new, sends_pack); |
| 205 | if let Some(pack) = pack { |
| 206 | body.extend(pack); |
| 207 | } |
| 208 | let response = post(target, "git-receive-pack", body).await?; |
| 209 | Ok(reported(&response, reference, sends_pack)) |
| 210 | } |
| 211 | |
| 212 | /// Moves `branch` on `target` from `old` to `new`, a commit that exists in |
| 213 | /// `source`. The caller must have checked that `new` descends from `old`. |
| 214 | /// The pack goes from the source's answer into the push as it arrives. |
| 215 | pub async fn fast_forward( |
| 216 | source: &GitAccess, |
| 217 | target: &GitAccess, |
| 218 | branch: &str, |
| 219 | old: Option<&str>, |
| 220 | new: &str, |
| 221 | ) -> Result<std::result::Result<(), String>> { |
| 222 | let request = want_request(new, old); |
| 223 | let sent = request.len() as u64; |
| 224 | let mut fetched = send(source, "git-upload-pack", Uint8Array::from(request.as_slice()).into(), sent).await?; |
| 225 | let reference = format!("refs/heads/{branch}"); |
| 226 | let failure: Rc<RefCell<Option<String>>> = Rc::default(); |
| 227 | let demux = Rc::new(RefCell::new(Sideband::default())); |
| 228 | let pack = { |
| 229 | let failure = failure.clone(); |
| 230 | let demux = demux.clone(); |
| 231 | fetched.stream()?.map(move |chunk| { |
| 232 | let chunk = chunk?; |
| 233 | demux.borrow_mut().feed(&chunk).map_err(|why| { |
| 234 | *failure.borrow_mut() = Some(why.clone()); |
| 235 | Error::RustError(why) |
| 236 | }) |
| 237 | }) |
| 238 | }; |
| 239 | // The source said all it had: a pack must have come. |
| 240 | let end = { |
| 241 | let failure = failure.clone(); |
| 242 | let demux = demux.clone(); |
| 243 | futures_util::stream::once(async move { |
| 244 | demux.borrow().finish().map(|()| Vec::new()).map_err(|why| { |
| 245 | *failure.borrow_mut() = Some(why.clone()); |
| 246 | Error::RustError(why) |
| 247 | }) |
| 248 | }) |
| 249 | }; |
| 250 | let head = command(&reference, old, new, true); |
| 251 | let body = futures_util::stream::once(async move { Ok::<Vec<u8>, Error>(head) }) |
| 252 | .chain(pack) |
| 253 | .chain(end) |
| 254 | .filter(|chunk| futures_util::future::ready(!matches!(chunk, Ok(bytes) if bytes.is_empty()))); |
| 255 | let pushed = send(target, "git-receive-pack", crate::git_http::stream_body(body)?, 0).await; |
| 256 | if let Some(why) = failure.borrow_mut().take() { |
| 257 | return Err(Error::RustError(why)); |
| 258 | } |
| 259 | let report = pushed?.bytes().await?; |
| 260 | let moved = demux.borrow().pack_bytes; |
| 261 | if let (Some(from), Some(to)) = (crate::store::key_from_remote(&source.remote), crate::store::key_from_remote(&target.remote)) { |
| 262 | meters::record_bytes("internal.git.fetch", &from, 0, moved); |
| 263 | meters::record_bytes("internal.git.receive_pack", &to, moved, 0); |
| 264 | } |
| 265 | Ok(reported(&report, &reference, true)) |
| 266 | } |
| 267 | |
| 268 | /// The commands at the start of a receive-pack request for many refs: |
| 269 | /// `(name, old, new)`, `new` the zero id to delete. |
| 270 | fn commands_block(commands: &[crate::copy::Command]) -> Vec<u8> { |
| 271 | let deletes = commands.iter().any(|(_, _, new)| new == ZERO_ID); |
| 272 | let capabilities = if deletes { "report-status delete-refs" } else { "report-status" }; |
| 273 | let mut body = Vec::new(); |
| 274 | for (index, (name, old, new)) in commands.iter().enumerate() { |
| 275 | let old = old.as_deref().unwrap_or(ZERO_ID); |
| 276 | let line = if index == 0 { format!("{old} {new} {name}\0 {capabilities}\n") } else { format!("{old} {new} {name}\n") }; |
| 277 | body.extend(pkt_line(&line)); |
| 278 | } |
| 279 | body.extend_from_slice(FLUSH); |
| 280 | body |
| 281 | } |
| 282 | |
| 283 | /// Makes `target`'s refs match `commands`, fetching from `source` the |
| 284 | /// objects `wants` names that `haves` (commits the target holds) do not |
| 285 | /// reach. The pack streams from one into the other, never held whole, so |
| 286 | /// a whole repository can be copied this way (moves.rs). `Err` in the |
| 287 | /// inner result says which refs the target refused, and why. |
| 288 | pub(crate) async fn copy_refs( |
| 289 | source: &GitAccess, |
| 290 | target: &GitAccess, |
| 291 | commands: &[crate::copy::Command], |
| 292 | wants: &[String], |
| 293 | haves: &[String], |
| 294 | ) -> Result<std::result::Result<(), String>> { |
| 295 | if commands.is_empty() { |
| 296 | return Ok(Ok(())); |
| 297 | } |
| 298 | let head = commands_block(commands); |
| 299 | let sends_pack = commands.iter().any(|(_, _, new)| new != ZERO_ID); |
| 300 | let report = if wants.is_empty() { |
| 301 | // Every object is there already: only refs move. |
| 302 | let mut body = head; |
| 303 | if sends_pack { |
| 304 | body.extend_from_slice(EMPTY_PACK); |
| 305 | } |
| 306 | post(target, "git-receive-pack", body).await? |
| 307 | } else { |
| 308 | let request = crate::copy::upload_request(wants, haves); |
| 309 | let sent = request.len() as u64; |
| 310 | let mut fetched = send(source, "git-upload-pack", Uint8Array::from(request.as_slice()).into(), sent).await?; |
| 311 | let failure: Rc<RefCell<Option<String>>> = Rc::default(); |
| 312 | let demux = Rc::new(RefCell::new(Sideband::default())); |
| 313 | let pack = { |
| 314 | let failure = failure.clone(); |
| 315 | let demux = demux.clone(); |
| 316 | fetched.stream()?.map(move |chunk| { |
| 317 | let chunk = chunk?; |
| 318 | demux.borrow_mut().feed(&chunk).map_err(|why| { |
| 319 | *failure.borrow_mut() = Some(why.clone()); |
| 320 | Error::RustError(why) |
| 321 | }) |
| 322 | }) |
| 323 | }; |
| 324 | let end = { |
| 325 | let failure = failure.clone(); |
| 326 | let demux = demux.clone(); |
| 327 | futures_util::stream::once(async move { |
| 328 | demux.borrow().finish().map(|()| Vec::new()).map_err(|why| { |
| 329 | *failure.borrow_mut() = Some(why.clone()); |
| 330 | Error::RustError(why) |
| 331 | }) |
| 332 | }) |
| 333 | }; |
| 334 | let body = futures_util::stream::once(async move { Ok::<Vec<u8>, Error>(head) }) |
| 335 | .chain(pack) |
| 336 | .chain(end) |
| 337 | .filter(|chunk| futures_util::future::ready(!matches!(chunk, Ok(bytes) if bytes.is_empty()))); |
| 338 | let pushed = send(target, "git-receive-pack", crate::git_http::stream_body(body)?, 0).await; |
| 339 | if let Some(why) = failure.borrow_mut().take() { |
| 340 | return Err(Error::RustError(why)); |
| 341 | } |
| 342 | let report = pushed?.bytes().await?; |
| 343 | let moved = demux.borrow().pack_bytes; |
| 344 | if let (Some(from), Some(to)) = (crate::store::key_from_remote(&source.remote), crate::store::key_from_remote(&target.remote)) { |
| 345 | meters::record_bytes("internal.git.fetch", &from, 0, moved); |
| 346 | meters::record_bytes("internal.git.receive_pack", &to, moved, 0); |
| 347 | } |
| 348 | report |
| 349 | }; |
| 350 | let problems = crate::copy::refused(&report, commands); |
| 351 | Ok(if problems.is_empty() { Ok(()) } else { Err(problems.join("; ")) }) |
| 352 | } |
| 353 | |
| 354 | #[cfg(test)] |
| 355 | mod tests { |
| 356 | use super::*; |
| 357 | |
| 358 | #[test] |
| 359 | fn landing_rejects_a_branch_replaced_after_checks() { |
| 360 | assert_eq!(checked_commit("tested", Some("tested")).as_deref(), Some("tested")); |
| 361 | assert_eq!(checked_commit("replacement", Some("tested")), None); |
| 362 | // Compatibility for callers that have no checked-state contract. |
| 363 | assert_eq!(checked_commit("tip", None).as_deref(), Some("tip")); |
| 364 | let commit = checked_commit("tested", Some("tested")).unwrap(); |
| 365 | let command = String::from_utf8(commands_block(&[("refs/heads/main".into(), Some("base".into()), commit)])).unwrap(); |
| 366 | assert!(command.contains("base tested refs/heads/main")); |
| 367 | } |
| 368 | |
| 369 | #[test] |
| 370 | fn many_refs_are_moved_by_one_block_of_commands() { |
| 371 | let a = "c71546fcd893ef8b0f57388b65e620d759705dda".to_owned(); |
| 372 | let b = "4807077b296e6edbf410d55e72749d3e1170c291".to_owned(); |
| 373 | let block = String::from_utf8(commands_block(&[ |
| 374 | ("refs/heads/main".to_owned(), None, a.clone()), |
| 375 | ("refs/tags/v1".to_owned(), Some(a.clone()), b.clone()), |
| 376 | ])) |
| 377 | .unwrap(); |
| 378 | // Capabilities ride on the first command only. |
| 379 | assert!(block.contains(&format!("{ZERO_ID} {a} refs/heads/main\0 report-status\n"))); |
| 380 | assert!(block.contains(&format!("{a} {b} refs/tags/v1\n"))); |
| 381 | assert!(block.ends_with("0000")); |
| 382 | let deleting = String::from_utf8(commands_block(&[("refs/heads/old".to_owned(), Some(a), ZERO_ID.to_owned())])).unwrap(); |
| 383 | assert!(deleting.contains("delete-refs")); |
| 384 | } |
| 385 | |
| 386 | #[test] |
| 387 | fn one_ref_is_moved_by_one_command() { |
| 388 | let new = "4807077b296e6edbf410d55e72749d3e1170c291"; |
| 389 | let body = String::from_utf8(command("refs/pull/pul_1/head", None, new, true)).unwrap(); |
| 390 | assert!(body.starts_with(&format!("{:04x}", body.len() - 4))); |
| 391 | assert!(body.contains(&format!("{ZERO_ID} {new} refs/pull/pul_1/head\0 report-status\n"))); |
| 392 | assert!(body.ends_with("0000")); |
| 393 | let deletion = String::from_utf8(command("refs/heads/x", Some(new), ZERO_ID, false)).unwrap(); |
| 394 | assert!(deletion.contains("delete-refs")); |
| 395 | } |
| 396 | |
| 397 | #[test] |
| 398 | fn a_report_says_whether_the_ref_moved() { |
| 399 | let ok = [pkt_line("unpack ok\n"), pkt_line("ok refs/heads/main\n"), FLUSH.to_vec()].concat(); |
| 400 | assert_eq!(reported(&ok, "refs/heads/main", true), Ok(())); |
| 401 | let refused = [pkt_line("unpack ok\n"), pkt_line("ng refs/heads/main non-fast-forward\n"), FLUSH.to_vec()].concat(); |
| 402 | assert!(reported(&refused, "refs/heads/main", true).unwrap_err().contains("non-fast-forward")); |
| 403 | // Nothing unpacked: a server may not say so. |
| 404 | let bare = [pkt_line("ok refs/heads/main\n"), FLUSH.to_vec()].concat(); |
| 405 | assert_eq!(reported(&bare, "refs/heads/main", false), Ok(())); |
| 406 | assert!(reported(&bare, "refs/heads/main", true).is_err()); |
| 407 | } |
| 408 | |
| 409 | #[test] |
| 410 | fn the_empty_pack_is_a_pack_of_nothing() { |
| 411 | let pack = g1t_scan::pack::write_pack(&[]); |
| 412 | assert_eq!(pack, EMPTY_PACK); |
| 413 | } |
| 414 | } |