Skip to content
Sign up

flagon-io/g1t

public
00

Where people and agents ship software together. The open-source git platform for the whole job: issues, agents, checks and deploys to the edge.

g1t.sh
gitagentscloudflarerustforgemcp
188 linesCodeBlameRaw
1//! Sizes, chunk plans and the range headers of a chunked upload.
2
3pub const MIB: u64 = 1024 * 1024;
4/// What a chunk is unless `--chunk-size` says otherwise.
5#[cfg_attr(not(test), allow(dead_code))]
6pub const DEFAULT_CHUNK: u64 = 90 * MIB;
7/// The largest chunk: well under the 100 MB a request to g1t.sh may carry.
8pub const MAX_CHUNK: u64 = 95 * MIB;
9/// The smallest: every chunk but the last becomes a part in storage.
10pub const MIN_CHUNK: u64 = 5 * MIB;
11
12/// Reads a size like `90MB`, `90MiB`, `90M`, `512K` or `94371840`. K, M and
13/// G count in 1,024s, with or without a trailing `B` or `iB`.
14pub fn parse_size(text: &str) -> Result<u64, String> {
15 let text = text.trim();
16 let split = text
17 .find(|c: char| !c.is_ascii_digit() && c != '.')
18 .unwrap_or(text.len());
19 let (number, unit) = text.split_at(split);
20 let number: f64 = number
21 .parse()
22 .map_err(|_| format!("`{text}` is not a size, like 90MB"))?;
23 let unit = unit.trim().to_ascii_lowercase();
24 let unit = unit
25 .strip_suffix("ib")
26 .or_else(|| unit.strip_suffix('b'))
27 .unwrap_or(&unit);
28 let scale = match unit {
29 "" => 1,
30 "k" => 1024,
31 "m" => MIB,
32 "g" => 1024 * MIB,
33 _ => return Err(format!("`{text}` is not a size, like 90MB")),
34 };
35 Ok((number * scale as f64) as u64)
36}
37
38/// Checks a chunk size against the limits.
39pub fn check_chunk(size: u64) -> Result<u64, String> {
40 if size > MAX_CHUNK {
41 return Err(format!(
42 "A chunk may be at most 95MiB ({MAX_CHUNK} bytes), to stay under the 100 MB a request may carry."
43 ));
44 }
45 if size < MIN_CHUNK {
46 return Err(format!(
47 "A chunk must be at least 5MiB ({MIN_CHUNK} bytes)."
48 ));
49 }
50 Ok(size)
51}
52
53/// The chunks that send bytes `from..total`: `(start, length)` each.
54pub fn plan(total: u64, chunk: u64, from: u64) -> Vec<(u64, u64)> {
55 let mut chunks = Vec::new();
56 let mut start = from;
57 while start < total {
58 let length = chunk.min(total - start);
59 chunks.push((start, length));
60 start += length;
61 }
62 chunks
63}
64
65/// How many chunks a blob of `total` bytes takes.
66pub fn count(total: u64, chunk: u64) -> u64 {
67 total.div_ceil(chunk).max(1)
68}
69
70/// A PATCH's `Content-Range`: `<first byte>-<last byte>`.
71pub fn content_range(start: u64, length: u64) -> String {
72 format!("{start}-{}", start + length.max(1) - 1)
73}
74
75/// The offset an upload's `Range` header (`0-<last byte>`) says comes next.
76/// `0-0` reads as nothing received yet: registries say it before the first
77/// byte, and a one-byte upload that was in fact received is caught by the
78/// registry refusing the resent byte.
79pub fn next_offset(range: &str) -> Option<u64> {
80 let text = range.trim();
81 let text = text
82 .strip_prefix("bytes=")
83 .or_else(|| text.strip_prefix("bytes "))
84 .unwrap_or(text);
85 let (start, end) = text.split_once('-')?;
86 let (start, end): (u64, u64) = (start.trim().parse().ok()?, end.trim().parse().ok()?);
87 if start != 0 {
88 return None;
89 }
90 Some(if end == 0 { 0 } else { end + 1 })
91}
92
93/// A size for people: `612 B`, `3.5 MiB`.
94pub fn human(bytes: u64) -> String {
95 const UNITS: [&str; 4] = ["KiB", "MiB", "GiB", "TiB"];
96 if bytes < 1024 {
97 return format!("{bytes} B");
98 }
99 let mut value = bytes as f64 / 1024.0;
100 let mut unit = 0;
101 while value >= 1024.0 && unit < UNITS.len() - 1 {
102 value /= 1024.0;
103 unit += 1;
104 }
105 format!("{value:.1} {}", UNITS[unit])
106}
107
108#[cfg(test)]
109mod tests {
110 use super::*;
111
112 #[test]
113 fn sizes_read_in_1024s_with_any_suffix() {
114 assert_eq!(parse_size("90MB"), Ok(90 * MIB));
115 assert_eq!(parse_size("90MiB"), Ok(90 * MIB));
116 assert_eq!(parse_size("90m"), Ok(90 * MIB));
117 assert_eq!(parse_size("512K"), Ok(512 * 1024));
118 assert_eq!(parse_size("94371840"), Ok(94_371_840));
119 assert_eq!(parse_size("1.5G"), Ok(1536 * MIB));
120 assert!(parse_size("ninety").is_err());
121 assert!(parse_size("90XB").is_err());
122 }
123
124 #[test]
125 fn chunks_stay_between_5_and_95_mib() {
126 assert_eq!(check_chunk(95 * MIB), Ok(95 * MIB));
127 assert!(check_chunk(95 * MIB + 1).is_err());
128 assert!(check_chunk(100_000_000).is_err());
129 assert!(check_chunk(MIN_CHUNK - 1).is_err());
130 }
131
132 #[test]
133 fn a_plan_covers_every_byte_once() {
134 let total = 150_000_000 + 4_096;
135 let chunks = plan(total, DEFAULT_CHUNK, 0);
136 assert_eq!(
137 chunks,
138 vec![(0, DEFAULT_CHUNK), (DEFAULT_CHUNK, total - DEFAULT_CHUNK)]
139 );
140 assert_eq!(chunks.iter().map(|c| c.1).sum::<u64>(), total);
141 assert_eq!(count(total, DEFAULT_CHUNK), 2);
142 }
143
144 #[test]
145 fn a_plan_on_an_exact_multiple_has_no_empty_tail() {
146 let chunks = plan(3 * MIB * 10, 10 * MIB, 0);
147 assert_eq!(chunks.len(), 3);
148 assert!(chunks.iter().all(|c| c.1 == 10 * MIB));
149 }
150
151 #[test]
152 fn a_plan_resumes_from_an_offset() {
153 let chunks = plan(25 * MIB, 10 * MIB, 10 * MIB);
154 assert_eq!(chunks, vec![(10 * MIB, 10 * MIB), (20 * MIB, 5 * MIB)]);
155 assert!(plan(25 * MIB, 10 * MIB, 25 * MIB).is_empty());
156 }
157
158 #[test]
159 fn small_and_empty_blobs_are_one_chunk_or_none() {
160 assert_eq!(plan(612, DEFAULT_CHUNK, 0), vec![(0, 612)]);
161 assert!(plan(0, DEFAULT_CHUNK, 0).is_empty());
162 assert_eq!(count(0, DEFAULT_CHUNK), 1);
163 }
164
165 #[test]
166 fn content_ranges_name_the_first_and_last_byte() {
167 assert_eq!(content_range(0, 1024), "0-1023");
168 assert_eq!(
169 content_range(DEFAULT_CHUNK, 10),
170 format!("{}-{}", DEFAULT_CHUNK, DEFAULT_CHUNK + 9)
171 );
172 }
173
174 #[test]
175 fn upload_ranges_give_the_next_offset() {
176 assert_eq!(next_offset("0-0"), Some(0));
177 assert_eq!(next_offset("0-94371839"), Some(94_371_840));
178 assert_eq!(next_offset("bytes=0-1023"), Some(1024));
179 assert_eq!(next_offset("5-10"), None);
180 assert_eq!(next_offset("junk"), None);
181 }
182
183 #[test]
184 fn sizes_print_for_people() {
185 assert_eq!(human(612), "612 B");
186 assert_eq!(human(150_000_000), "143.1 MiB");
187 }
188}