Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Packages, with a container registry on g1t.sh; workspaces deleted whole and kept 30 days; Members for every member | 1 | //! Content digests, and a SHA-256 whose progress can be kept between |
| 2 | //! requests. | |
| 3 | //! | |
| 4 | //! A layer arrives in as many requests as the client likes. Its digest is | |
| 5 | //! worked out as the bytes pass, and the hasher's state is written to the | |
| 6 | //! upload's row after each request, so finishing an upload never reads | |
| 7 | //! the stored bytes back. | |
| 8 | ||
| 9 | use sha2::compress256; | |
| 10 | use sha2::digest::generic_array::GenericArray; | |
| 11 | ||
| 12 | const BLOCK: usize = 64; | |
| 13 | const INITIAL: [u32; 8] = [ | |
| 14 | 0x6a09_e667, 0xbb67_ae85, 0x3c6e_f372, 0xa54f_f53a, 0x510e_527f, 0x9b05_688c, 0x1f83_d9ab, 0x5be0_cd19, | |
| 15 | ]; | |
| 16 | ||
| 17 | /// A `sha256:<hex>` digest. Only SHA-256 is accepted: it is what every | |
| 18 | /// client sends. | |
| 19 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] | |
| 20 | pub struct Digest(String); | |
| 21 | ||
| 22 | impl Digest { | |
| 23 | pub fn parse(text: &str) -> Option<Digest> { | |
| 24 | let hex = text.strip_prefix("sha256:")?; | |
| 25 | (hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) | |
| 26 | .then(|| Digest(text.to_owned())) | |
| 27 | } | |
| 28 | ||
| 29 | pub fn of(bytes: &[u8]) -> Digest { | |
| 30 | let mut hasher = Sha256::new(); | |
| 31 | hasher.update(bytes); | |
| 32 | hasher.finish() | |
| 33 | } | |
| 34 | ||
| 35 | pub fn as_str(&self) -> &str { | |
| 36 | &self.0 | |
| 37 | } | |
| 38 | ||
| 39 | pub fn hex(&self) -> &str { | |
| 40 | &self.0["sha256:".len()..] | |
| 41 | } | |
| 42 | ||
| 43 | /// Where a blob of this digest is kept when it is stored whole. | |
| 44 | pub fn object_key(&self) -> String { | |
| 45 | format!("blobs/sha256/{}", self.hex()) | |
| 46 | } | |
| 47 | } | |
| 48 | ||
| 49 | impl std::fmt::Display for Digest { | |
| 50 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
| 51 | f.write_str(&self.0) | |
| 52 | } | |
| 53 | } | |
| 54 | ||
| 55 | /// SHA-256, from its state words up, so the state can be written out | |
| 56 | /// part way and read back. | |
| 57 | #[derive(Clone, Debug, PartialEq, Eq)] | |
| 58 | pub struct Sha256 { | |
| 59 | state: [u32; 8], | |
| 60 | /// Bytes not yet a whole block. | |
| 61 | pending: Vec<u8>, | |
| 62 | /// Bytes hashed so far, `pending` included. | |
| 63 | length: u64, | |
| 64 | } | |
| 65 | ||
| 66 | impl Default for Sha256 { | |
| 67 | fn default() -> Self { | |
| 68 | Sha256::new() | |
| 69 | } | |
| 70 | } | |
| 71 | ||
| 72 | impl Sha256 { | |
| 73 | pub fn new() -> Self { | |
| 74 | Sha256 { | |
| 75 | state: INITIAL, | |
| 76 | pending: Vec::with_capacity(BLOCK), | |
| 77 | length: 0, | |
| 78 | } | |
| 79 | } | |
| 80 | ||
| 81 | pub fn length(&self) -> u64 { | |
| 82 | self.length | |
| 83 | } | |
| 84 | ||
| 85 | pub fn update(&mut self, mut bytes: &[u8]) { | |
| 86 | self.length += bytes.len() as u64; | |
| 87 | if !self.pending.is_empty() { | |
| 88 | let take = (BLOCK - self.pending.len()).min(bytes.len()); | |
| 89 | self.pending.extend_from_slice(&bytes[..take]); | |
| 90 | bytes = &bytes[take..]; | |
| 91 | if self.pending.len() < BLOCK { | |
| 92 | return; | |
| 93 | } | |
| 94 | let block = GenericArray::clone_from_slice(&self.pending); | |
| 95 | compress256(&mut self.state, &[block]); | |
| 96 | self.pending.clear(); | |
| 97 | } | |
| 98 | let whole = bytes.len() / BLOCK * BLOCK; | |
| 99 | if whole > 0 { | |
| 100 | let blocks: Vec<GenericArray<u8, _>> = | |
| 101 | bytes[..whole].as_chunks::<BLOCK>().0.iter().map(|block| GenericArray::clone_from_slice(block)).collect(); | |
| 102 | compress256(&mut self.state, &blocks); | |
| 103 | } | |
| 104 | self.pending.extend_from_slice(&bytes[whole..]); | |
| 105 | } | |
| 106 | ||
| 107 | pub fn finish(mut self) -> Digest { | |
| 108 | let bits = self.length.wrapping_mul(8); | |
| 109 | let mut tail = std::mem::take(&mut self.pending); | |
| 110 | tail.push(0x80); | |
| 111 | while tail.len() % BLOCK != BLOCK - 8 { | |
| 112 | tail.push(0); | |
| 113 | } | |
| 114 | tail.extend_from_slice(&bits.to_be_bytes()); | |
| 115 | let blocks: Vec<GenericArray<u8, _>> = | |
| 116 | tail.as_chunks::<BLOCK>().0.iter().map(|block| GenericArray::clone_from_slice(block)).collect(); | |
| 117 | compress256(&mut self.state, &blocks); | |
| 118 | let hex: String = self.state.iter().map(|word| format!("{word:08x}")).collect(); | |
| 119 | Digest(format!("sha256:{hex}")) | |
| 120 | } | |
| 121 | ||
| 122 | /// The state as text, for the upload's row: the eight words, the bytes | |
| 123 | /// pending, and the length, separated by `.`. | |
| 124 | pub fn save(&self) -> String { | |
| 125 | let words: String = self.state.iter().map(|word| format!("{word:08x}")).collect(); | |
| 126 | format!("{words}.{}.{}", hex::encode(&self.pending), self.length) | |
| 127 | } | |
| 128 | ||
| 129 | /// The state [`save`](Self::save) wrote, or `None` if it is not one. | |
| 130 | pub fn restore(text: &str) -> Option<Sha256> { | |
| 131 | let mut parts = text.split('.'); | |
| 132 | let (words, pending, length) = (parts.next()?, parts.next()?, parts.next()?); | |
| 133 | if parts.next().is_some() || words.len() != 64 { | |
| 134 | return None; | |
| 135 | } | |
| 136 | let mut state = [0u32; 8]; | |
| 137 | for (i, word) in state.iter_mut().enumerate() { | |
| 138 | *word = u32::from_str_radix(words.get(i * 8..i * 8 + 8)?, 16).ok()?; | |
| 139 | } | |
| 140 | let pending = hex::decode(pending).ok()?; | |
| 141 | let length: u64 = length.parse().ok()?; | |
| 142 | (pending.len() < BLOCK && length % BLOCK as u64 == pending.len() as u64).then_some(Sha256 { | |
| 143 | state, | |
| 144 | pending, | |
| 145 | length, | |
| 146 | }) | |
| 147 | } | |
| 148 | } | |
| 149 | ||
| 150 | #[cfg(test)] | |
| 151 | mod tests { | |
| 152 | use super::*; | |
| 153 | use sha2::Digest as _; | |
| 154 | ||
| 155 | fn reference(bytes: &[u8]) -> String { | |
| 156 | format!("sha256:{}", hex::encode(sha2::Sha256::digest(bytes))) | |
| 157 | } | |
| 158 | ||
| 159 | #[test] | |
| 160 | fn matches_sha2_for_every_length_around_a_block() { | |
| 161 | for length in [0, 1, 55, 56, 63, 64, 65, 119, 120, 128, 1000] { | |
| 162 | let bytes: Vec<u8> = (0..length).map(|i| (i * 7 % 251) as u8).collect(); | |
| 163 | assert_eq!(Digest::of(&bytes).as_str(), reference(&bytes), "{length}"); | |
| 164 | } | |
| 165 | assert_eq!( | |
| 166 | Digest::of(b"").as_str(), | |
| 167 | "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" | |
| 168 | ); | |
| 169 | } | |
| 170 | ||
| 171 | #[test] | |
| 172 | fn the_state_survives_being_written_out_between_chunks() { | |
| 173 | let bytes: Vec<u8> = (0..10_000u32).map(|i| (i % 256) as u8).collect(); | |
| 174 | let mut hasher = Sha256::new(); | |
| 175 | for chunk in bytes.chunks(333) { | |
| 176 | hasher.update(chunk); | |
| 177 | let saved = hasher.save(); | |
| 178 | hasher = Sha256::restore(&saved).expect("restores"); | |
| 179 | assert_eq!(hasher.save(), saved); | |
| 180 | } | |
| 181 | assert_eq!(hasher.length(), 10_000); | |
| 182 | assert_eq!(hasher.finish().as_str(), reference(&bytes)); | |
| 183 | } | |
| 184 | ||
| 185 | #[test] | |
| 186 | fn a_broken_state_is_refused() { | |
| 187 | assert!(Sha256::restore("").is_none()); | |
| 188 | assert!(Sha256::restore("zz.00.1").is_none()); | |
| 189 | let saved = Sha256::new().save(); | |
| 190 | assert!(Sha256::restore(&saved).is_some()); | |
| 191 | assert!(Sha256::restore(&saved.replace(".0", ".5")).is_none(), "length and pending bytes disagree"); | |
| 192 | } | |
| 193 | ||
| 194 | #[test] | |
| 195 | fn digests_are_sha256_in_lowercase_hex() { | |
| 196 | let hex = "a".repeat(64); | |
| 197 | let digest = Digest::parse(&format!("sha256:{hex}")).unwrap(); | |
| 198 | assert_eq!(digest.hex(), hex); | |
| 199 | assert_eq!(digest.object_key(), format!("blobs/sha256/{hex}")); | |
| 200 | assert!(Digest::parse(&format!("sha256:{}", "A".repeat(64))).is_none()); | |
| 201 | assert!(Digest::parse(&format!("sha512:{hex}")).is_none()); | |
| 202 | assert!(Digest::parse("sha256:abc").is_none()); | |
| 203 | } | |
| 204 | } |