Pick any line to see why it is the way it is: the commit, the pull request and issue it came from, and what the agent was thinking.
| Merge branch 'worktree-agent-a6a121745e81f639f' | 1 | //! Reading the archives packages arrive as: a `.nupkg` is a zip (read with |
| 2 | //! the CRC-32 the Composer zips are written with), a `.gem` is a tar | |
| 3 | //! holding gzipped files. Only what a registry needs is read: the entries' | |
| 4 | //! names, and the bytes of the few it asks for, each up to a limit. | |
| 5 | ||
| 6 | use crate::composer::crc32; | |
| 7 | ||
| 8 | /// One file in a zip, as its central directory lists it. | |
| 9 | #[derive(Clone, Debug, PartialEq, Eq)] | |
| 10 | pub struct ZipEntry { | |
| 11 | pub name: String, | |
| 12 | method: u16, | |
| 13 | crc: u32, | |
| 14 | compressed: u64, | |
| 15 | pub size: u64, | |
| 16 | offset: u64, | |
| 17 | } | |
| 18 | ||
| 19 | fn u16_at(bytes: &[u8], at: usize) -> Option<u16> { | |
| 20 | Some(u16::from_le_bytes(bytes.get(at..at + 2)?.try_into().ok()?)) | |
| 21 | } | |
| 22 | ||
| 23 | fn u32_at(bytes: &[u8], at: usize) -> Option<u32> { | |
| 24 | Some(u32::from_le_bytes(bytes.get(at..at + 4)?.try_into().ok()?)) | |
| 25 | } | |
| 26 | ||
| 27 | /// The files a zip holds, from its central directory. | |
| 28 | pub fn zip_entries(bytes: &[u8]) -> Result<Vec<ZipEntry>, String> { | |
| 29 | const END: u32 = 0x0605_4b50; | |
| 30 | const CENTRAL: u32 = 0x0201_4b50; | |
| 31 | let not_zip = || "The file is not a zip archive.".to_owned(); | |
| 32 | if bytes.len() < 22 { | |
| 33 | return Err(not_zip()); | |
| 34 | } | |
| 35 | // The end record is the last 22 bytes, before a comment of up to 64 KB. | |
| 36 | let earliest = bytes.len().saturating_sub(22 + 0xFFFF); | |
| 37 | let end = (earliest..=bytes.len() - 22).rev().find(|&at| u32_at(bytes, at) == Some(END)).ok_or_else(not_zip)?; | |
| 38 | let count = u16_at(bytes, end + 10).ok_or_else(not_zip)?; | |
| 39 | let mut at = u32_at(bytes, end + 16).ok_or_else(not_zip)? as usize; | |
| 40 | if count == 0xFFFF || at == 0xFFFF_FFFF_usize { | |
| 41 | return Err("The archive is a zip64 archive, which is not read here.".to_owned()); | |
| 42 | } | |
| 43 | let mut entries = Vec::with_capacity(count as usize); | |
| 44 | for _ in 0..count { | |
| 45 | if u32_at(bytes, at) != Some(CENTRAL) { | |
| 46 | return Err("The zip's directory is damaged.".to_owned()); | |
| 47 | } | |
| 48 | let field16 = |offset: usize| u16_at(bytes, at + offset).ok_or_else(not_zip); | |
| 49 | let field32 = |offset: usize| u32_at(bytes, at + offset).ok_or_else(not_zip); | |
| 50 | let (name_len, extra_len, comment_len) = (field16(28)? as usize, field16(30)? as usize, field16(32)? as usize); | |
| 51 | let name = bytes.get(at + 46..at + 46 + name_len).ok_or_else(not_zip)?; | |
| 52 | entries.push(ZipEntry { | |
| 53 | name: String::from_utf8_lossy(name).into_owned(), | |
| 54 | method: field16(10)?, | |
| 55 | crc: field32(16)?, | |
| 56 | compressed: u64::from(field32(20)?), | |
| 57 | size: u64::from(field32(24)?), | |
| 58 | offset: u64::from(field32(42)?), | |
| 59 | }); | |
| 60 | at += 46 + name_len + extra_len + comment_len; | |
| 61 | } | |
| 62 | Ok(entries) | |
| 63 | } | |
| 64 | ||
| 65 | /// The bytes of one entry, inflated and checked against its CRC-32. An | |
| 66 | /// entry larger than `limit` is refused. | |
| 67 | pub fn zip_read(bytes: &[u8], entry: &ZipEntry, limit: usize) -> Result<Vec<u8>, String> { | |
| 68 | const LOCAL: u32 = 0x0403_4b50; | |
| 69 | let damaged = || format!("{} is damaged in the archive.", entry.name); | |
| 70 | if entry.size > limit as u64 { | |
| 71 | return Err(format!("{} is larger than {} KB.", entry.name, limit / 1024)); | |
| 72 | } | |
| 73 | let at = entry.offset as usize; | |
| 74 | if u32_at(bytes, at) != Some(LOCAL) { | |
| 75 | return Err(damaged()); | |
| 76 | } | |
| 77 | let start = at + 30 + u16_at(bytes, at + 26).ok_or_else(damaged)? as usize + u16_at(bytes, at + 28).ok_or_else(damaged)? as usize; | |
| 78 | let body = bytes.get(start..start + entry.compressed as usize).ok_or_else(damaged)?; | |
| 79 | let data = match entry.method { | |
| 80 | 0 => body.to_vec(), | |
| 81 | 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(body, limit).map_err(|_| damaged())?, | |
| 82 | other => return Err(format!("{} is compressed with method {other}, which is not read here.", entry.name)), | |
| 83 | }; | |
| 84 | if data.len() as u64 != entry.size || crc32(&data) != entry.crc { | |
| 85 | return Err(damaged()); | |
| 86 | } | |
| 87 | Ok(data) | |
| 88 | } | |
| 89 | ||
| 90 | /// The bytes of a gzip file, inflated, up to `limit`. | |
| 91 | pub fn gunzip(bytes: &[u8], limit: usize) -> Result<Vec<u8>, String> { | |
| 92 | let bad = || "The file is not gzipped.".to_owned(); | |
| 93 | if bytes.len() < 18 || bytes[0] != 0x1f || bytes[1] != 0x8b || bytes[2] != 8 { | |
| 94 | return Err(bad()); | |
| 95 | } | |
| 96 | let flags = bytes[3]; | |
| 97 | let mut at = 10; | |
| 98 | if flags & 4 != 0 { | |
| 99 | at += 2 + u16_at(bytes, at).ok_or_else(bad)? as usize; | |
| 100 | } | |
| 101 | for flag in [8u8, 16] { | |
| 102 | if flags & flag != 0 { | |
| 103 | at += bytes.get(at..).ok_or_else(bad)?.iter().position(|b| *b == 0).ok_or_else(bad)? + 1; | |
| 104 | } | |
| 105 | } | |
| 106 | if flags & 2 != 0 { | |
| 107 | at += 2; | |
| 108 | } | |
| 109 | let body = bytes.get(at..bytes.len() - 8).ok_or_else(bad)?; | |
| 110 | miniz_oxide::inflate::decompress_to_vec_with_limit(body, limit).map_err(|_| "The gzipped file is damaged or too large.".to_owned()) | |
| 111 | } | |
| 112 | ||
| 113 | /// The regular files of a tar, as name and bytes. | |
| 114 | pub fn tar_files(bytes: &[u8]) -> Result<Vec<(String, &[u8])>, String> { | |
| 115 | let mut files = Vec::new(); | |
| 116 | let mut at = 0; | |
| 117 | while at + 512 <= bytes.len() { | |
| 118 | let header = &bytes[at..at + 512]; | |
| 119 | if header.iter().all(|b| *b == 0) { | |
| 120 | break; | |
| 121 | } | |
| 122 | let text = |range: std::ops::Range<usize>| { | |
| 123 | let field = &header[range]; | |
| 124 | let end = field.iter().position(|b| *b == 0).unwrap_or(field.len()); | |
| 125 | String::from_utf8_lossy(&field[..end]).into_owned() | |
| 126 | }; | |
| 127 | let size = u64::from_str_radix(text(124..136).trim(), 8).map_err(|_| "The tar's header is damaged.".to_owned())? as usize; | |
| 128 | let mut name = text(0..100); | |
| 129 | if &header[257..262] == b"ustar" { | |
| 130 | let prefix = text(345..500); | |
| 131 | if !prefix.is_empty() { | |
| 132 | name = format!("{prefix}/{name}"); | |
| 133 | } | |
| 134 | } | |
| 135 | let start = at + 512; | |
| 136 | let data = bytes.get(start..start + size).ok_or("The tar ends early.")?; | |
| 137 | if matches!(header[156], 0 | b'0') { | |
| 138 | files.push((name, data)); | |
| 139 | } | |
| 140 | at = start + size.div_ceil(512) * 512; | |
| 141 | } | |
| 142 | Ok(files) | |
| 143 | } | |
| 144 | ||
| 145 | #[cfg(test)] | |
| 146 | mod tests { | |
| 147 | use super::*; | |
| 148 | ||
| 149 | #[test] | |
| 150 | fn a_zip_written_here_reads_back() { | |
| 151 | let files = vec![ | |
| 152 | ("Acme.Web.nuspec".to_owned(), b"<package/>".to_vec()), | |
| 153 | ("lib/net8.0/Acme.Web.dll".to_owned(), "MZ".repeat(500).into_bytes()), | |
| 154 | ]; | |
| 155 | let zip = crate::composer::zip(&files); | |
| 156 | let entries = zip_entries(&zip).unwrap(); | |
| 157 | assert_eq!(entries.iter().map(|e| e.name.as_str()).collect::<Vec<_>>(), ["Acme.Web.nuspec", "lib/net8.0/Acme.Web.dll"]); | |
| 158 | assert_eq!(zip_read(&zip, &entries[0], 1024).unwrap(), b"<package/>"); | |
| 159 | assert_eq!(zip_read(&zip, &entries[1], 4096).unwrap(), "MZ".repeat(500).into_bytes(), "deflated"); | |
| 160 | assert!(zip_read(&zip, &entries[1], 100).is_err(), "over the limit"); | |
| 161 | let mut damaged = zip.clone(); | |
| 162 | damaged[30 + "Acme.Web.nuspec".len() + 2] ^= 0xFF; | |
| 163 | assert!(zip_read(&damaged, &entries[0], 1024).is_err(), "the CRC catches it"); | |
| 164 | assert!(zip_entries(b"not a zip at all, but long enough").is_err()); | |
| 165 | } | |
| 166 | ||
| 167 | fn gzip(data: &[u8]) -> Vec<u8> { | |
| 168 | let mut out = vec![0x1f, 0x8b, 8, 8, 0, 0, 0, 0, 0, 3]; | |
| 169 | out.extend_from_slice(b"metadata\0"); | |
| 170 | out.extend_from_slice(&miniz_oxide::deflate::compress_to_vec(data, 6)); | |
| 171 | out.extend_from_slice(&crc32(data).to_le_bytes()); | |
| 172 | out.extend_from_slice(&(data.len() as u32).to_le_bytes()); | |
| 173 | out | |
| 174 | } | |
| 175 | ||
| 176 | pub fn tar(files: &[(&str, &[u8])]) -> Vec<u8> { | |
| 177 | let mut out = Vec::new(); | |
| 178 | for (name, data) in files { | |
| 179 | let mut header = [0u8; 512]; | |
| 180 | header[..name.len()].copy_from_slice(name.as_bytes()); | |
| 181 | let size = format!("{:011o}\0", data.len()); | |
| 182 | header[124..136].copy_from_slice(size.as_bytes()); | |
| 183 | header[156] = b'0'; | |
| 184 | header[257..262].copy_from_slice(b"ustar"); | |
| 185 | out.extend_from_slice(&header); | |
| 186 | out.extend_from_slice(data); | |
| 187 | out.resize(out.len().div_ceil(512) * 512, 0); | |
| 188 | } | |
| 189 | out.extend_from_slice(&[0; 1024]); | |
| 190 | out | |
| 191 | } | |
| 192 | ||
| 193 | #[test] | |
| 194 | fn a_gem_is_a_tar_of_gzipped_files() { | |
| 195 | let metadata = gzip(b"--- !ruby/object:Gem::Specification\nname: hello\n"); | |
| 196 | let gem = tar(&[("metadata.gz", &metadata), ("data.tar.gz", b"data")]); | |
| 197 | let files = tar_files(&gem).unwrap(); | |
| 198 | assert_eq!(files.len(), 2); | |
| 199 | assert_eq!(files[0].0, "metadata.gz"); | |
| 200 | assert_eq!(files[1].1, b"data"); | |
| 201 | assert_eq!(gunzip(files[0].1, 1024).unwrap(), b"--- !ruby/object:Gem::Specification\nname: hello\n"); | |
| 202 | assert!(gunzip(b"plain text, not gzip at all", 1024).is_err()); | |
| 203 | assert!(tar_files(&gem[..1538]).is_err(), "cut short"); | |
| 204 | } | |
| 205 | } |
This file's history is long; its oldest lines are credited to the oldest commit read.