authorgravatar for johnnymarler@gmail.comJonathan Marler <johnnymarler@gmail.com> 2024-04-20 16:52:02-06:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-05-03 16:58:53-04:00
loga96b78c170ef0464e51a1c2fa226c51d49cfde04
tree69ca057d887d636bf2855570caed97b0d6534b15
parentb86c4bde64b2c0d01e4d582798ac1b84dd50b99b

add std.zip and support zip files in build.zig.zon

fixes #17408 Helpful reviewers/testers include Joshe Wolfe, Auguste Rame, Andrew Kelley and Jacob Young. Co-authored-by: Joel Gustafson <joelg@mit.edu>

7 files changed, 1206 insertions(+), 1 deletions(-)

lib/std/io.zig+4
...@@ -344,6 +344,10 @@ pub fn GenericWriter(...@@ -344,6 +344,10 @@ pub fn GenericWriter(
344 return @errorCast(self.any().writeStruct(value));344 return @errorCast(self.any().writeStruct(value));
345 }345 }
346346
347 pub inline fn writeStructEndian(self: Self, value: anytype, endian: std.builtin.Endian) Error!void {
348 return @errorCast(self.any().writeStructEndian(value, endian));
349 }
350
347 pub inline fn any(self: *const Self) AnyWriter {351 pub inline fn any(self: *const Self) AnyWriter {
348 return .{352 return .{
349 .context = @ptrCast(&self.context),353 .context = @ptrCast(&self.context),
lib/std/io/Writer.zig+12
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const mem = std.mem;3const mem = std.mem;
4const native_endian = @import("builtin").target.cpu.arch.endian();
45
5context: *const anyopaque,6context: *const anyopaque,
6writeFn: *const fn (context: *const anyopaque, bytes: []const u8) anyerror!usize,7writeFn: *const fn (context: *const anyopaque, bytes: []const u8) anyerror!usize,
...@@ -59,6 +60,17 @@ pub fn writeStruct(self: Self, value: anytype) anyerror!void {...@@ -59,6 +60,17 @@ pub fn writeStruct(self: Self, value: anytype) anyerror!void {
59 return self.writeAll(mem.asBytes(&value));60 return self.writeAll(mem.asBytes(&value));
60}61}
6162
63pub fn writeStructEndian(self: Self, value: anytype, endian: std.builtin.Endian) anyerror!void {
64 // TODO: make sure this value is not a reference type
65 if (native_endian == endian) {
66 return self.writeStruct(value);
67 } else {
68 var copy = value;
69 mem.byteSwapAllFields(@TypeOf(value), &copy);
70 return self.writeStruct(copy);
71 }
72}
73
62pub fn writeFile(self: Self, file: std.fs.File) anyerror!void {74pub fn writeFile(self: Self, file: std.fs.File) anyerror!void {
63 // TODO: figure out how to adjust std lib abstractions so that this ends up75 // TODO: figure out how to adjust std lib abstractions so that this ends up
64 // doing sendfile or maybe even copy_file_range under the right conditions.76 // doing sendfile or maybe even copy_file_range under the right conditions.
lib/std/mem.zig+6-1
...@@ -2008,7 +2008,12 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {...@@ -2008,7 +2008,12 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
2008 .Struct => {2008 .Struct => {
2009 inline for (std.meta.fields(S)) |f| {2009 inline for (std.meta.fields(S)) |f| {
2010 switch (@typeInfo(f.type)) {2010 switch (@typeInfo(f.type)) {
2011 .Struct, .Array => byteSwapAllFields(f.type, &@field(ptr, f.name)),2011 .Struct => |struct_info| if (struct_info.backing_integer) |Int| {
2012 @field(ptr, f.name) = @bitCast(@byteSwap(@as(Int, @bitCast(@field(ptr, f.name)))));
2013 } else {
2014 byteSwapAllFields(f.type, &@field(ptr, f.name));
2015 },
2016 .Array => byteSwapAllFields(f.type, &@field(ptr, f.name)),
2012 .Enum => {2017 .Enum => {
2013 @field(ptr, f.name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f.name))));2018 @field(ptr, f.name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f.name))));
2014 },2019 },
lib/std/std.zig+1
...@@ -104,6 +104,7 @@ pub const unicode = @import("unicode.zig");...@@ -104,6 +104,7 @@ pub const unicode = @import("unicode.zig");
104pub const valgrind = @import("valgrind.zig");104pub const valgrind = @import("valgrind.zig");
105pub const wasm = @import("wasm.zig");105pub const wasm = @import("wasm.zig");
106pub const zig = @import("zig.zig");106pub const zig = @import("zig.zig");
107pub const zip = @import("zip.zig");
107pub const start = @import("start.zig");108pub const start = @import("start.zig");
108109
109const root = @import("root");110const root = @import("root");
lib/std/zip.zig created+752
...@@ -0,0 +1,752 @@
1/// The .ZIP File Format Specification is found here:
2/// https://pkwaredownloads.blob.core.windows.net/pem/APPNOTE.txt
3///
4/// Note that this file uses the abbreviation "cd" for "central directory"
5///
6const builtin = @import("builtin");
7const std = @import("std");
8const testing = std.testing;
9
10pub const testutil = @import("zip/test.zig");
11const File = testutil.File;
12const FileStore = testutil.FileStore;
13
14pub const CompressionMethod = enum(u16) {
15 store = 0,
16 deflate = 8,
17 _,
18};
19
20pub const central_file_header_sig = [4]u8{ 'P', 'K', 1, 2 };
21pub const local_file_header_sig = [4]u8{ 'P', 'K', 3, 4 };
22pub const end_record_sig = [4]u8{ 'P', 'K', 5, 6 };
23pub const end_record64_sig = [4]u8{ 'P', 'K', 6, 6 };
24pub const end_locator64_sig = [4]u8{ 'P', 'K', 6, 7 };
25pub const ExtraHeader = enum(u16) {
26 zip64_info = 0x1,
27 _,
28};
29
30const GeneralPurposeFlags = packed struct(u16) {
31 encrypted: bool,
32 _: u15,
33};
34
35pub const LocalFileHeader = extern struct {
36 signature: [4]u8 align(1),
37 version_needed_to_extract: u16 align(1),
38 flags: GeneralPurposeFlags align(1),
39 compression_method: CompressionMethod align(1),
40 last_modification_time: u16 align(1),
41 last_modification_date: u16 align(1),
42 crc32: u32 align(1),
43 compressed_size: u32 align(1),
44 uncompressed_size: u32 align(1),
45 filename_len: u16 align(1),
46 extra_len: u16 align(1),
47};
48
49pub const CentralDirectoryFileHeader = extern struct {
50 signature: [4]u8 align(1),
51 version_made_by: u16 align(1),
52 version_needed_to_extract: u16 align(1),
53 flags: GeneralPurposeFlags align(1),
54 compression_method: CompressionMethod align(1),
55 last_modification_time: u16 align(1),
56 last_modification_date: u16 align(1),
57 crc32: u32 align(1),
58 compressed_size: u32 align(1),
59 uncompressed_size: u32 align(1),
60 filename_len: u16 align(1),
61 extra_len: u16 align(1),
62 comment_len: u16 align(1),
63 disk_number: u16 align(1),
64 internal_file_attributes: u16 align(1),
65 external_file_attributes: u32 align(1),
66 local_file_header_offset: u32 align(1),
67};
68
69pub const EndRecord64 = extern struct {
70 signature: [4]u8 align(1),
71 end_record_size: u64 align(1),
72 version_made_by: u16 align(1),
73 version_needed_to_extract: u16 align(1),
74 disk_number: u32 align(1),
75 central_directory_disk_number: u32 align(1),
76 record_count_disk: u64 align(1),
77 record_count_total: u64 align(1),
78 central_directory_size: u64 align(1),
79 central_directory_offset: u64 align(1),
80};
81
82pub const EndLocator64 = extern struct {
83 signature: [4]u8 align(1),
84 zip64_disk_count: u32 align(1),
85 record_file_offset: u64 align(1),
86 total_disk_count: u32 align(1),
87};
88
89pub const EndRecord = extern struct {
90 signature: [4]u8 align(1),
91 disk_number: u16 align(1),
92 central_directory_disk_number: u16 align(1),
93 record_count_disk: u16 align(1),
94 record_count_total: u16 align(1),
95 central_directory_size: u32 align(1),
96 central_directory_offset: u32 align(1),
97 comment_len: u16 align(1),
98 pub fn need_zip64(self: EndRecord) bool {
99 return isMaxInt(self.record_count_disk) or
100 isMaxInt(self.record_count_total) or
101 isMaxInt(self.central_directory_size) or
102 isMaxInt(self.central_directory_offset);
103 }
104};
105
106/// Find and return the end record for the given seekable zip stream.
107/// Note that `seekable_stream` must be an instance of `std.io.SeekabkeStream` and
108/// its context must also have a `.reader()` method that returns an instance of
109/// `std.io.Reader`.
110pub fn findEndRecord(seekable_stream: anytype, stream_len: u64) !EndRecord {
111 var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
112 const record_len_max = @min(stream_len, buf.len);
113 var loaded_len: u32 = 0;
114
115 var comment_len: u16 = 0;
116 while (true) {
117 const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
118 if (record_len > record_len_max)
119 return error.ZipNoEndRecord;
120
121 if (record_len > loaded_len) {
122 const new_loaded_len = @min(loaded_len + 300, record_len_max);
123 const read_len = new_loaded_len - loaded_len;
124
125 try seekable_stream.seekTo(stream_len - @as(u64, new_loaded_len));
126 const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
127 const len = try seekable_stream.context.reader().readAll(read_buf);
128 if (len != read_len)
129 return error.ZipTruncated;
130 loaded_len = new_loaded_len;
131 }
132
133 const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];
134 if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and
135 std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)
136 {
137 const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);
138 if (builtin.target.cpu.arch.endian() != .little) {
139 std.mem.byteSwapAllFields(@TypeOf(record.*), record);
140 }
141 return record.*;
142 }
143
144 if (comment_len == std.math.maxInt(u16))
145 return error.ZipNoEndRecord;
146 comment_len += 1;
147 }
148}
149
150/// Decompresses the given data from `reader` into `writer`. Stops early if more
151/// than `uncompressed_size` bytes are processed and verifies that exactly that
152/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data.
153/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method.
154pub fn decompress(
155 method: CompressionMethod,
156 uncompressed_size: u64,
157 reader: anytype,
158 writer: anytype,
159) !u32 {
160 var hash = std.hash.Crc32.init();
161
162 var total_uncompressed: u64 = 0;
163 switch (method) {
164 .store => {
165 var buf: [std.mem.page_size]u8 = undefined;
166 while (true) {
167 const len = try reader.read(&buf);
168 if (len == 0) break;
169 try writer.writeAll(buf[0..len]);
170 hash.update(buf[0..len]);
171 total_uncompressed += @intCast(len);
172 }
173 },
174 .deflate => {
175 var br = std.io.bufferedReader(reader);
176 var decompressor = std.compress.flate.decompressor(br.reader());
177 while (try decompressor.next()) |chunk| {
178 try writer.writeAll(chunk);
179 hash.update(chunk);
180 total_uncompressed += @intCast(chunk.len);
181 if (total_uncompressed > uncompressed_size)
182 return error.ZipUncompressSizeTooSmall;
183 }
184 if (br.end != br.start)
185 return error.ZipDeflateTruncated;
186 },
187 _ => return error.UnsupportedCompressionMethod,
188 }
189 if (total_uncompressed != uncompressed_size)
190 return error.ZipUncompressSizeMismatch;
191
192 return hash.final();
193}
194
195fn isBadFilename(filename: []const u8) bool {
196 if (filename.len == 0 or filename[0] == '/')
197 return true;
198
199 var it = std.mem.splitScalar(u8, filename, '/');
200 while (it.next()) |part| {
201 if (std.mem.eql(u8, part, ".."))
202 return true;
203 }
204
205 return false;
206}
207
208fn isMaxInt(uint: anytype) bool {
209 return uint == std.math.maxInt(@TypeOf(uint));
210}
211
212const FileExtents = struct {
213 uncompressed_size: u64,
214 compressed_size: u64,
215 local_file_header_offset: u64,
216};
217
218fn readZip64FileExtents(header: CentralDirectoryFileHeader, extents: *FileExtents, data: []u8) !void {
219 var data_offset: usize = 0;
220 if (isMaxInt(header.uncompressed_size)) {
221 if (data_offset + 8 > data.len)
222 return error.ZipBadCd64Size;
223 extents.uncompressed_size = std.mem.readInt(u64, data[data_offset..][0..8], .little);
224 data_offset += 8;
225 }
226 if (isMaxInt(header.compressed_size)) {
227 if (data_offset + 8 > data.len)
228 return error.ZipBadCd64Size;
229 extents.compressed_size = std.mem.readInt(u64, data[data_offset..][0..8], .little);
230 data_offset += 8;
231 }
232 if (isMaxInt(header.local_file_header_offset)) {
233 if (data_offset + 8 > data.len)
234 return error.ZipBadCd64Size;
235 extents.local_file_header_offset = std.mem.readInt(u64, data[data_offset..][0..8], .little);
236 data_offset += 8;
237 }
238 if (isMaxInt(header.disk_number)) {
239 if (data_offset + 4 > data.len)
240 return error.ZipInvalid;
241 const disk_number = std.mem.readInt(u32, data[data_offset..][0..4], .little);
242 if (disk_number != 0)
243 return error.ZipMultiDiskUnsupported;
244 data_offset += 4;
245 }
246 if (data_offset > data.len)
247 return error.ZipBadCd64Size;
248}
249
250pub fn Iterator(comptime SeekableStream: type) type {
251 return struct {
252 stream: SeekableStream,
253
254 cd_record_count: u64,
255 cd_zip_offset: u64,
256 cd_size: u64,
257
258 cd_record_index: u64 = 0,
259 cd_record_offset: u64 = 0,
260
261 const Self = @This();
262
263 pub fn init(stream: SeekableStream) !Self {
264 const stream_len = try stream.getEndPos();
265
266 const end_record = try findEndRecord(stream, stream_len);
267
268 if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
269 return error.ZipDiskRecordCountTooLarge;
270
271 if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
272 return error.ZipMultiDiskUnsupported;
273
274 {
275 const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);
276 if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
277 return error.ZipMultiDiskUnsupported;
278 }
279
280 var result = Self{
281 .stream = stream,
282 .cd_record_count = end_record.record_count_total,
283 .cd_zip_offset = end_record.central_directory_offset,
284 .cd_size = end_record.central_directory_size,
285 };
286 if (!end_record.need_zip64()) return result;
287
288 const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
289 if (locator_end_offset > stream_len)
290 return error.ZipTruncated;
291 try stream.seekTo(stream_len - locator_end_offset);
292 const locator = try stream.context.reader().readStructEndian(EndLocator64, .little);
293 if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
294 return error.ZipBadLocatorSig;
295 if (locator.zip64_disk_count != 0)
296 return error.ZipUnsupportedZip64DiskCount;
297 if (locator.total_disk_count != 1)
298 return error.ZipMultiDiskUnsupported;
299
300 try stream.seekTo(locator.record_file_offset);
301
302 const record64 = try stream.context.reader().readStructEndian(EndRecord64, .little);
303
304 if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
305 return error.ZipBadEndRecord64Sig;
306
307 if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
308 return error.ZipEndRecord64SizeTooSmall;
309 if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
310 return error.ZipEndRecord64UnhandledExtraData;
311
312 if (record64.version_needed_to_extract > 45)
313 return error.ZipUnsupportedVersion;
314
315 {
316 const is_multidisk = record64.disk_number != 0 or
317 record64.central_directory_disk_number != 0 or
318 record64.record_count_disk != record64.record_count_total;
319 if (is_multidisk)
320 return error.ZipMultiDiskUnsupported;
321 }
322
323 if (isMaxInt(end_record.record_count_total)) {
324 result.cd_record_count = record64.record_count_total;
325 } else if (end_record.record_count_total != record64.record_count_total)
326 return error.Zip64RecordCountTotalMismatch;
327
328 if (isMaxInt(end_record.central_directory_offset)) {
329 result.cd_zip_offset = record64.central_directory_offset;
330 } else if (end_record.central_directory_offset != record64.central_directory_offset)
331 return error.Zip64CentralDirectoryOffsetMismatch;
332
333 if (isMaxInt(end_record.central_directory_size)) {
334 result.cd_size = record64.central_directory_size;
335 } else if (end_record.central_directory_size != record64.central_directory_size)
336 return error.Zip64CentralDirectorySizeMismatch;
337
338 return result;
339 }
340
341 pub fn next(self: *Self) !?Entry {
342 if (self.cd_record_index == self.cd_record_count) {
343 if (self.cd_record_offset != self.cd_size)
344 return if (self.cd_size > self.cd_record_offset)
345 error.ZipCdOversized
346 else
347 error.ZipCdUndersized;
348
349 return null;
350 }
351
352 const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
353 try self.stream.seekTo(header_zip_offset);
354 const header = try self.stream.context.reader().readStructEndian(CentralDirectoryFileHeader, .little);
355 if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
356 return error.ZipBadCdOffset;
357
358 self.cd_record_index += 1;
359 self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;
360
361 // Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file
362 // at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip
363 // has an undocumented version 788 but extracts just fine.
364
365 if (header.flags.encrypted)
366 return error.ZipEncryptionUnsupported;
367 // TODO: check/verify more flags
368 if (header.disk_number != 0)
369 return error.ZipMultiDiskUnsupported;
370
371 var extents: FileExtents = .{
372 .uncompressed_size = header.uncompressed_size,
373 .compressed_size = header.compressed_size,
374 .local_file_header_offset = header.local_file_header_offset,
375 };
376
377 if (header.extra_len > 0) {
378 var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
379 const extra = extra_buf[0..header.extra_len];
380
381 {
382 try self.stream.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
383 const len = try self.stream.context.reader().readAll(extra);
384 if (len != extra.len)
385 return error.ZipTruncated;
386 }
387
388 var extra_offset: usize = 0;
389 while (extra_offset + 4 <= extra.len) {
390 const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
391 const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
392 const end = extra_offset + 4 + data_size;
393 if (end > extra.len)
394 return error.ZipBadExtraFieldSize;
395 const data = extra[extra_offset + 4 .. end];
396 switch (@as(ExtraHeader, @enumFromInt(header_id))) {
397 .zip64_info => try readZip64FileExtents(header, &extents, data),
398 else => {}, // ignore
399 }
400 extra_offset = end;
401 }
402 }
403
404 return .{
405 .version_needed_to_extract = header.version_needed_to_extract,
406 .flags = header.flags,
407 .compression_method = header.compression_method,
408 .last_modification_time = header.last_modification_time,
409 .last_modification_date = header.last_modification_date,
410 .header_zip_offset = header_zip_offset,
411 .crc32 = header.crc32,
412 .filename_len = header.filename_len,
413 .compressed_size = extents.compressed_size,
414 .uncompressed_size = extents.uncompressed_size,
415 .file_offset = extents.local_file_header_offset,
416 };
417 }
418
419 pub const Entry = struct {
420 version_needed_to_extract: u16,
421 flags: GeneralPurposeFlags,
422 compression_method: CompressionMethod,
423 last_modification_time: u16,
424 last_modification_date: u16,
425 header_zip_offset: u64,
426 crc32: u32,
427 filename_len: u32,
428 compressed_size: u64,
429 uncompressed_size: u64,
430 file_offset: u64,
431
432 pub fn extract(
433 self: Entry,
434 stream: SeekableStream,
435 options: ExtractOptions,
436 filename_buf: []u8,
437 dest: std.fs.Dir,
438 ) !u32 {
439 if (filename_buf.len < self.filename_len)
440 return error.ZipInsufficientBuffer;
441 const filename = filename_buf[0..self.filename_len];
442
443 try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
444
445 {
446 const len = try stream.context.reader().readAll(filename);
447 if (len != filename.len)
448 return error.ZipBadFileOffset;
449 }
450
451 const local_data_header_offset: u64 = local_data_header_offset: {
452 const local_header = blk: {
453 try stream.seekTo(self.file_offset);
454 break :blk try stream.context.reader().readStructEndian(LocalFileHeader, .little);
455 };
456 if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
457 return error.ZipBadFileOffset;
458 if (local_header.version_needed_to_extract != self.version_needed_to_extract)
459 return error.ZipMismatchVersionNeeded;
460 if (local_header.last_modification_time != self.last_modification_time)
461 return error.ZipMismatchModTime;
462 if (local_header.last_modification_date != self.last_modification_date)
463 return error.ZipMismatchModDate;
464
465 if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
466 return error.ZipMismatchFlags;
467 if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
468 return error.ZipMismatchCrc32;
469 if (local_header.compressed_size != 0 and
470 local_header.compressed_size != self.compressed_size)
471 return error.ZipMismatchCompLen;
472 if (local_header.uncompressed_size != 0 and
473 local_header.uncompressed_size != self.uncompressed_size)
474 return error.ZipMismatchUncompLen;
475 if (local_header.filename_len != self.filename_len)
476 return error.ZipMismatchFilenameLen;
477
478 break :local_data_header_offset @as(u64, local_header.filename_len) +
479 @as(u64, local_header.extra_len);
480 };
481
482 if (isBadFilename(filename))
483 return error.ZipBadFilename;
484
485 if (options.allow_backslashes) {
486 std.mem.replaceScalar(u8, filename, '\\', '/');
487 } else {
488 if (std.mem.indexOfScalar(u8, filename, '\\')) |_|
489 return error.ZipFilenameHasBackslash;
490 }
491
492 // All entries that end in '/' are directories
493 if (filename[filename.len - 1] == '/') {
494 if (self.uncompressed_size != 0)
495 return error.ZipBadDirectorySize;
496 try dest.makePath(filename[0 .. filename.len - 1]);
497 return std.hash.Crc32.hash(&.{});
498 }
499
500 const out_file = blk: {
501 if (std.fs.path.dirname(filename)) |dirname| {
502 var parent_dir = try dest.makeOpenPath(dirname, .{});
503 defer parent_dir.close();
504
505 const basename = std.fs.path.basename(filename);
506 break :blk try parent_dir.createFile(basename, .{ .exclusive = true });
507 }
508 break :blk try dest.createFile(filename, .{ .exclusive = true });
509 };
510 defer out_file.close();
511 const local_data_file_offset: u64 =
512 @as(u64, self.file_offset) +
513 @as(u64, @sizeOf(LocalFileHeader)) +
514 local_data_header_offset;
515 try stream.seekTo(local_data_file_offset);
516 var limited_reader = std.io.limitedReader(stream.context.reader(), self.compressed_size);
517 const crc = try decompress(
518 self.compression_method,
519 self.uncompressed_size,
520 limited_reader.reader(),
521 out_file.writer(),
522 );
523 if (limited_reader.bytes_left != 0)
524 return error.ZipDecompressTruncated;
525 return crc;
526 }
527 };
528 };
529}
530
531// returns true if `filename` starts with `root` followed by a forward slash
532fn filenameInRoot(filename: []const u8, root: []const u8) bool {
533 return (filename.len >= root.len + 1) and
534 (filename[root.len] == '/') and
535 std.mem.eql(u8, filename[0..root.len], root);
536}
537
538pub const Diagnostics = struct {
539 allocator: std.mem.Allocator,
540
541 /// The common root directory for all extracted files if there is one.
542 root_dir: []const u8 = "",
543
544 saw_first_file: bool = false,
545
546 pub fn deinit(self: *Diagnostics) void {
547 self.allocator.free(self.root_dir);
548 self.* = undefined;
549 }
550
551 // This function assumes name is a filename from a zip file which has already been verified to
552 // not start with a slash, backslashes have been normalized to forward slashes, and directories
553 // always end in a slash.
554 pub fn nextFilename(self: *Diagnostics, name: []const u8) error{OutOfMemory}!void {
555 if (!self.saw_first_file) {
556 self.saw_first_file = true;
557 std.debug.assert(self.root_dir.len == 0);
558 const root_len = std.mem.indexOfScalar(u8, name, '/') orelse return;
559 std.debug.assert(root_len > 0);
560 self.root_dir = try self.allocator.dupe(u8, name[0..root_len]);
561 } else if (self.root_dir.len > 0) {
562 if (!filenameInRoot(name, self.root_dir)) {
563 self.allocator.free(self.root_dir);
564 self.root_dir = "";
565 }
566 }
567 }
568};
569
570pub const ExtractOptions = struct {
571 /// Allow filenames within the zip to use backslashes. Back slashes are normalized
572 /// to forward slashes before forwarding them to platform APIs.
573 allow_backslashes: bool = false,
574
575 diagnostics: ?*Diagnostics = null,
576};
577
578/// Extract the zipped files inside `seekable_stream` to the given `dest` directory.
579/// Note that `seekable_stream` must be an instance of `std.io.SeekabkeStream` and
580/// its context must also have a `.reader()` method that returns an instance of
581/// `std.io.Reader`.
582pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptions) !void {
583 const SeekableStream = @TypeOf(seekable_stream);
584 var iter = try Iterator(SeekableStream).init(seekable_stream);
585
586 var filename_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
587 while (try iter.next()) |entry| {
588 const crc32 = try entry.extract(seekable_stream, options, &filename_buf, dest);
589 if (crc32 != entry.crc32)
590 return error.ZipCrcMismatch;
591 if (options.diagnostics) |d| {
592 try d.nextFilename(filename_buf[0..entry.filename_len]);
593 }
594 }
595}
596
597fn testZip(options: ExtractOptions, comptime files: []const File, write_opt: testutil.WriteZipOptions) !void {
598 var store: [files.len]FileStore = undefined;
599 try testZipWithStore(options, files, write_opt, &store);
600}
601fn testZipWithStore(
602 options: ExtractOptions,
603 test_files: []const File,
604 write_opt: testutil.WriteZipOptions,
605 store: []FileStore,
606) !void {
607 var zip_buf: [4096]u8 = undefined;
608 var fbs = try testutil.makeZipWithStore(&zip_buf, test_files, write_opt, store);
609
610 var tmp = testing.tmpDir(.{ .no_follow = true });
611 defer tmp.cleanup();
612 try extract(tmp.dir, fbs.seekableStream(), options);
613 try testutil.expectFiles(test_files, tmp.dir, .{});
614}
615fn testZipError(expected_error: anyerror, file: File, options: ExtractOptions) !void {
616 var zip_buf: [4096]u8 = undefined;
617 var store: [1]FileStore = undefined;
618 var fbs = try testutil.makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store);
619 var tmp = testing.tmpDir(.{ .no_follow = true });
620 defer tmp.cleanup();
621 try testing.expectError(expected_error, extract(tmp.dir, fbs.seekableStream(), options));
622}
623
624test "zip one file" {
625 try testZip(.{}, &[_]File{
626 .{ .name = "onefile.txt", .content = "Just a single file\n", .compression = .store },
627 }, .{});
628}
629test "zip multiple files" {
630 try testZip(.{ .allow_backslashes = true }, &[_]File{
631 .{ .name = "foo", .content = "a foo file\n", .compression = .store },
632 .{ .name = "subdir/bar", .content = "bar is this right?\nanother newline\n", .compression = .store },
633 .{ .name = "subdir\\whoa", .content = "you can do backslashes", .compression = .store },
634 .{ .name = "subdir/another/baz", .content = "bazzy mc bazzerson", .compression = .store },
635 }, .{});
636}
637test "zip deflated" {
638 try testZip(.{}, &[_]File{
639 .{ .name = "deflateme", .content = "This is a deflated file.\nIt should be smaller in the Zip file1\n", .compression = .deflate },
640 // TODO: re-enable this if/when we add support for deflate64
641 //.{ .name = "deflateme64", .content = "The 64k version of deflate!\n", .compression = .deflate64 },
642 .{ .name = "raw", .content = "Not all files need to be deflated in the same Zip.\n", .compression = .store },
643 }, .{});
644}
645test "zip verify filenames" {
646 // no empty filenames
647 try testZipError(error.ZipBadFilename, .{ .name = "", .content = "", .compression = .store }, .{});
648 // no absolute paths
649 try testZipError(error.ZipBadFilename, .{ .name = "/", .content = "", .compression = .store }, .{});
650 try testZipError(error.ZipBadFilename, .{ .name = "/foo", .content = "", .compression = .store }, .{});
651 try testZipError(error.ZipBadFilename, .{ .name = "/foo/bar", .content = "", .compression = .store }, .{});
652 // no '..' components
653 try testZipError(error.ZipBadFilename, .{ .name = "..", .content = "", .compression = .store }, .{});
654 try testZipError(error.ZipBadFilename, .{ .name = "foo/..", .content = "", .compression = .store }, .{});
655 try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/..", .content = "", .compression = .store }, .{});
656 try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/../", .content = "", .compression = .store }, .{});
657 // no backslashes
658 try testZipError(error.ZipFilenameHasBackslash, .{ .name = "foo\\bar", .content = "", .compression = .store }, .{});
659}
660
661test "zip64" {
662 const test_files = [_]File{
663 .{ .name = "fram", .content = "fram foo fro fraba", .compression = .store },
664 .{ .name = "subdir/barro", .content = "aljdk;jal;jfd;lajkf", .compression = .store },
665 };
666
667 try testZip(.{}, &test_files, .{
668 .end = .{
669 .zip64 = .{},
670 .record_count_disk = std.math.maxInt(u16), // trigger zip64
671 },
672 });
673 try testZip(.{}, &test_files, .{
674 .end = .{
675 .zip64 = .{},
676 .record_count_total = std.math.maxInt(u16), // trigger zip64
677 },
678 });
679 try testZip(.{}, &test_files, .{
680 .end = .{
681 .zip64 = .{},
682 .record_count_disk = std.math.maxInt(u16), // trigger zip64
683 .record_count_total = std.math.maxInt(u16), // trigger zip64
684 },
685 });
686 try testZip(.{}, &test_files, .{
687 .end = .{
688 .zip64 = .{},
689 .central_directory_size = std.math.maxInt(u32), // trigger zip64
690 },
691 });
692 try testZip(.{}, &test_files, .{
693 .end = .{
694 .zip64 = .{},
695 .central_directory_offset = std.math.maxInt(u32), // trigger zip64
696 },
697 });
698}
699
700test "bad zip files" {
701 var tmp = testing.tmpDir(.{ .no_follow = true });
702 defer tmp.cleanup();
703 var zip_buf: [4096]u8 = undefined;
704
705 const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }};
706
707 {
708 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
709 try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
710 }
711 {
712 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } });
713 try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
714 }
715 {
716 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
717 try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
718 }
719 {
720 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } });
721 try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
722 }
723 {
724 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
725 try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
726 }
727 {
728 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } });
729 try testing.expectError(error.ZipDiskRecordCountTooLarge, extract(tmp.dir, fbs.seekableStream(), .{}));
730 }
731 {
732 var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } });
733 try testing.expectError(error.ZipCdOversized, extract(tmp.dir, fbs.seekableStream(), .{}));
734 }
735 {
736 var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } });
737 try testing.expectError(error.ZipCdUndersized, extract(tmp.dir, fbs.seekableStream(), .{}));
738 }
739 {
740 var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
741 try testing.expectError(error.ZipBadCdOffset, extract(tmp.dir, fbs.seekableStream(), .{}));
742 }
743 {
744 var fbs = try testutil.makeZip(&zip_buf, &file_a, .{
745 .end = .{
746 .zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } },
747 .central_directory_size = std.math.maxInt(u32), // trigger 64
748 },
749 });
750 try testing.expectError(error.ZipBadLocatorSig, extract(tmp.dir, fbs.seekableStream(), .{}));
751 }
752}
lib/std/zip/test.zig created+267
...@@ -0,0 +1,267 @@
1const std = @import("std");
2const testing = std.testing;
3const zip = @import("../zip.zig");
4const maxInt = std.math.maxInt;
5
6pub const File = struct {
7 name: []const u8,
8 content: []const u8,
9 compression: zip.CompressionMethod,
10};
11
12pub fn expectFiles(
13 test_files: []const File,
14 dir: std.fs.Dir,
15 opt: struct {
16 strip_prefix: ?[]const u8 = null,
17 },
18) !void {
19 for (test_files) |test_file| {
20 var normalized_sub_path_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
21
22 const name = blk: {
23 if (opt.strip_prefix) |strip_prefix| {
24 try testing.expect(test_file.name.len >= strip_prefix.len);
25 try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]);
26 break :blk test_file.name[strip_prefix.len..];
27 }
28 break :blk test_file.name;
29 };
30 const normalized_sub_path = normalized_sub_path_buf[0..name.len];
31 @memcpy(normalized_sub_path, name);
32 std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
33 var file = try dir.openFile(normalized_sub_path, .{});
34 defer file.close();
35 var content_buf: [4096]u8 = undefined;
36 const n = try file.reader().readAll(&content_buf);
37 try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
38 }
39}
40
41// Used to store any data from writing a file to the zip archive that's needed
42// when writing the corresponding central directory record.
43pub const FileStore = struct {
44 compression: zip.CompressionMethod,
45 file_offset: u64,
46 crc32: u32,
47 compressed_size: u32,
48 uncompressed_size: usize,
49};
50
51pub fn makeZip(
52 buf: []u8,
53 comptime files: []const File,
54 options: WriteZipOptions,
55) !std.io.FixedBufferStream([]u8) {
56 var store: [files.len]FileStore = undefined;
57 return try makeZipWithStore(buf, files, options, &store);
58}
59
60pub fn makeZipWithStore(
61 buf: []u8,
62 files: []const File,
63 options: WriteZipOptions,
64 store: []FileStore,
65) !std.io.FixedBufferStream([]u8) {
66 var fbs = std.io.fixedBufferStream(buf);
67 try writeZip(fbs.writer(), files, store, options);
68 return std.io.fixedBufferStream(buf[0..fbs.pos]);
69}
70
71pub const WriteZipOptions = struct {
72 end: ?EndRecordOptions = null,
73};
74pub const EndRecordOptions = struct {
75 zip64: ?Zip64Options = null,
76 sig: ?[4]u8 = null,
77 disk_number: ?u16 = null,
78 central_directory_disk_number: ?u16 = null,
79 record_count_disk: ?u16 = null,
80 record_count_total: ?u16 = null,
81 central_directory_size: ?u32 = null,
82 central_directory_offset: ?u32 = null,
83 comment_len: ?u16 = null,
84 comment: ?[]const u8 = null,
85};
86pub const Zip64Options = struct {
87 locator_sig: ?[4]u8 = null,
88 locator_zip64_disk_count: ?u32 = null,
89 locator_record_file_offset: ?u64 = null,
90 locator_total_disk_count: ?u32 = null,
91 //record_size: ?u64 = null,
92 central_directory_size: ?u64 = null,
93};
94
95pub fn writeZip(
96 writer: anytype,
97 files: []const File,
98 store: []FileStore,
99 options: WriteZipOptions,
100) !void {
101 if (store.len < files.len) return error.FileStoreTooSmall;
102 var zipper = initZipper(writer);
103 for (files, 0..) |file, i| {
104 store[i] = try zipper.writeFile(.{
105 .name = file.name,
106 .content = file.content,
107 .compression = file.compression,
108 });
109 }
110 for (files, 0..) |file, i| {
111 try zipper.writeCentralRecord(store[i], .{
112 .name = file.name,
113 });
114 }
115 try zipper.writeEndRecord(if (options.end) |e| e else .{});
116}
117
118pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
119 return .{ .counting_writer = std.io.countingWriter(writer) };
120}
121
122/// Provides methods to format and write the contents of a zip archive
123/// to the underlying Writer.
124pub fn Zipper(comptime Writer: type) type {
125 return struct {
126 counting_writer: std.io.CountingWriter(Writer),
127 central_count: u64 = 0,
128 first_central_offset: ?u64 = null,
129 last_central_limit: ?u64 = null,
130
131 const Self = @This();
132
133 pub fn writeFile(
134 self: *Self,
135 opt: struct {
136 name: []const u8,
137 content: []const u8,
138 compression: zip.CompressionMethod,
139 },
140 ) !FileStore {
141 const writer = self.counting_writer.writer();
142
143 const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
144 const crc32 = std.hash.Crc32.hash(opt.content);
145
146 {
147 const hdr: zip.LocalFileHeader = .{
148 .signature = zip.local_file_header_sig,
149 .version_needed_to_extract = 10,
150 .flags = .{ .encrypted = false, ._ = 0 },
151 .compression_method = opt.compression,
152 .last_modification_time = 0,
153 .last_modification_date = 0,
154 .crc32 = crc32,
155 .compressed_size = 0,
156 .uncompressed_size = @intCast(opt.content.len),
157 .filename_len = @intCast(opt.name.len),
158 .extra_len = 0,
159 };
160 try writer.writeStructEndian(hdr, .little);
161 }
162 try writer.writeAll(opt.name);
163
164 var compressed_size: u32 = undefined;
165 switch (opt.compression) {
166 .store => {
167 try writer.writeAll(opt.content);
168 compressed_size = @intCast(opt.content.len);
169 },
170 .deflate => {
171 const offset = self.counting_writer.bytes_written;
172 var fbs = std.io.fixedBufferStream(opt.content);
173 try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
174 std.debug.assert(fbs.pos == opt.content.len);
175 compressed_size = @intCast(self.counting_writer.bytes_written - offset);
176 },
177 else => unreachable,
178 }
179 return .{
180 .compression = opt.compression,
181 .file_offset = file_offset,
182 .crc32 = crc32,
183 .compressed_size = compressed_size,
184 .uncompressed_size = opt.content.len,
185 };
186 }
187
188 pub fn writeCentralRecord(
189 self: *Self,
190 store: FileStore,
191 opt: struct {
192 name: []const u8,
193 version_needed_to_extract: u16 = 10,
194 },
195 ) !void {
196 if (self.first_central_offset == null) {
197 self.first_central_offset = self.counting_writer.bytes_written;
198 }
199 self.central_count += 1;
200
201 const hdr: zip.CentralDirectoryFileHeader = .{
202 .signature = zip.central_file_header_sig,
203 .version_made_by = 0,
204 .version_needed_to_extract = opt.version_needed_to_extract,
205 .flags = .{ .encrypted = false, ._ = 0 },
206 .compression_method = store.compression,
207 .last_modification_time = 0,
208 .last_modification_date = 0,
209 .crc32 = store.crc32,
210 .compressed_size = store.compressed_size,
211 .uncompressed_size = @intCast(store.uncompressed_size),
212 .filename_len = @intCast(opt.name.len),
213 .extra_len = 0,
214 .comment_len = 0,
215 .disk_number = 0,
216 .internal_file_attributes = 0,
217 .external_file_attributes = 0,
218 .local_file_header_offset = @intCast(store.file_offset),
219 };
220 try self.counting_writer.writer().writeStructEndian(hdr, .little);
221 try self.counting_writer.writer().writeAll(opt.name);
222 self.last_central_limit = self.counting_writer.bytes_written;
223 }
224
225 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
226 const cd_offset = self.first_central_offset orelse 0;
227 const cd_end = self.last_central_limit orelse 0;
228
229 if (opt.zip64) |zip64| {
230 const end64_off = cd_end;
231 const fixed: zip.EndRecord64 = .{
232 .signature = zip.end_record64_sig,
233 .end_record_size = @sizeOf(zip.EndRecord64) - 12,
234 .version_made_by = 0,
235 .version_needed_to_extract = 45,
236 .disk_number = 0,
237 .central_directory_disk_number = 0,
238 .record_count_disk = @intCast(self.central_count),
239 .record_count_total = @intCast(self.central_count),
240 .central_directory_size = @intCast(cd_end - cd_offset),
241 .central_directory_offset = @intCast(cd_offset),
242 };
243 try self.counting_writer.writer().writeStructEndian(fixed, .little);
244 const locator: zip.EndLocator64 = .{
245 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
246 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
247 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
248 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
249 };
250 try self.counting_writer.writer().writeStructEndian(locator, .little);
251 }
252 const hdr: zip.EndRecord = .{
253 .signature = if (opt.sig) |s| s else zip.end_record_sig,
254 .disk_number = if (opt.disk_number) |n| n else 0,
255 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
256 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
257 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
258 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
259 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
260 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
261 };
262 try self.counting_writer.writer().writeStructEndian(hdr, .little);
263 if (opt.comment) |c|
264 try self.counting_writer.writer().writeAll(c);
265 }
266 };
267}
src/Package/Fetch.zig+164
...@@ -840,6 +840,7 @@ const FileType = enum {...@@ -840,6 +840,7 @@ const FileType = enum {
840 @"tar.xz",840 @"tar.xz",
841 @"tar.zst",841 @"tar.zst",
842 git_pack,842 git_pack,
843 zip,
843844
844 fn fromPath(file_path: []const u8) ?FileType {845 fn fromPath(file_path: []const u8) ?FileType {
845 if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;846 if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
...@@ -849,6 +850,7 @@ const FileType = enum {...@@ -849,6 +850,7 @@ const FileType = enum {
849 if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";850 if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
850 if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";851 if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
851 if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";852 if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
853 if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
852 return null;854 return null;
853 }855 }
854856
...@@ -1077,6 +1079,9 @@ fn unpackResource(...@@ -1077,6 +1079,9 @@ fn unpackResource(
1077 if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))1079 if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
1078 break :ft .@"tar.zst";1080 break :ft .@"tar.zst";
10791081
1082 if (ascii.eqlIgnoreCase(mime_type, "application/zip"))
1083 break :ft .zip;
1084
1080 if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and1085 if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
1081 !ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))1086 !ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
1082 {1087 {
...@@ -1157,6 +1162,7 @@ fn unpackResource(...@@ -1157,6 +1162,7 @@ fn unpackResource(
1157 .{@errorName(e)},1162 .{@errorName(e)},
1158 )),1163 )),
1159 },1164 },
1165 .zip => return try unzip(f, tmp_directory.handle, resource.reader()),
1160 }1166 }
1161}1167}
11621168
...@@ -1190,6 +1196,98 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes...@@ -1190,6 +1196,98 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackRes
1190 return res;1196 return res;
1191}1197}
11921198
1199fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1200 // We write the entire contents to a file first because zip files
1201 // must be processed back to front and they could be too large to
1202 // load into memory.
1203
1204 const cache_root = f.job_queue.global_cache;
1205
1206 // TODO: the downside of this solution is if we get a failure/crash/oom/power out
1207 // during this process, we leave behind a zip file that would be
1208 // difficult to know if/when it can be cleaned up.
1209 // Might be worth it to use a mechanism that enables other processes
1210 // to see if the owning process of a file is still alive (on linux this
1211 // can be done with file locks).
1212 // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0,
1213 // zig-cache/tmp/1, etc) which would mean that subsequent runs would
1214 // automatically clean up old dead files.
1215 // This could all be done with a simple TmpFile abstraction.
1216 const prefix = "tmp/";
1217 const suffix = ".zip";
1218
1219 const random_bytes_count = 20;
1220 const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count);
1221 var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined;
1222 @memcpy(zip_path[0..prefix.len], prefix);
1223 @memcpy(zip_path[prefix.len + random_path_len ..], suffix);
1224 {
1225 var random_bytes: [random_bytes_count]u8 = undefined;
1226 std.crypto.random.bytes(&random_bytes);
1227 _ = std.fs.base64_encoder.encode(
1228 zip_path[prefix.len..][0..random_path_len],
1229 &random_bytes,
1230 );
1231 }
1232
1233 defer cache_root.handle.deleteFile(&zip_path) catch {};
1234
1235 const eb = &f.error_bundle;
1236
1237 {
1238 var zip_file = cache_root.handle.createFile(
1239 &zip_path,
1240 .{},
1241 ) catch |err| return f.fail(f.location_tok, try eb.printString(
1242 "failed to create tmp zip file: {s}",
1243 .{@errorName(err)},
1244 ));
1245 defer zip_file.close();
1246 var buf: [std.mem.page_size]u8 = undefined;
1247 while (true) {
1248 const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(
1249 "read zip stream failed: {s}",
1250 .{@errorName(err)},
1251 ));
1252 if (len == 0) break;
1253 zip_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString(
1254 "write temporary zip file failed: {s}",
1255 .{@errorName(err)},
1256 ));
1257 }
1258 }
1259
1260 var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
1261 // no need to deinit since we are using an arena allocator
1262
1263 {
1264 var zip_file = cache_root.handle.openFile(
1265 &zip_path,
1266 .{},
1267 ) catch |err| return f.fail(f.location_tok, try eb.printString(
1268 "failed to open temporary zip file: {s}",
1269 .{@errorName(err)},
1270 ));
1271 defer zip_file.close();
1272
1273 std.zip.extract(out_dir, zip_file.seekableStream(), .{
1274 .allow_backslashes = true,
1275 .diagnostics = &diagnostics,
1276 }) catch |err| return f.fail(f.location_tok, try eb.printString(
1277 "zip extract failed: {s}",
1278 .{@errorName(err)},
1279 ));
1280 }
1281
1282 cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString(
1283 "delete temporary zip failed: {s}",
1284 .{@errorName(err)},
1285 ));
1286
1287 const res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
1288 return res;
1289}
1290
1193fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!UnpackResult {1291fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!UnpackResult {
1194 const arena = f.arena.allocator();1292 const arena = f.arena.allocator();
1195 const gpa = f.arena.child_allocator;1293 const gpa = f.arena.child_allocator;
...@@ -1895,6 +1993,72 @@ const UnpackResult = struct {...@@ -1895,6 +1993,72 @@ const UnpackResult = struct {
1895 }1993 }
1896};1994};
18971995
1996test "zip" {
1997 const gpa = std.testing.allocator;
1998 var tmp = std.testing.tmpDir(.{});
1999 defer tmp.cleanup();
2000
2001 const test_files = [_]std.zip.testutil.File{
2002 .{ .name = "foo", .content = "this is just foo\n", .compression = .store },
2003 .{ .name = "bar", .content = "another file\n", .compression = .deflate },
2004 };
2005 {
2006 var zip_file = try tmp.dir.createFile("test.zip", .{});
2007 defer zip_file.close();
2008 var bw = std.io.bufferedWriter(zip_file.writer());
2009 var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2010 try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2011 try bw.flush();
2012 }
2013
2014 const zip_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2015 defer gpa.free(zip_path);
2016
2017 var fb: TestFetchBuilder = undefined;
2018 var fetch = try fb.build(gpa, tmp.dir, zip_path);
2019 defer fb.deinit();
2020
2021 try fetch.run();
2022
2023 var out = try fb.packageDir();
2024 defer out.close();
2025
2026 try std.zip.testutil.expectFiles(&test_files, out, .{});
2027}
2028
2029test "zip with one root folder" {
2030 const gpa = std.testing.allocator;
2031 var tmp = std.testing.tmpDir(.{});
2032 defer tmp.cleanup();
2033
2034 const test_files = [_]std.zip.testutil.File{
2035 .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store },
2036 .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store },
2037 };
2038 {
2039 var zip_file = try tmp.dir.createFile("test.zip", .{});
2040 defer zip_file.close();
2041 var bw = std.io.bufferedWriter(zip_file.writer());
2042 var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2043 try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2044 try bw.flush();
2045 }
2046
2047 const zip_path = try std.fmt.allocPrint(gpa, "zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2048 defer gpa.free(zip_path);
2049
2050 var fb: TestFetchBuilder = undefined;
2051 var fetch = try fb.build(gpa, tmp.dir, zip_path);
2052 defer fb.deinit();
2053
2054 try fetch.run();
2055
2056 var out = try fb.packageDir();
2057 defer out.close();
2058
2059 try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" });
2060}
2061
1898test "tarball with duplicate paths" {2062test "tarball with duplicate paths" {
1899 // This tarball has duplicate path 'dir1/file1' to simulate case sensitve2063 // This tarball has duplicate path 'dir1/file1' to simulate case sensitve
1900 // file system on any file sytstem.2064 // file system on any file sytstem.