authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-18 15:05:39-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-23 02:37:11-07:00
logc44a90283699246172a77429a04288b19d38f075
treeb7bc00c264bcf9828325d1af3f22375352851da9
parent5c12783094ef6c36e2d26d58116fdb992306d044

fix zstd compilation errors from previous commit

2 files changed, 39 insertions(+), 64 deletions(-)

lib/std/compress/zstandard.zig+16-25
...@@ -7,13 +7,16 @@ pub const compressed_block = types.compressed_block;...@@ -7,13 +7,16 @@ pub const compressed_block = types.compressed_block;
77
8pub const decompress = @import("zstandard/decompress.zig");8pub const decompress = @import("zstandard/decompress.zig");
99
10pub const DecompressStreamOptions = struct {10pub const DecompressorOptions = struct {
11 verify_checksum: bool = true,11 verify_checksum: bool = true,
12 window_buffer: []u8,
13
14 /// Recommended amount by the standard. Lower than this may result
15 /// in inability to decompress common streams.
16 pub const default_window_buffer_len = 8 * 1024 * 1024;
12};17};
1318
14pub fn DecompressStream(19pub fn Decompressor(comptime ReaderType: type) type {
15 comptime ReaderType: type,
16) type {
17 return struct {20 return struct {
18 const Self = @This();21 const Self = @This();
1922
...@@ -49,13 +52,13 @@ pub fn DecompressStream(...@@ -49,13 +52,13 @@ pub fn DecompressStream(
4952
50 pub const Reader = std.io.Reader(*Self, Error, read);53 pub const Reader = std.io.Reader(*Self, Error, read);
5154
52 pub fn init(source: ReaderType, window_buffer: []u8, options: DecompressStreamOptions) Self {55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {
53 return Self{56 return .{
54 .source = std.io.countingReader(source),57 .source = std.io.countingReader(source),
55 .state = .NewFrame,58 .state = .NewFrame,
56 .decode_state = undefined,59 .decode_state = undefined,
57 .frame_context = undefined,60 .frame_context = undefined,
58 .buffer = .{ .data = window_buffer },61 .buffer = .{ .data = options.window_buffer },
59 .literal_fse_buffer = undefined,62 .literal_fse_buffer = undefined,
60 .match_fse_buffer = undefined,63 .match_fse_buffer = undefined,
61 .offset_fse_buffer = undefined,64 .offset_fse_buffer = undefined,
...@@ -199,20 +202,8 @@ pub fn DecompressStream(...@@ -199,20 +202,8 @@ pub fn DecompressStream(
199 };202 };
200}203}
201204
202pub fn decompressStreamOptions(205pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
203 reader: anytype,206 return Decompressor(@TypeOf(reader)).init(reader, options);
204 comptime options: DecompressStreamOptions,
205 window_buffer: *[options.window_size_max]u8,
206) DecompressStream(@TypeOf(reader), options) {
207 return DecompressStream(@TypeOf(reader), options).init(reader, window_buffer);
208}
209
210pub fn decompressStream(
211 reader: anytype,
212 window_buffer: []u8,
213 options: DecompressStreamOptions,
214) DecompressStream(@TypeOf(reader)) {
215 return DecompressStream(@TypeOf(reader)).init(reader, window_buffer, options);
216}207}
217208
218fn testDecompress(data: []const u8) ![]u8 {209fn testDecompress(data: []const u8) ![]u8 {
...@@ -220,7 +211,7 @@ fn testDecompress(data: []const u8) ![]u8 {...@@ -220,7 +211,7 @@ fn testDecompress(data: []const u8) ![]u8 {
220 defer std.testing.allocator.free(window_buffer);211 defer std.testing.allocator.free(window_buffer);
221212
222 var in_stream = std.io.fixedBufferStream(data);213 var in_stream = std.io.fixedBufferStream(data);
223 var zstd_stream = decompressStream(in_stream.reader(), window_buffer, .{});214 var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
224 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));215 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
225 return result;216 return result;
226}217}
...@@ -249,7 +240,7 @@ test "zstandard decompression" {...@@ -249,7 +240,7 @@ test "zstandard decompression" {
249}240}
250241
251test "zstandard streaming decompression" {242test "zstandard streaming decompression" {
252 // default stack size for wasm32 is too low for DecompressStream - slightly243 // default stack size for wasm32 is too low for Decompressor - slightly
253 // over 1MiB stack space is needed via the --stack CLI flag244 // over 1MiB stack space is needed via the --stack CLI flag
254 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;245 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
255246
...@@ -282,7 +273,7 @@ fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {...@@ -282,7 +273,7 @@ fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
282 defer std.testing.allocator.free(window_buffer);273 defer std.testing.allocator.free(window_buffer);
283274
284 var in_stream = std.io.fixedBufferStream(input);275 var in_stream = std.io.fixedBufferStream(input);
285 var stream = decompressStream(in_stream.reader(), window_buffer, .{});276 var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
286277
287 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));278 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
288 defer std.testing.allocator.free(result);279 defer std.testing.allocator.free(result);
...@@ -307,7 +298,7 @@ test "zero sized block" {...@@ -307,7 +298,7 @@ test "zero sized block" {
307}298}
308299
309test "zero sized block streaming" {300test "zero sized block streaming" {
310 // default stack size for wasm32 is too low for DecompressStream - slightly301 // default stack size for wasm32 is too low for Decompressor - slightly
311 // over 1MiB stack space is needed via the --stack CLI flag302 // over 1MiB stack space is needed via the --stack CLI flag
312 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;303 if (@import("builtin").target.cpu.arch == .wasm32) return error.SkipZigTest;
313304
src/Package/Fetch.zig+23-39
...@@ -1109,8 +1109,29 @@ fn unpackResource(...@@ -1109,8 +1109,29 @@ fn unpackResource(
1109 var dcp = std.compress.gzip.decompressor(br.reader());1109 var dcp = std.compress.gzip.decompressor(br.reader());
1110 try unpackTarball(f, tmp_directory.handle, dcp.reader());1110 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1111 },1111 },
1112 .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),1112 .@"tar.xz" => {
1113 .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper),1113 const gpa = f.arena.child_allocator;
1114 const reader = resource.reader();
1115 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1116 var dcp = std.compress.xz.decompress(gpa, br.reader()) catch |err| {
1117 return f.fail(f.location_tok, try eb.printString(
1118 "unable to decompress tarball: {s}",
1119 .{@errorName(err)},
1120 ));
1121 };
1122 defer dcp.deinit();
1123 try unpackTarball(f, tmp_directory.handle, dcp.reader());
1124 },
1125 .@"tar.zst" => {
1126 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
1127 const window_buffer = try f.arena.allocator().create([window_size]u8);
1128 const reader = resource.reader();
1129 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1130 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1131 .window_buffer = window_buffer,
1132 });
1133 return unpackTarball(f, tmp_directory.handle, dcp.reader());
1134 },
1114 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {1135 .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {
1115 error.FetchFailed => return error.FetchFailed,1136 error.FetchFailed => return error.FetchFailed,
1116 error.OutOfMemory => return error.OutOfMemory,1137 error.OutOfMemory => return error.OutOfMemory,
...@@ -1122,43 +1143,6 @@ fn unpackResource(...@@ -1122,43 +1143,6 @@ fn unpackResource(
1122 }1143 }
1123}1144}
11241145
1125// due to slight differences in the API of std.compress.(gzip|xz) and std.compress.zstd, zstd is
1126// wrapped for generic use in unpackTarballCompressed: see github.com/ziglang/zig/issues/14739
1127const ZstdWrapper = struct {
1128 fn DecompressType(comptime T: type) type {
1129 return Allocator.Error!std.compress.zstd.DecompressStream(T, .{});
1130 }
1131
1132 fn decompress(allocator: Allocator, reader: anytype) DecompressType(@TypeOf(reader)) {
1133 const window_size = std.compress.zstd.DecompressStreamOptions.default_window_size_max;
1134 const window_buffer = try allocator.create([window_size]u8);
1135 defer allocator.destroy(window_buffer);
1136 return std.compress.zstd.decompressStream(reader, window_buffer);
1137 }
1138};
1139
1140fn unpackTarballCompressed(
1141 f: *Fetch,
1142 out_dir: fs.Dir,
1143 resource: *Resource,
1144 Compression: anytype,
1145) RunError!void {
1146 const gpa = f.arena.child_allocator;
1147 const eb = &f.error_bundle;
1148 const reader = resource.reader();
1149 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1150
1151 var decompress = Compression.decompress(gpa, br.reader()) catch |err| {
1152 return f.fail(f.location_tok, try eb.printString(
1153 "unable to decompress tarball: {s}",
1154 .{@errorName(err)},
1155 ));
1156 };
1157 defer if (@hasDecl(Compression, "deinit")) decompress.deinit();
1158
1159 return unpackTarball(f, out_dir, decompress.reader());
1160}
1161
1162fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {1146fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
1163 const eb = &f.error_bundle;1147 const eb = &f.error_bundle;
1164 const gpa = f.arena.child_allocator;1148 const gpa = f.arena.child_allocator;