authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-24 23:31:00-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-25 14:15:33-07:00
logee4f5b3f92988bb57849ea0e8d3d2651cda4b696
tree56d0173a273b0b61ae2d058fc4e45601a76dfedf
parent7f1c04423e5cfd628c58824aee543b4e79525c5f

std.compress.zstd: respect the window length

2 files changed, 29 insertions(+), 19 deletions(-)

lib/std/compress/zstd.zig+2-3
......@@ -1,12 +1,11 @@
11const std = @import("../std.zig");
22const assert = std.debug.assert;
33
4pub const Decompress = @import("zstd/Decompress.zig");
5
46/// Recommended amount by the standard. Lower than this may result in inability
57/// to decompress common streams.
68pub const default_window_len = 8 * 1024 * 1024;
7
8pub const Decompress = @import("zstd/Decompress.zig");
9
109pub const block_size_max = 1 << 17;
1110
1211pub const literals_length_default_distribution = [36]i16{
lib/std/compress/zstd/Decompress.zig+27-16
......@@ -10,6 +10,7 @@ input: *Reader,
1010reader: Reader,
1111state: State,
1212verify_checksum: bool,
13window_len: u32,
1314err: ?Error = null,
1415
1516const State = union(enum) {
......@@ -29,6 +30,8 @@ pub const Options = struct {
2930 /// Verifying checksums is not implemented yet and will cause a panic if
3031 /// you set this to true.
3132 verify_checksum: bool = false,
33 /// Affects the minimum capacity of the provided buffer.
34 window_len: u32 = zstd.default_window_len,
3235};
3336
3437pub const Error = error{
......@@ -65,11 +68,14 @@ pub const Error = error{
6568 WindowSizeUnknown,
6669};
6770
71/// If buffer that is written to is not big enough, some streams will fail with
72/// `error.OutputBufferUndersize`. A safe value is `zstd.default_window_len * 2`.
6873pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
6974 return .{
7075 .input = input,
7176 .state = .new_frame,
7277 .verify_checksum = options.verify_checksum,
78 .window_len = options.window_len,
7379 .reader = .{
7480 .vtable = &.{ .stream = stream },
7581 .buffer = buffer,
......@@ -143,6 +149,7 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
143149
144150fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame) !usize {
145151 const in = d.input;
152 const window_len = d.window_len;
146153
147154 const header_bytes = try in.takeArray(3);
148155 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
......@@ -153,12 +160,12 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)
153160 var bytes_written: usize = 0;
154161 switch (block_header.type) {
155162 .raw => {
156 try in.streamExact(w, block_size);
163 try in.streamExactPreserve(w, window_len, block_size);
157164 bytes_written = block_size;
158165 },
159166 .rle => {
160167 const byte = try in.takeByte();
161 try w.splatByteAll(byte, block_size);
168 try w.splatBytePreserve(window_len, byte, block_size);
162169 bytes_written = block_size;
163170 },
164171 .compressed => {
......@@ -167,7 +174,7 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)
167174 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
168175 var literals_buffer: [zstd.block_size_max]u8 = undefined;
169176 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
170 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer);
177 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer, window_len);
171178 var remaining: Limit = .limited(block_size);
172179 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
173180 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
......@@ -185,15 +192,16 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)
185192 try decode.readInitialFseState(&bit_stream);
186193
187194 // Ensures the following calls to `decodeSequence` will not flush.
188 if (frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
189 const dest = (try w.writableSliceGreedy(frame_block_size_max))[0..frame_block_size_max];
195 if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
196 const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];
197 const write_pos = dest.ptr - w.buffer.ptr;
190198 for (0..sequences_header.sequence_count - 1) |_| {
191 bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
199 bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
192200 try decode.updateState(.literal, &bit_stream);
193201 try decode.updateState(.match, &bit_stream);
194202 try decode.updateState(.offset, &bit_stream);
195203 }
196 bytes_written += try decode.decodeSequence(dest, bytes_written, &bit_stream);
204 bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
197205 if (bytes_written > dest.len) return error.MalformedSequence;
198206 w.advance(bytes_written);
199207 }
......@@ -363,6 +371,7 @@ pub const Frame = struct {
363371 };
364372
365373 pub const Decode = struct {
374 window_len: u32,
366375 repeat_offsets: [3]u32,
367376
368377 offset: StateData(8),
......@@ -397,8 +406,10 @@ pub const Frame = struct {
397406 literal_fse_buffer: []Table.Fse,
398407 match_fse_buffer: []Table.Fse,
399408 offset_fse_buffer: []Table.Fse,
409 window_len: u32,
400410 ) Decode {
401411 return .{
412 .window_len = window_len,
402413 .repeat_offsets = .{
403414 zstd.start_repeated_offset_1,
404415 zstd.start_repeated_offset_2,
......@@ -698,19 +709,19 @@ pub const Frame = struct {
698709 };
699710 }
700711
701 /// Decode `len` bytes of literals into `dest`.
702 fn decodeLiterals(self: *Decode, dest: *Writer, len: usize) !void {
703 switch (self.literal_header.block_type) {
712 /// Decode `len` bytes of literals into `w`.
713 fn decodeLiterals(d: *Decode, w: *Writer, len: usize) !void {
714 switch (d.literal_header.block_type) {
704715 .raw => {
705 try dest.writeAll(self.literal_streams.one[self.literal_written_count..][0..len]);
716 try w.writeAll(d.literal_streams.one[d.literal_written_count..][0..len]);
706717 },
707718 .rle => {
708 try dest.splatByteAll(self.literal_streams.one[0], len);
719 try w.splatByteAll(d.literal_streams.one[0], len);
709720 },
710721 .compressed, .treeless => {
711 if (len > dest.buffer.len) return error.OutputBufferUndersize;
712 const buf = try dest.writableSlice(len);
713 const huffman_tree = self.huffman_tree.?;
722 if (len > w.buffer.len) return error.OutputBufferUndersize;
723 const buf = try w.writableSlice(len);
724 const huffman_tree = d.huffman_tree.?;
714725 const max_bit_count = huffman_tree.max_bit_count;
715726 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
716727 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
......@@ -722,7 +733,7 @@ pub const Frame = struct {
722733 for (buf) |*out| {
723734 var prefix: u16 = 0;
724735 while (true) {
725 const new_bits = try self.readLiteralsBits(bit_count_to_read);
736 const new_bits = try d.readLiteralsBits(bit_count_to_read);
726737 prefix <<= bit_count_to_read;
727738 prefix |= new_bits;
728739 bits_read += bit_count_to_read;