authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-25 13:15:23-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-25 14:15:33-07:00
log7f43f78b5bea8d6af8fa29451a427b43cb37291f
treeae6ce626e5372bcbd9c104efa51c5ea5b7f83991
parent13e08ea6f0fe6caa4ab0a0e6795f6864c96b9336

std.compress.zstd: keep frame state between blocks

1 files changed, 32 insertions(+), 42 deletions(-)

lib/std/compress/zstd/Decompress.zig+32-42
...@@ -23,6 +23,7 @@ const State = union(enum) {...@@ -23,6 +23,7 @@ const State = union(enum) {
23 frame: Frame,23 frame: Frame,
24 checksum: ?u32,24 checksum: ?u32,
25 decompressed_size: usize,25 decompressed_size: usize,
26 decode: Frame.Zstandard.Decode,
26 };27 };
27};28};
2829
...@@ -138,6 +139,7 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {...@@ -138,6 +139,7 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
138 .frame = try Frame.init(header, window_size_max, d.verify_checksum),139 .frame = try Frame.init(header, window_size_max, d.verify_checksum),
139 .checksum = null,140 .checksum = null,
140 .decompressed_size = 0,141 .decompressed_size = 0,
142 .decode = .init,
141 } };143 } };
142 },144 },
143 .skippable => {145 .skippable => {
...@@ -168,16 +170,13 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)...@@ -168,16 +170,13 @@ fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame)
168 bytes_written = block_size;170 bytes_written = block_size;
169 },171 },
170 .compressed => {172 .compressed => {
171 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
172 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
173 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
174 var literals_buffer: [zstd.block_size_max]u8 = undefined;173 var literals_buffer: [zstd.block_size_max]u8 = undefined;
175 var sequence_buffer: [zstd.block_size_max]u8 = undefined;174 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
176 var decode: Frame.Zstandard.Decode = .init(&literal_fse_buffer, &match_fse_buffer, &offset_fse_buffer, window_len);
177 var remaining: Limit = .limited(block_size);175 var remaining: Limit = .limited(block_size);
178 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);176 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
179 const sequences_header = try SequencesSection.Header.decode(in, &remaining);177 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
180178
179 const decode = &state.decode;
181 try decode.prepare(in, &remaining, literals, sequences_header);180 try decode.prepare(in, &remaining, literals, sequences_header);
182181
183 {182 {
...@@ -370,16 +369,15 @@ pub const Frame = struct {...@@ -370,16 +369,15 @@ pub const Frame = struct {
370 };369 };
371370
372 pub const Decode = struct {371 pub const Decode = struct {
373 window_len: u32,
374 repeat_offsets: [3]u32,372 repeat_offsets: [3]u32,
375373
376 offset: StateData(8),374 offset: StateData(8),
377 match: StateData(9),375 match: StateData(9),
378 literal: StateData(9),376 literal: StateData(9),
379377
380 offset_fse_buffer: []Table.Fse,378 literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse,
381 match_fse_buffer: []Table.Fse,379 match_fse_buffer: [zstd.table_size_max.match]Table.Fse,
382 literal_fse_buffer: []Table.Fse,380 offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse,
383381
384 fse_tables_undefined: bool,382 fse_tables_undefined: bool,
385383
...@@ -401,38 +399,30 @@ pub const Frame = struct {...@@ -401,38 +399,30 @@ pub const Frame = struct {
401 };399 };
402 }400 }
403401
404 pub fn init(402 const init: Decode = .{
405 literal_fse_buffer: []Table.Fse,403 .repeat_offsets = .{
406 match_fse_buffer: []Table.Fse,404 zstd.start_repeated_offset_1,
407 offset_fse_buffer: []Table.Fse,405 zstd.start_repeated_offset_2,
408 window_len: u32,406 zstd.start_repeated_offset_3,
409 ) Decode {407 },
410 return .{
411 .window_len = window_len,
412 .repeat_offsets = .{
413 zstd.start_repeated_offset_1,
414 zstd.start_repeated_offset_2,
415 zstd.start_repeated_offset_3,
416 },
417408
418 .offset = undefined,409 .offset = undefined,
419 .match = undefined,410 .match = undefined,
420 .literal = undefined,411 .literal = undefined,
421412
422 .literal_fse_buffer = literal_fse_buffer,413 .literal_fse_buffer = undefined,
423 .match_fse_buffer = match_fse_buffer,414 .match_fse_buffer = undefined,
424 .offset_fse_buffer = offset_fse_buffer,415 .offset_fse_buffer = undefined,
425416
426 .fse_tables_undefined = true,417 .fse_tables_undefined = true,
427418
428 .literal_written_count = 0,419 .literal_written_count = 0,
429 .literal_header = undefined,420 .literal_header = undefined,
430 .literal_streams = undefined,421 .literal_streams = undefined,
431 .literal_stream_reader = undefined,422 .literal_stream_reader = undefined,
432 .literal_stream_index = undefined,423 .literal_stream_index = undefined,
433 .huffman_tree = null,424 .huffman_tree = null,
434 };425 };
435 }
436426
437 pub const PrepareError = error{427 pub const PrepareError = error{
438 /// the (reversed) literal bitstream's first byte does not have any bits set428 /// the (reversed) literal bitstream's first byte does not have any bits set
...@@ -514,12 +504,12 @@ pub const Frame = struct {...@@ -514,12 +504,12 @@ pub const Frame = struct {
514 return self.repeat_offsets[0];504 return self.repeat_offsets[0];
515 }505 }
516506
517 const DataType = enum { offset, match, literal };507 const WhichFse = enum { offset, match, literal };
518508
519 /// TODO: don't use `@field`509 /// TODO: don't use `@field`
520 fn updateState(510 fn updateState(
521 self: *Decode,511 self: *Decode,
522 comptime choice: DataType,512 comptime choice: WhichFse,
523 bit_reader: *ReverseBitReader,513 bit_reader: *ReverseBitReader,
524 ) error{ MalformedFseBits, EndOfStream }!void {514 ) error{ MalformedFseBits, EndOfStream }!void {
525 switch (@field(self, @tagName(choice)).table) {515 switch (@field(self, @tagName(choice)).table) {
...@@ -549,7 +539,7 @@ pub const Frame = struct {...@@ -549,7 +539,7 @@ pub const Frame = struct {
549 self: *Decode,539 self: *Decode,
550 in: *Reader,540 in: *Reader,
551 remaining: *Limit,541 remaining: *Limit,
552 comptime choice: DataType,542 comptime choice: WhichFse,
553 mode: SequencesSection.Header.Mode,543 mode: SequencesSection.Header.Mode,
554 ) !void {544 ) !void {
555 const field_name = @tagName(choice);545 const field_name = @tagName(choice);
...@@ -576,10 +566,10 @@ pub const Frame = struct {...@@ -576,10 +566,10 @@ pub const Frame = struct {
576 &bit_reader,566 &bit_reader,
577 @field(zstd.table_symbol_count_max, field_name),567 @field(zstd.table_symbol_count_max, field_name),
578 @field(zstd.table_accuracy_log_max, field_name),568 @field(zstd.table_accuracy_log_max, field_name),
579 @field(self, field_name ++ "_fse_buffer"),569 &@field(self, field_name ++ "_fse_buffer"),
580 );570 );
581 @field(self, field_name).table = .{571 @field(self, field_name).table = .{
582 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],572 .fse = (&@field(self, field_name ++ "_fse_buffer"))[0..table_size],
583 };573 };
584 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);574 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
585 in.toss(bit_reader.index);575 in.toss(bit_reader.index);
...@@ -762,7 +752,7 @@ pub const Frame = struct {...@@ -762,7 +752,7 @@ pub const Frame = struct {
762 }752 }
763753
764 /// TODO: don't use `@field`754 /// TODO: don't use `@field`
765 fn getCode(self: *Decode, comptime choice: DataType) u32 {755 fn getCode(self: *Decode, comptime choice: WhichFse) u32 {
766 return switch (@field(self, @tagName(choice)).table) {756 return switch (@field(self, @tagName(choice)).table) {
767 .rle => |value| value,757 .rle => |value| value,
768 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,758 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,