authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-01-23 16:26:03+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
logfc64c279a497263c15feb857eb5442aa615179c4
tree7fdec373c00100b8c11140b38cb479d979442632
parentcbfaa876d466a885c54ead16a5901399619ed0c8

std.compress.zstandard: clean up api


3 files changed, 113 insertions(+), 101 deletions(-)

lib/std/compress/zstandard.zig+1
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1const std = @import("std");1const std = @import("std");
22
3pub const decompress = @import("zstandard/decompress.zig");3pub const decompress = @import("zstandard/decompress.zig");
4pub usingnamespace @import("zstandard/types.zig");
45
5test "decompression" {6test "decompression" {
6 const uncompressed = @embedFile("testdata/rfc8478.txt");7 const uncompressed = @embedFile("testdata/rfc8478.txt");
lib/std/compress/zstandard/decompress.zig+109-98
...@@ -3,10 +3,10 @@ const assert = std.debug.assert;...@@ -3,10 +3,10 @@ const assert = std.debug.assert;
33
4const types = @import("types.zig");4const types = @import("types.zig");
5const frame = types.frame;5const frame = types.frame;
6const Literals = types.compressed_block.Literals;6const LiteralsSection = types.compressed_block.LiteralsSection;
7const Sequences = types.compressed_block.Sequences;7const SequencesSection = types.compressed_block.SequencesSection;
8const Table = types.compressed_block.Table;8const Table = types.compressed_block.Table;
9const RingBuffer = @import("RingBuffer.zig");9pub const RingBuffer = @import("RingBuffer.zig");
1010
11const readInt = std.mem.readIntLittle;11const readInt = std.mem.readIntLittle;
12const readIntSlice = std.mem.readIntSliceLittle;12const readIntSlice = std.mem.readIntSliceLittle;
...@@ -55,7 +55,7 @@ pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWrit...@@ -55,7 +55,7 @@ pub fn decodeFrame(dest: []u8, src: []const u8, verify_checksum: bool) !ReadWrit
55 };55 };
56}56}
5757
58const DecodeState = struct {58pub const DecodeState = struct {
59 repeat_offsets: [3]u32,59 repeat_offsets: [3]u32,
6060
61 offset: StateData(8),61 offset: StateData(8),
...@@ -70,7 +70,7 @@ const DecodeState = struct {...@@ -70,7 +70,7 @@ const DecodeState = struct {
7070
71 literal_stream_reader: ReverseBitReader,71 literal_stream_reader: ReverseBitReader,
72 literal_stream_index: usize,72 literal_stream_index: usize,
73 huffman_tree: ?Literals.HuffmanTree,73 huffman_tree: ?LiteralsSection.HuffmanTree,
7474
75 literal_written_count: usize,75 literal_written_count: usize,
7676
...@@ -84,7 +84,55 @@ const DecodeState = struct {...@@ -84,7 +84,55 @@ const DecodeState = struct {
84 };84 };
85 }85 }
8686
87 fn readInitialState(self: *DecodeState, bit_reader: anytype) !void {87 pub fn prepare(
88 self: *DecodeState,
89 src: []const u8,
90 literals: LiteralsSection,
91 sequences_header: SequencesSection.Header,
92 ) !usize {
93 if (literals.huffman_tree) |tree| {
94 self.huffman_tree = tree;
95 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
96 return error.TreelessLiteralsFirst;
97 }
98
99 switch (literals.header.block_type) {
100 .raw, .rle => {},
101 .compressed, .treeless => {
102 self.literal_stream_index = 0;
103 switch (literals.streams) {
104 .one => |slice| try self.initLiteralStream(slice),
105 .four => |streams| try self.initLiteralStream(streams[0]),
106 }
107 },
108 }
109
110 if (sequences_header.sequence_count > 0) {
111 var bytes_read = try self.updateFseTable(
112 src,
113 .literal,
114 sequences_header.literal_lengths,
115 );
116
117 bytes_read += try self.updateFseTable(
118 src[bytes_read..],
119 .offset,
120 sequences_header.offsets,
121 );
122
123 bytes_read += try self.updateFseTable(
124 src[bytes_read..],
125 .match,
126 sequences_header.match_lengths,
127 );
128 self.fse_tables_undefined = false;
129
130 return bytes_read;
131 }
132 return 0;
133 }
134
135 pub fn readInitialFseState(self: *DecodeState, bit_reader: anytype) !void {
88 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);136 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
89 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);137 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
90 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);138 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
...@@ -130,7 +178,7 @@ const DecodeState = struct {...@@ -130,7 +178,7 @@ const DecodeState = struct {
130 self: *DecodeState,178 self: *DecodeState,
131 src: []const u8,179 src: []const u8,
132 comptime choice: DataType,180 comptime choice: DataType,
133 mode: Sequences.Header.Mode,181 mode: SequencesSection.Header.Mode,
134 ) !usize {182 ) !usize {
135 const field_name = @tagName(choice);183 const field_name = @tagName(choice);
136 switch (mode) {184 switch (mode) {
...@@ -213,7 +261,13 @@ const DecodeState = struct {...@@ -213,7 +261,13 @@ const DecodeState = struct {
213 };261 };
214 }262 }
215263
216 fn executeSequenceSlice(self: *DecodeState, dest: []u8, write_pos: usize, literals: Literals, sequence: Sequence) !void {264 fn executeSequenceSlice(
265 self: *DecodeState,
266 dest: []u8,
267 write_pos: usize,
268 literals: LiteralsSection,
269 sequence: Sequence,
270 ) !void {
217 try self.decodeLiteralsSlice(dest[write_pos..], literals, sequence.literal_length);271 try self.decodeLiteralsSlice(dest[write_pos..], literals, sequence.literal_length);
218272
219 // TODO: should we validate offset against max_window_size?273 // TODO: should we validate offset against max_window_size?
...@@ -225,7 +279,12 @@ const DecodeState = struct {...@@ -225,7 +279,12 @@ const DecodeState = struct {
225 std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);279 std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);
226 }280 }
227281
228 fn executeSequenceRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: Literals, sequence: Sequence) !void {282 fn executeSequenceRingBuffer(
283 self: *DecodeState,
284 dest: *RingBuffer,
285 literals: LiteralsSection,
286 sequence: Sequence,
287 ) !void {
229 try self.decodeLiteralsRingBuffer(dest, literals, sequence.literal_length);288 try self.decodeLiteralsRingBuffer(dest, literals, sequence.literal_length);
230 // TODO: check that ring buffer window is full enough for match copies289 // TODO: check that ring buffer window is full enough for match copies
231 const copy_slice = dest.sliceAt(dest.write_index + dest.data.len - sequence.offset, sequence.match_length);290 const copy_slice = dest.sliceAt(dest.write_index + dest.data.len - sequence.offset, sequence.match_length);
...@@ -234,11 +293,11 @@ const DecodeState = struct {...@@ -234,11 +293,11 @@ const DecodeState = struct {
234 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);293 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);
235 }294 }
236295
237 fn decodeSequenceSlice(296 pub fn decodeSequenceSlice(
238 self: *DecodeState,297 self: *DecodeState,
239 dest: []u8,298 dest: []u8,
240 write_pos: usize,299 write_pos: usize,
241 literals: Literals,300 literals: LiteralsSection,
242 bit_reader: anytype,301 bit_reader: anytype,
243 last_sequence: bool,302 last_sequence: bool,
244 ) !usize {303 ) !usize {
...@@ -255,10 +314,10 @@ const DecodeState = struct {...@@ -255,10 +314,10 @@ const DecodeState = struct {
255 return sequence.match_length + sequence.literal_length;314 return sequence.match_length + sequence.literal_length;
256 }315 }
257316
258 fn decodeSequenceRingBuffer(317 pub fn decodeSequenceRingBuffer(
259 self: *DecodeState,318 self: *DecodeState,
260 dest: *RingBuffer,319 dest: *RingBuffer,
261 literals: Literals,320 literals: LiteralsSection,
262 bit_reader: anytype,321 bit_reader: anytype,
263 last_sequence: bool,322 last_sequence: bool,
264 ) !usize {323 ) !usize {
...@@ -280,7 +339,7 @@ const DecodeState = struct {...@@ -280,7 +339,7 @@ const DecodeState = struct {
280 return sequence.match_length + sequence.literal_length;339 return sequence.match_length + sequence.literal_length;
281 }340 }
282341
283 fn nextLiteralMultiStream(self: *DecodeState, literals: Literals) !void {342 fn nextLiteralMultiStream(self: *DecodeState, literals: LiteralsSection) !void {
284 self.literal_stream_index += 1;343 self.literal_stream_index += 1;
285 try self.initLiteralStream(literals.streams.four[self.literal_stream_index]);344 try self.initLiteralStream(literals.streams.four[self.literal_stream_index]);
286 }345 }
...@@ -290,7 +349,7 @@ const DecodeState = struct {...@@ -290,7 +349,7 @@ const DecodeState = struct {
290 try self.literal_stream_reader.init(bytes);349 try self.literal_stream_reader.init(bytes);
291 }350 }
292351
293 fn decodeLiteralsSlice(self: *DecodeState, dest: []u8, literals: Literals, len: usize) !void {352 pub fn decodeLiteralsSlice(self: *DecodeState, dest: []u8, literals: LiteralsSection, len: usize) !void {
294 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;353 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;
295 switch (literals.header.block_type) {354 switch (literals.header.block_type) {
296 .raw => {355 .raw => {
...@@ -310,7 +369,7 @@ const DecodeState = struct {...@@ -310,7 +369,7 @@ const DecodeState = struct {
310 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;369 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
311 const huffman_tree = self.huffman_tree orelse unreachable;370 const huffman_tree = self.huffman_tree orelse unreachable;
312 const max_bit_count = huffman_tree.max_bit_count;371 const max_bit_count = huffman_tree.max_bit_count;
313 const starting_bit_count = Literals.HuffmanTree.weightToBitCount(372 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
314 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,373 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
315 max_bit_count,374 max_bit_count,
316 );375 );
...@@ -345,7 +404,7 @@ const DecodeState = struct {...@@ -345,7 +404,7 @@ const DecodeState = struct {
345 },404 },
346 .index => |index| {405 .index => |index| {
347 huffman_tree_index = index;406 huffman_tree_index = index;
348 const bit_count = Literals.HuffmanTree.weightToBitCount(407 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
349 huffman_tree.nodes[index].weight,408 huffman_tree.nodes[index].weight,
350 max_bit_count,409 max_bit_count,
351 );410 );
...@@ -359,7 +418,7 @@ const DecodeState = struct {...@@ -359,7 +418,7 @@ const DecodeState = struct {
359 }418 }
360 }419 }
361420
362 fn decodeLiteralsRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: Literals, len: usize) !void {421 pub fn decodeLiteralsRingBuffer(self: *DecodeState, dest: *RingBuffer, literals: LiteralsSection, len: usize) !void {
363 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;422 if (self.literal_written_count + len > literals.header.regenerated_size) return error.MalformedLiteralsLength;
364 switch (literals.header.block_type) {423 switch (literals.header.block_type) {
365 .raw => {424 .raw => {
...@@ -378,7 +437,7 @@ const DecodeState = struct {...@@ -378,7 +437,7 @@ const DecodeState = struct {
378 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;437 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
379 const huffman_tree = self.huffman_tree orelse unreachable;438 const huffman_tree = self.huffman_tree orelse unreachable;
380 const max_bit_count = huffman_tree.max_bit_count;439 const max_bit_count = huffman_tree.max_bit_count;
381 const starting_bit_count = Literals.HuffmanTree.weightToBitCount(440 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
382 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,441 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
383 max_bit_count,442 max_bit_count,
384 );443 );
...@@ -413,7 +472,7 @@ const DecodeState = struct {...@@ -413,7 +472,7 @@ const DecodeState = struct {
413 },472 },
414 .index => |index| {473 .index => |index| {
415 huffman_tree_index = index;474 huffman_tree_index = index;
416 const bit_count = Literals.HuffmanTree.weightToBitCount(475 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
417 huffman_tree.nodes[index].weight,476 huffman_tree.nodes[index].weight,
418 max_bit_count,477 max_bit_count,
419 );478 );
...@@ -647,54 +706,6 @@ fn decodeRleBlockRingBuffer(dest: *RingBuffer, src: []const u8, block_size: u21,...@@ -647,54 +706,6 @@ fn decodeRleBlockRingBuffer(dest: *RingBuffer, src: []const u8, block_size: u21,
647 return block_size;706 return block_size;
648}707}
649708
650fn prepareDecodeState(
651 decode_state: *DecodeState,
652 src: []const u8,
653 literals: Literals,
654 sequences_header: Sequences.Header,
655) !usize {
656 if (literals.huffman_tree) |tree| {
657 decode_state.huffman_tree = tree;
658 } else if (literals.header.block_type == .treeless and decode_state.huffman_tree == null) {
659 return error.TreelessLiteralsFirst;
660 }
661
662 switch (literals.header.block_type) {
663 .raw, .rle => {},
664 .compressed, .treeless => {
665 decode_state.literal_stream_index = 0;
666 switch (literals.streams) {
667 .one => |slice| try decode_state.initLiteralStream(slice),
668 .four => |streams| try decode_state.initLiteralStream(streams[0]),
669 }
670 },
671 }
672
673 if (sequences_header.sequence_count > 0) {
674 var bytes_read = try decode_state.updateFseTable(
675 src,
676 .literal,
677 sequences_header.literal_lengths,
678 );
679
680 bytes_read += try decode_state.updateFseTable(
681 src[bytes_read..],
682 .offset,
683 sequences_header.offsets,
684 );
685
686 bytes_read += try decode_state.updateFseTable(
687 src[bytes_read..],
688 .match,
689 sequences_header.match_lengths,
690 );
691 decode_state.fse_tables_undefined = false;
692
693 return bytes_read;
694 }
695 return 0;
696}
697
698pub fn decodeBlock(709pub fn decodeBlock(
699 dest: []u8,710 dest: []u8,
700 src: []const u8,711 src: []const u8,
...@@ -715,7 +726,7 @@ pub fn decodeBlock(...@@ -715,7 +726,7 @@ pub fn decodeBlock(
715 const literals = try decodeLiteralsSection(src, &bytes_read);726 const literals = try decodeLiteralsSection(src, &bytes_read);
716 const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read);727 const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read);
717728
718 bytes_read += try prepareDecodeState(decode_state, src[bytes_read..], literals, sequences_header);729 bytes_read += try decode_state.prepare(src[bytes_read..], literals, sequences_header);
719730
720 var bytes_written: usize = 0;731 var bytes_written: usize = 0;
721 if (sequences_header.sequence_count > 0) {732 if (sequences_header.sequence_count > 0) {
...@@ -723,7 +734,7 @@ pub fn decodeBlock(...@@ -723,7 +734,7 @@ pub fn decodeBlock(
723 var bit_stream: ReverseBitReader = undefined;734 var bit_stream: ReverseBitReader = undefined;
724 try bit_stream.init(bit_stream_bytes);735 try bit_stream.init(bit_stream_bytes);
725736
726 try decode_state.readInitialState(&bit_stream);737 try decode_state.readInitialFseState(&bit_stream);
727738
728 var i: usize = 0;739 var i: usize = 0;
729 while (i < sequences_header.sequence_count) : (i += 1) {740 while (i < sequences_header.sequence_count) : (i += 1) {
...@@ -780,7 +791,7 @@ pub fn decodeBlockRingBuffer(...@@ -780,7 +791,7 @@ pub fn decodeBlockRingBuffer(
780 const literals = try decodeLiteralsSection(src, &bytes_read);791 const literals = try decodeLiteralsSection(src, &bytes_read);
781 const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read);792 const sequences_header = try decodeSequencesHeader(src[bytes_read..], &bytes_read);
782793
783 bytes_read += try prepareDecodeState(decode_state, src[bytes_read..], literals, sequences_header);794 bytes_read += try decode_state.prepare(src[bytes_read..], literals, sequences_header);
784795
785 var bytes_written: usize = 0;796 var bytes_written: usize = 0;
786 if (sequences_header.sequence_count > 0) {797 if (sequences_header.sequence_count > 0) {
...@@ -788,7 +799,7 @@ pub fn decodeBlockRingBuffer(...@@ -788,7 +799,7 @@ pub fn decodeBlockRingBuffer(
788 var bit_stream: ReverseBitReader = undefined;799 var bit_stream: ReverseBitReader = undefined;
789 try bit_stream.init(bit_stream_bytes);800 try bit_stream.init(bit_stream_bytes);
790801
791 try decode_state.readInitialState(&bit_stream);802 try decode_state.readInitialFseState(&bit_stream);
792803
793 var i: usize = 0;804 var i: usize = 0;
794 while (i < sequences_header.sequence_count) : (i += 1) {805 while (i < sequences_header.sequence_count) : (i += 1) {
...@@ -928,7 +939,7 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {...@@ -928,7 +939,7 @@ pub fn decodeBlockHeader(src: *const [3]u8) frame.ZStandard.Block.Header {
928 };939 };
929}940}
930941
931pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals {942pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !LiteralsSection {
932 // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)943 // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)
933 var bytes_read: usize = 0;944 var bytes_read: usize = 0;
934 const header = decodeLiteralsHeader(src, &bytes_read);945 const header = decodeLiteralsHeader(src, &bytes_read);
...@@ -936,7 +947,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals...@@ -936,7 +947,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals
936 .raw => {947 .raw => {
937 const stream = src[bytes_read .. bytes_read + header.regenerated_size];948 const stream = src[bytes_read .. bytes_read + header.regenerated_size];
938 consumed_count.* += header.regenerated_size + bytes_read;949 consumed_count.* += header.regenerated_size + bytes_read;
939 return Literals{950 return LiteralsSection{
940 .header = header,951 .header = header,
941 .huffman_tree = null,952 .huffman_tree = null,
942 .streams = .{ .one = stream },953 .streams = .{ .one = stream },
...@@ -945,7 +956,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals...@@ -945,7 +956,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals
945 .rle => {956 .rle => {
946 const stream = src[bytes_read .. bytes_read + 1];957 const stream = src[bytes_read .. bytes_read + 1];
947 consumed_count.* += 1 + bytes_read;958 consumed_count.* += 1 + bytes_read;
948 return Literals{959 return LiteralsSection{
949 .header = header,960 .header = header,
950 .huffman_tree = null,961 .huffman_tree = null,
951 .streams = .{ .one = stream },962 .streams = .{ .one = stream },
...@@ -966,7 +977,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals...@@ -966,7 +977,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals
966 const stream = src[bytes_read .. bytes_read + total_streams_size];977 const stream = src[bytes_read .. bytes_read + total_streams_size];
967 bytes_read += total_streams_size;978 bytes_read += total_streams_size;
968 consumed_count.* += bytes_read;979 consumed_count.* += bytes_read;
969 return Literals{980 return LiteralsSection{
970 .header = header,981 .header = header,
971 .huffman_tree = huffman_tree,982 .huffman_tree = huffman_tree,
972 .streams = .{ .one = stream },983 .streams = .{ .one = stream },
...@@ -988,7 +999,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals...@@ -988,7 +999,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals
988999
989 consumed_count.* += total_streams_size + bytes_read;1000 consumed_count.* += total_streams_size + bytes_read;
9901001
991 return Literals{1002 return LiteralsSection{
992 .header = header,1003 .header = header,
993 .huffman_tree = huffman_tree,1004 .huffman_tree = huffman_tree,
994 .streams = .{ .four = .{1005 .streams = .{ .four = .{
...@@ -1002,7 +1013,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals...@@ -1002,7 +1013,7 @@ pub fn decodeLiteralsSection(src: []const u8, consumed_count: *usize) !Literals
1002 }1013 }
1003}1014}
10041015
1005fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanTree {1016fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !LiteralsSection.HuffmanTree {
1006 var bytes_read: usize = 0;1017 var bytes_read: usize = 0;
1007 bytes_read += 1;1018 bytes_read += 1;
1008 const header = src[0];1019 const header = src[0];
...@@ -1094,7 +1105,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT...@@ -1094,7 +1105,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT
1094 weights[symbol_count - 1] = @intCast(u4, std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1);1105 weights[symbol_count - 1] = @intCast(u4, std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1);
1095 log.debug("weights[{d}] = {d}", .{ symbol_count - 1, weights[symbol_count - 1] });1106 log.debug("weights[{d}] = {d}", .{ symbol_count - 1, weights[symbol_count - 1] });
10961107
1097 var weight_sorted_prefixed_symbols: [256]Literals.HuffmanTree.PrefixedSymbol = undefined;1108 var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
1098 for (weight_sorted_prefixed_symbols[0..symbol_count]) |_, i| {1109 for (weight_sorted_prefixed_symbols[0..symbol_count]) |_, i| {
1099 weight_sorted_prefixed_symbols[i] = .{1110 weight_sorted_prefixed_symbols[i] = .{
1100 .symbol = @intCast(u8, i),1111 .symbol = @intCast(u8, i),
...@@ -1104,7 +1115,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT...@@ -1104,7 +1115,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT
1104 }1115 }
11051116
1106 std.sort.sort(1117 std.sort.sort(
1107 Literals.HuffmanTree.PrefixedSymbol,1118 LiteralsSection.HuffmanTree.PrefixedSymbol,
1108 weight_sorted_prefixed_symbols[0..symbol_count],1119 weight_sorted_prefixed_symbols[0..symbol_count],
1109 weights,1120 weights,
1110 lessThanByWeight,1121 lessThanByWeight,
...@@ -1137,7 +1148,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT...@@ -1137,7 +1148,7 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT
1137 }1148 }
1138 }1149 }
1139 consumed_count.* += bytes_read;1150 consumed_count.* += bytes_read;
1140 const tree = Literals.HuffmanTree{1151 const tree = LiteralsSection.HuffmanTree{
1141 .max_bit_count = max_number_of_bits,1152 .max_bit_count = max_number_of_bits,
1142 .symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1),1153 .symbol_count_minus_one = @intCast(u8, prefixed_symbol_count - 1),
1143 .nodes = weight_sorted_prefixed_symbols,1154 .nodes = weight_sorted_prefixed_symbols,
...@@ -1148,8 +1159,8 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT...@@ -1148,8 +1159,8 @@ fn decodeHuffmanTree(src: []const u8, consumed_count: *usize) !Literals.HuffmanT
11481159
1149fn lessThanByWeight(1160fn lessThanByWeight(
1150 weights: [256]u4,1161 weights: [256]u4,
1151 lhs: Literals.HuffmanTree.PrefixedSymbol,1162 lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
1152 rhs: Literals.HuffmanTree.PrefixedSymbol,1163 rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
1153) bool {1164) bool {
1154 // NOTE: this function relies on the use of a stable sorting algorithm,1165 // NOTE: this function relies on the use of a stable sorting algorithm,
1155 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;1166 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
...@@ -1157,11 +1168,11 @@ fn lessThanByWeight(...@@ -1157,11 +1168,11 @@ fn lessThanByWeight(
1157 return weights[lhs.symbol] < weights[rhs.symbol];1168 return weights[lhs.symbol] < weights[rhs.symbol];
1158}1169}
11591170
1160pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.Header {1171pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) LiteralsSection.Header {
1161 // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)1172 // TODO: we probably want to enable safety for release-fast and release-small (or insert custom checks)
1162 const start = consumed_count.*;1173 const start = consumed_count.*;
1163 const byte0 = src[0];1174 const byte0 = src[0];
1164 const block_type = @intToEnum(Literals.BlockType, byte0 & 0b11);1175 const block_type = @intToEnum(LiteralsSection.BlockType, byte0 & 0b11);
1165 const size_format = @intCast(u2, (byte0 & 0b1100) >> 2);1176 const size_format = @intCast(u2, (byte0 & 0b1100) >> 2);
1166 var regenerated_size: u20 = undefined;1177 var regenerated_size: u20 = undefined;
1167 var compressed_size: ?u18 = null;1178 var compressed_size: ?u18 = null;
...@@ -1220,7 +1231,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He...@@ -1220,7 +1231,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He
1220 compressed_size,1231 compressed_size,
1221 },1232 },
1222 );1233 );
1223 return Literals.Header{1234 return LiteralsSection.Header{
1224 .block_type = block_type,1235 .block_type = block_type,
1225 .size_format = size_format,1236 .size_format = size_format,
1226 .regenerated_size = regenerated_size,1237 .regenerated_size = regenerated_size,
...@@ -1228,7 +1239,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He...@@ -1228,7 +1239,7 @@ pub fn decodeLiteralsHeader(src: []const u8, consumed_count: *usize) Literals.He
1228 };1239 };
1229}1240}
12301241
1231fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Header {1242pub fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !SequencesSection.Header {
1232 var sequence_count: u24 = undefined;1243 var sequence_count: u24 = undefined;
12331244
1234 var bytes_read: usize = 0;1245 var bytes_read: usize = 0;
...@@ -1237,7 +1248,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea...@@ -1237,7 +1248,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea
1237 bytes_read += 1;1248 bytes_read += 1;
1238 log.debug("decoded sequences header '{}': sequence count = 0", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])});1249 log.debug("decoded sequences header '{}': sequence count = 0", .{std.fmt.fmtSliceHexUpper(src[0..bytes_read])});
1239 consumed_count.* += bytes_read;1250 consumed_count.* += bytes_read;
1240 return Sequences.Header{1251 return SequencesSection.Header{
1241 .sequence_count = 0,1252 .sequence_count = 0,
1242 .offsets = undefined,1253 .offsets = undefined,
1243 .match_lengths = undefined,1254 .match_lengths = undefined,
...@@ -1258,9 +1269,9 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea...@@ -1258,9 +1269,9 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea
1258 bytes_read += 1;1269 bytes_read += 1;
12591270
1260 consumed_count.* += bytes_read;1271 consumed_count.* += bytes_read;
1261 const matches_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00001100) >> 2);1272 const matches_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00001100) >> 2);
1262 const offsets_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b00110000) >> 4);1273 const offsets_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b00110000) >> 4);
1263 const literal_mode = @intToEnum(Sequences.Header.Mode, (compression_modes & 0b11000000) >> 6);1274 const literal_mode = @intToEnum(SequencesSection.Header.Mode, (compression_modes & 0b11000000) >> 6);
1264 log.debug("decoded sequences header '{}': (sc={d},o={s},m={s},l={s})", .{1275 log.debug("decoded sequences header '{}': (sc={d},o={s},m={s},l={s})", .{
1265 std.fmt.fmtSliceHexUpper(src[0..bytes_read]),1276 std.fmt.fmtSliceHexUpper(src[0..bytes_read]),
1266 sequence_count,1277 sequence_count,
...@@ -1270,7 +1281,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea...@@ -1270,7 +1281,7 @@ fn decodeSequencesHeader(src: []const u8, consumed_count: *usize) !Sequences.Hea
1270 });1281 });
1271 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;1282 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
12721283
1273 return Sequences.Header{1284 return SequencesSection.Header{
1274 .sequence_count = sequence_count,1285 .sequence_count = sequence_count,
1275 .offsets = offsets_mode,1286 .offsets = offsets_mode,
1276 .match_lengths = matches_mode,1287 .match_lengths = matches_mode,
...@@ -1428,25 +1439,25 @@ const ReversedByteReader = struct {...@@ -1428,25 +1439,25 @@ const ReversedByteReader = struct {
1428 }1439 }
1429};1440};
14301441
1431const ReverseBitReader = struct {1442pub const ReverseBitReader = struct {
1432 byte_reader: ReversedByteReader,1443 byte_reader: ReversedByteReader,
1433 bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader),1444 bit_reader: std.io.BitReader(.Big, ReversedByteReader.Reader),
14341445
1435 fn init(self: *ReverseBitReader, bytes: []const u8) !void {1446 pub fn init(self: *ReverseBitReader, bytes: []const u8) !void {
1436 self.byte_reader = ReversedByteReader.init(bytes);1447 self.byte_reader = ReversedByteReader.init(bytes);
1437 self.bit_reader = std.io.bitReader(.Big, self.byte_reader.reader());1448 self.bit_reader = std.io.bitReader(.Big, self.byte_reader.reader());
1438 while (0 == self.readBitsNoEof(u1, 1) catch return error.BitStreamHasNoStartBit) {}1449 while (0 == self.readBitsNoEof(u1, 1) catch return error.BitStreamHasNoStartBit) {}
1439 }1450 }
14401451
1441 fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U {1452 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: usize) !U {
1442 return self.bit_reader.readBitsNoEof(U, num_bits);1453 return self.bit_reader.readBitsNoEof(U, num_bits);
1443 }1454 }
14441455
1445 fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) !U {1456 pub fn readBits(self: *@This(), comptime U: type, num_bits: usize, out_bits: *usize) !U {
1446 return try self.bit_reader.readBits(U, num_bits, out_bits);1457 return try self.bit_reader.readBits(U, num_bits, out_bits);
1447 }1458 }
14481459
1449 fn alignToByte(self: *@This()) void {1460 pub fn alignToByte(self: *@This()) void {
1450 self.bit_reader.alignToByte();1461 self.bit_reader.alignToByte();
1451 }1462 }
1452};1463};
...@@ -1514,7 +1525,7 @@ fn dumpFseTable(prefix: []const u8, table: []const Table.Fse) void {...@@ -1514,7 +1525,7 @@ fn dumpFseTable(prefix: []const u8, table: []const Table.Fse) void {
1514 }1525 }
1515}1526}
15161527
1517fn dumpHuffmanTree(tree: Literals.HuffmanTree) void {1528fn dumpHuffmanTree(tree: LiteralsSection.HuffmanTree) void {
1518 log.debug("Huffman tree: max bit count = {}, symbol count = {}", .{ tree.max_bit_count, tree.symbol_count_minus_one + 1 });1529 log.debug("Huffman tree: max bit count = {}, symbol count = {}", .{ tree.max_bit_count, tree.symbol_count_minus_one + 1 });
1519 for (tree.nodes[0 .. tree.symbol_count_minus_one + 1]) |node| {1530 for (tree.nodes[0 .. tree.symbol_count_minus_one + 1]) |node| {
1520 log.debug("symbol = {[symbol]d}, prefix = {[prefix]d}, weight = {[weight]d}", node);1531 log.debug("symbol = {[symbol]d}, prefix = {[prefix]d}, weight = {[weight]d}", node);
lib/std/compress/zstandard/types.zig+3-3
...@@ -52,7 +52,7 @@ pub const frame = struct {...@@ -52,7 +52,7 @@ pub const frame = struct {
52};52};
5353
54pub const compressed_block = struct {54pub const compressed_block = struct {
55 pub const Literals = struct {55 pub const LiteralsSection = struct {
56 header: Header,56 header: Header,
57 huffman_tree: ?HuffmanTree,57 huffman_tree: ?HuffmanTree,
58 streams: Streams,58 streams: Streams,
...@@ -119,8 +119,8 @@ pub const compressed_block = struct {...@@ -119,8 +119,8 @@ pub const compressed_block = struct {
119 }119 }
120 };120 };
121121
122 pub const Sequences = struct {122 pub const SequencesSection = struct {
123 header: Sequences.Header,123 header: SequencesSection.Header,
124 literals_length_table: Table,124 literals_length_table: Table,
125 offset_table: Table,125 offset_table: Table,
126 match_length_table: Table,126 match_length_table: Table,