authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-25 20:02:53-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-07-25 20:02:53-07:00
log66e49d93b7b3815a5213b39d19b3c5f39c21ffc3
tree311d2907194f6d87ba62823622250260b18a349c
parent9e11727c7c88289554251646166f844376dcbbe9
parent733b2082565a86daac6f2b22a1a27f0b60621b4e
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #24559 from ziglang/zstd

std: rework zstd for new I/O API

20 files changed, 2188 insertions(+), 3072 deletions(-)

lib/std/Io/DeprecatedReader.zig+2-2
......@@ -373,11 +373,11 @@ pub fn discard(self: Self) anyerror!u64 {
373373}
374374
375375/// Helper for bridging to the new `Reader` API while upgrading.
376pub fn adaptToNewApi(self: *const Self) Adapter {
376pub fn adaptToNewApi(self: *const Self, buffer: []u8) Adapter {
377377 return .{
378378 .derp_reader = self.*,
379379 .new_interface = .{
380 .buffer = &.{},
380 .buffer = buffer,
381381 .vtable = &.{ .stream = Adapter.stream },
382382 .seek = 0,
383383 .end = 0,
lib/std/Io/Reader.zig+28-2
......@@ -185,6 +185,32 @@ pub fn streamExact64(r: *Reader, w: *Writer, n: u64) StreamError!void {
185185 while (remaining != 0) remaining -= try r.stream(w, .limited64(remaining));
186186}
187187
188/// "Pump" exactly `n` bytes from the reader to the writer.
189///
190/// When draining `w`, ensures that at least `preserve_len` bytes remain
191/// buffered.
192///
193/// Asserts `Writer.buffer` capacity exceeds `preserve_len`.
194pub fn streamExactPreserve(r: *Reader, w: *Writer, preserve_len: usize, n: usize) StreamError!void {
195 if (w.end + n <= w.buffer.len) {
196 @branchHint(.likely);
197 return streamExact(r, w, n);
198 }
199 // If `n` is large, we can ignore `preserve_len` up to a point.
200 var remaining = n;
201 while (remaining > preserve_len) {
202 assert(remaining != 0);
203 remaining -= try r.stream(w, .limited(remaining - preserve_len));
204 if (w.end + remaining <= w.buffer.len) return streamExact(r, w, remaining);
205 }
206 // All the next bytes received must be preserved.
207 if (preserve_len < w.end) {
208 @memmove(w.buffer[0..preserve_len], w.buffer[w.end - preserve_len ..][0..preserve_len]);
209 w.end = preserve_len;
210 }
211 return streamExact(r, w, remaining);
212}
213
188214/// "Pump" data from the reader to the writer, handling `error.EndOfStream` as
189215/// a success case.
190216///
......@@ -240,7 +266,7 @@ pub fn allocRemaining(r: *Reader, gpa: Allocator, limit: Limit) LimitedAllocErro
240266/// such case, the next byte that would be read will be the first one to exceed
241267/// `limit`, and all preceeding bytes have been appended to `list`.
242268///
243/// Asserts `buffer` has nonzero capacity.
269/// If `limit` is not `Limit.unlimited`, asserts `buffer` has nonzero capacity.
244270///
245271/// See also:
246272/// * `allocRemaining`
......@@ -251,7 +277,7 @@ pub fn appendRemaining(
251277 list: *std.ArrayListAlignedUnmanaged(u8, alignment),
252278 limit: Limit,
253279) LimitedAllocError!void {
254 assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
280 if (limit != .unlimited) assert(r.buffer.len != 0); // Needed to detect limit exceeded without losing data.
255281 const buffer_contents = r.buffer[r.seek..r.end];
256282 const copy_len = limit.minInt(buffer_contents.len);
257283 try list.appendSlice(gpa, r.buffer[0..copy_len]);
lib/std/Io/Writer.zig+70-23
......@@ -256,10 +256,10 @@ test "fixed buffer flush" {
256256 try testing.expectEqual(10, buffer[0]);
257257}
258258
259/// Calls `VTable.drain` but hides the last `preserve_length` bytes from the
259/// Calls `VTable.drain` but hides the last `preserve_len` bytes from the
260260/// implementation, keeping them buffered.
261pub fn drainPreserve(w: *Writer, preserve_length: usize) Error!void {
262 const temp_end = w.end -| preserve_length;
261pub fn drainPreserve(w: *Writer, preserve_len: usize) Error!void {
262 const temp_end = w.end -| preserve_len;
263263 const preserved = w.buffer[temp_end..w.end];
264264 w.end = temp_end;
265265 defer w.end += preserved.len;
......@@ -310,24 +310,38 @@ pub fn writableSliceGreedy(w: *Writer, minimum_length: usize) Error![]u8 {
310310}
311311
312312/// Asserts the provided buffer has total capacity enough for `minimum_length`
313/// and `preserve_length` combined.
313/// and `preserve_len` combined.
314314///
315315/// Does not `advance` the buffer end position.
316316///
317/// When draining the buffer, ensures that at least `preserve_length` bytes
317/// When draining the buffer, ensures that at least `preserve_len` bytes
318318/// remain buffered.
319319///
320/// If `preserve_length` is zero, this is equivalent to `writableSliceGreedy`.
321pub fn writableSliceGreedyPreserve(w: *Writer, preserve_length: usize, minimum_length: usize) Error![]u8 {
322 assert(w.buffer.len >= preserve_length + minimum_length);
320/// If `preserve_len` is zero, this is equivalent to `writableSliceGreedy`.
321pub fn writableSliceGreedyPreserve(w: *Writer, preserve_len: usize, minimum_length: usize) Error![]u8 {
322 assert(w.buffer.len >= preserve_len + minimum_length);
323323 while (w.buffer.len - w.end < minimum_length) {
324 try drainPreserve(w, preserve_length);
324 try drainPreserve(w, preserve_len);
325325 } else {
326326 @branchHint(.likely);
327327 return w.buffer[w.end..];
328328 }
329329}
330330
331/// Asserts the provided buffer has total capacity enough for `len`.
332///
333/// Advances the buffer end position by `len`.
334///
335/// When draining the buffer, ensures that at least `preserve_len` bytes
336/// remain buffered.
337///
338/// If `preserve_len` is zero, this is equivalent to `writableSlice`.
339pub fn writableSlicePreserve(w: *Writer, preserve_len: usize, len: usize) Error![]u8 {
340 const big_slice = try w.writableSliceGreedyPreserve(preserve_len, len);
341 advance(w, len);
342 return big_slice[0..len];
343}
344
331345pub const WritableVectorIterator = struct {
332346 first: []u8,
333347 middle: []const []u8 = &.{},
......@@ -523,16 +537,16 @@ pub fn write(w: *Writer, bytes: []const u8) Error!usize {
523537 return w.vtable.drain(w, &.{bytes}, 1);
524538}
525539
526/// Asserts `buffer` capacity exceeds `preserve_length`.
527pub fn writePreserve(w: *Writer, preserve_length: usize, bytes: []const u8) Error!usize {
528 assert(preserve_length <= w.buffer.len);
540/// Asserts `buffer` capacity exceeds `preserve_len`.
541pub fn writePreserve(w: *Writer, preserve_len: usize, bytes: []const u8) Error!usize {
542 assert(preserve_len <= w.buffer.len);
529543 if (w.end + bytes.len <= w.buffer.len) {
530544 @branchHint(.likely);
531545 @memcpy(w.buffer[w.end..][0..bytes.len], bytes);
532546 w.end += bytes.len;
533547 return bytes.len;
534548 }
535 const temp_end = w.end -| preserve_length;
549 const temp_end = w.end -| preserve_len;
536550 const preserved = w.buffer[temp_end..w.end];
537551 w.end = temp_end;
538552 defer w.end += preserved.len;
......@@ -552,13 +566,13 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void {
552566/// Calls `drain` as many times as necessary such that all of `bytes` are
553567/// transferred.
554568///
555/// When draining the buffer, ensures that at least `preserve_length` bytes
569/// When draining the buffer, ensures that at least `preserve_len` bytes
556570/// remain buffered.
557571///
558/// Asserts `buffer` capacity exceeds `preserve_length`.
559pub fn writeAllPreserve(w: *Writer, preserve_length: usize, bytes: []const u8) Error!void {
572/// Asserts `buffer` capacity exceeds `preserve_len`.
573pub fn writeAllPreserve(w: *Writer, preserve_len: usize, bytes: []const u8) Error!void {
560574 var index: usize = 0;
561 while (index < bytes.len) index += try w.writePreserve(preserve_length, bytes[index..]);
575 while (index < bytes.len) index += try w.writePreserve(preserve_len, bytes[index..]);
562576}
563577
564578/// Renders fmt string with args, calling `writer` with slices of bytes.
......@@ -761,11 +775,11 @@ pub fn writeByte(w: *Writer, byte: u8) Error!void {
761775 }
762776}
763777
764/// When draining the buffer, ensures that at least `preserve_length` bytes
778/// When draining the buffer, ensures that at least `preserve_len` bytes
765779/// remain buffered.
766pub fn writeBytePreserve(w: *Writer, preserve_length: usize, byte: u8) Error!void {
780pub fn writeBytePreserve(w: *Writer, preserve_len: usize, byte: u8) Error!void {
767781 while (w.buffer.len - w.end == 0) {
768 try drainPreserve(w, preserve_length);
782 try drainPreserve(w, preserve_len);
769783 } else {
770784 @branchHint(.likely);
771785 w.buffer[w.end] = byte;
......@@ -788,10 +802,42 @@ test splatByteAll {
788802 try testing.expectEqualStrings("7" ** 45, aw.writer.buffered());
789803}
790804
805pub fn splatBytePreserve(w: *Writer, preserve_len: usize, byte: u8, n: usize) Error!void {
806 const new_end = w.end + n;
807 if (new_end <= w.buffer.len) {
808 @memset(w.buffer[w.end..][0..n], byte);
809 w.end = new_end;
810 return;
811 }
812 // If `n` is large, we can ignore `preserve_len` up to a point.
813 var remaining = n;
814 while (remaining > preserve_len) {
815 assert(remaining != 0);
816 remaining -= try splatByte(w, byte, remaining - preserve_len);
817 if (w.end + remaining <= w.buffer.len) {
818 @memset(w.buffer[w.end..][0..remaining], byte);
819 w.end += remaining;
820 return;
821 }
822 }
823 // All the next bytes received must be preserved.
824 if (preserve_len < w.end) {
825 @memmove(w.buffer[0..preserve_len], w.buffer[w.end - preserve_len ..][0..preserve_len]);
826 w.end = preserve_len;
827 }
828 while (remaining > 0) remaining -= try w.splatByte(byte, remaining);
829}
830
791831/// Writes the same byte many times, allowing short writes.
792832///
793833/// Does maximum of one underlying `VTable.drain`.
794834pub fn splatByte(w: *Writer, byte: u8, n: usize) Error!usize {
835 if (w.end + n <= w.buffer.len) {
836 @branchHint(.likely);
837 @memset(w.buffer[w.end..][0..n], byte);
838 w.end += n;
839 return n;
840 }
795841 return writeSplat(w, &.{&.{byte}}, n);
796842}
797843
......@@ -801,9 +847,10 @@ pub fn splatBytesAll(w: *Writer, bytes: []const u8, splat: usize) Error!void {
801847 var remaining_bytes: usize = bytes.len * splat;
802848 remaining_bytes -= try w.splatBytes(bytes, splat);
803849 while (remaining_bytes > 0) {
804 const leftover = remaining_bytes % bytes.len;
805 const buffers: [2][]const u8 = .{ bytes[bytes.len - leftover ..], bytes };
806 remaining_bytes -= try w.writeSplat(&buffers, splat);
850 const leftover_splat = remaining_bytes / bytes.len;
851 const leftover_bytes = remaining_bytes % bytes.len;
852 const buffers: [2][]const u8 = .{ bytes[bytes.len - leftover_bytes ..], bytes };
853 remaining_bytes -= try w.writeSplat(&buffers, leftover_splat);
807854 }
808855}
809856
lib/std/compress.zig+2-58
......@@ -1,75 +1,19 @@
11//! Compression algorithms.
22
3const std = @import("std.zig");
4
53pub const flate = @import("compress/flate.zig");
64pub const gzip = @import("compress/gzip.zig");
75pub const zlib = @import("compress/zlib.zig");
86pub const lzma = @import("compress/lzma.zig");
97pub const lzma2 = @import("compress/lzma2.zig");
108pub const xz = @import("compress/xz.zig");
11pub const zstd = @import("compress/zstandard.zig");
12
13pub fn HashedReader(ReaderType: type, HasherType: type) type {
14 return struct {
15 child_reader: ReaderType,
16 hasher: HasherType,
17
18 pub const Error = ReaderType.Error;
19 pub const Reader = std.io.GenericReader(*@This(), Error, read);
20
21 pub fn read(self: *@This(), buf: []u8) Error!usize {
22 const amt = try self.child_reader.read(buf);
23 self.hasher.update(buf[0..amt]);
24 return amt;
25 }
26
27 pub fn reader(self: *@This()) Reader {
28 return .{ .context = self };
29 }
30 };
31}
32
33pub fn hashedReader(
34 reader: anytype,
35 hasher: anytype,
36) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
37 return .{ .child_reader = reader, .hasher = hasher };
38}
39
40pub fn HashedWriter(WriterType: type, HasherType: type) type {
41 return struct {
42 child_writer: WriterType,
43 hasher: HasherType,
44
45 pub const Error = WriterType.Error;
46 pub const Writer = std.io.GenericWriter(*@This(), Error, write);
47
48 pub fn write(self: *@This(), buf: []const u8) Error!usize {
49 const amt = try self.child_writer.write(buf);
50 self.hasher.update(buf[0..amt]);
51 return amt;
52 }
53
54 pub fn writer(self: *@This()) Writer {
55 return .{ .context = self };
56 }
57 };
58}
59
60pub fn hashedWriter(
61 writer: anytype,
62 hasher: anytype,
63) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
64 return .{ .child_writer = writer, .hasher = hasher };
65}
9pub const zstd = @import("compress/zstd.zig");
6610
6711test {
12 _ = flate;
6813 _ = lzma;
6914 _ = lzma2;
7015 _ = xz;
7116 _ = zstd;
72 _ = flate;
7317 _ = gzip;
7418 _ = zlib;
7519}
lib/std/compress/xz.zig+30-3
......@@ -47,7 +47,7 @@ pub fn Decompress(comptime ReaderType: type) type {
4747
4848 var check: Check = undefined;
4949 const hash_a = blk: {
50 var hasher = std.compress.hashedReader(source, Crc32.init());
50 var hasher = hashedReader(source, Crc32.init());
5151 try readStreamFlags(hasher.reader(), &check);
5252 break :blk hasher.hasher.final();
5353 };
......@@ -80,7 +80,7 @@ pub fn Decompress(comptime ReaderType: type) type {
8080 return r;
8181
8282 const index_size = blk: {
83 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
83 var hasher = hashedReader(self.in_reader, Crc32.init());
8484 hasher.hasher.update(&[1]u8{0x00});
8585
8686 var counter = std.io.countingReader(hasher.reader());
......@@ -115,7 +115,7 @@ pub fn Decompress(comptime ReaderType: type) type {
115115 const hash_a = try self.in_reader.readInt(u32, .little);
116116
117117 const hash_b = blk: {
118 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
118 var hasher = hashedReader(self.in_reader, Crc32.init());
119119 const hashed_reader = hasher.reader();
120120
121121 const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
......@@ -140,6 +140,33 @@ pub fn Decompress(comptime ReaderType: type) type {
140140 };
141141}
142142
143pub fn HashedReader(ReaderType: type, HasherType: type) type {
144 return struct {
145 child_reader: ReaderType,
146 hasher: HasherType,
147
148 pub const Error = ReaderType.Error;
149 pub const Reader = std.io.GenericReader(*@This(), Error, read);
150
151 pub fn read(self: *@This(), buf: []u8) Error!usize {
152 const amt = try self.child_reader.read(buf);
153 self.hasher.update(buf[0..amt]);
154 return amt;
155 }
156
157 pub fn reader(self: *@This()) Reader {
158 return .{ .context = self };
159 }
160 };
161}
162
163pub fn hashedReader(
164 reader: anytype,
165 hasher: anytype,
166) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
167 return .{ .child_reader = reader, .hasher = hasher };
168}
169
143170test {
144171 _ = @import("xz/test.zig");
145172}
lib/std/compress/xz/block.zig+1-1
......@@ -91,7 +91,7 @@ pub fn Decoder(comptime ReaderType: type) type {
9191
9292 // Block Header
9393 {
94 var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());
94 var header_hasher = xz.hashedReader(block_reader, Crc32.init());
9595 const header_reader = header_hasher.reader();
9696
9797 const header_size = @as(u64, try header_reader.readByte()) * 4;
lib/std/compress/zstandard.zig deleted-310
......@@ -1,310 +0,0 @@
1const std = @import("std");
2const RingBuffer = std.RingBuffer;
3
4const types = @import("zstandard/types.zig");
5pub const frame = types.frame;
6pub const compressed_block = types.compressed_block;
7
8pub const decompress = @import("zstandard/decompress.zig");
9
10pub const DecompressorOptions = struct {
11 verify_checksum: bool = true,
12 window_buffer: []u8,
13
14 /// Recommended amount by the standard. Lower than this may result
15 /// in inability to decompress common streams.
16 pub const default_window_buffer_len = 8 * 1024 * 1024;
17};
18
19pub fn Decompressor(comptime ReaderType: type) type {
20 return struct {
21 const Self = @This();
22
23 const table_size_max = types.compressed_block.table_size_max;
24
25 source: std.io.CountingReader(ReaderType),
26 state: enum { NewFrame, InFrame, LastBlock },
27 decode_state: decompress.block.DecodeState,
28 frame_context: decompress.FrameContext,
29 buffer: WindowBuffer,
30 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
31 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
32 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
33 literals_buffer: [types.block_size_max]u8,
34 sequence_buffer: [types.block_size_max]u8,
35 verify_checksum: bool,
36 checksum: ?u32,
37 current_frame_decompressed_size: usize,
38
39 const WindowBuffer = struct {
40 data: []u8 = undefined,
41 read_index: usize = 0,
42 write_index: usize = 0,
43 };
44
45 pub const Error = ReaderType.Error || error{
46 ChecksumFailure,
47 DictionaryIdFlagUnsupported,
48 MalformedBlock,
49 MalformedFrame,
50 OutOfMemory,
51 };
52
53 pub const Reader = std.io.GenericReader(*Self, Error, read);
54
55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {
56 return .{
57 .source = std.io.countingReader(source),
58 .state = .NewFrame,
59 .decode_state = undefined,
60 .frame_context = undefined,
61 .buffer = .{ .data = options.window_buffer },
62 .literal_fse_buffer = undefined,
63 .match_fse_buffer = undefined,
64 .offset_fse_buffer = undefined,
65 .literals_buffer = undefined,
66 .sequence_buffer = undefined,
67 .verify_checksum = options.verify_checksum,
68 .checksum = undefined,
69 .current_frame_decompressed_size = undefined,
70 };
71 }
72
73 fn frameInit(self: *Self) !void {
74 const source_reader = self.source.reader();
75 switch (try decompress.decodeFrameHeader(source_reader)) {
76 .skippable => |header| {
77 try source_reader.skipBytes(header.frame_size, .{});
78 self.state = .NewFrame;
79 },
80 .zstandard => |header| {
81 const frame_context = try decompress.FrameContext.init(
82 header,
83 self.buffer.data.len,
84 self.verify_checksum,
85 );
86
87 const decode_state = decompress.block.DecodeState.init(
88 &self.literal_fse_buffer,
89 &self.match_fse_buffer,
90 &self.offset_fse_buffer,
91 );
92
93 self.decode_state = decode_state;
94 self.frame_context = frame_context;
95
96 self.checksum = null;
97 self.current_frame_decompressed_size = 0;
98
99 self.state = .InFrame;
100 },
101 }
102 }
103
104 pub fn reader(self: *Self) Reader {
105 return .{ .context = self };
106 }
107
108 pub fn read(self: *Self, buffer: []u8) Error!usize {
109 if (buffer.len == 0) return 0;
110
111 var size: usize = 0;
112 while (size == 0) {
113 while (self.state == .NewFrame) {
114 const initial_count = self.source.bytes_read;
115 self.frameInit() catch |err| switch (err) {
116 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
117 error.EndOfStream => return if (self.source.bytes_read == initial_count)
118 0
119 else
120 error.MalformedFrame,
121 else => return error.MalformedFrame,
122 };
123 }
124 size = try self.readInner(buffer);
125 }
126 return size;
127 }
128
129 fn readInner(self: *Self, buffer: []u8) Error!usize {
130 std.debug.assert(self.state != .NewFrame);
131
132 var ring_buffer = RingBuffer{
133 .data = self.buffer.data,
134 .read_index = self.buffer.read_index,
135 .write_index = self.buffer.write_index,
136 };
137 defer {
138 self.buffer.read_index = ring_buffer.read_index;
139 self.buffer.write_index = ring_buffer.write_index;
140 }
141
142 const source_reader = self.source.reader();
143 while (ring_buffer.isEmpty() and self.state != .LastBlock) {
144 const header_bytes = source_reader.readBytesNoEof(3) catch
145 return error.MalformedFrame;
146 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
147
148 decompress.block.decodeBlockReader(
149 &ring_buffer,
150 source_reader,
151 block_header,
152 &self.decode_state,
153 self.frame_context.block_size_max,
154 &self.literals_buffer,
155 &self.sequence_buffer,
156 ) catch
157 return error.MalformedBlock;
158
159 if (self.frame_context.content_size) |size| {
160 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
161 }
162
163 const size = ring_buffer.len();
164 self.current_frame_decompressed_size += size;
165
166 if (self.frame_context.hasher_opt) |*hasher| {
167 if (size > 0) {
168 const written_slice = ring_buffer.sliceLast(size);
169 hasher.update(written_slice.first);
170 hasher.update(written_slice.second);
171 }
172 }
173 if (block_header.last_block) {
174 self.state = .LastBlock;
175 if (self.frame_context.has_checksum) {
176 const checksum = source_reader.readInt(u32, .little) catch
177 return error.MalformedFrame;
178 if (self.verify_checksum) {
179 if (self.frame_context.hasher_opt) |*hasher| {
180 if (checksum != decompress.computeChecksum(hasher))
181 return error.ChecksumFailure;
182 }
183 }
184 }
185 if (self.frame_context.content_size) |content_size| {
186 if (content_size != self.current_frame_decompressed_size) {
187 return error.MalformedFrame;
188 }
189 }
190 }
191 }
192
193 const size = @min(ring_buffer.len(), buffer.len);
194 if (size > 0) {
195 ring_buffer.readFirstAssumeLength(buffer, size);
196 }
197 if (self.state == .LastBlock and ring_buffer.len() == 0) {
198 self.state = .NewFrame;
199 }
200 return size;
201 }
202 };
203}
204
205pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206 return Decompressor(@TypeOf(reader)).init(reader, options);
207}
208
209fn testDecompress(data: []const u8) ![]u8 {
210 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
211 defer std.testing.allocator.free(window_buffer);
212
213 var in_stream = std.io.fixedBufferStream(data);
214 var zstd_stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
215 const result = zstd_stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
216 return result;
217}
218
219fn testReader(data: []const u8, comptime expected: []const u8) !void {
220 const buf = try testDecompress(data);
221 defer std.testing.allocator.free(buf);
222 try std.testing.expectEqualSlices(u8, expected, buf);
223}
224
225test "decompression" {
226 const uncompressed = @embedFile("testdata/rfc8478.txt");
227 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
228 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
229
230 const buffer = try std.testing.allocator.alloc(u8, uncompressed.len);
231 defer std.testing.allocator.free(buffer);
232
233 const res3 = try decompress.decode(buffer, compressed3, true);
234 try std.testing.expectEqual(uncompressed.len, res3);
235 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
236
237 @memset(buffer, undefined);
238 const res19 = try decompress.decode(buffer, compressed19, true);
239 try std.testing.expectEqual(uncompressed.len, res19);
240 try std.testing.expectEqualSlices(u8, uncompressed, buffer);
241
242 try testReader(compressed3, uncompressed);
243 try testReader(compressed19, uncompressed);
244}
245
246fn expectEqualDecoded(expected: []const u8, input: []const u8) !void {
247 {
248 const result = try decompress.decodeAlloc(std.testing.allocator, input, false, 1 << 23);
249 defer std.testing.allocator.free(result);
250 try std.testing.expectEqualStrings(expected, result);
251 }
252
253 {
254 var buffer = try std.testing.allocator.alloc(u8, 2 * expected.len);
255 defer std.testing.allocator.free(buffer);
256
257 const size = try decompress.decode(buffer, input, false);
258 try std.testing.expectEqualStrings(expected, buffer[0..size]);
259 }
260}
261
262fn expectEqualDecodedStreaming(expected: []const u8, input: []const u8) !void {
263 const window_buffer = try std.testing.allocator.alloc(u8, 1 << 23);
264 defer std.testing.allocator.free(window_buffer);
265
266 var in_stream = std.io.fixedBufferStream(input);
267 var stream = decompressor(in_stream.reader(), .{ .window_buffer = window_buffer });
268
269 const result = try stream.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
270 defer std.testing.allocator.free(result);
271
272 try std.testing.expectEqualStrings(expected, result);
273}
274
275test "zero sized block" {
276 const input_raw =
277 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
278 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
279 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
280
281 const input_rle =
282 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
283 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
284 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
285 "\xaa"; // block_content
286
287 try expectEqualDecoded("", input_raw);
288 try expectEqualDecoded("", input_rle);
289 try expectEqualDecodedStreaming("", input_raw);
290 try expectEqualDecodedStreaming("", input_rle);
291}
292
293test "declared raw literals size too large" {
294 const input_raw =
295 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
296 "\x00\x00" ++ // frame header: everything unset, window descriptor zero
297 "\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
298 "\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
299 "\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
300
301 // Note that the regenerated_size in the above input is larger than block maximum size, so the
302 // block can't be valid as it is a raw literals block.
303
304 var fbs = std.io.fixedBufferStream(input_raw);
305 var window: [1024]u8 = undefined;
306 var stream = decompressor(fbs.reader(), .{ .window_buffer = &window });
307
308 var buf: [1024]u8 = undefined;
309 try std.testing.expectError(error.MalformedBlock, stream.read(&buf));
310}
lib/std/compress/zstandard/decode/block.zig deleted-1149
......@@ -1,1149 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const RingBuffer = std.RingBuffer;
4
5const types = @import("../types.zig");
6const frame = types.frame;
7const Table = types.compressed_block.Table;
8const LiteralsSection = types.compressed_block.LiteralsSection;
9const SequencesSection = types.compressed_block.SequencesSection;
10
11const huffman = @import("huffman.zig");
12const readers = @import("../readers.zig");
13
14const decodeFseTable = @import("fse.zig").decodeFseTable;
15
16pub const Error = error{
17 BlockSizeOverMaximum,
18 MalformedBlockSize,
19 ReservedBlock,
20 MalformedRleBlock,
21 MalformedCompressedBlock,
22};
23
24pub const DecodeState = struct {
25 repeat_offsets: [3]u32,
26
27 offset: StateData(8),
28 match: StateData(9),
29 literal: StateData(9),
30
31 offset_fse_buffer: []Table.Fse,
32 match_fse_buffer: []Table.Fse,
33 literal_fse_buffer: []Table.Fse,
34
35 fse_tables_undefined: bool,
36
37 literal_stream_reader: readers.ReverseBitReader,
38 literal_stream_index: usize,
39 literal_streams: LiteralsSection.Streams,
40 literal_header: LiteralsSection.Header,
41 huffman_tree: ?LiteralsSection.HuffmanTree,
42
43 literal_written_count: usize,
44 written_count: usize = 0,
45
46 fn StateData(comptime max_accuracy_log: comptime_int) type {
47 return struct {
48 state: State,
49 table: Table,
50 accuracy_log: u8,
51
52 const State = std.meta.Int(.unsigned, max_accuracy_log);
53 };
54 }
55
56 pub fn init(
57 literal_fse_buffer: []Table.Fse,
58 match_fse_buffer: []Table.Fse,
59 offset_fse_buffer: []Table.Fse,
60 ) DecodeState {
61 return DecodeState{
62 .repeat_offsets = .{
63 types.compressed_block.start_repeated_offset_1,
64 types.compressed_block.start_repeated_offset_2,
65 types.compressed_block.start_repeated_offset_3,
66 },
67
68 .offset = undefined,
69 .match = undefined,
70 .literal = undefined,
71
72 .literal_fse_buffer = literal_fse_buffer,
73 .match_fse_buffer = match_fse_buffer,
74 .offset_fse_buffer = offset_fse_buffer,
75
76 .fse_tables_undefined = true,
77
78 .literal_written_count = 0,
79 .literal_header = undefined,
80 .literal_streams = undefined,
81 .literal_stream_reader = undefined,
82 .literal_stream_index = undefined,
83 .huffman_tree = null,
84
85 .written_count = 0,
86 };
87 }
88
89 /// Prepare the decoder to decode a compressed block. Loads the literals
90 /// stream and Huffman tree from `literals` and reads the FSE tables from
91 /// `source`.
92 ///
93 /// Errors returned:
94 /// - `error.BitStreamHasNoStartBit` if the (reversed) literal bitstream's
95 /// first byte does not have any bits set
96 /// - `error.TreelessLiteralsFirst` `literals` is a treeless literals
97 /// section and the decode state does not have a Huffman tree from a
98 /// previous block
99 /// - `error.RepeatModeFirst` on the first call if one of the sequence FSE
100 /// tables is set to repeat mode
101 /// - `error.MalformedAccuracyLog` if an FSE table has an invalid accuracy
102 /// - `error.MalformedFseTable` if there are errors decoding an FSE table
103 /// - `error.EndOfStream` if `source` ends before all FSE tables are read
104 pub fn prepare(
105 self: *DecodeState,
106 source: anytype,
107 literals: LiteralsSection,
108 sequences_header: SequencesSection.Header,
109 ) !void {
110 self.literal_written_count = 0;
111 self.literal_header = literals.header;
112 self.literal_streams = literals.streams;
113
114 if (literals.huffman_tree) |tree| {
115 self.huffman_tree = tree;
116 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
117 return error.TreelessLiteralsFirst;
118 }
119
120 switch (literals.header.block_type) {
121 .raw, .rle => {},
122 .compressed, .treeless => {
123 self.literal_stream_index = 0;
124 switch (literals.streams) {
125 .one => |slice| try self.initLiteralStream(slice),
126 .four => |streams| try self.initLiteralStream(streams[0]),
127 }
128 },
129 }
130
131 if (sequences_header.sequence_count > 0) {
132 try self.updateFseTable(source, .literal, sequences_header.literal_lengths);
133 try self.updateFseTable(source, .offset, sequences_header.offsets);
134 try self.updateFseTable(source, .match, sequences_header.match_lengths);
135 self.fse_tables_undefined = false;
136 }
137 }
138
139 /// Read initial FSE states for sequence decoding.
140 ///
141 /// Errors returned:
142 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
143 pub fn readInitialFseState(self: *DecodeState, bit_reader: *readers.ReverseBitReader) error{EndOfStream}!void {
144 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
145 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
146 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
147 }
148
149 fn updateRepeatOffset(self: *DecodeState, offset: u32) void {
150 self.repeat_offsets[2] = self.repeat_offsets[1];
151 self.repeat_offsets[1] = self.repeat_offsets[0];
152 self.repeat_offsets[0] = offset;
153 }
154
155 fn useRepeatOffset(self: *DecodeState, index: usize) u32 {
156 if (index == 1)
157 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
158 else if (index == 2) {
159 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
160 std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
161 }
162 return self.repeat_offsets[0];
163 }
164
165 const DataType = enum { offset, match, literal };
166
167 fn updateState(
168 self: *DecodeState,
169 comptime choice: DataType,
170 bit_reader: *readers.ReverseBitReader,
171 ) error{ MalformedFseBits, EndOfStream }!void {
172 switch (@field(self, @tagName(choice)).table) {
173 .rle => {},
174 .fse => |table| {
175 const data = table[@field(self, @tagName(choice)).state];
176 const T = @TypeOf(@field(self, @tagName(choice))).State;
177 const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
178 const next_state = std.math.cast(
179 @TypeOf(@field(self, @tagName(choice))).State,
180 data.baseline + bits_summand,
181 ) orelse return error.MalformedFseBits;
182 @field(self, @tagName(choice)).state = next_state;
183 },
184 }
185 }
186
187 const FseTableError = error{
188 MalformedFseTable,
189 MalformedAccuracyLog,
190 RepeatModeFirst,
191 EndOfStream,
192 };
193
194 fn updateFseTable(
195 self: *DecodeState,
196 source: anytype,
197 comptime choice: DataType,
198 mode: SequencesSection.Header.Mode,
199 ) !void {
200 const field_name = @tagName(choice);
201 switch (mode) {
202 .predefined => {
203 @field(self, field_name).accuracy_log =
204 @field(types.compressed_block.default_accuracy_log, field_name);
205
206 @field(self, field_name).table =
207 @field(types.compressed_block, "predefined_" ++ field_name ++ "_fse_table");
208 },
209 .rle => {
210 @field(self, field_name).accuracy_log = 0;
211 @field(self, field_name).table = .{ .rle = try source.readByte() };
212 },
213 .fse => {
214 var bit_reader = readers.bitReader(source);
215
216 const table_size = try decodeFseTable(
217 &bit_reader,
218 @field(types.compressed_block.table_symbol_count_max, field_name),
219 @field(types.compressed_block.table_accuracy_log_max, field_name),
220 @field(self, field_name ++ "_fse_buffer"),
221 );
222 @field(self, field_name).table = .{
223 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
224 };
225 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
226 },
227 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
228 }
229 }
230
231 const Sequence = struct {
232 literal_length: u32,
233 match_length: u32,
234 offset: u32,
235 };
236
237 fn nextSequence(
238 self: *DecodeState,
239 bit_reader: *readers.ReverseBitReader,
240 ) error{ InvalidBitStream, EndOfStream }!Sequence {
241 const raw_code = self.getCode(.offset);
242 const offset_code = std.math.cast(u5, raw_code) orelse {
243 return error.InvalidBitStream;
244 };
245 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
246
247 const match_code = self.getCode(.match);
248 if (match_code >= types.compressed_block.match_length_code_table.len)
249 return error.InvalidBitStream;
250 const match = types.compressed_block.match_length_code_table[match_code];
251 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
252
253 const literal_code = self.getCode(.literal);
254 if (literal_code >= types.compressed_block.literals_length_code_table.len)
255 return error.InvalidBitStream;
256 const literal = types.compressed_block.literals_length_code_table[literal_code];
257 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
258
259 const offset = if (offset_value > 3) offset: {
260 const offset = offset_value - 3;
261 self.updateRepeatOffset(offset);
262 break :offset offset;
263 } else offset: {
264 if (literal_length == 0) {
265 if (offset_value == 3) {
266 const offset = self.repeat_offsets[0] - 1;
267 self.updateRepeatOffset(offset);
268 break :offset offset;
269 }
270 break :offset self.useRepeatOffset(offset_value);
271 }
272 break :offset self.useRepeatOffset(offset_value - 1);
273 };
274
275 if (offset == 0) return error.InvalidBitStream;
276
277 return .{
278 .literal_length = literal_length,
279 .match_length = match_length,
280 .offset = offset,
281 };
282 }
283
284 fn executeSequenceSlice(
285 self: *DecodeState,
286 dest: []u8,
287 write_pos: usize,
288 sequence: Sequence,
289 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
290 if (sequence.offset > write_pos + sequence.literal_length) return error.MalformedSequence;
291
292 try self.decodeLiteralsSlice(dest[write_pos..], sequence.literal_length);
293 const copy_start = write_pos + sequence.literal_length - sequence.offset;
294 for (
295 dest[write_pos + sequence.literal_length ..][0..sequence.match_length],
296 dest[copy_start..][0..sequence.match_length],
297 ) |*d, s| d.* = s;
298 self.written_count += sequence.match_length;
299 }
300
301 fn executeSequenceRingBuffer(
302 self: *DecodeState,
303 dest: *RingBuffer,
304 sequence: Sequence,
305 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
306 if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
307 return error.MalformedSequence;
308
309 try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
310 const copy_start = dest.write_index + dest.data.len - sequence.offset;
311 const copy_slice = dest.sliceAt(copy_start, sequence.match_length);
312 dest.writeSliceForwardsAssumeCapacity(copy_slice.first);
313 dest.writeSliceForwardsAssumeCapacity(copy_slice.second);
314 self.written_count += sequence.match_length;
315 }
316
317 const DecodeSequenceError = error{
318 InvalidBitStream,
319 EndOfStream,
320 MalformedSequence,
321 MalformedFseBits,
322 } || DecodeLiteralsError;
323
324 /// Decode one sequence from `bit_reader` into `dest`, written starting at
325 /// `write_pos` and update FSE states if `last_sequence` is `false`.
326 /// `prepare()` must be called for the block before attempting to decode
327 /// sequences.
328 ///
329 /// Errors returned:
330 /// - `error.MalformedSequence` if the decompressed sequence would be
331 /// longer than `sequence_size_limit` or the sequence's offset is too
332 /// large
333 /// - `error.UnexpectedEndOfLiteralStream` if the decoder state's literal
334 /// streams do not contain enough literals for the sequence (this may
335 /// mean the literal stream or the sequence is malformed).
336 /// - `error.InvalidBitStream` if the FSE sequence bitstream is malformed
337 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits
338 /// - `error.DestTooSmall` if `dest` is not large enough to holde the
339 /// decompressed sequence
340 pub fn decodeSequenceSlice(
341 self: *DecodeState,
342 dest: []u8,
343 write_pos: usize,
344 bit_reader: *readers.ReverseBitReader,
345 sequence_size_limit: usize,
346 last_sequence: bool,
347 ) (error{DestTooSmall} || DecodeSequenceError)!usize {
348 const sequence = try self.nextSequence(bit_reader);
349 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
350 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
351 if (sequence_length > dest[write_pos..].len) return error.DestTooSmall;
352
353 try self.executeSequenceSlice(dest, write_pos, sequence);
354 if (!last_sequence) {
355 try self.updateState(.literal, bit_reader);
356 try self.updateState(.match, bit_reader);
357 try self.updateState(.offset, bit_reader);
358 }
359 return sequence_length;
360 }
361
362 /// Decode one sequence from `bit_reader` into `dest`; see
363 /// `decodeSequenceSlice`.
364 pub fn decodeSequenceRingBuffer(
365 self: *DecodeState,
366 dest: *RingBuffer,
367 bit_reader: anytype,
368 sequence_size_limit: usize,
369 last_sequence: bool,
370 ) DecodeSequenceError!usize {
371 const sequence = try self.nextSequence(bit_reader);
372 const sequence_length = @as(usize, sequence.literal_length) + sequence.match_length;
373 if (sequence_length > sequence_size_limit) return error.MalformedSequence;
374
375 try self.executeSequenceRingBuffer(dest, sequence);
376 if (!last_sequence) {
377 try self.updateState(.literal, bit_reader);
378 try self.updateState(.match, bit_reader);
379 try self.updateState(.offset, bit_reader);
380 }
381 return sequence_length;
382 }
383
384 fn nextLiteralMultiStream(
385 self: *DecodeState,
386 ) error{BitStreamHasNoStartBit}!void {
387 self.literal_stream_index += 1;
388 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
389 }
390
391 fn initLiteralStream(self: *DecodeState, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
392 try self.literal_stream_reader.init(bytes);
393 }
394
395 fn isLiteralStreamEmpty(self: *DecodeState) bool {
396 switch (self.literal_streams) {
397 .one => return self.literal_stream_reader.isEmpty(),
398 .four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
399 }
400 }
401
402 const LiteralBitsError = error{
403 BitStreamHasNoStartBit,
404 UnexpectedEndOfLiteralStream,
405 };
406 fn readLiteralsBits(
407 self: *DecodeState,
408 bit_count_to_read: u16,
409 ) LiteralBitsError!u16 {
410 return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
411 if (self.literal_streams == .four and self.literal_stream_index < 3) {
412 try self.nextLiteralMultiStream();
413 break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
414 return error.UnexpectedEndOfLiteralStream;
415 } else {
416 return error.UnexpectedEndOfLiteralStream;
417 }
418 };
419 }
420
421 const DecodeLiteralsError = error{
422 MalformedLiteralsLength,
423 NotFound,
424 } || LiteralBitsError;
425
426 /// Decode `len` bytes of literals into `dest`.
427 ///
428 /// Errors returned:
429 /// - `error.MalformedLiteralsLength` if the number of literal bytes
430 /// decoded by `self` plus `len` is greater than the regenerated size of
431 /// `literals`
432 /// - `error.UnexpectedEndOfLiteralStream` and `error.NotFound` if there
433 /// are problems decoding Huffman compressed literals
434 pub fn decodeLiteralsSlice(
435 self: *DecodeState,
436 dest: []u8,
437 len: usize,
438 ) DecodeLiteralsError!void {
439 if (self.literal_written_count + len > self.literal_header.regenerated_size)
440 return error.MalformedLiteralsLength;
441
442 switch (self.literal_header.block_type) {
443 .raw => {
444 const literal_data = self.literal_streams.one[self.literal_written_count..][0..len];
445 @memcpy(dest[0..len], literal_data);
446 self.literal_written_count += len;
447 self.written_count += len;
448 },
449 .rle => {
450 for (0..len) |i| {
451 dest[i] = self.literal_streams.one[0];
452 }
453 self.literal_written_count += len;
454 self.written_count += len;
455 },
456 .compressed, .treeless => {
457 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
458 const huffman_tree = self.huffman_tree orelse unreachable;
459 const max_bit_count = huffman_tree.max_bit_count;
460 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
461 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
462 max_bit_count,
463 );
464 var bits_read: u4 = 0;
465 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
466 var bit_count_to_read: u4 = starting_bit_count;
467 for (0..len) |i| {
468 var prefix: u16 = 0;
469 while (true) {
470 const new_bits = self.readLiteralsBits(bit_count_to_read) catch |err| {
471 return err;
472 };
473 prefix <<= bit_count_to_read;
474 prefix |= new_bits;
475 bits_read += bit_count_to_read;
476 const result = huffman_tree.query(huffman_tree_index, prefix) catch |err| {
477 return err;
478 };
479
480 switch (result) {
481 .symbol => |sym| {
482 dest[i] = sym;
483 bit_count_to_read = starting_bit_count;
484 bits_read = 0;
485 huffman_tree_index = huffman_tree.symbol_count_minus_one;
486 break;
487 },
488 .index => |index| {
489 huffman_tree_index = index;
490 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
491 huffman_tree.nodes[index].weight,
492 max_bit_count,
493 );
494 bit_count_to_read = bit_count - bits_read;
495 },
496 }
497 }
498 }
499 self.literal_written_count += len;
500 self.written_count += len;
501 },
502 }
503 }
504
505 /// Decode literals into `dest`; see `decodeLiteralsSlice()`.
506 pub fn decodeLiteralsRingBuffer(
507 self: *DecodeState,
508 dest: *RingBuffer,
509 len: usize,
510 ) DecodeLiteralsError!void {
511 if (self.literal_written_count + len > self.literal_header.regenerated_size)
512 return error.MalformedLiteralsLength;
513
514 switch (self.literal_header.block_type) {
515 .raw => {
516 const literals_end = self.literal_written_count + len;
517 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
518 dest.writeSliceAssumeCapacity(literal_data);
519 self.literal_written_count += len;
520 self.written_count += len;
521 },
522 .rle => {
523 for (0..len) |_| {
524 dest.writeAssumeCapacity(self.literal_streams.one[0]);
525 }
526 self.literal_written_count += len;
527 self.written_count += len;
528 },
529 .compressed, .treeless => {
530 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
531 const huffman_tree = self.huffman_tree orelse unreachable;
532 const max_bit_count = huffman_tree.max_bit_count;
533 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
534 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
535 max_bit_count,
536 );
537 var bits_read: u4 = 0;
538 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
539 var bit_count_to_read: u4 = starting_bit_count;
540 for (0..len) |_| {
541 var prefix: u16 = 0;
542 while (true) {
543 const new_bits = try self.readLiteralsBits(bit_count_to_read);
544 prefix <<= bit_count_to_read;
545 prefix |= new_bits;
546 bits_read += bit_count_to_read;
547 const result = try huffman_tree.query(huffman_tree_index, prefix);
548
549 switch (result) {
550 .symbol => |sym| {
551 dest.writeAssumeCapacity(sym);
552 bit_count_to_read = starting_bit_count;
553 bits_read = 0;
554 huffman_tree_index = huffman_tree.symbol_count_minus_one;
555 break;
556 },
557 .index => |index| {
558 huffman_tree_index = index;
559 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
560 huffman_tree.nodes[index].weight,
561 max_bit_count,
562 );
563 bit_count_to_read = bit_count - bits_read;
564 },
565 }
566 }
567 }
568 self.literal_written_count += len;
569 self.written_count += len;
570 },
571 }
572 }
573
574 fn getCode(self: *DecodeState, comptime choice: DataType) u32 {
575 return switch (@field(self, @tagName(choice)).table) {
576 .rle => |value| value,
577 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,
578 };
579 }
580};
581
582/// Decode a single block from `src` into `dest`. The beginning of `src` must be
583/// the start of the block content (i.e. directly after the block header).
584/// Increments `consumed_count` by the number of bytes read from `src` to decode
585/// the block and returns the decompressed size of the block.
586///
587/// Errors returned:
588///
589/// - `error.BlockSizeOverMaximum` if block's size is larger than 1 << 17 or
590/// `dest[written_count..].len`
591/// - `error.MalformedBlockSize` if `src.len` is smaller than the block size
592/// and the block is a raw or compressed block
593/// - `error.ReservedBlock` if the block is a reserved block
594/// - `error.MalformedRleBlock` if the block is an RLE block and `src.len < 1`
595/// - `error.MalformedCompressedBlock` if there are errors decoding a
596/// compressed block
597/// - `error.DestTooSmall` is `dest` is not large enough to hold the
598/// decompressed block
599pub fn decodeBlock(
600 dest: []u8,
601 src: []const u8,
602 block_header: frame.Zstandard.Block.Header,
603 decode_state: *DecodeState,
604 consumed_count: *usize,
605 block_size_max: usize,
606 written_count: usize,
607) (error{DestTooSmall} || Error)!usize {
608 const block_size = block_header.block_size;
609 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
610 switch (block_header.block_type) {
611 .raw => {
612 if (src.len < block_size) return error.MalformedBlockSize;
613 if (dest[written_count..].len < block_size) return error.DestTooSmall;
614 @memcpy(dest[written_count..][0..block_size], src[0..block_size]);
615 consumed_count.* += block_size;
616 decode_state.written_count += block_size;
617 return block_size;
618 },
619 .rle => {
620 if (src.len < 1) return error.MalformedRleBlock;
621 if (dest[written_count..].len < block_size) return error.DestTooSmall;
622 for (written_count..block_size + written_count) |write_pos| {
623 dest[write_pos] = src[0];
624 }
625 consumed_count.* += 1;
626 decode_state.written_count += block_size;
627 return block_size;
628 },
629 .compressed => {
630 if (src.len < block_size) return error.MalformedBlockSize;
631 var bytes_read: usize = 0;
632 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
633 return error.MalformedCompressedBlock;
634 var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
635 const fbs_reader = fbs.reader();
636 const sequences_header = decodeSequencesHeader(fbs_reader) catch
637 return error.MalformedCompressedBlock;
638
639 decode_state.prepare(fbs_reader, literals, sequences_header) catch
640 return error.MalformedCompressedBlock;
641
642 bytes_read += fbs.pos;
643
644 var bytes_written: usize = 0;
645 {
646 const bit_stream_bytes = src[bytes_read..block_size];
647 var bit_stream: readers.ReverseBitReader = undefined;
648 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
649
650 if (sequences_header.sequence_count > 0) {
651 decode_state.readInitialFseState(&bit_stream) catch
652 return error.MalformedCompressedBlock;
653
654 var sequence_size_limit = block_size_max;
655 for (0..sequences_header.sequence_count) |i| {
656 const write_pos = written_count + bytes_written;
657 const decompressed_size = decode_state.decodeSequenceSlice(
658 dest,
659 write_pos,
660 &bit_stream,
661 sequence_size_limit,
662 i == sequences_header.sequence_count - 1,
663 ) catch |err| switch (err) {
664 error.DestTooSmall => return error.DestTooSmall,
665 else => return error.MalformedCompressedBlock,
666 };
667 bytes_written += decompressed_size;
668 sequence_size_limit -= decompressed_size;
669 }
670 }
671
672 if (!bit_stream.isEmpty()) {
673 return error.MalformedCompressedBlock;
674 }
675 }
676
677 if (decode_state.literal_written_count < literals.header.regenerated_size) {
678 const len = literals.header.regenerated_size - decode_state.literal_written_count;
679 if (len > dest[written_count + bytes_written ..].len) return error.DestTooSmall;
680 decode_state.decodeLiteralsSlice(dest[written_count + bytes_written ..], len) catch
681 return error.MalformedCompressedBlock;
682 bytes_written += len;
683 }
684
685 switch (decode_state.literal_header.block_type) {
686 .treeless, .compressed => {
687 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
688 },
689 .raw, .rle => {},
690 }
691
692 consumed_count.* += block_size;
693 return bytes_written;
694 },
695 .reserved => return error.ReservedBlock,
696 }
697}
698
699/// Decode a single block from `src` into `dest`; see `decodeBlock()`. Returns
700/// the size of the decompressed block, which can be used with `dest.sliceLast()`
701/// to get the decompressed bytes. `error.BlockSizeOverMaximum` is returned if
702/// the block's compressed or decompressed size is larger than `block_size_max`.
703pub fn decodeBlockRingBuffer(
704 dest: *RingBuffer,
705 src: []const u8,
706 block_header: frame.Zstandard.Block.Header,
707 decode_state: *DecodeState,
708 consumed_count: *usize,
709 block_size_max: usize,
710) Error!usize {
711 const block_size = block_header.block_size;
712 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
713 switch (block_header.block_type) {
714 .raw => {
715 if (src.len < block_size) return error.MalformedBlockSize;
716 // dest may have length zero if block_size == 0, causing division by zero in
717 // writeSliceAssumeCapacity()
718 if (block_size > 0) {
719 const data = src[0..block_size];
720 dest.writeSliceAssumeCapacity(data);
721 consumed_count.* += block_size;
722 decode_state.written_count += block_size;
723 }
724 return block_size;
725 },
726 .rle => {
727 if (src.len < 1) return error.MalformedRleBlock;
728 for (0..block_size) |_| {
729 dest.writeAssumeCapacity(src[0]);
730 }
731 consumed_count.* += 1;
732 decode_state.written_count += block_size;
733 return block_size;
734 },
735 .compressed => {
736 if (src.len < block_size) return error.MalformedBlockSize;
737 var bytes_read: usize = 0;
738 const literals = decodeLiteralsSectionSlice(src[0..block_size], &bytes_read) catch
739 return error.MalformedCompressedBlock;
740 var fbs = std.io.fixedBufferStream(src[bytes_read..block_size]);
741 const fbs_reader = fbs.reader();
742 const sequences_header = decodeSequencesHeader(fbs_reader) catch
743 return error.MalformedCompressedBlock;
744
745 decode_state.prepare(fbs_reader, literals, sequences_header) catch
746 return error.MalformedCompressedBlock;
747
748 bytes_read += fbs.pos;
749
750 var bytes_written: usize = 0;
751 {
752 const bit_stream_bytes = src[bytes_read..block_size];
753 var bit_stream: readers.ReverseBitReader = undefined;
754 bit_stream.init(bit_stream_bytes) catch return error.MalformedCompressedBlock;
755
756 if (sequences_header.sequence_count > 0) {
757 decode_state.readInitialFseState(&bit_stream) catch
758 return error.MalformedCompressedBlock;
759
760 var sequence_size_limit = block_size_max;
761 for (0..sequences_header.sequence_count) |i| {
762 const decompressed_size = decode_state.decodeSequenceRingBuffer(
763 dest,
764 &bit_stream,
765 sequence_size_limit,
766 i == sequences_header.sequence_count - 1,
767 ) catch return error.MalformedCompressedBlock;
768 bytes_written += decompressed_size;
769 sequence_size_limit -= decompressed_size;
770 }
771 }
772
773 if (!bit_stream.isEmpty()) {
774 return error.MalformedCompressedBlock;
775 }
776 }
777
778 if (decode_state.literal_written_count < literals.header.regenerated_size) {
779 const len = literals.header.regenerated_size - decode_state.literal_written_count;
780 decode_state.decodeLiteralsRingBuffer(dest, len) catch
781 return error.MalformedCompressedBlock;
782 bytes_written += len;
783 }
784
785 switch (decode_state.literal_header.block_type) {
786 .treeless, .compressed => {
787 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
788 },
789 .raw, .rle => {},
790 }
791
792 consumed_count.* += block_size;
793 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
794 return bytes_written;
795 },
796 .reserved => return error.ReservedBlock,
797 }
798}
799
800/// Decode a single block from `source` into `dest`. Literal and sequence data
801/// from the block is copied into `literals_buffer` and `sequence_buffer`, which
802/// must be large enough or `error.LiteralsBufferTooSmall` and
803/// `error.SequenceBufferTooSmall` are returned (the maximum block size is an
804/// upper bound for the size of both buffers). See `decodeBlock`
805/// and `decodeBlockRingBuffer` for function that can decode a block without
806/// these extra copies. `error.EndOfStream` is returned if `source` does not
807/// contain enough bytes.
808pub fn decodeBlockReader(
809 dest: *RingBuffer,
810 source: anytype,
811 block_header: frame.Zstandard.Block.Header,
812 decode_state: *DecodeState,
813 block_size_max: usize,
814 literals_buffer: []u8,
815 sequence_buffer: []u8,
816) !void {
817 const block_size = block_header.block_size;
818 var block_reader_limited = std.io.limitedReader(source, block_size);
819 const block_reader = block_reader_limited.reader();
820 if (block_size_max < block_size) return error.BlockSizeOverMaximum;
821 switch (block_header.block_type) {
822 .raw => {
823 if (block_size == 0) return;
824 const slice = dest.sliceAt(dest.write_index, block_size);
825 try source.readNoEof(slice.first);
826 try source.readNoEof(slice.second);
827 dest.write_index = dest.mask2(dest.write_index + block_size);
828 decode_state.written_count += block_size;
829 },
830 .rle => {
831 const byte = try source.readByte();
832 for (0..block_size) |_| {
833 dest.writeAssumeCapacity(byte);
834 }
835 decode_state.written_count += block_size;
836 },
837 .compressed => {
838 const literals = try decodeLiteralsSection(block_reader, literals_buffer);
839 const sequences_header = try decodeSequencesHeader(block_reader);
840
841 try decode_state.prepare(block_reader, literals, sequences_header);
842
843 var bytes_written: usize = 0;
844 {
845 const size = try block_reader.readAll(sequence_buffer);
846 var bit_stream: readers.ReverseBitReader = undefined;
847 try bit_stream.init(sequence_buffer[0..size]);
848
849 if (sequences_header.sequence_count > 0) {
850 if (sequence_buffer.len < block_reader_limited.bytes_left)
851 return error.SequenceBufferTooSmall;
852
853 decode_state.readInitialFseState(&bit_stream) catch
854 return error.MalformedCompressedBlock;
855
856 var sequence_size_limit = block_size_max;
857 for (0..sequences_header.sequence_count) |i| {
858 const decompressed_size = decode_state.decodeSequenceRingBuffer(
859 dest,
860 &bit_stream,
861 sequence_size_limit,
862 i == sequences_header.sequence_count - 1,
863 ) catch return error.MalformedCompressedBlock;
864 sequence_size_limit -= decompressed_size;
865 bytes_written += decompressed_size;
866 }
867 }
868
869 if (!bit_stream.isEmpty()) {
870 return error.MalformedCompressedBlock;
871 }
872 }
873
874 if (decode_state.literal_written_count < literals.header.regenerated_size) {
875 const len = literals.header.regenerated_size - decode_state.literal_written_count;
876 decode_state.decodeLiteralsRingBuffer(dest, len) catch
877 return error.MalformedCompressedBlock;
878 bytes_written += len;
879 }
880
881 switch (decode_state.literal_header.block_type) {
882 .treeless, .compressed => {
883 if (!decode_state.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
884 },
885 .raw, .rle => {},
886 }
887
888 if (bytes_written > block_size_max) return error.BlockSizeOverMaximum;
889 if (block_reader_limited.bytes_left != 0) return error.MalformedCompressedBlock;
890 decode_state.literal_written_count = 0;
891 },
892 .reserved => return error.ReservedBlock,
893 }
894}
895
896/// Decode the header of a block.
897pub fn decodeBlockHeader(src: *const [3]u8) frame.Zstandard.Block.Header {
898 const last_block = src[0] & 1 == 1;
899 const block_type = @as(frame.Zstandard.Block.Type, @enumFromInt((src[0] & 0b110) >> 1));
900 const block_size = ((src[0] & 0b11111000) >> 3) + (@as(u21, src[1]) << 5) + (@as(u21, src[2]) << 13);
901 return .{
902 .last_block = last_block,
903 .block_type = block_type,
904 .block_size = block_size,
905 };
906}
907
908/// Decode the header of a block.
909///
910/// Errors returned:
911/// - `error.EndOfStream` if `src.len < 3`
912pub fn decodeBlockHeaderSlice(src: []const u8) error{EndOfStream}!frame.Zstandard.Block.Header {
913 if (src.len < 3) return error.EndOfStream;
914 return decodeBlockHeader(src[0..3]);
915}
916
917/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
918/// number of bytes the section uses.
919///
920/// Errors returned:
921/// - `error.MalformedLiteralsHeader` if the header is invalid
922/// - `error.MalformedLiteralsSection` if there are decoding errors
923/// - `error.MalformedAccuracyLog` if compressed literals have invalid
924/// accuracy
925/// - `error.MalformedFseTable` if compressed literals have invalid FSE table
926/// - `error.MalformedHuffmanTree` if there are errors decoding a Huffamn tree
927/// - `error.EndOfStream` if there are not enough bytes in `src`
928pub fn decodeLiteralsSectionSlice(
929 src: []const u8,
930 consumed_count: *usize,
931) (error{ MalformedLiteralsHeader, MalformedLiteralsSection, EndOfStream } || huffman.Error)!LiteralsSection {
932 var bytes_read: usize = 0;
933 const header = header: {
934 var fbs = std.io.fixedBufferStream(src);
935 defer bytes_read = fbs.pos;
936 break :header decodeLiteralsHeader(fbs.reader()) catch return error.MalformedLiteralsHeader;
937 };
938 switch (header.block_type) {
939 .raw => {
940 if (src.len < bytes_read + header.regenerated_size) return error.MalformedLiteralsSection;
941 const stream = src[bytes_read..][0..header.regenerated_size];
942 consumed_count.* += header.regenerated_size + bytes_read;
943 return LiteralsSection{
944 .header = header,
945 .huffman_tree = null,
946 .streams = .{ .one = stream },
947 };
948 },
949 .rle => {
950 if (src.len < bytes_read + 1) return error.MalformedLiteralsSection;
951 const stream = src[bytes_read..][0..1];
952 consumed_count.* += 1 + bytes_read;
953 return LiteralsSection{
954 .header = header,
955 .huffman_tree = null,
956 .streams = .{ .one = stream },
957 };
958 },
959 .compressed, .treeless => {
960 const huffman_tree_start = bytes_read;
961 const huffman_tree = if (header.block_type == .compressed)
962 try huffman.decodeHuffmanTreeSlice(src[bytes_read..], &bytes_read)
963 else
964 null;
965 const huffman_tree_size = bytes_read - huffman_tree_start;
966 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
967 return error.MalformedLiteralsSection;
968
969 if (src.len < bytes_read + total_streams_size) return error.MalformedLiteralsSection;
970 const stream_data = src[bytes_read .. bytes_read + total_streams_size];
971
972 const streams = try decodeStreams(header.size_format, stream_data);
973 consumed_count.* += bytes_read + total_streams_size;
974 return LiteralsSection{
975 .header = header,
976 .huffman_tree = huffman_tree,
977 .streams = streams,
978 };
979 },
980 }
981}
982
983/// Decode a `LiteralsSection` from `src`, incrementing `consumed_count` by the
984/// number of bytes the section uses. See `decodeLiterasSectionSlice()`.
985pub fn decodeLiteralsSection(
986 source: anytype,
987 buffer: []u8,
988) !LiteralsSection {
989 const header = try decodeLiteralsHeader(source);
990 switch (header.block_type) {
991 .raw => {
992 if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
993 try source.readNoEof(buffer[0..header.regenerated_size]);
994 return LiteralsSection{
995 .header = header,
996 .huffman_tree = null,
997 .streams = .{ .one = buffer },
998 };
999 },
1000 .rle => {
1001 buffer[0] = try source.readByte();
1002 return LiteralsSection{
1003 .header = header,
1004 .huffman_tree = null,
1005 .streams = .{ .one = buffer[0..1] },
1006 };
1007 },
1008 .compressed, .treeless => {
1009 var counting_reader = std.io.countingReader(source);
1010 const huffman_tree = if (header.block_type == .compressed)
1011 try huffman.decodeHuffmanTree(counting_reader.reader(), buffer)
1012 else
1013 null;
1014 const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1015 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1016 return error.MalformedLiteralsSection;
1017
1018 if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1019 try source.readNoEof(buffer[0..total_streams_size]);
1020 const stream_data = buffer[0..total_streams_size];
1021
1022 const streams = try decodeStreams(header.size_format, stream_data);
1023 return LiteralsSection{
1024 .header = header,
1025 .huffman_tree = huffman_tree,
1026 .streams = streams,
1027 };
1028 },
1029 }
1030}
1031
1032fn decodeStreams(size_format: u2, stream_data: []const u8) !LiteralsSection.Streams {
1033 if (size_format == 0) {
1034 return .{ .one = stream_data };
1035 }
1036
1037 if (stream_data.len < 6) return error.MalformedLiteralsSection;
1038
1039 const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
1040 const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
1041 const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
1042
1043 const stream_1_start = 6;
1044 const stream_2_start = stream_1_start + stream_1_length;
1045 const stream_3_start = stream_2_start + stream_2_length;
1046 const stream_4_start = stream_3_start + stream_3_length;
1047
1048 if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
1049
1050 return .{ .four = .{
1051 stream_data[stream_1_start .. stream_1_start + stream_1_length],
1052 stream_data[stream_2_start .. stream_2_start + stream_2_length],
1053 stream_data[stream_3_start .. stream_3_start + stream_3_length],
1054 stream_data[stream_4_start..],
1055 } };
1056}
1057
1058/// Decode a literals section header.
1059///
1060/// Errors returned:
1061/// - `error.EndOfStream` if there are not enough bytes in `source`
1062pub fn decodeLiteralsHeader(source: anytype) !LiteralsSection.Header {
1063 const byte0 = try source.readByte();
1064 const block_type = @as(LiteralsSection.BlockType, @enumFromInt(byte0 & 0b11));
1065 const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
1066 var regenerated_size: u20 = undefined;
1067 var compressed_size: ?u18 = null;
1068 switch (block_type) {
1069 .raw, .rle => {
1070 switch (size_format) {
1071 0, 2 => {
1072 regenerated_size = byte0 >> 3;
1073 },
1074 1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
1075 3 => regenerated_size = (byte0 >> 4) +
1076 (@as(u20, try source.readByte()) << 4) +
1077 (@as(u20, try source.readByte()) << 12),
1078 }
1079 },
1080 .compressed, .treeless => {
1081 const byte1 = try source.readByte();
1082 const byte2 = try source.readByte();
1083 switch (size_format) {
1084 0, 1 => {
1085 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
1086 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
1087 },
1088 2 => {
1089 const byte3 = try source.readByte();
1090 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
1091 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
1092 },
1093 3 => {
1094 const byte3 = try source.readByte();
1095 const byte4 = try source.readByte();
1096 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
1097 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
1098 },
1099 }
1100 },
1101 }
1102 return LiteralsSection.Header{
1103 .block_type = block_type,
1104 .size_format = size_format,
1105 .regenerated_size = regenerated_size,
1106 .compressed_size = compressed_size,
1107 };
1108}
1109
1110/// Decode a sequences section header.
1111///
1112/// Errors returned:
1113/// - `error.ReservedBitSet` if the reserved bit is set
1114/// - `error.EndOfStream` if there are not enough bytes in `source`
1115pub fn decodeSequencesHeader(
1116 source: anytype,
1117) !SequencesSection.Header {
1118 var sequence_count: u24 = undefined;
1119
1120 const byte0 = try source.readByte();
1121 if (byte0 == 0) {
1122 return SequencesSection.Header{
1123 .sequence_count = 0,
1124 .offsets = undefined,
1125 .match_lengths = undefined,
1126 .literal_lengths = undefined,
1127 };
1128 } else if (byte0 < 128) {
1129 sequence_count = byte0;
1130 } else if (byte0 < 255) {
1131 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1132 } else {
1133 sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1134 }
1135
1136 const compression_modes = try source.readByte();
1137
1138 const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1139 const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1140 const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1141 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1142
1143 return SequencesSection.Header{
1144 .sequence_count = sequence_count,
1145 .offsets = offsets_mode,
1146 .match_lengths = matches_mode,
1147 .literal_lengths = literal_mode,
1148 };
1149}
lib/std/compress/zstandard/decode/fse.zig deleted-153
......@@ -1,153 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
4const types = @import("../types.zig");
5const Table = types.compressed_block.Table;
6
7pub fn decodeFseTable(
8 bit_reader: anytype,
9 expected_symbol_count: usize,
10 max_accuracy_log: u4,
11 entries: []Table.Fse,
12) !usize {
13 const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
14 if (accuracy_log_biased > max_accuracy_log -| 5) return error.MalformedAccuracyLog;
15 const accuracy_log = accuracy_log_biased + 5;
16
17 var values: [256]u16 = undefined;
18 var value_count: usize = 0;
19
20 const total_probability = @as(u16, 1) << accuracy_log;
21 var accumulated_probability: u16 = 0;
22
23 while (accumulated_probability < total_probability) {
24 // WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
25 // but power of two (remaining probabilities + 1) need max bits set to 1 more.
26 const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
27 const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
28
29 const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
30
31 const value = if (small < cutoff)
32 small
33 else value: {
34 const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
35 break :value if (value_read < @as(u16, 1) << (max_bits - 1))
36 value_read
37 else
38 value_read - cutoff;
39 };
40
41 accumulated_probability += if (value != 0) value - 1 else 1;
42
43 values[value_count] = value;
44 value_count += 1;
45
46 if (value == 1) {
47 while (true) {
48 const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
49 if (repeat_flag + value_count > 256) return error.MalformedFseTable;
50 for (0..repeat_flag) |_| {
51 values[value_count] = 1;
52 value_count += 1;
53 }
54 if (repeat_flag < 3) break;
55 }
56 }
57 if (value_count == 256) break;
58 }
59 bit_reader.alignToByte();
60
61 if (value_count < 2) return error.MalformedFseTable;
62 if (accumulated_probability != total_probability) return error.MalformedFseTable;
63 if (value_count > expected_symbol_count) return error.MalformedFseTable;
64
65 const table_size = total_probability;
66
67 try buildFseTable(values[0..value_count], entries[0..table_size]);
68 return table_size;
69}
70
71fn buildFseTable(values: []const u16, entries: []Table.Fse) !void {
72 const total_probability = @as(u16, @intCast(entries.len));
73 const accuracy_log = std.math.log2_int(u16, total_probability);
74 assert(total_probability <= 1 << 9);
75
76 var less_than_one_count: usize = 0;
77 for (values, 0..) |value, i| {
78 if (value == 0) {
79 entries[entries.len - 1 - less_than_one_count] = Table.Fse{
80 .symbol = @as(u8, @intCast(i)),
81 .baseline = 0,
82 .bits = accuracy_log,
83 };
84 less_than_one_count += 1;
85 }
86 }
87
88 var position: usize = 0;
89 var temp_states: [1 << 9]u16 = undefined;
90 for (values, 0..) |value, symbol| {
91 if (value == 0 or value == 1) continue;
92 const probability = value - 1;
93
94 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
95 return error.MalformedFseTable;
96 const share_size = @divExact(total_probability, state_share_dividend);
97 const double_state_count = state_share_dividend - probability;
98 const single_state_count = probability - double_state_count;
99 const share_size_log = std.math.log2_int(u16, share_size);
100
101 for (0..probability) |i| {
102 temp_states[i] = @as(u16, @intCast(position));
103 position += (entries.len >> 1) + (entries.len >> 3) + 3;
104 position &= entries.len - 1;
105 while (position >= entries.len - less_than_one_count) {
106 position += (entries.len >> 1) + (entries.len >> 3) + 3;
107 position &= entries.len - 1;
108 }
109 }
110 std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
111 for (0..probability) |i| {
112 entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
113 .symbol = @as(u8, @intCast(symbol)),
114 .bits = share_size_log + 1,
115 .baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
116 } else Table.Fse{
117 .symbol = @as(u8, @intCast(symbol)),
118 .bits = share_size_log,
119 .baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
120 };
121 }
122 }
123}
124
125test buildFseTable {
126 const literals_length_default_values = [36]u16{
127 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
128 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
129 0, 0, 0, 0,
130 };
131
132 const match_lengths_default_values = [53]u16{
133 2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
134 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
135 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
136 0, 0, 0, 0, 0,
137 };
138
139 const offset_codes_default_values = [29]u16{
140 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
141 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
142 };
143
144 var entries: [64]Table.Fse = undefined;
145 try buildFseTable(&literals_length_default_values, &entries);
146 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_literal_fse_table.fse, &entries);
147
148 try buildFseTable(&match_lengths_default_values, &entries);
149 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_match_fse_table.fse, &entries);
150
151 try buildFseTable(&offset_codes_default_values, entries[0..32]);
152 try std.testing.expectEqualSlices(Table.Fse, types.compressed_block.predefined_offset_fse_table.fse, entries[0..32]);
153}
lib/std/compress/zstandard/decode/huffman.zig deleted-234
......@@ -1,234 +0,0 @@
1const std = @import("std");
2
3const types = @import("../types.zig");
4const LiteralsSection = types.compressed_block.LiteralsSection;
5const Table = types.compressed_block.Table;
6
7const readers = @import("../readers.zig");
8
9const decodeFseTable = @import("fse.zig").decodeFseTable;
10
11pub const Error = error{
12 MalformedHuffmanTree,
13 MalformedFseTable,
14 MalformedAccuracyLog,
15 EndOfStream,
16};
17
18fn decodeFseHuffmanTree(
19 source: anytype,
20 compressed_size: usize,
21 buffer: []u8,
22 weights: *[256]u4,
23) !usize {
24 var stream = std.io.limitedReader(source, compressed_size);
25 var bit_reader = readers.bitReader(stream.reader());
26
27 var entries: [1 << 6]Table.Fse = undefined;
28 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
29 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
30 error.EndOfStream => return error.MalformedFseTable,
31 else => |e| return e,
32 };
33 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
34
35 const amount = try stream.reader().readAll(buffer);
36 var huff_bits: readers.ReverseBitReader = undefined;
37 huff_bits.init(buffer[0..amount]) catch return error.MalformedHuffmanTree;
38
39 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
40}
41
42fn decodeFseHuffmanTreeSlice(src: []const u8, compressed_size: usize, weights: *[256]u4) !usize {
43 if (src.len < compressed_size) return error.MalformedHuffmanTree;
44 var stream = std.io.fixedBufferStream(src[0..compressed_size]);
45 var counting_reader = std.io.countingReader(stream.reader());
46 var bit_reader = readers.bitReader(counting_reader.reader());
47
48 var entries: [1 << 6]Table.Fse = undefined;
49 const table_size = decodeFseTable(&bit_reader, 256, 6, &entries) catch |err| switch (err) {
50 error.MalformedAccuracyLog, error.MalformedFseTable => |e| return e,
51 error.EndOfStream => return error.MalformedFseTable,
52 };
53 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
54
55 const start_index = std.math.cast(usize, counting_reader.bytes_read) orelse
56 return error.MalformedHuffmanTree;
57 const huff_data = src[start_index..compressed_size];
58 var huff_bits: readers.ReverseBitReader = undefined;
59 huff_bits.init(huff_data) catch return error.MalformedHuffmanTree;
60
61 return assignWeights(&huff_bits, accuracy_log, &entries, weights);
62}
63
64fn assignWeights(
65 huff_bits: *readers.ReverseBitReader,
66 accuracy_log: u16,
67 entries: *[1 << 6]Table.Fse,
68 weights: *[256]u4,
69) !usize {
70 var i: usize = 0;
71 var even_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
72 var odd_state: u32 = huff_bits.readBitsNoEof(u32, accuracy_log) catch return error.MalformedHuffmanTree;
73
74 while (i < 254) {
75 const even_data = entries[even_state];
76 var read_bits: u16 = 0;
77 const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
78 weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
79 i += 1;
80 if (read_bits < even_data.bits) {
81 weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
82 i += 1;
83 break;
84 }
85 even_state = even_data.baseline + even_bits;
86
87 read_bits = 0;
88 const odd_data = entries[odd_state];
89 const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
90 weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
91 i += 1;
92 if (read_bits < odd_data.bits) {
93 if (i == 255) return error.MalformedHuffmanTree;
94 weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
95 i += 1;
96 break;
97 }
98 odd_state = odd_data.baseline + odd_bits;
99 } else return error.MalformedHuffmanTree;
100
101 if (!huff_bits.isEmpty()) {
102 return error.MalformedHuffmanTree;
103 }
104
105 return i + 1; // stream contains all but the last symbol
106}
107
108fn decodeDirectHuffmanTree(source: anytype, encoded_symbol_count: usize, weights: *[256]u4) !usize {
109 const weights_byte_count = (encoded_symbol_count + 1) / 2;
110 for (0..weights_byte_count) |i| {
111 const byte = try source.readByte();
112 weights[2 * i] = @as(u4, @intCast(byte >> 4));
113 weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
114 }
115 return encoded_symbol_count + 1;
116}
117
118fn assignSymbols(weight_sorted_prefixed_symbols: []LiteralsSection.HuffmanTree.PrefixedSymbol, weights: [256]u4) usize {
119 for (0..weight_sorted_prefixed_symbols.len) |i| {
120 weight_sorted_prefixed_symbols[i] = .{
121 .symbol = @as(u8, @intCast(i)),
122 .weight = undefined,
123 .prefix = undefined,
124 };
125 }
126
127 std.mem.sort(
128 LiteralsSection.HuffmanTree.PrefixedSymbol,
129 weight_sorted_prefixed_symbols,
130 weights,
131 lessThanByWeight,
132 );
133
134 var prefix: u16 = 0;
135 var prefixed_symbol_count: usize = 0;
136 var sorted_index: usize = 0;
137 const symbol_count = weight_sorted_prefixed_symbols.len;
138 while (sorted_index < symbol_count) {
139 var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
140 const weight = weights[symbol];
141 if (weight == 0) {
142 sorted_index += 1;
143 continue;
144 }
145
146 while (sorted_index < symbol_count) : ({
147 sorted_index += 1;
148 prefixed_symbol_count += 1;
149 prefix += 1;
150 }) {
151 symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
152 if (weights[symbol] != weight) {
153 prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
154 break;
155 }
156 weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
157 weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
158 weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
159 }
160 }
161 return prefixed_symbol_count;
162}
163
164fn buildHuffmanTree(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!LiteralsSection.HuffmanTree {
165 var weight_power_sum_big: u32 = 0;
166 for (weights[0 .. symbol_count - 1]) |value| {
167 weight_power_sum_big += (@as(u16, 1) << value) >> 1;
168 }
169 if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
170 const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
171
172 // advance to next power of two (even if weight_power_sum is a power of 2)
173 // TODO: is it valid to have weight_power_sum == 0?
174 const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
175 const next_power_of_two = @as(u16, 1) << max_number_of_bits;
176 weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
177
178 var weight_sorted_prefixed_symbols: [256]LiteralsSection.HuffmanTree.PrefixedSymbol = undefined;
179 const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
180 const tree = LiteralsSection.HuffmanTree{
181 .max_bit_count = max_number_of_bits,
182 .symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
183 .nodes = weight_sorted_prefixed_symbols,
184 };
185 return tree;
186}
187
188pub fn decodeHuffmanTree(
189 source: anytype,
190 buffer: []u8,
191) (@TypeOf(source).Error || Error)!LiteralsSection.HuffmanTree {
192 const header = try source.readByte();
193 var weights: [256]u4 = undefined;
194 const symbol_count = if (header < 128)
195 // FSE compressed weights
196 try decodeFseHuffmanTree(source, header, buffer, &weights)
197 else
198 try decodeDirectHuffmanTree(source, header - 127, &weights);
199
200 return buildHuffmanTree(&weights, symbol_count);
201}
202
203pub fn decodeHuffmanTreeSlice(
204 src: []const u8,
205 consumed_count: *usize,
206) Error!LiteralsSection.HuffmanTree {
207 if (src.len == 0) return error.MalformedHuffmanTree;
208 const header = src[0];
209 var bytes_read: usize = 1;
210 var weights: [256]u4 = undefined;
211 const symbol_count = if (header < 128) count: {
212 // FSE compressed weights
213 bytes_read += header;
214 break :count try decodeFseHuffmanTreeSlice(src[1..], header, &weights);
215 } else count: {
216 var fbs = std.io.fixedBufferStream(src[1..]);
217 defer bytes_read += fbs.pos;
218 break :count try decodeDirectHuffmanTree(fbs.reader(), header - 127, &weights);
219 };
220
221 consumed_count.* += bytes_read;
222 return buildHuffmanTree(&weights, symbol_count);
223}
224
225fn lessThanByWeight(
226 weights: [256]u4,
227 lhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
228 rhs: LiteralsSection.HuffmanTree.PrefixedSymbol,
229) bool {
230 // NOTE: this function relies on the use of a stable sorting algorithm,
231 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
232 // should be added
233 return weights[lhs.symbol] < weights[rhs.symbol];
234}
lib/std/compress/zstandard/decompress.zig deleted-633
......@@ -1,633 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;
4const RingBuffer = std.RingBuffer;
5
6const types = @import("types.zig");
7const frame = types.frame;
8const LiteralsSection = types.compressed_block.LiteralsSection;
9const SequencesSection = types.compressed_block.SequencesSection;
10const SkippableHeader = types.frame.Skippable.Header;
11const ZstandardHeader = types.frame.Zstandard.Header;
12const Table = types.compressed_block.Table;
13
14pub const block = @import("decode/block.zig");
15
16const readers = @import("readers.zig");
17
18/// Returns `true` is `magic` is a valid magic number for a skippable frame
19pub fn isSkippableMagic(magic: u32) bool {
20 return frame.Skippable.magic_number_min <= magic and magic <= frame.Skippable.magic_number_max;
21}
22
23/// Returns the kind of frame at the beginning of `source`.
24///
25/// Errors returned:
26/// - `error.BadMagic` if `source` begins with bytes not equal to the
27/// Zstandard frame magic number, or outside the range of magic numbers for
28/// skippable frames.
29/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
30pub fn decodeFrameType(source: anytype) error{ BadMagic, EndOfStream }!frame.Kind {
31 const magic = try source.readInt(u32, .little);
32 return frameType(magic);
33}
34
35/// Returns the kind of frame associated to `magic`.
36///
37/// Errors returned:
38/// - `error.BadMagic` if `magic` is not a valid magic number.
39pub fn frameType(magic: u32) error{BadMagic}!frame.Kind {
40 return if (magic == frame.Zstandard.magic_number)
41 .zstandard
42 else if (isSkippableMagic(magic))
43 .skippable
44 else
45 error.BadMagic;
46}
47
48pub const FrameHeader = union(enum) {
49 zstandard: ZstandardHeader,
50 skippable: SkippableHeader,
51};
52
53pub const HeaderError = error{ BadMagic, EndOfStream, ReservedBitSet };
54
55/// Returns the header of the frame at the beginning of `source`.
56///
57/// Errors returned:
58/// - `error.BadMagic` if `source` begins with bytes not equal to the
59/// Zstandard frame magic number, or outside the range of magic numbers for
60/// skippable frames.
61/// - `error.EndOfStream` if `source` contains fewer than 4 bytes
62/// - `error.ReservedBitSet` if the frame is a Zstandard frame and any of the
63/// reserved bits are set
64pub fn decodeFrameHeader(source: anytype) (@TypeOf(source).Error || HeaderError)!FrameHeader {
65 const magic = try source.readInt(u32, .little);
66 const frame_type = try frameType(magic);
67 switch (frame_type) {
68 .zstandard => return FrameHeader{ .zstandard = try decodeZstandardHeader(source) },
69 .skippable => return FrameHeader{
70 .skippable = .{
71 .magic_number = magic,
72 .frame_size = try source.readInt(u32, .little),
73 },
74 },
75 }
76}
77
78pub const ReadWriteCount = struct {
79 read_count: usize,
80 write_count: usize,
81};
82
83/// Decodes frames from `src` into `dest`; returns the length of the result.
84/// The stream should not have extra trailing bytes - either all bytes in `src`
85/// will be decoded, or an error will be returned. An error will be returned if
86/// a Zstandard frame in `src` does not declare its content size.
87///
88/// Errors returned:
89/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
90/// uses a dictionary
91/// - `error.MalformedFrame` if a frame in `src` is invalid
92/// - `error.UnknownContentSizeUnsupported` if a frame in `src` does not
93/// declare its content size
94pub fn decode(dest: []u8, src: []const u8, verify_checksum: bool) error{
95 MalformedFrame,
96 UnknownContentSizeUnsupported,
97 DictionaryIdFlagUnsupported,
98}!usize {
99 var write_count: usize = 0;
100 var read_count: usize = 0;
101 while (read_count < src.len) {
102 const counts = decodeFrame(dest, src[read_count..], verify_checksum) catch |err| {
103 switch (err) {
104 error.UnknownContentSizeUnsupported => return error.UnknownContentSizeUnsupported,
105 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
106 else => return error.MalformedFrame,
107 }
108 };
109 read_count += counts.read_count;
110 write_count += counts.write_count;
111 }
112 return write_count;
113}
114
115/// Decodes a stream of frames from `src`; returns the decoded bytes. The stream
116/// should not have extra trailing bytes - either all bytes in `src` will be
117/// decoded, or an error will be returned.
118///
119/// Errors returned:
120/// - `error.DictionaryIdFlagUnsupported` if a `src` contains a frame that
121/// uses a dictionary
122/// - `error.MalformedFrame` if a frame in `src` is invalid
123/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
124pub fn decodeAlloc(
125 allocator: Allocator,
126 src: []const u8,
127 verify_checksum: bool,
128 window_size_max: usize,
129) error{ DictionaryIdFlagUnsupported, MalformedFrame, OutOfMemory }![]u8 {
130 var result = std.ArrayList(u8).init(allocator);
131 errdefer result.deinit();
132
133 var read_count: usize = 0;
134 while (read_count < src.len) {
135 read_count += decodeFrameArrayList(
136 allocator,
137 &result,
138 src[read_count..],
139 verify_checksum,
140 window_size_max,
141 ) catch |err| switch (err) {
142 error.OutOfMemory => return error.OutOfMemory,
143 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
144 else => return error.MalformedFrame,
145 };
146 }
147 return result.toOwnedSlice();
148}
149
150/// Decodes the frame at the start of `src` into `dest`. Returns the number of
151/// bytes read from `src` and written to `dest`. This function can only decode
152/// frames that declare the decompressed content size.
153///
154/// Errors returned:
155/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
156/// number for a Zstandard or skippable frame
157/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
158/// uncompressed content size
159/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
160/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
161/// size declared by the frame header
162/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
163/// that is larger than `std.math.maxInt(usize)`
164/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
165/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
166/// contains a checksum that does not match the checksum of the decompressed
167/// data
168/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
169/// are set
170/// - `error.EndOfStream` if `src` does not contain a complete frame
171/// - `error.BadContentSize` if the content size declared by the frame does
172/// not equal the actual size of decompressed data
173/// - an error in `block.Error` if there are errors decoding a block
174/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
175/// size greater than `src.len`
176pub fn decodeFrame(
177 dest: []u8,
178 src: []const u8,
179 verify_checksum: bool,
180) (error{
181 BadMagic,
182 UnknownContentSizeUnsupported,
183 ContentTooLarge,
184 ContentSizeTooLarge,
185 WindowSizeUnknown,
186 DictionaryIdFlagUnsupported,
187 SkippableSizeTooLarge,
188} || FrameError)!ReadWriteCount {
189 var fbs = std.io.fixedBufferStream(src);
190 switch (try decodeFrameType(fbs.reader())) {
191 .zstandard => return decodeZstandardFrame(dest, src, verify_checksum),
192 .skippable => {
193 const content_size = try fbs.reader().readInt(u32, .little);
194 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
195 const read_count = @as(usize, content_size) + 8;
196 if (read_count > src.len) return error.SkippableSizeTooLarge;
197 return ReadWriteCount{
198 .read_count = read_count,
199 .write_count = 0,
200 };
201 },
202 }
203}
204
205/// Decodes the frame at the start of `src` into `dest`. Returns the number of
206/// bytes read from `src`.
207///
208/// Errors returned:
209/// - `error.BadMagic` if the first 4 bytes of `src` is not a valid magic
210/// number for a Zstandard or skippable frame
211/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
212/// - `error.WindowTooLarge` if the window size is larger than
213/// `window_size_max`
214/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
215/// that is larger than `std.math.maxInt(usize)`
216/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
217/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
218/// contains a checksum that does not match the checksum of the decompressed
219/// data
220/// - `error.ReservedBitSet` if any of the reserved bits of the frame header
221/// are set
222/// - `error.EndOfStream` if `src` does not contain a complete frame
223/// - `error.BadContentSize` if the content size declared by the frame does
224/// not equal the actual size of decompressed data
225/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
226/// - an error in `block.Error` if there are errors decoding a block
227/// - `error.SkippableSizeTooLarge` if the frame is skippable and reports a
228/// size greater than `src.len`
229pub fn decodeFrameArrayList(
230 allocator: Allocator,
231 dest: *std.ArrayList(u8),
232 src: []const u8,
233 verify_checksum: bool,
234 window_size_max: usize,
235) (error{ BadMagic, OutOfMemory, SkippableSizeTooLarge } || FrameContext.Error || FrameError)!usize {
236 var fbs = std.io.fixedBufferStream(src);
237 const reader = fbs.reader();
238 const magic = try reader.readInt(u32, .little);
239 switch (try frameType(magic)) {
240 .zstandard => return decodeZstandardFrameArrayList(
241 allocator,
242 dest,
243 src,
244 verify_checksum,
245 window_size_max,
246 ),
247 .skippable => {
248 const content_size = try fbs.reader().readInt(u32, .little);
249 if (content_size > std.math.maxInt(usize) - 8) return error.SkippableSizeTooLarge;
250 const read_count = @as(usize, content_size) + 8;
251 if (read_count > src.len) return error.SkippableSizeTooLarge;
252 return read_count;
253 },
254 }
255}
256
257/// Returns the frame checksum corresponding to the data fed into `hasher`
258pub fn computeChecksum(hasher: *std.hash.XxHash64) u32 {
259 const hash = hasher.final();
260 return @as(u32, @intCast(hash & 0xFFFFFFFF));
261}
262
263const FrameError = error{
264 ChecksumFailure,
265 BadContentSize,
266 EndOfStream,
267 ReservedBitSet,
268} || block.Error;
269
270/// Decode a Zstandard frame from `src` into `dest`, returning the number of
271/// bytes read from `src` and written to `dest`. The first four bytes of `src`
272/// must be the magic number for a Zstandard frame.
273///
274/// Error returned:
275/// - `error.UnknownContentSizeUnsupported` if the frame does not declare the
276/// uncompressed content size
277/// - `error.ContentTooLarge` if `dest` is smaller than the uncompressed data
278/// size declared by the frame header
279/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
280/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
281/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
282/// that is larger than `std.math.maxInt(usize)`
283/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
284/// contains a checksum that does not match the checksum of the decompressed
285/// data
286/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
287/// - `error.EndOfStream` if `src` does not contain a complete frame
288/// - an error in `block.Error` if there are errors decoding a block
289/// - `error.BadContentSize` if the content size declared by the frame does
290/// not equal the actual size of decompressed data
291pub fn decodeZstandardFrame(
292 dest: []u8,
293 src: []const u8,
294 verify_checksum: bool,
295) (error{
296 UnknownContentSizeUnsupported,
297 ContentTooLarge,
298 ContentSizeTooLarge,
299 WindowSizeUnknown,
300 DictionaryIdFlagUnsupported,
301} || FrameError)!ReadWriteCount {
302 assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
303 var consumed_count: usize = 4;
304
305 var frame_context = context: {
306 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
307 const source = fbs.reader();
308 const frame_header = try decodeZstandardHeader(source);
309 consumed_count += fbs.pos;
310 break :context FrameContext.init(
311 frame_header,
312 std.math.maxInt(usize),
313 verify_checksum,
314 ) catch |err| switch (err) {
315 error.WindowTooLarge => unreachable,
316 inline else => |e| return e,
317 };
318 };
319 const counts = try decodeZStandardFrameBlocks(
320 dest,
321 src[consumed_count..],
322 &frame_context,
323 );
324 return ReadWriteCount{
325 .read_count = counts.read_count + consumed_count,
326 .write_count = counts.write_count,
327 };
328}
329
330pub fn decodeZStandardFrameBlocks(
331 dest: []u8,
332 src: []const u8,
333 frame_context: *FrameContext,
334) (error{ ContentTooLarge, UnknownContentSizeUnsupported } || FrameError)!ReadWriteCount {
335 const content_size = frame_context.content_size orelse
336 return error.UnknownContentSizeUnsupported;
337 if (dest.len < content_size) return error.ContentTooLarge;
338
339 var consumed_count: usize = 0;
340 const written_count = decodeFrameBlocksInner(
341 dest[0..content_size],
342 src[consumed_count..],
343 &consumed_count,
344 if (frame_context.hasher_opt) |*hasher| hasher else null,
345 frame_context.block_size_max,
346 ) catch |err| switch (err) {
347 error.DestTooSmall => return error.BadContentSize,
348 inline else => |e| return e,
349 };
350
351 if (written_count != content_size) return error.BadContentSize;
352 if (frame_context.has_checksum) {
353 if (src.len < consumed_count + 4) return error.EndOfStream;
354 const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
355 consumed_count += 4;
356 if (frame_context.hasher_opt) |*hasher| {
357 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
358 }
359 }
360 return ReadWriteCount{ .read_count = consumed_count, .write_count = written_count };
361}
362
363pub const FrameContext = struct {
364 hasher_opt: ?std.hash.XxHash64,
365 window_size: usize,
366 has_checksum: bool,
367 block_size_max: usize,
368 content_size: ?usize,
369
370 const Error = error{
371 DictionaryIdFlagUnsupported,
372 WindowSizeUnknown,
373 WindowTooLarge,
374 ContentSizeTooLarge,
375 };
376 /// Validates `frame_header` and returns the associated `FrameContext`.
377 ///
378 /// Errors returned:
379 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
380 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
381 /// size
382 /// - `error.WindowTooLarge` if the window size is larger than
383 /// `window_size_max` or `std.math.intMax(usize)`
384 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
385 /// size larger than `std.math.maxInt(usize)`
386 pub fn init(
387 frame_header: ZstandardHeader,
388 window_size_max: usize,
389 verify_checksum: bool,
390 ) Error!FrameContext {
391 if (frame_header.descriptor.dictionary_id_flag != 0)
392 return error.DictionaryIdFlagUnsupported;
393
394 const window_size_raw = frameWindowSize(frame_header) orelse return error.WindowSizeUnknown;
395 const window_size = if (window_size_raw > window_size_max)
396 return error.WindowTooLarge
397 else
398 std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
399
400 const should_compute_checksum =
401 frame_header.descriptor.content_checksum_flag and verify_checksum;
402
403 const content_size = if (frame_header.content_size) |size|
404 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
405 else
406 null;
407
408 return .{
409 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
410 .window_size = window_size,
411 .has_checksum = frame_header.descriptor.content_checksum_flag,
412 .block_size_max = @min(types.block_size_max, window_size),
413 .content_size = content_size,
414 };
415 }
416};
417
418/// Decode a Zstandard from from `src` and return number of bytes read; see
419/// `decodeZstandardFrame()`. The first four bytes of `src` must be the magic
420/// number for a Zstandard frame.
421///
422/// Errors returned:
423/// - `error.WindowSizeUnknown` if the frame does not have a valid window size
424/// - `error.WindowTooLarge` if the window size is larger than
425/// `window_size_max`
426/// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
427/// - `error.ContentSizeTooLarge` if the frame header indicates a content size
428/// that is larger than `std.math.maxInt(usize)`
429/// - `error.ChecksumFailure` if `verify_checksum` is true and the frame
430/// contains a checksum that does not match the checksum of the decompressed
431/// data
432/// - `error.ReservedBitSet` if the reserved bit of the frame header is set
433/// - `error.EndOfStream` if `src` does not contain a complete frame
434/// - `error.OutOfMemory` if `allocator` cannot allocate enough memory
435/// - an error in `block.Error` if there are errors decoding a block
436/// - `error.BadContentSize` if the content size declared by the frame does
437/// not equal the size of decompressed data
438pub fn decodeZstandardFrameArrayList(
439 allocator: Allocator,
440 dest: *std.ArrayList(u8),
441 src: []const u8,
442 verify_checksum: bool,
443 window_size_max: usize,
444) (error{OutOfMemory} || FrameContext.Error || FrameError)!usize {
445 assert(std.mem.readInt(u32, src[0..4], .little) == frame.Zstandard.magic_number);
446 var consumed_count: usize = 4;
447
448 var frame_context = context: {
449 var fbs = std.io.fixedBufferStream(src[consumed_count..]);
450 const source = fbs.reader();
451 const frame_header = try decodeZstandardHeader(source);
452 consumed_count += fbs.pos;
453 break :context try FrameContext.init(frame_header, window_size_max, verify_checksum);
454 };
455
456 consumed_count += try decodeZstandardFrameBlocksArrayList(
457 allocator,
458 dest,
459 src[consumed_count..],
460 &frame_context,
461 );
462 return consumed_count;
463}
464
465pub fn decodeZstandardFrameBlocksArrayList(
466 allocator: Allocator,
467 dest: *std.ArrayList(u8),
468 src: []const u8,
469 frame_context: *FrameContext,
470) (error{OutOfMemory} || FrameError)!usize {
471 const initial_len = dest.items.len;
472
473 var ring_buffer = try RingBuffer.init(allocator, frame_context.window_size);
474 defer ring_buffer.deinit(allocator);
475
476 // These tables take 7680 bytes
477 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
478 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
479 var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
480
481 var block_header = try block.decodeBlockHeaderSlice(src);
482 var consumed_count: usize = 3;
483 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
484 while (true) : ({
485 block_header = try block.decodeBlockHeaderSlice(src[consumed_count..]);
486 consumed_count += 3;
487 }) {
488 const written_size = try block.decodeBlockRingBuffer(
489 &ring_buffer,
490 src[consumed_count..],
491 block_header,
492 &decode_state,
493 &consumed_count,
494 frame_context.block_size_max,
495 );
496 if (frame_context.content_size) |size| {
497 if (dest.items.len - initial_len > size) {
498 return error.BadContentSize;
499 }
500 }
501 if (written_size > 0) {
502 const written_slice = ring_buffer.sliceLast(written_size);
503 try dest.appendSlice(written_slice.first);
504 try dest.appendSlice(written_slice.second);
505 if (frame_context.hasher_opt) |*hasher| {
506 hasher.update(written_slice.first);
507 hasher.update(written_slice.second);
508 }
509 }
510 if (block_header.last_block) break;
511 }
512 if (frame_context.content_size) |size| {
513 if (dest.items.len - initial_len != size) {
514 return error.BadContentSize;
515 }
516 }
517
518 if (frame_context.has_checksum) {
519 if (src.len < consumed_count + 4) return error.EndOfStream;
520 const checksum = std.mem.readInt(u32, src[consumed_count..][0..4], .little);
521 consumed_count += 4;
522 if (frame_context.hasher_opt) |*hasher| {
523 if (checksum != computeChecksum(hasher)) return error.ChecksumFailure;
524 }
525 }
526 return consumed_count;
527}
528
529fn decodeFrameBlocksInner(
530 dest: []u8,
531 src: []const u8,
532 consumed_count: *usize,
533 hash: ?*std.hash.XxHash64,
534 block_size_max: usize,
535) (error{ EndOfStream, DestTooSmall } || block.Error)!usize {
536 // These tables take 7680 bytes
537 var literal_fse_data: [types.compressed_block.table_size_max.literal]Table.Fse = undefined;
538 var match_fse_data: [types.compressed_block.table_size_max.match]Table.Fse = undefined;
539 var offset_fse_data: [types.compressed_block.table_size_max.offset]Table.Fse = undefined;
540
541 var block_header = try block.decodeBlockHeaderSlice(src);
542 var bytes_read: usize = 3;
543 defer consumed_count.* += bytes_read;
544 var decode_state = block.DecodeState.init(&literal_fse_data, &match_fse_data, &offset_fse_data);
545 var count: usize = 0;
546 while (true) : ({
547 block_header = try block.decodeBlockHeaderSlice(src[bytes_read..]);
548 bytes_read += 3;
549 }) {
550 const written_size = try block.decodeBlock(
551 dest,
552 src[bytes_read..],
553 block_header,
554 &decode_state,
555 &bytes_read,
556 block_size_max,
557 count,
558 );
559 if (hash) |hash_state| hash_state.update(dest[count .. count + written_size]);
560 count += written_size;
561 if (block_header.last_block) break;
562 }
563 return count;
564}
565
566/// Decode the header of a skippable frame. The first four bytes of `src` must
567/// be a valid magic number for a skippable frame.
568pub fn decodeSkippableHeader(src: *const [8]u8) SkippableHeader {
569 const magic = std.mem.readInt(u32, src[0..4], .little);
570 assert(isSkippableMagic(magic));
571 const frame_size = std.mem.readInt(u32, src[4..8], .little);
572 return .{
573 .magic_number = magic,
574 .frame_size = frame_size,
575 };
576}
577
578/// Returns the window size required to decompress a frame, or `null` if it
579/// cannot be determined (which indicates a malformed frame header).
580pub fn frameWindowSize(header: ZstandardHeader) ?u64 {
581 if (header.window_descriptor) |descriptor| {
582 const exponent = (descriptor & 0b11111000) >> 3;
583 const mantissa = descriptor & 0b00000111;
584 const window_log = 10 + exponent;
585 const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
586 const window_add = (window_base / 8) * mantissa;
587 return window_base + window_add;
588 } else return header.content_size;
589}
590
591/// Decode the header of a Zstandard frame.
592///
593/// Errors returned:
594/// - `error.ReservedBitSet` if any of the reserved bits of the header are set
595/// - `error.EndOfStream` if `source` does not contain a complete header
596pub fn decodeZstandardHeader(
597 source: anytype,
598) (@TypeOf(source).Error || error{ EndOfStream, ReservedBitSet })!ZstandardHeader {
599 const descriptor = @as(ZstandardHeader.Descriptor, @bitCast(try source.readByte()));
600
601 if (descriptor.reserved) return error.ReservedBitSet;
602
603 var window_descriptor: ?u8 = null;
604 if (!descriptor.single_segment_flag) {
605 window_descriptor = try source.readByte();
606 }
607
608 var dictionary_id: ?u32 = null;
609 if (descriptor.dictionary_id_flag > 0) {
610 // if flag is 3 then field_size = 4, else field_size = flag
611 const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
612 dictionary_id = try source.readVarInt(u32, .little, field_size);
613 }
614
615 var content_size: ?u64 = null;
616 if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) {
617 const field_size = @as(u4, 1) << descriptor.content_size_flag;
618 content_size = try source.readVarInt(u64, .little, field_size);
619 if (field_size == 2) content_size.? += 256;
620 }
621
622 const header = ZstandardHeader{
623 .descriptor = descriptor,
624 .window_descriptor = window_descriptor,
625 .dictionary_id = dictionary_id,
626 .content_size = content_size,
627 };
628 return header;
629}
630
631test {
632 std.testing.refAllDecls(@This());
633}
lib/std/compress/zstandard/readers.zig deleted-82
......@@ -1,82 +0,0 @@
1const std = @import("std");
2
3pub const ReversedByteReader = struct {
4 remaining_bytes: usize,
5 bytes: []const u8,
6
7 const Reader = std.io.GenericReader(*ReversedByteReader, error{}, readFn);
8
9 pub fn init(bytes: []const u8) ReversedByteReader {
10 return .{
11 .bytes = bytes,
12 .remaining_bytes = bytes.len,
13 };
14 }
15
16 pub fn reader(self: *ReversedByteReader) Reader {
17 return .{ .context = self };
18 }
19
20 fn readFn(ctx: *ReversedByteReader, buffer: []u8) !usize {
21 if (ctx.remaining_bytes == 0) return 0;
22 const byte_index = ctx.remaining_bytes - 1;
23 buffer[0] = ctx.bytes[byte_index];
24 // buffer[0] = @bitReverse(ctx.bytes[byte_index]);
25 ctx.remaining_bytes = byte_index;
26 return 1;
27 }
28};
29
30/// A bit reader for reading the reversed bit streams used to encode
31/// FSE compressed data.
32pub const ReverseBitReader = struct {
33 byte_reader: ReversedByteReader,
34 bit_reader: std.io.BitReader(.big, ReversedByteReader.Reader),
35
36 pub fn init(self: *ReverseBitReader, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
37 self.byte_reader = ReversedByteReader.init(bytes);
38 self.bit_reader = std.io.bitReader(.big, self.byte_reader.reader());
39 if (bytes.len == 0) return;
40 var i: usize = 0;
41 while (i < 8 and 0 == self.readBitsNoEof(u1, 1) catch unreachable) : (i += 1) {}
42 if (i == 8) return error.BitStreamHasNoStartBit;
43 }
44
45 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) error{EndOfStream}!U {
46 return self.bit_reader.readBitsNoEof(U, num_bits);
47 }
48
49 pub fn readBits(self: *@This(), comptime U: type, num_bits: u16, out_bits: *u16) error{}!U {
50 return try self.bit_reader.readBits(U, num_bits, out_bits);
51 }
52
53 pub fn alignToByte(self: *@This()) void {
54 self.bit_reader.alignToByte();
55 }
56
57 pub fn isEmpty(self: ReverseBitReader) bool {
58 return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
59 }
60};
61
62pub fn BitReader(comptime Reader: type) type {
63 return struct {
64 underlying: std.io.BitReader(.little, Reader),
65
66 pub fn readBitsNoEof(self: *@This(), comptime U: type, num_bits: u16) !U {
67 return self.underlying.readBitsNoEof(U, num_bits);
68 }
69
70 pub fn readBits(self: *@This(), comptime U: type, num_bits: u16, out_bits: *u16) !U {
71 return self.underlying.readBits(U, num_bits, out_bits);
72 }
73
74 pub fn alignToByte(self: *@This()) void {
75 self.underlying.alignToByte();
76 }
77 };
78}
79
80pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
81 return .{ .underlying = std.io.bitReader(.little, reader) };
82}
lib/std/compress/zstandard/types.zig deleted-403
......@@ -1,403 +0,0 @@
1pub const block_size_max = 1 << 17;
2
3pub const frame = struct {
4 pub const Kind = enum { zstandard, skippable };
5
6 pub const Zstandard = struct {
7 pub const magic_number = 0xFD2FB528;
8
9 header: Header,
10 data_blocks: []Block,
11 checksum: ?u32,
12
13 pub const Header = struct {
14 descriptor: Descriptor,
15 window_descriptor: ?u8,
16 dictionary_id: ?u32,
17 content_size: ?u64,
18
19 pub const Descriptor = packed struct {
20 dictionary_id_flag: u2,
21 content_checksum_flag: bool,
22 reserved: bool,
23 unused: bool,
24 single_segment_flag: bool,
25 content_size_flag: u2,
26 };
27 };
28
29 pub const Block = struct {
30 pub const Header = struct {
31 last_block: bool,
32 block_type: Block.Type,
33 block_size: u21,
34 };
35
36 pub const Type = enum(u2) {
37 raw,
38 rle,
39 compressed,
40 reserved,
41 };
42 };
43 };
44
45 pub const Skippable = struct {
46 pub const magic_number_min = 0x184D2A50;
47 pub const magic_number_max = 0x184D2A5F;
48
49 pub const Header = struct {
50 magic_number: u32,
51 frame_size: u32,
52 };
53 };
54};
55
56pub const compressed_block = struct {
57 pub const LiteralsSection = struct {
58 header: Header,
59 huffman_tree: ?HuffmanTree,
60 streams: Streams,
61
62 pub const Streams = union(enum) {
63 one: []const u8,
64 four: [4][]const u8,
65 };
66
67 pub const Header = struct {
68 block_type: BlockType,
69 size_format: u2,
70 regenerated_size: u20,
71 compressed_size: ?u18,
72 };
73
74 pub const BlockType = enum(u2) {
75 raw,
76 rle,
77 compressed,
78 treeless,
79 };
80
81 pub const HuffmanTree = struct {
82 max_bit_count: u4,
83 symbol_count_minus_one: u8,
84 nodes: [256]PrefixedSymbol,
85
86 pub const PrefixedSymbol = struct {
87 symbol: u8,
88 prefix: u16,
89 weight: u4,
90 };
91
92 pub const Result = union(enum) {
93 symbol: u8,
94 index: usize,
95 };
96
97 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
98 var node = self.nodes[index];
99 const weight = node.weight;
100 var i: usize = index;
101 while (node.weight == weight) {
102 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
103 if (i == 0) return error.NotFound;
104 i -= 1;
105 node = self.nodes[i];
106 }
107 return Result{ .index = i };
108 }
109
110 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
111 return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
112 }
113 };
114
115 pub const StreamCount = enum { one, four };
116 pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
117 return switch (block_type) {
118 .raw, .rle => .one,
119 .compressed, .treeless => if (size_format == 0) .one else .four,
120 };
121 }
122 };
123
124 pub const SequencesSection = struct {
125 header: SequencesSection.Header,
126 literals_length_table: Table,
127 offset_table: Table,
128 match_length_table: Table,
129
130 pub const Header = struct {
131 sequence_count: u24,
132 match_lengths: Mode,
133 offsets: Mode,
134 literal_lengths: Mode,
135
136 pub const Mode = enum(u2) {
137 predefined,
138 rle,
139 fse,
140 repeat,
141 };
142 };
143 };
144
145 pub const Table = union(enum) {
146 fse: []const Fse,
147 rle: u8,
148
149 pub const Fse = struct {
150 symbol: u8,
151 baseline: u16,
152 bits: u8,
153 };
154 };
155
156 pub const literals_length_code_table = [36]struct { u32, u5 }{
157 .{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
158 .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
159 .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
160 .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
161 .{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
162 .{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
163 .{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
164 .{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
165 .{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
166 };
167
168 pub const match_length_code_table = [53]struct { u32, u5 }{
169 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
170 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
171 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
172 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
173 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
174 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
175 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
176 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
177 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
178 };
179
180 pub const literals_length_default_distribution = [36]i16{
181 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
182 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
183 -1, -1, -1, -1,
184 };
185
186 pub const match_lengths_default_distribution = [53]i16{
187 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
190 -1, -1, -1, -1, -1,
191 };
192
193 pub const offset_codes_default_distribution = [29]i16{
194 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
195 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
196 };
197
198 pub const predefined_literal_fse_table = Table{
199 .fse = &[64]Table.Fse{
200 .{ .symbol = 0, .bits = 4, .baseline = 0 },
201 .{ .symbol = 0, .bits = 4, .baseline = 16 },
202 .{ .symbol = 1, .bits = 5, .baseline = 32 },
203 .{ .symbol = 3, .bits = 5, .baseline = 0 },
204 .{ .symbol = 4, .bits = 5, .baseline = 0 },
205 .{ .symbol = 6, .bits = 5, .baseline = 0 },
206 .{ .symbol = 7, .bits = 5, .baseline = 0 },
207 .{ .symbol = 9, .bits = 5, .baseline = 0 },
208 .{ .symbol = 10, .bits = 5, .baseline = 0 },
209 .{ .symbol = 12, .bits = 5, .baseline = 0 },
210 .{ .symbol = 14, .bits = 6, .baseline = 0 },
211 .{ .symbol = 16, .bits = 5, .baseline = 0 },
212 .{ .symbol = 18, .bits = 5, .baseline = 0 },
213 .{ .symbol = 19, .bits = 5, .baseline = 0 },
214 .{ .symbol = 21, .bits = 5, .baseline = 0 },
215 .{ .symbol = 22, .bits = 5, .baseline = 0 },
216 .{ .symbol = 24, .bits = 5, .baseline = 0 },
217 .{ .symbol = 25, .bits = 5, .baseline = 32 },
218 .{ .symbol = 26, .bits = 5, .baseline = 0 },
219 .{ .symbol = 27, .bits = 6, .baseline = 0 },
220 .{ .symbol = 29, .bits = 6, .baseline = 0 },
221 .{ .symbol = 31, .bits = 6, .baseline = 0 },
222 .{ .symbol = 0, .bits = 4, .baseline = 32 },
223 .{ .symbol = 1, .bits = 4, .baseline = 0 },
224 .{ .symbol = 2, .bits = 5, .baseline = 0 },
225 .{ .symbol = 4, .bits = 5, .baseline = 32 },
226 .{ .symbol = 5, .bits = 5, .baseline = 0 },
227 .{ .symbol = 7, .bits = 5, .baseline = 32 },
228 .{ .symbol = 8, .bits = 5, .baseline = 0 },
229 .{ .symbol = 10, .bits = 5, .baseline = 32 },
230 .{ .symbol = 11, .bits = 5, .baseline = 0 },
231 .{ .symbol = 13, .bits = 6, .baseline = 0 },
232 .{ .symbol = 16, .bits = 5, .baseline = 32 },
233 .{ .symbol = 17, .bits = 5, .baseline = 0 },
234 .{ .symbol = 19, .bits = 5, .baseline = 32 },
235 .{ .symbol = 20, .bits = 5, .baseline = 0 },
236 .{ .symbol = 22, .bits = 5, .baseline = 32 },
237 .{ .symbol = 23, .bits = 5, .baseline = 0 },
238 .{ .symbol = 25, .bits = 4, .baseline = 0 },
239 .{ .symbol = 25, .bits = 4, .baseline = 16 },
240 .{ .symbol = 26, .bits = 5, .baseline = 32 },
241 .{ .symbol = 28, .bits = 6, .baseline = 0 },
242 .{ .symbol = 30, .bits = 6, .baseline = 0 },
243 .{ .symbol = 0, .bits = 4, .baseline = 48 },
244 .{ .symbol = 1, .bits = 4, .baseline = 16 },
245 .{ .symbol = 2, .bits = 5, .baseline = 32 },
246 .{ .symbol = 3, .bits = 5, .baseline = 32 },
247 .{ .symbol = 5, .bits = 5, .baseline = 32 },
248 .{ .symbol = 6, .bits = 5, .baseline = 32 },
249 .{ .symbol = 8, .bits = 5, .baseline = 32 },
250 .{ .symbol = 9, .bits = 5, .baseline = 32 },
251 .{ .symbol = 11, .bits = 5, .baseline = 32 },
252 .{ .symbol = 12, .bits = 5, .baseline = 32 },
253 .{ .symbol = 15, .bits = 6, .baseline = 0 },
254 .{ .symbol = 17, .bits = 5, .baseline = 32 },
255 .{ .symbol = 18, .bits = 5, .baseline = 32 },
256 .{ .symbol = 20, .bits = 5, .baseline = 32 },
257 .{ .symbol = 21, .bits = 5, .baseline = 32 },
258 .{ .symbol = 23, .bits = 5, .baseline = 32 },
259 .{ .symbol = 24, .bits = 5, .baseline = 32 },
260 .{ .symbol = 35, .bits = 6, .baseline = 0 },
261 .{ .symbol = 34, .bits = 6, .baseline = 0 },
262 .{ .symbol = 33, .bits = 6, .baseline = 0 },
263 .{ .symbol = 32, .bits = 6, .baseline = 0 },
264 },
265 };
266
267 pub const predefined_match_fse_table = Table{
268 .fse = &[64]Table.Fse{
269 .{ .symbol = 0, .bits = 6, .baseline = 0 },
270 .{ .symbol = 1, .bits = 4, .baseline = 0 },
271 .{ .symbol = 2, .bits = 5, .baseline = 32 },
272 .{ .symbol = 3, .bits = 5, .baseline = 0 },
273 .{ .symbol = 5, .bits = 5, .baseline = 0 },
274 .{ .symbol = 6, .bits = 5, .baseline = 0 },
275 .{ .symbol = 8, .bits = 5, .baseline = 0 },
276 .{ .symbol = 10, .bits = 6, .baseline = 0 },
277 .{ .symbol = 13, .bits = 6, .baseline = 0 },
278 .{ .symbol = 16, .bits = 6, .baseline = 0 },
279 .{ .symbol = 19, .bits = 6, .baseline = 0 },
280 .{ .symbol = 22, .bits = 6, .baseline = 0 },
281 .{ .symbol = 25, .bits = 6, .baseline = 0 },
282 .{ .symbol = 28, .bits = 6, .baseline = 0 },
283 .{ .symbol = 31, .bits = 6, .baseline = 0 },
284 .{ .symbol = 33, .bits = 6, .baseline = 0 },
285 .{ .symbol = 35, .bits = 6, .baseline = 0 },
286 .{ .symbol = 37, .bits = 6, .baseline = 0 },
287 .{ .symbol = 39, .bits = 6, .baseline = 0 },
288 .{ .symbol = 41, .bits = 6, .baseline = 0 },
289 .{ .symbol = 43, .bits = 6, .baseline = 0 },
290 .{ .symbol = 45, .bits = 6, .baseline = 0 },
291 .{ .symbol = 1, .bits = 4, .baseline = 16 },
292 .{ .symbol = 2, .bits = 4, .baseline = 0 },
293 .{ .symbol = 3, .bits = 5, .baseline = 32 },
294 .{ .symbol = 4, .bits = 5, .baseline = 0 },
295 .{ .symbol = 6, .bits = 5, .baseline = 32 },
296 .{ .symbol = 7, .bits = 5, .baseline = 0 },
297 .{ .symbol = 9, .bits = 6, .baseline = 0 },
298 .{ .symbol = 12, .bits = 6, .baseline = 0 },
299 .{ .symbol = 15, .bits = 6, .baseline = 0 },
300 .{ .symbol = 18, .bits = 6, .baseline = 0 },
301 .{ .symbol = 21, .bits = 6, .baseline = 0 },
302 .{ .symbol = 24, .bits = 6, .baseline = 0 },
303 .{ .symbol = 27, .bits = 6, .baseline = 0 },
304 .{ .symbol = 30, .bits = 6, .baseline = 0 },
305 .{ .symbol = 32, .bits = 6, .baseline = 0 },
306 .{ .symbol = 34, .bits = 6, .baseline = 0 },
307 .{ .symbol = 36, .bits = 6, .baseline = 0 },
308 .{ .symbol = 38, .bits = 6, .baseline = 0 },
309 .{ .symbol = 40, .bits = 6, .baseline = 0 },
310 .{ .symbol = 42, .bits = 6, .baseline = 0 },
311 .{ .symbol = 44, .bits = 6, .baseline = 0 },
312 .{ .symbol = 1, .bits = 4, .baseline = 32 },
313 .{ .symbol = 1, .bits = 4, .baseline = 48 },
314 .{ .symbol = 2, .bits = 4, .baseline = 16 },
315 .{ .symbol = 4, .bits = 5, .baseline = 32 },
316 .{ .symbol = 5, .bits = 5, .baseline = 32 },
317 .{ .symbol = 7, .bits = 5, .baseline = 32 },
318 .{ .symbol = 8, .bits = 5, .baseline = 32 },
319 .{ .symbol = 11, .bits = 6, .baseline = 0 },
320 .{ .symbol = 14, .bits = 6, .baseline = 0 },
321 .{ .symbol = 17, .bits = 6, .baseline = 0 },
322 .{ .symbol = 20, .bits = 6, .baseline = 0 },
323 .{ .symbol = 23, .bits = 6, .baseline = 0 },
324 .{ .symbol = 26, .bits = 6, .baseline = 0 },
325 .{ .symbol = 29, .bits = 6, .baseline = 0 },
326 .{ .symbol = 52, .bits = 6, .baseline = 0 },
327 .{ .symbol = 51, .bits = 6, .baseline = 0 },
328 .{ .symbol = 50, .bits = 6, .baseline = 0 },
329 .{ .symbol = 49, .bits = 6, .baseline = 0 },
330 .{ .symbol = 48, .bits = 6, .baseline = 0 },
331 .{ .symbol = 47, .bits = 6, .baseline = 0 },
332 .{ .symbol = 46, .bits = 6, .baseline = 0 },
333 },
334 };
335
336 pub const predefined_offset_fse_table = Table{
337 .fse = &[32]Table.Fse{
338 .{ .symbol = 0, .bits = 5, .baseline = 0 },
339 .{ .symbol = 6, .bits = 4, .baseline = 0 },
340 .{ .symbol = 9, .bits = 5, .baseline = 0 },
341 .{ .symbol = 15, .bits = 5, .baseline = 0 },
342 .{ .symbol = 21, .bits = 5, .baseline = 0 },
343 .{ .symbol = 3, .bits = 5, .baseline = 0 },
344 .{ .symbol = 7, .bits = 4, .baseline = 0 },
345 .{ .symbol = 12, .bits = 5, .baseline = 0 },
346 .{ .symbol = 18, .bits = 5, .baseline = 0 },
347 .{ .symbol = 23, .bits = 5, .baseline = 0 },
348 .{ .symbol = 5, .bits = 5, .baseline = 0 },
349 .{ .symbol = 8, .bits = 4, .baseline = 0 },
350 .{ .symbol = 14, .bits = 5, .baseline = 0 },
351 .{ .symbol = 20, .bits = 5, .baseline = 0 },
352 .{ .symbol = 2, .bits = 5, .baseline = 0 },
353 .{ .symbol = 7, .bits = 4, .baseline = 16 },
354 .{ .symbol = 11, .bits = 5, .baseline = 0 },
355 .{ .symbol = 17, .bits = 5, .baseline = 0 },
356 .{ .symbol = 22, .bits = 5, .baseline = 0 },
357 .{ .symbol = 4, .bits = 5, .baseline = 0 },
358 .{ .symbol = 8, .bits = 4, .baseline = 16 },
359 .{ .symbol = 13, .bits = 5, .baseline = 0 },
360 .{ .symbol = 19, .bits = 5, .baseline = 0 },
361 .{ .symbol = 1, .bits = 5, .baseline = 0 },
362 .{ .symbol = 6, .bits = 4, .baseline = 16 },
363 .{ .symbol = 10, .bits = 5, .baseline = 0 },
364 .{ .symbol = 16, .bits = 5, .baseline = 0 },
365 .{ .symbol = 28, .bits = 5, .baseline = 0 },
366 .{ .symbol = 27, .bits = 5, .baseline = 0 },
367 .{ .symbol = 26, .bits = 5, .baseline = 0 },
368 .{ .symbol = 25, .bits = 5, .baseline = 0 },
369 .{ .symbol = 24, .bits = 5, .baseline = 0 },
370 },
371 };
372 pub const start_repeated_offset_1 = 1;
373 pub const start_repeated_offset_2 = 4;
374 pub const start_repeated_offset_3 = 8;
375
376 pub const table_accuracy_log_max = struct {
377 pub const literal = 9;
378 pub const match = 9;
379 pub const offset = 8;
380 };
381
382 pub const table_symbol_count_max = struct {
383 pub const literal = 36;
384 pub const match = 53;
385 pub const offset = 32;
386 };
387
388 pub const default_accuracy_log = struct {
389 pub const literal = 6;
390 pub const match = 6;
391 pub const offset = 5;
392 };
393 pub const table_size_max = struct {
394 pub const literal = 1 << table_accuracy_log_max.literal;
395 pub const match = 1 << table_accuracy_log_max.match;
396 pub const offset = 1 << table_accuracy_log_max.offset;
397 };
398};
399
400test {
401 const testing = @import("std").testing;
402 testing.refAllDeclsRecursive(@This());
403}
lib/std/compress/zstd.zig created+152
......@@ -0,0 +1,152 @@
1const std = @import("../std.zig");
2const assert = std.debug.assert;
3
4pub const Decompress = @import("zstd/Decompress.zig");
5
6/// Recommended amount by the standard. Lower than this may result in inability
7/// to decompress common streams.
8pub const default_window_len = 8 * 1024 * 1024;
9pub const block_size_max = 1 << 17;
10
11pub const literals_length_default_distribution = [36]i16{
12 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
13 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
14 -1, -1, -1, -1,
15};
16
17pub const match_lengths_default_distribution = [53]i16{
18 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
19 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
20 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1,
21 -1, -1, -1, -1, -1,
22};
23
24pub const offset_codes_default_distribution = [29]i16{
25 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
26 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
27};
28
29pub const start_repeated_offset_1 = 1;
30pub const start_repeated_offset_2 = 4;
31pub const start_repeated_offset_3 = 8;
32
33pub const literals_length_code_table = [36]struct { u32, u5 }{
34 .{ 0, 0 }, .{ 1, 0 }, .{ 2, 0 }, .{ 3, 0 },
35 .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 },
36 .{ 8, 0 }, .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 },
37 .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 }, .{ 15, 0 },
38 .{ 16, 1 }, .{ 18, 1 }, .{ 20, 1 }, .{ 22, 1 },
39 .{ 24, 2 }, .{ 28, 2 }, .{ 32, 3 }, .{ 40, 3 },
40 .{ 48, 4 }, .{ 64, 6 }, .{ 128, 7 }, .{ 256, 8 },
41 .{ 512, 9 }, .{ 1024, 10 }, .{ 2048, 11 }, .{ 4096, 12 },
42 .{ 8192, 13 }, .{ 16384, 14 }, .{ 32768, 15 }, .{ 65536, 16 },
43};
44
45pub const match_length_code_table = [53]struct { u32, u5 }{
46 .{ 3, 0 }, .{ 4, 0 }, .{ 5, 0 }, .{ 6, 0 }, .{ 7, 0 }, .{ 8, 0 },
47 .{ 9, 0 }, .{ 10, 0 }, .{ 11, 0 }, .{ 12, 0 }, .{ 13, 0 }, .{ 14, 0 },
48 .{ 15, 0 }, .{ 16, 0 }, .{ 17, 0 }, .{ 18, 0 }, .{ 19, 0 }, .{ 20, 0 },
49 .{ 21, 0 }, .{ 22, 0 }, .{ 23, 0 }, .{ 24, 0 }, .{ 25, 0 }, .{ 26, 0 },
50 .{ 27, 0 }, .{ 28, 0 }, .{ 29, 0 }, .{ 30, 0 }, .{ 31, 0 }, .{ 32, 0 },
51 .{ 33, 0 }, .{ 34, 0 }, .{ 35, 1 }, .{ 37, 1 }, .{ 39, 1 }, .{ 41, 1 },
52 .{ 43, 2 }, .{ 47, 2 }, .{ 51, 3 }, .{ 59, 3 }, .{ 67, 4 }, .{ 83, 4 },
53 .{ 99, 5 }, .{ 131, 7 }, .{ 259, 8 }, .{ 515, 9 }, .{ 1027, 10 }, .{ 2051, 11 },
54 .{ 4099, 12 }, .{ 8195, 13 }, .{ 16387, 14 }, .{ 32771, 15 }, .{ 65539, 16 },
55};
56
57pub const table_accuracy_log_max = struct {
58 pub const literal = 9;
59 pub const match = 9;
60 pub const offset = 8;
61};
62
63pub const table_symbol_count_max = struct {
64 pub const literal = 36;
65 pub const match = 53;
66 pub const offset = 32;
67};
68
69pub const default_accuracy_log = struct {
70 pub const literal = 6;
71 pub const match = 6;
72 pub const offset = 5;
73};
74pub const table_size_max = struct {
75 pub const literal = 1 << table_accuracy_log_max.literal;
76 pub const match = 1 << table_accuracy_log_max.match;
77 pub const offset = 1 << table_accuracy_log_max.offset;
78};
79
80fn testDecompress(gpa: std.mem.Allocator, compressed: []const u8) ![]u8 {
81 var out: std.ArrayListUnmanaged(u8) = .empty;
82 defer out.deinit(gpa);
83 try out.ensureUnusedCapacity(gpa, default_window_len);
84
85 var in: std.io.Reader = .fixed(compressed);
86 var zstd_stream: Decompress = .init(&in, &.{}, .{});
87 try zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited);
88
89 return out.toOwnedSlice(gpa);
90}
91
92fn testExpectDecompress(uncompressed: []const u8, compressed: []const u8) !void {
93 const gpa = std.testing.allocator;
94 const result = try testDecompress(gpa, compressed);
95 defer gpa.free(result);
96 try std.testing.expectEqualSlices(u8, uncompressed, result);
97}
98
99fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
100 const gpa = std.testing.allocator;
101
102 var out: std.ArrayListUnmanaged(u8) = .empty;
103 defer out.deinit(gpa);
104 try out.ensureUnusedCapacity(gpa, default_window_len);
105
106 var in: std.io.Reader = .fixed(compressed);
107 var zstd_stream: Decompress = .init(&in, &.{}, .{});
108 try std.testing.expectError(
109 error.ReadFailed,
110 zstd_stream.reader.appendRemaining(gpa, null, &out, .unlimited),
111 );
112 try std.testing.expectError(err, zstd_stream.err orelse {});
113}
114
115test Decompress {
116 const uncompressed = @embedFile("testdata/rfc8478.txt");
117 const compressed3 = @embedFile("testdata/rfc8478.txt.zst.3");
118 const compressed19 = @embedFile("testdata/rfc8478.txt.zst.19");
119
120 try testExpectDecompress(uncompressed, compressed3);
121 try testExpectDecompress(uncompressed, compressed19);
122}
123
124test "zero sized raw block" {
125 const input_raw =
126 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
127 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
128 "\x01\x00\x00"; // block header with: last_block set, block_type raw, block_size zero
129 try testExpectDecompress("", input_raw);
130}
131
132test "zero sized rle block" {
133 const input_rle =
134 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
135 "\x20\x00" ++ // frame header: only single_segment_flag set, frame_content_size zero
136 "\x03\x00\x00" ++ // block header with: last_block set, block_type rle, block_size zero
137 "\xaa"; // block_content
138 try testExpectDecompress("", input_rle);
139}
140
141test "declared raw literals size too large" {
142 const input_raw =
143 "\x28\xb5\x2f\xfd" ++ // zstandard frame magic number
144 "\x00\x00" ++ // frame header: everything unset, window descriptor zero
145 "\x95\x00\x00" ++ // block header with: last_block set, block_type compressed, block_size 18
146 "\xbc\xf3\xae" ++ // literals section header with: type raw, size_format 3, regenerated_size 716603
147 "\xa5\x9f\xe3"; // some bytes of literal content - the content is shorter than regenerated_size
148
149 // Note that the regenerated_size in the above input is larger than block maximum size, so the
150 // block can't be valid as it is a raw literals block.
151 try testExpectDecompressError(error.MalformedLiteralsSection, input_raw);
152}
lib/std/compress/zstd/Decompress.zig created+1818
......@@ -0,0 +1,1818 @@
1const Decompress = @This();
2const std = @import("std");
3const assert = std.debug.assert;
4const Reader = std.io.Reader;
5const Limit = std.io.Limit;
6const zstd = @import("../zstd.zig");
7const Writer = std.io.Writer;
8
9input: *Reader,
10reader: Reader,
11state: State,
12verify_checksum: bool,
13window_len: u32,
14err: ?Error = null,
15
16const State = union(enum) {
17 new_frame,
18 in_frame: InFrame,
19 skipping_frame: usize,
20 end,
21
22 const InFrame = struct {
23 frame: Frame,
24 checksum: ?u32,
25 decompressed_size: usize,
26 decode: Frame.Zstandard.Decode,
27 };
28};
29
30pub const Options = struct {
31 /// Verifying checksums is not implemented yet and will cause a panic if
32 /// you set this to true.
33 verify_checksum: bool = false,
34 /// Affects the minimum capacity of the provided buffer.
35 window_len: u32 = zstd.default_window_len,
36};
37
38pub const Error = error{
39 BadMagic,
40 BlockOversize,
41 ChecksumFailure,
42 ContentOversize,
43 DictionaryIdFlagUnsupported,
44 EndOfStream,
45 HuffmanTreeIncomplete,
46 InvalidBitStream,
47 MalformedAccuracyLog,
48 MalformedBlock,
49 MalformedCompressedBlock,
50 MalformedFrame,
51 MalformedFseBits,
52 MalformedFseTable,
53 MalformedHuffmanTree,
54 MalformedLiteralsHeader,
55 MalformedLiteralsLength,
56 MalformedLiteralsSection,
57 MalformedSequence,
58 MissingStartBit,
59 OutputBufferUndersize,
60 InputBufferUndersize,
61 ReadFailed,
62 RepeatModeFirst,
63 ReservedBitSet,
64 ReservedBlock,
65 SequenceBufferUndersize,
66 TreelessLiteralsFirst,
67 UnexpectedEndOfLiteralStream,
68 WindowOversize,
69 WindowSizeUnknown,
70};
71
72/// If buffer that is written to is not big enough, some streams will fail with
73/// `error.OutputBufferUndersize`. A safe value is `zstd.default_window_len * 2`.
74pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
75 return .{
76 .input = input,
77 .state = .new_frame,
78 .verify_checksum = options.verify_checksum,
79 .window_len = options.window_len,
80 .reader = .{
81 .vtable = &.{ .stream = stream },
82 .buffer = buffer,
83 .seek = 0,
84 .end = 0,
85 },
86 };
87}
88
89fn stream(r: *Reader, w: *Writer, limit: Limit) Reader.StreamError!usize {
90 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
91 const in = d.input;
92
93 switch (d.state) {
94 .new_frame => {
95 // Allow error.EndOfStream only on the frame magic.
96 const magic = try in.takeEnumNonexhaustive(Frame.Magic, .little);
97 initFrame(d, w.buffer.len, magic) catch |err| {
98 d.err = err;
99 return error.ReadFailed;
100 };
101 return readInFrame(d, w, limit, &d.state.in_frame) catch |err| switch (err) {
102 error.ReadFailed => return error.ReadFailed,
103 error.WriteFailed => return error.WriteFailed,
104 else => |e| {
105 d.err = e;
106 return error.ReadFailed;
107 },
108 };
109 },
110 .in_frame => |*in_frame| {
111 return readInFrame(d, w, limit, in_frame) catch |err| switch (err) {
112 error.ReadFailed => return error.ReadFailed,
113 error.WriteFailed => return error.WriteFailed,
114 else => |e| {
115 d.err = e;
116 return error.ReadFailed;
117 },
118 };
119 },
120 .skipping_frame => |*remaining| {
121 const n = in.discard(.limited(remaining.*)) catch |err| {
122 d.err = err;
123 return error.ReadFailed;
124 };
125 remaining.* -= n;
126 if (remaining.* == 0) d.state = .new_frame;
127 return 0;
128 },
129 .end => return error.EndOfStream,
130 }
131}
132
133fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
134 const in = d.input;
135 switch (magic.kind() orelse return error.BadMagic) {
136 .zstandard => {
137 const header = try Frame.Zstandard.Header.decode(in);
138 d.state = .{ .in_frame = .{
139 .frame = try Frame.init(header, window_size_max, d.verify_checksum),
140 .checksum = null,
141 .decompressed_size = 0,
142 .decode = .init,
143 } };
144 },
145 .skippable => {
146 const frame_size = try in.takeInt(u32, .little);
147 d.state = .{ .skipping_frame = frame_size };
148 },
149 }
150}
151
152fn readInFrame(d: *Decompress, w: *Writer, limit: Limit, state: *State.InFrame) !usize {
153 const in = d.input;
154 const window_len = d.window_len;
155
156 const block_header = try in.takeStruct(Frame.Zstandard.Block.Header, .little);
157 const block_size = block_header.size;
158 const frame_block_size_max = state.frame.block_size_max;
159 if (frame_block_size_max < block_size) return error.BlockOversize;
160 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
161 var bytes_written: usize = 0;
162 switch (block_header.type) {
163 .raw => {
164 try in.streamExactPreserve(w, window_len, block_size);
165 bytes_written = block_size;
166 },
167 .rle => {
168 const byte = try in.takeByte();
169 try w.splatBytePreserve(window_len, byte, block_size);
170 bytes_written = block_size;
171 },
172 .compressed => {
173 var literals_buffer: [zstd.block_size_max]u8 = undefined;
174 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
175 var remaining: Limit = .limited(block_size);
176 const literals = try LiteralsSection.decode(in, &remaining, &literals_buffer);
177 const sequences_header = try SequencesSection.Header.decode(in, &remaining);
178
179 const decode = &state.decode;
180 try decode.prepare(in, &remaining, literals, sequences_header);
181
182 {
183 if (sequence_buffer.len < @intFromEnum(remaining))
184 return error.SequenceBufferUndersize;
185 const seq_slice = remaining.slice(&sequence_buffer);
186 try in.readSliceAll(seq_slice);
187 var bit_stream = try ReverseBitReader.init(seq_slice);
188
189 if (sequences_header.sequence_count > 0) {
190 try decode.readInitialFseState(&bit_stream);
191
192 // Ensures the following calls to `decodeSequence` will not flush.
193 if (window_len + frame_block_size_max > w.buffer.len) return error.OutputBufferUndersize;
194 const dest = (try w.writableSliceGreedyPreserve(window_len, frame_block_size_max))[0..frame_block_size_max];
195 const write_pos = dest.ptr - w.buffer.ptr;
196 for (0..sequences_header.sequence_count - 1) |_| {
197 bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
198 try decode.updateState(.literal, &bit_stream);
199 try decode.updateState(.match, &bit_stream);
200 try decode.updateState(.offset, &bit_stream);
201 }
202 bytes_written += try decode.decodeSequence(w.buffer, write_pos + bytes_written, &bit_stream);
203 if (bytes_written > dest.len) return error.MalformedSequence;
204 w.advance(bytes_written);
205 }
206
207 if (!bit_stream.isEmpty()) {
208 return error.MalformedCompressedBlock;
209 }
210 }
211
212 if (decode.literal_written_count < literals.header.regenerated_size) {
213 const len = literals.header.regenerated_size - decode.literal_written_count;
214 try decode.decodeLiterals(w, len);
215 decode.literal_written_count += len;
216 bytes_written += len;
217 }
218
219 switch (decode.literal_header.block_type) {
220 .treeless, .compressed => {
221 if (!decode.isLiteralStreamEmpty()) return error.MalformedCompressedBlock;
222 },
223 .raw, .rle => {},
224 }
225
226 if (bytes_written > frame_block_size_max) return error.BlockOversize;
227 },
228 .reserved => return error.ReservedBlock,
229 }
230
231 if (state.frame.hasher_opt) |*hasher| {
232 if (bytes_written > 0) {
233 _ = hasher;
234 @panic("TODO all those bytes written needed to go through the hasher too");
235 }
236 }
237
238 state.decompressed_size += bytes_written;
239
240 if (block_header.last) {
241 if (state.frame.has_checksum) {
242 const expected_checksum = try in.takeInt(u32, .little);
243 if (state.frame.hasher_opt) |*hasher| {
244 const actual_checksum: u32 = @truncate(hasher.final());
245 if (expected_checksum != actual_checksum) return error.ChecksumFailure;
246 }
247 }
248 if (state.frame.content_size) |content_size| {
249 if (content_size != state.decompressed_size) {
250 return error.MalformedFrame;
251 }
252 }
253 d.state = .new_frame;
254 } else if (state.frame.content_size) |content_size| {
255 if (state.decompressed_size > content_size) return error.MalformedFrame;
256 }
257
258 return bytes_written;
259}
260
261pub const Frame = struct {
262 hasher_opt: ?std.hash.XxHash64,
263 window_size: usize,
264 has_checksum: bool,
265 block_size_max: usize,
266 content_size: ?usize,
267
268 pub const Magic = enum(u32) {
269 zstandard = 0xFD2FB528,
270 _,
271
272 pub fn kind(m: Magic) ?Kind {
273 return switch (@intFromEnum(m)) {
274 @intFromEnum(Magic.zstandard) => .zstandard,
275 @intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => .skippable,
276 else => null,
277 };
278 }
279
280 pub fn isSkippable(m: Magic) bool {
281 return switch (@intFromEnum(m)) {
282 @intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => true,
283 else => false,
284 };
285 }
286 };
287
288 pub const Kind = enum { zstandard, skippable };
289
290 pub const Zstandard = struct {
291 pub const magic: Magic = .zstandard;
292
293 header: Header,
294 data_blocks: []Block,
295 checksum: ?u32,
296
297 pub const Header = struct {
298 descriptor: Descriptor,
299 window_descriptor: ?u8,
300 dictionary_id: ?u32,
301 content_size: ?u64,
302
303 pub const Descriptor = packed struct {
304 dictionary_id_flag: u2,
305 content_checksum_flag: bool,
306 reserved: bool,
307 unused: bool,
308 single_segment_flag: bool,
309 content_size_flag: u2,
310 };
311
312 pub const DecodeError = Reader.Error || error{ReservedBitSet};
313
314 pub fn decode(in: *Reader) DecodeError!Header {
315 const descriptor: Descriptor = @bitCast(try in.takeByte());
316
317 if (descriptor.reserved) return error.ReservedBitSet;
318
319 const window_descriptor: ?u8 = if (descriptor.single_segment_flag) null else try in.takeByte();
320
321 const dictionary_id: ?u32 = if (descriptor.dictionary_id_flag > 0) d: {
322 // if flag is 3 then field_size = 4, else field_size = flag
323 const field_size = (@as(u4, 1) << descriptor.dictionary_id_flag) >> 1;
324 break :d try in.takeVarInt(u32, .little, field_size);
325 } else null;
326
327 const content_size: ?u64 = if (descriptor.single_segment_flag or descriptor.content_size_flag > 0) c: {
328 const field_size = @as(u4, 1) << descriptor.content_size_flag;
329 const content_size = try in.takeVarInt(u64, .little, field_size);
330 break :c if (field_size == 2) content_size + 256 else content_size;
331 } else null;
332
333 return .{
334 .descriptor = descriptor,
335 .window_descriptor = window_descriptor,
336 .dictionary_id = dictionary_id,
337 .content_size = content_size,
338 };
339 }
340
341 /// Returns the window size required to decompress a frame, or `null` if it
342 /// cannot be determined (which indicates a malformed frame header).
343 pub fn windowSize(header: Header) ?u64 {
344 if (header.window_descriptor) |descriptor| {
345 const exponent = (descriptor & 0b11111000) >> 3;
346 const mantissa = descriptor & 0b00000111;
347 const window_log = 10 + exponent;
348 const window_base = @as(u64, 1) << @as(u6, @intCast(window_log));
349 const window_add = (window_base / 8) * mantissa;
350 return window_base + window_add;
351 } else return header.content_size;
352 }
353 };
354
355 pub const Block = struct {
356 pub const Header = packed struct(u24) {
357 last: bool,
358 type: Type,
359 size: u21,
360 };
361
362 pub const Type = enum(u2) {
363 raw,
364 rle,
365 compressed,
366 reserved,
367 };
368 };
369
370 pub const Decode = struct {
371 repeat_offsets: [3]u32,
372
373 offset: StateData(8),
374 match: StateData(9),
375 literal: StateData(9),
376
377 literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse,
378 match_fse_buffer: [zstd.table_size_max.match]Table.Fse,
379 offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse,
380
381 fse_tables_undefined: bool,
382
383 literal_stream_reader: ReverseBitReader,
384 literal_stream_index: usize,
385 literal_streams: LiteralsSection.Streams,
386 literal_header: LiteralsSection.Header,
387 huffman_tree: ?LiteralsSection.HuffmanTree,
388
389 literal_written_count: usize,
390
391 fn StateData(comptime max_accuracy_log: comptime_int) type {
392 return struct {
393 state: @This().State,
394 table: Table,
395 accuracy_log: u8,
396
397 const State = std.meta.Int(.unsigned, max_accuracy_log);
398 };
399 }
400
401 const init: Decode = .{
402 .repeat_offsets = .{
403 zstd.start_repeated_offset_1,
404 zstd.start_repeated_offset_2,
405 zstd.start_repeated_offset_3,
406 },
407
408 .offset = undefined,
409 .match = undefined,
410 .literal = undefined,
411
412 .literal_fse_buffer = undefined,
413 .match_fse_buffer = undefined,
414 .offset_fse_buffer = undefined,
415
416 .fse_tables_undefined = true,
417
418 .literal_written_count = 0,
419 .literal_header = undefined,
420 .literal_streams = undefined,
421 .literal_stream_reader = undefined,
422 .literal_stream_index = undefined,
423 .huffman_tree = null,
424 };
425
426 pub const PrepareError = error{
427 /// the (reversed) literal bitstream's first byte does not have any bits set
428 MissingStartBit,
429 /// `literals` is a treeless literals section and the decode state does not
430 /// have a Huffman tree from a previous block
431 TreelessLiteralsFirst,
432 /// on the first call if one of the sequence FSE tables is set to repeat mode
433 RepeatModeFirst,
434 /// an FSE table has an invalid accuracy
435 MalformedAccuracyLog,
436 /// failed decoding an FSE table
437 MalformedFseTable,
438 /// input stream ends before all FSE tables are read
439 EndOfStream,
440 ReadFailed,
441 InputBufferUndersize,
442 };
443
444 /// Prepare the decoder to decode a compressed block. Loads the
445 /// literals stream and Huffman tree from `literals` and reads the
446 /// FSE tables from `in`.
447 pub fn prepare(
448 self: *Decode,
449 in: *Reader,
450 remaining: *Limit,
451 literals: LiteralsSection,
452 sequences_header: SequencesSection.Header,
453 ) PrepareError!void {
454 self.literal_written_count = 0;
455 self.literal_header = literals.header;
456 self.literal_streams = literals.streams;
457
458 if (literals.huffman_tree) |tree| {
459 self.huffman_tree = tree;
460 } else if (literals.header.block_type == .treeless and self.huffman_tree == null) {
461 return error.TreelessLiteralsFirst;
462 }
463
464 switch (literals.header.block_type) {
465 .raw, .rle => {},
466 .compressed, .treeless => {
467 self.literal_stream_index = 0;
468 switch (literals.streams) {
469 .one => |slice| try self.initLiteralStream(slice),
470 .four => |streams| try self.initLiteralStream(streams[0]),
471 }
472 },
473 }
474
475 if (sequences_header.sequence_count > 0) {
476 try self.updateFseTable(in, remaining, .literal, sequences_header.literal_lengths);
477 try self.updateFseTable(in, remaining, .offset, sequences_header.offsets);
478 try self.updateFseTable(in, remaining, .match, sequences_header.match_lengths);
479 self.fse_tables_undefined = false;
480 }
481 }
482
483 /// Read initial FSE states for sequence decoding.
484 pub fn readInitialFseState(self: *Decode, bit_reader: *ReverseBitReader) error{EndOfStream}!void {
485 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
486 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
487 self.match.state = try bit_reader.readBitsNoEof(u9, self.match.accuracy_log);
488 }
489
490 fn updateRepeatOffset(self: *Decode, offset: u32) void {
491 self.repeat_offsets[2] = self.repeat_offsets[1];
492 self.repeat_offsets[1] = self.repeat_offsets[0];
493 self.repeat_offsets[0] = offset;
494 }
495
496 fn useRepeatOffset(self: *Decode, index: usize) u32 {
497 if (index == 1)
498 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[1])
499 else if (index == 2) {
500 std.mem.swap(u32, &self.repeat_offsets[0], &self.repeat_offsets[2]);
501 std.mem.swap(u32, &self.repeat_offsets[1], &self.repeat_offsets[2]);
502 }
503 return self.repeat_offsets[0];
504 }
505
506 const WhichFse = enum { offset, match, literal };
507
508 /// TODO: don't use `@field`
509 fn updateState(
510 self: *Decode,
511 comptime choice: WhichFse,
512 bit_reader: *ReverseBitReader,
513 ) error{ MalformedFseBits, EndOfStream }!void {
514 switch (@field(self, @tagName(choice)).table) {
515 .rle => {},
516 .fse => |table| {
517 const data = table[@field(self, @tagName(choice)).state];
518 const T = @TypeOf(@field(self, @tagName(choice))).State;
519 const bits_summand = try bit_reader.readBitsNoEof(T, data.bits);
520 const next_state = std.math.cast(
521 @TypeOf(@field(self, @tagName(choice))).State,
522 data.baseline + bits_summand,
523 ) orelse return error.MalformedFseBits;
524 @field(self, @tagName(choice)).state = next_state;
525 },
526 }
527 }
528
529 const FseTableError = error{
530 MalformedFseTable,
531 MalformedAccuracyLog,
532 RepeatModeFirst,
533 EndOfStream,
534 };
535
536 /// TODO: don't use `@field`
537 fn updateFseTable(
538 self: *Decode,
539 in: *Reader,
540 remaining: *Limit,
541 comptime choice: WhichFse,
542 mode: SequencesSection.Header.Mode,
543 ) !void {
544 const field_name = @tagName(choice);
545 switch (mode) {
546 .predefined => {
547 @field(self, field_name).accuracy_log =
548 @field(zstd.default_accuracy_log, field_name);
549
550 @field(self, field_name).table =
551 @field(Table, "predefined_" ++ field_name);
552 },
553 .rle => {
554 @field(self, field_name).accuracy_log = 0;
555 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
556 @field(self, field_name).table = .{ .rle = try in.takeByte() };
557 },
558 .fse => {
559 const max_table_size = 2048;
560 const peek_len: usize = remaining.minInt(max_table_size);
561 if (in.buffer.len < peek_len) return error.InputBufferUndersize;
562 const limited_buffer = try in.peek(peek_len);
563 var bit_reader: BitReader = .{ .bytes = limited_buffer };
564 const table_size = try Table.decode(
565 &bit_reader,
566 @field(zstd.table_symbol_count_max, field_name),
567 @field(zstd.table_accuracy_log_max, field_name),
568 &@field(self, field_name ++ "_fse_buffer"),
569 );
570 @field(self, field_name).table = .{
571 .fse = (&@field(self, field_name ++ "_fse_buffer"))[0..table_size],
572 };
573 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
574 in.toss(bit_reader.index);
575 remaining.* = remaining.subtract(bit_reader.index).?;
576 },
577 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
578 }
579 }
580
581 const Sequence = struct {
582 literal_length: u32,
583 match_length: u32,
584 offset: u32,
585 };
586
587 fn nextSequence(
588 self: *Decode,
589 bit_reader: *ReverseBitReader,
590 ) error{ InvalidBitStream, EndOfStream }!Sequence {
591 const raw_code = self.getCode(.offset);
592 const offset_code = std.math.cast(u5, raw_code) orelse {
593 return error.InvalidBitStream;
594 };
595 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
596
597 const match_code = self.getCode(.match);
598 if (match_code >= zstd.match_length_code_table.len)
599 return error.InvalidBitStream;
600 const match = zstd.match_length_code_table[match_code];
601 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
602
603 const literal_code = self.getCode(.literal);
604 if (literal_code >= zstd.literals_length_code_table.len)
605 return error.InvalidBitStream;
606 const literal = zstd.literals_length_code_table[literal_code];
607 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
608
609 const offset = if (offset_value > 3) offset: {
610 const offset = offset_value - 3;
611 self.updateRepeatOffset(offset);
612 break :offset offset;
613 } else offset: {
614 if (literal_length == 0) {
615 if (offset_value == 3) {
616 const offset = self.repeat_offsets[0] - 1;
617 self.updateRepeatOffset(offset);
618 break :offset offset;
619 }
620 break :offset self.useRepeatOffset(offset_value);
621 }
622 break :offset self.useRepeatOffset(offset_value - 1);
623 };
624
625 if (offset == 0) return error.InvalidBitStream;
626
627 return .{
628 .literal_length = literal_length,
629 .match_length = match_length,
630 .offset = offset,
631 };
632 }
633
634 /// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
635 /// if `last_sequence` is `false`. Assumes `prepare` called for the block
636 /// before attempting to decode sequences.
637 fn decodeSequence(
638 decode: *Decode,
639 dest: []u8,
640 write_pos: usize,
641 bit_reader: *ReverseBitReader,
642 ) !usize {
643 const sequence = try decode.nextSequence(bit_reader);
644 const literal_length: usize = sequence.literal_length;
645 const match_length: usize = sequence.match_length;
646 const sequence_length = literal_length + match_length;
647
648 const copy_start = std.math.sub(usize, write_pos + sequence.literal_length, sequence.offset) catch
649 return error.MalformedSequence;
650
651 if (decode.literal_written_count + literal_length > decode.literal_header.regenerated_size)
652 return error.MalformedLiteralsLength;
653 var sub_bw: Writer = .fixed(dest[write_pos..]);
654 try decodeLiterals(decode, &sub_bw, literal_length);
655 decode.literal_written_count += literal_length;
656 // This is not a @memmove; it intentionally repeats patterns
657 // caused by iterating one byte at a time.
658 for (
659 dest[write_pos + literal_length ..][0..match_length],
660 dest[copy_start..][0..match_length],
661 ) |*d, s| d.* = s;
662 return sequence_length;
663 }
664
665 fn nextLiteralMultiStream(self: *Decode) error{MissingStartBit}!void {
666 self.literal_stream_index += 1;
667 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
668 }
669
670 fn initLiteralStream(self: *Decode, bytes: []const u8) error{MissingStartBit}!void {
671 self.literal_stream_reader = try ReverseBitReader.init(bytes);
672 }
673
674 fn isLiteralStreamEmpty(self: *Decode) bool {
675 switch (self.literal_streams) {
676 .one => return self.literal_stream_reader.isEmpty(),
677 .four => return self.literal_stream_index == 3 and self.literal_stream_reader.isEmpty(),
678 }
679 }
680
681 const LiteralBitsError = error{
682 MissingStartBit,
683 UnexpectedEndOfLiteralStream,
684 };
685 fn readLiteralsBits(
686 self: *Decode,
687 bit_count_to_read: u16,
688 ) LiteralBitsError!u16 {
689 return self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch bits: {
690 if (self.literal_streams == .four and self.literal_stream_index < 3) {
691 try self.nextLiteralMultiStream();
692 break :bits self.literal_stream_reader.readBitsNoEof(u16, bit_count_to_read) catch
693 return error.UnexpectedEndOfLiteralStream;
694 } else {
695 return error.UnexpectedEndOfLiteralStream;
696 }
697 };
698 }
699
700 /// Decode `len` bytes of literals into `w`.
701 fn decodeLiterals(d: *Decode, w: *Writer, len: usize) !void {
702 switch (d.literal_header.block_type) {
703 .raw => {
704 try w.writeAll(d.literal_streams.one[d.literal_written_count..][0..len]);
705 },
706 .rle => {
707 try w.splatByteAll(d.literal_streams.one[0], len);
708 },
709 .compressed, .treeless => {
710 if (len > w.buffer.len) return error.OutputBufferUndersize;
711 const buf = try w.writableSlice(len);
712 const huffman_tree = d.huffman_tree.?;
713 const max_bit_count = huffman_tree.max_bit_count;
714 const starting_bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
715 huffman_tree.nodes[huffman_tree.symbol_count_minus_one].weight,
716 max_bit_count,
717 );
718 var bits_read: u4 = 0;
719 var huffman_tree_index: usize = huffman_tree.symbol_count_minus_one;
720 var bit_count_to_read: u4 = starting_bit_count;
721 for (buf) |*out| {
722 var prefix: u16 = 0;
723 while (true) {
724 const new_bits = try d.readLiteralsBits(bit_count_to_read);
725 prefix <<= bit_count_to_read;
726 prefix |= new_bits;
727 bits_read += bit_count_to_read;
728 const result = try huffman_tree.query(huffman_tree_index, prefix);
729
730 switch (result) {
731 .symbol => |sym| {
732 out.* = sym;
733 bit_count_to_read = starting_bit_count;
734 bits_read = 0;
735 huffman_tree_index = huffman_tree.symbol_count_minus_one;
736 break;
737 },
738 .index => |index| {
739 huffman_tree_index = index;
740 const bit_count = LiteralsSection.HuffmanTree.weightToBitCount(
741 huffman_tree.nodes[index].weight,
742 max_bit_count,
743 );
744 bit_count_to_read = bit_count - bits_read;
745 },
746 }
747 }
748 }
749 },
750 }
751 }
752
753 /// TODO: don't use `@field`
754 fn getCode(self: *Decode, comptime choice: WhichFse) u32 {
755 return switch (@field(self, @tagName(choice)).table) {
756 .rle => |value| value,
757 .fse => |table| table[@field(self, @tagName(choice)).state].symbol,
758 };
759 }
760 };
761 };
762
763 pub const Skippable = struct {
764 pub const magic_min: Magic = @enumFromInt(0x184D2A50);
765 pub const magic_max: Magic = @enumFromInt(0x184D2A5F);
766
767 pub const Header = struct {
768 magic_number: u32,
769 frame_size: u32,
770 };
771 };
772
773 const InitError = error{
774 /// Frame uses a dictionary.
775 DictionaryIdFlagUnsupported,
776 /// Frame does not have a valid window size.
777 WindowSizeUnknown,
778 /// Window size exceeds `window_size_max` or max `usize` value.
779 WindowOversize,
780 /// Frame header indicates a content size exceeding max `usize` value.
781 ContentOversize,
782 };
783
784 /// Validates `frame_header` and returns the associated `Frame`.
785 pub fn init(
786 frame_header: Frame.Zstandard.Header,
787 window_size_max: usize,
788 verify_checksum: bool,
789 ) InitError!Frame {
790 if (frame_header.descriptor.dictionary_id_flag != 0)
791 return error.DictionaryIdFlagUnsupported;
792
793 const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
794 const window_size = if (window_size_raw > window_size_max)
795 return error.WindowOversize
796 else
797 std.math.cast(usize, window_size_raw) orelse return error.WindowOversize;
798
799 const should_compute_checksum =
800 frame_header.descriptor.content_checksum_flag and verify_checksum;
801
802 const content_size = if (frame_header.content_size) |size|
803 std.math.cast(usize, size) orelse return error.ContentOversize
804 else
805 null;
806
807 return .{
808 .hasher_opt = if (should_compute_checksum) std.hash.XxHash64.init(0) else null,
809 .window_size = window_size,
810 .has_checksum = frame_header.descriptor.content_checksum_flag,
811 .block_size_max = @min(zstd.block_size_max, window_size),
812 .content_size = content_size,
813 };
814 }
815};
816
817pub const LiteralsSection = struct {
818 header: Header,
819 huffman_tree: ?HuffmanTree,
820 streams: Streams,
821
822 pub const Streams = union(enum) {
823 one: []const u8,
824 four: [4][]const u8,
825
826 fn decode(size_format: u2, stream_data: []const u8) !Streams {
827 if (size_format == 0) {
828 return .{ .one = stream_data };
829 }
830
831 if (stream_data.len < 6) return error.MalformedLiteralsSection;
832
833 const stream_1_length: usize = std.mem.readInt(u16, stream_data[0..2], .little);
834 const stream_2_length: usize = std.mem.readInt(u16, stream_data[2..4], .little);
835 const stream_3_length: usize = std.mem.readInt(u16, stream_data[4..6], .little);
836
837 const stream_1_start = 6;
838 const stream_2_start = stream_1_start + stream_1_length;
839 const stream_3_start = stream_2_start + stream_2_length;
840 const stream_4_start = stream_3_start + stream_3_length;
841
842 if (stream_data.len < stream_4_start) return error.MalformedLiteralsSection;
843
844 return .{ .four = .{
845 stream_data[stream_1_start .. stream_1_start + stream_1_length],
846 stream_data[stream_2_start .. stream_2_start + stream_2_length],
847 stream_data[stream_3_start .. stream_3_start + stream_3_length],
848 stream_data[stream_4_start..],
849 } };
850 }
851 };
852
853 pub const Header = struct {
854 block_type: BlockType,
855 size_format: u2,
856 regenerated_size: u20,
857 compressed_size: ?u18,
858
859 /// Decode a literals section header.
860 pub fn decode(in: *Reader, remaining: *Limit) !Header {
861 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
862 const byte0 = try in.takeByte();
863 const block_type: BlockType = @enumFromInt(byte0 & 0b11);
864 const size_format: u2 = @intCast((byte0 & 0b1100) >> 2);
865 var regenerated_size: u20 = undefined;
866 var compressed_size: ?u18 = null;
867 switch (block_type) {
868 .raw, .rle => {
869 switch (size_format) {
870 0, 2 => {
871 regenerated_size = byte0 >> 3;
872 },
873 1 => {
874 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
875 regenerated_size = (byte0 >> 4) + (@as(u20, try in.takeByte()) << 4);
876 },
877 3 => {
878 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
879 regenerated_size = (byte0 >> 4) +
880 (@as(u20, try in.takeByte()) << 4) +
881 (@as(u20, try in.takeByte()) << 12);
882 },
883 }
884 },
885 .compressed, .treeless => {
886 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
887 const byte1 = try in.takeByte();
888 const byte2 = try in.takeByte();
889 switch (size_format) {
890 0, 1 => {
891 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
892 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
893 },
894 2 => {
895 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
896 const byte3 = try in.takeByte();
897 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
898 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
899 },
900 3 => {
901 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
902 const byte3 = try in.takeByte();
903 const byte4 = try in.takeByte();
904 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
905 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
906 },
907 }
908 },
909 }
910 return .{
911 .block_type = block_type,
912 .size_format = size_format,
913 .regenerated_size = regenerated_size,
914 .compressed_size = compressed_size,
915 };
916 }
917 };
918
919 pub const BlockType = enum(u2) {
920 raw,
921 rle,
922 compressed,
923 treeless,
924 };
925
926 pub const HuffmanTree = struct {
927 max_bit_count: u4,
928 symbol_count_minus_one: u8,
929 nodes: [256]PrefixedSymbol,
930
931 pub const PrefixedSymbol = struct {
932 symbol: u8,
933 prefix: u16,
934 weight: u4,
935 };
936
937 pub const Result = union(enum) {
938 symbol: u8,
939 index: usize,
940 };
941
942 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{HuffmanTreeIncomplete}!Result {
943 var node = self.nodes[index];
944 const weight = node.weight;
945 var i: usize = index;
946 while (node.weight == weight) {
947 if (node.prefix == prefix) return .{ .symbol = node.symbol };
948 if (i == 0) return error.HuffmanTreeIncomplete;
949 i -= 1;
950 node = self.nodes[i];
951 }
952 return .{ .index = i };
953 }
954
955 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
956 return if (weight == 0) 0 else ((max_bit_count + 1) - weight);
957 }
958
959 pub const DecodeError = Reader.Error || error{
960 MalformedHuffmanTree,
961 MalformedFseTable,
962 MalformedAccuracyLog,
963 EndOfStream,
964 MissingStartBit,
965 };
966
967 pub fn decode(in: *Reader, remaining: *Limit) HuffmanTree.DecodeError!HuffmanTree {
968 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
969 const header = try in.takeByte();
970 if (header < 128) {
971 return decodeFse(in, remaining, header);
972 } else {
973 return decodeDirect(in, remaining, header - 127);
974 }
975 }
976
977 fn decodeDirect(
978 in: *Reader,
979 remaining: *Limit,
980 encoded_symbol_count: usize,
981 ) HuffmanTree.DecodeError!HuffmanTree {
982 var weights: [256]u4 = undefined;
983 const weights_byte_count = (encoded_symbol_count + 1) / 2;
984 remaining.* = remaining.subtract(weights_byte_count) orelse return error.EndOfStream;
985 for (0..weights_byte_count) |i| {
986 const byte = try in.takeByte();
987 weights[2 * i] = @as(u4, @intCast(byte >> 4));
988 weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
989 }
990 const symbol_count = encoded_symbol_count + 1;
991 return build(&weights, symbol_count);
992 }
993
994 fn decodeFse(
995 in: *Reader,
996 remaining: *Limit,
997 compressed_size: usize,
998 ) HuffmanTree.DecodeError!HuffmanTree {
999 var weights: [256]u4 = undefined;
1000 remaining.* = remaining.subtract(compressed_size) orelse return error.EndOfStream;
1001 const compressed_buffer = try in.take(compressed_size);
1002 var bit_reader: BitReader = .{ .bytes = compressed_buffer };
1003 var entries: [1 << 6]Table.Fse = undefined;
1004 const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
1005 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
1006 const remaining_buffer = bit_reader.bytes[bit_reader.index..];
1007 const symbol_count = try assignWeights(remaining_buffer, accuracy_log, &entries, &weights);
1008 return build(&weights, symbol_count);
1009 }
1010
1011 fn assignWeights(
1012 huff_bits_buffer: []const u8,
1013 accuracy_log: u16,
1014 entries: *[1 << 6]Table.Fse,
1015 weights: *[256]u4,
1016 ) !usize {
1017 var huff_bits = try ReverseBitReader.init(huff_bits_buffer);
1018
1019 var i: usize = 0;
1020 var even_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1021 var odd_state: u32 = try huff_bits.readBitsNoEof(u32, accuracy_log);
1022
1023 while (i < 254) {
1024 const even_data = entries[even_state];
1025 var read_bits: u16 = 0;
1026 const even_bits = huff_bits.readBits(u32, even_data.bits, &read_bits) catch unreachable;
1027 weights[i] = std.math.cast(u4, even_data.symbol) orelse return error.MalformedHuffmanTree;
1028 i += 1;
1029 if (read_bits < even_data.bits) {
1030 weights[i] = std.math.cast(u4, entries[odd_state].symbol) orelse return error.MalformedHuffmanTree;
1031 i += 1;
1032 break;
1033 }
1034 even_state = even_data.baseline + even_bits;
1035
1036 read_bits = 0;
1037 const odd_data = entries[odd_state];
1038 const odd_bits = huff_bits.readBits(u32, odd_data.bits, &read_bits) catch unreachable;
1039 weights[i] = std.math.cast(u4, odd_data.symbol) orelse return error.MalformedHuffmanTree;
1040 i += 1;
1041 if (read_bits < odd_data.bits) {
1042 if (i == 255) return error.MalformedHuffmanTree;
1043 weights[i] = std.math.cast(u4, entries[even_state].symbol) orelse return error.MalformedHuffmanTree;
1044 i += 1;
1045 break;
1046 }
1047 odd_state = odd_data.baseline + odd_bits;
1048 } else return error.MalformedHuffmanTree;
1049
1050 if (!huff_bits.isEmpty()) {
1051 return error.MalformedHuffmanTree;
1052 }
1053
1054 return i + 1; // stream contains all but the last symbol
1055 }
1056
1057 fn assignSymbols(weight_sorted_prefixed_symbols: []PrefixedSymbol, weights: [256]u4) usize {
1058 for (0..weight_sorted_prefixed_symbols.len) |i| {
1059 weight_sorted_prefixed_symbols[i] = .{
1060 .symbol = @as(u8, @intCast(i)),
1061 .weight = undefined,
1062 .prefix = undefined,
1063 };
1064 }
1065
1066 std.mem.sort(
1067 PrefixedSymbol,
1068 weight_sorted_prefixed_symbols,
1069 weights,
1070 lessThanByWeight,
1071 );
1072
1073 var prefix: u16 = 0;
1074 var prefixed_symbol_count: usize = 0;
1075 var sorted_index: usize = 0;
1076 const symbol_count = weight_sorted_prefixed_symbols.len;
1077 while (sorted_index < symbol_count) {
1078 var symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1079 const weight = weights[symbol];
1080 if (weight == 0) {
1081 sorted_index += 1;
1082 continue;
1083 }
1084
1085 while (sorted_index < symbol_count) : ({
1086 sorted_index += 1;
1087 prefixed_symbol_count += 1;
1088 prefix += 1;
1089 }) {
1090 symbol = weight_sorted_prefixed_symbols[sorted_index].symbol;
1091 if (weights[symbol] != weight) {
1092 prefix = ((prefix - 1) >> (weights[symbol] - weight)) + 1;
1093 break;
1094 }
1095 weight_sorted_prefixed_symbols[prefixed_symbol_count].symbol = symbol;
1096 weight_sorted_prefixed_symbols[prefixed_symbol_count].prefix = prefix;
1097 weight_sorted_prefixed_symbols[prefixed_symbol_count].weight = weight;
1098 }
1099 }
1100 return prefixed_symbol_count;
1101 }
1102
1103 fn build(weights: *[256]u4, symbol_count: usize) error{MalformedHuffmanTree}!HuffmanTree {
1104 var weight_power_sum_big: u32 = 0;
1105 for (weights[0 .. symbol_count - 1]) |value| {
1106 weight_power_sum_big += (@as(u16, 1) << value) >> 1;
1107 }
1108 if (weight_power_sum_big >= 1 << 11) return error.MalformedHuffmanTree;
1109 const weight_power_sum = @as(u16, @intCast(weight_power_sum_big));
1110
1111 // advance to next power of two (even if weight_power_sum is a power of 2)
1112 // TODO: is it valid to have weight_power_sum == 0?
1113 const max_number_of_bits = if (weight_power_sum == 0) 1 else std.math.log2_int(u16, weight_power_sum) + 1;
1114 const next_power_of_two = @as(u16, 1) << max_number_of_bits;
1115 weights[symbol_count - 1] = std.math.log2_int(u16, next_power_of_two - weight_power_sum) + 1;
1116
1117 var weight_sorted_prefixed_symbols: [256]PrefixedSymbol = undefined;
1118 const prefixed_symbol_count = assignSymbols(weight_sorted_prefixed_symbols[0..symbol_count], weights.*);
1119 const tree: HuffmanTree = .{
1120 .max_bit_count = max_number_of_bits,
1121 .symbol_count_minus_one = @as(u8, @intCast(prefixed_symbol_count - 1)),
1122 .nodes = weight_sorted_prefixed_symbols,
1123 };
1124 return tree;
1125 }
1126
1127 fn lessThanByWeight(
1128 weights: [256]u4,
1129 lhs: PrefixedSymbol,
1130 rhs: PrefixedSymbol,
1131 ) bool {
1132 // NOTE: this function relies on the use of a stable sorting algorithm,
1133 // otherwise a special case of if (weights[lhs] == weights[rhs]) return lhs < rhs;
1134 // should be added
1135 return weights[lhs.symbol] < weights[rhs.symbol];
1136 }
1137 };
1138
1139 pub const StreamCount = enum { one, four };
1140 pub fn streamCount(size_format: u2, block_type: BlockType) StreamCount {
1141 return switch (block_type) {
1142 .raw, .rle => .one,
1143 .compressed, .treeless => if (size_format == 0) .one else .four,
1144 };
1145 }
1146
1147 pub const DecodeError = error{
1148 /// Invalid header.
1149 MalformedLiteralsHeader,
1150 /// Decoding errors.
1151 MalformedLiteralsSection,
1152 /// Compressed literals have invalid accuracy.
1153 MalformedAccuracyLog,
1154 /// Compressed literals have invalid FSE table.
1155 MalformedFseTable,
1156 /// Failed decoding a Huffamn tree.
1157 MalformedHuffmanTree,
1158 /// Not enough bytes to complete the section.
1159 EndOfStream,
1160 ReadFailed,
1161 MissingStartBit,
1162 };
1163
1164 pub fn decode(in: *Reader, remaining: *Limit, buffer: []u8) DecodeError!LiteralsSection {
1165 const header = try Header.decode(in, remaining);
1166 switch (header.block_type) {
1167 .raw => {
1168 if (buffer.len < header.regenerated_size) return error.MalformedLiteralsSection;
1169 remaining.* = remaining.subtract(header.regenerated_size) orelse return error.EndOfStream;
1170 try in.readSliceAll(buffer[0..header.regenerated_size]);
1171 return .{
1172 .header = header,
1173 .huffman_tree = null,
1174 .streams = .{ .one = buffer },
1175 };
1176 },
1177 .rle => {
1178 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1179 buffer[0] = try in.takeByte();
1180 return .{
1181 .header = header,
1182 .huffman_tree = null,
1183 .streams = .{ .one = buffer[0..1] },
1184 };
1185 },
1186 .compressed, .treeless => {
1187 const before_remaining = remaining.*;
1188 const huffman_tree = if (header.block_type == .compressed)
1189 try HuffmanTree.decode(in, remaining)
1190 else
1191 null;
1192 const huffman_tree_size = @intFromEnum(before_remaining) - @intFromEnum(remaining.*);
1193 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
1194 return error.MalformedLiteralsSection;
1195 if (total_streams_size > buffer.len) return error.MalformedLiteralsSection;
1196 remaining.* = remaining.subtract(total_streams_size) orelse return error.EndOfStream;
1197 try in.readSliceAll(buffer[0..total_streams_size]);
1198 const stream_data = buffer[0..total_streams_size];
1199 const streams = try Streams.decode(header.size_format, stream_data);
1200 return .{
1201 .header = header,
1202 .huffman_tree = huffman_tree,
1203 .streams = streams,
1204 };
1205 },
1206 }
1207 }
1208};
1209
1210pub const SequencesSection = struct {
1211 header: Header,
1212 literals_length_table: Table,
1213 offset_table: Table,
1214 match_length_table: Table,
1215
1216 pub const Header = struct {
1217 sequence_count: u24,
1218 match_lengths: Mode,
1219 offsets: Mode,
1220 literal_lengths: Mode,
1221
1222 pub const Mode = enum(u2) {
1223 predefined,
1224 rle,
1225 fse,
1226 repeat,
1227 };
1228
1229 pub const DecodeError = error{
1230 ReservedBitSet,
1231 EndOfStream,
1232 ReadFailed,
1233 };
1234
1235 pub fn decode(in: *Reader, remaining: *Limit) DecodeError!Header {
1236 var sequence_count: u24 = undefined;
1237
1238 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1239 const byte0 = try in.takeByte();
1240 if (byte0 == 0) {
1241 return .{
1242 .sequence_count = 0,
1243 .offsets = undefined,
1244 .match_lengths = undefined,
1245 .literal_lengths = undefined,
1246 };
1247 } else if (byte0 < 128) {
1248 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1249 sequence_count = byte0;
1250 } else if (byte0 < 255) {
1251 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
1252 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try in.takeByte();
1253 } else {
1254 remaining.* = remaining.subtract(3) orelse return error.EndOfStream;
1255 sequence_count = (try in.takeByte()) + (@as(u24, try in.takeByte()) << 8) + 0x7F00;
1256 }
1257
1258 const compression_modes = try in.takeByte();
1259
1260 const matches_mode: Header.Mode = @enumFromInt((compression_modes & 0b00001100) >> 2);
1261 const offsets_mode: Header.Mode = @enumFromInt((compression_modes & 0b00110000) >> 4);
1262 const literal_mode: Header.Mode = @enumFromInt((compression_modes & 0b11000000) >> 6);
1263 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
1264
1265 return .{
1266 .sequence_count = sequence_count,
1267 .offsets = offsets_mode,
1268 .match_lengths = matches_mode,
1269 .literal_lengths = literal_mode,
1270 };
1271 }
1272 };
1273};
1274
1275pub const Table = union(enum) {
1276 fse: []const Fse,
1277 rle: u8,
1278
1279 pub const Fse = struct {
1280 symbol: u8,
1281 baseline: u16,
1282 bits: u8,
1283 };
1284
1285 pub fn decode(
1286 bit_reader: *BitReader,
1287 expected_symbol_count: usize,
1288 max_accuracy_log: u4,
1289 entries: []Table.Fse,
1290 ) !usize {
1291 const accuracy_log_biased = try bit_reader.readBitsNoEof(u4, 4);
1292 if (accuracy_log_biased > max_accuracy_log -| 5) return error.MalformedAccuracyLog;
1293 const accuracy_log = accuracy_log_biased + 5;
1294
1295 var values: [256]u16 = undefined;
1296 var value_count: usize = 0;
1297
1298 const total_probability = @as(u16, 1) << accuracy_log;
1299 var accumulated_probability: u16 = 0;
1300
1301 while (accumulated_probability < total_probability) {
1302 // WARNING: The RFC is poorly worded, and would suggest std.math.log2_int_ceil is correct here,
1303 // but power of two (remaining probabilities + 1) need max bits set to 1 more.
1304 const max_bits = std.math.log2_int(u16, total_probability - accumulated_probability + 1) + 1;
1305 const small = try bit_reader.readBitsNoEof(u16, max_bits - 1);
1306
1307 const cutoff = (@as(u16, 1) << max_bits) - 1 - (total_probability - accumulated_probability + 1);
1308
1309 const value = if (small < cutoff)
1310 small
1311 else value: {
1312 const value_read = small + (try bit_reader.readBitsNoEof(u16, 1) << (max_bits - 1));
1313 break :value if (value_read < @as(u16, 1) << (max_bits - 1))
1314 value_read
1315 else
1316 value_read - cutoff;
1317 };
1318
1319 accumulated_probability += if (value != 0) value - 1 else 1;
1320
1321 values[value_count] = value;
1322 value_count += 1;
1323
1324 if (value == 1) {
1325 while (true) {
1326 const repeat_flag = try bit_reader.readBitsNoEof(u2, 2);
1327 if (repeat_flag + value_count > 256) return error.MalformedFseTable;
1328 for (0..repeat_flag) |_| {
1329 values[value_count] = 1;
1330 value_count += 1;
1331 }
1332 if (repeat_flag < 3) break;
1333 }
1334 }
1335 if (value_count == 256) break;
1336 }
1337 bit_reader.alignToByte();
1338
1339 if (value_count < 2) return error.MalformedFseTable;
1340 if (accumulated_probability != total_probability) return error.MalformedFseTable;
1341 if (value_count > expected_symbol_count) return error.MalformedFseTable;
1342
1343 const table_size = total_probability;
1344
1345 try build(values[0..value_count], entries[0..table_size]);
1346 return table_size;
1347 }
1348
1349 pub fn build(values: []const u16, entries: []Table.Fse) !void {
1350 const total_probability = @as(u16, @intCast(entries.len));
1351 const accuracy_log = std.math.log2_int(u16, total_probability);
1352 assert(total_probability <= 1 << 9);
1353
1354 var less_than_one_count: usize = 0;
1355 for (values, 0..) |value, i| {
1356 if (value == 0) {
1357 entries[entries.len - 1 - less_than_one_count] = Table.Fse{
1358 .symbol = @as(u8, @intCast(i)),
1359 .baseline = 0,
1360 .bits = accuracy_log,
1361 };
1362 less_than_one_count += 1;
1363 }
1364 }
1365
1366 var position: usize = 0;
1367 var temp_states: [1 << 9]u16 = undefined;
1368 for (values, 0..) |value, symbol| {
1369 if (value == 0 or value == 1) continue;
1370 const probability = value - 1;
1371
1372 const state_share_dividend = std.math.ceilPowerOfTwo(u16, probability) catch
1373 return error.MalformedFseTable;
1374 const share_size = @divExact(total_probability, state_share_dividend);
1375 const double_state_count = state_share_dividend - probability;
1376 const single_state_count = probability - double_state_count;
1377 const share_size_log = std.math.log2_int(u16, share_size);
1378
1379 for (0..probability) |i| {
1380 temp_states[i] = @as(u16, @intCast(position));
1381 position += (entries.len >> 1) + (entries.len >> 3) + 3;
1382 position &= entries.len - 1;
1383 while (position >= entries.len - less_than_one_count) {
1384 position += (entries.len >> 1) + (entries.len >> 3) + 3;
1385 position &= entries.len - 1;
1386 }
1387 }
1388 std.mem.sort(u16, temp_states[0..probability], {}, std.sort.asc(u16));
1389 for (0..probability) |i| {
1390 entries[temp_states[i]] = if (i < double_state_count) Table.Fse{
1391 .symbol = @as(u8, @intCast(symbol)),
1392 .bits = share_size_log + 1,
1393 .baseline = single_state_count * share_size + @as(u16, @intCast(i)) * 2 * share_size,
1394 } else Table.Fse{
1395 .symbol = @as(u8, @intCast(symbol)),
1396 .bits = share_size_log,
1397 .baseline = (@as(u16, @intCast(i)) - double_state_count) * share_size,
1398 };
1399 }
1400 }
1401 }
1402
1403 test build {
1404 const literals_length_default_values = [36]u16{
1405 5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2,
1406 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 2, 2, 2, 2, 2,
1407 0, 0, 0, 0,
1408 };
1409
1410 const match_lengths_default_values = [53]u16{
1411 2, 5, 4, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1412 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1413 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0,
1414 0, 0, 0, 0, 0,
1415 };
1416
1417 const offset_codes_default_values = [29]u16{
1418 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2,
1419 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0,
1420 };
1421
1422 var entries: [64]Table.Fse = undefined;
1423 try build(&literals_length_default_values, &entries);
1424 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_literal.fse, &entries);
1425
1426 try build(&match_lengths_default_values, &entries);
1427 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_match.fse, &entries);
1428
1429 try build(&offset_codes_default_values, entries[0..32]);
1430 try std.testing.expectEqualSlices(Table.Fse, Table.predefined_offset.fse, entries[0..32]);
1431 }
1432
1433 pub const predefined_literal: Table = .{
1434 .fse = &[64]Table.Fse{
1435 .{ .symbol = 0, .bits = 4, .baseline = 0 },
1436 .{ .symbol = 0, .bits = 4, .baseline = 16 },
1437 .{ .symbol = 1, .bits = 5, .baseline = 32 },
1438 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1439 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1440 .{ .symbol = 6, .bits = 5, .baseline = 0 },
1441 .{ .symbol = 7, .bits = 5, .baseline = 0 },
1442 .{ .symbol = 9, .bits = 5, .baseline = 0 },
1443 .{ .symbol = 10, .bits = 5, .baseline = 0 },
1444 .{ .symbol = 12, .bits = 5, .baseline = 0 },
1445 .{ .symbol = 14, .bits = 6, .baseline = 0 },
1446 .{ .symbol = 16, .bits = 5, .baseline = 0 },
1447 .{ .symbol = 18, .bits = 5, .baseline = 0 },
1448 .{ .symbol = 19, .bits = 5, .baseline = 0 },
1449 .{ .symbol = 21, .bits = 5, .baseline = 0 },
1450 .{ .symbol = 22, .bits = 5, .baseline = 0 },
1451 .{ .symbol = 24, .bits = 5, .baseline = 0 },
1452 .{ .symbol = 25, .bits = 5, .baseline = 32 },
1453 .{ .symbol = 26, .bits = 5, .baseline = 0 },
1454 .{ .symbol = 27, .bits = 6, .baseline = 0 },
1455 .{ .symbol = 29, .bits = 6, .baseline = 0 },
1456 .{ .symbol = 31, .bits = 6, .baseline = 0 },
1457 .{ .symbol = 0, .bits = 4, .baseline = 32 },
1458 .{ .symbol = 1, .bits = 4, .baseline = 0 },
1459 .{ .symbol = 2, .bits = 5, .baseline = 0 },
1460 .{ .symbol = 4, .bits = 5, .baseline = 32 },
1461 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1462 .{ .symbol = 7, .bits = 5, .baseline = 32 },
1463 .{ .symbol = 8, .bits = 5, .baseline = 0 },
1464 .{ .symbol = 10, .bits = 5, .baseline = 32 },
1465 .{ .symbol = 11, .bits = 5, .baseline = 0 },
1466 .{ .symbol = 13, .bits = 6, .baseline = 0 },
1467 .{ .symbol = 16, .bits = 5, .baseline = 32 },
1468 .{ .symbol = 17, .bits = 5, .baseline = 0 },
1469 .{ .symbol = 19, .bits = 5, .baseline = 32 },
1470 .{ .symbol = 20, .bits = 5, .baseline = 0 },
1471 .{ .symbol = 22, .bits = 5, .baseline = 32 },
1472 .{ .symbol = 23, .bits = 5, .baseline = 0 },
1473 .{ .symbol = 25, .bits = 4, .baseline = 0 },
1474 .{ .symbol = 25, .bits = 4, .baseline = 16 },
1475 .{ .symbol = 26, .bits = 5, .baseline = 32 },
1476 .{ .symbol = 28, .bits = 6, .baseline = 0 },
1477 .{ .symbol = 30, .bits = 6, .baseline = 0 },
1478 .{ .symbol = 0, .bits = 4, .baseline = 48 },
1479 .{ .symbol = 1, .bits = 4, .baseline = 16 },
1480 .{ .symbol = 2, .bits = 5, .baseline = 32 },
1481 .{ .symbol = 3, .bits = 5, .baseline = 32 },
1482 .{ .symbol = 5, .bits = 5, .baseline = 32 },
1483 .{ .symbol = 6, .bits = 5, .baseline = 32 },
1484 .{ .symbol = 8, .bits = 5, .baseline = 32 },
1485 .{ .symbol = 9, .bits = 5, .baseline = 32 },
1486 .{ .symbol = 11, .bits = 5, .baseline = 32 },
1487 .{ .symbol = 12, .bits = 5, .baseline = 32 },
1488 .{ .symbol = 15, .bits = 6, .baseline = 0 },
1489 .{ .symbol = 17, .bits = 5, .baseline = 32 },
1490 .{ .symbol = 18, .bits = 5, .baseline = 32 },
1491 .{ .symbol = 20, .bits = 5, .baseline = 32 },
1492 .{ .symbol = 21, .bits = 5, .baseline = 32 },
1493 .{ .symbol = 23, .bits = 5, .baseline = 32 },
1494 .{ .symbol = 24, .bits = 5, .baseline = 32 },
1495 .{ .symbol = 35, .bits = 6, .baseline = 0 },
1496 .{ .symbol = 34, .bits = 6, .baseline = 0 },
1497 .{ .symbol = 33, .bits = 6, .baseline = 0 },
1498 .{ .symbol = 32, .bits = 6, .baseline = 0 },
1499 },
1500 };
1501
1502 pub const predefined_match: Table = .{
1503 .fse = &[64]Table.Fse{
1504 .{ .symbol = 0, .bits = 6, .baseline = 0 },
1505 .{ .symbol = 1, .bits = 4, .baseline = 0 },
1506 .{ .symbol = 2, .bits = 5, .baseline = 32 },
1507 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1508 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1509 .{ .symbol = 6, .bits = 5, .baseline = 0 },
1510 .{ .symbol = 8, .bits = 5, .baseline = 0 },
1511 .{ .symbol = 10, .bits = 6, .baseline = 0 },
1512 .{ .symbol = 13, .bits = 6, .baseline = 0 },
1513 .{ .symbol = 16, .bits = 6, .baseline = 0 },
1514 .{ .symbol = 19, .bits = 6, .baseline = 0 },
1515 .{ .symbol = 22, .bits = 6, .baseline = 0 },
1516 .{ .symbol = 25, .bits = 6, .baseline = 0 },
1517 .{ .symbol = 28, .bits = 6, .baseline = 0 },
1518 .{ .symbol = 31, .bits = 6, .baseline = 0 },
1519 .{ .symbol = 33, .bits = 6, .baseline = 0 },
1520 .{ .symbol = 35, .bits = 6, .baseline = 0 },
1521 .{ .symbol = 37, .bits = 6, .baseline = 0 },
1522 .{ .symbol = 39, .bits = 6, .baseline = 0 },
1523 .{ .symbol = 41, .bits = 6, .baseline = 0 },
1524 .{ .symbol = 43, .bits = 6, .baseline = 0 },
1525 .{ .symbol = 45, .bits = 6, .baseline = 0 },
1526 .{ .symbol = 1, .bits = 4, .baseline = 16 },
1527 .{ .symbol = 2, .bits = 4, .baseline = 0 },
1528 .{ .symbol = 3, .bits = 5, .baseline = 32 },
1529 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1530 .{ .symbol = 6, .bits = 5, .baseline = 32 },
1531 .{ .symbol = 7, .bits = 5, .baseline = 0 },
1532 .{ .symbol = 9, .bits = 6, .baseline = 0 },
1533 .{ .symbol = 12, .bits = 6, .baseline = 0 },
1534 .{ .symbol = 15, .bits = 6, .baseline = 0 },
1535 .{ .symbol = 18, .bits = 6, .baseline = 0 },
1536 .{ .symbol = 21, .bits = 6, .baseline = 0 },
1537 .{ .symbol = 24, .bits = 6, .baseline = 0 },
1538 .{ .symbol = 27, .bits = 6, .baseline = 0 },
1539 .{ .symbol = 30, .bits = 6, .baseline = 0 },
1540 .{ .symbol = 32, .bits = 6, .baseline = 0 },
1541 .{ .symbol = 34, .bits = 6, .baseline = 0 },
1542 .{ .symbol = 36, .bits = 6, .baseline = 0 },
1543 .{ .symbol = 38, .bits = 6, .baseline = 0 },
1544 .{ .symbol = 40, .bits = 6, .baseline = 0 },
1545 .{ .symbol = 42, .bits = 6, .baseline = 0 },
1546 .{ .symbol = 44, .bits = 6, .baseline = 0 },
1547 .{ .symbol = 1, .bits = 4, .baseline = 32 },
1548 .{ .symbol = 1, .bits = 4, .baseline = 48 },
1549 .{ .symbol = 2, .bits = 4, .baseline = 16 },
1550 .{ .symbol = 4, .bits = 5, .baseline = 32 },
1551 .{ .symbol = 5, .bits = 5, .baseline = 32 },
1552 .{ .symbol = 7, .bits = 5, .baseline = 32 },
1553 .{ .symbol = 8, .bits = 5, .baseline = 32 },
1554 .{ .symbol = 11, .bits = 6, .baseline = 0 },
1555 .{ .symbol = 14, .bits = 6, .baseline = 0 },
1556 .{ .symbol = 17, .bits = 6, .baseline = 0 },
1557 .{ .symbol = 20, .bits = 6, .baseline = 0 },
1558 .{ .symbol = 23, .bits = 6, .baseline = 0 },
1559 .{ .symbol = 26, .bits = 6, .baseline = 0 },
1560 .{ .symbol = 29, .bits = 6, .baseline = 0 },
1561 .{ .symbol = 52, .bits = 6, .baseline = 0 },
1562 .{ .symbol = 51, .bits = 6, .baseline = 0 },
1563 .{ .symbol = 50, .bits = 6, .baseline = 0 },
1564 .{ .symbol = 49, .bits = 6, .baseline = 0 },
1565 .{ .symbol = 48, .bits = 6, .baseline = 0 },
1566 .{ .symbol = 47, .bits = 6, .baseline = 0 },
1567 .{ .symbol = 46, .bits = 6, .baseline = 0 },
1568 },
1569 };
1570
1571 pub const predefined_offset: Table = .{
1572 .fse = &[32]Table.Fse{
1573 .{ .symbol = 0, .bits = 5, .baseline = 0 },
1574 .{ .symbol = 6, .bits = 4, .baseline = 0 },
1575 .{ .symbol = 9, .bits = 5, .baseline = 0 },
1576 .{ .symbol = 15, .bits = 5, .baseline = 0 },
1577 .{ .symbol = 21, .bits = 5, .baseline = 0 },
1578 .{ .symbol = 3, .bits = 5, .baseline = 0 },
1579 .{ .symbol = 7, .bits = 4, .baseline = 0 },
1580 .{ .symbol = 12, .bits = 5, .baseline = 0 },
1581 .{ .symbol = 18, .bits = 5, .baseline = 0 },
1582 .{ .symbol = 23, .bits = 5, .baseline = 0 },
1583 .{ .symbol = 5, .bits = 5, .baseline = 0 },
1584 .{ .symbol = 8, .bits = 4, .baseline = 0 },
1585 .{ .symbol = 14, .bits = 5, .baseline = 0 },
1586 .{ .symbol = 20, .bits = 5, .baseline = 0 },
1587 .{ .symbol = 2, .bits = 5, .baseline = 0 },
1588 .{ .symbol = 7, .bits = 4, .baseline = 16 },
1589 .{ .symbol = 11, .bits = 5, .baseline = 0 },
1590 .{ .symbol = 17, .bits = 5, .baseline = 0 },
1591 .{ .symbol = 22, .bits = 5, .baseline = 0 },
1592 .{ .symbol = 4, .bits = 5, .baseline = 0 },
1593 .{ .symbol = 8, .bits = 4, .baseline = 16 },
1594 .{ .symbol = 13, .bits = 5, .baseline = 0 },
1595 .{ .symbol = 19, .bits = 5, .baseline = 0 },
1596 .{ .symbol = 1, .bits = 5, .baseline = 0 },
1597 .{ .symbol = 6, .bits = 4, .baseline = 16 },
1598 .{ .symbol = 10, .bits = 5, .baseline = 0 },
1599 .{ .symbol = 16, .bits = 5, .baseline = 0 },
1600 .{ .symbol = 28, .bits = 5, .baseline = 0 },
1601 .{ .symbol = 27, .bits = 5, .baseline = 0 },
1602 .{ .symbol = 26, .bits = 5, .baseline = 0 },
1603 .{ .symbol = 25, .bits = 5, .baseline = 0 },
1604 .{ .symbol = 24, .bits = 5, .baseline = 0 },
1605 },
1606 };
1607};
1608
1609const low_bit_mask = [9]u8{
1610 0b00000000,
1611 0b00000001,
1612 0b00000011,
1613 0b00000111,
1614 0b00001111,
1615 0b00011111,
1616 0b00111111,
1617 0b01111111,
1618 0b11111111,
1619};
1620
1621fn Bits(comptime T: type) type {
1622 return struct { T, u16 };
1623}
1624
1625/// For reading the reversed bit streams used to encode FSE compressed data.
1626const ReverseBitReader = struct {
1627 bytes: []const u8,
1628 remaining: usize,
1629 bits: u8,
1630 count: u4,
1631
1632 fn init(bytes: []const u8) error{MissingStartBit}!ReverseBitReader {
1633 var result: ReverseBitReader = .{
1634 .bytes = bytes,
1635 .remaining = bytes.len,
1636 .bits = 0,
1637 .count = 0,
1638 };
1639 if (bytes.len == 0) return result;
1640 for (0..8) |_| if (0 != (result.readBitsNoEof(u1, 1) catch unreachable)) return result;
1641 return error.MissingStartBit;
1642 }
1643
1644 fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1645 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1646 return .{
1647 @bitCast(@as(UT, @intCast(out))),
1648 num,
1649 };
1650 }
1651
1652 fn readBitsNoEof(self: *ReverseBitReader, comptime T: type, num: u16) error{EndOfStream}!T {
1653 const b, const c = try self.readBitsTuple(T, num);
1654 if (c < num) return error.EndOfStream;
1655 return b;
1656 }
1657
1658 fn readBits(self: *ReverseBitReader, comptime T: type, num: u16, out_bits: *u16) !T {
1659 const b, const c = try self.readBitsTuple(T, num);
1660 out_bits.* = c;
1661 return b;
1662 }
1663
1664 fn readBitsTuple(self: *ReverseBitReader, comptime T: type, num: u16) !Bits(T) {
1665 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1666 const U = if (@bitSizeOf(T) < 8) u8 else UT;
1667
1668 if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1669
1670 var out_count: u16 = self.count;
1671 var out: U = self.removeBits(self.count);
1672
1673 const full_bytes_left = (num - out_count) / 8;
1674
1675 for (0..full_bytes_left) |_| {
1676 const byte = takeByte(self) catch |err| switch (err) {
1677 error.EndOfStream => return initBits(T, out, out_count),
1678 };
1679 if (U == u8) out = 0 else out <<= 8;
1680 out |= byte;
1681 out_count += 8;
1682 }
1683
1684 const bits_left = num - out_count;
1685 const keep = 8 - bits_left;
1686
1687 if (bits_left == 0) return initBits(T, out, out_count);
1688
1689 const final_byte = takeByte(self) catch |err| switch (err) {
1690 error.EndOfStream => return initBits(T, out, out_count),
1691 };
1692
1693 out <<= @intCast(bits_left);
1694 out |= final_byte >> @intCast(keep);
1695 self.bits = final_byte & low_bit_mask[keep];
1696
1697 self.count = @intCast(keep);
1698 return initBits(T, out, num);
1699 }
1700
1701 fn takeByte(rbr: *ReverseBitReader) error{EndOfStream}!u8 {
1702 if (rbr.remaining == 0) return error.EndOfStream;
1703 rbr.remaining -= 1;
1704 return rbr.bytes[rbr.remaining];
1705 }
1706
1707 fn isEmpty(self: *const ReverseBitReader) bool {
1708 return self.remaining == 0 and self.count == 0;
1709 }
1710
1711 fn removeBits(self: *ReverseBitReader, num: u4) u8 {
1712 if (num == 8) {
1713 self.count = 0;
1714 return self.bits;
1715 }
1716
1717 const keep = self.count - num;
1718 const bits = self.bits >> @intCast(keep);
1719 self.bits &= low_bit_mask[keep];
1720
1721 self.count = keep;
1722 return bits;
1723 }
1724};
1725
1726const BitReader = struct {
1727 bytes: []const u8,
1728 index: usize = 0,
1729 bits: u8 = 0,
1730 count: u4 = 0,
1731
1732 fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1733 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1734 return .{
1735 @bitCast(@as(UT, @intCast(out))),
1736 num,
1737 };
1738 }
1739
1740 fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
1741 const b, const c = try self.readBitsTuple(T, num);
1742 if (c < num) return error.EndOfStream;
1743 return b;
1744 }
1745
1746 fn readBits(self: *@This(), comptime T: type, num: u16, out_bits: *u16) !T {
1747 const b, const c = try self.readBitsTuple(T, num);
1748 out_bits.* = c;
1749 return b;
1750 }
1751
1752 fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
1753 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1754 const U = if (@bitSizeOf(T) < 8) u8 else UT;
1755
1756 if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1757
1758 var out_count: u16 = self.count;
1759 var out: U = self.removeBits(self.count);
1760
1761 const full_bytes_left = (num - out_count) / 8;
1762
1763 for (0..full_bytes_left) |_| {
1764 const byte = takeByte(self) catch |err| switch (err) {
1765 error.EndOfStream => return initBits(T, out, out_count),
1766 };
1767
1768 const pos = @as(U, byte) << @intCast(out_count);
1769 out |= pos;
1770 out_count += 8;
1771 }
1772
1773 const bits_left = num - out_count;
1774 const keep = 8 - bits_left;
1775
1776 if (bits_left == 0) return initBits(T, out, out_count);
1777
1778 const final_byte = takeByte(self) catch |err| switch (err) {
1779 error.EndOfStream => return initBits(T, out, out_count),
1780 };
1781
1782 const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
1783 out |= pos;
1784 self.bits = final_byte >> @intCast(bits_left);
1785
1786 self.count = @intCast(keep);
1787 return initBits(T, out, num);
1788 }
1789
1790 fn takeByte(br: *BitReader) error{EndOfStream}!u8 {
1791 if (br.bytes.len - br.index == 0) return error.EndOfStream;
1792 const result = br.bytes[br.index];
1793 br.index += 1;
1794 return result;
1795 }
1796
1797 fn removeBits(self: *@This(), num: u4) u8 {
1798 if (num == 8) {
1799 self.count = 0;
1800 return self.bits;
1801 }
1802
1803 const keep = self.count - num;
1804 const bits = self.bits & low_bit_mask[num];
1805 self.bits >>= @intCast(num);
1806 self.count = keep;
1807 return bits;
1808 }
1809
1810 fn alignToByte(self: *@This()) void {
1811 self.bits = 0;
1812 self.count = 0;
1813 }
1814};
1815
1816test {
1817 _ = Table;
1818}
lib/std/fs/File.zig+13-1
......@@ -2026,7 +2026,19 @@ pub const Writer = struct {
20262026 /// along with other write failures.
20272027 pub fn end(w: *Writer) EndError!void {
20282028 try w.interface.flush();
2029 return w.file.setEndPos(w.pos);
2029 switch (w.mode) {
2030 .positional,
2031 .positional_reading,
2032 => w.file.setEndPos(w.pos) catch |err| switch (err) {
2033 error.NonResizable => return,
2034 else => |e| return e,
2035 },
2036
2037 .streaming,
2038 .streaming_reading,
2039 .failure,
2040 => {},
2041 }
20302042 }
20312043};
20322044
lib/std/http/Server.zig+1-2
......@@ -129,11 +129,10 @@ pub const Request = struct {
129129 pub const Compression = union(enum) {
130130 pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131131 pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132 pub const ZstdDecompressor = std.compress.zstd.Decompressor(std.io.AnyReader);
133132
134133 deflate: DeflateDecompressor,
135134 gzip: GzipDecompressor,
136 zstd: ZstdDecompressor,
135 zstd: std.compress.zstd.Decompress,
137136 none: void,
138137 };
139138
lib/std/posix.zig+3-2
......@@ -1052,6 +1052,7 @@ pub const TruncateError = error{
10521052 FileBusy,
10531053 AccessDenied,
10541054 PermissionDenied,
1055 NonResizable,
10551056} || UnexpectedError;
10561057
10571058/// Length must be positive when treated as an i64.
......@@ -1091,7 +1092,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {
10911092 .PERM => return error.PermissionDenied,
10921093 .TXTBSY => return error.FileBusy,
10931094 .BADF => unreachable, // Handle not open for writing
1094 .INVAL => unreachable, // Handle not open for writing, negative length, or non-resizable handle
1095 .INVAL => return error.NonResizable,
10951096 .NOTCAPABLE => return error.AccessDenied,
10961097 else => |err| return unexpectedErrno(err),
10971098 }
......@@ -1107,7 +1108,7 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void {
11071108 .PERM => return error.PermissionDenied,
11081109 .TXTBSY => return error.FileBusy,
11091110 .BADF => unreachable, // Handle not open for writing
1110 .INVAL => unreachable, // Handle not open for writing, negative length, or non-resizable handle
1111 .INVAL => return error.NonResizable, // This is returned for /dev/null for example.
11111112 else => |err| return unexpectedErrno(err),
11121113 }
11131114 }
src/Package/Fetch.zig+8-9
......@@ -1198,7 +1198,8 @@ fn unpackResource(
11981198
11991199 switch (file_type) {
12001200 .tar => {
1201 var adapter = resource.reader().adaptToNewApi();
1201 var adapter_buffer: [1024]u8 = undefined;
1202 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
12021203 return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
12031204 },
12041205 .@"tar.gz" => {
......@@ -1225,16 +1226,14 @@ fn unpackResource(
12251226 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
12261227 },
12271228 .@"tar.zst" => {
1228 const window_size = std.compress.zstd.DecompressorOptions.default_window_buffer_len;
1229 const window_size = std.compress.zstd.default_window_len;
12291230 const window_buffer = try f.arena.allocator().create([window_size]u8);
1230 const reader = resource.reader();
1231 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1232 var dcp = std.compress.zstd.decompressor(br.reader(), .{
1233 .window_buffer = window_buffer,
1231 var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
1232 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1233 var decompress: std.compress.zstd.Decompress = .init(&adapter.new_interface, window_buffer, .{
1234 .verify_checksum = false,
12341235 });
1235 var adapter_buffer: [1024]u8 = undefined;
1236 var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
1237 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1236 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
12381237 },
12391238 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
12401239 error.FetchFailed => return error.FetchFailed,
src/Package/Fetch/git.zig+60-5
......@@ -1281,7 +1281,7 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
12811281 }
12821282 @memset(fan_out_table[fan_out_index..], count);
12831283
1284 var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format));
1284 var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format));
12851285 const writer = index_hashed_writer.writer();
12861286 try writer.writeAll(IndexHeader.signature);
12871287 try writer.writeInt(u32, IndexHeader.supported_version, .big);
......@@ -1331,7 +1331,7 @@ fn indexPackFirstPass(
13311331) !Oid {
13321332 var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());
13331333 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1334 var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1334 var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
13351335 const pack_reader = pack_hashed_reader.reader();
13361336
13371337 const pack_header = try PackHeader.read(pack_reader);
......@@ -1339,13 +1339,13 @@ fn indexPackFirstPass(
13391339 var current_entry: u32 = 0;
13401340 while (current_entry < pack_header.total_objects) : (current_entry += 1) {
13411341 const entry_offset = pack_counting_reader.bytes_read;
1342 var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
1342 var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
13431343 const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
13441344 switch (entry_header) {
13451345 .commit, .tree, .blob, .tag => |object| {
13461346 var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
13471347 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1348 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1348 var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
13491349 const entry_writer = entry_hashed_writer.writer();
13501350 // The object header is not included in the pack data but is
13511351 // part of the object's ID
......@@ -1432,7 +1432,7 @@ fn indexPackHashDelta(
14321432 const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
14331433
14341434 var entry_hasher: Oid.Hasher = .init(format);
1435 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);
1435 var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
14361436 try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
14371437 entry_hasher.update(base_data);
14381438 return entry_hasher.finalResult();
......@@ -1703,3 +1703,58 @@ pub fn main() !void {
17031703 std.debug.print("Diagnostic: {}\n", .{err});
17041704 }
17051705}
1706
1707/// Deprecated
1708fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
1709 return .{ .child_reader = reader, .hasher = hasher };
1710}
1711
1712/// Deprecated
1713fn HashedReader(ReaderType: type, HasherType: type) type {
1714 return struct {
1715 child_reader: ReaderType,
1716 hasher: HasherType,
1717
1718 pub const Error = ReaderType.Error;
1719 pub const Reader = std.io.GenericReader(*@This(), Error, read);
1720
1721 pub fn read(self: *@This(), buf: []u8) Error!usize {
1722 const amt = try self.child_reader.read(buf);
1723 self.hasher.update(buf[0..amt]);
1724 return amt;
1725 }
1726
1727 pub fn reader(self: *@This()) Reader {
1728 return .{ .context = self };
1729 }
1730 };
1731}
1732
1733/// Deprecated
1734pub fn HashedWriter(WriterType: type, HasherType: type) type {
1735 return struct {
1736 child_writer: WriterType,
1737 hasher: HasherType,
1738
1739 pub const Error = WriterType.Error;
1740 pub const Writer = std.io.GenericWriter(*@This(), Error, write);
1741
1742 pub fn write(self: *@This(), buf: []const u8) Error!usize {
1743 const amt = try self.child_writer.write(buf);
1744 self.hasher.update(buf[0..amt]);
1745 return amt;
1746 }
1747
1748 pub fn writer(self: *@This()) Writer {
1749 return .{ .context = self };
1750 }
1751 };
1752}
1753
1754/// Deprecated
1755pub fn hashedWriter(
1756 writer: anytype,
1757 hasher: anytype,
1758) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1759 return .{ .child_writer = writer, .hasher = hasher };
1760}