Commit 742956865c0c55b6650d7b89d923b63ea11cde4f | zig

lib/std/Io.zig-12

 @@ -438,8 +438,6 @@ pub fn GenericWriter(
 pub const AnyReader = @import("Io/DeprecatedReader.zig");
 /// Deprecated in favor of `Writer`.
 pub const AnyWriter = @import("Io/DeprecatedWriter.zig");
-/// Deprecated in favor of `File.Reader` and `File.Writer`.
-pub const SeekableStream = @import("Io/seekable_stream.zig").SeekableStream;
 /// Deprecated in favor of `Writer`.
 pub const BufferedWriter = @import("Io/buffered_writer.zig").BufferedWriter;
 /// Deprecated in favor of `Writer`.
 @@ -467,12 +465,6 @@ pub const CountingReader = @import("Io/counting_reader.zig").CountingReader;
 /// Deprecated with no replacement; inefficient pattern
 pub const countingReader = @import("Io/counting_reader.zig").countingReader;
-pub const BitReader = @import("Io/bit_reader.zig").BitReader;
-pub const bitReader = @import("Io/bit_reader.zig").bitReader;
-pub const BitWriter = @import("Io/bit_writer.zig").BitWriter;
-pub const bitWriter = @import("Io/bit_writer.zig").bitWriter;
 pub const tty = @import("Io/tty.zig");
 /// Deprecated in favor of `Writer.Discarding`.
 @@ -948,16 +940,12 @@ pub fn PollFiles(comptime StreamEnum: type) type {
 test {
     _ = Reader;
-    _ = Reader.Limited;
     _ = Writer;
-    _ = BitReader;
-    _ = BitWriter;
     _ = BufferedReader;
     _ = BufferedWriter;
     _ = CountingWriter;
     _ = CountingReader;
     _ = FixedBufferStream;
-    _ = SeekableStream;
     _ = tty;
     _ = @import("Io/test.zig");
 }

lib/std/Io/Reader.zig+48-44

 @@ -74,6 +74,10 @@ pub const VTable = struct {
     ///
     /// `data` may not contain an alias to `Reader.buffer`.
     ///
+    /// `data` is mutable because the implementation may to temporarily modify
+    /// the fields in order to handle partial reads. Implementations must
+    /// restore the original value before returning.
+    ///
     /// Implementations may ignore `data`, writing directly to `Reader.buffer`,
     /// modifying `seek` and `end` accordingly, and returning 0 from this
     /// function. Implementations are encouraged to take advantage of this if
 @@ -81,7 +85,7 @@ pub const VTable = struct {
     ///
     /// The default implementation calls `stream` with either `data[0]` or
     /// `Reader.buffer`, whichever is bigger.
-    readVec: *const fn (r: *Reader, data: []const []u8) Error!usize = defaultReadVec,
+    readVec: *const fn (r: *Reader, data: [][]u8) Error!usize = defaultReadVec,
     /// Ensures `capacity` more data can be buffered without rebasing.
     ///
 @@ -262,8 +266,7 @@ pub fn streamRemaining(r: *Reader, w: *Writer) StreamRemainingError!usize {
 /// number of bytes discarded.
 pub fn discardRemaining(r: *Reader) ShortError!usize {
     var offset: usize = r.end - r.seek;
-    r.seek = 0;
+    r.seek = r.end;
-    r.end = 0;
     while (true) {
         offset += r.vtable.discard(r, .unlimited) catch |err| switch (err) {
             error.EndOfStream => return offset,
 @@ -417,7 +420,7 @@ pub fn readVec(r: *Reader, data: [][]u8) Error!usize {
 }
 /// Writes to `Reader.buffer` or `data`, whichever has larger capacity.
-pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize {
+pub fn defaultReadVec(r: *Reader, data: [][]u8) Error!usize {
     assert(r.seek == r.end);
     r.seek = 0;
     r.end = 0;
 @@ -438,23 +441,6 @@ pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize {
     return 0;
 }
-/// Always writes to `Reader.buffer` and returns 0.
-pub fn indirectReadVec(r: *Reader, data: []const []u8) Error!usize {
-    _ = data;
-    assert(r.seek == r.end);
-    var writer: Writer = .{
-        .buffer = r.buffer,
-        .end = r.end,
-        .vtable = &.{ .drain = Writer.fixedDrain },
-    };
-    const limit: Limit = .limited(writer.buffer.len - writer.end);
-    r.end += r.vtable.stream(r, &writer, limit) catch |err| switch (err) {
-        error.WriteFailed => unreachable,
-        else => |e| return e,
-    };
-    return 0;
-}
 pub fn buffered(r: *Reader) []u8 {
     return r.buffer[r.seek..r.end];
 }
 @@ -463,8 +449,8 @@ pub fn bufferedLen(r: *const Reader) usize {
     return r.end - r.seek;
 }
-pub fn hashed(r: *Reader, hasher: anytype) Hashed(@TypeOf(hasher)) {
+pub fn hashed(r: *Reader, hasher: anytype, buffer: []u8) Hashed(@TypeOf(hasher)) {
-    return .{ .in = r, .hasher = hasher };
+    return .init(r, hasher, buffer);
 }
 pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {
 @@ -539,8 +525,7 @@ pub fn toss(r: *Reader, n: usize) void {
 /// Equivalent to `toss(r.bufferedLen())`.
 pub fn tossBuffered(r: *Reader) void {
-    r.seek = 0;
+    r.seek = r.end;
-    r.end = 0;
 }
 /// Equivalent to `peek` followed by `toss`.
 @@ -627,8 +612,7 @@ pub fn discardShort(r: *Reader, n: usize) ShortError!usize {
         return n;
     }
     var remaining = n - (r.end - r.seek);
-    r.end = 0;
+    r.seek = r.end;
-    r.seek = 0;
     while (true) {
         const discard_len = r.vtable.discard(r, .limited(remaining)) catch |err| switch (err) {
             error.EndOfStream => return n - remaining,
 @@ -1678,7 +1662,7 @@ fn endingStream(r: *Reader, w: *Writer, limit: Limit) StreamError!usize {
     return error.EndOfStream;
 }
-fn endingReadVec(r: *Reader, data: []const []u8) Error!usize {
+fn endingReadVec(r: *Reader, data: [][]u8) Error!usize {
     _ = r;
     _ = data;
     return error.EndOfStream;
 @@ -1709,6 +1693,15 @@ fn failingDiscard(r: *Reader, limit: Limit) Error!usize {
     return error.ReadFailed;
 }
+pub fn adaptToOldInterface(r: *Reader) std.Io.AnyReader {
+    return .{ .context = r, .readFn = derpRead };
+}
+fn derpRead(context: *const anyopaque, buffer: []u8) anyerror!usize {
+    const r: *Reader = @constCast(@alignCast(@ptrCast(context)));
+    return r.readSliceShort(buffer);
+}
 test "readAlloc when the backing reader provides one byte at a time" {
     const str = "This is a test";
     var tiny_buffer: [1]u8 = undefined;
 @@ -1772,15 +1765,16 @@ pub fn Hashed(comptime Hasher: type) type {
     return struct {
         in: *Reader,
         hasher: Hasher,
-        interface: Reader,
+        reader: Reader,
         pub fn init(in: *Reader, hasher: Hasher, buffer: []u8) @This() {
             return .{
                 .in = in,
                 .hasher = hasher,
-                .interface = .{
+                .reader = .{
                     .vtable = &.{
-                        .read = @This().read,
+                        .stream = @This().stream,
+                        .readVec = @This().readVec,
                         .discard = @This().discard,
                     },
                     .buffer = buffer,
 @@ -1790,33 +1784,39 @@ pub fn Hashed(comptime Hasher: type) type {
             };
         }
-        fn read(r: *Reader, w: *Writer, limit: Limit) StreamError!usize {
+        fn stream(r: *Reader, w: *Writer, limit: Limit) StreamError!usize {
-            const this: *@This() = @alignCast(@fieldParentPtr("interface", r));
+            const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
-            const data = w.writableVector(limit);
+            const data = limit.slice(try w.writableSliceGreedy(1));
+            var vec: [1][]u8 = .{data};
+            const n = try this.in.readVec(&vec);
+            this.hasher.update(data[0..n]);
+            w.advance(n);
+            return n;
+        }
+        fn readVec(r: *Reader, data: [][]u8) Error!usize {
+            const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
             const n = try this.in.readVec(data);
-            const result = w.advanceVector(n);
             var remaining: usize = n;
             for (data) |slice| {
                 if (remaining < slice.len) {
                     this.hasher.update(slice[0..remaining]);
-                    return result;
+                    return n;
                 } else {
                     remaining -= slice.len;
                     this.hasher.update(slice);
                 }
             }
             assert(remaining == 0);
-            return result;
+            return n;
         }
         fn discard(r: *Reader, limit: Limit) Error!usize {
-            const this: *@This() = @alignCast(@fieldParentPtr("interface", r));
+            const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
-            var w = this.hasher.writer(&.{});
+            const peeked = limit.slice(try this.in.peekGreedy(1));
-            const n = this.in.stream(&w, limit) catch |err| switch (err) {
+            this.hasher.update(peeked);
-                error.WriteFailed => unreachable,
+            this.in.toss(peeked.len);
-                else => |e| return e,
+            return peeked.len;
-            };
-            return n;
         }
     };
 }
 @@ -1874,3 +1874,7 @@ pub fn writableVectorWsa(
     }
     return .{ i, n };
 }
+test {
+    _ = Limited;
+}

lib/std/Io/Writer.zig+57-2

...	@@ -2266,7 +2266,7 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz	...	@@ -2266,7 +2266,7 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2266	const pattern = data[data.len - 1];	2266	const pattern = data[data.len - 1];
2267	const dest = w.buffer[w.end..];	2267	const dest = w.buffer[w.end..];
2268	switch (pattern.len) {	2268	switch (pattern.len) {
2269	0 => return w.end,	2269	0 => return 0,
2270	1 => {	2270	1 => {
2271	assert(splat >= dest.len);	2271	assert(splat >= dest.len);
2272	@memset(dest, pattern[0]);	2272	@memset(dest, pattern[0]);
...	@@ -2286,6 +2286,13 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz	...	@@ -2286,6 +2286,13 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2286	}	2286	}
2287	}	2287	}
2288		2288
		2289	pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usize {
		2290	_ = w;
		2291	_ = data;
		2292	_ = splat;
		2293	unreachable;
		2294	}
		2295
2289	/// Provides a `Writer` implementation based on calling `Hasher.update`, sending	2296	/// Provides a `Writer` implementation based on calling `Hasher.update`, sending
2290	/// all data also to an underlying `Writer`.	2297	/// all data also to an underlying `Writer`.
2291	///	2298	///
...	@@ -2296,6 +2303,8 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz	...	@@ -2296,6 +2303,8 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2296	/// generic. A better solution will involve creating a writer for each hash	2303	/// generic. A better solution will involve creating a writer for each hash
2297	/// function, where the splat buffer can be tailored to the hash implementation	2304	/// function, where the splat buffer can be tailored to the hash implementation
2298	/// details.	2305	/// details.
		2306	///
		2307	/// Contrast with `Hashing` which terminates the stream pipeline.
2299	pub fn Hashed(comptime Hasher: type) type {	2308	pub fn Hashed(comptime Hasher: type) type {
2300	return struct {	2309	return struct {
2301	out: *Writer,	2310	out: *Writer,
...	@@ -2341,7 +2350,7 @@ pub fn Hashed(comptime Hasher: type) type {	...	@@ -2341,7 +2350,7 @@ pub fn Hashed(comptime Hasher: type) type {
2341	this.hasher.update(slice);	2350	this.hasher.update(slice);
2342	}	2351	}
2343	const pattern = data[data.len - 1];	2352	const pattern = data[data.len - 1];
2344	assert(remaining == splat * pattern.len);	2353	assert(remaining <= splat * pattern.len);
2345	switch (pattern.len) {	2354	switch (pattern.len) {
2346	0 => {	2355	0 => {
2347	assert(remaining == 0);	2356	assert(remaining == 0);
...	@@ -2368,6 +2377,52 @@ pub fn Hashed(comptime Hasher: type) type {	...	@@ -2368,6 +2377,52 @@ pub fn Hashed(comptime Hasher: type) type {
2368	};	2377	};
2369	}	2378	}
2370		2379
		2380	/// Provides a `Writer` implementation based on calling `Hasher.update`,
		2381	/// discarding all data.
		2382	///
		2383	/// This implementation makes suboptimal buffering decisions due to being
		2384	/// generic. A better solution will involve creating a writer for each hash
		2385	/// function, where the splat buffer can be tailored to the hash implementation
		2386	/// details.
		2387	///
		2388	/// The total number of bytes written is stored in `hasher`.
		2389	///
		2390	/// Contrast with `Hashed` which also passes the data to an underlying stream.
		2391	pub fn Hashing(comptime Hasher: type) type {
		2392	return struct {
		2393	hasher: Hasher,
		2394	writer: Writer,
		2395
		2396	pub fn init(buffer: []u8) @This() {
		2397	return .initHasher(.init(.{}), buffer);
		2398	}
		2399
		2400	pub fn initHasher(hasher: Hasher, buffer: []u8) @This() {
		2401	return .{
		2402	.hasher = hasher,
		2403	.writer = .{
		2404	.buffer = buffer,
		2405	.vtable = &.{ .drain = @This().drain },
		2406	},
		2407	};
		2408	}
		2409
		2410	fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize {
		2411	const this: *@This() = @alignCast(@fieldParentPtr("writer", w));
		2412	const hasher = &this.hasher;
		2413	hasher.update(w.buffered());
		2414	w.end = 0;
		2415	var n: usize = 0;
		2416	for (data[0 .. data.len - 1]) \|slice\| {
		2417	hasher.update(slice);
		2418	n += slice.len;
		2419	}
		2420	for (0..splat) \|_\| hasher.update(data[data.len - 1]);
		2421	return n + splat * data[data.len - 1].len;
		2422	}
		2423	};
		2424	}
		2425
2371	/// Maintains `Writer` state such that it writes to the unused capacity of an	2426	/// Maintains `Writer` state such that it writes to the unused capacity of an
2372	/// array list, filling it up completely before making a call through the	2427	/// array list, filling it up completely before making a call through the
2373	/// vtable, causing a resize. Consequently, the same, optimized, non-generic	2428	/// vtable, causing a resize. Consequently, the same, optimized, non-generic

lib/std/Io/bit_reader.zig deleted-238

...	@@ -1,238 +0,0 @@
1	const std = @import("../std.zig");
2
3	//General note on endianess:
4	//Big endian is packed starting in the most significant part of the byte and subsequent
5	// bytes contain less significant bits. Thus we always take bits from the high
6	// end and place them below existing bits in our output.
7	//Little endian is packed starting in the least significant part of the byte and
8	// subsequent bytes contain more significant bits. Thus we always take bits from
9	// the low end and place them above existing bits in our output.
10	//Regardless of endianess, within any given byte the bits are always in most
11	// to least significant order.
12	//Also regardless of endianess, the buffer always aligns bits to the low end
13	// of the byte.
14
15	/// Creates a bit reader which allows for reading bits from an underlying standard reader
16	pub fn BitReader(comptime endian: std.builtin.Endian, comptime Reader: type) type {
17	return struct {
18	reader: Reader,
19	bits: u8 = 0,
20	count: u4 = 0,
21
22	const low_bit_mask = [9]u8{
23	0b00000000,
24	0b00000001,
25	0b00000011,
26	0b00000111,
27	0b00001111,
28	0b00011111,
29	0b00111111,
30	0b01111111,
31	0b11111111,
32	};
33
34	fn Bits(comptime T: type) type {
35	return struct {
36	T,
37	u16,
38	};
39	}
40
41	fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
42	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
43	return .{
44	@bitCast(@as(UT, @intCast(out))),
45	num,
46	};
47	}
48
49	/// Reads `bits` bits from the reader and returns a specified type
50	/// containing them in the least significant end, returning an error if the
51	/// specified number of bits could not be read.
52	pub fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
53	const b, const c = try self.readBitsTuple(T, num);
54	if (c < num) return error.EndOfStream;
55	return b;
56	}
57
58	/// Reads `bits` bits from the reader and returns a specified type
59	/// containing them in the least significant end. The number of bits successfully
60	/// read is placed in `out_bits`, as reaching the end of the stream is not an error.
61	pub fn readBits(self: @This(), comptime T: type, num: u16, out_bits: u16) !T {
62	const b, const c = try self.readBitsTuple(T, num);
63	out_bits.* = c;
64	return b;
65	}
66
67	/// Reads `bits` bits from the reader and returns a tuple of the specified type
68	/// containing them in the least significant end, and the number of bits successfully
69	/// read. Reaching the end of the stream is not an error.
70	pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
71	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
72	const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with <u8
73
74	//dump any bits in our buffer first
75	if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
76
77	var out_count: u16 = self.count;
78	var out: U = self.removeBits(self.count);
79
80	//grab all the full bytes we need and put their
81	//bits where they belong
82	const full_bytes_left = (num - out_count) / 8;
83
84	for (0..full_bytes_left) \|_\| {
85	const byte = self.reader.readByte() catch \|err\| switch (err) {
86	error.EndOfStream => return initBits(T, out, out_count),
87	else => \|e\| return e,
88	};
89
90	switch (endian) {
91	.big => {
92	if (U == u8) out = 0 else out <<= 8; //shifting u8 by 8 is illegal in Zig
93	out \|= byte;
94	},
95	.little => {
96	const pos = @as(U, byte) << @intCast(out_count);
97	out \|= pos;
98	},
99	}
100	out_count += 8;
101	}
102
103	const bits_left = num - out_count;
104	const keep = 8 - bits_left;
105
106	if (bits_left == 0) return initBits(T, out, out_count);
107
108	const final_byte = self.reader.readByte() catch \|err\| switch (err) {
109	error.EndOfStream => return initBits(T, out, out_count),
110	else => \|e\| return e,
111	};
112
113	switch (endian) {
114	.big => {
115	out <<= @intCast(bits_left);
116	out \|= final_byte >> @intCast(keep);
117	self.bits = final_byte & low_bit_mask[keep];
118	},
119	.little => {
120	const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
121	out \|= pos;
122	self.bits = final_byte >> @intCast(bits_left);
123	},
124	}
125
126	self.count = @intCast(keep);
127	return initBits(T, out, num);
128	}
129
130	//convenience function for removing bits from
131	//the appropriate part of the buffer based on
132	//endianess.
133	fn removeBits(self: *@This(), num: u4) u8 {
134	if (num == 8) {
135	self.count = 0;
136	return self.bits;
137	}
138
139	const keep = self.count - num;
140	const bits = switch (endian) {
141	.big => self.bits >> @intCast(keep),
142	.little => self.bits & low_bit_mask[num],
143	};
144	switch (endian) {
145	.big => self.bits &= low_bit_mask[keep],
146	.little => self.bits >>= @intCast(num),
147	}
148
149	self.count = keep;
150	return bits;
151	}
152
153	pub fn alignToByte(self: *@This()) void {
154	self.bits = 0;
155	self.count = 0;
156	}
157	};
158	}
159
160	pub fn bitReader(comptime endian: std.builtin.Endian, reader: anytype) BitReader(endian, @TypeOf(reader)) {
161	return .{ .reader = reader };
162	}
163
164	///////////////////////////////
165
166	test "api coverage" {
167	const mem_be = [_]u8{ 0b11001101, 0b00001011 };
168	const mem_le = [_]u8{ 0b00011101, 0b10010101 };
169
170	var mem_in_be = std.io.fixedBufferStream(&mem_be);
171	var bit_stream_be = bitReader(.big, mem_in_be.reader());
172
173	var out_bits: u16 = undefined;
174
175	const expect = std.testing.expect;
176	const expectError = std.testing.expectError;
177
178	try expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits));
179	try expect(out_bits == 1);
180	try expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits));
181	try expect(out_bits == 2);
182	try expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits));
183	try expect(out_bits == 3);
184	try expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits));
185	try expect(out_bits == 4);
186	try expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits));
187	try expect(out_bits == 5);
188	try expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits));
189	try expect(out_bits == 1);
190
191	mem_in_be.pos = 0;
192	bit_stream_be.count = 0;
193	try expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits));
194	try expect(out_bits == 15);
195
196	mem_in_be.pos = 0;
197	bit_stream_be.count = 0;
198	try expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits));
199	try expect(out_bits == 16);
200
201	_ = try bit_stream_be.readBits(u0, 0, &out_bits);
202
203	try expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits));
204	try expect(out_bits == 0);
205	try expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1));
206
207	var mem_in_le = std.io.fixedBufferStream(&mem_le);
208	var bit_stream_le = bitReader(.little, mem_in_le.reader());
209
210	try expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits));
211	try expect(out_bits == 1);
212	try expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits));
213	try expect(out_bits == 2);
214	try expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits));
215	try expect(out_bits == 3);
216	try expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits));
217	try expect(out_bits == 4);
218	try expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits));
219	try expect(out_bits == 5);
220	try expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits));
221	try expect(out_bits == 1);
222
223	mem_in_le.pos = 0;
224	bit_stream_le.count = 0;
225	try expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits));
226	try expect(out_bits == 15);
227
228	mem_in_le.pos = 0;
229	bit_stream_le.count = 0;
230	try expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits));
231	try expect(out_bits == 16);
232
233	_ = try bit_stream_le.readBits(u0, 0, &out_bits);
234
235	try expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits));
236	try expect(out_bits == 0);
237	try expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1));
238	}

lib/std/Io/bit_writer.zig deleted-179

...	@@ -1,179 +0,0 @@
1	const std = @import("../std.zig");
2
3	//General note on endianess:
4	//Big endian is packed starting in the most significant part of the byte and subsequent
5	// bytes contain less significant bits. Thus we write out bits from the high end
6	// of our input first.
7	//Little endian is packed starting in the least significant part of the byte and
8	// subsequent bytes contain more significant bits. Thus we write out bits from
9	// the low end of our input first.
10	//Regardless of endianess, within any given byte the bits are always in most
11	// to least significant order.
12	//Also regardless of endianess, the buffer always aligns bits to the low end
13	// of the byte.
14
15	/// Creates a bit writer which allows for writing bits to an underlying standard writer
16	pub fn BitWriter(comptime endian: std.builtin.Endian, comptime Writer: type) type {
17	return struct {
18	writer: Writer,
19	bits: u8 = 0,
20	count: u4 = 0,
21
22	const low_bit_mask = [9]u8{
23	0b00000000,
24	0b00000001,
25	0b00000011,
26	0b00000111,
27	0b00001111,
28	0b00011111,
29	0b00111111,
30	0b01111111,
31	0b11111111,
32	};
33
34	/// Write the specified number of bits to the writer from the least significant bits of
35	/// the specified value. Bits will only be written to the writer when there
36	/// are enough to fill a byte.
37	pub fn writeBits(self: *@This(), value: anytype, num: u16) !void {
38	const T = @TypeOf(value);
39	const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
40	const U = if (@bitSizeOf(T) < 8) u8 else UT; //<u8 is a pain to work with
41
42	var in: U = @as(UT, @bitCast(value));
43	var in_count: u16 = num;
44
45	if (self.count > 0) {
46	//if we can't fill the buffer, add what we have
47	const bits_free = 8 - self.count;
48	if (num < bits_free) {
49	self.addBits(@truncate(in), @intCast(num));
50	return;
51	}
52
53	//finish filling the buffer and flush it
54	if (num == bits_free) {
55	self.addBits(@truncate(in), @intCast(num));
56	return self.flushBits();
57	}
58
59	switch (endian) {
60	.big => {
61	const bits = in >> @intCast(in_count - bits_free);
62	self.addBits(@truncate(bits), bits_free);
63	},
64	.little => {
65	self.addBits(@truncate(in), bits_free);
66	in >>= @intCast(bits_free);
67	},
68	}
69	in_count -= bits_free;
70	try self.flushBits();
71	}
72
73	//write full bytes while we can
74	const full_bytes_left = in_count / 8;
75	for (0..full_bytes_left) \|_\| {
76	switch (endian) {
77	.big => {
78	const bits = in >> @intCast(in_count - 8);
79	try self.writer.writeByte(@truncate(bits));
80	},
81	.little => {
82	try self.writer.writeByte(@truncate(in));
83	if (U == u8) in = 0 else in >>= 8;
84	},
85	}
86	in_count -= 8;
87	}
88
89	//save the remaining bits in the buffer
90	self.addBits(@truncate(in), @intCast(in_count));
91	}
92
93	//convenience funciton for adding bits to the buffer
94	//in the appropriate position based on endianess
95	fn addBits(self: *@This(), bits: u8, num: u4) void {
96	if (num == 8) self.bits = bits else switch (endian) {
97	.big => {
98	self.bits <<= @intCast(num);
99	self.bits \|= bits & low_bit_mask[num];
100	},
101	.little => {
102	const pos = bits << @intCast(self.count);
103	self.bits \|= pos;
104	},
105	}
106	self.count += num;
107	}
108
109	/// Flush any remaining bits to the writer, filling
110	/// unused bits with 0s.
111	pub fn flushBits(self: *@This()) !void {
112	if (self.count == 0) return;
113	if (endian == .big) self.bits <<= @intCast(8 - self.count);
114	try self.writer.writeByte(self.bits);
115	self.bits = 0;
116	self.count = 0;
117	}
118	};
119	}
120
121	pub fn bitWriter(comptime endian: std.builtin.Endian, writer: anytype) BitWriter(endian, @TypeOf(writer)) {
122	return .{ .writer = writer };
123	}
124
125	///////////////////////////////
126
127	test "api coverage" {
128	var mem_be = [_]u8{0} ** 2;
129	var mem_le = [_]u8{0} ** 2;
130
131	var mem_out_be = std.io.fixedBufferStream(&mem_be);
132	var bit_stream_be = bitWriter(.big, mem_out_be.writer());
133
134	const testing = std.testing;
135
136	try bit_stream_be.writeBits(@as(u2, 1), 1);
137	try bit_stream_be.writeBits(@as(u5, 2), 2);
138	try bit_stream_be.writeBits(@as(u128, 3), 3);
139	try bit_stream_be.writeBits(@as(u8, 4), 4);
140	try bit_stream_be.writeBits(@as(u9, 5), 5);
141	try bit_stream_be.writeBits(@as(u1, 1), 1);
142
143	try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011);
144
145	mem_out_be.pos = 0;
146
147	try bit_stream_be.writeBits(@as(u15, 0b110011010000101), 15);
148	try bit_stream_be.flushBits();
149	try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010);
150
151	mem_out_be.pos = 0;
152	try bit_stream_be.writeBits(@as(u32, 0b110011010000101), 16);
153	try testing.expect(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101);
154
155	try bit_stream_be.writeBits(@as(u0, 0), 0);
156
157	var mem_out_le = std.io.fixedBufferStream(&mem_le);
158	var bit_stream_le = bitWriter(.little, mem_out_le.writer());
159
160	try bit_stream_le.writeBits(@as(u2, 1), 1);
161	try bit_stream_le.writeBits(@as(u5, 2), 2);
162	try bit_stream_le.writeBits(@as(u128, 3), 3);
163	try bit_stream_le.writeBits(@as(u8, 4), 4);
164	try bit_stream_le.writeBits(@as(u9, 5), 5);
165	try bit_stream_le.writeBits(@as(u1, 1), 1);
166
167	try testing.expect(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101);
168
169	mem_out_le.pos = 0;
170	try bit_stream_le.writeBits(@as(u15, 0b110011010000101), 15);
171	try bit_stream_le.flushBits();
172	try testing.expect(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110);
173
174	mem_out_le.pos = 0;
175	try bit_stream_le.writeBits(@as(u32, 0b1100110100001011), 16);
176	try testing.expect(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101);
177
178	try bit_stream_le.writeBits(@as(u0, 0), 0);
179	}

lib/std/Io/fixed_buffer_stream.zig+1-16

 @@ -4,8 +4,7 @@ const testing = std.testing;
 const mem = std.mem;
 const assert = std.debug.assert;
-/// This turns a byte buffer into an `io.GenericWriter`, `io.GenericReader`, or `io.SeekableStream`.
+/// Deprecated in favor of `std.Io.Reader.fixed` and `std.Io.Writer.fixed`.
-/// If the supplied byte buffer is const, then `io.GenericWriter` is not available.
 pub fn FixedBufferStream(comptime Buffer: type) type {
     return struct {
         /// `Buffer` is either a `[]u8` or `[]const u8`.
 @@ -20,16 +19,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type {
         pub const Reader = io.GenericReader(*Self, ReadError, read);
         pub const Writer = io.GenericWriter(*Self, WriteError, write);
-        pub const SeekableStream = io.SeekableStream(
-            *Self,
-            SeekError,
-            GetSeekPosError,
-            seekTo,
-            seekBy,
-            getPos,
-            getEndPos,
-        );
         const Self = @This();
         pub fn reader(self: *Self) Reader {
 @@ -40,10 +29,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type {
             return .{ .context = self };
         }
-        pub fn seekableStream(self: *Self) SeekableStream {
-            return .{ .context = self };
-        }
         pub fn read(self: *Self, dest: []u8) ReadError!usize {
             const size = @min(dest.len, self.buffer.len - self.pos);
             const end = self.pos + size;

lib/std/Io/seekable_stream.zig deleted-35

...	@@ -1,35 +0,0 @@
1	const std = @import("../std.zig");
2
3	pub fn SeekableStream(
4	comptime Context: type,
5	comptime SeekErrorType: type,
6	comptime GetSeekPosErrorType: type,
7	comptime seekToFn: fn (context: Context, pos: u64) SeekErrorType!void,
8	comptime seekByFn: fn (context: Context, pos: i64) SeekErrorType!void,
9	comptime getPosFn: fn (context: Context) GetSeekPosErrorType!u64,
10	comptime getEndPosFn: fn (context: Context) GetSeekPosErrorType!u64,
11	) type {
12	return struct {
13	context: Context,
14
15	const Self = @This();
16	pub const SeekError = SeekErrorType;
17	pub const GetSeekPosError = GetSeekPosErrorType;
18
19	pub fn seekTo(self: Self, pos: u64) SeekError!void {
20	return seekToFn(self.context, pos);
21	}
22
23	pub fn seekBy(self: Self, amt: i64) SeekError!void {
24	return seekByFn(self.context, amt);
25	}
26
27	pub fn getEndPos(self: Self) GetSeekPosError!u64 {
28	return getEndPosFn(self.context);
29	}
30
31	pub fn getPos(self: Self) GetSeekPosError!u64 {
32	return getPosFn(self.context);
33	}
34	};
35	}

lib/std/Io/test.zig-45

...	@@ -57,51 +57,6 @@ test "write a file, read it, then delete it" {	...	@@ -57,51 +57,6 @@ test "write a file, read it, then delete it" {
57	try tmp.dir.deleteFile(tmp_file_name);	57	try tmp.dir.deleteFile(tmp_file_name);
58	}	58	}
59		59
60	test "BitStreams with File Stream" {
61	var tmp = tmpDir(.{});
62	defer tmp.cleanup();
63
64	const tmp_file_name = "temp_test_file.txt";
65	{
66	var file = try tmp.dir.createFile(tmp_file_name, .{});
67	defer file.close();
68
69	var bit_stream = io.bitWriter(native_endian, file.deprecatedWriter());
70
71	try bit_stream.writeBits(@as(u2, 1), 1);
72	try bit_stream.writeBits(@as(u5, 2), 2);
73	try bit_stream.writeBits(@as(u128, 3), 3);
74	try bit_stream.writeBits(@as(u8, 4), 4);
75	try bit_stream.writeBits(@as(u9, 5), 5);
76	try bit_stream.writeBits(@as(u1, 1), 1);
77	try bit_stream.flushBits();
78	}
79	{
80	var file = try tmp.dir.openFile(tmp_file_name, .{});
81	defer file.close();
82
83	var bit_stream = io.bitReader(native_endian, file.deprecatedReader());
84
85	var out_bits: u16 = undefined;
86
87	try expect(1 == try bit_stream.readBits(u2, 1, &out_bits));
88	try expect(out_bits == 1);
89	try expect(2 == try bit_stream.readBits(u5, 2, &out_bits));
90	try expect(out_bits == 2);
91	try expect(3 == try bit_stream.readBits(u128, 3, &out_bits));
92	try expect(out_bits == 3);
93	try expect(4 == try bit_stream.readBits(u8, 4, &out_bits));
94	try expect(out_bits == 4);
95	try expect(5 == try bit_stream.readBits(u9, 5, &out_bits));
96	try expect(out_bits == 5);
97	try expect(1 == try bit_stream.readBits(u1, 1, &out_bits));
98	try expect(out_bits == 1);
99
100	try expectError(error.EndOfStream, bit_stream.readBitsNoEof(u1, 1));
101	}
102	try tmp.dir.deleteFile(tmp_file_name);
103	}
104
105	test "File seek ops" {	60	test "File seek ops" {
106	var tmp = tmpDir(.{});	61	var tmp = tmpDir(.{});
107	defer tmp.cleanup();	62	defer tmp.cleanup();

lib/std/compress.zig+1-4

...	@@ -1,8 +1,7 @@	...	@@ -1,8 +1,7 @@
1	//! Compression algorithms.	1	//! Compression algorithms.
2		2
		3	/// gzip and zlib are here.
3	pub const flate = @import("compress/flate.zig");	4	pub const flate = @import("compress/flate.zig");
4	pub const gzip = @import("compress/gzip.zig");
5	pub const zlib = @import("compress/zlib.zig");
6	pub const lzma = @import("compress/lzma.zig");	5	pub const lzma = @import("compress/lzma.zig");
7	pub const lzma2 = @import("compress/lzma2.zig");	6	pub const lzma2 = @import("compress/lzma2.zig");
8	pub const xz = @import("compress/xz.zig");	7	pub const xz = @import("compress/xz.zig");
...	@@ -14,6 +13,4 @@ test {	...	@@ -14,6 +13,4 @@ test {
14	_ = lzma2;	13	_ = lzma2;
15	_ = xz;	14	_ = xz;
16	_ = zstd;	15	_ = zstd;
17	_ = gzip;
18	_ = zlib;
19	}	16	}

lib/std/compress/flate.zig+151-448

...	@@ -1,477 +1,180 @@	...	@@ -1,477 +1,180 @@
1	/// Deflate is a lossless data compression file format that uses a combination	1	const std = @import("../std.zig");
2	/// of LZ77 and Huffman coding.
3	pub const deflate = @import("flate/deflate.zig");
4
5	/// Inflate is the decoding process that takes a Deflate bitstream for
6	/// decompression and correctly produces the original full-size data or file.
7	pub const inflate = @import("flate/inflate.zig");
8
9	/// Decompress compressed data from reader and write plain data to the writer.
10	pub fn decompress(reader: anytype, writer: anytype) !void {
11	try inflate.decompress(.raw, reader, writer);
12	}
13
14	/// Decompressor type
15	pub fn Decompressor(comptime ReaderType: type) type {
16	return inflate.Decompressor(.raw, ReaderType);
17	}
18
19	/// Create Decompressor which will read compressed data from reader.
20	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21	return inflate.decompressor(.raw, reader);
22	}
23
24	/// Compression level, trades between speed and compression size.
25	pub const Options = deflate.Options;
26		2
27	/// Compress plain data from reader and write compressed data to the writer.	3	/// When decompressing, the output buffer is used as the history window, so
28	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {	4	/// less than this may result in failure to decompress streams that were
29	try deflate.compress(.raw, reader, writer, options);	5	/// compressed with a larger window.
30	}	6	pub const max_window_len = history_len * 2;
31		7
32	/// Compressor type	8	pub const history_len = 32768;
33	pub fn Compressor(comptime WriterType: type) type {
34	return deflate.Compressor(.raw, WriterType);
35	}
36		9
37	/// Create Compressor which outputs compressed data to the writer.	10	/// Deflate is a lossless data compression file format that uses a combination
38	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {	11	/// of LZ77 and Huffman coding.
39	return try deflate.compressor(.raw, writer, options);	12	pub const Compress = @import("flate/Compress.zig");
40	}	13
41		14	/// Inflate is the decoding process that consumes a Deflate bitstream and
42	/// Huffman only compression. Without Lempel-Ziv match searching. Faster	15	/// produces the original full-size data.
43	/// compression, less memory requirements but bigger compressed sizes.	16	pub const Decompress = @import("flate/Decompress.zig");
44	pub const huffman = struct {	17
45	pub fn compress(reader: anytype, writer: anytype) !void {	18	/// Compression without Lempel-Ziv match searching. Faster compression, less
46	try deflate.huffman.compress(.raw, reader, writer);	19	/// memory requirements but bigger compressed sizes.
		20	pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
		21
		22	/// Container of the deflate bit stream body. Container adds header before
		23	/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
		24	/// no footer, raw bit stream).
		25	///
		26	/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
		27	/// addler 32 checksum.
		28	///
		29	/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
		30	/// crc32 checksum and 4 bytes of uncompressed data length.
		31	///
		32	/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
		33	/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
		34	pub const Container = enum {
		35	raw, // no header or footer
		36	gzip, // gzip header and footer
		37	zlib, // zlib header and footer
		38
		39	pub fn size(w: Container) usize {
		40	return headerSize(w) + footerSize(w);
47	}	41	}
48		42
49	pub fn Compressor(comptime WriterType: type) type {	43	pub fn headerSize(w: Container) usize {
50	return deflate.huffman.Compressor(.raw, WriterType);	44	return header(w).len;
51	}	45	}
52		46
53	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {	47	pub fn footerSize(w: Container) usize {
54	return deflate.huffman.compressor(.raw, writer);	48	return switch (w) {
		49	.gzip => 8,
		50	.zlib => 4,
		51	.raw => 0,
		52	};
55	}	53	}
56	};
57		54
58	// No compression store only. Compressed size is slightly bigger than plain.	55	pub const list = [_]Container{ .raw, .gzip, .zlib };
59	pub const store = struct {
60	pub fn compress(reader: anytype, writer: anytype) !void {
61	try deflate.store.compress(.raw, reader, writer);
62	}
63		56
64	pub fn Compressor(comptime WriterType: type) type {	57	pub const Error = error{
65	return deflate.store.Compressor(.raw, WriterType);	58	BadGzipHeader,
66	}	59	BadZlibHeader,
		60	WrongGzipChecksum,
		61	WrongGzipSize,
		62	WrongZlibChecksum,
		63	};
67		64
68	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {	65	pub fn header(container: Container) []const u8 {
69	return deflate.store.compressor(.raw, writer);	66	return switch (container) {
		67	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
		68	// - ID1 (IDentification 1), always 0x1f
		69	// - ID2 (IDentification 2), always 0x8b
		70	// - CM (Compression Method), always 8 = deflate
		71	// - FLG (Flags), all set to 0
		72	// - 4 bytes, MTIME (Modification time), not used, all set to zero
		73	// - XFL (eXtra FLags), all set to zero
		74	// - OS (Operating System), 03 = Unix
		75	.gzip => &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 },
		76	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
		77	// 1st byte:
		78	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
		79	// - The next four bits is the CM (compression method), which is 8 for deflate.
		80	// 2nd byte:
		81	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
		82	// - The next bit, FDICT, is set if a dictionary is given.
		83	// - The final five FCHECK bits form a mod-31 checksum.
		84	//
		85	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
		86	.zlib => &[_]u8{ 0x78, 0b10_0_11100 },
		87	.raw => &.{},
		88	};
70	}	89	}
71	};
72
73	/// Container defines header/footer around deflate bit stream. Gzip and zlib
74	/// compression algorithms are containers around deflate bit stream body.
75	const Container = @import("flate/container.zig").Container;
76	const std = @import("std");
77	const testing = std.testing;
78	const fixedBufferStream = std.io.fixedBufferStream;
79	const print = std.debug.print;
80	const builtin = @import("builtin");
81
82	test {
83	_ = deflate;
84	_ = inflate;
85	}
86		90
87	test "compress/decompress" {	91	pub const Hasher = union(Container) {
88	var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer	92	raw: void,
89	var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer	93	gzip: struct {
90		94	crc: std.hash.Crc32 = .init(),
91	const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };	95	count: u32 = 0,
92	const cases = [_]struct {
93	data: []const u8, // uncompressed content
94	// compressed data sizes per level 4-9
95	gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
96	huffman_only_size: usize = 0,
97	store_size: usize = 0,
98	}{
99	.{
100	.data = @embedFile("flate/testdata/rfc1951.txt"),
101	.gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
102	.huffman_only_size = 20287,
103	.store_size = 36967,
104	},
105	.{
106	.data = @embedFile("flate/testdata/fuzz/roundtrip1.input"),
107	.gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
108	.huffman_only_size = 393,
109	.store_size = 393,
110	},
111	.{
112	.data = @embedFile("flate/testdata/fuzz/roundtrip2.input"),
113	.gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
114	.huffman_only_size = 394,
115	.store_size = 394,
116	},	96	},
117	.{	97	zlib: std.hash.Adler32,
118	.data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"),	98
119	.gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },	99	pub fn init(containter: Container) Hasher {
120	.huffman_only_size = 498,	100	return switch (containter) {
121	.store_size = 747,	101	.gzip => .{ .gzip = .{} },
122	},	102	.zlib => .{ .zlib = .{} },
123	};	103	.raw => .raw,
124		104	};
125	for (cases, 0..) \|case, case_no\| { // for each case
126	const data = case.data;
127
128	for (levels, 0..) \|level, i\| { // for each compression level
129
130	inline for (Container.list) \|container\| { // for each wrapping
131	var compressed_size: usize = if (case.gzip_sizes[i] > 0)
132	case.gzip_sizes[i] - Container.gzip.size() + container.size()
133	else
134	0;
135
136	// compress original stream to compressed stream
137	{
138	var original = fixedBufferStream(data);
139	var compressed = fixedBufferStream(&cmp_buf);
140	try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });
141	if (compressed_size == 0) {
142	if (container == .gzip)
143	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
144	compressed_size = compressed.pos;
145	}
146	try testing.expectEqual(compressed_size, compressed.pos);
147	}
148	// decompress compressed stream to decompressed stream
149	{
150	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
151	var decompressed = fixedBufferStream(&dcm_buf);
152	try inflate.decompress(container, compressed.reader(), decompressed.writer());
153	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
154	}
155
156	// compressor writer interface
157	{
158	var compressed = fixedBufferStream(&cmp_buf);
159	var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });
160	var cmp_wrt = cmp.writer();
161	try cmp_wrt.writeAll(data);
162	try cmp.finish();
163
164	try testing.expectEqual(compressed_size, compressed.pos);
165	}
166	// decompressor reader interface
167	{
168	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
169	var dcm = inflate.decompressor(container, compressed.reader());
170	var dcm_rdr = dcm.reader();
171	const n = try dcm_rdr.readAll(&dcm_buf);
172	try testing.expectEqual(data.len, n);
173	try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
174	}
175	}
176	}	105	}
177	// huffman only compression
178	{
179	inline for (Container.list) \|container\| { // for each wrapping
180	var compressed_size: usize = if (case.huffman_only_size > 0)
181	case.huffman_only_size - Container.gzip.size() + container.size()
182	else
183	0;
184		106
185	// compress original stream to compressed stream	107	pub fn container(h: Hasher) Container {
186	{	108	return h;
187	var original = fixedBufferStream(data);
188	var compressed = fixedBufferStream(&cmp_buf);
189	var cmp = try deflate.huffman.compressor(container, compressed.writer());
190	try cmp.compress(original.reader());
191	try cmp.finish();
192	if (compressed_size == 0) {
193	if (container == .gzip)
194	print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
195	compressed_size = compressed.pos;
196	}
197	try testing.expectEqual(compressed_size, compressed.pos);
198	}
199	// decompress compressed stream to decompressed stream
200	{
201	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
202	var decompressed = fixedBufferStream(&dcm_buf);
203	try inflate.decompress(container, compressed.reader(), decompressed.writer());
204	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
205	}
206	}
207	}	109	}
208		110
209	// store only	111	pub fn update(h: *Hasher, buf: []const u8) void {
210	{	112	switch (h.*) {
211	inline for (Container.list) \|container\| { // for each wrapping	113	.raw => {},
212	var compressed_size: usize = if (case.store_size > 0)	114	.gzip => \|*gzip\| {
213	case.store_size - Container.gzip.size() + container.size()	115	gzip.update(buf);
214	else	116	gzip.count +%= buf.len;
215	0;	117	},
216		118	.zlib => \|*zlib\| {
217	// compress original stream to compressed stream	119	zlib.update(buf);
218	{	120	},
219	var original = fixedBufferStream(data);	121	inline .gzip, .zlib => \|*x\| x.update(buf),
220	var compressed = fixedBufferStream(&cmp_buf);
221	var cmp = try deflate.store.compressor(container, compressed.writer());
222	try cmp.compress(original.reader());
223	try cmp.finish();
224	if (compressed_size == 0) {
225	if (container == .gzip)
226	print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
227	compressed_size = compressed.pos;
228	}
229
230	try testing.expectEqual(compressed_size, compressed.pos);
231	}
232	// decompress compressed stream to decompressed stream
233	{
234	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
235	var decompressed = fixedBufferStream(&dcm_buf);
236	try inflate.decompress(container, compressed.reader(), decompressed.writer());
237	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
238	}
239	}	122	}
240	}	123	}
241	}
242	}
243
244	fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void {
245	var in = fixedBufferStream(compressed);
246	var out = std.ArrayList(u8).init(testing.allocator);
247	defer out.deinit();
248
249	try inflate.decompress(container, in.reader(), out.writer());
250	try testing.expectEqualSlices(u8, expected_plain, out.items);
251	}
252
253	test "don't read past deflate stream's end" {
254	try testDecompress(.zlib, &[_]u8{
255	0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
256	0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
257	0x83, 0x95, 0x0b, 0xf5,
258	}, &[_]u8{
259	0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
260	0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
261	0x00, 0x00, 0xff, 0xff, 0xff,
262	});
263	}
264
265	test "zlib header" {
266	// Truncated header
267	try testing.expectError(
268	error.EndOfStream,
269	testDecompress(.zlib, &[_]u8{0x78}, ""),
270	);
271	// Wrong CM
272	try testing.expectError(
273	error.BadZlibHeader,
274	testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
275	);
276	// Wrong CINFO
277	try testing.expectError(
278	error.BadZlibHeader,
279	testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
280	);
281	// Wrong checksum
282	try testing.expectError(
283	error.WrongZlibChecksum,
284	testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
285	);
286	// Truncated checksum
287	try testing.expectError(
288	error.EndOfStream,
289	testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
290	);
291	}
292
293	test "gzip header" {
294	// Truncated header
295	try testing.expectError(
296	error.EndOfStream,
297	testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
298	);
299	// Wrong CM
300	try testing.expectError(
301	error.BadGzipHeader,
302	testDecompress(.gzip, &[_]u8{
303	0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
304	0x00, 0x03,
305	}, undefined),
306	);
307
308	// Wrong checksum
309	try testing.expectError(
310	error.WrongGzipChecksum,
311	testDecompress(.gzip, &[_]u8{
312	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
313	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
314	0x00, 0x00, 0x00, 0x00,
315	}, undefined),
316	);
317	// Truncated checksum
318	try testing.expectError(
319	error.EndOfStream,
320	testDecompress(.gzip, &[_]u8{
321	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
322	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
323	}, undefined),
324	);
325	// Wrong initial size
326	try testing.expectError(
327	error.WrongGzipSize,
328	testDecompress(.gzip, &[_]u8{
329	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
330	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
331	0x00, 0x00, 0x00, 0x01,
332	}, undefined),
333	);
334	// Truncated initial size field
335	try testing.expectError(
336	error.EndOfStream,
337	testDecompress(.gzip, &[_]u8{
338	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
339	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
340	0x00, 0x00, 0x00,
341	}, undefined),
342	);
343
344	try testDecompress(.gzip, &[_]u8{
345	// GZIP header
346	0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
347	// header.FHCRC (should cover entire header)
348	0x99, 0xd6,
349	// GZIP data
350	0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
351	}, "");
352	}
353
354	test "public interface" {
355	const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
356
357	// deflate final stored block, header + plain (stored) data
358	const deflate_block = [_]u8{
359	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
360	} ++ plain_data;
361
362	// gzip header/footer + deflate block
363	const gzip_data =
364	[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
365	deflate_block ++
366	[_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
367
368	// zlib header/footer + deflate block
369	const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
370	deflate_block ++
371	[_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
372
373	const gzip = @import("gzip.zig");
374	const zlib = @import("zlib.zig");
375	const flate = @This();
376
377	try testInterface(gzip, &gzip_data, &plain_data);
378	try testInterface(zlib, &zlib_data, &plain_data);
379	try testInterface(flate, &deflate_block, &plain_data);
380	}
381		124
382	fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void {	125	pub fn writeFooter(hasher: Hasher, writer: std.Io.Writer) std.Io.Writer.Error!void {
383	var buffer1: [64]u8 = undefined;	126	var bits: [4]u8 = undefined;
384	var buffer2: [64]u8 = undefined;	127	switch (hasher.*) {
385		128	.gzip => \|*gzip\| {
386	var compressed = fixedBufferStream(&buffer1);	129	// GZIP 8 bytes footer
387	var plain = fixedBufferStream(&buffer2);	130	// - 4 bytes, CRC32 (CRC-32)
388		131	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
389	// decompress	132	std.mem.writeInt(u32, &bits, gzip.final(), .little);
390	{	133	try writer.writeAll(&bits);
391	var in = fixedBufferStream(gzip_data);	134
392	try pkg.decompress(in.reader(), plain.writer());	135	std.mem.writeInt(u32, &bits, gzip.bytes_read, .little);
393	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	136	try writer.writeAll(&bits);
394	}	137	},
395	plain.reset();	138	.zlib => \|*zlib\| {
396	compressed.reset();	139	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
397		140	// 4 bytes of ADLER32 (Adler-32 checksum)
398	// compress/decompress	141	// Checksum value of the uncompressed data (excluding any
399	{	142	// dictionary data) computed according to Adler-32
400	var in = fixedBufferStream(plain_data);	143	// algorithm.
401	try pkg.compress(in.reader(), compressed.writer(), .{});	144	std.mem.writeInt(u32, &bits, zlib.final, .big);
402	compressed.reset();	145	try writer.writeAll(&bits);
403	try pkg.decompress(compressed.reader(), plain.writer());	146	},
404	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	147	.raw => {},
405	}	148	}
406	plain.reset();
407	compressed.reset();
408
409	// compressor/decompressor
410	{
411	var in = fixedBufferStream(plain_data);
412	var cmp = try pkg.compressor(compressed.writer(), .{});
413	try cmp.compress(in.reader());
414	try cmp.finish();
415
416	compressed.reset();
417	var dcp = pkg.decompressor(compressed.reader());
418	try dcp.decompress(plain.writer());
419	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
420	}
421	plain.reset();
422	compressed.reset();
423
424	// huffman
425	{
426	// huffman compress/decompress
427	{
428	var in = fixedBufferStream(plain_data);
429	try pkg.huffman.compress(in.reader(), compressed.writer());
430	compressed.reset();
431	try pkg.decompress(compressed.reader(), plain.writer());
432	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
433	}	149	}
434	plain.reset();	150	};
435	compressed.reset();
436		151
437	// huffman compressor/decompressor	152	pub const Metadata = union(Container) {
438	{	153	raw: void,
439	var in = fixedBufferStream(plain_data);	154	gzip: struct {
440	var cmp = try pkg.huffman.compressor(compressed.writer());	155	crc: u32 = 0,
441	try cmp.compress(in.reader());	156	count: u32 = 0,
442	try cmp.finish();	157	},
		158	zlib: struct {
		159	adler: u32 = 0,
		160	},
443		161
444	compressed.reset();	162	pub fn init(containter: Container) Metadata {
445	try pkg.decompress(compressed.reader(), plain.writer());	163	return switch (containter) {
446	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	164	.gzip => .{ .gzip = .{} },
		165	.zlib => .{ .zlib = .{} },
		166	.raw => .raw,
		167	};
447	}	168	}
448	}
449	plain.reset();
450	compressed.reset();
451		169
452	// store	170	pub fn container(m: Metadata) Container {
453	{	171	return m;
454	// store compress/decompress
455	{
456	var in = fixedBufferStream(plain_data);
457	try pkg.store.compress(in.reader(), compressed.writer());
458	compressed.reset();
459	try pkg.decompress(compressed.reader(), plain.writer());
460	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
461	}	172	}
462	plain.reset();	173	};
463	compressed.reset();	174	};
464
465	// store compressor/decompressor
466	{
467	var in = fixedBufferStream(plain_data);
468	var cmp = try pkg.store.compressor(compressed.writer());
469	try cmp.compress(in.reader());
470	try cmp.finish();
471		175
472	compressed.reset();	176	test {
473	try pkg.decompress(compressed.reader(), plain.writer());	177	_ = HuffmanEncoder;
474	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	178	_ = Compress;
475	}	179	_ = Decompress;
476	}
477	}	180	}

lib/std/compress/flate/BlockWriter.zig created+592

...	@@ -0,0 +1,592 @@
1	//! Accepts list of tokens, decides what is best block type to write. What block
2	//! type will provide best compression. Writes header and body of the block.
3	const std = @import("std");
4	const io = std.io;
5	const assert = std.debug.assert;
6	const Writer = std.io.Writer;
7
8	const BlockWriter = @This();
9	const flate = @import("../flate.zig");
10	const Compress = flate.Compress;
11	const HuffmanEncoder = flate.HuffmanEncoder;
12	const Token = @import("Token.zig");
13
14	const codegen_order = HuffmanEncoder.codegen_order;
15	const end_code_mark = 255;
16
17	output: *Writer,
18
19	codegen_freq: [HuffmanEncoder.codegen_code_count]u16,
20	literal_freq: [HuffmanEncoder.max_num_lit]u16,
21	distance_freq: [HuffmanEncoder.distance_code_count]u16,
22	codegen: [HuffmanEncoder.max_num_lit + HuffmanEncoder.distance_code_count + 1]u8,
23	literal_encoding: HuffmanEncoder,
24	distance_encoding: HuffmanEncoder,
25	codegen_encoding: HuffmanEncoder,
26	fixed_literal_encoding: HuffmanEncoder,
27	fixed_distance_encoding: HuffmanEncoder,
28	huff_distance: HuffmanEncoder,
29
30	fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,
31	fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
32	distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
33
34	pub fn init(output: *Writer) BlockWriter {
35	return .{
36	.output = output,
37	.codegen_freq = undefined,
38	.literal_freq = undefined,
39	.distance_freq = undefined,
40	.codegen = undefined,
41	.literal_encoding = undefined,
42	.distance_encoding = undefined,
43	.codegen_encoding = undefined,
44	.fixed_literal_encoding = undefined,
45	.fixed_distance_encoding = undefined,
46	.huff_distance = undefined,
47	.fixed_literal_codes = undefined,
48	.fixed_distance_codes = undefined,
49	.distance_codes = undefined,
50	};
51	}
52
53	pub fn initBuffers(bw: *BlockWriter) void {
54	bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);
55	bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);
56	bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);
57	}
58
59	/// Flush intrenal bit buffer to the writer.
60	/// Should be called only when bit stream is at byte boundary.
61	///
62	/// That is after final block; when last byte could be incomplete or
63	/// after stored block; which is aligned to the byte boundary (it has x
64	/// padding bits after first 3 bits).
65	pub fn flush(self: *BlockWriter) Writer.Error!void {
66	try self.bit_writer.flush();
67	}
68
69	fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void {
70	try self.bit_writer.writeBits(c.code, c.len);
71	}
72
73	/// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
74	/// the literal and distance lengths arrays (which are concatenated into a single
75	/// array). This method generates that run-length encoding.
76	///
77	/// The result is written into the codegen array, and the frequencies
78	/// of each code is written into the codegen_freq array.
79	/// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
80	/// information. Code bad_code is an end marker
81	///
82	/// num_literals: The number of literals in literal_encoding
83	/// num_distances: The number of distances in distance_encoding
84	/// lit_enc: The literal encoder to use
85	/// dist_enc: The distance encoder to use
86	fn generateCodegen(
87	self: *BlockWriter,
88	num_literals: u32,
89	num_distances: u32,
90	lit_enc: *Compress.LiteralEncoder,
91	dist_enc: *Compress.DistanceEncoder,
92	) void {
93	for (self.codegen_freq, 0..) \|_, i\| {
94	self.codegen_freq[i] = 0;
95	}
96
97	// Note that we are using codegen both as a temporary variable for holding
98	// a copy of the frequencies, and as the place where we put the result.
99	// This is fine because the output is always shorter than the input used
100	// so far.
101	var codegen = &self.codegen; // cache
102	// Copy the concatenated code sizes to codegen. Put a marker at the end.
103	var cgnl = codegen[0..num_literals];
104	for (cgnl, 0..) \|_, i\| {
105	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
106	}
107
108	cgnl = codegen[num_literals .. num_literals + num_distances];
109	for (cgnl, 0..) \|_, i\| {
110	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
111	}
112	codegen[num_literals + num_distances] = end_code_mark;
113
114	var size = codegen[0];
115	var count: i32 = 1;
116	var out_index: u32 = 0;
117	var in_index: u32 = 1;
118	while (size != end_code_mark) : (in_index += 1) {
119	// INVARIANT: We have seen "count" copies of size that have not yet
120	// had output generated for them.
121	const next_size = codegen[in_index];
122	if (next_size == size) {
123	count += 1;
124	continue;
125	}
126	// We need to generate codegen indicating "count" of size.
127	if (size != 0) {
128	codegen[out_index] = size;
129	out_index += 1;
130	self.codegen_freq[size] += 1;
131	count -= 1;
132	while (count >= 3) {
133	var n: i32 = 6;
134	if (n > count) {
135	n = count;
136	}
137	codegen[out_index] = 16;
138	out_index += 1;
139	codegen[out_index] = @as(u8, @intCast(n - 3));
140	out_index += 1;
141	self.codegen_freq[16] += 1;
142	count -= n;
143	}
144	} else {
145	while (count >= 11) {
146	var n: i32 = 138;
147	if (n > count) {
148	n = count;
149	}
150	codegen[out_index] = 18;
151	out_index += 1;
152	codegen[out_index] = @as(u8, @intCast(n - 11));
153	out_index += 1;
154	self.codegen_freq[18] += 1;
155	count -= n;
156	}
157	if (count >= 3) {
158	// 3 <= count <= 10
159	codegen[out_index] = 17;
160	out_index += 1;
161	codegen[out_index] = @as(u8, @intCast(count - 3));
162	out_index += 1;
163	self.codegen_freq[17] += 1;
164	count = 0;
165	}
166	}
167	count -= 1;
168	while (count >= 0) : (count -= 1) {
169	codegen[out_index] = size;
170	out_index += 1;
171	self.codegen_freq[size] += 1;
172	}
173	// Set up invariant for next time through the loop.
174	size = next_size;
175	count = 1;
176	}
177	// Marker indicating the end of the codegen.
178	codegen[out_index] = end_code_mark;
179	}
180
181	const DynamicSize = struct {
182	size: u32,
183	num_codegens: u32,
184	};
185
186	/// dynamicSize returns the size of dynamically encoded data in bits.
187	fn dynamicSize(
188	self: *BlockWriter,
189	lit_enc: *Compress.LiteralEncoder, // literal encoder
190	dist_enc: *Compress.DistanceEncoder, // distance encoder
191	extra_bits: u32,
192	) DynamicSize {
193	var num_codegens = self.codegen_freq.len;
194	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
195	num_codegens -= 1;
196	}
197	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
198	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
199	self.codegen_freq[16] * 2 +
200	self.codegen_freq[17] * 3 +
201	self.codegen_freq[18] * 7;
202	const size = header +
203	lit_enc.bitLength(&self.literal_freq) +
204	dist_enc.bitLength(&self.distance_freq) +
205	extra_bits;
206
207	return DynamicSize{
208	.size = @as(u32, @intCast(size)),
209	.num_codegens = @as(u32, @intCast(num_codegens)),
210	};
211	}
212
213	/// fixedSize returns the size of dynamically encoded data in bits.
214	fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 {
215	return 3 +
216	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
217	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
218	extra_bits;
219	}
220
221	const StoredSize = struct {
222	size: u32,
223	storable: bool,
224	};
225
226	/// storedSizeFits calculates the stored size, including header.
227	/// The function returns the size in bits and whether the block
228	/// fits inside a single block.
229	fn storedSizeFits(in: ?[]const u8) StoredSize {
230	if (in == null) {
231	return .{ .size = 0, .storable = false };
232	}
233	if (in.?.len <= HuffmanEncoder.max_store_block_size) {
234	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
235	}
236	return .{ .size = 0, .storable = false };
237	}
238
239	/// Write the header of a dynamic Huffman block to the output stream.
240	///
241	/// num_literals: The number of literals specified in codegen
242	/// num_distances: The number of distances specified in codegen
243	/// num_codegens: The number of codegens used in codegen
244	/// eof: Is it the end-of-file? (end of stream)
245	fn dynamicHeader(
246	self: *BlockWriter,
247	num_literals: u32,
248	num_distances: u32,
249	num_codegens: u32,
250	eof: bool,
251	) Writer.Error!void {
252	const first_bits: u32 = if (eof) 5 else 4;
253	try self.bit_writer.writeBits(first_bits, 3);
254	try self.bit_writer.writeBits(num_literals - 257, 5);
255	try self.bit_writer.writeBits(num_distances - 1, 5);
256	try self.bit_writer.writeBits(num_codegens - 4, 4);
257
258	var i: u32 = 0;
259	while (i < num_codegens) : (i += 1) {
260	const value = self.codegen_encoding.codes[codegen_order[i]].len;
261	try self.bit_writer.writeBits(value, 3);
262	}
263
264	i = 0;
265	while (true) {
266	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
267	i += 1;
268	if (code_word == end_code_mark) {
269	break;
270	}
271	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
272
273	switch (code_word) {
274	16 => {
275	try self.bit_writer.writeBits(self.codegen[i], 2);
276	i += 1;
277	},
278	17 => {
279	try self.bit_writer.writeBits(self.codegen[i], 3);
280	i += 1;
281	},
282	18 => {
283	try self.bit_writer.writeBits(self.codegen[i], 7);
284	i += 1;
285	},
286	else => {},
287	}
288	}
289	}
290
291	fn storedHeader(self: *BlockWriter, length: usize, eof: bool) Writer.Error!void {
292	assert(length <= 65535);
293	const flag: u32 = if (eof) 1 else 0;
294	try self.bit_writer.writeBits(flag, 3);
295	try self.flush();
296	const l: u16 = @intCast(length);
297	try self.bit_writer.writeBits(l, 16);
298	try self.bit_writer.writeBits(~l, 16);
299	}
300
301	fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void {
302	// Indicate that we are a fixed Huffman block
303	var value: u32 = 2;
304	if (eof) {
305	value = 3;
306	}
307	try self.bit_writer.writeBits(value, 3);
308	}
309
310	/// Write a block of tokens with the smallest encoding. Will choose block type.
311	/// The original input can be supplied, and if the huffman encoded data
312	/// is larger than the original bytes, the data will be written as a
313	/// stored block.
314	/// If the input is null, the tokens will always be Huffman encoded.
315	pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void {
316	const lit_and_dist = self.indexTokens(tokens);
317	const num_literals = lit_and_dist.num_literals;
318	const num_distances = lit_and_dist.num_distances;
319
320	var extra_bits: u32 = 0;
321	const ret = storedSizeFits(input);
322	const stored_size = ret.size;
323	const storable = ret.storable;
324
325	if (storable) {
326	// We only bother calculating the costs of the extra bits required by
327	// the length of distance fields (which will be the same for both fixed
328	// and dynamic encoding), if we need to compare those two encodings
329	// against stored encoding.
330	var length_code: u16 = Token.length_codes_start + 8;
331	while (length_code < num_literals) : (length_code += 1) {
332	// First eight length codes have extra size = 0.
333	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
334	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
335	}
336	var distance_code: u16 = 4;
337	while (distance_code < num_distances) : (distance_code += 1) {
338	// First four distance codes have extra size = 0.
339	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
340	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
341	}
342	}
343
344	// Figure out smallest code.
345	// Fixed Huffman baseline.
346	var literal_encoding = &self.fixed_literal_encoding;
347	var distance_encoding = &self.fixed_distance_encoding;
348	var size = self.fixedSize(extra_bits);
349
350	// Dynamic Huffman?
351	var num_codegens: u32 = 0;
352
353	// Generate codegen and codegenFrequencies, which indicates how to encode
354	// the literal_encoding and the distance_encoding.
355	self.generateCodegen(
356	num_literals,
357	num_distances,
358	&self.literal_encoding,
359	&self.distance_encoding,
360	);
361	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
362	const dynamic_size = self.dynamicSize(
363	&self.literal_encoding,
364	&self.distance_encoding,
365	extra_bits,
366	);
367	const dyn_size = dynamic_size.size;
368	num_codegens = dynamic_size.num_codegens;
369
370	if (dyn_size < size) {
371	size = dyn_size;
372	literal_encoding = &self.literal_encoding;
373	distance_encoding = &self.distance_encoding;
374	}
375
376	// Stored bytes?
377	if (storable and stored_size < size) {
378	try self.storedBlock(input.?, eof);
379	return;
380	}
381
382	// Huffman.
383	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
384	try self.fixedHeader(eof);
385	} else {
386	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
387	}
388
389	// Write the tokens.
390	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
391	}
392
393	pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
394	try self.storedHeader(input.len, eof);
395	try self.bit_writer.writeBytes(input);
396	}
397
398	/// writeBlockDynamic encodes a block using a dynamic Huffman table.
399	/// This should be used if the symbols used have a disproportionate
400	/// histogram distribution.
401	/// If input is supplied and the compression savings are below 1/16th of the
402	/// input size the block is stored.
403	fn dynamicBlock(
404	self: *BlockWriter,
405	tokens: []const Token,
406	eof: bool,
407	input: ?[]const u8,
408	) Writer.Error!void {
409	const total_tokens = self.indexTokens(tokens);
410	const num_literals = total_tokens.num_literals;
411	const num_distances = total_tokens.num_distances;
412
413	// Generate codegen and codegenFrequencies, which indicates how to encode
414	// the literal_encoding and the distance_encoding.
415	self.generateCodegen(
416	num_literals,
417	num_distances,
418	&self.literal_encoding,
419	&self.distance_encoding,
420	);
421	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
422	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
423	const size = dynamic_size.size;
424	const num_codegens = dynamic_size.num_codegens;
425
426	// Store bytes, if we don't get a reasonable improvement.
427
428	const stored_size = storedSizeFits(input);
429	const ssize = stored_size.size;
430	const storable = stored_size.storable;
431	if (storable and ssize < (size + (size >> 4))) {
432	try self.storedBlock(input.?, eof);
433	return;
434	}
435
436	// Write Huffman table.
437	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
438
439	// Write the tokens.
440	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
441	}
442
443	const TotalIndexedTokens = struct {
444	num_literals: u32,
445	num_distances: u32,
446	};
447
448	/// Indexes a slice of tokens followed by an end_block_marker, and updates
449	/// literal_freq and distance_freq, and generates literal_encoding
450	/// and distance_encoding.
451	/// The number of literal and distance tokens is returned.
452	fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
453	var num_literals: u32 = 0;
454	var num_distances: u32 = 0;
455
456	for (self.literal_freq, 0..) \|_, i\| {
457	self.literal_freq[i] = 0;
458	}
459	for (self.distance_freq, 0..) \|_, i\| {
460	self.distance_freq[i] = 0;
461	}
462
463	for (tokens) \|t\| {
464	if (t.kind == Token.Kind.literal) {
465	self.literal_freq[t.literal()] += 1;
466	continue;
467	}
468	self.literal_freq[t.lengthCode()] += 1;
469	self.distance_freq[t.distanceCode()] += 1;
470	}
471	// add end_block_marker token at the end
472	self.literal_freq[HuffmanEncoder.end_block_marker] += 1;
473
474	// get the number of literals
475	num_literals = @as(u32, @intCast(self.literal_freq.len));
476	while (self.literal_freq[num_literals - 1] == 0) {
477	num_literals -= 1;
478	}
479	// get the number of distances
480	num_distances = @as(u32, @intCast(self.distance_freq.len));
481	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
482	num_distances -= 1;
483	}
484	if (num_distances == 0) {
485	// We haven't found a single match. If we want to go with the dynamic encoding,
486	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
487	self.distance_freq[0] = 1;
488	num_distances = 1;
489	}
490	self.literal_encoding.generate(&self.literal_freq, 15);
491	self.distance_encoding.generate(&self.distance_freq, 15);
492	return TotalIndexedTokens{
493	.num_literals = num_literals,
494	.num_distances = num_distances,
495	};
496	}
497
498	/// Writes a slice of tokens to the output followed by and end_block_marker.
499	/// codes for literal and distance encoding must be supplied.
500	fn writeTokens(
501	self: *BlockWriter,
502	tokens: []const Token,
503	le_codes: []Compress.HuffCode,
504	oe_codes: []Compress.HuffCode,
505	) Writer.Error!void {
506	for (tokens) \|t\| {
507	if (t.kind == Token.Kind.literal) {
508	try self.writeCode(le_codes[t.literal()]);
509	continue;
510	}
511
512	// Write the length
513	const le = t.lengthEncoding();
514	try self.writeCode(le_codes[le.code]);
515	if (le.extra_bits > 0) {
516	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
517	}
518
519	// Write the distance
520	const oe = t.distanceEncoding();
521	try self.writeCode(oe_codes[oe.code]);
522	if (oe.extra_bits > 0) {
523	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
524	}
525	}
526	// add end_block_marker at the end
527	try self.writeCode(le_codes[HuffmanEncoder.end_block_marker]);
528	}
529
530	/// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
531	/// if the results only gains very little from compression.
532	pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
533	// Add everything as literals
534	histogram(input, &self.literal_freq);
535
536	self.literal_freq[HuffmanEncoder.end_block_marker] = 1;
537
538	const num_literals = HuffmanEncoder.end_block_marker + 1;
539	self.distance_freq[0] = 1;
540	const num_distances = 1;
541
542	self.literal_encoding.generate(&self.literal_freq, 15);
543
544	// Figure out smallest code.
545	// Always use dynamic Huffman or Store
546	var num_codegens: u32 = 0;
547
548	// Generate codegen and codegenFrequencies, which indicates how to encode
549	// the literal_encoding and the distance_encoding.
550	self.generateCodegen(
551	num_literals,
552	num_distances,
553	&self.literal_encoding,
554	&self.huff_distance,
555	);
556	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
557	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
558	const size = dynamic_size.size;
559	num_codegens = dynamic_size.num_codegens;
560
561	// Store bytes, if we don't get a reasonable improvement.
562	const stored_size_ret = storedSizeFits(input);
563	const ssize = stored_size_ret.size;
564	const storable = stored_size_ret.storable;
565
566	if (storable and ssize < (size + (size >> 4))) {
567	try self.storedBlock(input, eof);
568	return;
569	}
570
571	// Huffman.
572	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
573	const encoding = self.literal_encoding.codes[0..257];
574
575	for (input) \|t\| {
576	const c = encoding[t];
577	try self.bit_writer.writeBits(c.code, c.len);
578	}
579	try self.writeCode(encoding[HuffmanEncoder.end_block_marker]);
580	}
581
582	fn histogram(b: []const u8, h: *[286]u16) void {
583	// Clear histogram
584	for (h, 0..) \|_, i\| {
585	h[i] = 0;
586	}
587
588	var lh = h.*[0..256];
589	for (b) \|t\| {
590	lh[t] += 1;
591	}
592	}

lib/std/compress/flate/CircularBuffer.zig deleted-240

...	@@ -1,240 +0,0 @@
1	//! 64K buffer of uncompressed data created in inflate (decompression). Has enough
2	//! history to support writing match<length, distance>; copying length of bytes
3	//! from the position distance backward from current.
4	//!
5	//! Reads can return less than available bytes if they are spread across
6	//! different circles. So reads should repeat until get required number of bytes
7	//! or until returned slice is zero length.
8	//!
9	//! Note on deflate limits:
10	//! * non-compressible block is limited to 65,535 bytes.
11	//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12	//!
13	//! Whole non-compressed block can be written without overlap. We always have
14	//! history of up to 64K, more then 32K needed.
15	//!
16	const std = @import("std");
17	const assert = std.debug.assert;
18	const testing = std.testing;
19
20	const consts = @import("consts.zig").match;
21
22	const mask = 0xffff; // 64K - 1
23	const buffer_len = mask + 1; // 64K buffer
24
25	const Self = @This();
26
27	buffer: [buffer_len]u8 = undefined,
28	wp: usize = 0, // write position
29	rp: usize = 0, // read position
30
31	fn writeAll(self: *Self, buf: []const u8) void {
32	for (buf) \|c\| self.write(c);
33	}
34
35	/// Write literal.
36	pub fn write(self: *Self, b: u8) void {
37	assert(self.wp - self.rp < mask);
38	self.buffer[self.wp & mask] = b;
39	self.wp += 1;
40	}
41
42	/// Write match (back-reference to the same data slice) starting at `distance`
43	/// back from current write position, and `length` of bytes.
44	pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45	if (self.wp < distance or
46	length < consts.base_length or length > consts.max_length or
47	distance < consts.min_distance or distance > consts.max_distance)
48	{
49	return error.InvalidMatch;
50	}
51	assert(self.wp - self.rp < mask);
52
53	var from: usize = self.wp - distance & mask;
54	const from_end: usize = from + length;
55	var to: usize = self.wp & mask;
56	const to_end: usize = to + length;
57
58	self.wp += length;
59
60	// Fast path using memcpy
61	if (from_end < buffer_len and to_end < buffer_len) // start and end at the same circle
62	{
63	var cur_len = distance;
64	var remaining_len = length;
65	while (cur_len < remaining_len) {
66	@memcpy(self.buffer[to..][0..cur_len], self.buffer[from..][0..cur_len]);
67	to += cur_len;
68	remaining_len -= cur_len;
69	cur_len = cur_len * 2;
70	}
71	@memcpy(self.buffer[to..][0..remaining_len], self.buffer[from..][0..remaining_len]);
72	return;
73	}
74
75	// Slow byte by byte
76	while (to < to_end) {
77	self.buffer[to & mask] = self.buffer[from & mask];
78	to += 1;
79	from += 1;
80	}
81	}
82
83	/// Returns writable part of the internal buffer of size `n` at most. Advances
84	/// write pointer, assumes that returned buffer will be filled with data.
85	pub fn getWritable(self: *Self, n: usize) []u8 {
86	const wp = self.wp & mask;
87	const len = @min(n, buffer_len - wp);
88	self.wp += len;
89	return self.buffer[wp .. wp + len];
90	}
91
92	/// Read available data. Can return part of the available data if it is
93	/// spread across two circles. So read until this returns zero length.
94	pub fn read(self: *Self) []const u8 {
95	return self.readAtMost(buffer_len);
96	}
97
98	/// Read part of available data. Can return less than max even if there are
99	/// more than max decoded data.
100	pub fn readAtMost(self: *Self, limit: usize) []const u8 {
101	const rb = self.readBlock(if (limit == 0) buffer_len else limit);
102	defer self.rp += rb.len;
103	return self.buffer[rb.head..rb.tail];
104	}
105
106	const ReadBlock = struct {
107	head: usize,
108	tail: usize,
109	len: usize,
110	};
111
112	/// Returns position of continuous read block data.
113	fn readBlock(self: *Self, max: usize) ReadBlock {
114	const r = self.rp & mask;
115	const w = self.wp & mask;
116	const n = @min(
117	max,
118	if (w >= r) w - r else buffer_len - r,
119	);
120	return .{
121	.head = r,
122	.tail = r + n,
123	.len = n,
124	};
125	}
126
127	/// Number of free bytes for write.
128	pub fn free(self: *Self) usize {
129	return buffer_len - (self.wp - self.rp);
130	}
131
132	/// Full if largest match can't fit. 258 is largest match length. That much
133	/// bytes can be produced in single decode step.
134	pub fn full(self: *Self) bool {
135	return self.free() < 258 + 1;
136	}
137
138	// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
139	test writeMatch {
140	var cb: Self = .{};
141
142	cb.writeAll("a salad; ");
143	try cb.writeMatch(5, 9);
144	try cb.writeMatch(3, 3);
145
146	try testing.expectEqualStrings("a salad; a salsal", cb.read());
147	}
148
149	test "writeMatch overlap" {
150	var cb: Self = .{};
151
152	cb.writeAll("a b c ");
153	try cb.writeMatch(8, 4);
154	cb.write('d');
155
156	try testing.expectEqualStrings("a b c b c b c d", cb.read());
157	}
158
159	test readAtMost {
160	var cb: Self = .{};
161
162	cb.writeAll("0123456789");
163	try cb.writeMatch(50, 10);
164
165	try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
166	for (0..6) \|i\| {
167	try testing.expectEqual(i * 10, cb.rp);
168	try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
169	}
170	try testing.expectEqualStrings("", cb.readAtMost(10));
171	try testing.expectEqualStrings("", cb.read());
172	}
173
174	test Self {
175	var cb: Self = .{};
176
177	const data = "0123456789abcdef" ** (1024 / 16);
178	cb.writeAll(data);
179	try testing.expectEqual(@as(usize, 0), cb.rp);
180	try testing.expectEqual(@as(usize, 1024), cb.wp);
181	try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
182
183	for (0..62 * 4) \|_\|
184	try cb.writeMatch(256, 1024); // write 62K
185
186	try testing.expectEqual(@as(usize, 0), cb.rp);
187	try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
188	try testing.expectEqual(@as(usize, 1024), cb.free());
189
190	cb.writeAll(data[0..200]);
191	_ = cb.readAtMost(1024); // make some space
192	cb.writeAll(data); // overflows write position
193	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
194	try testing.expectEqual(@as(usize, 1024), cb.rp);
195	try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
196
197	const rb = cb.readBlock(Self.buffer_len);
198	try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
199	try testing.expectEqual(@as(usize, 1024), rb.head);
200	try testing.expectEqual(@as(usize, 65536), rb.tail);
201
202	try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
203	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
204	try testing.expectEqual(@as(usize, 65536), cb.rp);
205	try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
206
207	try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
208	}
209
210	test "write overlap" {
211	var cb: Self = .{};
212	cb.wp = cb.buffer.len - 15;
213	cb.rp = cb.wp;
214
215	cb.writeAll("0123456789");
216	cb.writeAll("abcdefghij");
217
218	try testing.expectEqual(cb.buffer.len + 5, cb.wp);
219	try testing.expectEqual(cb.buffer.len - 15, cb.rp);
220
221	try testing.expectEqualStrings("0123456789abcde", cb.read());
222	try testing.expectEqualStrings("fghij", cb.read());
223
224	try testing.expect(cb.wp == cb.rp);
225	}
226
227	test "writeMatch/read overlap" {
228	var cb: Self = .{};
229	cb.wp = cb.buffer.len - 15;
230	cb.rp = cb.wp;
231
232	cb.writeAll("0123456789");
233	try cb.writeMatch(15, 5);
234
235	try testing.expectEqualStrings("012345678956789", cb.read());
236	try testing.expectEqualStrings("5678956789", cb.read());
237
238	try cb.writeMatch(20, 25);
239	try testing.expectEqualStrings("01234567895678956789", cb.read());
240	}

lib/std/compress/flate/Compress.zig created+332

...	@@ -0,0 +1,332 @@
1	//! Default compression algorithm. Has two steps: tokenization and token
2	//! encoding.
3	//!
4	//! Tokenization takes uncompressed input stream and produces list of tokens.
5	//! Each token can be literal (byte of data) or match (backrefernce to previous
6	//! data with length and distance). Tokenization accumulators 32K tokens, when
7	//! full or `flush` is called tokens are passed to the `block_writer`. Level
8	//! defines how hard (how slow) it tries to find match.
9	//!
10	//! Block writer will decide which type of deflate block to write (stored, fixed,
11	//! dynamic) and encode tokens to the output byte stream. Client has to call
12	//! `finish` to write block with the final bit set.
13	//!
14	//! Container defines type of header and footer which can be gzip, zlib or raw.
15	//! They all share same deflate body. Raw has no header or footer just deflate
16	//! body.
17	//!
18	//! Compression algorithm explained in rfc-1951 (slightly edited for this case):
19	//!
20	//! The compressor uses a chained hash table `lookup` to find duplicated
21	//! strings, using a hash function that operates on 4-byte sequences. At any
22	//! given point during compression, let XYZW be the next 4 input bytes
23	//! (lookahead) to be examined (not necessarily all different, of course).
24	//! First, the compressor examines the hash chain for XYZW. If the chain is
25	//! empty, the compressor simply writes out X as a literal byte and advances
26	//! one byte in the input. If the hash chain is not empty, indicating that the
27	//! sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
28	//! hash function value) has occurred recently, the compressor compares all
29	//! strings on the XYZW hash chain with the actual input data sequence
30	//! starting at the current point, and selects the longest match.
31	//!
32	//! To improve overall compression, the compressor defers the selection of
33	//! matches ("lazy matching"): after a match of length N has been found, the
34	//! compressor searches for a longer match starting at the next input byte. If
35	//! it finds a longer match, it truncates the previous match to a length of
36	//! one (thus producing a single literal byte) and then emits the longer
37	//! match. Otherwise, it emits the original match, and, as described above,
38	//! advances N bytes before continuing.
39	//!
40	//!
41	//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
42
43	const builtin = @import("builtin");
44	const std = @import("std");
45	const assert = std.debug.assert;
46	const testing = std.testing;
47	const expect = testing.expect;
48	const mem = std.mem;
49	const math = std.math;
50	const Writer = std.Io.Writer;
51
52	const Compress = @This();
53	const Token = @import("Token.zig");
54	const BlockWriter = @import("BlockWriter.zig");
55	const flate = @import("../flate.zig");
56	const Container = flate.Container;
57	const Lookup = @import("Lookup.zig");
58	const HuffmanEncoder = flate.HuffmanEncoder;
59	const LiteralNode = HuffmanEncoder.LiteralNode;
60
61	lookup: Lookup = .{},
62	tokens: Tokens = .{},
63	block_writer: BlockWriter,
64	level: LevelArgs,
65	hasher: Container.Hasher,
66	writer: Writer,
67	state: State,
68
69	// Match and literal at the previous position.
70	// Used for lazy match finding in processWindow.
71	prev_match: ?Token = null,
72	prev_literal: ?u8 = null,
73
74	pub const State = enum { header, middle, ended };
75
76	/// Trades between speed and compression size.
77	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
78	/// levels 1-3 are using different algorithm to perform faster but with less
79	/// compression. That is not implemented here.
80	pub const Level = enum(u4) {
81	level_4 = 4,
82	level_5 = 5,
83	level_6 = 6,
84	level_7 = 7,
85	level_8 = 8,
86	level_9 = 9,
87
88	fast = 0xb,
89	default = 0xc,
90	best = 0xd,
91	};
92
93	/// Number of tokens to accumulate in deflate before starting block encoding.
94	///
95	/// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
96	/// 8 and max 9 that gives 14 or 15 bits.
97	pub const n_tokens = 1 << 15;
98
99	/// Algorithm knobs for each level.
100	const LevelArgs = struct {
101	good: u16, // Do less lookups if we already have match of this length.
102	nice: u16, // Stop looking for better match if we found match with at least this length.
103	lazy: u16, // Don't do lazy match find if got match with at least this length.
104	chain: u16, // How many lookups for previous match to perform.
105
106	pub fn get(level: Level) LevelArgs {
107	return switch (level) {
108	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
109	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
110	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
111	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
112	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
113	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
114	};
115	}
116	};
117
118	pub const Options = struct {
119	level: Level = .default,
120	container: Container = .raw,
121	};
122
123	pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {
124	return .{
125	.block_writer = .init(output),
126	.level = .get(options.level),
127	.hasher = .init(options.container),
128	.state = .header,
129	.writer = .{
130	.buffer = buffer,
131	.vtable = &.{ .drain = drain },
132	},
133	};
134	}
135
136	// Tokens store
137	const Tokens = struct {
138	list: [n_tokens]Token = undefined,
139	pos: usize = 0,
140
141	fn add(self: *Tokens, t: Token) void {
142	self.list[self.pos] = t;
143	self.pos += 1;
144	}
145
146	fn full(self: *Tokens) bool {
147	return self.pos == self.list.len;
148	}
149
150	fn reset(self: *Tokens) void {
151	self.pos = 0;
152	}
153
154	fn tokens(self: *Tokens) []const Token {
155	return self.list[0..self.pos];
156	}
157	};
158
159	fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {
160	_ = data;
161	_ = splat;
162	const c: *Compress = @fieldParentPtr("writer", me);
163	const out = c.block_writer.output;
164	switch (c.state) {
165	.header => {
166	c.state = .middle;
167	const header = c.hasher.container().header();
168	try out.writeAll(header);
169	return header.len;
170	},
171	.middle => {},
172	.ended => unreachable,
173	}
174
175	const buffered = me.buffered();
176	const min_lookahead = Token.min_length + Token.max_length;
177	const history_plus_lookahead_len = flate.history_len + min_lookahead;
178	if (buffered.len < history_plus_lookahead_len) return 0;
179	const lookahead = buffered[flate.history_len..];
180
181	// TODO tokenize
182	_ = lookahead;
183	//c.hasher.update(lookahead[0..n]);
184	@panic("TODO");
185	}
186
187	pub fn end(c: *Compress) !void {
188	try endUnflushed(c);
189	const out = c.block_writer.output;
190	try out.flush();
191	}
192
193	pub fn endUnflushed(c: *Compress) !void {
194	while (c.writer.end != 0) _ = try drain(&c.writer, &.{""}, 1);
195	c.state = .ended;
196
197	const out = c.block_writer.output;
198
199	// TODO flush tokens
200
201	switch (c.hasher) {
202	.gzip => \|*gzip\| {
203	// GZIP 8 bytes footer
204	// - 4 bytes, CRC32 (CRC-32)
205	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
206	const footer = try out.writableArray(8);
207	std.mem.writeInt(u32, footer[0..4], gzip.crc.final(), .little);
208	std.mem.writeInt(u32, footer[4..8], @truncate(gzip.count), .little);
209	},
210	.zlib => \|*zlib\| {
211	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
212	// 4 bytes of ADLER32 (Adler-32 checksum)
213	// Checksum value of the uncompressed data (excluding any
214	// dictionary data) computed according to Adler-32
215	// algorithm.
216	std.mem.writeInt(u32, try out.writableArray(4), zlib.adler, .big);
217	},
218	.raw => {},
219	}
220	}
221
222	pub const Simple = struct {
223	/// Note that store blocks are limited to 65535 bytes.
224	buffer: []u8,
225	wp: usize,
226	block_writer: BlockWriter,
227	hasher: Container.Hasher,
228	strategy: Strategy,
229
230	pub const Strategy = enum { huffman, store };
231
232	pub fn init(output: *Writer, buffer: []u8, container: Container, strategy: Strategy) !Simple {
233	const header = container.header();
234	try output.writeAll(header);
235	return .{
236	.buffer = buffer,
237	.wp = 0,
238	.block_writer = .init(output),
239	.hasher = .init(container),
240	.strategy = strategy,
241	};
242	}
243
244	pub fn flush(self: *Simple) !void {
245	try self.flushBuffer(false);
246	try self.block_writer.storedBlock("", false);
247	try self.block_writer.flush();
248	}
249
250	pub fn finish(self: *Simple) !void {
251	try self.flushBuffer(true);
252	try self.block_writer.flush();
253	try self.hasher.container().writeFooter(&self.hasher, self.block_writer.output);
254	}
255
256	fn flushBuffer(self: *Simple, final: bool) !void {
257	const buf = self.buffer[0..self.wp];
258	switch (self.strategy) {
259	.huffman => try self.block_writer.huffmanBlock(buf, final),
260	.store => try self.block_writer.storedBlock(buf, final),
261	}
262	self.wp = 0;
263	}
264	};
265
266	test "generate a Huffman code from an array of frequencies" {
267	var freqs: [19]u16 = [_]u16{
268	8, // 0
269	1, // 1
270	1, // 2
271	2, // 3
272	5, // 4
273	10, // 5
274	9, // 6
275	1, // 7
276	0, // 8
277	0, // 9
278	0, // 10
279	0, // 11
280	0, // 12
281	0, // 13
282	0, // 14
283	0, // 15
284	1, // 16
285	3, // 17
286	5, // 18
287	};
288
289	var codes: [19]HuffmanEncoder.Code = undefined;
290	var enc: HuffmanEncoder = .{
291	.codes = &codes,
292	.freq_cache = undefined,
293	.bit_count = undefined,
294	.lns = undefined,
295	.lfs = undefined,
296	};
297	enc.generate(freqs[0..], 7);
298
299	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
300
301	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
302	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
303	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
304	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
305	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
306	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
307	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
308	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
309	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
310	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
311	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
312	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
313	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
314	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
315	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
316	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
317	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
318	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
319	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
320
321	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
322	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
323	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
324	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
325	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
326	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
327	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
328	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
329	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
330	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
331	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
332	}

lib/std/compress/flate/Decompress.zig created+1270

...	@@ -0,0 +1,1270 @@
1	const std = @import("../../std.zig");
2	const assert = std.debug.assert;
3	const flate = std.compress.flate;
4	const testing = std.testing;
5	const Writer = std.Io.Writer;
6	const Reader = std.Io.Reader;
7	const Container = flate.Container;
8
9	const Decompress = @This();
10	const Token = @import("Token.zig");
11
12	input: *Reader,
13	next_bits: Bits,
14	remaining_bits: std.math.Log2Int(Bits),
15
16	reader: Reader,
17
18	container_metadata: Container.Metadata,
19
20	lit_dec: LiteralDecoder,
21	dst_dec: DistanceDecoder,
22
23	final_block: bool,
24	state: State,
25
26	err: ?Error,
27
28	/// TODO: change this to usize
29	const Bits = u64;
30
31	const BlockType = enum(u2) {
32	stored = 0,
33	fixed = 1,
34	dynamic = 2,
35	};
36
37	const State = union(enum) {
38	protocol_header,
39	block_header,
40	stored_block: u16,
41	fixed_block,
42	dynamic_block,
43	dynamic_block_literal: u8,
44	dynamic_block_match: u16,
45	protocol_footer,
46	end,
47	};
48
49	pub const Error = Container.Error \|\| error{
50	InvalidCode,
51	InvalidMatch,
52	WrongStoredBlockNlen,
53	InvalidDynamicBlockHeader,
54	ReadFailed,
55	OversubscribedHuffmanTree,
56	IncompleteHuffmanTree,
57	MissingEndOfBlockCode,
58	EndOfStream,
59	};
60
61	const direct_vtable: Reader.VTable = .{
62	.stream = streamDirect,
63	.rebase = rebaseFallible,
64	.discard = discard,
65	.readVec = readVec,
66	};
67
68	const indirect_vtable: Reader.VTable = .{
69	.stream = streamIndirect,
70	.rebase = rebaseFallible,
71	.discard = discardIndirect,
72	.readVec = readVec,
73	};
74
75	pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress {
76	return .{
77	.reader = .{
78	.vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable,
79	.buffer = buffer,
80	.seek = 0,
81	.end = 0,
82	},
83	.input = input,
84	.next_bits = 0,
85	.remaining_bits = 0,
86	.container_metadata = .init(container),
87	.lit_dec = .{},
88	.dst_dec = .{},
89	.final_block = false,
90	.state = .protocol_header,
91	.err = null,
92	};
93	}
94
95	fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
96	rebase(r, capacity);
97	}
98
99	fn rebase(r: *Reader, capacity: usize) void {
100	assert(capacity <= r.buffer.len - flate.history_len);
101	assert(r.end + capacity > r.buffer.len);
102	const discard_n = r.end - flate.history_len;
103	const keep = r.buffer[discard_n..r.end];
104	@memmove(r.buffer[0..keep.len], keep);
105	assert(keep.len != 0);
106	r.end = keep.len;
107	r.seek -= discard_n;
108	}
109
110	/// This could be improved so that when an amount is discarded that includes an
111	/// entire frame, skip decoding that frame.
112	fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
113	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
114	var writer: Writer = .{
115	.vtable = &.{
116	.drain = std.Io.Writer.Discarding.drain,
117	.sendFile = std.Io.Writer.Discarding.sendFile,
118	},
119	.buffer = r.buffer,
120	.end = r.end,
121	};
122	defer {
123	assert(writer.end != 0);
124	r.end = writer.end;
125	r.seek = r.end;
126	}
127	const n = r.stream(&writer, limit) catch \|err\| switch (err) {
128	error.WriteFailed => unreachable,
129	error.ReadFailed => return error.ReadFailed,
130	error.EndOfStream => return error.EndOfStream,
131	};
132	assert(n <= @intFromEnum(limit));
133	return n;
134	}
135
136	fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
137	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
138	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
139	var writer: Writer = .{
140	.buffer = r.buffer,
141	.end = r.end,
142	.vtable = &.{ .drain = Writer.unreachableDrain },
143	};
144	{
145	defer r.end = writer.end;
146	_ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
147	error.WriteFailed => unreachable,
148	else => \|e\| return e,
149	};
150	}
151	const n = limit.minInt(r.end - r.seek);
152	r.seek += n;
153	return n;
154	}
155
156	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
157	_ = data;
158	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
159	return streamIndirectInner(d);
160	}
161
162	fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
163	const r = &d.reader;
164	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
165	var writer: Writer = .{
166	.buffer = r.buffer,
167	.end = r.end,
168	.vtable = &.{ .drain = Writer.unreachableDrain },
169	};
170	defer r.end = writer.end;
171	_ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
172	error.WriteFailed => unreachable,
173	else => \|e\| return e,
174	};
175	return 0;
176	}
177
178	fn decodeLength(self: *Decompress, code: u8) !u16 {
179	if (code > 28) return error.InvalidCode;
180	const ml = Token.matchLength(code);
181	return if (ml.extra_bits == 0) // 0 - 5 extra bits
182	ml.base
183	else
184	ml.base + try self.takeBitsRuntime(ml.extra_bits);
185	}
186
187	fn decodeDistance(self: *Decompress, code: u8) !u16 {
188	if (code > 29) return error.InvalidCode;
189	const md = Token.matchDistance(code);
190	return if (md.extra_bits == 0) // 0 - 13 extra bits
191	md.base
192	else
193	md.base + try self.takeBitsRuntime(md.extra_bits);
194	}
195
196	// Decode code length symbol to code length. Writes decoded length into
197	// lens slice starting at position pos. Returns number of positions
198	// advanced.
199	fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usize {
200	if (pos >= lens.len)
201	return error.InvalidDynamicBlockHeader;
202
203	switch (code) {
204	0...15 => {
205	// Represent code lengths of 0 - 15
206	lens[pos] = @intCast(code);
207	return 1;
208	},
209	16 => {
210	// Copy the previous code length 3 - 6 times.
211	// The next 2 bits indicate repeat length
212	const n: u8 = @as(u8, try self.takeBits(u2)) + 3;
213	if (pos == 0 or pos + n > lens.len)
214	return error.InvalidDynamicBlockHeader;
215	for (0..n) \|i\| {
216	lens[pos + i] = lens[pos + i - 1];
217	}
218	return n;
219	},
220	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
221	17 => return @as(u8, try self.takeBits(u3)) + 3,
222	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
223	18 => return @as(u8, try self.takeBits(u7)) + 11,
224	else => return error.InvalidDynamicBlockHeader,
225	}
226	}
227
228	fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
229	// Maximum code len is 15 bits.
230	const sym = try decoder.find(@bitReverse(try self.peekBits(u15)));
231	try self.tossBits(sym.code_bits);
232	return sym;
233	}
234
235	fn streamDirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
236	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
237	return streamFallible(d, w, limit);
238	}
239
240	fn streamIndirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
241	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
242	_ = limit;
243	_ = w;
244	return streamIndirectInner(d);
245	}
246
247	fn streamFallible(d: Decompress, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
248	return streamInner(d, w, limit) catch \|err\| switch (err) {
249	error.EndOfStream => {
250	if (d.state == .end) {
251	return error.EndOfStream;
252	} else {
253	d.err = error.EndOfStream;
254	return error.ReadFailed;
255	}
256	},
257	error.WriteFailed => return error.WriteFailed,
258	else => \|e\| {
259	// In the event of an error, state is unmodified so that it can be
260	// better used to diagnose the failure.
261	d.err = e;
262	return error.ReadFailed;
263	},
264	};
265	}
266
267	fn streamInner(d: Decompress, w: Writer, limit: std.Io.Limit) (Error \|\| Reader.StreamError)!usize {
268	var remaining = @intFromEnum(limit);
269	const in = d.input;
270	sw: switch (d.state) {
271	.protocol_header => switch (d.container_metadata.container()) {
272	.gzip => {
273	const Header = extern struct {
274	magic: u16 align(1),
275	method: u8,
276	flags: packed struct(u8) {
277	text: bool,
278	hcrc: bool,
279	extra: bool,
280	name: bool,
281	comment: bool,
282	reserved: u3,
283	},
284	mtime: u32 align(1),
285	xfl: u8,
286	os: u8,
287	};
288	const header = try in.takeStruct(Header, .little);
289	if (header.magic != 0x8b1f or header.method != 0x08)
290	return error.BadGzipHeader;
291	if (header.flags.extra) {
292	const extra_len = try in.takeInt(u16, .little);
293	try in.discardAll(extra_len);
294	}
295	if (header.flags.name) {
296	_ = try in.discardDelimiterInclusive(0);
297	}
298	if (header.flags.comment) {
299	_ = try in.discardDelimiterInclusive(0);
300	}
301	if (header.flags.hcrc) {
302	try in.discardAll(2);
303	}
304	continue :sw .block_header;
305	},
306	.zlib => {
307	const header = try in.takeArray(2);
308	const cmf: packed struct(u8) { cm: u4, cinfo: u4 } = @bitCast(header[0]);
309	if (cmf.cm != 8 or cmf.cinfo > 7) return error.BadZlibHeader;
310	continue :sw .block_header;
311	},
312	.raw => continue :sw .block_header,
313	},
314	.block_header => {
315	d.final_block = (try d.takeBits(u1)) != 0;
316	const block_type: BlockType = @enumFromInt(try d.takeBits(u2));
317	switch (block_type) {
318	.stored => {
319	d.alignBitsDiscarding();
320	// everything after this is byte aligned in stored block
321	const len = try in.takeInt(u16, .little);
322	const nlen = try in.takeInt(u16, .little);
323	if (len != ~nlen) return error.WrongStoredBlockNlen;
324	continue :sw .{ .stored_block = len };
325	},
326	.fixed => continue :sw .fixed_block,
327	.dynamic => {
328	const hlit: u16 = @as(u16, try d.takeBits(u5)) + 257; // number of ll code entries present - 257
329	const hdist: u16 = @as(u16, try d.takeBits(u5)) + 1; // number of distance code entries - 1
330	const hclen: u8 = @as(u8, try d.takeBits(u4)) + 4; // hclen + 4 code lengths are encoded
331
332	if (hlit > 286 or hdist > 30)
333	return error.InvalidDynamicBlockHeader;
334
335	// lengths for code lengths
336	var cl_lens: [19]u4 = @splat(0);
337	for (flate.HuffmanEncoder.codegen_order[0..hclen]) \|i\| {
338	cl_lens[i] = try d.takeBits(u3);
339	}
340	var cl_dec: CodegenDecoder = .{};
341	try cl_dec.generate(&cl_lens);
342
343	// decoded code lengths
344	var dec_lens: [286 + 30]u4 = @splat(0);
345	var pos: usize = 0;
346	while (pos < hlit + hdist) {
347	const peeked = @bitReverse(try d.peekBits(u7));
348	const sym = try cl_dec.find(peeked);
349	try d.tossBits(sym.code_bits);
350	pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos);
351	}
352	if (pos > hlit + hdist) {
353	return error.InvalidDynamicBlockHeader;
354	}
355
356	// literal code lengths to literal decoder
357	try d.lit_dec.generate(dec_lens[0..hlit]);
358
359	// distance code lengths to distance decoder
360	try d.dst_dec.generate(dec_lens[hlit..][0..hdist]);
361
362	continue :sw .dynamic_block;
363	},
364	}
365	},
366	.stored_block => \|remaining_len\| {
367	const out = try w.writableSliceGreedyPreserve(flate.history_len, 1);
368	var limited_out: [1][]u8 = .{limit.min(.limited(remaining_len)).slice(out)};
369	const n = try d.input.readVec(&limited_out);
370	if (remaining_len - n == 0) {
371	d.state = if (d.final_block) .protocol_footer else .block_header;
372	} else {
373	d.state = .{ .stored_block = @intCast(remaining_len - n) };
374	}
375	w.advance(n);
376	return n;
377	},
378	.fixed_block => {
379	while (remaining > 0) {
380	const code = try d.readFixedCode();
381	switch (code) {
382	0...255 => {
383	try w.writeBytePreserve(flate.history_len, @intCast(code));
384	remaining -= 1;
385	},
386	256 => {
387	d.state = if (d.final_block) .protocol_footer else .block_header;
388	return @intFromEnum(limit) - remaining;
389	},
390	257...285 => {
391	// Handles fixed block non literal (length) code.
392	// Length code is followed by 5 bits of distance code.
393	const length = try d.decodeLength(@intCast(code - 257));
394	const distance = try d.decodeDistance(@bitReverse(try d.takeBits(u5)));
395	try writeMatch(w, length, distance);
396	remaining -= length;
397	},
398	else => return error.InvalidCode,
399	}
400	}
401	d.state = .fixed_block;
402	return @intFromEnum(limit) - remaining;
403	},
404	.dynamic_block => {
405	// In larger archives most blocks are usually dynamic, so
406	// decompression performance depends on this logic.
407	var sym = try d.decodeSymbol(&d.lit_dec);
408	sym: switch (sym.kind) {
409	.literal => {
410	if (remaining != 0) {
411	@branchHint(.likely);
412	remaining -= 1;
413	try w.writeBytePreserve(flate.history_len, sym.symbol);
414	sym = try d.decodeSymbol(&d.lit_dec);
415	continue :sym sym.kind;
416	} else {
417	d.state = .{ .dynamic_block_literal = sym.symbol };
418	return @intFromEnum(limit) - remaining;
419	}
420	},
421	.match => {
422	// Decode match backreference <length, distance>
423	const length = try d.decodeLength(sym.symbol);
424	continue :sw .{ .dynamic_block_match = length };
425	},
426	.end_of_block => {
427	d.state = if (d.final_block) .protocol_footer else .block_header;
428	continue :sw d.state;
429	},
430	}
431	},
432	.dynamic_block_literal => \|symbol\| {
433	assert(remaining != 0);
434	remaining -= 1;
435	try w.writeBytePreserve(flate.history_len, symbol);
436	continue :sw .dynamic_block;
437	},
438	.dynamic_block_match => \|length\| {
439	if (remaining >= length) {
440	@branchHint(.likely);
441	remaining -= length;
442	const dsm = try d.decodeSymbol(&d.dst_dec);
443	const distance = try d.decodeDistance(dsm.symbol);
444	try writeMatch(w, length, distance);
445	continue :sw .dynamic_block;
446	} else {
447	d.state = .{ .dynamic_block_match = length };
448	return @intFromEnum(limit) - remaining;
449	}
450	},
451	.protocol_footer => {
452	switch (d.container_metadata) {
453	.gzip => \|*gzip\| {
454	d.alignBitsDiscarding();
455	gzip.* = .{
456	.crc = try in.takeInt(u32, .little),
457	.count = try in.takeInt(u32, .little),
458	};
459	},
460	.zlib => \|*zlib\| {
461	d.alignBitsDiscarding();
462	zlib.* = .{
463	.adler = try in.takeInt(u32, .little),
464	};
465	},
466	.raw => {
467	d.alignBitsPreserving();
468	},
469	}
470	d.state = .end;
471	return @intFromEnum(limit) - remaining;
472	},
473	.end => return error.EndOfStream,
474	}
475	}
476
477	/// Write match (back-reference to the same data slice) starting at `distance`
478	/// back from current write position, and `length` of bytes.
479	fn writeMatch(w: *Writer, length: u16, distance: u16) !void {
480	if (w.end < distance) return error.InvalidMatch;
481	if (length < Token.base_length) return error.InvalidMatch;
482	if (length > Token.max_length) return error.InvalidMatch;
483	if (distance < Token.min_distance) return error.InvalidMatch;
484	if (distance > Token.max_distance) return error.InvalidMatch;
485
486	// This is not a @memmove; it intentionally repeats patterns caused by
487	// iterating one byte at a time.
488	const dest = try w.writableSlicePreserve(flate.history_len, length);
489	const end = dest.ptr - w.buffer.ptr;
490	const src = w.buffer[end - distance ..][0..length];
491	for (dest, src) \|d, s\| d. = s;
492	}
493
494	fn takeBits(d: *Decompress, comptime U: type) !U {
495	const remaining_bits = d.remaining_bits;
496	const next_bits = d.next_bits;
497	if (remaining_bits >= @bitSizeOf(U)) {
498	const u: U = @truncate(next_bits);
499	d.next_bits = next_bits >> @bitSizeOf(U);
500	d.remaining_bits = remaining_bits - @bitSizeOf(U);
501	return u;
502	}
503	const in = d.input;
504	const next_int = in.takeInt(Bits, .little) catch \|err\| switch (err) {
505	error.ReadFailed => return error.ReadFailed,
506	error.EndOfStream => return takeBitsEnding(d, U),
507	};
508	const needed_bits = @bitSizeOf(U) - remaining_bits;
509	const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
510	d.next_bits = next_int >> needed_bits;
511	d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits));
512	return u;
513	}
514
515	fn takeBitsEnding(d: *Decompress, comptime U: type) !U {
516	const remaining_bits = d.remaining_bits;
517	const next_bits = d.next_bits;
518	const in = d.input;
519	const n = in.bufferedLen();
520	assert(n < @sizeOf(Bits));
521	const needed_bits = @bitSizeOf(U) - remaining_bits;
522	if (n * 8 < needed_bits) return error.EndOfStream;
523	const next_int = in.takeVarInt(Bits, .little, n) catch \|err\| switch (err) {
524	error.ReadFailed => return error.ReadFailed,
525	error.EndOfStream => unreachable,
526	};
527	const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
528	d.next_bits = next_int >> needed_bits;
529	d.remaining_bits = @intCast(n * 8 - @as(usize, needed_bits));
530	return u;
531	}
532
533	fn peekBits(d: *Decompress, comptime U: type) !U {
534	const remaining_bits = d.remaining_bits;
535	const next_bits = d.next_bits;
536	if (remaining_bits >= @bitSizeOf(U)) return @truncate(next_bits);
537	const in = d.input;
538	const next_int = in.peekInt(Bits, .little) catch \|err\| switch (err) {
539	error.ReadFailed => return error.ReadFailed,
540	error.EndOfStream => return peekBitsEnding(d, U),
541	};
542	const needed_bits = @bitSizeOf(U) - remaining_bits;
543	return @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
544	}
545
546	fn peekBitsEnding(d: *Decompress, comptime U: type) !U {
547	const remaining_bits = d.remaining_bits;
548	const next_bits = d.next_bits;
549	const in = d.input;
550	var u: Bits = 0;
551	var remaining_needed_bits = @bitSizeOf(U) - remaining_bits;
552	var i: usize = 0;
553	while (remaining_needed_bits >= 8) {
554	const byte = try specialPeek(in, next_bits, i);
555	u \|= @as(Bits, byte) << @intCast(i * 8);
556	remaining_needed_bits -= 8;
557	i += 1;
558	}
559	if (remaining_needed_bits != 0) {
560	const byte = try specialPeek(in, next_bits, i);
561	u \|= @as(Bits, byte) << @intCast((i * 8) + remaining_needed_bits);
562	}
563	return @truncate((u << remaining_bits) \| next_bits);
564	}
565
566	/// If there is any unconsumed data, handles EndOfStream by pretending there
567	/// are zeroes afterwards.
568	fn specialPeek(in: *Reader, next_bits: Bits, i: usize) Reader.Error!u8 {
569	const peeked = in.peek(i + 1) catch \|err\| switch (err) {
570	error.ReadFailed => return error.ReadFailed,
571	error.EndOfStream => if (next_bits == 0 and i == 0) return error.EndOfStream else return 0,
572	};
573	return peeked[i];
574	}
575
576	fn tossBits(d: *Decompress, n: u4) !void {
577	const remaining_bits = d.remaining_bits;
578	const next_bits = d.next_bits;
579	if (remaining_bits >= n) {
580	d.next_bits = next_bits >> n;
581	d.remaining_bits = remaining_bits - n;
582	} else {
583	const in = d.input;
584	const next_int = in.takeInt(Bits, .little) catch \|err\| switch (err) {
585	error.ReadFailed => return error.ReadFailed,
586	error.EndOfStream => return tossBitsEnding(d, n),
587	};
588	const needed_bits = n - remaining_bits;
589	d.next_bits = next_int >> needed_bits;
590	d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits));
591	}
592	}
593
594	fn tossBitsEnding(d: *Decompress, n: u4) !void {
595	const remaining_bits = d.remaining_bits;
596	const in = d.input;
597	const buffered_n = in.bufferedLen();
598	if (buffered_n == 0) return error.EndOfStream;
599	assert(buffered_n < @sizeOf(Bits));
600	const needed_bits = n - remaining_bits;
601	const next_int = in.takeVarInt(Bits, .little, buffered_n) catch \|err\| switch (err) {
602	error.ReadFailed => return error.ReadFailed,
603	error.EndOfStream => unreachable,
604	};
605	d.next_bits = next_int >> needed_bits;
606	d.remaining_bits = @intCast(@as(usize, n) * 8 -\| @as(usize, needed_bits));
607	}
608
609	fn takeBitsRuntime(d: *Decompress, n: u4) !u16 {
610	const x = try peekBits(d, u16);
611	const mask: u16 = (@as(u16, 1) << n) - 1;
612	const u: u16 = @as(u16, @truncate(x)) & mask;
613	try tossBits(d, n);
614	return u;
615	}
616
617	fn alignBitsDiscarding(d: *Decompress) void {
618	const remaining_bits = d.remaining_bits;
619	if (remaining_bits == 0) return;
620	const n_bytes = remaining_bits / 8;
621	const in = d.input;
622	in.seek -= n_bytes;
623	d.remaining_bits = 0;
624	d.next_bits = 0;
625	}
626
627	fn alignBitsPreserving(d: *Decompress) void {
628	const remaining_bits: usize = d.remaining_bits;
629	if (remaining_bits == 0) return;
630	const n_bytes = (remaining_bits + 7) / 8;
631	const in = d.input;
632	in.seek -= n_bytes;
633	d.remaining_bits = 0;
634	d.next_bits = 0;
635	}
636
637	/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
638	/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
639	/// Lit Value Bits Codes
640	/// --------- ---- -----
641	/// 0 - 143 8 00110000 through
642	/// 10111111
643	/// 144 - 255 9 110010000 through
644	/// 111111111
645	/// 256 - 279 7 0000000 through
646	/// 0010111
647	/// 280 - 287 8 11000000 through
648	/// 11000111
649	fn readFixedCode(d: *Decompress) !u16 {
650	const code7 = @bitReverse(try d.takeBits(u7));
651	return switch (code7) {
652	0...0b0010_111 => @as(u16, code7) + 256,
653	0b0010_111 + 1...0b1011_111 => (@as(u16, code7) << 1) + @as(u16, try d.takeBits(u1)) - 0b0011_0000,
654	0b1011_111 + 1...0b1100_011 => (@as(u16, code7 - 0b1100000) << 1) + try d.takeBits(u1) + 280,
655	else => (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, @bitReverse(try d.takeBits(u2))) + 144,
656	};
657	}
658
659	pub const Symbol = packed struct {
660	pub const Kind = enum(u2) {
661	literal,
662	end_of_block,
663	match,
664	};
665
666	symbol: u8 = 0, // symbol from alphabet
667	code_bits: u4 = 0, // number of bits in code 0-15
668	kind: Kind = .literal,
669
670	code: u16 = 0, // huffman code of the symbol
671	next: u16 = 0, // pointer to the next symbol in linked list
672	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
673
674	// Sorting less than function.
675	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
676	if (a.code_bits == b.code_bits) {
677	if (a.kind == b.kind) {
678	return a.symbol < b.symbol;
679	}
680	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
681	}
682	return a.code_bits < b.code_bits;
683	}
684	};
685
686	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
687	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
688	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
689
690	/// Creates huffman tree codes from list of code lengths (in `build`).
691	///
692	/// `find` then finds symbol for code bits. Code can be any length between 1 and
693	/// 15 bits. When calling `find` we don't know how many bits will be used to
694	/// find symbol. When symbol is returned it has code_bits field which defines
695	/// how much we should advance in bit stream.
696	///
697	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
698	/// many times in this table; 32K places for 286 (at most) symbols.
699	/// Small lookup table is optimization for faster search.
700	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
701	/// with difference that we here use statically allocated arrays.
702	///
703	fn HuffmanDecoder(
704	comptime alphabet_size: u16,
705	comptime max_code_bits: u4,
706	comptime lookup_bits: u4,
707	) type {
708	const lookup_shift = max_code_bits - lookup_bits;
709
710	return struct {
711	// all symbols in alaphabet, sorted by code_len, symbol
712	symbols: [alphabet_size]Symbol = undefined,
713	// lookup table code -> symbol
714	lookup: [1 << lookup_bits]Symbol = undefined,
715
716	const Self = @This();
717
718	/// Generates symbols and lookup tables from list of code lens for each symbol.
719	pub fn generate(self: *Self, lens: []const u4) !void {
720	try checkCompleteness(lens);
721
722	// init alphabet with code_bits
723	for (self.symbols, 0..) \|_, i\| {
724	const cb: u4 = if (i < lens.len) lens[i] else 0;
725	self.symbols[i] = if (i < 256)
726	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
727	else if (i == 256)
728	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
729	else
730	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
731	}
732	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
733
734	// reset lookup table
735	for (0..self.lookup.len) \|i\| {
736	self.lookup[i] = .{};
737	}
738
739	// assign code to symbols
740	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
741	var code: u16 = 0;
742	var idx: u16 = 0;
743	for (&self.symbols, 0..) \|*sym, pos\| {
744	if (sym.code_bits == 0) continue; // skip unused
745	sym.code = code;
746
747	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
748	const next_idx = next_code >> lookup_shift;
749
750	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
751	if (sym.code_bits <= lookup_bits) {
752	// fill small lookup table
753	for (idx..next_idx) \|j\|
754	self.lookup[j] = sym.*;
755	} else {
756	// insert into linked table starting at root
757	const root = &self.lookup[idx];
758	const root_next = root.next;
759	root.next = @intCast(pos);
760	sym.next = root_next;
761	}
762
763	idx = next_idx;
764	code = next_code;
765	}
766	}
767
768	/// Given the list of code lengths check that it represents a canonical
769	/// Huffman code for n symbols.
770	///
771	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
772	fn checkCompleteness(lens: []const u4) !void {
773	if (alphabet_size == 286)
774	if (lens[256] == 0) return error.MissingEndOfBlockCode;
775
776	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
777	var max: usize = 0;
778	for (lens) \|n\| {
779	if (n == 0) continue;
780	if (n > max) max = n;
781	count[n] += 1;
782	}
783	if (max == 0) // empty tree
784	return;
785
786	// check for an over-subscribed or incomplete set of lengths
787	var left: usize = 1; // one possible code of zero length
788	for (1..count.len) \|len\| {
789	left <<= 1; // one more bit, double codes left
790	if (count[len] > left)
791	return error.OversubscribedHuffmanTree;
792	left -= count[len]; // deduct count from possible codes
793	}
794	if (left > 0) { // left > 0 means incomplete
795	// incomplete code ok only for single length 1 code
796	if (max_code_bits > 7 and max == count[0] + count[1]) return;
797	return error.IncompleteHuffmanTree;
798	}
799	}
800
801	/// Finds symbol for lookup table code.
802	pub fn find(self: *Self, code: u16) !Symbol {
803	// try to find in lookup table
804	const idx = code >> lookup_shift;
805	const sym = self.lookup[idx];
806	if (sym.code_bits != 0) return sym;
807	// if not use linked list of symbols with same prefix
808	return self.findLinked(code, sym.next);
809	}
810
811	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
812	var pos = start;
813	while (pos > 0) {
814	const sym = self.symbols[pos];
815	const shift = max_code_bits - sym.code_bits;
816	// compare code_bits number of upper bits
817	if ((code ^ sym.code) >> shift == 0) return sym;
818	pos = sym.next;
819	}
820	return error.InvalidCode;
821	}
822	};
823	}
824
825	test "init/find" {
826	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
827	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
828	var h: CodegenDecoder = .{};
829	try h.generate(&code_lens);
830
831	const expected = [_]struct {
832	sym: Symbol,
833	code: u16,
834	}{
835	.{
836	.code = 0b00_00000,
837	.sym = .{ .symbol = 3, .code_bits = 2 },
838	},
839	.{
840	.code = 0b01_00000,
841	.sym = .{ .symbol = 18, .code_bits = 2 },
842	},
843	.{
844	.code = 0b100_0000,
845	.sym = .{ .symbol = 1, .code_bits = 3 },
846	},
847	.{
848	.code = 0b101_0000,
849	.sym = .{ .symbol = 4, .code_bits = 3 },
850	},
851	.{
852	.code = 0b110_0000,
853	.sym = .{ .symbol = 17, .code_bits = 3 },
854	},
855	.{
856	.code = 0b1110_000,
857	.sym = .{ .symbol = 0, .code_bits = 4 },
858	},
859	.{
860	.code = 0b1111_000,
861	.sym = .{ .symbol = 16, .code_bits = 4 },
862	},
863	};
864
865	// unused symbols
866	for (0..12) \|i\| {
867	try testing.expectEqual(0, h.symbols[i].code_bits);
868	}
869	// used, from index 12
870	for (expected, 12..) \|e, i\| {
871	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
872	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
873	const sym_from_code = try h.find(e.code);
874	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
875	}
876
877	// All possible codes for each symbol.
878	// Lookup table has 126 elements, to cover all possible 7 bit codes.
879	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
880	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
881
882	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
883	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
884
885	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
886	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
887
888	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
889	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
890
891	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
892	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
893
894	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
895	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
896
897	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
898	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
899	}
900
901	test "encode/decode literals" {
902	var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;
903	for (1..286) \|j\| { // for all different number of codes
904	var enc: flate.HuffmanEncoder = .{
905	.codes = &codes,
906	.freq_cache = undefined,
907	.bit_count = undefined,
908	.lns = undefined,
909	.lfs = undefined,
910	};
911	// create frequencies
912	var freq = [_]u16{0} ** 286;
913	freq[256] = 1; // ensure we have end of block code
914	for (&freq, 1..) \|*f, i\| {
915	if (i % j == 0)
916	f.* = @intCast(i);
917	}
918
919	// encoder from frequencies
920	enc.generate(&freq, 15);
921
922	// get code_lens from encoder
923	var code_lens = [_]u4{0} ** 286;
924	for (code_lens, 0..) \|_, i\| {
925	code_lens[i] = @intCast(enc.codes[i].len);
926	}
927	// generate decoder from code lens
928	var dec: LiteralDecoder = .{};
929	try dec.generate(&code_lens);
930
931	// expect decoder code to match original encoder code
932	for (dec.symbols) \|s\| {
933	if (s.code_bits == 0) continue;
934	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
935	const symbol: u16 = switch (s.kind) {
936	.literal => s.symbol,
937	.end_of_block => 256,
938	.match => @as(u16, s.symbol) + 257,
939	};
940
941	const c = enc.codes[symbol];
942	try testing.expect(c.code == c_code);
943	}
944
945	// find each symbol by code
946	for (enc.codes) \|c\| {
947	if (c.len == 0) continue;
948
949	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
950	const s = try dec.find(s_code);
951	try testing.expect(s.code == s_code);
952	try testing.expect(s.code_bits == c.len);
953	}
954	}
955	}
956
957	test "non compressed block (type 0)" {
958	try testDecompress(.raw, &[_]u8{
959	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
960	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
961	}, "Hello world\n");
962	}
963
964	test "fixed code block (type 1)" {
965	try testDecompress(.raw, &[_]u8{
966	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
967	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
968	}, "Hello world\n");
969	}
970
971	test "dynamic block (type 2)" {
972	try testDecompress(.raw, &[_]u8{
973	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
974	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
975	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
976	}, "ABCDEABCD ABCDEABCD");
977	}
978
979	test "gzip non compressed block (type 0)" {
980	try testDecompress(.gzip, &[_]u8{
981	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
982	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
983	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
984	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
985	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
986	}, "Hello world\n");
987	}
988
989	test "gzip fixed code block (type 1)" {
990	try testDecompress(.gzip, &[_]u8{
991	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
992	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
993	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
994	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
995	}, "Hello world\n");
996	}
997
998	test "gzip dynamic block (type 2)" {
999	try testDecompress(.gzip, &[_]u8{
1000	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
1001	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
1002	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
1003	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
1004	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
1005	}, "ABCDEABCD ABCDEABCD");
1006	}
1007
1008	test "gzip header with name" {
1009	try testDecompress(.gzip, &[_]u8{
1010	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
1011	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
1012	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
1013	}, "Hello world\n");
1014	}
1015
1016	test "zlib decompress non compressed block (type 0)" {
1017	try testDecompress(.zlib, &[_]u8{
1018	0x78, 0b10_0_11100, // zlib header (2 bytes)
1019	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
1020	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
1021	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
1022	}, "Hello world\n");
1023	}
1024
1025	test "failing end-of-stream" {
1026	try testFailure(.raw, @embedFile("testdata/fuzz/end-of-stream.input"), error.EndOfStream);
1027	}
1028	test "failing invalid-distance" {
1029	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-distance.input"), error.InvalidMatch);
1030	}
1031	test "failing invalid-tree01" {
1032	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree);
1033	}
1034	test "failing invalid-tree02" {
1035	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree02.input"), error.EndOfStream);
1036	}
1037	test "failing invalid-tree03" {
1038	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree);
1039	}
1040	test "failing lengths-overflow" {
1041	try testFailure(.raw, @embedFile("testdata/fuzz/lengths-overflow.input"), error.InvalidDynamicBlockHeader);
1042	}
1043	test "failing out-of-codes" {
1044	try testFailure(.raw, @embedFile("testdata/fuzz/out-of-codes.input"), error.InvalidCode);
1045	}
1046	test "failing puff01" {
1047	try testFailure(.raw, @embedFile("testdata/fuzz/puff01.input"), error.WrongStoredBlockNlen);
1048	}
1049	test "failing puff02" {
1050	try testFailure(.raw, @embedFile("testdata/fuzz/puff02.input"), error.EndOfStream);
1051	}
1052	test "failing puff04" {
1053	try testFailure(.raw, @embedFile("testdata/fuzz/puff04.input"), error.InvalidCode);
1054	}
1055	test "failing puff05" {
1056	try testFailure(.raw, @embedFile("testdata/fuzz/puff05.input"), error.EndOfStream);
1057	}
1058	test "failing puff06" {
1059	try testFailure(.raw, @embedFile("testdata/fuzz/puff06.input"), error.EndOfStream);
1060	}
1061	test "failing puff08" {
1062	try testFailure(.raw, @embedFile("testdata/fuzz/puff08.input"), error.InvalidCode);
1063	}
1064	test "failing puff10" {
1065	try testFailure(.raw, @embedFile("testdata/fuzz/puff10.input"), error.InvalidCode);
1066	}
1067	test "failing puff11" {
1068	try testFailure(.raw, @embedFile("testdata/fuzz/puff11.input"), error.EndOfStream);
1069	}
1070	test "failing puff12" {
1071	try testFailure(.raw, @embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader);
1072	}
1073	test "failing puff13" {
1074	try testFailure(.raw, @embedFile("testdata/fuzz/puff13.input"), error.IncompleteHuffmanTree);
1075	}
1076	test "failing puff14" {
1077	try testFailure(.raw, @embedFile("testdata/fuzz/puff14.input"), error.EndOfStream);
1078	}
1079	test "failing puff15" {
1080	try testFailure(.raw, @embedFile("testdata/fuzz/puff15.input"), error.IncompleteHuffmanTree);
1081	}
1082	test "failing puff16" {
1083	try testFailure(.raw, @embedFile("testdata/fuzz/puff16.input"), error.InvalidDynamicBlockHeader);
1084	}
1085	test "failing puff17" {
1086	try testFailure(.raw, @embedFile("testdata/fuzz/puff17.input"), error.MissingEndOfBlockCode);
1087	}
1088	test "failing fuzz1" {
1089	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz1.input"), error.InvalidDynamicBlockHeader);
1090	}
1091	test "failing fuzz2" {
1092	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz2.input"), error.InvalidDynamicBlockHeader);
1093	}
1094	test "failing fuzz3" {
1095	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz3.input"), error.InvalidMatch);
1096	}
1097	test "failing fuzz4" {
1098	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz4.input"), error.OversubscribedHuffmanTree);
1099	}
1100	test "failing puff18" {
1101	try testFailure(.raw, @embedFile("testdata/fuzz/puff18.input"), error.OversubscribedHuffmanTree);
1102	}
1103	test "failing puff19" {
1104	try testFailure(.raw, @embedFile("testdata/fuzz/puff19.input"), error.OversubscribedHuffmanTree);
1105	}
1106	test "failing puff20" {
1107	try testFailure(.raw, @embedFile("testdata/fuzz/puff20.input"), error.OversubscribedHuffmanTree);
1108	}
1109	test "failing puff21" {
1110	try testFailure(.raw, @embedFile("testdata/fuzz/puff21.input"), error.OversubscribedHuffmanTree);
1111	}
1112	test "failing puff22" {
1113	try testFailure(.raw, @embedFile("testdata/fuzz/puff22.input"), error.OversubscribedHuffmanTree);
1114	}
1115	test "failing puff23" {
1116	try testFailure(.raw, @embedFile("testdata/fuzz/puff23.input"), error.OversubscribedHuffmanTree);
1117	}
1118	test "failing puff24" {
1119	try testFailure(.raw, @embedFile("testdata/fuzz/puff24.input"), error.IncompleteHuffmanTree);
1120	}
1121	test "failing puff25" {
1122	try testFailure(.raw, @embedFile("testdata/fuzz/puff25.input"), error.OversubscribedHuffmanTree);
1123	}
1124	test "failing puff26" {
1125	try testFailure(.raw, @embedFile("testdata/fuzz/puff26.input"), error.InvalidDynamicBlockHeader);
1126	}
1127	test "failing puff27" {
1128	try testFailure(.raw, @embedFile("testdata/fuzz/puff27.input"), error.InvalidDynamicBlockHeader);
1129	}
1130
1131	test "deflate-stream" {
1132	try testDecompress(
1133	.raw,
1134	@embedFile("testdata/fuzz/deflate-stream.input"),
1135	@embedFile("testdata/fuzz/deflate-stream.expect"),
1136	);
1137	}
1138
1139	test "empty-distance-alphabet01" {
1140	try testFailure(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet01.input"), error.EndOfStream);
1141	}
1142
1143	test "empty-distance-alphabet02" {
1144	try testDecompress(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet02.input"), "");
1145	}
1146
1147	test "puff03" {
1148	try testDecompress(.raw, @embedFile("testdata/fuzz/puff03.input"), &.{0xa});
1149	}
1150
1151	test "puff09" {
1152	try testDecompress(.raw, @embedFile("testdata/fuzz/puff09.input"), "P");
1153	}
1154
1155	test "bug 18966" {
1156	try testDecompress(
1157	.gzip,
1158	@embedFile("testdata/fuzz/bug_18966.input"),
1159	@embedFile("testdata/fuzz/bug_18966.expect"),
1160	);
1161	}
1162
1163	test "reading into empty buffer" {
1164	// Inspired by https://github.com/ziglang/zig/issues/19895
1165	const input = &[_]u8{
1166	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
1167	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
1168	};
1169	var in: Reader = .fixed(input);
1170	var decomp: Decompress = .init(&in, .raw, &.{});
1171	const r = &decomp.reader;
1172	var bufs: [1][]u8 = .{&.{}};
1173	try testing.expectEqual(0, try r.readVec(&bufs));
1174	}
1175
1176	test "zlib header" {
1177	// Truncated header
1178	try testFailure(.zlib, &[_]u8{0x78}, error.EndOfStream);
1179
1180	// Wrong CM
1181	try testFailure(.zlib, &[_]u8{ 0x79, 0x94 }, error.BadZlibHeader);
1182
1183	// Wrong CINFO
1184	try testFailure(.zlib, &[_]u8{ 0x88, 0x98 }, error.BadZlibHeader);
1185
1186	// Truncated checksum
1187	try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, error.EndOfStream);
1188	}
1189
1190	test "gzip header" {
1191	// Truncated header
1192	try testFailure(.gzip, &[_]u8{ 0x1f, 0x8B }, error.EndOfStream);
1193
1194	// Wrong CM
1195	try testFailure(.gzip, &[_]u8{
1196	0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
1197	0x00, 0x03,
1198	}, error.BadGzipHeader);
1199
1200	// Truncated checksum
1201	try testFailure(.gzip, &[_]u8{
1202	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1203	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
1204	}, error.EndOfStream);
1205
1206	// Truncated initial size field
1207	try testFailure(.gzip, &[_]u8{
1208	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
1209	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
1210	0x00, 0x00, 0x00,
1211	}, error.EndOfStream);
1212
1213	try testDecompress(.gzip, &[_]u8{
1214	// GZIP header
1215	0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
1216	// header.FHCRC (should cover entire header)
1217	0x99, 0xd6,
1218	// GZIP data
1219	0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1220	}, "");
1221	}
1222
1223	test "zlib should not overshoot" {
1224	// Compressed zlib data with extra 4 bytes at the end.
1225	const data = [_]u8{
1226	0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
1227	0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
1228	0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
1229	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
1230	};
1231
1232	var reader: std.Io.Reader = .fixed(&data);
1233
1234	var decompress_buffer: [flate.max_window_len]u8 = undefined;
1235	var decompress: Decompress = .init(&reader, .zlib, &decompress_buffer);
1236	var out: [128]u8 = undefined;
1237
1238	{
1239	const n = try decompress.reader.readSliceShort(&out);
1240	try std.testing.expectEqual(46, n);
1241	try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
1242	}
1243
1244	// 4 bytes after compressed chunk are available in reader.
1245	const n = try reader.readSliceShort(&out);
1246	try std.testing.expectEqual(n, 4);
1247	try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
1248	}
1249
1250	fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void {
1251	var reader: Reader = .fixed(in);
1252	var aw: Writer.Allocating = .init(testing.allocator);
1253	try aw.ensureUnusedCapacity(flate.history_len);
1254	defer aw.deinit();
1255
1256	var decompress: Decompress = .init(&reader, container, &.{});
1257	try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer));
1258	try testing.expectEqual(expected_err, decompress.err orelse return error.TestFailed);
1259	}
1260
1261	fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
1262	var in: std.Io.Reader = .fixed(compressed);
1263	var aw: std.Io.Writer.Allocating = .init(testing.allocator);
1264	try aw.ensureUnusedCapacity(flate.history_len);
1265	defer aw.deinit();
1266
1267	var decompress: Decompress = .init(&in, container, &.{});
1268	_ = try decompress.reader.streamRemaining(&aw.writer);
1269	try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
1270	}

lib/std/compress/flate/HuffmanEncoder.zig created+463

...	@@ -0,0 +1,463 @@
1	const HuffmanEncoder = @This();
2	const std = @import("std");
3	const assert = std.debug.assert;
4	const testing = std.testing;
5
6	codes: []Code,
7	// Reusable buffer with the longest possible frequency table.
8	freq_cache: [max_num_frequencies + 1]LiteralNode,
9	bit_count: [17]u32,
10	lns: []LiteralNode, // sorted by literal, stored to avoid repeated allocation in generate
11	lfs: []LiteralNode, // sorted by frequency, stored to avoid repeated allocation in generate
12
13	pub const LiteralNode = struct {
14	literal: u16,
15	freq: u16,
16
17	pub fn max() LiteralNode {
18	return .{
19	.literal = std.math.maxInt(u16),
20	.freq = std.math.maxInt(u16),
21	};
22	}
23	};
24
25	pub const Code = struct {
26	code: u16 = 0,
27	len: u16 = 0,
28	};
29
30	/// The odd order in which the codegen code sizes are written.
31	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
32	/// The number of codegen codes.
33	pub const codegen_code_count = 19;
34
35	/// The largest distance code.
36	pub const distance_code_count = 30;
37
38	/// Maximum number of literals.
39	pub const max_num_lit = 286;
40
41	/// Max number of frequencies used for a Huffman Code
42	/// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
43	/// The largest of these is max_num_lit.
44	pub const max_num_frequencies = max_num_lit;
45
46	/// Biggest block size for uncompressed block.
47	pub const max_store_block_size = 65535;
48	/// The special code used to mark the end of a block.
49	pub const end_block_marker = 256;
50
51	/// Update this Huffman Code object to be the minimum code for the specified frequency count.
52	///
53	/// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
54	/// max_bits The maximum number of bits to use for any literal.
55	pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
56	var list = self.freq_cache[0 .. freq.len + 1];
57	// Number of non-zero literals
58	var count: u32 = 0;
59	// Set list to be the set of all non-zero literals and their frequencies
60	for (freq, 0..) \|f, i\| {
61	if (f != 0) {
62	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
63	count += 1;
64	} else {
65	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
66	self.codes[i].len = 0;
67	}
68	}
69	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
70
71	list = list[0..count];
72	if (count <= 2) {
73	// Handle the small cases here, because they are awkward for the general case code. With
74	// two or fewer literals, everything has bit length 1.
75	for (list, 0..) \|node, i\| {
76	// "list" is in order of increasing literal value.
77	self.codes[node.literal] = .{
78	.code = @intCast(i),
79	.len = 1,
80	};
81	}
82	return;
83	}
84	self.lfs = list;
85	std.mem.sort(LiteralNode, self.lfs, {}, byFreq);
86
87	// Get the number of literals for each bit count
88	const bit_count = self.bitCounts(list, max_bits);
89	// And do the assignment
90	self.assignEncodingAndSize(bit_count, list);
91	}
92
93	pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
94	var total: u32 = 0;
95	for (freq, 0..) \|f, i\| {
96	if (f != 0) {
97	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
98	}
99	}
100	return total;
101	}
102
103	/// Return the number of literals assigned to each bit size in the Huffman encoding
104	///
105	/// This method is only called when list.len >= 3
106	/// The cases of 0, 1, and 2 literals are handled by special case code.
107	///
108	/// list: An array of the literals with non-zero frequencies
109	/// and their associated frequencies. The array is in order of increasing
110	/// frequency, and has as its last element a special element with frequency
111	/// `math.maxInt(i32)`
112	///
113	/// max_bits: The maximum number of bits that should be used to encode any literal.
114	/// Must be less than 16.
115	///
116	/// Returns an integer array in which array[i] indicates the number of literals
117	/// that should be encoded in i bits.
118	fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
119	var max_bits = max_bits_to_use;
120	const n = list.len;
121	const max_bits_limit = 16;
122
123	assert(max_bits < max_bits_limit);
124
125	// The tree can't have greater depth than n - 1, no matter what. This
126	// saves a little bit of work in some small cases
127	max_bits = @min(max_bits, n - 1);
128
129	// Create information about each of the levels.
130	// A bogus "Level 0" whose sole purpose is so that
131	// level1.prev.needed == 0. This makes level1.next_pair_freq
132	// be a legitimate value that never gets chosen.
133	var levels: [max_bits_limit]LevelInfo = std.mem.zeroes([max_bits_limit]LevelInfo);
134	// leaf_counts[i] counts the number of literals at the left
135	// of ancestors of the rightmost node at level i.
136	// leaf_counts[i][j] is the number of literals at the left
137	// of the level j ancestor.
138	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(@splat(0));
139
140	{
141	var level = @as(u32, 1);
142	while (level <= max_bits) : (level += 1) {
143	// For every level, the first two items are the first two characters.
144	// We initialize the levels as if we had already figured this out.
145	levels[level] = LevelInfo{
146	.level = level,
147	.last_freq = list[1].freq,
148	.next_char_freq = list[2].freq,
149	.next_pair_freq = list[0].freq + list[1].freq,
150	.needed = 0,
151	};
152	leaf_counts[level][level] = 2;
153	if (level == 1) {
154	levels[level].next_pair_freq = std.math.maxInt(i32);
155	}
156	}
157	}
158
159	// We need a total of 2*n - 2 items at top level and have already generated 2.
160	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
161
162	{
163	var level = max_bits;
164	while (true) {
165	var l = &levels[level];
166	if (l.next_pair_freq == std.math.maxInt(i32) and l.next_char_freq == std.math.maxInt(i32)) {
167	// We've run out of both leaves and pairs.
168	// End all calculations for this level.
169	// To make sure we never come back to this level or any lower level,
170	// set next_pair_freq impossibly large.
171	l.needed = 0;
172	levels[level + 1].next_pair_freq = std.math.maxInt(i32);
173	level += 1;
174	continue;
175	}
176
177	const prev_freq = l.last_freq;
178	if (l.next_char_freq < l.next_pair_freq) {
179	// The next item on this row is a leaf node.
180	const next = leaf_counts[level][level] + 1;
181	l.last_freq = l.next_char_freq;
182	// Lower leaf_counts are the same of the previous node.
183	leaf_counts[level][level] = next;
184	if (next >= list.len) {
185	l.next_char_freq = LiteralNode.max().freq;
186	} else {
187	l.next_char_freq = list[next].freq;
188	}
189	} else {
190	// The next item on this row is a pair from the previous row.
191	// next_pair_freq isn't valid until we generate two
192	// more values in the level below
193	l.last_freq = l.next_pair_freq;
194	// Take leaf counts from the lower level, except counts[level] remains the same.
195	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
196	levels[l.level - 1].needed = 2;
197	}
198
199	l.needed -= 1;
200	if (l.needed == 0) {
201	// We've done everything we need to do for this level.
202	// Continue calculating one level up. Fill in next_pair_freq
203	// of that level with the sum of the two nodes we've just calculated on
204	// this level.
205	if (l.level == max_bits) {
206	// All done!
207	break;
208	}
209	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
210	level += 1;
211	} else {
212	// If we stole from below, move down temporarily to replenish it.
213	while (levels[level - 1].needed > 0) {
214	level -= 1;
215	if (level == 0) {
216	break;
217	}
218	}
219	}
220	}
221	}
222
223	// Somethings is wrong if at the end, the top level is null or hasn't used
224	// all of the leaves.
225	assert(leaf_counts[max_bits][max_bits] == n);
226
227	var bit_count = self.bit_count[0 .. max_bits + 1];
228	var bits: u32 = 1;
229	const counts = &leaf_counts[max_bits];
230	{
231	var level = max_bits;
232	while (level > 0) : (level -= 1) {
233	// counts[level] gives the number of literals requiring at least "bits"
234	// bits to encode.
235	bit_count[bits] = counts[level] - counts[level - 1];
236	bits += 1;
237	if (level == 0) {
238	break;
239	}
240	}
241	}
242	return bit_count;
243	}
244
245	/// Look at the leaves and assign them a bit count and an encoding as specified
246	/// in RFC 1951 3.2.2
247	fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
248	var code = @as(u16, 0);
249	var list = list_arg;
250
251	for (bit_count, 0..) \|bits, n\| {
252	code <<= 1;
253	if (n == 0 or bits == 0) {
254	continue;
255	}
256	// The literals list[list.len-bits] .. list[list.len-bits]
257	// are encoded using "bits" bits, and get the values
258	// code, code + 1, .... The code values are
259	// assigned in literal order (not frequency order).
260	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
261
262	self.lns = chunk;
263	std.mem.sort(LiteralNode, self.lns, {}, byLiteral);
264
265	for (chunk) \|node\| {
266	self.codes[node.literal] = .{
267	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
268	.len = @as(u16, @intCast(n)),
269	};
270	code += 1;
271	}
272	list = list[0 .. list.len - @as(u32, @intCast(bits))];
273	}
274	}
275
276	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
277	_ = context;
278	if (a.freq == b.freq) {
279	return a.literal < b.literal;
280	}
281	return a.freq < b.freq;
282	}
283
284	/// Describes the state of the constructed tree for a given depth.
285	const LevelInfo = struct {
286	/// Our level. for better printing
287	level: u32,
288	/// The frequency of the last node at this level
289	last_freq: u32,
290	/// The frequency of the next character to add to this level
291	next_char_freq: u32,
292	/// The frequency of the next pair (from level below) to add to this level.
293	/// Only valid if the "needed" value of the next lower level is 0.
294	next_pair_freq: u32,
295	/// The number of chains remaining to generate for this level before moving
296	/// up to the next level
297	needed: u32,
298	};
299
300	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
301	_ = context;
302	return a.literal < b.literal;
303	}
304
305	/// Reverse bit-by-bit a N-bit code.
306	fn bitReverse(comptime T: type, value: T, n: usize) T {
307	const r = @bitReverse(value);
308	return r >> @as(std.math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
309	}
310
311	test bitReverse {
312	const ReverseBitsTest = struct {
313	in: u16,
314	bit_count: u5,
315	out: u16,
316	};
317
318	const reverse_bits_tests = [_]ReverseBitsTest{
319	.{ .in = 1, .bit_count = 1, .out = 1 },
320	.{ .in = 1, .bit_count = 2, .out = 2 },
321	.{ .in = 1, .bit_count = 3, .out = 4 },
322	.{ .in = 1, .bit_count = 4, .out = 8 },
323	.{ .in = 1, .bit_count = 5, .out = 16 },
324	.{ .in = 17, .bit_count = 5, .out = 17 },
325	.{ .in = 257, .bit_count = 9, .out = 257 },
326	.{ .in = 29, .bit_count = 5, .out = 23 },
327	};
328
329	for (reverse_bits_tests) \|h\| {
330	const v = bitReverse(u16, h.in, h.bit_count);
331	try std.testing.expectEqual(h.out, v);
332	}
333	}
334
335	/// Generates a HuffmanCode corresponding to the fixed literal table
336	pub fn fixedLiteralEncoder(codes: *[max_num_frequencies]Code) HuffmanEncoder {
337	var h: HuffmanEncoder = undefined;
338	h.codes = codes;
339	var ch: u16 = 0;
340
341	while (ch < max_num_frequencies) : (ch += 1) {
342	var bits: u16 = undefined;
343	var size: u16 = undefined;
344	switch (ch) {
345	0...143 => {
346	// size 8, 000110000 .. 10111111
347	bits = ch + 48;
348	size = 8;
349	},
350	144...255 => {
351	// size 9, 110010000 .. 111111111
352	bits = ch + 400 - 144;
353	size = 9;
354	},
355	256...279 => {
356	// size 7, 0000000 .. 0010111
357	bits = ch - 256;
358	size = 7;
359	},
360	else => {
361	// size 8, 11000000 .. 11000111
362	bits = ch + 192 - 280;
363	size = 8;
364	},
365	}
366	h.codes[ch] = .{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
367	}
368	return h;
369	}
370
371	pub fn fixedDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
372	var h: HuffmanEncoder = undefined;
373	h.codes = codes;
374	for (h.codes, 0..) \|_, ch\| {
375	h.codes[ch] = .{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
376	}
377	return h;
378	}
379
380	pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
381	var distance_freq: [distance_code_count]u16 = @splat(0);
382	distance_freq[0] = 1;
383	// huff_distance is a static distance encoder used for huffman only encoding.
384	// It can be reused since we will not be encoding distance values.
385	var h: HuffmanEncoder = .{};
386	h.codes = codes;
387	h.generate(distance_freq[0..], 15);
388	return h;
389	}
390
391	test "generate a Huffman code for the fixed literal table specific to Deflate" {
392	var codes: [max_num_frequencies]Code = undefined;
393	const enc: HuffmanEncoder = .fixedLiteralEncoder(&codes);
394	for (enc.codes) \|c\| {
395	switch (c.len) {
396	7 => {
397	const v = @bitReverse(@as(u7, @intCast(c.code)));
398	try testing.expect(v <= 0b0010111);
399	},
400	8 => {
401	const v = @bitReverse(@as(u8, @intCast(c.code)));
402	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
403	(v >= 0b11000000 and v <= 11000111));
404	},
405	9 => {
406	const v = @bitReverse(@as(u9, @intCast(c.code)));
407	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
408	},
409	else => unreachable,
410	}
411	}
412	}
413
414	test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
415	var codes: [distance_code_count]Code = undefined;
416	const enc = fixedDistanceEncoder(&codes);
417	for (enc.codes) \|c\| {
418	const v = @bitReverse(@as(u5, @intCast(c.code)));
419	try testing.expect(v <= 29);
420	try testing.expect(c.len == 5);
421	}
422	}
423
424	pub const fixed_codes = [_]u8{
425	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
426	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
427	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
428	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
429	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
430	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
431	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
432	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
433	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
434	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
435	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
436	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
437	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
438	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
439	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
440	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
441	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
442	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
443	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
444	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
445	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
446	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
447	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
448	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
449	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
450	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
451	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
452	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
453	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
454	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
455	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
456	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
457	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
458	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
459	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
460	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
461	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
462	0b10100011,
463	};

lib/std/compress/flate/Lookup.zig+21-16

 @@ -5,22 +5,27 @@
 const std = @import("std");
 const testing = std.testing;
 const expect = testing.expect;
-const consts = @import("consts.zig");
+const flate = @import("../flate.zig");
+const Token = @import("Token.zig");
-const Self = @This();
+const Lookup = @This();
 const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
-const chain_len = 2 * consts.history.len;
+const chain_len = 2 * flate.history_len;
+pub const bits = 15;
+pub const len = 1 << bits;
+pub const shift = 32 - bits;
 // Maps hash => first position
-head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,
+head: [len]u16 = [_]u16{0} ** len,
 // Maps position => previous positions for the same hash value
 chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
 // Calculates hash of the 4 bytes from data.
 // Inserts `pos` position of that hash in the lookup tables.
 // Returns previous location with the same hash value.
-pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
+pub fn add(self: *Lookup, data: []const u8, pos: u16) u16 {
     if (data.len < 4) return 0;
     const h = hash(data[0..4]);
     return self.set(h, pos);
 @@ -28,11 +33,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
 // Returns previous location with the same hash value given the current
 // position.
-pub fn prev(self: *Self, pos: u16) u16 {
+pub fn prev(self: *Lookup, pos: u16) u16 {
     return self.chain[pos];
 }
-fn set(self: *Self, h: u32, pos: u16) u16 {
+fn set(self: *Lookup, h: u32, pos: u16) u16 {
     const p = self.head[h];
     self.head[h] = pos;
     self.chain[pos] = p;
 @@ -40,7 +45,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 {
 }
 // Slide all positions in head and chain for `n`
-pub fn slide(self: *Self, n: u16) void {
+pub fn slide(self: *Lookup, n: u16) void {
     for (&self.head) |*v| {
         v.* -|= n;
     }
 @@ -52,8 +57,8 @@ pub fn slide(self: *Self, n: u16) void {
 // Add `len` 4 bytes hashes from `data` into lookup.
 // Position of the first byte is `pos`.
-pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
+pub fn bulkAdd(self: *Lookup, data: []const u8, length: u16, pos: u16) void {
-    if (len == 0 or data.len < consts.match.min_length) {
+    if (length == 0 or data.len < Token.min_length) {
         return;
     }
     var hb =
 @@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
     _ = self.set(hashu(hb), pos);
     var i = pos;
-    for (4..@min(len + 3, data.len)) |j| {
+    for (4..@min(length + 3, data.len)) |j| {
         hb = (hb << 8) | @as(u32, data[j]);
         i += 1;
         _ = self.set(hashu(hb), i);
 @@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 {
 }
 fn hashu(v: u32) u32 {
-    return @intCast((v *% prime4) >> consts.lookup.shift);
+    return @intCast((v *% prime4) >> shift);
 }
 test add {
 @@ -91,7 +96,7 @@ test add {
         0x01, 0x02, 0x03,
     };
-    var h: Self = .{};
+    var h: Lookup = .{};
     for (data, 0..) |_, i| {
         const p = h.add(data[i..], @intCast(i));
         if (i >= 8 and i < 24) {
 @@ -101,7 +106,7 @@ test add {
         }
     }
-    const v = Self.hash(data[2 .. 2 + 4]);
+    const v = Lookup.hash(data[2 .. 2 + 4]);
     try expect(h.head[v] == 2 + 16);
     try expect(h.chain[2 + 16] == 2 + 8);
     try expect(h.chain[2 + 8] == 2);
 @@ -111,13 +116,13 @@ test bulkAdd {
     const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
     // one by one
-    var h: Self = .{};
+    var h: Lookup = .{};
     for (data, 0..) |_, i| {
         _ = h.add(data[i..], @intCast(i));
     }
     // in bulk
-    var bh: Self = .{};
+    var bh: Lookup = .{};
     bh.bulkAdd(data, data.len, 0);
     try testing.expectEqualSlices(u16, &h.head, &bh.head);

lib/std/compress/flate/SlidingWindow.zig deleted-160

...	@@ -1,160 +0,0 @@
1	//! Used in deflate (compression), holds uncompressed data form which Tokens are
2	//! produces. In combination with Lookup it is used to find matches in history data.
3	//!
4	const std = @import("std");
5	const consts = @import("consts.zig");
6
7	const expect = testing.expect;
8	const assert = std.debug.assert;
9	const testing = std.testing;
10
11	const hist_len = consts.history.len;
12	const buffer_len = 2 * hist_len;
13	const min_lookahead = consts.match.min_length + consts.match.max_length;
14	const max_rp = buffer_len - min_lookahead;
15
16	const Self = @This();
17
18	buffer: [buffer_len]u8 = undefined,
19	wp: usize = 0, // write position
20	rp: usize = 0, // read position
21	fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23	/// Returns number of bytes written, or 0 if buffer is full and need to slide.
24	pub fn write(self: *Self, buf: []const u8) usize {
25	if (self.rp >= max_rp) return 0; // need to slide
26
27	const n = @min(buf.len, buffer_len - self.wp);
28	@memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29	self.wp += n;
30	return n;
31	}
32
33	/// Slide buffer for hist_len.
34	/// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35	/// Returns number of bytes removed.
36	pub fn slide(self: *Self) u16 {
37	assert(self.rp >= max_rp and self.wp >= self.rp);
38	const n = self.wp - hist_len;
39	@memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40	self.rp -= hist_len;
41	self.wp -= hist_len;
42	self.fp -= hist_len;
43	return @intCast(n);
44	}
45
46	/// Data from the current position (read position). Those part of the buffer is
47	/// not converted to tokens yet.
48	fn lookahead(self: *Self) []const u8 {
49	assert(self.wp >= self.rp);
50	return self.buffer[self.rp..self.wp];
51	}
52
53	/// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54	/// preserved otherwise preserves enough data for the longest match. Returns
55	/// null if there is not enough data.
56	pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57	const min: usize = if (should_flush) 0 else min_lookahead;
58	const lh = self.lookahead();
59	return if (lh.len > min) lh else null;
60	}
61
62	/// Advances read position, shrinks lookahead.
63	pub fn advance(self: *Self, n: u16) void {
64	assert(self.wp >= self.rp + n);
65	self.rp += n;
66	}
67
68	/// Returns writable part of the buffer, where new uncompressed data can be
69	/// written.
70	pub fn writable(self: *Self) []u8 {
71	return self.buffer[self.wp..];
72	}
73
74	/// Notification of what part of writable buffer is filled with data.
75	pub fn written(self: *Self, n: usize) void {
76	self.wp += n;
77	}
78
79	/// Finds match length between previous and current position.
80	/// Used in hot path!
81	pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82	const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83	// lookahead buffers from previous and current positions
84	const prev_lh = self.buffer[prev_pos..][0..max_len];
85	const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87	// If we already have match (min_len > 0),
88	// test the first byte above previous len a[min_len] != b[min_len]
89	// and then all the bytes from that position to zero.
90	// That is likely positions to find difference than looping from first bytes.
91	var i: usize = min_len;
92	if (i > 0) {
93	if (max_len <= i) return 0;
94	while (true) {
95	if (prev_lh[i] != curr_lh[i]) return 0;
96	if (i == 0) break;
97	i -= 1;
98	}
99	i = min_len;
100	}
101	while (i < max_len) : (i += 1)
102	if (prev_lh[i] != curr_lh[i]) break;
103	return if (i >= consts.match.min_length) @intCast(i) else 0;
104	}
105
106	/// Current position of non-compressed data. Data before rp are already converted
107	/// to tokens.
108	pub fn pos(self: *Self) u16 {
109	return @intCast(self.rp);
110	}
111
112	/// Notification that token list is cleared.
113	pub fn flush(self: *Self) void {
114	self.fp = @intCast(self.rp);
115	}
116
117	/// Part of the buffer since last flush or null if there was slide in between (so
118	/// fp becomes negative).
119	pub fn tokensBuffer(self: *Self) ?[]const u8 {
120	assert(self.fp <= self.rp);
121	if (self.fp < 0) return null;
122	return self.buffer[@intCast(self.fp)..self.rp];
123	}
124
125	test match {
126	const data = "Blah blah blah blah blah!";
127	var win: Self = .{};
128	try expect(win.write(data) == data.len);
129	try expect(win.wp == data.len);
130	try expect(win.rp == 0);
131
132	// length between l symbols
133	try expect(win.match(1, 6, 0) == 18);
134	try expect(win.match(1, 11, 0) == 13);
135	try expect(win.match(1, 16, 0) == 8);
136	try expect(win.match(1, 21, 0) == 0);
137
138	// position 15 = "blah blah!"
139	// position 20 = "blah!"
140	try expect(win.match(15, 20, 0) == 4);
141	try expect(win.match(15, 20, 3) == 4);
142	try expect(win.match(15, 20, 4) == 0);
143	}
144
145	test slide {
146	var win: Self = .{};
147	win.wp = Self.buffer_len - 11;
148	win.rp = Self.buffer_len - 111;
149	win.buffer[win.rp] = 0xab;
150	try expect(win.lookahead().len == 100);
151	try expect(win.tokensBuffer().?.len == win.rp);
152
153	const n = win.slide();
154	try expect(n == 32757);
155	try expect(win.buffer[win.rp] == 0xab);
156	try expect(win.rp == Self.hist_len - 111);
157	try expect(win.wp == Self.hist_len - 11);
158	try expect(win.lookahead().len == 100);
159	try expect(win.tokensBuffer() == null);
160	}

lib/std/compress/flate/Token.zig+13-7

 @@ -6,7 +6,6 @@ const std = @import("std");
 const assert = std.debug.assert;
 const print = std.debug.print;
 const expect = std.testing.expect;
-const consts = @import("consts.zig").match;
 const Token = @This();
 @@ -21,16 +20,23 @@ dist: u15 = 0,
 len_lit: u8 = 0,
 kind: Kind = .literal,
+pub const base_length = 3; // smallest match length per the RFC section 3.2.5
+pub const min_length = 4; // min length used in this algorithm
+pub const max_length = 258;
+pub const min_distance = 1;
+pub const max_distance = std.compress.flate.history_len;
 pub fn literal(t: Token) u8 {
     return t.len_lit;
 }
 pub fn distance(t: Token) u16 {
-    return @as(u16, t.dist) + consts.min_distance;
+    return @as(u16, t.dist) + min_distance;
 }
 pub fn length(t: Token) u16 {
-    return @as(u16, t.len_lit) + consts.base_length;
+    return @as(u16, t.len_lit) + base_length;
 }
 pub fn initLiteral(lit: u8) Token {
 @@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token {
 // distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
 // length range 3 - 258, stored in len_lit as 0 - 255 (u8)
 pub fn initMatch(dist: u16, len: u16) Token {
-    assert(len >= consts.min_length and len <= consts.max_length);
+    assert(len >= min_length and len <= max_length);
-    assert(dist >= consts.min_distance and dist <= consts.max_distance);
+    assert(dist >= min_distance and dist <= max_distance);
     return .{
         .kind = .match,
-        .dist = @intCast(dist - consts.min_distance),
+        .dist = @intCast(dist - min_distance),
-        .len_lit = @intCast(len - consts.base_length),
+        .len_lit = @intCast(len - base_length),
     };
 }

lib/std/compress/flate/bit_reader.zig deleted-422

...	@@ -1,422 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) {
6	return BitReader(T, @TypeOf(reader)).init(reader);
7	}
8
9	pub fn BitReader64(comptime ReaderType: type) type {
10	return BitReader(u64, ReaderType);
11	}
12
13	pub fn BitReader32(comptime ReaderType: type) type {
14	return BitReader(u32, ReaderType);
15	}
16
17	/// Bit reader used during inflate (decompression). Has internal buffer of 64
18	/// bits which shifts right after bits are consumed. Uses forward_reader to fill
19	/// that internal buffer when needed.
20	///
21	/// readF is the core function. Supports few different ways of getting bits
22	/// controlled by flags. In hot path we try to avoid checking whether we need to
23	/// fill buffer from forward_reader by calling fill in advance and readF with
24	/// buffered flag set.
25	///
26	pub fn BitReader(comptime T: type, comptime ReaderType: type) type {
27	assert(T == u32 or T == u64);
28	const t_bytes: usize = @sizeOf(T);
29	const Tshift = if (T == u64) u6 else u5;
30
31	return struct {
32	// Underlying reader used for filling internal bits buffer
33	forward_reader: ReaderType = undefined,
34	// Internal buffer of 64 bits
35	bits: T = 0,
36	// Number of bits in the buffer
37	nbits: u32 = 0,
38
39	const Self = @This();
40
41	pub const Error = ReaderType.Error \|\| error{EndOfStream};
42
43	pub fn init(rdr: ReaderType) Self {
44	var self = Self{ .forward_reader = rdr };
45	self.fill(1) catch {};
46	return self;
47	}
48
49	/// Try to have `nice` bits are available in buffer. Reads from
50	/// forward reader if there is no `nice` bits in buffer. Returns error
51	/// if end of forward stream is reached and internal buffer is empty.
52	/// It will not error if less than `nice` bits are in buffer, only when
53	/// all bits are exhausted. During inflate we usually know what is the
54	/// maximum bits for the next step but usually that step will need less
55	/// bits to decode. So `nice` is not hard limit, it will just try to have
56	/// that number of bits available. If end of forward stream is reached
57	/// it may be some extra zero bits in buffer.
58	pub inline fn fill(self: *Self, nice: u6) !void {
59	if (self.nbits >= nice and nice != 0) {
60	return; // We have enough bits
61	}
62	// Read more bits from forward reader
63
64	// Number of empty bytes in bits, round nbits to whole bytes.
65	const empty_bytes =
66	@as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
67	(self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
68
69	var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
70	const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
71	if (bytes_read > 0) {
72	const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
73	self.bits \|= u << @as(Tshift, @intCast(self.nbits));
74	self.nbits += 8 * @as(u8, @intCast(bytes_read));
75	return;
76	}
77
78	if (self.nbits == 0)
79	return error.EndOfStream;
80	}
81
82	/// Read exactly buf.len bytes into buf.
83	pub fn readAll(self: *Self, buf: []u8) !void {
84	assert(self.alignBits() == 0); // internal bits must be at byte boundary
85
86	// First read from internal bits buffer.
87	var n: usize = 0;
88	while (self.nbits > 0 and n < buf.len) {
89	buf[n] = try self.readF(u8, flag.buffered);
90	n += 1;
91	}
92	// Then use forward reader for all other bytes.
93	try self.forward_reader.readNoEof(buf[n..]);
94	}
95
96	pub const flag = struct {
97	pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer
98	pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before
99	pub const reverse: u3 = 0b100; // bit reverse read bits
100	};
101
102	/// Alias for readF(U, 0).
103	pub fn read(self: *Self, comptime U: type) !U {
104	return self.readF(U, 0);
105	}
106
107	/// Alias for readF with flag.peak set.
108	pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U {
109	return self.readF(U, how \| flag.peek);
110	}
111
112	/// Read with flags provided.
113	pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
114	if (U == T) {
115	assert(how == 0);
116	assert(self.alignBits() == 0);
117	try self.fill(@bitSizeOf(T));
118	if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
119	const v = self.bits;
120	self.nbits = 0;
121	self.bits = 0;
122	return v;
123	}
124	const n: Tshift = @bitSizeOf(U);
125	switch (how) {
126	0 => { // `normal` read
127	try self.fill(n); // ensure that there are n bits in the buffer
128	const u: U = @truncate(self.bits); // get n bits
129	try self.shift(n); // advance buffer for n
130	return u;
131	},
132	(flag.peek) => { // no shift, leave bits in the buffer
133	try self.fill(n);
134	return @truncate(self.bits);
135	},
136	flag.buffered => { // no fill, assume that buffer has enough bits
137	const u: U = @truncate(self.bits);
138	try self.shift(n);
139	return u;
140	},
141	(flag.reverse) => { // same as 0 with bit reverse
142	try self.fill(n);
143	const u: U = @truncate(self.bits);
144	try self.shift(n);
145	return @bitReverse(u);
146	},
147	(flag.peek \| flag.reverse) => {
148	try self.fill(n);
149	return @bitReverse(@as(U, @truncate(self.bits)));
150	},
151	(flag.buffered \| flag.reverse) => {
152	const u: U = @truncate(self.bits);
153	try self.shift(n);
154	return @bitReverse(u);
155	},
156	(flag.peek \| flag.buffered) => {
157	return @truncate(self.bits);
158	},
159	(flag.peek \| flag.buffered \| flag.reverse) => {
160	return @bitReverse(@as(U, @truncate(self.bits)));
161	},
162	}
163	}
164
165	/// Read n number of bits.
166	/// Only buffered flag can be used in how.
167	pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
168	switch (how) {
169	0 => {
170	try self.fill(n);
171	},
172	flag.buffered => {},
173	else => unreachable,
174	}
175	const mask: u16 = (@as(u16, 1) << n) - 1;
176	const u: u16 = @as(u16, @truncate(self.bits)) & mask;
177	try self.shift(n);
178	return u;
179	}
180
181	/// Advance buffer for n bits.
182	pub fn shift(self: *Self, n: Tshift) !void {
183	if (n > self.nbits) return error.EndOfStream;
184	self.bits >>= n;
185	self.nbits -= n;
186	}
187
188	/// Skip n bytes.
189	pub fn skipBytes(self: *Self, n: u16) !void {
190	for (0..n) \|_\| {
191	try self.fill(8);
192	try self.shift(8);
193	}
194	}
195
196	// Number of bits to align stream to the byte boundary.
197	fn alignBits(self: *Self) u3 {
198	return @intCast(self.nbits & 0x7);
199	}
200
201	/// Align stream to the byte boundary.
202	pub fn alignToByte(self: *Self) void {
203	const ab = self.alignBits();
204	if (ab > 0) self.shift(ab) catch unreachable;
205	}
206
207	/// Skip zero terminated string.
208	pub fn skipStringZ(self: *Self) !void {
209	while (true) {
210	if (try self.readF(u8, 0) == 0) break;
211	}
212	}
213
214	/// Read deflate fixed fixed code.
215	/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
216	/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
217	/// Lit Value Bits Codes
218	/// --------- ---- -----
219	/// 0 - 143 8 00110000 through
220	/// 10111111
221	/// 144 - 255 9 110010000 through
222	/// 111111111
223	/// 256 - 279 7 0000000 through
224	/// 0010111
225	/// 280 - 287 8 11000000 through
226	/// 11000111
227	pub fn readFixedCode(self: *Self) !u16 {
228	try self.fill(7 + 2);
229	const code7 = try self.readF(u7, flag.buffered \| flag.reverse);
230	if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
231	return @as(u16, code7) + 256;
232	} else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
233	return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000;
234	} else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
235	return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280;
236	} else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
237	return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered \| flag.reverse)) + 144;
238	}
239	}
240	};
241	}
242
243	test "readF" {
244	var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
245	var br = bitReader(u64, fbs.reader());
246	const F = BitReader64(@TypeOf(fbs.reader())).flag;
247
248	try testing.expectEqual(@as(u8, 48), br.nbits);
249	try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
250
251	try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
252	try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
253	try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
254	try testing.expectEqual(@as(u3, 5), br.alignBits());
255
256	try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110);
257	try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110);
258	try br.shift(9);
259	try testing.expectEqual(@as(u8, 36), br.nbits);
260	try testing.expectEqual(@as(u3, 4), br.alignBits());
261
262	try testing.expect(try br.readF(u4, 0) == 0b0100);
263	try testing.expectEqual(@as(u8, 32), br.nbits);
264	try testing.expectEqual(@as(u3, 0), br.alignBits());
265
266	try br.shift(1);
267	try testing.expectEqual(@as(u3, 7), br.alignBits());
268	try br.shift(1);
269	try testing.expectEqual(@as(u3, 6), br.alignBits());
270	br.alignToByte();
271	try testing.expectEqual(@as(u3, 0), br.alignBits());
272
273	try testing.expectEqual(@as(u64, 0xc9), br.bits);
274	try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
275	try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
276	}
277
278	test "read block type 1 data" {
279	inline for ([_]type{ u64, u32 }) \|T\| {
280	const data = [_]u8{
281	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
282	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
283	0x0c, 0x01, 0x02, 0x03, //
284	0xaa, 0xbb, 0xcc, 0xdd,
285	};
286	var fbs = std.io.fixedBufferStream(&data);
287	var br = bitReader(T, fbs.reader());
288	const F = BitReader(T, @TypeOf(fbs.reader())).flag;
289
290	try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
291	try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
292
293	for ("Hello world\n") \|c\| {
294	try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
295	}
296	try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
297	br.alignToByte();
298	try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
299	try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
300	try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
301	}
302	}
303
304	test "shift/fill" {
305	const data = [_]u8{
306	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
307	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308	};
309	var fbs = std.io.fixedBufferStream(&data);
310	var br = bitReader(u64, fbs.reader());
311
312	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
313	try br.shift(8);
314	try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
315	try br.fill(60); // fill with 1 byte
316	try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
317	try br.shift(8 * 4 + 4);
318	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
319
320	try br.fill(60); // fill with 4 bytes (shift by 4)
321	try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
322	try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
323
324	try br.shift(@intCast(br.nbits)); // clear buffer
325	try br.fill(8); // refill with the rest of the bytes
326	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
327	}
328
329	test "readAll" {
330	inline for ([_]type{ u64, u32 }) \|T\| {
331	const data = [_]u8{
332	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
333	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
334	};
335	var fbs = std.io.fixedBufferStream(&data);
336	var br = bitReader(T, fbs.reader());
337
338	switch (T) {
339	u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
340	u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
341	else => unreachable,
342	}
343
344	var out: [16]u8 = undefined;
345	try br.readAll(out[0..]);
346	try testing.expect(br.nbits == 0);
347	try testing.expect(br.bits == 0);
348
349	try testing.expectEqualSlices(u8, data[0..16], &out);
350	}
351	}
352
353	test "readFixedCode" {
354	inline for ([_]type{ u64, u32 }) \|T\| {
355	const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
356
357	var fbs = std.io.fixedBufferStream(&fixed_codes);
358	var rdr = bitReader(T, fbs.reader());
359
360	for (0..286) \|c\| {
361	try testing.expectEqual(c, try rdr.readFixedCode());
362	}
363	try testing.expect(rdr.nbits == 0);
364	}
365	}
366
367	test "u32 leaves no bits on u32 reads" {
368	const data = [_]u8{
369	0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
370	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
371	};
372	var fbs = std.io.fixedBufferStream(&data);
373	var br = bitReader(u32, fbs.reader());
374
375	_ = try br.read(u3);
376	try testing.expectEqual(29, br.nbits);
377	br.alignToByte();
378	try testing.expectEqual(24, br.nbits);
379	try testing.expectEqual(0x04_03_02_01, try br.read(u32));
380	try testing.expectEqual(0, br.nbits);
381	try testing.expectEqual(0x08_07_06_05, try br.read(u32));
382	try testing.expectEqual(0, br.nbits);
383
384	_ = try br.read(u9);
385	try testing.expectEqual(23, br.nbits);
386	br.alignToByte();
387	try testing.expectEqual(16, br.nbits);
388	try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
389	try testing.expectEqual(0, br.nbits);
390	}
391
392	test "u64 need fill after alignToByte" {
393	const data = [_]u8{
394	0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
395	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
396	};
397
398	// without fill
399	var fbs = std.io.fixedBufferStream(&data);
400	var br = bitReader(u64, fbs.reader());
401	_ = try br.read(u23);
402	try testing.expectEqual(41, br.nbits);
403	br.alignToByte();
404	try testing.expectEqual(40, br.nbits);
405	try testing.expectEqual(0x06_05_04_03, try br.read(u32));
406	try testing.expectEqual(8, br.nbits);
407	try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
408	try testing.expectEqual(32, br.nbits);
409
410	// fill after align ensures all bits filled
411	fbs.reset();
412	br = bitReader(u64, fbs.reader());
413	_ = try br.read(u23);
414	try testing.expectEqual(41, br.nbits);
415	br.alignToByte();
416	try br.fill(0);
417	try testing.expectEqual(64, br.nbits);
418	try testing.expectEqual(0x06_05_04_03, try br.read(u32));
419	try testing.expectEqual(32, br.nbits);
420	try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
421	try testing.expectEqual(0, br.nbits);
422	}

lib/std/compress/flate/bit_writer.zig deleted-99

...	@@ -1,99 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	/// Bit writer for use in deflate (compression).
5	///
6	/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7	/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8	/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9	///
10	pub fn BitWriter(comptime WriterType: type) type {
11	// buffer_flush_size indicates the buffer size
12	// after which bytes are flushed to the writer.
13	// Should preferably be a multiple of 6, since
14	// we accumulate 6 bytes between writes to the buffer.
15	const buffer_flush_size = 240;
16
17	// buffer_size is the actual output byte buffer size.
18	// It must have additional headroom for a flush
19	// which can contain up to 8 bytes.
20	const buffer_size = buffer_flush_size + 8;
21
22	return struct {
23	inner_writer: WriterType,
24
25	// Data waiting to be written is bytes[0 .. nbytes]
26	// and then the low nbits of bits. Data is always written
27	// sequentially into the bytes array.
28	bits: u64 = 0,
29	nbits: u32 = 0, // number of bits
30	bytes: [buffer_size]u8 = undefined,
31	nbytes: u32 = 0, // number of bytes
32
33	const Self = @This();
34
35	pub const Error = WriterType.Error \|\| error{UnfinishedBits};
36
37	pub fn init(writer: WriterType) Self {
38	return .{ .inner_writer = writer };
39	}
40
41	pub fn setWriter(self: *Self, new_writer: WriterType) void {
42	//assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43	self.inner_writer = new_writer;
44	}
45
46	pub fn flush(self: *Self) Error!void {
47	var n = self.nbytes;
48	while (self.nbits != 0) {
49	self.bytes[n] = @as(u8, @truncate(self.bits));
50	self.bits >>= 8;
51	if (self.nbits > 8) { // Avoid underflow
52	self.nbits -= 8;
53	} else {
54	self.nbits = 0;
55	}
56	n += 1;
57	}
58	self.bits = 0;
59	_ = try self.inner_writer.write(self.bytes[0..n]);
60	self.nbytes = 0;
61	}
62
63	pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64	self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65	self.nbits += nb;
66	if (self.nbits < 48)
67	return;
68
69	var n = self.nbytes;
70	std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71	n += 6;
72	if (n >= buffer_flush_size) {
73	_ = try self.inner_writer.write(self.bytes[0..n]);
74	n = 0;
75	}
76	self.nbytes = n;
77	self.bits >>= 48;
78	self.nbits -= 48;
79	}
80
81	pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82	var n = self.nbytes;
83	if (self.nbits & 7 != 0) {
84	return error.UnfinishedBits;
85	}
86	while (self.nbits != 0) {
87	self.bytes[n] = @as(u8, @truncate(self.bits));
88	self.bits >>= 8;
89	self.nbits -= 8;
90	n += 1;
91	}
92	if (n != 0) {
93	_ = try self.inner_writer.write(self.bytes[0..n]);
94	}
95	self.nbytes = 0;
96	_ = try self.inner_writer.write(bytes);
97	}
98	};
99	}

lib/std/compress/flate/block_writer.zig deleted-706

...	@@ -1,706 +0,0 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4
5	const hc = @import("huffman_encoder.zig");
6	const consts = @import("consts.zig").huffman;
7	const Token = @import("Token.zig");
8	const BitWriter = @import("bit_writer.zig").BitWriter;
9
10	pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11	return BlockWriter(@TypeOf(writer)).init(writer);
12	}
13
14	/// Accepts list of tokens, decides what is best block type to write. What block
15	/// type will provide best compression. Writes header and body of the block.
16	///
17	pub fn BlockWriter(comptime WriterType: type) type {
18	const BitWriterType = BitWriter(WriterType);
19	return struct {
20	const codegen_order = consts.codegen_order;
21	const end_code_mark = 255;
22	const Self = @This();
23
24	pub const Error = BitWriterType.Error;
25	bit_writer: BitWriterType,
26
27	codegen_freq: [consts.codegen_code_count]u16 = undefined,
28	literal_freq: [consts.max_num_lit]u16 = undefined,
29	distance_freq: [consts.distance_code_count]u16 = undefined,
30	codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31	literal_encoding: hc.LiteralEncoder = .{},
32	distance_encoding: hc.DistanceEncoder = .{},
33	codegen_encoding: hc.CodegenEncoder = .{},
34	fixed_literal_encoding: hc.LiteralEncoder,
35	fixed_distance_encoding: hc.DistanceEncoder,
36	huff_distance: hc.DistanceEncoder,
37
38	pub fn init(writer: WriterType) Self {
39	return .{
40	.bit_writer = BitWriterType.init(writer),
41	.fixed_literal_encoding = hc.fixedLiteralEncoder(),
42	.fixed_distance_encoding = hc.fixedDistanceEncoder(),
43	.huff_distance = hc.huffmanDistanceEncoder(),
44	};
45	}
46
47	/// Flush intrenal bit buffer to the writer.
48	/// Should be called only when bit stream is at byte boundary.
49	///
50	/// That is after final block; when last byte could be incomplete or
51	/// after stored block; which is aligned to the byte boundary (it has x
52	/// padding bits after first 3 bits).
53	pub fn flush(self: *Self) Error!void {
54	try self.bit_writer.flush();
55	}
56
57	pub fn setWriter(self: *Self, new_writer: WriterType) void {
58	self.bit_writer.setWriter(new_writer);
59	}
60
61	fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62	try self.bit_writer.writeBits(c.code, c.len);
63	}
64
65	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66	// the literal and distance lengths arrays (which are concatenated into a single
67	// array). This method generates that run-length encoding.
68	//
69	// The result is written into the codegen array, and the frequencies
70	// of each code is written into the codegen_freq array.
71	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72	// information. Code bad_code is an end marker
73	//
74	// num_literals: The number of literals in literal_encoding
75	// num_distances: The number of distances in distance_encoding
76	// lit_enc: The literal encoder to use
77	// dist_enc: The distance encoder to use
78	fn generateCodegen(
79	self: *Self,
80	num_literals: u32,
81	num_distances: u32,
82	lit_enc: *hc.LiteralEncoder,
83	dist_enc: *hc.DistanceEncoder,
84	) void {
85	for (self.codegen_freq, 0..) \|_, i\| {
86	self.codegen_freq[i] = 0;
87	}
88
89	// Note that we are using codegen both as a temporary variable for holding
90	// a copy of the frequencies, and as the place where we put the result.
91	// This is fine because the output is always shorter than the input used
92	// so far.
93	var codegen = &self.codegen; // cache
94	// Copy the concatenated code sizes to codegen. Put a marker at the end.
95	var cgnl = codegen[0..num_literals];
96	for (cgnl, 0..) \|_, i\| {
97	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98	}
99
100	cgnl = codegen[num_literals .. num_literals + num_distances];
101	for (cgnl, 0..) \|_, i\| {
102	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103	}
104	codegen[num_literals + num_distances] = end_code_mark;
105
106	var size = codegen[0];
107	var count: i32 = 1;
108	var out_index: u32 = 0;
109	var in_index: u32 = 1;
110	while (size != end_code_mark) : (in_index += 1) {
111	// INVARIANT: We have seen "count" copies of size that have not yet
112	// had output generated for them.
113	const next_size = codegen[in_index];
114	if (next_size == size) {
115	count += 1;
116	continue;
117	}
118	// We need to generate codegen indicating "count" of size.
119	if (size != 0) {
120	codegen[out_index] = size;
121	out_index += 1;
122	self.codegen_freq[size] += 1;
123	count -= 1;
124	while (count >= 3) {
125	var n: i32 = 6;
126	if (n > count) {
127	n = count;
128	}
129	codegen[out_index] = 16;
130	out_index += 1;
131	codegen[out_index] = @as(u8, @intCast(n - 3));
132	out_index += 1;
133	self.codegen_freq[16] += 1;
134	count -= n;
135	}
136	} else {
137	while (count >= 11) {
138	var n: i32 = 138;
139	if (n > count) {
140	n = count;
141	}
142	codegen[out_index] = 18;
143	out_index += 1;
144	codegen[out_index] = @as(u8, @intCast(n - 11));
145	out_index += 1;
146	self.codegen_freq[18] += 1;
147	count -= n;
148	}
149	if (count >= 3) {
150	// 3 <= count <= 10
151	codegen[out_index] = 17;
152	out_index += 1;
153	codegen[out_index] = @as(u8, @intCast(count - 3));
154	out_index += 1;
155	self.codegen_freq[17] += 1;
156	count = 0;
157	}
158	}
159	count -= 1;
160	while (count >= 0) : (count -= 1) {
161	codegen[out_index] = size;
162	out_index += 1;
163	self.codegen_freq[size] += 1;
164	}
165	// Set up invariant for next time through the loop.
166	size = next_size;
167	count = 1;
168	}
169	// Marker indicating the end of the codegen.
170	codegen[out_index] = end_code_mark;
171	}
172
173	const DynamicSize = struct {
174	size: u32,
175	num_codegens: u32,
176	};
177
178	// dynamicSize returns the size of dynamically encoded data in bits.
179	fn dynamicSize(
180	self: *Self,
181	lit_enc: *hc.LiteralEncoder, // literal encoder
182	dist_enc: *hc.DistanceEncoder, // distance encoder
183	extra_bits: u32,
184	) DynamicSize {
185	var num_codegens = self.codegen_freq.len;
186	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187	num_codegens -= 1;
188	}
189	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191	self.codegen_freq[16] * 2 +
192	self.codegen_freq[17] * 3 +
193	self.codegen_freq[18] * 7;
194	const size = header +
195	lit_enc.bitLength(&self.literal_freq) +
196	dist_enc.bitLength(&self.distance_freq) +
197	extra_bits;
198
199	return DynamicSize{
200	.size = @as(u32, @intCast(size)),
201	.num_codegens = @as(u32, @intCast(num_codegens)),
202	};
203	}
204
205	// fixedSize returns the size of dynamically encoded data in bits.
206	fn fixedSize(self: *Self, extra_bits: u32) u32 {
207	return 3 +
208	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210	extra_bits;
211	}
212
213	const StoredSize = struct {
214	size: u32,
215	storable: bool,
216	};
217
218	// storedSizeFits calculates the stored size, including header.
219	// The function returns the size in bits and whether the block
220	// fits inside a single block.
221	fn storedSizeFits(in: ?[]const u8) StoredSize {
222	if (in == null) {
223	return .{ .size = 0, .storable = false };
224	}
225	if (in.?.len <= consts.max_store_block_size) {
226	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227	}
228	return .{ .size = 0, .storable = false };
229	}
230
231	// Write the header of a dynamic Huffman block to the output stream.
232	//
233	// num_literals: The number of literals specified in codegen
234	// num_distances: The number of distances specified in codegen
235	// num_codegens: The number of codegens used in codegen
236	// eof: Is it the end-of-file? (end of stream)
237	fn dynamicHeader(
238	self: *Self,
239	num_literals: u32,
240	num_distances: u32,
241	num_codegens: u32,
242	eof: bool,
243	) Error!void {
244	const first_bits: u32 = if (eof) 5 else 4;
245	try self.bit_writer.writeBits(first_bits, 3);
246	try self.bit_writer.writeBits(num_literals - 257, 5);
247	try self.bit_writer.writeBits(num_distances - 1, 5);
248	try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250	var i: u32 = 0;
251	while (i < num_codegens) : (i += 1) {
252	const value = self.codegen_encoding.codes[codegen_order[i]].len;
253	try self.bit_writer.writeBits(value, 3);
254	}
255
256	i = 0;
257	while (true) {
258	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259	i += 1;
260	if (code_word == end_code_mark) {
261	break;
262	}
263	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265	switch (code_word) {
266	16 => {
267	try self.bit_writer.writeBits(self.codegen[i], 2);
268	i += 1;
269	},
270	17 => {
271	try self.bit_writer.writeBits(self.codegen[i], 3);
272	i += 1;
273	},
274	18 => {
275	try self.bit_writer.writeBits(self.codegen[i], 7);
276	i += 1;
277	},
278	else => {},
279	}
280	}
281	}
282
283	fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284	assert(length <= 65535);
285	const flag: u32 = if (eof) 1 else 0;
286	try self.bit_writer.writeBits(flag, 3);
287	try self.flush();
288	const l: u16 = @intCast(length);
289	try self.bit_writer.writeBits(l, 16);
290	try self.bit_writer.writeBits(~l, 16);
291	}
292
293	fn fixedHeader(self: *Self, eof: bool) Error!void {
294	// Indicate that we are a fixed Huffman block
295	var value: u32 = 2;
296	if (eof) {
297	value = 3;
298	}
299	try self.bit_writer.writeBits(value, 3);
300	}
301
302	// Write a block of tokens with the smallest encoding. Will choose block type.
303	// The original input can be supplied, and if the huffman encoded data
304	// is larger than the original bytes, the data will be written as a
305	// stored block.
306	// If the input is null, the tokens will always be Huffman encoded.
307	pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308	const lit_and_dist = self.indexTokens(tokens);
309	const num_literals = lit_and_dist.num_literals;
310	const num_distances = lit_and_dist.num_distances;
311
312	var extra_bits: u32 = 0;
313	const ret = storedSizeFits(input);
314	const stored_size = ret.size;
315	const storable = ret.storable;
316
317	if (storable) {
318	// We only bother calculating the costs of the extra bits required by
319	// the length of distance fields (which will be the same for both fixed
320	// and dynamic encoding), if we need to compare those two encodings
321	// against stored encoding.
322	var length_code: u16 = Token.length_codes_start + 8;
323	while (length_code < num_literals) : (length_code += 1) {
324	// First eight length codes have extra size = 0.
325	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
327	}
328	var distance_code: u16 = 4;
329	while (distance_code < num_distances) : (distance_code += 1) {
330	// First four distance codes have extra size = 0.
331	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333	}
334	}
335
336	// Figure out smallest code.
337	// Fixed Huffman baseline.
338	var literal_encoding = &self.fixed_literal_encoding;
339	var distance_encoding = &self.fixed_distance_encoding;
340	var size = self.fixedSize(extra_bits);
341
342	// Dynamic Huffman?
343	var num_codegens: u32 = 0;
344
345	// Generate codegen and codegenFrequencies, which indicates how to encode
346	// the literal_encoding and the distance_encoding.
347	self.generateCodegen(
348	num_literals,
349	num_distances,
350	&self.literal_encoding,
351	&self.distance_encoding,
352	);
353	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354	const dynamic_size = self.dynamicSize(
355	&self.literal_encoding,
356	&self.distance_encoding,
357	extra_bits,
358	);
359	const dyn_size = dynamic_size.size;
360	num_codegens = dynamic_size.num_codegens;
361
362	if (dyn_size < size) {
363	size = dyn_size;
364	literal_encoding = &self.literal_encoding;
365	distance_encoding = &self.distance_encoding;
366	}
367
368	// Stored bytes?
369	if (storable and stored_size < size) {
370	try self.storedBlock(input.?, eof);
371	return;
372	}
373
374	// Huffman.
375	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376	try self.fixedHeader(eof);
377	} else {
378	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379	}
380
381	// Write the tokens.
382	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383	}
384
385	pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386	try self.storedHeader(input.len, eof);
387	try self.bit_writer.writeBytes(input);
388	}
389
390	// writeBlockDynamic encodes a block using a dynamic Huffman table.
391	// This should be used if the symbols used have a disproportionate
392	// histogram distribution.
393	// If input is supplied and the compression savings are below 1/16th of the
394	// input size the block is stored.
395	fn dynamicBlock(
396	self: *Self,
397	tokens: []const Token,
398	eof: bool,
399	input: ?[]const u8,
400	) Error!void {
401	const total_tokens = self.indexTokens(tokens);
402	const num_literals = total_tokens.num_literals;
403	const num_distances = total_tokens.num_distances;
404
405	// Generate codegen and codegenFrequencies, which indicates how to encode
406	// the literal_encoding and the distance_encoding.
407	self.generateCodegen(
408	num_literals,
409	num_distances,
410	&self.literal_encoding,
411	&self.distance_encoding,
412	);
413	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415	const size = dynamic_size.size;
416	const num_codegens = dynamic_size.num_codegens;
417
418	// Store bytes, if we don't get a reasonable improvement.
419
420	const stored_size = storedSizeFits(input);
421	const ssize = stored_size.size;
422	const storable = stored_size.storable;
423	if (storable and ssize < (size + (size >> 4))) {
424	try self.storedBlock(input.?, eof);
425	return;
426	}
427
428	// Write Huffman table.
429	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431	// Write the tokens.
432	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433	}
434
435	const TotalIndexedTokens = struct {
436	num_literals: u32,
437	num_distances: u32,
438	};
439
440	// Indexes a slice of tokens followed by an end_block_marker, and updates
441	// literal_freq and distance_freq, and generates literal_encoding
442	// and distance_encoding.
443	// The number of literal and distance tokens is returned.
444	fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445	var num_literals: u32 = 0;
446	var num_distances: u32 = 0;
447
448	for (self.literal_freq, 0..) \|_, i\| {
449	self.literal_freq[i] = 0;
450	}
451	for (self.distance_freq, 0..) \|_, i\| {
452	self.distance_freq[i] = 0;
453	}
454
455	for (tokens) \|t\| {
456	if (t.kind == Token.Kind.literal) {
457	self.literal_freq[t.literal()] += 1;
458	continue;
459	}
460	self.literal_freq[t.lengthCode()] += 1;
461	self.distance_freq[t.distanceCode()] += 1;
462	}
463	// add end_block_marker token at the end
464	self.literal_freq[consts.end_block_marker] += 1;
465
466	// get the number of literals
467	num_literals = @as(u32, @intCast(self.literal_freq.len));
468	while (self.literal_freq[num_literals - 1] == 0) {
469	num_literals -= 1;
470	}
471	// get the number of distances
472	num_distances = @as(u32, @intCast(self.distance_freq.len));
473	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474	num_distances -= 1;
475	}
476	if (num_distances == 0) {
477	// We haven't found a single match. If we want to go with the dynamic encoding,
478	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
479	self.distance_freq[0] = 1;
480	num_distances = 1;
481	}
482	self.literal_encoding.generate(&self.literal_freq, 15);
483	self.distance_encoding.generate(&self.distance_freq, 15);
484	return TotalIndexedTokens{
485	.num_literals = num_literals,
486	.num_distances = num_distances,
487	};
488	}
489
490	// Writes a slice of tokens to the output followed by and end_block_marker.
491	// codes for literal and distance encoding must be supplied.
492	fn writeTokens(
493	self: *Self,
494	tokens: []const Token,
495	le_codes: []hc.HuffCode,
496	oe_codes: []hc.HuffCode,
497	) Error!void {
498	for (tokens) \|t\| {
499	if (t.kind == Token.Kind.literal) {
500	try self.writeCode(le_codes[t.literal()]);
501	continue;
502	}
503
504	// Write the length
505	const le = t.lengthEncoding();
506	try self.writeCode(le_codes[le.code]);
507	if (le.extra_bits > 0) {
508	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509	}
510
511	// Write the distance
512	const oe = t.distanceEncoding();
513	try self.writeCode(oe_codes[oe.code]);
514	if (oe.extra_bits > 0) {
515	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516	}
517	}
518	// add end_block_marker at the end
519	try self.writeCode(le_codes[consts.end_block_marker]);
520	}
521
522	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523	// if the results only gains very little from compression.
524	pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525	// Add everything as literals
526	histogram(input, &self.literal_freq);
527
528	self.literal_freq[consts.end_block_marker] = 1;
529
530	const num_literals = consts.end_block_marker + 1;
531	self.distance_freq[0] = 1;
532	const num_distances = 1;
533
534	self.literal_encoding.generate(&self.literal_freq, 15);
535
536	// Figure out smallest code.
537	// Always use dynamic Huffman or Store
538	var num_codegens: u32 = 0;
539
540	// Generate codegen and codegenFrequencies, which indicates how to encode
541	// the literal_encoding and the distance_encoding.
542	self.generateCodegen(
543	num_literals,
544	num_distances,
545	&self.literal_encoding,
546	&self.huff_distance,
547	);
548	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550	const size = dynamic_size.size;
551	num_codegens = dynamic_size.num_codegens;
552
553	// Store bytes, if we don't get a reasonable improvement.
554	const stored_size_ret = storedSizeFits(input);
555	const ssize = stored_size_ret.size;
556	const storable = stored_size_ret.storable;
557
558	if (storable and ssize < (size + (size >> 4))) {
559	try self.storedBlock(input, eof);
560	return;
561	}
562
563	// Huffman.
564	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565	const encoding = self.literal_encoding.codes[0..257];
566
567	for (input) \|t\| {
568	const c = encoding[t];
569	try self.bit_writer.writeBits(c.code, c.len);
570	}
571	try self.writeCode(encoding[consts.end_block_marker]);
572	}
573
574	// histogram accumulates a histogram of b in h.
575	fn histogram(b: []const u8, h: *[286]u16) void {
576	// Clear histogram
577	for (h, 0..) \|_, i\| {
578	h[i] = 0;
579	}
580
581	var lh = h.*[0..256];
582	for (b) \|t\| {
583	lh[t] += 1;
584	}
585	}
586	};
587	}
588
589	// tests
590	const expect = std.testing.expect;
591	const fmt = std.fmt;
592	const testing = std.testing;
593	const ArrayList = std.ArrayList;
594
595	const TestCase = @import("testdata/block_writer.zig").TestCase;
596	const testCases = @import("testdata/block_writer.zig").testCases;
597
598	// tests if the writeBlock encoding has changed.
599	test "write" {
600	inline for (0..testCases.len) \|i\| {
601	try testBlock(testCases[i], .write_block);
602	}
603	}
604
605	// tests if the writeBlockDynamic encoding has changed.
606	test "dynamicBlock" {
607	inline for (0..testCases.len) \|i\| {
608	try testBlock(testCases[i], .write_dyn_block);
609	}
610	}
611
612	test "huffmanBlock" {
613	inline for (0..testCases.len) \|i\| {
614	try testBlock(testCases[i], .write_huffman_block);
615	}
616	try testBlock(.{
617	.tokens = &[_]Token{},
618	.input = "huffman-rand-max.input",
619	.want = "huffman-rand-max.{s}.expect",
620	}, .write_huffman_block);
621	}
622
623	const TestFn = enum {
624	write_block,
625	write_dyn_block, // write dynamic block
626	write_huffman_block,
627
628	fn to_s(self: TestFn) []const u8 {
629	return switch (self) {
630	.write_block => "wb",
631	.write_dyn_block => "dyn",
632	.write_huffman_block => "huff",
633	};
634	}
635
636	fn write(
637	comptime self: TestFn,
638	bw: anytype,
639	tok: []const Token,
640	input: ?[]const u8,
641	final: bool,
642	) !void {
643	switch (self) {
644	.write_block => try bw.write(tok, final, input),
645	.write_dyn_block => try bw.dynamicBlock(tok, final, input),
646	.write_huffman_block => try bw.huffmanBlock(input.?, final),
647	}
648	try bw.flush();
649	}
650	};
651
652	// testBlock tests a block against its references
653	//
654	// size
655	// 64K [file-name].input - input non compressed file
656	// 8.1K [file-name].golden -
657	// 78 [file-name].dyn.expect - output with writeBlockDynamic
658	// 78 [file-name].wb.expect - output with writeBlock
659	// 8.1K [file-name].huff.expect - output with writeBlockHuff
660	// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661	// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662	//
663	// wb - writeBlock
664	// dyn - writeBlockDynamic
665	// huff - writeBlockHuff
666	//
667	fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668	if (tc.input.len != 0 and tc.want.len != 0) {
669	const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670	const input = @embedFile("testdata/block_writer/" ++ tc.input);
671	const want = @embedFile("testdata/block_writer/" ++ want_name);
672	try testWriteBlock(tfn, input, want, tc.tokens);
673	}
674
675	if (tfn == .write_huffman_block) {
676	return;
677	}
678
679	const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680	const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681	try testWriteBlock(tfn, null, want, tc.tokens);
682	}
683
684	// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685	fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686	var buf = ArrayList(u8).init(testing.allocator);
687	var bw = blockWriter(buf.writer());
688	try tfn.write(&bw, tokens, input, false);
689	var got = buf.items;
690	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691	try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692	//
693	// Test if the writer produces the same output after reset.
694	buf.deinit();
695	buf = ArrayList(u8).init(testing.allocator);
696	defer buf.deinit();
697	bw.setWriter(buf.writer());
698
699	try tfn.write(&bw, tokens, input, true);
700	try bw.flush();
701	got = buf.items;
702
703	try expect(got[0] & 1 == 1); // bfinal is set
704	buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706	}

lib/std/compress/flate/consts.zig deleted-49

...	@@ -1,49 +0,0 @@
1	pub const deflate = struct {
2	// Number of tokens to accumulate in deflate before starting block encoding.
3	//
4	// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5	// 8 and max 9 that gives 14 or 15 bits.
6	pub const tokens = 1 << 15;
7	};
8
9	pub const match = struct {
10	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11	pub const min_length = 4; // min length used in this algorithm
12	pub const max_length = 258;
13
14	pub const min_distance = 1;
15	pub const max_distance = 32768;
16	};
17
18	pub const history = struct {
19	pub const len = match.max_distance;
20	};
21
22	pub const lookup = struct {
23	pub const bits = 15;
24	pub const len = 1 << bits;
25	pub const shift = 32 - bits;
26	};
27
28	pub const huffman = struct {
29	// The odd order in which the codegen code sizes are written.
30	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31	// The number of codegen codes.
32	pub const codegen_code_count = 19;
33
34	// The largest distance code.
35	pub const distance_code_count = 30;
36
37	// Maximum number of literals.
38	pub const max_num_lit = 286;
39
40	// Max number of frequencies used for a Huffman Code
41	// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42	// The largest of these is max_num_lit.
43	pub const max_num_frequencies = max_num_lit;
44
45	// Biggest block size for uncompressed block.
46	pub const max_store_block_size = 65535;
47	// The special code used to mark the end of a block.
48	pub const end_block_marker = 256;
49	};

lib/std/compress/flate/container.zig deleted-208

...	@@ -1,208 +0,0 @@
1	//! Container of the deflate bit stream body. Container adds header before
2	//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
3	//! no footer, raw bit stream).
4	//!
5	//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
6	//! addler 32 checksum.
7	//!
8	//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
9	//! crc32 checksum and 4 bytes of uncompressed data length.
10	//!
11	//!
12	//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
13	//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
14	//!
15
16	const std = @import("std");
17
18	pub const Container = enum {
19	raw, // no header or footer
20	gzip, // gzip header and footer
21	zlib, // zlib header and footer
22
23	pub fn size(w: Container) usize {
24	return headerSize(w) + footerSize(w);
25	}
26
27	pub fn headerSize(w: Container) usize {
28	return switch (w) {
29	.gzip => 10,
30	.zlib => 2,
31	.raw => 0,
32	};
33	}
34
35	pub fn footerSize(w: Container) usize {
36	return switch (w) {
37	.gzip => 8,
38	.zlib => 4,
39	.raw => 0,
40	};
41	}
42
43	pub const list = [_]Container{ .raw, .gzip, .zlib };
44
45	pub const Error = error{
46	BadGzipHeader,
47	BadZlibHeader,
48	WrongGzipChecksum,
49	WrongGzipSize,
50	WrongZlibChecksum,
51	};
52
53	pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
54	switch (wrap) {
55	.gzip => {
56	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
57	// - ID1 (IDentification 1), always 0x1f
58	// - ID2 (IDentification 2), always 0x8b
59	// - CM (Compression Method), always 8 = deflate
60	// - FLG (Flags), all set to 0
61	// - 4 bytes, MTIME (Modification time), not used, all set to zero
62	// - XFL (eXtra FLags), all set to zero
63	// - OS (Operating System), 03 = Unix
64	const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
65	try writer.writeAll(&gzipHeader);
66	},
67	.zlib => {
68	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
69	// 1st byte:
70	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
71	// - The next four bits is the CM (compression method), which is 8 for deflate.
72	// 2nd byte:
73	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
74	// - The next bit, FDICT, is set if a dictionary is given.
75	// - The final five FCHECK bits form a mod-31 checksum.
76	//
77	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
78	const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
79	try writer.writeAll(&zlibHeader);
80	},
81	.raw => {},
82	}
83	}
84
85	pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
86	var bits: [4]u8 = undefined;
87	switch (wrap) {
88	.gzip => {
89	// GZIP 8 bytes footer
90	// - 4 bytes, CRC32 (CRC-32)
91	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
92	std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
93	try writer.writeAll(&bits);
94
95	std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
96	try writer.writeAll(&bits);
97	},
98	.zlib => {
99	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
100	// 4 bytes of ADLER32 (Adler-32 checksum)
101	// Checksum value of the uncompressed data (excluding any
102	// dictionary data) computed according to Adler-32
103	// algorithm.
104	std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
105	try writer.writeAll(&bits);
106	},
107	.raw => {},
108	}
109	}
110
111	pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
112	switch (wrap) {
113	.gzip => try parseGzipHeader(reader),
114	.zlib => try parseZlibHeader(reader),
115	.raw => {},
116	}
117	}
118
119	fn parseGzipHeader(reader: anytype) !void {
120	const magic1 = try reader.read(u8);
121	const magic2 = try reader.read(u8);
122	const method = try reader.read(u8);
123	const flags = try reader.read(u8);
124	try reader.skipBytes(6); // mtime(4), xflags, os
125	if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
126	return error.BadGzipHeader;
127	// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
128	if (flags != 0) {
129	if (flags & 0b0000_0100 != 0) { // FEXTRA
130	const extra_len = try reader.read(u16);
131	try reader.skipBytes(extra_len);
132	}
133	if (flags & 0b0000_1000 != 0) { // FNAME
134	try reader.skipStringZ();
135	}
136	if (flags & 0b0001_0000 != 0) { // FCOMMENT
137	try reader.skipStringZ();
138	}
139	if (flags & 0b0000_0010 != 0) { // FHCRC
140	try reader.skipBytes(2);
141	}
142	}
143	}
144
145	fn parseZlibHeader(reader: anytype) !void {
146	const cm = try reader.read(u4);
147	const cinfo = try reader.read(u4);
148	_ = try reader.read(u8);
149	if (cm != 8 or cinfo > 7) {
150	return error.BadZlibHeader;
151	}
152	}
153
154	pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155	switch (wrap) {
156	.gzip => {
157	try reader.fill(0);
158	if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
159	if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
160	},
161	.zlib => {
162	const chksum: u32 = @byteSwap(hasher.chksum());
163	if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
164	},
165	.raw => {},
166	}
167	}
168
169	pub fn Hasher(comptime wrap: Container) type {
170	const HasherType = switch (wrap) {
171	.gzip => std.hash.Crc32,
172	.zlib => std.hash.Adler32,
173	.raw => struct {
174	pub fn init() @This() {
175	return .{};
176	}
177	},
178	};
179
180	return struct {
181	hasher: HasherType = HasherType.init(),
182	bytes: usize = 0,
183
184	const Self = @This();
185
186	pub fn update(self: *Self, buf: []const u8) void {
187	switch (wrap) {
188	.raw => {},
189	else => {
190	self.hasher.update(buf);
191	self.bytes += buf.len;
192	},
193	}
194	}
195
196	pub fn chksum(self: *Self) u32 {
197	switch (wrap) {
198	.raw => return 0,
199	else => return self.hasher.final(),
200	}
201	}
202
203	pub fn bytesRead(self: *Self) u32 {
204	return @truncate(self.bytes);
205	}
206	};
207	}
208	};

lib/std/compress/flate/deflate.zig deleted-744

...	@@ -1,744 +0,0 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4	const testing = std.testing;
5	const expect = testing.expect;
6	const print = std.debug.print;
7
8	const Token = @import("Token.zig");
9	const consts = @import("consts.zig");
10	const BlockWriter = @import("block_writer.zig").BlockWriter;
11	const Container = @import("container.zig").Container;
12	const SlidingWindow = @import("SlidingWindow.zig");
13	const Lookup = @import("Lookup.zig");
14
15	pub const Options = struct {
16	level: Level = .default,
17	};
18
19	/// Trades between speed and compression size.
20	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21	/// levels 1-3 are using different algorithm to perform faster but with less
22	/// compression. That is not implemented here.
23	pub const Level = enum(u4) {
24	// zig fmt: off
25	fast = 0xb, level_4 = 4,
26	level_5 = 5,
27	default = 0xc, level_6 = 6,
28	level_7 = 7,
29	level_8 = 8,
30	best = 0xd, level_9 = 9,
31	// zig fmt: on
32	};
33
34	/// Algorithm knobs for each level.
35	const LevelArgs = struct {
36	good: u16, // Do less lookups if we already have match of this length.
37	nice: u16, // Stop looking for better match if we found match with at least this length.
38	lazy: u16, // Don't do lazy match find if got match with at least this length.
39	chain: u16, // How many lookups for previous match to perform.
40
41	pub fn get(level: Level) LevelArgs {
42	// zig fmt: off
43	return switch (level) {
44	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50	};
51	// zig fmt: on
52	}
53	};
54
55	/// Compress plain data from reader into compressed stream written to writer.
56	pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57	var c = try compressor(container, writer, options);
58	try c.compress(reader);
59	try c.finish();
60	}
61
62	/// Create compressor for writer type.
63	pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64	container,
65	@TypeOf(writer),
66	) {
67	return try Compressor(container, @TypeOf(writer)).init(writer, options);
68	}
69
70	/// Compressor type.
71	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72	const TokenWriterType = BlockWriter(WriterType);
73	return Deflate(container, WriterType, TokenWriterType);
74	}
75
76	/// Default compression algorithm. Has two steps: tokenization and token
77	/// encoding.
78	///
79	/// Tokenization takes uncompressed input stream and produces list of tokens.
80	/// Each token can be literal (byte of data) or match (backrefernce to previous
81	/// data with length and distance). Tokenization accumulators 32K tokens, when
82	/// full or `flush` is called tokens are passed to the `block_writer`. Level
83	/// defines how hard (how slow) it tries to find match.
84	///
85	/// Block writer will decide which type of deflate block to write (stored, fixed,
86	/// dynamic) and encode tokens to the output byte stream. Client has to call
87	/// `finish` to write block with the final bit set.
88	///
89	/// Container defines type of header and footer which can be gzip, zlib or raw.
90	/// They all share same deflate body. Raw has no header or footer just deflate
91	/// body.
92	///
93	/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
94	///
95	/// The compressor uses a chained hash table `lookup` to find duplicated
96	/// strings, using a hash function that operates on 4-byte sequences. At any
97	/// given point during compression, let XYZW be the next 4 input bytes
98	/// (lookahead) to be examined (not necessarily all different, of course).
99	/// First, the compressor examines the hash chain for XYZW. If the chain is
100	/// empty, the compressor simply writes out X as a literal byte and advances
101	/// one byte in the input. If the hash chain is not empty, indicating that the
102	/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
103	/// hash function value) has occurred recently, the compressor compares all
104	/// strings on the XYZW hash chain with the actual input data sequence
105	/// starting at the current point, and selects the longest match.
106	///
107	/// To improve overall compression, the compressor defers the selection of
108	/// matches ("lazy matching"): after a match of length N has been found, the
109	/// compressor searches for a longer match starting at the next input byte. If
110	/// it finds a longer match, it truncates the previous match to a length of
111	/// one (thus producing a single literal byte) and then emits the longer
112	/// match. Otherwise, it emits the original match, and, as described above,
113	/// advances N bytes before continuing.
114	///
115	///
116	/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
117	///
118	/// Deflate function accepts BlockWriterType so we can change that in test to test
119	/// just tokenization part.
120	///
121	fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
122	return struct {
123	lookup: Lookup = .{},
124	win: SlidingWindow = .{},
125	tokens: Tokens = .{},
126	wrt: WriterType,
127	block_writer: BlockWriterType,
128	level: LevelArgs,
129	hasher: container.Hasher() = .{},
130
131	// Match and literal at the previous position.
132	// Used for lazy match finding in processWindow.
133	prev_match: ?Token = null,
134	prev_literal: ?u8 = null,
135
136	const Self = @This();
137
138	pub fn init(wrt: WriterType, options: Options) !Self {
139	const self = Self{
140	.wrt = wrt,
141	.block_writer = BlockWriterType.init(wrt),
142	.level = LevelArgs.get(options.level),
143	};
144	try container.writeHeader(self.wrt);
145	return self;
146	}
147
148	const FlushOption = enum { none, flush, final };
149
150	// Process data in window and create tokens. If token buffer is full
151	// flush tokens to the token writer. In the case of `flush` or `final`
152	// option it will process all data from the window. In the `none` case
153	// it will preserve some data for the next match.
154	fn tokenize(self: *Self, flush_opt: FlushOption) !void {
155	// flush - process all data from window
156	const should_flush = (flush_opt != .none);
157
158	// While there is data in active lookahead buffer.
159	while (self.win.activeLookahead(should_flush)) \|lh\| {
160	var step: u16 = 1; // 1 in the case of literal, match length otherwise
161	const pos: u16 = self.win.pos();
162	const literal = lh[0]; // literal at current position
163	const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
164
165	// Try to find match at least min_len long.
166	if (self.findMatch(pos, lh, min_len)) \|match\| {
167	// Found better match than previous.
168	try self.addPrevLiteral();
169
170	// Is found match length good enough?
171	if (match.length() >= self.level.lazy) {
172	// Don't try to lazy find better match, use this.
173	step = try self.addMatch(match);
174	} else {
175	// Store this match.
176	self.prev_literal = literal;
177	self.prev_match = match;
178	}
179	} else {
180	// There is no better match at current pos then it was previous.
181	// Write previous match or literal.
182	if (self.prev_match) \|m\| {
183	// Write match from previous position.
184	step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
185	} else {
186	// No match at previous position.
187	// Write previous literal if any, and remember this literal.
188	try self.addPrevLiteral();
189	self.prev_literal = literal;
190	}
191	}
192	// Advance window and add hashes.
193	self.windowAdvance(step, lh, pos);
194	}
195
196	if (should_flush) {
197	// In the case of flushing, last few lookahead buffers were smaller then min match len.
198	// So only last literal can be unwritten.
199	assert(self.prev_match == null);
200	try self.addPrevLiteral();
201	self.prev_literal = null;
202
203	try self.flushTokens(flush_opt);
204	}
205	}
206
207	fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
208	// current position is already added in findMatch
209	self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
210	self.win.advance(step);
211	}
212
213	// Add previous literal (if any) to the tokens list.
214	fn addPrevLiteral(self: *Self) !void {
215	if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
216	}
217
218	// Add match to the tokens list, reset prev pointers.
219	// Returns length of the added match.
220	fn addMatch(self: *Self, m: Token) !u16 {
221	try self.addToken(m);
222	self.prev_literal = null;
223	self.prev_match = null;
224	return m.length();
225	}
226
227	fn addToken(self: *Self, token: Token) !void {
228	self.tokens.add(token);
229	if (self.tokens.full()) try self.flushTokens(.none);
230	}
231
232	// Finds largest match in the history window with the data at current pos.
233	fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
234	var len: u16 = min_len;
235	// Previous location with the same hash (same 4 bytes).
236	var prev_pos = self.lookup.add(lh, pos);
237	// Last found match.
238	var match: ?Token = null;
239
240	// How much back-references to try, performance knob.
241	var chain: usize = self.level.chain;
242	if (len >= self.level.good) {
243	// If we've got a match that's good enough, only look in 1/4 the chain.
244	chain >>= 2;
245	}
246
247	// Hot path loop!
248	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
249	const distance = pos - prev_pos;
250	if (distance > consts.match.max_distance)
251	break;
252
253	const new_len = self.win.match(prev_pos, pos, len);
254	if (new_len > len) {
255	match = Token.initMatch(@intCast(distance), new_len);
256	if (new_len >= self.level.nice) {
257	// The match is good enough that we don't try to find a better one.
258	return match;
259	}
260	len = new_len;
261	}
262	prev_pos = self.lookup.prev(prev_pos);
263	}
264
265	return match;
266	}
267
268	fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
269	// Pass tokens to the token writer
270	try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
271	// Stored block ensures byte alignment.
272	// It has 3 bits (final, block_type) and then padding until byte boundary.
273	// After that everything is aligned to the boundary in the stored block.
274	// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
275	// Last 4 bytes are byte aligned.
276	if (flush_opt == .flush) {
277	try self.block_writer.storedBlock("", false);
278	}
279	if (flush_opt != .none) {
280	// Safe to call only when byte aligned or it is OK to add
281	// padding bits (on last byte of the final block).
282	try self.block_writer.flush();
283	}
284	// Reset internal tokens store.
285	self.tokens.reset();
286	// Notify win that tokens are flushed.
287	self.win.flush();
288	}
289
290	// Slide win and if needed lookup tables.
291	fn slide(self: *Self) void {
292	const n = self.win.slide();
293	self.lookup.slide(n);
294	}
295
296	/// Compresses as much data as possible, stops when the reader becomes
297	/// empty. It will introduce some output latency (reading input without
298	/// producing all output) because some data are still in internal
299	/// buffers.
300	///
301	/// It is up to the caller to call flush (if needed) or finish (required)
302	/// when is need to output any pending data or complete stream.
303	///
304	pub fn compress(self: *Self, reader: anytype) !void {
305	while (true) {
306	// Fill window from reader
307	const buf = self.win.writable();
308	if (buf.len == 0) {
309	try self.tokenize(.none);
310	self.slide();
311	continue;
312	}
313	const n = try reader.readAll(buf);
314	self.hasher.update(buf[0..n]);
315	self.win.written(n);
316	// Process window
317	try self.tokenize(.none);
318	// Exit when no more data in reader
319	if (n < buf.len) break;
320	}
321	}
322
323	/// Flushes internal buffers to the output writer. Outputs empty stored
324	/// block to sync bit stream to the byte boundary, so that the
325	/// decompressor can get all input data available so far.
326	///
327	/// It is useful mainly in compressed network protocols, to ensure that
328	/// deflate bit stream can be used as byte stream. May degrade
329	/// compression so it should be used only when necessary.
330	///
331	/// Completes the current deflate block and follows it with an empty
332	/// stored block that is three zero bits plus filler bits to the next
333	/// byte, followed by four bytes (00 00 ff ff).
334	///
335	pub fn flush(self: *Self) !void {
336	try self.tokenize(.flush);
337	}
338
339	/// Completes deflate bit stream by writing any pending data as deflate
340	/// final deflate block. HAS to be called once all data are written to
341	/// the compressor as a signal that next block has to have final bit
342	/// set.
343	///
344	pub fn finish(self: *Self) !void {
345	try self.tokenize(.final);
346	try container.writeFooter(&self.hasher, self.wrt);
347	}
348
349	/// Use another writer while preserving history. Most probably flush
350	/// should be called on old writer before setting new.
351	pub fn setWriter(self: *Self, new_writer: WriterType) void {
352	self.block_writer.setWriter(new_writer);
353	self.wrt = new_writer;
354	}
355
356	// Writer interface
357
358	pub const Writer = io.GenericWriter(*Self, Error, write);
359	pub const Error = BlockWriterType.Error;
360
361	/// Write `input` of uncompressed data.
362	/// See compress.
363	pub fn write(self: *Self, input: []const u8) !usize {
364	var fbs = io.fixedBufferStream(input);
365	try self.compress(fbs.reader());
366	return input.len;
367	}
368
369	pub fn writer(self: *Self) Writer {
370	return .{ .context = self };
371	}
372	};
373	}
374
375	// Tokens store
376	const Tokens = struct {
377	list: [consts.deflate.tokens]Token = undefined,
378	pos: usize = 0,
379
380	fn add(self: *Tokens, t: Token) void {
381	self.list[self.pos] = t;
382	self.pos += 1;
383	}
384
385	fn full(self: *Tokens) bool {
386	return self.pos == self.list.len;
387	}
388
389	fn reset(self: *Tokens) void {
390	self.pos = 0;
391	}
392
393	fn tokens(self: *Tokens) []const Token {
394	return self.list[0..self.pos];
395	}
396	};
397
398	/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
399	/// only performs Huffman entropy encoding. Results in faster compression, much
400	/// less memory requirements during compression but bigger compressed sizes.
401	pub const huffman = struct {
402	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
403	var c = try huffman.compressor(container, writer);
404	try c.compress(reader);
405	try c.finish();
406	}
407
408	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
409	return SimpleCompressor(.huffman, container, WriterType);
410	}
411
412	pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
413	return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
414	}
415	};
416
417	/// Creates store blocks only. Data are not compressed only packed into deflate
418	/// store blocks. That adds 9 bytes of header for each block. Max stored block
419	/// size is 64K. Block is emitted when flush is called on on finish.
420	pub const store = struct {
421	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
422	var c = try store.compressor(container, writer);
423	try c.compress(reader);
424	try c.finish();
425	}
426
427	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
428	return SimpleCompressor(.store, container, WriterType);
429	}
430
431	pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
432	return try store.Compressor(container, @TypeOf(writer)).init(writer);
433	}
434	};
435
436	const SimpleCompressorKind = enum {
437	huffman,
438	store,
439	};
440
441	fn simpleCompressor(
442	comptime kind: SimpleCompressorKind,
443	comptime container: Container,
444	writer: anytype,
445	) !SimpleCompressor(kind, container, @TypeOf(writer)) {
446	return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
447	}
448
449	fn SimpleCompressor(
450	comptime kind: SimpleCompressorKind,
451	comptime container: Container,
452	comptime WriterType: type,
453	) type {
454	const BlockWriterType = BlockWriter(WriterType);
455	return struct {
456	buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
457	wp: usize = 0,
458
459	wrt: WriterType,
460	block_writer: BlockWriterType,
461	hasher: container.Hasher() = .{},
462
463	const Self = @This();
464
465	pub fn init(wrt: WriterType) !Self {
466	const self = Self{
467	.wrt = wrt,
468	.block_writer = BlockWriterType.init(wrt),
469	};
470	try container.writeHeader(self.wrt);
471	return self;
472	}
473
474	pub fn flush(self: *Self) !void {
475	try self.flushBuffer(false);
476	try self.block_writer.storedBlock("", false);
477	try self.block_writer.flush();
478	}
479
480	pub fn finish(self: *Self) !void {
481	try self.flushBuffer(true);
482	try self.block_writer.flush();
483	try container.writeFooter(&self.hasher, self.wrt);
484	}
485
486	fn flushBuffer(self: *Self, final: bool) !void {
487	const buf = self.buffer[0..self.wp];
488	switch (kind) {
489	.huffman => try self.block_writer.huffmanBlock(buf, final),
490	.store => try self.block_writer.storedBlock(buf, final),
491	}
492	self.wp = 0;
493	}
494
495	// Writes all data from the input reader of uncompressed data.
496	// It is up to the caller to call flush or finish if there is need to
497	// output compressed blocks.
498	pub fn compress(self: *Self, reader: anytype) !void {
499	while (true) {
500	// read from rdr into buffer
501	const buf = self.buffer[self.wp..];
502	if (buf.len == 0) {
503	try self.flushBuffer(false);
504	continue;
505	}
506	const n = try reader.readAll(buf);
507	self.hasher.update(buf[0..n]);
508	self.wp += n;
509	if (n < buf.len) break; // no more data in reader
510	}
511	}
512
513	// Writer interface
514
515	pub const Writer = io.GenericWriter(*Self, Error, write);
516	pub const Error = BlockWriterType.Error;
517
518	// Write `input` of uncompressed data.
519	pub fn write(self: *Self, input: []const u8) !usize {
520	var fbs = io.fixedBufferStream(input);
521	try self.compress(fbs.reader());
522	return input.len;
523	}
524
525	pub fn writer(self: *Self) Writer {
526	return .{ .context = self };
527	}
528	};
529	}
530
531	const builtin = @import("builtin");
532
533	test "tokenization" {
534	const L = Token.initLiteral;
535	const M = Token.initMatch;
536
537	const cases = [_]struct {
538	data: []const u8,
539	tokens: []const Token,
540	}{
541	.{
542	.data = "Blah blah blah blah blah!",
543	.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
544	},
545	.{
546	.data = "ABCDEABCD ABCDEABCD",
547	.tokens = &[_]Token{
548	L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
549	L('A'), M(10, 8),
550	},
551	},
552	};
553
554	for (cases) \|c\| {
555	inline for (Container.list) \|container\| { // for each wrapping
556
557	var cw = io.countingWriter(io.null_writer);
558	const cww = cw.writer();
559	var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
560
561	_ = try df.write(c.data);
562	try df.flush();
563
564	// df.token_writer.show();
565	try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
566	try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
567
568	try testing.expectEqual(container.headerSize(), cw.bytes_written);
569	try df.finish();
570	try testing.expectEqual(container.size(), cw.bytes_written);
571	}
572	}
573	}
574
575	// Tests that tokens written are equal to expected token list.
576	const TestTokenWriter = struct {
577	const Self = @This();
578
579	pos: usize = 0,
580	actual: [128]Token = undefined,
581
582	pub fn init(_: anytype) Self {
583	return .{};
584	}
585	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
586	for (tokens) \|t\| {
587	self.actual[self.pos] = t;
588	self.pos += 1;
589	}
590	}
591
592	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
593
594	pub fn get(self: *Self) []Token {
595	return self.actual[0..self.pos];
596	}
597
598	pub fn show(self: *Self) void {
599	print("\n", .{});
600	for (self.get()) \|t\| {
601	t.show();
602	}
603	}
604
605	pub fn flush(_: *Self) !void {}
606	};
607
608	test "file tokenization" {
609	const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
610	const cases = [_]struct {
611	data: []const u8, // uncompressed content
612	// expected number of tokens producet in deflate tokenization
613	tokens_count: [levels.len]usize = .{0} ** levels.len,
614	}{
615	.{
616	.data = @embedFile("testdata/rfc1951.txt"),
617	.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
618	},
619
620	.{
621	.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
622	.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
623	},
624	.{
625	.data = @embedFile("testdata/block_writer/huffman-pi.input"),
626	.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
627	},
628	.{
629	.data = @embedFile("testdata/block_writer/huffman-text.input"),
630	.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
631	},
632	.{
633	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
634	.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
635	},
636	.{
637	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
638	.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
639	},
640	};
641
642	for (cases) \|case\| { // for each case
643	const data = case.data;
644
645	for (levels, 0..) \|level, i\| { // for each compression level
646	var original = io.fixedBufferStream(data);
647
648	// buffer for decompressed data
649	var al = std.ArrayList(u8).init(testing.allocator);
650	defer al.deinit();
651	const writer = al.writer();
652
653	// create compressor
654	const WriterType = @TypeOf(writer);
655	const TokenWriter = TokenDecoder(@TypeOf(writer));
656	var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
657
658	// Stream uncompressed `original` data to the compressor. It will
659	// produce tokens list and pass that list to the TokenDecoder. This
660	// TokenDecoder uses CircularBuffer from inflate to convert list of
661	// tokens back to the uncompressed stream.
662	try cmp.compress(original.reader());
663	try cmp.flush();
664	const expected_count = case.tokens_count[i];
665	const actual = cmp.block_writer.tokens_count;
666	if (expected_count == 0) {
667	print("actual token count {d}\n", .{actual});
668	} else {
669	try testing.expectEqual(expected_count, actual);
670	}
671
672	try testing.expectEqual(data.len, al.items.len);
673	try testing.expectEqualSlices(u8, data, al.items);
674	}
675	}
676	}
677
678	fn TokenDecoder(comptime WriterType: type) type {
679	return struct {
680	const CircularBuffer = @import("CircularBuffer.zig");
681	hist: CircularBuffer = .{},
682	wrt: WriterType,
683	tokens_count: usize = 0,
684
685	const Self = @This();
686
687	pub fn init(wrt: WriterType) Self {
688	return .{ .wrt = wrt };
689	}
690
691	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
692	self.tokens_count += tokens.len;
693	for (tokens) \|t\| {
694	switch (t.kind) {
695	.literal => self.hist.write(t.literal()),
696	.match => try self.hist.writeMatch(t.length(), t.distance()),
697	}
698	if (self.hist.free() < 285) try self.flushWin();
699	}
700	try self.flushWin();
701	}
702
703	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
704
705	fn flushWin(self: *Self) !void {
706	while (true) {
707	const buf = self.hist.read();
708	if (buf.len == 0) break;
709	try self.wrt.writeAll(buf);
710	}
711	}
712
713	pub fn flush(_: *Self) !void {}
714	};
715	}
716
717	test "store simple compressor" {
718	const data = "Hello world!";
719	const expected = [_]u8{
720	0x1, // block type 0, final bit set
721	0xc, 0x0, // len = 12
722	0xf3, 0xff, // ~len
723	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
724	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
725	};
726
727	var fbs = std.io.fixedBufferStream(data);
728	var al = std.ArrayList(u8).init(testing.allocator);
729	defer al.deinit();
730
731	var cmp = try store.compressor(.raw, al.writer());
732	try cmp.compress(fbs.reader());
733	try cmp.finish();
734	try testing.expectEqualSlices(u8, &expected, al.items);
735
736	fbs.reset();
737	try al.resize(0);
738
739	// huffman only compresoor will also emit store block for this small sample
740	var hc = try huffman.compressor(.raw, al.writer());
741	try hc.compress(fbs.reader());
742	try hc.finish();
743	try testing.expectEqualSlices(u8, &expected, al.items);
744	}

lib/std/compress/flate/huffman_decoder.zig deleted-302

...	@@ -1,302 +0,0 @@
1	const std = @import("std");
2	const testing = std.testing;
3
4	pub const Symbol = packed struct {
5	pub const Kind = enum(u2) {
6	literal,
7	end_of_block,
8	match,
9	};
10
11	symbol: u8 = 0, // symbol from alphabet
12	code_bits: u4 = 0, // number of bits in code 0-15
13	kind: Kind = .literal,
14
15	code: u16 = 0, // huffman code of the symbol
16	next: u16 = 0, // pointer to the next symbol in linked list
17	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19	// Sorting less than function.
20	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21	if (a.code_bits == b.code_bits) {
22	if (a.kind == b.kind) {
23	return a.symbol < b.symbol;
24	}
25	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26	}
27	return a.code_bits < b.code_bits;
28	}
29	};
30
31	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35	pub const Error = error{
36	InvalidCode,
37	OversubscribedHuffmanTree,
38	IncompleteHuffmanTree,
39	MissingEndOfBlockCode,
40	};
41
42	/// Creates huffman tree codes from list of code lengths (in `build`).
43	///
44	/// `find` then finds symbol for code bits. Code can be any length between 1 and
45	/// 15 bits. When calling `find` we don't know how many bits will be used to
46	/// find symbol. When symbol is returned it has code_bits field which defines
47	/// how much we should advance in bit stream.
48	///
49	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50	/// many times in this table; 32K places for 286 (at most) symbols.
51	/// Small lookup table is optimization for faster search.
52	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53	/// with difference that we here use statically allocated arrays.
54	///
55	fn HuffmanDecoder(
56	comptime alphabet_size: u16,
57	comptime max_code_bits: u4,
58	comptime lookup_bits: u4,
59	) type {
60	const lookup_shift = max_code_bits - lookup_bits;
61
62	return struct {
63	// all symbols in alaphabet, sorted by code_len, symbol
64	symbols: [alphabet_size]Symbol = undefined,
65	// lookup table code -> symbol
66	lookup: [1 << lookup_bits]Symbol = undefined,
67
68	const Self = @This();
69
70	/// Generates symbols and lookup tables from list of code lens for each symbol.
71	pub fn generate(self: *Self, lens: []const u4) !void {
72	try checkCompleteness(lens);
73
74	// init alphabet with code_bits
75	for (self.symbols, 0..) \|_, i\| {
76	const cb: u4 = if (i < lens.len) lens[i] else 0;
77	self.symbols[i] = if (i < 256)
78	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79	else if (i == 256)
80	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81	else
82	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83	}
84	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86	// reset lookup table
87	for (0..self.lookup.len) \|i\| {
88	self.lookup[i] = .{};
89	}
90
91	// assign code to symbols
92	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93	var code: u16 = 0;
94	var idx: u16 = 0;
95	for (&self.symbols, 0..) \|*sym, pos\| {
96	if (sym.code_bits == 0) continue; // skip unused
97	sym.code = code;
98
99	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
100	const next_idx = next_code >> lookup_shift;
101
102	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
103	if (sym.code_bits <= lookup_bits) {
104	// fill small lookup table
105	for (idx..next_idx) \|j\|
106	self.lookup[j] = sym.*;
107	} else {
108	// insert into linked table starting at root
109	const root = &self.lookup[idx];
110	const root_next = root.next;
111	root.next = @intCast(pos);
112	sym.next = root_next;
113	}
114
115	idx = next_idx;
116	code = next_code;
117	}
118	}
119
120	/// Given the list of code lengths check that it represents a canonical
121	/// Huffman code for n symbols.
122	///
123	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
124	fn checkCompleteness(lens: []const u4) !void {
125	if (alphabet_size == 286)
126	if (lens[256] == 0) return error.MissingEndOfBlockCode;
127
128	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
129	var max: usize = 0;
130	for (lens) \|n\| {
131	if (n == 0) continue;
132	if (n > max) max = n;
133	count[n] += 1;
134	}
135	if (max == 0) // empty tree
136	return;
137
138	// check for an over-subscribed or incomplete set of lengths
139	var left: usize = 1; // one possible code of zero length
140	for (1..count.len) \|len\| {
141	left <<= 1; // one more bit, double codes left
142	if (count[len] > left)
143	return error.OversubscribedHuffmanTree;
144	left -= count[len]; // deduct count from possible codes
145	}
146	if (left > 0) { // left > 0 means incomplete
147	// incomplete code ok only for single length 1 code
148	if (max_code_bits > 7 and max == count[0] + count[1]) return;
149	return error.IncompleteHuffmanTree;
150	}
151	}
152
153	/// Finds symbol for lookup table code.
154	pub fn find(self: *Self, code: u16) !Symbol {
155	// try to find in lookup table
156	const idx = code >> lookup_shift;
157	const sym = self.lookup[idx];
158	if (sym.code_bits != 0) return sym;
159	// if not use linked list of symbols with same prefix
160	return self.findLinked(code, sym.next);
161	}
162
163	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
164	var pos = start;
165	while (pos > 0) {
166	const sym = self.symbols[pos];
167	const shift = max_code_bits - sym.code_bits;
168	// compare code_bits number of upper bits
169	if ((code ^ sym.code) >> shift == 0) return sym;
170	pos = sym.next;
171	}
172	return error.InvalidCode;
173	}
174	};
175	}
176
177	test "init/find" {
178	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
179	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
180	var h: CodegenDecoder = .{};
181	try h.generate(&code_lens);
182
183	const expected = [_]struct {
184	sym: Symbol,
185	code: u16,
186	}{
187	.{
188	.code = 0b00_00000,
189	.sym = .{ .symbol = 3, .code_bits = 2 },
190	},
191	.{
192	.code = 0b01_00000,
193	.sym = .{ .symbol = 18, .code_bits = 2 },
194	},
195	.{
196	.code = 0b100_0000,
197	.sym = .{ .symbol = 1, .code_bits = 3 },
198	},
199	.{
200	.code = 0b101_0000,
201	.sym = .{ .symbol = 4, .code_bits = 3 },
202	},
203	.{
204	.code = 0b110_0000,
205	.sym = .{ .symbol = 17, .code_bits = 3 },
206	},
207	.{
208	.code = 0b1110_000,
209	.sym = .{ .symbol = 0, .code_bits = 4 },
210	},
211	.{
212	.code = 0b1111_000,
213	.sym = .{ .symbol = 16, .code_bits = 4 },
214	},
215	};
216
217	// unused symbols
218	for (0..12) \|i\| {
219	try testing.expectEqual(0, h.symbols[i].code_bits);
220	}
221	// used, from index 12
222	for (expected, 12..) \|e, i\| {
223	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
224	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
225	const sym_from_code = try h.find(e.code);
226	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
227	}
228
229	// All possible codes for each symbol.
230	// Lookup table has 126 elements, to cover all possible 7 bit codes.
231	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
232	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
233
234	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
235	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
236
237	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
238	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
239
240	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
241	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
242
243	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
244	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
245
246	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
247	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
248
249	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
250	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
251	}
252
253	test "encode/decode literals" {
254	const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
255
256	for (1..286) \|j\| { // for all different number of codes
257	var enc: LiteralEncoder = .{};
258	// create frequencies
259	var freq = [_]u16{0} ** 286;
260	freq[256] = 1; // ensure we have end of block code
261	for (&freq, 1..) \|*f, i\| {
262	if (i % j == 0)
263	f.* = @intCast(i);
264	}
265
266	// encoder from frequencies
267	enc.generate(&freq, 15);
268
269	// get code_lens from encoder
270	var code_lens = [_]u4{0} ** 286;
271	for (code_lens, 0..) \|_, i\| {
272	code_lens[i] = @intCast(enc.codes[i].len);
273	}
274	// generate decoder from code lens
275	var dec: LiteralDecoder = .{};
276	try dec.generate(&code_lens);
277
278	// expect decoder code to match original encoder code
279	for (dec.symbols) \|s\| {
280	if (s.code_bits == 0) continue;
281	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
282	const symbol: u16 = switch (s.kind) {
283	.literal => s.symbol,
284	.end_of_block => 256,
285	.match => @as(u16, s.symbol) + 257,
286	};
287
288	const c = enc.codes[symbol];
289	try testing.expect(c.code == c_code);
290	}
291
292	// find each symbol by code
293	for (enc.codes) \|c\| {
294	if (c.len == 0) continue;
295
296	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
297	const s = try dec.find(s_code);
298	try testing.expect(s.code == s_code);
299	try testing.expect(s.code_bits == c.len);
300	}
301	}
302	}

lib/std/compress/flate/huffman_encoder.zig deleted-536

...	@@ -1,536 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const consts = @import("consts.zig").huffman;
9
10	const LiteralNode = struct {
11	literal: u16,
12	freq: u16,
13	};
14
15	// Describes the state of the constructed tree for a given depth.
16	const LevelInfo = struct {
17	// Our level. for better printing
18	level: u32,
19
20	// The frequency of the last node at this level
21	last_freq: u32,
22
23	// The frequency of the next character to add to this level
24	next_char_freq: u32,
25
26	// The frequency of the next pair (from level below) to add to this level.
27	// Only valid if the "needed" value of the next lower level is 0.
28	next_pair_freq: u32,
29
30	// The number of chains remaining to generate for this level before moving
31	// up to the next level
32	needed: u32,
33	};
34
35	// hcode is a huffman code with a bit code and bit length.
36	pub const HuffCode = struct {
37	code: u16 = 0,
38	len: u16 = 0,
39
40	// set sets the code and length of an hcode.
41	fn set(self: *HuffCode, code: u16, length: u16) void {
42	self.len = length;
43	self.code = code;
44	}
45	};
46
47	pub fn HuffmanEncoder(comptime size: usize) type {
48	return struct {
49	codes: [size]HuffCode = undefined,
50	// Reusable buffer with the longest possible frequency table.
51	freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52	bit_count: [17]u32 = undefined,
53	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56	const Self = @This();
57
58	// Update this Huffman Code object to be the minimum code for the specified frequency count.
59	//
60	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61	// max_bits The maximum number of bits to use for any literal.
62	pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63	var list = self.freq_cache[0 .. freq.len + 1];
64	// Number of non-zero literals
65	var count: u32 = 0;
66	// Set list to be the set of all non-zero literals and their frequencies
67	for (freq, 0..) \|f, i\| {
68	if (f != 0) {
69	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70	count += 1;
71	} else {
72	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73	self.codes[i].len = 0;
74	}
75	}
76	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78	list = list[0..count];
79	if (count <= 2) {
80	// Handle the small cases here, because they are awkward for the general case code. With
81	// two or fewer literals, everything has bit length 1.
82	for (list, 0..) \|node, i\| {
83	// "list" is in order of increasing literal value.
84	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85	}
86	return;
87	}
88	self.lfs = list;
89	mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91	// Get the number of literals for each bit count
92	const bit_count = self.bitCounts(list, max_bits);
93	// And do the assignment
94	self.assignEncodingAndSize(bit_count, list);
95	}
96
97	pub fn bitLength(self: *Self, freq: []u16) u32 {
98	var total: u32 = 0;
99	for (freq, 0..) \|f, i\| {
100	if (f != 0) {
101	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102	}
103	}
104	return total;
105	}
106
107	// Return the number of literals assigned to each bit size in the Huffman encoding
108	//
109	// This method is only called when list.len >= 3
110	// The cases of 0, 1, and 2 literals are handled by special case code.
111	//
112	// list: An array of the literals with non-zero frequencies
113	// and their associated frequencies. The array is in order of increasing
114	// frequency, and has as its last element a special element with frequency
115	// std.math.maxInt(i32)
116	//
117	// max_bits: The maximum number of bits that should be used to encode any literal.
118	// Must be less than 16.
119	//
120	// Returns an integer array in which array[i] indicates the number of literals
121	// that should be encoded in i bits.
122	fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123	var max_bits = max_bits_to_use;
124	const n = list.len;
125	const max_bits_limit = 16;
126
127	assert(max_bits < max_bits_limit);
128
129	// The tree can't have greater depth than n - 1, no matter what. This
130	// saves a little bit of work in some small cases
131	max_bits = @min(max_bits, n - 1);
132
133	// Create information about each of the levels.
134	// A bogus "Level 0" whose sole purpose is so that
135	// level1.prev.needed == 0. This makes level1.next_pair_freq
136	// be a legitimate value that never gets chosen.
137	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138	// leaf_counts[i] counts the number of literals at the left
139	// of ancestors of the rightmost node at level i.
140	// leaf_counts[i][j] is the number of literals at the left
141	// of the level j ancestor.
142	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144	{
145	var level = @as(u32, 1);
146	while (level <= max_bits) : (level += 1) {
147	// For every level, the first two items are the first two characters.
148	// We initialize the levels as if we had already figured this out.
149	levels[level] = LevelInfo{
150	.level = level,
151	.last_freq = list[1].freq,
152	.next_char_freq = list[2].freq,
153	.next_pair_freq = list[0].freq + list[1].freq,
154	.needed = 0,
155	};
156	leaf_counts[level][level] = 2;
157	if (level == 1) {
158	levels[level].next_pair_freq = math.maxInt(i32);
159	}
160	}
161	}
162
163	// We need a total of 2*n - 2 items at top level and have already generated 2.
164	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166	{
167	var level = max_bits;
168	while (true) {
169	var l = &levels[level];
170	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171	// We've run out of both leaves and pairs.
172	// End all calculations for this level.
173	// To make sure we never come back to this level or any lower level,
174	// set next_pair_freq impossibly large.
175	l.needed = 0;
176	levels[level + 1].next_pair_freq = math.maxInt(i32);
177	level += 1;
178	continue;
179	}
180
181	const prev_freq = l.last_freq;
182	if (l.next_char_freq < l.next_pair_freq) {
183	// The next item on this row is a leaf node.
184	const next = leaf_counts[level][level] + 1;
185	l.last_freq = l.next_char_freq;
186	// Lower leaf_counts are the same of the previous node.
187	leaf_counts[level][level] = next;
188	if (next >= list.len) {
189	l.next_char_freq = maxNode().freq;
190	} else {
191	l.next_char_freq = list[next].freq;
192	}
193	} else {
194	// The next item on this row is a pair from the previous row.
195	// next_pair_freq isn't valid until we generate two
196	// more values in the level below
197	l.last_freq = l.next_pair_freq;
198	// Take leaf counts from the lower level, except counts[level] remains the same.
199	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200	levels[l.level - 1].needed = 2;
201	}
202
203	l.needed -= 1;
204	if (l.needed == 0) {
205	// We've done everything we need to do for this level.
206	// Continue calculating one level up. Fill in next_pair_freq
207	// of that level with the sum of the two nodes we've just calculated on
208	// this level.
209	if (l.level == max_bits) {
210	// All done!
211	break;
212	}
213	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214	level += 1;
215	} else {
216	// If we stole from below, move down temporarily to replenish it.
217	while (levels[level - 1].needed > 0) {
218	level -= 1;
219	if (level == 0) {
220	break;
221	}
222	}
223	}
224	}
225	}
226
227	// Somethings is wrong if at the end, the top level is null or hasn't used
228	// all of the leaves.
229	assert(leaf_counts[max_bits][max_bits] == n);
230
231	var bit_count = self.bit_count[0 .. max_bits + 1];
232	var bits: u32 = 1;
233	const counts = &leaf_counts[max_bits];
234	{
235	var level = max_bits;
236	while (level > 0) : (level -= 1) {
237	// counts[level] gives the number of literals requiring at least "bits"
238	// bits to encode.
239	bit_count[bits] = counts[level] - counts[level - 1];
240	bits += 1;
241	if (level == 0) {
242	break;
243	}
244	}
245	}
246	return bit_count;
247	}
248
249	// Look at the leaves and assign them a bit count and an encoding as specified
250	// in RFC 1951 3.2.2
251	fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252	var code = @as(u16, 0);
253	var list = list_arg;
254
255	for (bit_count, 0..) \|bits, n\| {
256	code <<= 1;
257	if (n == 0 or bits == 0) {
258	continue;
259	}
260	// The literals list[list.len-bits] .. list[list.len-bits]
261	// are encoded using "bits" bits, and get the values
262	// code, code + 1, .... The code values are
263	// assigned in literal order (not frequency order).
264	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266	self.lns = chunk;
267	mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269	for (chunk) \|node\| {
270	self.codes[node.literal] = HuffCode{
271	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
272	.len = @as(u16, @intCast(n)),
273	};
274	code += 1;
275	}
276	list = list[0 .. list.len - @as(u32, @intCast(bits))];
277	}
278	}
279	};
280	}
281
282	fn maxNode() LiteralNode {
283	return LiteralNode{
284	.literal = math.maxInt(u16),
285	.freq = math.maxInt(u16),
286	};
287	}
288
289	pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290	return .{};
291	}
292
293	pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294	pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295	pub const CodegenEncoder = HuffmanEncoder(19);
296
297	// Generates a HuffmanCode corresponding to the fixed literal table
298	pub fn fixedLiteralEncoder() LiteralEncoder {
299	var h: LiteralEncoder = undefined;
300	var ch: u16 = 0;
301
302	while (ch < consts.max_num_frequencies) : (ch += 1) {
303	var bits: u16 = undefined;
304	var size: u16 = undefined;
305	switch (ch) {
306	0...143 => {
307	// size 8, 000110000 .. 10111111
308	bits = ch + 48;
309	size = 8;
310	},
311	144...255 => {
312	// size 9, 110010000 .. 111111111
313	bits = ch + 400 - 144;
314	size = 9;
315	},
316	256...279 => {
317	// size 7, 0000000 .. 0010111
318	bits = ch - 256;
319	size = 7;
320	},
321	else => {
322	// size 8, 11000000 .. 11000111
323	bits = ch + 192 - 280;
324	size = 8;
325	},
326	}
327	h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328	}
329	return h;
330	}
331
332	pub fn fixedDistanceEncoder() DistanceEncoder {
333	var h: DistanceEncoder = undefined;
334	for (h.codes, 0..) \|_, ch\| {
335	h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336	}
337	return h;
338	}
339
340	pub fn huffmanDistanceEncoder() DistanceEncoder {
341	var distance_freq = [1]u16{0} ** consts.distance_code_count;
342	distance_freq[0] = 1;
343	// huff_distance is a static distance encoder used for huffman only encoding.
344	// It can be reused since we will not be encoding distance values.
345	var h: DistanceEncoder = .{};
346	h.generate(distance_freq[0..], 15);
347	return h;
348	}
349
350	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351	_ = context;
352	return a.literal < b.literal;
353	}
354
355	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356	_ = context;
357	if (a.freq == b.freq) {
358	return a.literal < b.literal;
359	}
360	return a.freq < b.freq;
361	}
362
363	test "generate a Huffman code from an array of frequencies" {
364	var freqs: [19]u16 = [_]u16{
365	8, // 0
366	1, // 1
367	1, // 2
368	2, // 3
369	5, // 4
370	10, // 5
371	9, // 6
372	1, // 7
373	0, // 8
374	0, // 9
375	0, // 10
376	0, // 11
377	0, // 12
378	0, // 13
379	0, // 14
380	0, // 15
381	1, // 16
382	3, // 17
383	5, // 18
384	};
385
386	var enc = huffmanEncoder(19);
387	enc.generate(freqs[0..], 7);
388
389	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422	}
423
424	test "generate a Huffman code for the fixed literal table specific to Deflate" {
425	const enc = fixedLiteralEncoder();
426	for (enc.codes) \|c\| {
427	switch (c.len) {
428	7 => {
429	const v = @bitReverse(@as(u7, @intCast(c.code)));
430	try testing.expect(v <= 0b0010111);
431	},
432	8 => {
433	const v = @bitReverse(@as(u8, @intCast(c.code)));
434	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435	(v >= 0b11000000 and v <= 11000111));
436	},
437	9 => {
438	const v = @bitReverse(@as(u9, @intCast(c.code)));
439	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440	},
441	else => unreachable,
442	}
443	}
444	}
445
446	test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447	const enc = fixedDistanceEncoder();
448	for (enc.codes) \|c\| {
449	const v = @bitReverse(@as(u5, @intCast(c.code)));
450	try testing.expect(v <= 29);
451	try testing.expect(c.len == 5);
452	}
453	}
454
455	// Reverse bit-by-bit a N-bit code.
456	fn bitReverse(comptime T: type, value: T, n: usize) T {
457	const r = @bitReverse(value);
458	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
459	}
460
461	test bitReverse {
462	const ReverseBitsTest = struct {
463	in: u16,
464	bit_count: u5,
465	out: u16,
466	};
467
468	const reverse_bits_tests = [_]ReverseBitsTest{
469	.{ .in = 1, .bit_count = 1, .out = 1 },
470	.{ .in = 1, .bit_count = 2, .out = 2 },
471	.{ .in = 1, .bit_count = 3, .out = 4 },
472	.{ .in = 1, .bit_count = 4, .out = 8 },
473	.{ .in = 1, .bit_count = 5, .out = 16 },
474	.{ .in = 17, .bit_count = 5, .out = 17 },
475	.{ .in = 257, .bit_count = 9, .out = 257 },
476	.{ .in = 29, .bit_count = 5, .out = 23 },
477	};
478
479	for (reverse_bits_tests) \|h\| {
480	const v = bitReverse(u16, h.in, h.bit_count);
481	try std.testing.expectEqual(h.out, v);
482	}
483	}
484
485	test "fixedLiteralEncoder codes" {
486	var al = std.ArrayList(u8).init(testing.allocator);
487	defer al.deinit();
488	var bw = std.io.bitWriter(.little, al.writer());
489
490	const f = fixedLiteralEncoder();
491	for (f.codes) \|c\| {
492	try bw.writeBits(c.code, c.len);
493	}
494	try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495	}
496
497	pub const fixed_codes = [_]u8{
498	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535	0b10100011,
536	};

lib/std/compress/flate/inflate.zig deleted-570

...	@@ -1,570 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	const hfd = @import("huffman_decoder.zig");
6	const BitReader = @import("bit_reader.zig").BitReader;
7	const CircularBuffer = @import("CircularBuffer.zig");
8	const Container = @import("container.zig").Container;
9	const Token = @import("Token.zig");
10	const codegen_order = @import("consts.zig").huffman.codegen_order;
11
12	/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
13	/// `writer` stream.
14	pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
15	var d = decompressor(container, reader);
16	try d.decompress(writer);
17	}
18
19	/// Inflate decompressor for the reader type.
20	pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) {
21	return Decompressor(container, @TypeOf(reader)).init(reader);
22	}
23
24	pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type {
25	// zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
26	// gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
27	// For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
28	const lookahead: type = if (container == .zlib) u32 else u64;
29	return Inflate(container, lookahead, ReaderType);
30	}
31
32	/// Inflate decompresses deflate bit stream. Reads compressed data from reader
33	/// provided in init. Decompressed data are stored in internal hist buffer and
34	/// can be accesses iterable `next` or reader interface.
35	///
36	/// Container defines header/footer wrapper around deflate bit stream. Can be
37	/// gzip or zlib.
38	///
39	/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
40	/// * stored, non compressed, max 64k in size
41	/// * fixed, huffman codes are predefined
42	/// * dynamic, huffman code tables are encoded at the block start
43	///
44	/// `step` function runs decoder until internal `hist` buffer is full. Client
45	/// than needs to read that data in order to proceed with decoding.
46	///
47	/// Allocates 74.5K of internal buffers, most important are:
48	/// * 64K for history (CircularBuffer)
49	/// * ~10K huffman decoders (Literal and DistanceDecoder)
50	///
51	pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type {
52	assert(LookaheadType == u32 or LookaheadType == u64);
53	const BitReaderType = BitReader(LookaheadType, ReaderType);
54
55	return struct {
56	//const BitReaderType = BitReader(ReaderType);
57	const F = BitReaderType.flag;
58
59	bits: BitReaderType = .{},
60	hist: CircularBuffer = .{},
61	// Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
62	hasher: container.Hasher() = .{},
63
64	// dynamic block huffman code decoders
65	lit_dec: hfd.LiteralDecoder = .{}, // literals
66	dst_dec: hfd.DistanceDecoder = .{}, // distances
67
68	// current read state
69	bfinal: u1 = 0,
70	block_type: u2 = 0b11,
71	state: ReadState = .protocol_header,
72
73	const ReadState = enum {
74	protocol_header,
75	block_header,
76	block,
77	protocol_footer,
78	end,
79	};
80
81	const Self = @This();
82
83	pub const Error = BitReaderType.Error \|\| Container.Error \|\| hfd.Error \|\| error{
84	InvalidCode,
85	InvalidMatch,
86	InvalidBlockType,
87	WrongStoredBlockNlen,
88	InvalidDynamicBlockHeader,
89	};
90
91	pub fn init(rt: ReaderType) Self {
92	return .{ .bits = BitReaderType.init(rt) };
93	}
94
95	fn blockHeader(self: *Self) !void {
96	self.bfinal = try self.bits.read(u1);
97	self.block_type = try self.bits.read(u2);
98	}
99
100	fn storedBlock(self: *Self) !bool {
101	self.bits.alignToByte(); // skip padding until byte boundary
102	// everything after this is byte aligned in stored block
103	var len = try self.bits.read(u16);
104	const nlen = try self.bits.read(u16);
105	if (len != ~nlen) return error.WrongStoredBlockNlen;
106
107	while (len > 0) {
108	const buf = self.hist.getWritable(len);
109	try self.bits.readAll(buf);
110	len -= @intCast(buf.len);
111	}
112	return true;
113	}
114
115	fn fixedBlock(self: *Self) !bool {
116	while (!self.hist.full()) {
117	const code = try self.bits.readFixedCode();
118	switch (code) {
119	0...255 => self.hist.write(@intCast(code)),
120	256 => return true, // end of block
121	257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
122	else => return error.InvalidCode,
123	}
124	}
125	return false;
126	}
127
128	// Handles fixed block non literal (length) code.
129	// Length code is followed by 5 bits of distance code.
130	fn fixedDistanceCode(self: *Self, code: u8) !void {
131	try self.bits.fill(5 + 5 + 13);
132	const length = try self.decodeLength(code);
133	const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered \| F.reverse));
134	try self.hist.writeMatch(length, distance);
135	}
136
137	inline fn decodeLength(self: *Self, code: u8) !u16 {
138	if (code > 28) return error.InvalidCode;
139	const ml = Token.matchLength(code);
140	return if (ml.extra_bits == 0) // 0 - 5 extra bits
141	ml.base
142	else
143	ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
144	}
145
146	fn decodeDistance(self: *Self, code: u8) !u16 {
147	if (code > 29) return error.InvalidCode;
148	const md = Token.matchDistance(code);
149	return if (md.extra_bits == 0) // 0 - 13 extra bits
150	md.base
151	else
152	md.base + try self.bits.readN(md.extra_bits, F.buffered);
153	}
154
155	fn dynamicBlockHeader(self: *Self) !void {
156	const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
157	const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
158	const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
159
160	if (hlit > 286 or hdist > 30)
161	return error.InvalidDynamicBlockHeader;
162
163	// lengths for code lengths
164	var cl_lens = [_]u4{0} ** 19;
165	for (0..hclen) \|i\| {
166	cl_lens[codegen_order[i]] = try self.bits.read(u3);
167	}
168	var cl_dec: hfd.CodegenDecoder = .{};
169	try cl_dec.generate(&cl_lens);
170
171	// decoded code lengths
172	var dec_lens = [_]u4{0} ** (286 + 30);
173	var pos: usize = 0;
174	while (pos < hlit + hdist) {
175	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176	try self.bits.shift(sym.code_bits);
177	pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
178	}
179	if (pos > hlit + hdist) {
180	return error.InvalidDynamicBlockHeader;
181	}
182
183	// literal code lengths to literal decoder
184	try self.lit_dec.generate(dec_lens[0..hlit]);
185
186	// distance code lengths to distance decoder
187	try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
188	}
189
190	// Decode code length symbol to code length. Writes decoded length into
191	// lens slice starting at position pos. Returns number of positions
192	// advanced.
193	fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
194	if (pos >= lens.len)
195	return error.InvalidDynamicBlockHeader;
196
197	switch (code) {
198	0...15 => {
199	// Represent code lengths of 0 - 15
200	lens[pos] = @intCast(code);
201	return 1;
202	},
203	16 => {
204	// Copy the previous code length 3 - 6 times.
205	// The next 2 bits indicate repeat length
206	const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
207	if (pos == 0 or pos + n > lens.len)
208	return error.InvalidDynamicBlockHeader;
209	for (0..n) \|i\| {
210	lens[pos + i] = lens[pos + i - 1];
211	}
212	return n;
213	},
214	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
215	17 => return @as(u8, try self.bits.read(u3)) + 3,
216	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
217	18 => return @as(u8, try self.bits.read(u7)) + 11,
218	else => return error.InvalidDynamicBlockHeader,
219	}
220	}
221
222	// In larger archives most blocks are usually dynamic, so decompression
223	// performance depends on this function.
224	fn dynamicBlock(self: *Self) !bool {
225	// Hot path loop!
226	while (!self.hist.full()) {
227	try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
228	const sym = try self.decodeSymbol(&self.lit_dec);
229
230	switch (sym.kind) {
231	.literal => self.hist.write(sym.symbol),
232	.match => { // Decode match backreference <length, distance>
233	// fill so we can use buffered reads
234	if (LookaheadType == u32)
235	try self.bits.fill(5 + 15)
236	else
237	try self.bits.fill(5 + 15 + 13);
238	const length = try self.decodeLength(sym.symbol);
239	const dsm = try self.decodeSymbol(&self.dst_dec);
240	if (LookaheadType == u32) try self.bits.fill(13);
241	const distance = try self.decodeDistance(dsm.symbol);
242	try self.hist.writeMatch(length, distance);
243	},
244	.end_of_block => return true,
245	}
246	}
247	return false;
248	}
249
250	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
251	// decoder to find symbol for that code. We then know how many bits is
252	// used. Shift bit reader for that much bits, those bits are used. And
253	// return symbol.
254	fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
255	const sym = try decoder.find(try self.bits.peekF(u15, F.buffered \| F.reverse));
256	try self.bits.shift(sym.code_bits);
257	return sym;
258	}
259
260	fn step(self: *Self) !void {
261	switch (self.state) {
262	.protocol_header => {
263	try container.parseHeader(&self.bits);
264	self.state = .block_header;
265	},
266	.block_header => {
267	try self.blockHeader();
268	self.state = .block;
269	if (self.block_type == 2) try self.dynamicBlockHeader();
270	},
271	.block => {
272	const done = switch (self.block_type) {
273	0 => try self.storedBlock(),
274	1 => try self.fixedBlock(),
275	2 => try self.dynamicBlock(),
276	else => return error.InvalidBlockType,
277	};
278	if (done) {
279	self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
280	}
281	},
282	.protocol_footer => {
283	self.bits.alignToByte();
284	try container.parseFooter(&self.hasher, &self.bits);
285	self.state = .end;
286	},
287	.end => {},
288	}
289	}
290
291	/// Replaces the inner reader with new reader.
292	pub fn setReader(self: *Self, new_reader: ReaderType) void {
293	self.bits.forward_reader = new_reader;
294	if (self.state == .end or self.state == .protocol_footer) {
295	self.state = .protocol_header;
296	}
297	}
298
299	// Reads all compressed data from the internal reader and outputs plain
300	// (uncompressed) data to the provided writer.
301	pub fn decompress(self: *Self, writer: anytype) !void {
302	while (try self.next()) \|buf\| {
303	try writer.writeAll(buf);
304	}
305	}
306
307	/// Returns the number of bytes that have been read from the internal
308	/// reader but not yet consumed by the decompressor.
309	pub fn unreadBytes(self: Self) usize {
310	// There can be no error here: the denominator is not zero, and
311	// overflow is not possible since the type is unsigned.
312	return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
313	}
314
315	// Iterator interface
316
317	/// Can be used in iterator like loop without memcpy to another buffer:
318	/// while (try inflate.next()) \|buf\| { ... }
319	pub fn next(self: *Self) Error!?[]const u8 {
320	const out = try self.get(0);
321	if (out.len == 0) return null;
322	return out;
323	}
324
325	/// Returns decompressed data from internal sliding window buffer.
326	/// Returned buffer can be any length between 0 and `limit` bytes. 0
327	/// returned bytes means end of stream reached. With limit=0 returns as
328	/// much data it can. It newer will be more than 65536 bytes, which is
329	/// size of internal buffer.
330	pub fn get(self: *Self, limit: usize) Error![]const u8 {
331	while (true) {
332	const out = self.hist.readAtMost(limit);
333	if (out.len > 0) {
334	self.hasher.update(out);
335	return out;
336	}
337	if (self.state == .end) return out;
338	try self.step();
339	}
340	}
341
342	// Reader interface
343
344	pub const Reader = std.io.GenericReader(*Self, Error, read);
345
346	/// Returns the number of bytes read. It may be less than buffer.len.
347	/// If the number of bytes read is 0, it means end of stream.
348	/// End of stream is not an error condition.
349	pub fn read(self: *Self, buffer: []u8) Error!usize {
350	if (buffer.len == 0) return 0;
351	const out = try self.get(buffer.len);
352	@memcpy(buffer[0..out.len], out);
353	return out.len;
354	}
355
356	pub fn reader(self: *Self) Reader {
357	return .{ .context = self };
358	}
359	};
360	}
361
362	test "decompress" {
363	const cases = [_]struct {
364	in: []const u8,
365	out: []const u8,
366	}{
367	// non compressed block (type 0)
368	.{
369	.in = &[_]u8{
370	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
371	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
372	},
373	.out = "Hello world\n",
374	},
375	// fixed code block (type 1)
376	.{
377	.in = &[_]u8{
378	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
379	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
380	},
381	.out = "Hello world\n",
382	},
383	// dynamic block (type 2)
384	.{
385	.in = &[_]u8{
386	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
387	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
388	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
389	},
390	.out = "ABCDEABCD ABCDEABCD",
391	},
392	};
393	for (cases) \|c\| {
394	var fb = std.io.fixedBufferStream(c.in);
395	var al = std.ArrayList(u8).init(testing.allocator);
396	defer al.deinit();
397
398	try decompress(.raw, fb.reader(), al.writer());
399	try testing.expectEqualStrings(c.out, al.items);
400	}
401	}
402
403	test "gzip decompress" {
404	const cases = [_]struct {
405	in: []const u8,
406	out: []const u8,
407	}{
408	// non compressed block (type 0)
409	.{
410	.in = &[_]u8{
411	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
412	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
413	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
414	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
415	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
416	},
417	.out = "Hello world\n",
418	},
419	// fixed code block (type 1)
420	.{
421	.in = &[_]u8{
422	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
423	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
424	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
425	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
426	},
427	.out = "Hello world\n",
428	},
429	// dynamic block (type 2)
430	.{
431	.in = &[_]u8{
432	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
433	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
434	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
435	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
436	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
437	},
438	.out = "ABCDEABCD ABCDEABCD",
439	},
440	// gzip header with name
441	.{
442	.in = &[_]u8{
443	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
444	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
445	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
446	},
447	.out = "Hello world\n",
448	},
449	};
450	for (cases) \|c\| {
451	var fb = std.io.fixedBufferStream(c.in);
452	var al = std.ArrayList(u8).init(testing.allocator);
453	defer al.deinit();
454
455	try decompress(.gzip, fb.reader(), al.writer());
456	try testing.expectEqualStrings(c.out, al.items);
457	}
458	}
459
460	test "zlib decompress" {
461	const cases = [_]struct {
462	in: []const u8,
463	out: []const u8,
464	}{
465	// non compressed block (type 0)
466	.{
467	.in = &[_]u8{
468	0x78, 0b10_0_11100, // zlib header (2 bytes)
469	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
470	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
471	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
472	},
473	.out = "Hello world\n",
474	},
475	};
476	for (cases) \|c\| {
477	var fb = std.io.fixedBufferStream(c.in);
478	var al = std.ArrayList(u8).init(testing.allocator);
479	defer al.deinit();
480
481	try decompress(.zlib, fb.reader(), al.writer());
482	try testing.expectEqualStrings(c.out, al.items);
483	}
484	}
485
486	test "fuzzing tests" {
487	const cases = [_]struct {
488	input: []const u8,
489	out: []const u8 = "",
490	err: ?anyerror = null,
491	}{
492	.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
493	.{ .input = "empty-distance-alphabet01" },
494	.{ .input = "empty-distance-alphabet02" },
495	.{ .input = "end-of-stream", .err = error.EndOfStream },
496	.{ .input = "invalid-distance", .err = error.InvalidMatch },
497	.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
498	.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
499	.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
500	.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
501	.{ .input = "out-of-codes", .err = error.InvalidCode },
502	.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
503	.{ .input = "puff02", .err = error.EndOfStream },
504	.{ .input = "puff03", .out = &[_]u8{0xa} },
505	.{ .input = "puff04", .err = error.InvalidCode },
506	.{ .input = "puff05", .err = error.EndOfStream },
507	.{ .input = "puff06", .err = error.EndOfStream },
508	.{ .input = "puff08", .err = error.InvalidCode },
509	.{ .input = "puff09", .out = "P" },
510	.{ .input = "puff10", .err = error.InvalidCode },
511	.{ .input = "puff11", .err = error.InvalidMatch },
512	.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
513	.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
514	.{ .input = "puff14", .err = error.EndOfStream },
515	.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
516	.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
517	.{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
518	.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
519	.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
520	.{ .input = "fuzz3", .err = error.InvalidMatch },
521	.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
522	.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
523	.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
524	.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
525	.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
526	.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
527	.{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
528	.{ .input = "puff24", .err = error.IncompleteHuffmanTree },
529	.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
530	.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
531	.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
532	};
533
534	inline for (cases, 0..) \|c, case_no\| {
535	var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
536	var out = std.ArrayList(u8).init(testing.allocator);
537	defer out.deinit();
538	errdefer std.debug.print("test case failed {}\n", .{case_no});
539
540	if (c.err) \|expected_err\| {
541	try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
542	} else {
543	try decompress(.raw, in.reader(), out.writer());
544	try testing.expectEqualStrings(c.out, out.items);
545	}
546	}
547	}
548
549	test "bug 18966" {
550	const input = @embedFile("testdata/fuzz/bug_18966.input");
551	const expect = @embedFile("testdata/fuzz/bug_18966.expect");
552
553	var in = std.io.fixedBufferStream(input);
554	var out = std.ArrayList(u8).init(testing.allocator);
555	defer out.deinit();
556
557	try decompress(.gzip, in.reader(), out.writer());
558	try testing.expectEqualStrings(expect, out.items);
559	}
560
561	test "bug 19895" {
562	const input = &[_]u8{
563	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
564	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
565	};
566	var in = std.io.fixedBufferStream(input);
567	var decomp = decompressor(.raw, in.reader());
568	var buf: [0]u8 = undefined;
569	try testing.expectEqual(0, try decomp.read(&buf));
570	}

lib/std/compress/flate/testdata/block_writer.zig deleted-606

...	@@ -1,606 +0,0 @@
1	const Token = @import("../Token.zig");
2
3	pub const TestCase = struct {
4	tokens: []const Token,
5	input: []const u8 = "", // File name of input data matching the tokens.
6	want: []const u8 = "", // File name of data with the expected output with input available.
7	want_no_input: []const u8 = "", // File name of the expected output when no input is available.
8	};
9
10	pub const testCases = blk: {
11	@setEvalBranchQuota(4096 * 2);
12
13	const L = Token.initLiteral;
14	const M = Token.initMatch;
15	const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests
16
17	break :blk &[_]TestCase{
18	TestCase{
19	.input = "huffman-null-max.input",
20	.want = "huffman-null-max.{s}.expect",
21	.want_no_input = "huffman-null-max.{s}.expect-noinput",
22	.tokens = &[_]Token{
23	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
24	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
25	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
26	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
27	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
28	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
29	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
30	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
31	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
32	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
33	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
34	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
35	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0),
36	},
37	},
38	TestCase{
39	.input = "huffman-pi.input",
40	.want = "huffman-pi.{s}.expect",
41	.want_no_input = "huffman-pi.{s}.expect-noinput",
42	.tokens = &[_]Token{
43	L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'),
44	L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'),
45	L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'),
46	L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'),
47	L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'),
48	L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'),
49	L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'),
50	L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'),
51	L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'),
52	L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'),
53	L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'),
54	L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'),
55	L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'),
56	L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'),
57	L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'),
58	L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'),
59	L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'),
60	L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'),
61	L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'),
62	L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4),
63	L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'),
64	L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'),
65	L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'),
66	L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'),
67	L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'),
68	L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'),
69	L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4),
70	L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'),
71	L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'),
72	L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'),
73	L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'),
74	L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'),
75	L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4),
76	L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'),
77	L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'),
78	L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'),
79	L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'),
80	L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'),
81	L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'),
82	L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'),
83	L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'),
84	L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'),
85	L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'),
86	L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'),
87	L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'),
88	L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'),
89	L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'),
90	L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'),
91	L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'),
92	L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'),
93	L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'),
94	L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'),
95	M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'),
96	L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'),
97	L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'),
98	L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'),
99	L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'),
100	L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'),
101	L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'),
102	L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'),
103	L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'),
104	L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'),
105	L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'),
106	L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'),
107	L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'),
108	L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'),
109	L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'),
110	L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'),
111	L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'),
112	L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'),
113	M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'),
114	L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'),
115	L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'),
116	L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'),
117	L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'),
118	L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'),
119	L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'),
120	L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'),
121	L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'),
122	L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'),
123	L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'),
124	L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'),
125	L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'),
126	L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'),
127	L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'),
128	L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'),
129	L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'),
130	L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'),
131	L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'),
132	L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'),
133	M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'),
134	L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4),
135	M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'),
136	L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'),
137	L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'),
138	L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'),
139	L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'),
140	L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'),
141	L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'),
142	L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'),
143	L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4),
144	M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'),
145	L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'),
146	L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'),
147	L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'),
148	L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'),
149	M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'),
150	L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'),
151	L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'),
152	L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'),
153	L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'),
154	L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'),
155	L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'),
156	L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'),
157	L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'),
158	L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'),
159	L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'),
160	L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'),
161	L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'),
162	L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'),
163	L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'),
164	L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'),
165	L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'),
166	M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'),
167	L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'),
168	L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'),
169	M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'),
170	L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'),
171	L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'),
172	L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'),
173	L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'),
174	L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'),
175	L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'),
176	L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'),
177	M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5),
178	M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4),
179	L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'),
180	L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4),
181	L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6),
182	L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'),
183	L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'),
184	M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'),
185	L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4),
186	L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'),
187	L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'),
188	L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'),
189	L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'),
190	M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'),
191	L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'),
192	L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4),
193	L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'),
194	L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'),
195	L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'),
196	M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'),
197	L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'),
198	L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'),
199	M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'),
200	L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'),
201	L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4),
202	L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'),
203	M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'),
204	L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4),
205	L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4),
206	L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'),
207	L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'),
208	M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4),
209	L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'),
210	L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'),
211	L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'),
212	M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'),
213	L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'),
214	L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4),
215	L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'),
216	L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'),
217	L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4),
218	L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'),
219	L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'),
220	L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'),
221	L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'),
222	M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'),
223	L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'),
224	M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4),
225	L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'),
226	M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'),
227	L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'),
228	L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'),
229	L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'),
230	L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'),
231	M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'),
232	L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'),
233	L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'),
234	L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'),
235	L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'),
236	L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'),
237	L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'),
238	L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'),
239	L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'),
240	L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'),
241	M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'),
242	M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'),
243	L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'),
244	L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'),
245	M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'),
246	L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4),
247	L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'),
248	L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'),
249	L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'),
250	L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'),
251	L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4),
252	L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'),
253	L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'),
254	L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'),
255	L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4),
256	L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'),
257	M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'),
258	L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'),
259	L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'),
260	L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'),
261	L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'),
262	L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4),
263	L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'),
264	M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'),
265	L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'),
266	M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'),
267	L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'),
268	L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'),
269	L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'),
270	M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'),
271	L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'),
272	L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4),
273	L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'),
274	L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'),
275	L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'),
276	L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4),
277	L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'),
278	L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'),
279	L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'),
280	M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'),
281	L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'),
282	L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'),
283	M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'),
284	M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'),
285	L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'),
286	M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'),
287	L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4),
288	L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4),
289	L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'),
290	M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'),
291	L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'),
292	L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4),
293	L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'),
294	L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'),
295	M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'),
296	L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'),
297	L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'),
298	L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4),
299	L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4),
300	M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'),
301	L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4),
302	L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'),
303	L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'),
304	L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'),
305	L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'),
306	M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'),
307	L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'),
308	L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'),
309	L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'),
310	L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'),
311	L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'),
312	L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'),
313	L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'),
314	L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5),
315	M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4),
316	L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'),
317	M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'),
318	L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'),
319	L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'),
320	L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'),
321	L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'),
322	L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'),
323	M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4),
324	L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'),
325	L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'),
326	L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'),
327	L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'),
328	M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'),
329	L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4),
330	M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'),
331	L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'),
332	L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4),
333	L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4),
334	M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'),
335	L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'),
336	L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'),
337	L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'),
338	L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4),
339	L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4),
340	L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4),
341	L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'),
342	M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'),
343	L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'),
344	L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'),
345	L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'),
346	L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'),
347	M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4),
348	M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'),
349	L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'),
350	M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'),
351	L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'),
352	M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'),
353	M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'),
354	L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'),
355	L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'),
356	L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'),
357	L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4),
358	L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'),
359	M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'),
360	L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4),
361	M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'),
362	M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4),
363	L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'),
364	L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'),
365	L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'),
366	},
367	},
368	TestCase{
369	.input = "huffman-rand-1k.input",
370	.want = "huffman-rand-1k.{s}.expect",
371	.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
372	.tokens = &[_]Token{
373	L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d),
374	L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b),
375	L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4),
376	L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde),
377	L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14),
378	L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c),
379	L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5),
380	L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20),
381	L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4),
382	L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
383	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0),
384	L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda),
385	L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb),
386	L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43),
387	L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95),
388	L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe),
389	L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1),
390	L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9),
391	L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66),
392	L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1),
393	L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47),
394	L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5),
395	L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8),
396	L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba),
397	L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb),
398	L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62),
399	L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2),
400	L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a),
401	L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b),
402	L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7),
403	L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba),
404	L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5),
405	L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14),
406	L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67),
407	L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70),
408	L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68),
409	L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1),
410	L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa),
411	L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab),
412	L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69),
413	L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37),
414	L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff),
415	L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3),
416	L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c),
417	L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1),
418	L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67),
419	L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8),
420	L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e),
421	L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6),
422	L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69),
423	L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e),
424	L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75),
425	L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0),
426	L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99),
427	L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55),
428	L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34),
429	L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a),
430	L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f),
431	L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8),
432	L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2),
433	L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb),
434	L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48),
435	L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e),
436	L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee),
437	L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90),
438	L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3),
439	L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0),
440	L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba),
441	L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef),
442	L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75),
443	L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75),
444	L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a),
445	},
446	},
447	TestCase{
448	.input = "huffman-rand-limit.input",
449	.want = "huffman-rand-limit.{s}.expect",
450	.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
451	.tokens = &[_]Token{
452	L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80),
453	L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde),
454	L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73),
455	L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15),
456	L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9),
457	L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa),
458	L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70),
459	L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59),
460	L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46),
461	L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b),
462	L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
463	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f),
464	L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4),
465	L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa),
466	},
467	},
468	TestCase{
469	.input = "huffman-shifts.input",
470	.want = "huffman-shifts.{s}.expect",
471	.want_no_input = "huffman-shifts.{s}.expect-noinput",
472	.tokens = &[_]Token{
473	L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
474	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
475	M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258),
476	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256),
477	},
478	},
479	TestCase{
480	.input = "huffman-text-shift.input",
481	.want = "huffman-text-shift.{s}.expect",
482	.want_no_input = "huffman-text-shift.{s}.expect-noinput",
483	.tokens = &[_]Token{
484	L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'),
485	L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'),
486	L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'),
487	L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'),
488	L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'),
489	L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'),
490	L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'),
491	L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'),
492	L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'),
493	L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'),
494	L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'),
495	L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'),
496	L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'),
497	L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'),
498	L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'),
499	M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'),
500	L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'),
501	L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'),
502	L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'),
503	L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'),
504	L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'),
505	L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'),
506	L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'),
507	L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'),
508	L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa),
509	L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'),
510	L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'),
511	L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'),
512	L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'),
513	L('&'), L('/'), L('?'), L('"'),
514	},
515	},
516	TestCase{
517	.input = "huffman-text.input",
518	.want = "huffman-text.{s}.expect",
519	.want_no_input = "huffman-text.{s}.expect-noinput",
520	.tokens = &[_]Token{
521	L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'),
522	L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'),
523	L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'),
524	L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4),
525	L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'),
526	L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'),
527	L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '),
528	L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'),
529	L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'),
530	L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'),
531	L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'),
532	L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'),
533	L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'),
534	L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'),
535	L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L(''), L(''),
536	L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5),
537	M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'),
538	M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'),
539	L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'),
540	L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'),
541	L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'),
542	L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'),
543	L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'),
544	L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6),
545	L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '),
546	L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('),
547	M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'),
548	L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'),
549	L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa),
550	L('}'), L(0xa),
551	},
552	},
553	TestCase{
554	.input = "huffman-zero.input",
555	.want = "huffman-zero.{s}.expect",
556	.want_no_input = "huffman-zero.{s}.expect-noinput",
557	.tokens = &[_]Token{ L(0x30), ml, M(1, 49) },
558	},
559	TestCase{
560	.input = "",
561	.want = "",
562	.want_no_input = "null-long-match.{s}.expect-noinput",
563	.tokens = &[_]Token{
564	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
565	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
566	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
567	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
568	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
569	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
570	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
571	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
572	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
573	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
574	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
575	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
576	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
577	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
578	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
579	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
580	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
581	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
582	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
583	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
584	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
585	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
586	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
587	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
588	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
589	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
590	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
591	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
592	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
593	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
594	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
595	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
596	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
597	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
598	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
599	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
600	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
601	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
602	ml, ml, ml, M(1, 8),
603	},
604	},
605	};
606	};

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.input deleted-1

...	@@ -1 +0,0 @@
1	3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input deleted-4

...	@@ -1,4 +0,0 @@
1	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2	��vH
3	��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4	�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.input deleted-2

...	@@ -1,2 +0,0 @@
1	101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2	232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input deleted-14

...	@@ -1,14 +0,0 @@
1	//Copyright2009ThGoAuthor.Allrightrrvd.
2	//UofthiourccodigovrndbyBSD-tyl
3	//licnthtcnbfoundinthLICENSEfil.
4
5	pckgmin
6
7	import"o"
8
9	funcmin(){
10	vrb=mk([]byt,65535)
11	f,_:=o.Crt("huffmn-null-mx.in")
12	f.Writ(b)
13	}
14	ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.input deleted-14

...	@@ -1,14 +0,0 @@
1	// zig v0.10.0
2	// create a file filled with 0x00
3	const std = @import("std");
4
5	pub fn main() !void {
6	var b = [1]u8{0} ** 65535;
7	const f = try std.fs.cwd().createFile(
8	"huffman-null-max.in",
9	.{ .read = true },
10	);
11	defer f.close();
12
13	_ = try f.writeAll(b[0..]);
14	}

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.input deleted-1

...	@@ -1 +0,0 @@
1	00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput and /dev/null differ

lib/std/compress/gzip.zig deleted-66

...	@@ -1,66 +0,0 @@
1	const deflate = @import("flate/deflate.zig");
2	const inflate = @import("flate/inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.gzip, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Decompressor(.gzip, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.gzip, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.gzip, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.gzip, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.gzip, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.gzip, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.gzip, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.gzip, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.gzip, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.gzip, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.gzip, writer);
65	}
66	};

lib/std/compress/zlib.zig deleted-101

...	@@ -1,101 +0,0 @@
1	const deflate = @import("flate/deflate.zig");
2	const inflate = @import("flate/inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.zlib, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Decompressor(.zlib, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.zlib, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.zlib, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.zlib, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.zlib, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.zlib, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.zlib, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.zlib, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.zlib, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.zlib, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.zlib, writer);
65	}
66	};
67
68	test "should not overshoot" {
69	const std = @import("std");
70
71	// Compressed zlib data with extra 4 bytes at the end.
72	const data = [_]u8{
73	0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
74	0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
75	0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
76	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
77	};
78
79	var stream = std.io.fixedBufferStream(data[0..]);
80	const reader = stream.reader();
81
82	var dcp = decompressor(reader);
83	var out: [128]u8 = undefined;
84
85	// Decompress
86	var n = try dcp.reader().readAll(out[0..]);
87
88	// Expected decompressed data
89	try std.testing.expectEqual(46, n);
90	try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
91
92	// Decompressor don't overshoot underlying reader.
93	// It is leaving it at the end of compressed data chunk.
94	try std.testing.expectEqual(data.len - 4, stream.getPos());
95	try std.testing.expectEqual(0, dcp.unreadBytes());
96
97	// 4 bytes after compressed chunk are available in reader.
98	n = try reader.readAll(out[0..]);
99	try std.testing.expectEqual(n, 4);
100	try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
101	}

lib/std/compress/zstd/Decompress.zig+36-5

...	@@ -89,7 +89,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {	...	@@ -89,7 +89,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
89	.stream = stream,	89	.stream = stream,
90	.rebase = rebase,	90	.rebase = rebase,
91	.discard = discard,	91	.discard = discard,
92	.readVec = Reader.indirectReadVec,	92	.readVec = readVec,
93	},	93	},
94	.buffer = buffer,	94	.buffer = buffer,
95	.seek = 0,	95	.seek = 0,
...	@@ -109,10 +109,24 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {	...	@@ -109,10 +109,24 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {
109	r.seek -= discard_n;	109	r.seek -= discard_n;
110	}	110	}
111		111
112	fn discard(r: *Reader, limit: Limit) Reader.Error!usize {	112	/// This could be improved so that when an amount is discarded that includes an
113	r.rebase(zstd.block_size_max) catch unreachable;	113	/// entire frame, skip decoding that frame.
114	var d: Writer.Discarding = .init(r.buffer);	114	fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
115	const n = r.stream(&d.writer, limit) catch \|err\| switch (err) {	115	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
		116	r.rebase(d.window_len) catch unreachable;
		117	var writer: Writer = .{
		118	.vtable = &.{
		119	.drain = std.Io.Writer.Discarding.drain,
		120	.sendFile = std.Io.Writer.Discarding.sendFile,
		121	},
		122	.buffer = r.buffer,
		123	.end = r.end,
		124	};
		125	defer {
		126	r.end = writer.end;
		127	r.seek = r.end;
		128	}
		129	const n = r.stream(&writer, limit) catch \|err\| switch (err) {
116	error.WriteFailed => unreachable,	130	error.WriteFailed => unreachable,
117	error.ReadFailed => return error.ReadFailed,	131	error.ReadFailed => return error.ReadFailed,
118	error.EndOfStream => return error.EndOfStream,	132	error.EndOfStream => return error.EndOfStream,
...	@@ -121,6 +135,23 @@ fn discard(r: *Reader, limit: Limit) Reader.Error!usize {	...	@@ -121,6 +135,23 @@ fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
121	return n;	135	return n;
122	}	136	}
123		137
		138	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
		139	_ = data;
		140	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
		141	assert(r.seek == r.end);
		142	r.rebase(d.window_len) catch unreachable;
		143	var writer: Writer = .{
		144	.buffer = r.buffer,
		145	.end = r.end,
		146	.vtable = &.{ .drain = Writer.fixedDrain },
		147	};
		148	r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
		149	error.WriteFailed => unreachable,
		150	else => \|e\| return e,
		151	};
		152	return 0;
		153	}
		154
124	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {	155	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
125	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));	156	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
126	const in = d.input;	157	const in = d.input;

lib/std/debug/Dwarf.zig+19-13

...	@@ -2019,10 +2019,14 @@ pub fn compactUnwindToDwarfRegNumber(unwind_reg_number: u3) !u8 {	...	@@ -2019,10 +2019,14 @@ pub fn compactUnwindToDwarfRegNumber(unwind_reg_number: u3) !u8 {
2019	/// This function is to make it handy to comment out the return and make it	2019	/// This function is to make it handy to comment out the return and make it
2020	/// into a crash when working on this file.	2020	/// into a crash when working on this file.
2021	pub fn bad() error{InvalidDebugInfo} {	2021	pub fn bad() error{InvalidDebugInfo} {
2022	if (debug_debug_mode) @panic("bad dwarf");	2022	invalidDebugInfoDetected();
2023	return error.InvalidDebugInfo;	2023	return error.InvalidDebugInfo;
2024	}	2024	}
2025		2025
		2026	fn invalidDebugInfoDetected() void {
		2027	if (debug_debug_mode) @panic("bad dwarf");
		2028	}
		2029
2026	fn missing() error{MissingDebugInfo} {	2030	fn missing() error{MissingDebugInfo} {
2027	if (debug_debug_mode) @panic("missing dwarf");	2031	if (debug_debug_mode) @panic("missing dwarf");
2028	return error.MissingDebugInfo;	2032	return error.MissingDebugInfo;
...	@@ -2235,21 +2239,23 @@ pub const ElfModule = struct {	...	@@ -2235,21 +2239,23 @@ pub const ElfModule = struct {
2235		2239
2236	const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size);	2240	const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size);
2237	sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: {	2241	sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: {
2238	var section_stream = std.io.fixedBufferStream(section_bytes);	2242	var section_reader: std.Io.Reader = .fixed(section_bytes);
2239	const section_reader = section_stream.reader();	2243	const chdr = section_reader.takeStruct(elf.Chdr, endian) catch continue;
2240	const chdr = section_reader.readStruct(elf.Chdr) catch continue;
2241	if (chdr.ch_type != .ZLIB) continue;	2244	if (chdr.ch_type != .ZLIB) continue;
2242		2245
2243	var zlib_stream = std.compress.zlib.decompressor(section_reader);	2246	var decompress: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});
2244		2247	var decompressed_section: std.ArrayListUnmanaged(u8) = .empty;
2245	const decompressed_section = try gpa.alloc(u8, chdr.ch_size);	2248	defer decompressed_section.deinit(gpa);
2246	errdefer gpa.free(decompressed_section);	2249	decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch {
2247		2250	invalidDebugInfoDetected();
2248	const read = zlib_stream.reader().readAll(decompressed_section) catch continue;	2251	continue;
2249	assert(read == decompressed_section.len);	2252	};
2250		2253	if (chdr.ch_size != decompressed_section.items.len) {
		2254	invalidDebugInfoDetected();
		2255	continue;
		2256	}
2251	break :blk .{	2257	break :blk .{
2252	.data = decompressed_section,	2258	.data = try decompressed_section.toOwnedSlice(gpa),
2253	.virtual_address = shdr.sh_addr,	2259	.virtual_address = shdr.sh_addr,
2254	.owned = true,	2260	.owned = true,
2255	};	2261	};

lib/std/fs/File.zig+1-17

...	@@ -1105,22 +1105,6 @@ pub fn deprecatedWriter(file: File) DeprecatedWriter {	...	@@ -1105,22 +1105,6 @@ pub fn deprecatedWriter(file: File) DeprecatedWriter {
1105	return .{ .context = file };	1105	return .{ .context = file };
1106	}	1106	}
1107		1107
1108	/// Deprecated in favor of `Reader` and `Writer`.
1109	pub const SeekableStream = io.SeekableStream(
1110	File,
1111	SeekError,
1112	GetSeekPosError,
1113	seekTo,
1114	seekBy,
1115	getPos,
1116	getEndPos,
1117	);
1118
1119	/// Deprecated in favor of `Reader` and `Writer`.
1120	pub fn seekableStream(file: File) SeekableStream {
1121	return .{ .context = file };
1122	}
1123
1124	/// Memoizes key information about a file handle such as:	1108	/// Memoizes key information about a file handle such as:
1125	/// * The size from calling stat, or the error that occurred therein.	1109	/// * The size from calling stat, or the error that occurred therein.
1126	/// * The current seek position.	1110	/// * The current seek position.
...	@@ -1321,7 +1305,7 @@ pub const Reader = struct {	...	@@ -1321,7 +1305,7 @@ pub const Reader = struct {
1321	}	1305	}
1322	}	1306	}
1323		1307
1324	fn readVec(io_reader: *std.Io.Reader, data: []const []u8) std.Io.Reader.Error!usize {	1308	fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize {
1325	const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader));	1309	const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader));
1326	switch (r.mode) {	1310	switch (r.mode) {
1327	.positional, .positional_reading => {	1311	.positional, .positional_reading => {

lib/std/hash.zig+2-3

...	@@ -1,5 +1,4 @@	...	@@ -1,5 +1,4 @@
1	const adler = @import("hash/adler.zig");	1	pub const Adler32 = @import("hash/Adler32.zig");
2	pub const Adler32 = adler.Adler32;
3		2
4	const auto_hash = @import("hash/auto_hash.zig");	3	const auto_hash = @import("hash/auto_hash.zig");
5	pub const autoHash = auto_hash.autoHash;	4	pub const autoHash = auto_hash.autoHash;
...	@@ -116,7 +115,7 @@ test int {	...	@@ -116,7 +115,7 @@ test int {
116	}	115	}
117		116
118	test {	117	test {
119	_ = adler;	118	_ = Adler32;
120	_ = auto_hash;	119	_ = auto_hash;
121	_ = crc;	120	_ = crc;
122	_ = fnv;	121	_ = fnv;

lib/std/hash/Adler32.zig created+117

...	@@ -0,0 +1,117 @@
1	//! https://tools.ietf.org/html/rfc1950#section-9
2	//! https://github.com/madler/zlib/blob/master/adler32.c
3
4	const Adler32 = @This();
5	const std = @import("std");
6	const testing = std.testing;
7
8	adler: u32 = 1,
9
10	pub fn permute(state: u32, input: []const u8) u32 {
11	const base = 65521;
12	const nmax = 5552;
13
14	var s1 = state & 0xffff;
15	var s2 = (state >> 16) & 0xffff;
16
17	if (input.len == 1) {
18	s1 +%= input[0];
19	if (s1 >= base) {
20	s1 -= base;
21	}
22	s2 +%= s1;
23	if (s2 >= base) {
24	s2 -= base;
25	}
26	} else if (input.len < 16) {
27	for (input) \|b\| {
28	s1 +%= b;
29	s2 +%= s1;
30	}
31	if (s1 >= base) {
32	s1 -= base;
33	}
34
35	s2 %= base;
36	} else {
37	const n = nmax / 16; // note: 16 \| nmax
38
39	var i: usize = 0;
40
41	while (i + nmax <= input.len) {
42	var rounds: usize = 0;
43	while (rounds < n) : (rounds += 1) {
44	comptime var j: usize = 0;
45	inline while (j < 16) : (j += 1) {
46	s1 +%= input[i + j];
47	s2 +%= s1;
48	}
49	i += 16;
50	}
51
52	s1 %= base;
53	s2 %= base;
54	}
55
56	if (i < input.len) {
57	while (i + 16 <= input.len) : (i += 16) {
58	comptime var j: usize = 0;
59	inline while (j < 16) : (j += 1) {
60	s1 +%= input[i + j];
61	s2 +%= s1;
62	}
63	}
64	while (i < input.len) : (i += 1) {
65	s1 +%= input[i];
66	s2 +%= s1;
67	}
68
69	s1 %= base;
70	s2 %= base;
71	}
72	}
73
74	return s1 \| (s2 << 16);
75	}
76
77	pub fn update(a: *Adler32, input: []const u8) void {
78	a.adler = permute(a.adler, input);
79	}
80
81	pub fn hash(input: []const u8) u32 {
82	return permute(1, input);
83	}
84
85	test "sanity" {
86	try testing.expectEqual(@as(u32, 0x620062), hash("a"));
87	try testing.expectEqual(@as(u32, 0xbc002ed), hash("example"));
88	}
89
90	test "long" {
91	const long1 = [_]u8{1} ** 1024;
92	try testing.expectEqual(@as(u32, 0x06780401), hash(long1[0..]));
93
94	const long2 = [_]u8{1} ** 1025;
95	try testing.expectEqual(@as(u32, 0x0a7a0402), hash(long2[0..]));
96	}
97
98	test "very long" {
99	const long = [_]u8{1} ** 5553;
100	try testing.expectEqual(@as(u32, 0x707f15b2), hash(long[0..]));
101	}
102
103	test "very long with variation" {
104	const long = comptime blk: {
105	@setEvalBranchQuota(7000);
106	var result: [6000]u8 = undefined;
107
108	var i: usize = 0;
109	while (i < result.len) : (i += 1) {
110	result[i] = @as(u8, @truncate(i));
111	}
112
113	break :blk result;
114	};
115
116	try testing.expectEqual(@as(u32, 0x5af38d6e), hash(long[0..]));
117	}

lib/std/hash/adler.zig deleted-134

...	@@ -1,134 +0,0 @@
1	// Adler32 checksum.
2	//
3	// https://tools.ietf.org/html/rfc1950#section-9
4	// https://github.com/madler/zlib/blob/master/adler32.c
5
6	const std = @import("std");
7	const testing = std.testing;
8
9	pub const Adler32 = struct {
10	const base = 65521;
11	const nmax = 5552;
12
13	adler: u32,
14
15	pub fn init() Adler32 {
16	return Adler32{ .adler = 1 };
17	}
18
19	// This fast variant is taken from zlib. It reduces the required modulos and unrolls longer
20	// buffer inputs and should be much quicker.
21	pub fn update(self: *Adler32, input: []const u8) void {
22	var s1 = self.adler & 0xffff;
23	var s2 = (self.adler >> 16) & 0xffff;
24
25	if (input.len == 1) {
26	s1 +%= input[0];
27	if (s1 >= base) {
28	s1 -= base;
29	}
30	s2 +%= s1;
31	if (s2 >= base) {
32	s2 -= base;
33	}
34	} else if (input.len < 16) {
35	for (input) \|b\| {
36	s1 +%= b;
37	s2 +%= s1;
38	}
39	if (s1 >= base) {
40	s1 -= base;
41	}
42
43	s2 %= base;
44	} else {
45	const n = nmax / 16; // note: 16 \| nmax
46
47	var i: usize = 0;
48
49	while (i + nmax <= input.len) {
50	var rounds: usize = 0;
51	while (rounds < n) : (rounds += 1) {
52	comptime var j: usize = 0;
53	inline while (j < 16) : (j += 1) {
54	s1 +%= input[i + j];
55	s2 +%= s1;
56	}
57	i += 16;
58	}
59
60	s1 %= base;
61	s2 %= base;
62	}
63
64	if (i < input.len) {
65	while (i + 16 <= input.len) : (i += 16) {
66	comptime var j: usize = 0;
67	inline while (j < 16) : (j += 1) {
68	s1 +%= input[i + j];
69	s2 +%= s1;
70	}
71	}
72	while (i < input.len) : (i += 1) {
73	s1 +%= input[i];
74	s2 +%= s1;
75	}
76
77	s1 %= base;
78	s2 %= base;
79	}
80	}
81
82	self.adler = s1 \| (s2 << 16);
83	}
84
85	pub fn final(self: *Adler32) u32 {
86	return self.adler;
87	}
88
89	pub fn hash(input: []const u8) u32 {
90	var c = Adler32.init();
91	c.update(input);
92	return c.final();
93	}
94	};
95
96	test "adler32 sanity" {
97	try testing.expectEqual(@as(u32, 0x620062), Adler32.hash("a"));
98	try testing.expectEqual(@as(u32, 0xbc002ed), Adler32.hash("example"));
99	}
100
101	test "adler32 long" {
102	const long1 = [_]u8{1} ** 1024;
103	try testing.expectEqual(@as(u32, 0x06780401), Adler32.hash(long1[0..]));
104
105	const long2 = [_]u8{1} ** 1025;
106	try testing.expectEqual(@as(u32, 0x0a7a0402), Adler32.hash(long2[0..]));
107	}
108
109	test "adler32 very long" {
110	const long = [_]u8{1} ** 5553;
111	try testing.expectEqual(@as(u32, 0x707f15b2), Adler32.hash(long[0..]));
112	}
113
114	test "adler32 very long with variation" {
115	const long = comptime blk: {
116	@setEvalBranchQuota(7000);
117	var result: [6000]u8 = undefined;
118
119	var i: usize = 0;
120	while (i < result.len) : (i += 1) {
121	result[i] = @as(u8, @truncate(i));
122	}
123
124	break :blk result;
125	};
126
127	try testing.expectEqual(@as(u32, 0x5af38d6e), std.hash.Adler32.hash(long[0..]));
128	}
129
130	const verify = @import("verify.zig");
131
132	test "adler32 iterative" {
133	try verify.iterativeApi(Adler32);
134	}

lib/std/hash/verify.zig+1-1

...	@@ -45,7 +45,7 @@ pub fn smhasher(comptime hash_fn: anytype) u32 {	...	@@ -45,7 +45,7 @@ pub fn smhasher(comptime hash_fn: anytype) u32 {
45		45
46	pub fn iterativeApi(comptime Hash: anytype) !void {	46	pub fn iterativeApi(comptime Hash: anytype) !void {
47	// Sum(1..32) = 528	47	// Sum(1..32) = 528
48	var buf: [528]u8 = [_]u8{0} ** 528;	48	var buf: [528]u8 = @splat(0);
49	var len: usize = 0;	49	var len: usize = 0;
50	const seed = 0;	50	const seed = 0;
51		51

lib/std/http/Client.zig+9-15

 @@ -405,13 +405,8 @@ pub const RequestTransfer = union(enum) {
 /// The decompressor for response messages.
 pub const Compression = union(enum) {
-    pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
+    //deflate: std.compress.flate.Decompress,
-    pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
+    //gzip: std.compress.flate.Decompress,
-    // https://github.com/ziglang/zig/issues/18937
-    //pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
-    deflate: DeflateDecompressor,
-    gzip: GzipDecompressor,
     // https://github.com/ziglang/zig/issues/18937
     //zstd: ZstdDecompressor,
     none: void,
 @@ -1079,12 +1074,10 @@ pub const Request = struct {
                     switch (req.response.transfer_compression) {
                         .identity => req.response.compression = .none,
                         .compress, .@"x-compress" => return error.CompressionUnsupported,
-                        .deflate => req.response.compression = .{
+                        // I'm about to upstream my http.Client rewrite
-                            .deflate = std.compress.zlib.decompressor(req.transferReader()),
+                        .deflate => return error.CompressionUnsupported,
-                        },
+                        // I'm about to upstream my http.Client rewrite
-                        .gzip, .@"x-gzip" => req.response.compression = .{
+                        .gzip, .@"x-gzip" => return error.CompressionUnsupported,
-                            .gzip = std.compress.gzip.decompressor(req.transferReader()),
-                        },
                         // https://github.com/ziglang/zig/issues/18937
                         //.zstd => req.response.compression = .{
                         //    .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),
 @@ -1110,8 +1103,9 @@ pub const Request = struct {
     /// Reads data from the response body. Must be called after `wait`.
     pub fn read(req: *Request, buffer: []u8) ReadError!usize {
         const out_index = switch (req.response.compression) {
-            .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,
+            // I'm about to upstream my http client rewrite
-            .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
+            //.deflate => |*deflate| deflate.readSlice(buffer) catch return error.DecompressionFailure,
+            //.gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
             // https://github.com/ziglang/zig/issues/18937
             //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
             else => try req.transferRead(buffer),

lib/std/http/Server.zig+2-2

...	@@ -130,8 +130,8 @@ pub const Request = struct {	...	@@ -130,8 +130,8 @@ pub const Request = struct {
130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);	130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);	131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132		132
133	deflate: DeflateDecompressor,	133	deflate: std.compress.flate.Decompress,
134	gzip: GzipDecompressor,	134	gzip: std.compress.flate.Decompress,
135	zstd: std.compress.zstd.Decompress,	135	zstd: std.compress.zstd.Decompress,
136	none: void,	136	none: void,
137	};	137	};

lib/std/net.zig+3-2

 @@ -1973,12 +1973,13 @@ pub const Stream = struct {
             fn stream(io_r: *Io.Reader, io_w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
                 const dest = limit.slice(try io_w.writableSliceGreedy(1));
-                const n = try readVec(io_r, &.{dest});
+                var bufs: [1][]u8 = .{dest};
+                const n = try readVec(io_r, &bufs);
                 io_w.advance(n);
                 return n;
             }
-            fn readVec(io_r: *std.Io.Reader, data: []const []u8) Io.Reader.Error!usize {
+            fn readVec(io_r: *std.Io.Reader, data: [][]u8) Io.Reader.Error!usize {
                 const r: *Reader = @alignCast(@fieldParentPtr("interface_state", io_r));
                 var iovecs: [max_buffers_len]windows.ws2_32.WSABUF = undefined;
                 const bufs_n, const data_size = try io_r.writableVectorWsa(&iovecs, data);

lib/std/zip.zig+392-535

...	@@ -5,11 +5,11 @@	...	@@ -5,11 +5,11 @@
5		5
6	const builtin = @import("builtin");	6	const builtin = @import("builtin");
7	const std = @import("std");	7	const std = @import("std");
8	const testing = std.testing;	8	const File = std.fs.File;
9		9	const is_le = builtin.target.cpu.arch.endian() == .little;
10	pub const testutil = @import("zip/test.zig");	10	const Writer = std.Io.Writer;
11	const File = testutil.File;	11	const Reader = std.Io.Reader;
12	const FileStore = testutil.FileStore;	12	const flate = std.compress.flate;
13		13
14	pub const CompressionMethod = enum(u16) {	14	pub const CompressionMethod = enum(u16) {
15	store = 0,	15	store = 0,
...	@@ -95,102 +95,116 @@ pub const EndRecord = extern struct {	...	@@ -95,102 +95,116 @@ pub const EndRecord = extern struct {
95	central_directory_size: u32 align(1),	95	central_directory_size: u32 align(1),
96	central_directory_offset: u32 align(1),	96	central_directory_offset: u32 align(1),
97	comment_len: u16 align(1),	97	comment_len: u16 align(1),
		98
98	pub fn need_zip64(self: EndRecord) bool {	99	pub fn need_zip64(self: EndRecord) bool {
99	return isMaxInt(self.record_count_disk) or	100	return isMaxInt(self.record_count_disk) or
100	isMaxInt(self.record_count_total) or	101	isMaxInt(self.record_count_total) or
101	isMaxInt(self.central_directory_size) or	102	isMaxInt(self.central_directory_size) or
102	isMaxInt(self.central_directory_offset);	103	isMaxInt(self.central_directory_offset);
103	}	104	}
104	};
105		105
106	/// Find and return the end record for the given seekable zip stream.	106	pub const FindBufferError = error{ ZipNoEndRecord, ZipTruncated };
107	/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and
108	/// its context must also have a `.reader()` method that returns an instance of
109	/// `std.io.GenericReader`.
110	pub fn findEndRecord(seekable_stream: anytype, stream_len: u64) !EndRecord {
111	var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
112	const record_len_max = @min(stream_len, buf.len);
113	var loaded_len: u32 = 0;
114
115	var comment_len: u16 = 0;
116	while (true) {
117	const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
118	if (record_len > record_len_max)
119	return error.ZipNoEndRecord;
120
121	if (record_len > loaded_len) {
122	const new_loaded_len = @min(loaded_len + 300, record_len_max);
123	const read_len = new_loaded_len - loaded_len;
124
125	try seekable_stream.seekTo(stream_len - @as(u64, new_loaded_len));
126	const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
127	const len = try (if (@TypeOf(seekable_stream.context) == std.fs.File) seekable_stream.context.deprecatedReader() else seekable_stream.context.reader()).readAll(read_buf);
128	if (len != read_len)
129	return error.ZipTruncated;
130	loaded_len = new_loaded_len;
131	}
132		107
133	const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];	108	/// TODO audit this logic
134	if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and	109	pub fn findBuffer(buffer: []const u8) FindBufferError!EndRecord {
135	std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)	110	const pos = std.mem.lastIndexOf(u8, buffer, &end_record_sig) orelse return error.ZipNoEndRecord;
136	{	111	if (pos + @sizeOf(EndRecord) > buffer.len) return error.EndOfStream;
137	const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);	112	const record_ptr: *EndRecord = @ptrCast(buffer[pos..][0..@sizeOf(EndRecord)]);
138	if (builtin.target.cpu.arch.endian() != .little) {	113	var record = record_ptr.*;
139	std.mem.byteSwapAllFields(@TypeOf(record.*), record);	114	if (!is_le) std.mem.byteSwapAllFields(EndRecord, &record);
		115	return record;
		116	}
		117
		118	pub const FindFileError = File.GetEndPosError \|\| File.SeekError \|\| File.ReadError \|\| error{
		119	ZipNoEndRecord,
		120	EndOfStream,
		121	ReadFailed,
		122	};
		123
		124	pub fn findFile(fr: *File.Reader) FindFileError!EndRecord {
		125	const end_pos = try fr.getSize();
		126
		127	var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
		128	const record_len_max = @min(end_pos, buf.len);
		129	var loaded_len: u32 = 0;
		130	var comment_len: u16 = 0;
		131	while (true) {
		132	const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
		133	if (record_len > record_len_max)
		134	return error.ZipNoEndRecord;
		135
		136	if (record_len > loaded_len) {
		137	const new_loaded_len = @min(loaded_len + 300, record_len_max);
		138	const read_len = new_loaded_len - loaded_len;
		139
		140	try fr.seekTo(end_pos - @as(u64, new_loaded_len));
		141	const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
		142	fr.interface.readSliceAll(read_buf) catch \|err\| switch (err) {
		143	error.ReadFailed => return fr.err.?,
		144	error.EndOfStream => return error.EndOfStream,
		145	};
		146	loaded_len = new_loaded_len;
		147	}
		148
		149	const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];
		150	if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and
		151	std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)
		152	{
		153	const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);
		154	if (!is_le) std.mem.byteSwapAllFields(EndRecord, record);
		155	return record.*;
140	}	156	}
141	return record.*;	157
		158	if (comment_len == std.math.maxInt(u16))
		159	return error.ZipNoEndRecord;
		160	comment_len += 1;
142	}	161	}
		162	}
		163	};
143		164
144	if (comment_len == std.math.maxInt(u16))	165	pub const Decompress = struct {
145	return error.ZipNoEndRecord;	166	interface: Reader,
146	comment_len += 1;	167	state: union {
		168	inflate: flate.Decompress,
		169	store: *Reader,
		170	},
		171
		172	pub fn init(reader: *Reader, method: CompressionMethod, buffer: []u8) Reader {
		173	return switch (method) {
		174	.store => .{
		175	.state = .{ .store = reader },
		176	.interface = .{
		177	.context = undefined,
		178	.vtable = &.{ .stream = streamStore },
		179	.buffer = buffer,
		180	.end = 0,
		181	.seek = 0,
		182	},
		183	},
		184	.deflate => .{
		185	.state = .{ .inflate = .init(reader, .raw) },
		186	.interface = .{
		187	.context = undefined,
		188	.vtable = &.{ .stream = streamDeflate },
		189	.buffer = buffer,
		190	.end = 0,
		191	.seek = 0,
		192	},
		193	},
		194	else => unreachable,
		195	};
147	}	196	}
148	}
149		197
150	/// Decompresses the given data from `reader` into `writer`. Stops early if more	198	fn streamStore(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
151	/// than `uncompressed_size` bytes are processed and verifies that exactly that	199	const d: *Decompress = @fieldParentPtr("interface", r);
152	/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data.	200	return d.store.read(w, limit);
153	/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method.
154	pub fn decompress(
155	method: CompressionMethod,
156	uncompressed_size: u64,
157	reader: anytype,
158	writer: anytype,
159	) !u32 {
160	var hash = std.hash.Crc32.init();
161
162	var total_uncompressed: u64 = 0;
163	switch (method) {
164	.store => {
165	var buf: [4096]u8 = undefined;
166	while (true) {
167	const len = try reader.read(&buf);
168	if (len == 0) break;
169	try writer.writeAll(buf[0..len]);
170	hash.update(buf[0..len]);
171	total_uncompressed += @intCast(len);
172	}
173	},
174	.deflate => {
175	var br = std.io.bufferedReader(reader);
176	var decompressor = std.compress.flate.decompressor(br.reader());
177	while (try decompressor.next()) \|chunk\| {
178	try writer.writeAll(chunk);
179	hash.update(chunk);
180	total_uncompressed += @intCast(chunk.len);
181	if (total_uncompressed > uncompressed_size)
182	return error.ZipUncompressSizeTooSmall;
183	}
184	if (br.end != br.start)
185	return error.ZipDeflateTruncated;
186	},
187	_ => return error.UnsupportedCompressionMethod,
188	}	201	}
189	if (total_uncompressed != uncompressed_size)
190	return error.ZipUncompressSizeMismatch;
191		202
192	return hash.final();	203	fn streamDeflate(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
193	}	204	const d: *Decompress = @fieldParentPtr("interface", r);
		205	return flate.Decompress.read(&d.inflate, w, limit);
		206	}
		207	};
194		208
195	fn isBadFilename(filename: []const u8) bool {	209	fn isBadFilename(filename: []const u8) bool {
196	if (filename.len == 0 or filename[0] == '/')	210	if (filename.len == 0 or filename[0] == '/')
...	@@ -253,319 +267,337 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data	...	@@ -253,319 +267,337 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data
253	}	267	}
254	}	268	}
255		269
256	pub fn Iterator(comptime SeekableStream: type) type {	270	pub const Iterator = struct {
257	return struct {	271	input: *File.Reader,
258	stream: SeekableStream,
259		272
260	cd_record_count: u64,	273	cd_record_count: u64,
261	cd_zip_offset: u64,	274	cd_zip_offset: u64,
262	cd_size: u64,	275	cd_size: u64,
263		276
264	cd_record_index: u64 = 0,	277	cd_record_index: u64 = 0,
265	cd_record_offset: u64 = 0,	278	cd_record_offset: u64 = 0,
266		279
267	const Self = @This();	280	pub fn init(input: *File.Reader) !Iterator {
		281	const end_record = try EndRecord.findFile(input);
268		282
269	pub fn init(stream: SeekableStream) !Self {	283	if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
270	const stream_len = try stream.getEndPos();	284	return error.ZipDiskRecordCountTooLarge;
271
272	const end_record = try findEndRecord(stream, stream_len);
273
274	if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
275	return error.ZipDiskRecordCountTooLarge;
276
277	if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
278	return error.ZipMultiDiskUnsupported;
279		285
280	{	286	if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
281	const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);	287	return error.ZipMultiDiskUnsupported;
282	if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
283	return error.ZipMultiDiskUnsupported;
284	}
285		288
286	var result = Self{	289	{
287	.stream = stream,	290	const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);
288	.cd_record_count = end_record.record_count_total,	291	if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
289	.cd_zip_offset = end_record.central_directory_offset,
290	.cd_size = end_record.central_directory_size,
291	};
292	if (!end_record.need_zip64()) return result;
293
294	const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
295	if (locator_end_offset > stream_len)
296	return error.ZipTruncated;
297	try stream.seekTo(stream_len - locator_end_offset);
298	const locator = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndLocator64, .little);
299	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
300	return error.ZipBadLocatorSig;
301	if (locator.zip64_disk_count != 0)
302	return error.ZipUnsupportedZip64DiskCount;
303	if (locator.total_disk_count != 1)
304	return error.ZipMultiDiskUnsupported;	292	return error.ZipMultiDiskUnsupported;
		293	}
305		294
306	try stream.seekTo(locator.record_file_offset);	295	var result: Iterator = .{
307		296	.input = input,
308	const record64 = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndRecord64, .little);	297	.cd_record_count = end_record.record_count_total,
309		298	.cd_zip_offset = end_record.central_directory_offset,
310	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))	299	.cd_size = end_record.central_directory_size,
311	return error.ZipBadEndRecord64Sig;	300	};
312		301	if (!end_record.need_zip64()) return result;
313	if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
314	return error.ZipEndRecord64SizeTooSmall;
315	if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
316	return error.ZipEndRecord64UnhandledExtraData;
317		302
318	if (record64.version_needed_to_extract > 45)	303	const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
319	return error.ZipUnsupportedVersion;	304	const stream_len = try input.getSize();
320		305
321	{	306	if (locator_end_offset > stream_len)
322	const is_multidisk = record64.disk_number != 0 or	307	return error.ZipTruncated;
323	record64.central_directory_disk_number != 0 or	308	try input.seekTo(stream_len - locator_end_offset);
324	record64.record_count_disk != record64.record_count_total;	309	const locator = input.interface.takeStruct(EndLocator64, .little) catch \|err\| switch (err) {
325	if (is_multidisk)	310	error.ReadFailed => return input.err.?,
326	return error.ZipMultiDiskUnsupported;	311	error.EndOfStream => return error.EndOfStream,
327	}	312	};
		313	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
		314	return error.ZipBadLocatorSig;
		315	if (locator.zip64_disk_count != 0)
		316	return error.ZipUnsupportedZip64DiskCount;
		317	if (locator.total_disk_count != 1)
		318	return error.ZipMultiDiskUnsupported;
		319
		320	try input.seekTo(locator.record_file_offset);
		321
		322	const record64 = input.interface.takeStruct(EndRecord64, .little) catch \|err\| switch (err) {
		323	error.ReadFailed => return input.err.?,
		324	error.EndOfStream => return error.EndOfStream,
		325	};
328		326
329	if (isMaxInt(end_record.record_count_total)) {	327	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
330	result.cd_record_count = record64.record_count_total;	328	return error.ZipBadEndRecord64Sig;
331	} else if (end_record.record_count_total != record64.record_count_total)
332	return error.Zip64RecordCountTotalMismatch;
333		329
334	if (isMaxInt(end_record.central_directory_offset)) {	330	if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
335	result.cd_zip_offset = record64.central_directory_offset;	331	return error.ZipEndRecord64SizeTooSmall;
336	} else if (end_record.central_directory_offset != record64.central_directory_offset)	332	if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
337	return error.Zip64CentralDirectoryOffsetMismatch;	333	return error.ZipEndRecord64UnhandledExtraData;
338		334
339	if (isMaxInt(end_record.central_directory_size)) {	335	if (record64.version_needed_to_extract > 45)
340	result.cd_size = record64.central_directory_size;	336	return error.ZipUnsupportedVersion;
341	} else if (end_record.central_directory_size != record64.central_directory_size)
342	return error.Zip64CentralDirectorySizeMismatch;
343		337
344	return result;	338	{
		339	const is_multidisk = record64.disk_number != 0 or
		340	record64.central_directory_disk_number != 0 or
		341	record64.record_count_disk != record64.record_count_total;
		342	if (is_multidisk)
		343	return error.ZipMultiDiskUnsupported;
345	}	344	}
346		345
347	pub fn next(self: *Self) !?Entry {	346	if (isMaxInt(end_record.record_count_total)) {
348	if (self.cd_record_index == self.cd_record_count) {	347	result.cd_record_count = record64.record_count_total;
349	if (self.cd_record_offset != self.cd_size)	348	} else if (end_record.record_count_total != record64.record_count_total)
350	return if (self.cd_size > self.cd_record_offset)	349	return error.Zip64RecordCountTotalMismatch;
351	error.ZipCdOversized
352	else
353	error.ZipCdUndersized;
354		350
355	return null;	351	if (isMaxInt(end_record.central_directory_offset)) {
356	}	352	result.cd_zip_offset = record64.central_directory_offset;
		353	} else if (end_record.central_directory_offset != record64.central_directory_offset)
		354	return error.Zip64CentralDirectoryOffsetMismatch;
357		355
358	const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;	356	if (isMaxInt(end_record.central_directory_size)) {
359	try self.stream.seekTo(header_zip_offset);	357	result.cd_size = record64.central_directory_size;
360	const header = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readStructEndian(CentralDirectoryFileHeader, .little);	358	} else if (end_record.central_directory_size != record64.central_directory_size)
361	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))	359	return error.Zip64CentralDirectorySizeMismatch;
362	return error.ZipBadCdOffset;
363		360
364	self.cd_record_index += 1;	361	return result;
365	self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;	362	}
366		363
367	// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file	364	pub fn next(self: *Iterator) !?Entry {
368	// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip	365	if (self.cd_record_index == self.cd_record_count) {
369	// has an undocumented version 788 but extracts just fine.	366	if (self.cd_record_offset != self.cd_size)
		367	return if (self.cd_size > self.cd_record_offset)
		368	error.ZipCdOversized
		369	else
		370	error.ZipCdUndersized;
370		371
371	if (header.flags.encrypted)	372	return null;
372	return error.ZipEncryptionUnsupported;	373	}
373	// TODO: check/verify more flags
374	if (header.disk_number != 0)
375	return error.ZipMultiDiskUnsupported;
376		374
377	var extents: FileExtents = .{	375	const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
378	.uncompressed_size = header.uncompressed_size,	376	const input = self.input;
379	.compressed_size = header.compressed_size,	377	try input.seekTo(header_zip_offset);
380	.local_file_header_offset = header.local_file_header_offset,	378	const header = input.interface.takeStruct(CentralDirectoryFileHeader, .little) catch \|err\| switch (err) {
381	};	379	error.ReadFailed => return input.err.?,
		380	error.EndOfStream => return error.EndOfStream,
		381	};
		382	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
		383	return error.ZipBadCdOffset;
		384
		385	self.cd_record_index += 1;
		386	self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;
		387
		388	// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file
		389	// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip
		390	// has an undocumented version 788 but extracts just fine.
		391
		392	if (header.flags.encrypted)
		393	return error.ZipEncryptionUnsupported;
		394	// TODO: check/verify more flags
		395	if (header.disk_number != 0)
		396	return error.ZipMultiDiskUnsupported;
		397
		398	var extents: FileExtents = .{
		399	.uncompressed_size = header.uncompressed_size,
		400	.compressed_size = header.compressed_size,
		401	.local_file_header_offset = header.local_file_header_offset,
		402	};
382		403
383	if (header.extra_len > 0) {	404	if (header.extra_len > 0) {
384	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;	405	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
385	const extra = extra_buf[0..header.extra_len];	406	const extra = extra_buf[0..header.extra_len];
386		407
387	{	408	try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
388	try self.stream.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);	409	input.interface.readSliceAll(extra) catch \|err\| switch (err) {
389	const len = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readAll(extra);	410	error.ReadFailed => return input.err.?,
390	if (len != extra.len)	411	error.EndOfStream => return error.EndOfStream,
391	return error.ZipTruncated;	412	};
392	}
393		413
394	var extra_offset: usize = 0;	414	var extra_offset: usize = 0;
395	while (extra_offset + 4 <= extra.len) {	415	while (extra_offset + 4 <= extra.len) {
396	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);	416	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
397	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);	417	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
398	const end = extra_offset + 4 + data_size;	418	const end = extra_offset + 4 + data_size;
399	if (end > extra.len)	419	if (end > extra.len)
400	return error.ZipBadExtraFieldSize;	420	return error.ZipBadExtraFieldSize;
401	const data = extra[extra_offset + 4 .. end];	421	const data = extra[extra_offset + 4 .. end];
402	switch (@as(ExtraHeader, @enumFromInt(header_id))) {	422	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
403	.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),	423	.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),
404	else => {}, // ignore	424	else => {}, // ignore
405	}
406	extra_offset = end;
407	}	425	}
		426	extra_offset = end;
408	}	427	}
409
410	return .{
411	.version_needed_to_extract = header.version_needed_to_extract,
412	.flags = header.flags,
413	.compression_method = header.compression_method,
414	.last_modification_time = header.last_modification_time,
415	.last_modification_date = header.last_modification_date,
416	.header_zip_offset = header_zip_offset,
417	.crc32 = header.crc32,
418	.filename_len = header.filename_len,
419	.compressed_size = extents.compressed_size,
420	.uncompressed_size = extents.uncompressed_size,
421	.file_offset = extents.local_file_header_offset,
422	};
423	}	428	}
424		429
425	pub const Entry = struct {	430	return .{
426	version_needed_to_extract: u16,	431	.version_needed_to_extract = header.version_needed_to_extract,
427	flags: GeneralPurposeFlags,	432	.flags = header.flags,
428	compression_method: CompressionMethod,	433	.compression_method = header.compression_method,
429	last_modification_time: u16,	434	.last_modification_time = header.last_modification_time,
430	last_modification_date: u16,	435	.last_modification_date = header.last_modification_date,
431	header_zip_offset: u64,	436	.header_zip_offset = header_zip_offset,
432	crc32: u32,	437	.crc32 = header.crc32,
433	filename_len: u32,	438	.filename_len = header.filename_len,
434	compressed_size: u64,	439	.compressed_size = extents.compressed_size,
435	uncompressed_size: u64,	440	.uncompressed_size = extents.uncompressed_size,
436	file_offset: u64,	441	.file_offset = extents.local_file_header_offset,
437		442	};
438	pub fn extract(	443	}
439	self: Entry,
440	stream: SeekableStream,
441	options: ExtractOptions,
442	filename_buf: []u8,
443	dest: std.fs.Dir,
444	) !u32 {
445	if (filename_buf.len < self.filename_len)
446	return error.ZipInsufficientBuffer;
447	const filename = filename_buf[0..self.filename_len];
448		444
		445	pub const Entry = struct {
		446	version_needed_to_extract: u16,
		447	flags: GeneralPurposeFlags,
		448	compression_method: CompressionMethod,
		449	last_modification_time: u16,
		450	last_modification_date: u16,
		451	header_zip_offset: u64,
		452	crc32: u32,
		453	filename_len: u32,
		454	compressed_size: u64,
		455	uncompressed_size: u64,
		456	file_offset: u64,
		457
		458	pub fn extract(
		459	self: Entry,
		460	stream: *File.Reader,
		461	options: ExtractOptions,
		462	filename_buf: []u8,
		463	dest: std.fs.Dir,
		464	) !void {
		465	if (filename_buf.len < self.filename_len)
		466	return error.ZipInsufficientBuffer;
		467	switch (self.compression_method) {
		468	.store, .deflate => {},
		469	else => return error.UnsupportedCompressionMethod,
		470	}
		471	const filename = filename_buf[0..self.filename_len];
		472	{
449	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));	473	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
		474	try stream.interface.readSliceAll(filename);
		475	}
450		476
451	{	477	const local_data_header_offset: u64 = local_data_header_offset: {
452	const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(filename);	478	const local_header = blk: {
453	if (len != filename.len)	479	try stream.seekTo(self.file_offset);
454	return error.ZipBadFileOffset;	480	break :blk try stream.interface.takeStruct(LocalFileHeader, .little);
455	}	481	};
		482	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
		483	return error.ZipBadFileOffset;
		484	if (local_header.version_needed_to_extract != self.version_needed_to_extract)
		485	return error.ZipMismatchVersionNeeded;
		486	if (local_header.last_modification_time != self.last_modification_time)
		487	return error.ZipMismatchModTime;
		488	if (local_header.last_modification_date != self.last_modification_date)
		489	return error.ZipMismatchModDate;
		490
		491	if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
		492	return error.ZipMismatchFlags;
		493	if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
		494	return error.ZipMismatchCrc32;
		495	var extents: FileExtents = .{
		496	.uncompressed_size = local_header.uncompressed_size,
		497	.compressed_size = local_header.compressed_size,
		498	.local_file_header_offset = 0,
		499	};
		500	if (local_header.extra_len > 0) {
		501	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
		502	const extra = extra_buf[0..local_header.extra_len];
456		503
457	const local_data_header_offset: u64 = local_data_header_offset: {	504	{
458	const local_header = blk: {	505	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
459	try stream.seekTo(self.file_offset);	506	try stream.interface.readSliceAll(extra);
460	break :blk try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(LocalFileHeader, .little);	507	}
461	};
462	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
463	return error.ZipBadFileOffset;
464	if (local_header.version_needed_to_extract != self.version_needed_to_extract)
465	return error.ZipMismatchVersionNeeded;
466	if (local_header.last_modification_time != self.last_modification_time)
467	return error.ZipMismatchModTime;
468	if (local_header.last_modification_date != self.last_modification_date)
469	return error.ZipMismatchModDate;
470
471	if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
472	return error.ZipMismatchFlags;
473	if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
474	return error.ZipMismatchCrc32;
475	var extents: FileExtents = .{
476	.uncompressed_size = local_header.uncompressed_size,
477	.compressed_size = local_header.compressed_size,
478	.local_file_header_offset = 0,
479	};
480	if (local_header.extra_len > 0) {
481	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
482	const extra = extra_buf[0..local_header.extra_len];
483
484	{
485	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
486	const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(extra);
487	if (len != extra.len)
488	return error.ZipTruncated;
489	}
490		508
491	var extra_offset: usize = 0;	509	var extra_offset: usize = 0;
492	while (extra_offset + 4 <= local_header.extra_len) {	510	while (extra_offset + 4 <= local_header.extra_len) {
493	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);	511	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
494	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);	512	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
495	const end = extra_offset + 4 + data_size;	513	const end = extra_offset + 4 + data_size;
496	if (end > local_header.extra_len)	514	if (end > local_header.extra_len)
497	return error.ZipBadExtraFieldSize;	515	return error.ZipBadExtraFieldSize;
498	const data = extra[extra_offset + 4 .. end];	516	const data = extra[extra_offset + 4 .. end];
499	switch (@as(ExtraHeader, @enumFromInt(header_id))) {	517	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
500	.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),	518	.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),
501	else => {}, // ignore	519	else => {}, // ignore
502	}
503	extra_offset = end;
504	}	520	}
		521	extra_offset = end;
505	}	522	}
		523	}
506		524
507	if (extents.compressed_size != 0 and	525	if (extents.compressed_size != 0 and
508	extents.compressed_size != self.compressed_size)	526	extents.compressed_size != self.compressed_size)
509	return error.ZipMismatchCompLen;	527	return error.ZipMismatchCompLen;
510	if (extents.uncompressed_size != 0 and	528	if (extents.uncompressed_size != 0 and
511	extents.uncompressed_size != self.uncompressed_size)	529	extents.uncompressed_size != self.uncompressed_size)
512	return error.ZipMismatchUncompLen;	530	return error.ZipMismatchUncompLen;
513		531
514	if (local_header.filename_len != self.filename_len)	532	if (local_header.filename_len != self.filename_len)
515	return error.ZipMismatchFilenameLen;	533	return error.ZipMismatchFilenameLen;
516		534
517	break :local_data_header_offset @as(u64, local_header.filename_len) +	535	break :local_data_header_offset @as(u64, local_header.filename_len) +
518	@as(u64, local_header.extra_len);	536	@as(u64, local_header.extra_len);
519	};	537	};
520		538
521	if (isBadFilename(filename))	539	if (isBadFilename(filename))
522	return error.ZipBadFilename;	540	return error.ZipBadFilename;
523		541
524	if (options.allow_backslashes) {	542	if (options.allow_backslashes) {
525	std.mem.replaceScalar(u8, filename, '\\', '/');	543	std.mem.replaceScalar(u8, filename, '\\', '/');
526	} else {	544	} else {
527	if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|	545	if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|
528	return error.ZipFilenameHasBackslash;	546	return error.ZipFilenameHasBackslash;
529	}	547	}
530		548
531	// All entries that end in '/' are directories	549	// All entries that end in '/' are directories
532	if (filename[filename.len - 1] == '/') {	550	if (filename[filename.len - 1] == '/') {
533	if (self.uncompressed_size != 0)	551	if (self.uncompressed_size != 0)
534	return error.ZipBadDirectorySize;	552	return error.ZipBadDirectorySize;
535	try dest.makePath(filename[0 .. filename.len - 1]);	553	try dest.makePath(filename[0 .. filename.len - 1]);
536	return std.hash.Crc32.hash(&.{});	554	return;
537	}	555	}
538		556
539	const out_file = blk: {	557	const out_file = blk: {
540	if (std.fs.path.dirname(filename)) \|dirname\| {	558	if (std.fs.path.dirname(filename)) \|dirname\| {
541	var parent_dir = try dest.makeOpenPath(dirname, .{});	559	var parent_dir = try dest.makeOpenPath(dirname, .{});
542	defer parent_dir.close();	560	defer parent_dir.close();
543		561
544	const basename = std.fs.path.basename(filename);	562	const basename = std.fs.path.basename(filename);
545	break :blk try parent_dir.createFile(basename, .{ .exclusive = true });	563	break :blk try parent_dir.createFile(basename, .{ .exclusive = true });
546	}	564	}
547	break :blk try dest.createFile(filename, .{ .exclusive = true });	565	break :blk try dest.createFile(filename, .{ .exclusive = true });
548	};	566	};
549	defer out_file.close();	567	defer out_file.close();
550	const local_data_file_offset: u64 =	568	var out_file_buffer: [1024]u8 = undefined;
551	@as(u64, self.file_offset) +	569	var file_writer = out_file.writer(&out_file_buffer);
552	@as(u64, @sizeOf(LocalFileHeader)) +	570	const local_data_file_offset: u64 =
553	local_data_header_offset;	571	@as(u64, self.file_offset) +
554	try stream.seekTo(local_data_file_offset);	572	@as(u64, @sizeOf(LocalFileHeader)) +
555	var limited_reader = std.io.limitedReader((if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()), self.compressed_size);	573	local_data_header_offset;
556	const crc = try decompress(	574	try stream.seekTo(local_data_file_offset);
557	self.compression_method,	575
558	self.uncompressed_size,	576	// TODO limit based on self.compressed_size
559	limited_reader.reader(),	577
560	out_file.deprecatedWriter(),	578	switch (self.compression_method) {
561	);	579	.store => {
562	if (limited_reader.bytes_left != 0)	580	stream.interface.streamExact64(&file_writer.interface, self.uncompressed_size) catch \|err\| switch (err) {
563	return error.ZipDecompressTruncated;	581	error.ReadFailed => return stream.err.?,
564	return crc;	582	error.WriteFailed => return file_writer.err.?,
		583	error.EndOfStream => return error.ZipDecompressTruncated,
		584	};
		585	},
		586	.deflate => {
		587	var flate_buffer: [flate.max_window_len]u8 = undefined;
		588	var decompress: flate.Decompress = .init(&stream.interface, .raw, &flate_buffer);
		589	decompress.reader.streamExact64(&file_writer.interface, self.uncompressed_size) catch \|err\| switch (err) {
		590	error.ReadFailed => return stream.err.?,
		591	error.WriteFailed => return file_writer.err orelse decompress.err.?,
		592	error.EndOfStream => return error.ZipDecompressTruncated,
		593	};
		594	},
		595	else => return error.UnsupportedCompressionMethod,
565	}	596	}
566	};	597	try file_writer.end();
		598	}
567	};	599	};
568	}	600	};
569		601
570	// returns true if `filename` starts with `root` followed by a forward slash	602	// returns true if `filename` starts with `root` followed by a forward slash
571	fn filenameInRoot(filename: []const u8, root: []const u8) bool {	603	fn filenameInRoot(filename: []const u8, root: []const u8) bool {
...	@@ -610,196 +642,21 @@ pub const ExtractOptions = struct {	...	@@ -610,196 +642,21 @@ pub const ExtractOptions = struct {
610	/// Allow filenames within the zip to use backslashes. Back slashes are normalized	642	/// Allow filenames within the zip to use backslashes. Back slashes are normalized
611	/// to forward slashes before forwarding them to platform APIs.	643	/// to forward slashes before forwarding them to platform APIs.
612	allow_backslashes: bool = false,	644	allow_backslashes: bool = false,
613
614	diagnostics: ?*Diagnostics = null,	645	diagnostics: ?*Diagnostics = null,
		646	verify_checksums: bool = false,
615	};	647	};
616		648
617	/// Extract the zipped files inside `seekable_stream` to the given `dest` directory.	649	/// Extract the zipped files to the given `dest` directory.
618	/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and	650	pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void {
619	/// its context must also have a `.reader()` method that returns an instance of	651	if (options.verify_checksums) @panic("TODO unimplemented");
620	/// `std.io.GenericReader`.	652
621	pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptions) !void {	653	var iter = try Iterator.init(fr);
622	const SeekableStream = @TypeOf(seekable_stream);
623	var iter = try Iterator(SeekableStream).init(seekable_stream);
624		654
625	var filename_buf: [std.fs.max_path_bytes]u8 = undefined;	655	var filename_buf: [std.fs.max_path_bytes]u8 = undefined;
626	while (try iter.next()) \|entry\| {	656	while (try iter.next()) \|entry\| {
627	const crc32 = try entry.extract(seekable_stream, options, &filename_buf, dest);	657	try entry.extract(fr, options, &filename_buf, dest);
628	if (crc32 != entry.crc32)
629	return error.ZipCrcMismatch;
630	if (options.diagnostics) \|d\| {	658	if (options.diagnostics) \|d\| {
631	try d.nextFilename(filename_buf[0..entry.filename_len]);	659	try d.nextFilename(filename_buf[0..entry.filename_len]);
632	}	660	}
633	}	661	}
634	}	662	}
635
636	fn testZip(options: ExtractOptions, comptime files: []const File, write_opt: testutil.WriteZipOptions) !void {
637	var store: [files.len]FileStore = undefined;
638	try testZipWithStore(options, files, write_opt, &store);
639	}
640	fn testZipWithStore(
641	options: ExtractOptions,
642	test_files: []const File,
643	write_opt: testutil.WriteZipOptions,
644	store: []FileStore,
645	) !void {
646	var zip_buf: [4096]u8 = undefined;
647	var fbs = try testutil.makeZipWithStore(&zip_buf, test_files, write_opt, store);
648
649	var tmp = testing.tmpDir(.{ .no_follow = true });
650	defer tmp.cleanup();
651	try extract(tmp.dir, fbs.seekableStream(), options);
652	try testutil.expectFiles(test_files, tmp.dir, .{});
653	}
654	fn testZipError(expected_error: anyerror, file: File, options: ExtractOptions) !void {
655	var zip_buf: [4096]u8 = undefined;
656	var store: [1]FileStore = undefined;
657	var fbs = try testutil.makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store);
658	var tmp = testing.tmpDir(.{ .no_follow = true });
659	defer tmp.cleanup();
660	try testing.expectError(expected_error, extract(tmp.dir, fbs.seekableStream(), options));
661	}
662
663	test "zip one file" {
664	try testZip(.{}, &[_]File{
665	.{ .name = "onefile.txt", .content = "Just a single file\n", .compression = .store },
666	}, .{});
667	}
668	test "zip multiple files" {
669	try testZip(.{ .allow_backslashes = true }, &[_]File{
670	.{ .name = "foo", .content = "a foo file\n", .compression = .store },
671	.{ .name = "subdir/bar", .content = "bar is this right?\nanother newline\n", .compression = .store },
672	.{ .name = "subdir\\whoa", .content = "you can do backslashes", .compression = .store },
673	.{ .name = "subdir/another/baz", .content = "bazzy mc bazzerson", .compression = .store },
674	}, .{});
675	}
676	test "zip deflated" {
677	try testZip(.{}, &[_]File{
678	.{ .name = "deflateme", .content = "This is a deflated file.\nIt should be smaller in the Zip file1\n", .compression = .deflate },
679	// TODO: re-enable this if/when we add support for deflate64
680	//.{ .name = "deflateme64", .content = "The 64k version of deflate!\n", .compression = .deflate64 },
681	.{ .name = "raw", .content = "Not all files need to be deflated in the same Zip.\n", .compression = .store },
682	}, .{});
683	}
684	test "zip verify filenames" {
685	// no empty filenames
686	try testZipError(error.ZipBadFilename, .{ .name = "", .content = "", .compression = .store }, .{});
687	// no absolute paths
688	try testZipError(error.ZipBadFilename, .{ .name = "/", .content = "", .compression = .store }, .{});
689	try testZipError(error.ZipBadFilename, .{ .name = "/foo", .content = "", .compression = .store }, .{});
690	try testZipError(error.ZipBadFilename, .{ .name = "/foo/bar", .content = "", .compression = .store }, .{});
691	// no '..' components
692	try testZipError(error.ZipBadFilename, .{ .name = "..", .content = "", .compression = .store }, .{});
693	try testZipError(error.ZipBadFilename, .{ .name = "foo/..", .content = "", .compression = .store }, .{});
694	try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/..", .content = "", .compression = .store }, .{});
695	try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/../", .content = "", .compression = .store }, .{});
696	// no backslashes
697	try testZipError(error.ZipFilenameHasBackslash, .{ .name = "foo\\bar", .content = "", .compression = .store }, .{});
698	}
699
700	test "zip64" {
701	const test_files = [_]File{
702	.{ .name = "fram", .content = "fram foo fro fraba", .compression = .store },
703	.{ .name = "subdir/barro", .content = "aljdk;jal;jfd;lajkf", .compression = .store },
704	};
705
706	try testZip(.{}, &test_files, .{
707	.end = .{
708	.zip64 = .{},
709	.record_count_disk = std.math.maxInt(u16), // trigger zip64
710	},
711	});
712	try testZip(.{}, &test_files, .{
713	.end = .{
714	.zip64 = .{},
715	.record_count_total = std.math.maxInt(u16), // trigger zip64
716	},
717	});
718	try testZip(.{}, &test_files, .{
719	.end = .{
720	.zip64 = .{},
721	.record_count_disk = std.math.maxInt(u16), // trigger zip64
722	.record_count_total = std.math.maxInt(u16), // trigger zip64
723	},
724	});
725	try testZip(.{}, &test_files, .{
726	.end = .{
727	.zip64 = .{},
728	.central_directory_size = std.math.maxInt(u32), // trigger zip64
729	},
730	});
731	try testZip(.{}, &test_files, .{
732	.end = .{
733	.zip64 = .{},
734	.central_directory_offset = std.math.maxInt(u32), // trigger zip64
735	},
736	});
737	try testZip(.{}, &test_files, .{
738	.end = .{
739	.zip64 = .{},
740	.central_directory_offset = std.math.maxInt(u32), // trigger zip64
741	},
742	.local_header = .{
743	.zip64 = .{ // trigger local header zip64
744	.data_size = 16,
745	},
746	.compressed_size = std.math.maxInt(u32),
747	.uncompressed_size = std.math.maxInt(u32),
748	.extra_len = 20,
749	},
750	});
751	}
752
753	test "bad zip files" {
754	var tmp = testing.tmpDir(.{ .no_follow = true });
755	defer tmp.cleanup();
756	var zip_buf: [4096]u8 = undefined;
757
758	const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }};
759
760	{
761	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
762	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
763	}
764	{
765	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } });
766	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
767	}
768	{
769	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
770	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
771	}
772	{
773	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } });
774	try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
775	}
776	{
777	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
778	try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
779	}
780	{
781	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } });
782	try testing.expectError(error.ZipDiskRecordCountTooLarge, extract(tmp.dir, fbs.seekableStream(), .{}));
783	}
784	{
785	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } });
786	try testing.expectError(error.ZipCdOversized, extract(tmp.dir, fbs.seekableStream(), .{}));
787	}
788	{
789	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } });
790	try testing.expectError(error.ZipCdUndersized, extract(tmp.dir, fbs.seekableStream(), .{}));
791	}
792	{
793	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
794	try testing.expectError(error.ZipBadCdOffset, extract(tmp.dir, fbs.seekableStream(), .{}));
795	}
796	{
797	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{
798	.end = .{
799	.zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } },
800	.central_directory_size = std.math.maxInt(u32), // trigger 64
801	},
802	});
803	try testing.expectError(error.ZipBadLocatorSig, extract(tmp.dir, fbs.seekableStream(), .{}));
804	}
805	}

lib/std/zip/test.zig deleted-298

...	@@ -1,298 +0,0 @@
1	const std = @import("std");
2	const testing = std.testing;
3	const zip = @import("../zip.zig");
4	const maxInt = std.math.maxInt;
5
6	pub const File = struct {
7	name: []const u8,
8	content: []const u8,
9	compression: zip.CompressionMethod,
10	};
11
12	pub fn expectFiles(
13	test_files: []const File,
14	dir: std.fs.Dir,
15	opt: struct {
16	strip_prefix: ?[]const u8 = null,
17	},
18	) !void {
19	for (test_files) \|test_file\| {
20	var normalized_sub_path_buf: [std.fs.max_path_bytes]u8 = undefined;
21
22	const name = blk: {
23	if (opt.strip_prefix) \|strip_prefix\| {
24	try testing.expect(test_file.name.len >= strip_prefix.len);
25	try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]);
26	break :blk test_file.name[strip_prefix.len..];
27	}
28	break :blk test_file.name;
29	};
30	const normalized_sub_path = normalized_sub_path_buf[0..name.len];
31	@memcpy(normalized_sub_path, name);
32	std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
33	var file = try dir.openFile(normalized_sub_path, .{});
34	defer file.close();
35	var content_buf: [4096]u8 = undefined;
36	const n = try file.deprecatedReader().readAll(&content_buf);
37	try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
38	}
39	}
40
41	// Used to store any data from writing a file to the zip archive that's needed
42	// when writing the corresponding central directory record.
43	pub const FileStore = struct {
44	compression: zip.CompressionMethod,
45	file_offset: u64,
46	crc32: u32,
47	compressed_size: u32,
48	uncompressed_size: usize,
49	};
50
51	pub fn makeZip(
52	buf: []u8,
53	comptime files: []const File,
54	options: WriteZipOptions,
55	) !std.io.FixedBufferStream([]u8) {
56	var store: [files.len]FileStore = undefined;
57	return try makeZipWithStore(buf, files, options, &store);
58	}
59
60	pub fn makeZipWithStore(
61	buf: []u8,
62	files: []const File,
63	options: WriteZipOptions,
64	store: []FileStore,
65	) !std.io.FixedBufferStream([]u8) {
66	var fbs = std.io.fixedBufferStream(buf);
67	try writeZip(fbs.writer(), files, store, options);
68	return std.io.fixedBufferStream(buf[0..fbs.pos]);
69	}
70
71	pub const WriteZipOptions = struct {
72	end: ?EndRecordOptions = null,
73	local_header: ?LocalHeaderOptions = null,
74	};
75	pub const LocalHeaderOptions = struct {
76	zip64: ?LocalHeaderZip64Options = null,
77	compressed_size: ?u32 = null,
78	uncompressed_size: ?u32 = null,
79	extra_len: ?u16 = null,
80	};
81	pub const LocalHeaderZip64Options = struct {
82	data_size: ?u16 = null,
83	};
84	pub const EndRecordOptions = struct {
85	zip64: ?Zip64Options = null,
86	sig: ?[4]u8 = null,
87	disk_number: ?u16 = null,
88	central_directory_disk_number: ?u16 = null,
89	record_count_disk: ?u16 = null,
90	record_count_total: ?u16 = null,
91	central_directory_size: ?u32 = null,
92	central_directory_offset: ?u32 = null,
93	comment_len: ?u16 = null,
94	comment: ?[]const u8 = null,
95	};
96	pub const Zip64Options = struct {
97	locator_sig: ?[4]u8 = null,
98	locator_zip64_disk_count: ?u32 = null,
99	locator_record_file_offset: ?u64 = null,
100	locator_total_disk_count: ?u32 = null,
101	//record_size: ?u64 = null,
102	central_directory_size: ?u64 = null,
103	};
104
105	pub fn writeZip(
106	writer: anytype,
107	files: []const File,
108	store: []FileStore,
109	options: WriteZipOptions,
110	) !void {
111	if (store.len < files.len) return error.FileStoreTooSmall;
112	var zipper = initZipper(writer);
113	for (files, 0..) \|file, i\| {
114	store[i] = try zipper.writeFile(.{
115	.name = file.name,
116	.content = file.content,
117	.compression = file.compression,
118	.write_options = options,
119	});
120	}
121	for (files, 0..) \|file, i\| {
122	try zipper.writeCentralRecord(store[i], .{
123	.name = file.name,
124	});
125	}
126	try zipper.writeEndRecord(if (options.end) \|e\| e else .{});
127	}
128
129	pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
130	return .{ .counting_writer = std.io.countingWriter(writer) };
131	}
132
133	/// Provides methods to format and write the contents of a zip archive
134	/// to the underlying Writer.
135	pub fn Zipper(comptime Writer: type) type {
136	return struct {
137	counting_writer: std.io.CountingWriter(Writer),
138	central_count: u64 = 0,
139	first_central_offset: ?u64 = null,
140	last_central_limit: ?u64 = null,
141
142	const Self = @This();
143
144	pub fn writeFile(
145	self: *Self,
146	opt: struct {
147	name: []const u8,
148	content: []const u8,
149	compression: zip.CompressionMethod,
150	write_options: WriteZipOptions,
151	},
152	) !FileStore {
153	const writer = self.counting_writer.writer();
154
155	const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
156	const crc32 = std.hash.Crc32.hash(opt.content);
157
158	const header_options = opt.write_options.local_header;
159	{
160	var compressed_size: u32 = 0;
161	var uncompressed_size: u32 = 0;
162	var extra_len: u16 = 0;
163	if (header_options) \|hdr_options\| {
164	compressed_size = if (hdr_options.compressed_size) \|size\| size else 0;
165	uncompressed_size = if (hdr_options.uncompressed_size) \|size\| size else @intCast(opt.content.len);
166	extra_len = if (hdr_options.extra_len) \|len\| len else 0;
167	}
168	const hdr: zip.LocalFileHeader = .{
169	.signature = zip.local_file_header_sig,
170	.version_needed_to_extract = 10,
171	.flags = .{ .encrypted = false, ._ = 0 },
172	.compression_method = opt.compression,
173	.last_modification_time = 0,
174	.last_modification_date = 0,
175	.crc32 = crc32,
176	.compressed_size = compressed_size,
177	.uncompressed_size = uncompressed_size,
178	.filename_len = @intCast(opt.name.len),
179	.extra_len = extra_len,
180	};
181	try writer.writeStructEndian(hdr, .little);
182	}
183	try writer.writeAll(opt.name);
184
185	if (header_options) \|hdr\| {
186	if (hdr.zip64) \|options\| {
187	try writer.writeInt(u16, 0x0001, .little);
188	const data_size = if (options.data_size) \|size\| size else 8;
189	try writer.writeInt(u16, data_size, .little);
190	try writer.writeInt(u64, 0, .little);
191	try writer.writeInt(u64, @intCast(opt.content.len), .little);
192	}
193	}
194
195	var compressed_size: u32 = undefined;
196	switch (opt.compression) {
197	.store => {
198	try writer.writeAll(opt.content);
199	compressed_size = @intCast(opt.content.len);
200	},
201	.deflate => {
202	const offset = self.counting_writer.bytes_written;
203	var fbs = std.io.fixedBufferStream(opt.content);
204	try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205	std.debug.assert(fbs.pos == opt.content.len);
206	compressed_size = @intCast(self.counting_writer.bytes_written - offset);
207	},
208	else => unreachable,
209	}
210	return .{
211	.compression = opt.compression,
212	.file_offset = file_offset,
213	.crc32 = crc32,
214	.compressed_size = compressed_size,
215	.uncompressed_size = opt.content.len,
216	};
217	}
218
219	pub fn writeCentralRecord(
220	self: *Self,
221	store: FileStore,
222	opt: struct {
223	name: []const u8,
224	version_needed_to_extract: u16 = 10,
225	},
226	) !void {
227	if (self.first_central_offset == null) {
228	self.first_central_offset = self.counting_writer.bytes_written;
229	}
230	self.central_count += 1;
231
232	const hdr: zip.CentralDirectoryFileHeader = .{
233	.signature = zip.central_file_header_sig,
234	.version_made_by = 0,
235	.version_needed_to_extract = opt.version_needed_to_extract,
236	.flags = .{ .encrypted = false, ._ = 0 },
237	.compression_method = store.compression,
238	.last_modification_time = 0,
239	.last_modification_date = 0,
240	.crc32 = store.crc32,
241	.compressed_size = store.compressed_size,
242	.uncompressed_size = @intCast(store.uncompressed_size),
243	.filename_len = @intCast(opt.name.len),
244	.extra_len = 0,
245	.comment_len = 0,
246	.disk_number = 0,
247	.internal_file_attributes = 0,
248	.external_file_attributes = 0,
249	.local_file_header_offset = @intCast(store.file_offset),
250	};
251	try self.counting_writer.writer().writeStructEndian(hdr, .little);
252	try self.counting_writer.writer().writeAll(opt.name);
253	self.last_central_limit = self.counting_writer.bytes_written;
254	}
255
256	pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
257	const cd_offset = self.first_central_offset orelse 0;
258	const cd_end = self.last_central_limit orelse 0;
259
260	if (opt.zip64) \|zip64\| {
261	const end64_off = cd_end;
262	const fixed: zip.EndRecord64 = .{
263	.signature = zip.end_record64_sig,
264	.end_record_size = @sizeOf(zip.EndRecord64) - 12,
265	.version_made_by = 0,
266	.version_needed_to_extract = 45,
267	.disk_number = 0,
268	.central_directory_disk_number = 0,
269	.record_count_disk = @intCast(self.central_count),
270	.record_count_total = @intCast(self.central_count),
271	.central_directory_size = @intCast(cd_end - cd_offset),
272	.central_directory_offset = @intCast(cd_offset),
273	};
274	try self.counting_writer.writer().writeStructEndian(fixed, .little);
275	const locator: zip.EndLocator64 = .{
276	.signature = if (zip64.locator_sig) \|s\| s else zip.end_locator64_sig,
277	.zip64_disk_count = if (zip64.locator_zip64_disk_count) \|c\| c else 0,
278	.record_file_offset = if (zip64.locator_record_file_offset) \|o\| o else @intCast(end64_off),
279	.total_disk_count = if (zip64.locator_total_disk_count) \|c\| c else 1,
280	};
281	try self.counting_writer.writer().writeStructEndian(locator, .little);
282	}
283	const hdr: zip.EndRecord = .{
284	.signature = if (opt.sig) \|s\| s else zip.end_record_sig,
285	.disk_number = if (opt.disk_number) \|n\| n else 0,
286	.central_directory_disk_number = if (opt.central_directory_disk_number) \|n\| n else 0,
287	.record_count_disk = if (opt.record_count_disk) \|c\| c else @intCast(self.central_count),
288	.record_count_total = if (opt.record_count_total) \|c\| c else @intCast(self.central_count),
289	.central_directory_size = if (opt.central_directory_size) \|s\| s else @intCast(cd_end - cd_offset),
290	.central_directory_offset = if (opt.central_directory_offset) \|o\| o else @intCast(cd_offset),
291	.comment_len = if (opt.comment_len) \|l\| l else (if (opt.comment) \|c\| @as(u16, @intCast(c.len)) else 0),
292	};
293	try self.counting_writer.writer().writeStructEndian(hdr, .little);
294	if (opt.comment) \|c\|
295	try self.counting_writer.writer().writeAll(c);
296	}
297	};
298	}

src/Package/Fetch.zig+19-78

...	@@ -1203,12 +1203,11 @@ fn unpackResource(	...	@@ -1203,12 +1203,11 @@ fn unpackResource(
1203	return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);	1203	return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1204	},	1204	},
1205	.@"tar.gz" => {	1205	.@"tar.gz" => {
1206	const reader = resource.reader();	1206	var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
1207	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);	1207	var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1208	var dcp = std.compress.gzip.decompressor(br.reader());	1208	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1209	var adapter_buffer: [1024]u8 = undefined;	1209	var decompress: std.compress.flate.Decompress = .init(&adapter.new_interface, .gzip, &flate_buffer);
1210	var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);	1210	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1211	return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1212	},	1211	},
1213	.@"tar.xz" => {	1212	.@"tar.xz" => {
1214	const gpa = f.arena.child_allocator;	1213	const gpa = f.arena.child_allocator;
...	@@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {	...	@@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1352	));	1351	));
1353	defer zip_file.close();	1352	defer zip_file.close();
1354		1353
1355	std.zip.extract(out_dir, zip_file.seekableStream(), .{	1354	var zip_file_buffer: [1024]u8 = undefined;
		1355	var zip_file_reader = zip_file.reader(&zip_file_buffer);
		1356
		1357	std.zip.extract(out_dir, &zip_file_reader, .{
1356	.allow_backslashes = true,	1358	.allow_backslashes = true,
1357	.diagnostics = &diagnostics,	1359	.diagnostics = &diagnostics,
1358	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(	1360	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(
...	@@ -1384,23 +1386,28 @@ fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource.Git) anyerror!U	...	@@ -1384,23 +1386,28 @@ fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource.Git) anyerror!U
1384	defer pack_dir.close();	1386	defer pack_dir.close();
1385	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });	1387	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1386	defer pack_file.close();	1388	defer pack_file.close();
1387	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	1389	var pack_file_buffer: [4096]u8 = undefined;
		1390	var fifo = std.fifo.LinearFifo(u8, .{ .Slice = {} }).init(&pack_file_buffer);
1388	try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());	1391	try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());
1389		1392
		1393	var pack_file_reader = pack_file.reader(&pack_file_buffer);
		1394
1390	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });	1395	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
1391	defer index_file.close();	1396	defer index_file.close();
		1397	var index_file_buffer: [2000]u8 = undefined;
		1398	var index_file_writer = index_file.writer(&index_file_buffer);
1392	{	1399	{
1393	const index_prog_node = f.prog_node.start("Index pack", 0);	1400	const index_prog_node = f.prog_node.start("Index pack", 0);
1394	defer index_prog_node.end();	1401	defer index_prog_node.end();
1395	var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());	1402	try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
1396	try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer());
1397	try index_buffered_writer.flush();
1398	}	1403	}
1399		1404
1400	{	1405	{
		1406	var index_file_reader = index_file.reader(&index_file_buffer);
1401	const checkout_prog_node = f.prog_node.start("Checkout", 0);	1407	const checkout_prog_node = f.prog_node.start("Checkout", 0);
1402	defer checkout_prog_node.end();	1408	defer checkout_prog_node.end();
1403	var repository = try git.Repository.init(gpa, object_format, pack_file, index_file);	1409	var repository: git.Repository = undefined;
		1410	try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
1404	defer repository.deinit();	1411	defer repository.deinit();
1405	var diagnostics: git.Diagnostics = .{ .allocator = arena };	1412	var diagnostics: git.Diagnostics = .{ .allocator = arena };
1406	try repository.checkout(out_dir, resource.want_oid, &diagnostics);	1413	try repository.checkout(out_dir, resource.want_oid, &diagnostics);
...	@@ -2071,72 +2078,6 @@ const UnpackResult = struct {	...	@@ -2071,72 +2078,6 @@ const UnpackResult = struct {
2071	}	2078	}
2072	};	2079	};
2073		2080
2074	test "zip" {
2075	const gpa = std.testing.allocator;
2076	var tmp = std.testing.tmpDir(.{});
2077	defer tmp.cleanup();
2078
2079	const test_files = [_]std.zip.testutil.File{
2080	.{ .name = "foo", .content = "this is just foo\n", .compression = .store },
2081	.{ .name = "bar", .content = "another file\n", .compression = .deflate },
2082	};
2083	{
2084	var zip_file = try tmp.dir.createFile("test.zip", .{});
2085	defer zip_file.close();
2086	var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2087	var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2088	try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2089	try bw.flush();
2090	}
2091
2092	const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2093	defer gpa.free(zip_path);
2094
2095	var fb: TestFetchBuilder = undefined;
2096	var fetch = try fb.build(gpa, tmp.dir, zip_path);
2097	defer fb.deinit();
2098
2099	try fetch.run();
2100
2101	var out = try fb.packageDir();
2102	defer out.close();
2103
2104	try std.zip.testutil.expectFiles(&test_files, out, .{});
2105	}
2106
2107	test "zip with one root folder" {
2108	const gpa = std.testing.allocator;
2109	var tmp = std.testing.tmpDir(.{});
2110	defer tmp.cleanup();
2111
2112	const test_files = [_]std.zip.testutil.File{
2113	.{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store },
2114	.{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store },
2115	};
2116	{
2117	var zip_file = try tmp.dir.createFile("test.zip", .{});
2118	defer zip_file.close();
2119	var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2120	var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2121	try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2122	try bw.flush();
2123	}
2124
2125	const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2126	defer gpa.free(zip_path);
2127
2128	var fb: TestFetchBuilder = undefined;
2129	var fetch = try fb.build(gpa, tmp.dir, zip_path);
2130	defer fb.deinit();
2131
2132	try fetch.run();
2133
2134	var out = try fb.packageDir();
2135	defer out.close();
2136
2137	try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" });
2138	}
2139
2140	test "tarball with duplicate paths" {	2081	test "tarball with duplicate paths" {
2141	// This tarball has duplicate path 'dir1/file1' to simulate case sensitve	2082	// This tarball has duplicate path 'dir1/file1' to simulate case sensitve
2142	// file system on any file sytstem.	2083	// file system on any file sytstem.

src/Package/Fetch/git.zig+177-215

...	@@ -66,6 +66,33 @@ pub const Oid = union(Format) {	...	@@ -66,6 +66,33 @@ pub const Oid = union(Format) {
66	}	66	}
67	};	67	};
68		68
		69	const Hashing = union(Format) {
		70	sha1: std.Io.Writer.Hashing(Sha1),
		71	sha256: std.Io.Writer.Hashing(Sha256),
		72
		73	fn init(oid_format: Format, buffer: []u8) Hashing {
		74	return switch (oid_format) {
		75	.sha1 => .{ .sha1 = .init(buffer) },
		76	.sha256 => .{ .sha256 = .init(buffer) },
		77	};
		78	}
		79
		80	fn writer(h: @This()) std.Io.Writer {
		81	return switch (h.*) {
		82	inline else => \|*inner\| &inner.writer,
		83	};
		84	}
		85
		86	fn final(h: *@This()) Oid {
		87	switch (h.*) {
		88	inline else => \|*inner, tag\| {
		89	inner.writer.flush() catch unreachable; // hashers cannot fail
		90	return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
		91	},
		92	}
		93	}
		94	};
		95
69	pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {	96	pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
70	assert(bytes.len == oid_format.byteLength());	97	assert(bytes.len == oid_format.byteLength());
71	return switch (oid_format) {	98	return switch (oid_format) {
...	@@ -73,9 +100,9 @@ pub const Oid = union(Format) {	...	@@ -73,9 +100,9 @@ pub const Oid = union(Format) {
73	};	100	};
74	}	101	}
75		102
76	pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid {	103	pub fn readBytes(oid_format: Format, reader: *std.Io.Reader) !Oid {
77	return switch (oid_format) {	104	return switch (oid_format) {
78	inline else => \|tag\| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),	105	inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
79	};	106	};
80	}	107	}
81		108
...	@@ -166,8 +193,15 @@ pub const Diagnostics = struct {	...	@@ -166,8 +193,15 @@ pub const Diagnostics = struct {
166	pub const Repository = struct {	193	pub const Repository = struct {
167	odb: Odb,	194	odb: Odb,
168		195
169	pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository {	196	pub fn init(
170	return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };	197	repo: *Repository,
		198	allocator: Allocator,
		199	format: Oid.Format,
		200	pack_file: *std.fs.File.Reader,
		201	index_file: *std.fs.File.Reader,
		202	) !void {
		203	repo.* = .{ .odb = undefined };
		204	try repo.odb.init(allocator, format, pack_file, index_file);
171	}	205	}
172		206
173	pub fn deinit(repository: *Repository) void {	207	pub fn deinit(repository: *Repository) void {
...	@@ -335,24 +369,30 @@ pub const Repository = struct {	...	@@ -335,24 +369,30 @@ pub const Repository = struct {
335	/// [pack-format](https://git-scm.com/docs/pack-format).	369	/// [pack-format](https://git-scm.com/docs/pack-format).
336	const Odb = struct {	370	const Odb = struct {
337	format: Oid.Format,	371	format: Oid.Format,
338	pack_file: std.fs.File,	372	pack_file: *std.fs.File.Reader,
339	index_header: IndexHeader,	373	index_header: IndexHeader,
340	index_file: std.fs.File,	374	index_file: *std.fs.File.Reader,
341	cache: ObjectCache = .{},	375	cache: ObjectCache = .{},
342	allocator: Allocator,	376	allocator: Allocator,
343		377
344	/// Initializes the database from open pack and index files.	378	/// Initializes the database from open pack and index files.
345	fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb {	379	fn init(
		380	odb: *Odb,
		381	allocator: Allocator,
		382	format: Oid.Format,
		383	pack_file: *std.fs.File.Reader,
		384	index_file: *std.fs.File.Reader,
		385	) !void {
346	try pack_file.seekTo(0);	386	try pack_file.seekTo(0);
347	try index_file.seekTo(0);	387	try index_file.seekTo(0);
348	const index_header = try IndexHeader.read(index_file.deprecatedReader());	388	odb.* = .{
349	return .{
350	.format = format,	389	.format = format,
351	.pack_file = pack_file,	390	.pack_file = pack_file,
352	.index_header = index_header,	391	.index_header = undefined,
353	.index_file = index_file,	392	.index_file = index_file,
354	.allocator = allocator,	393	.allocator = allocator,
355	};	394	};
		395	try odb.index_header.read(&index_file.interface);
356	}	396	}
357		397
358	fn deinit(odb: *Odb) void {	398	fn deinit(odb: *Odb) void {
...	@@ -362,14 +402,14 @@ const Odb = struct {	...	@@ -362,14 +402,14 @@ const Odb = struct {
362		402
363	/// Reads the object at the current position in the database.	403	/// Reads the object at the current position in the database.
364	fn readObject(odb: *Odb) !Object {	404	fn readObject(odb: *Odb) !Object {
365	var base_offset = try odb.pack_file.getPos();	405	var base_offset = odb.pack_file.logicalPos();
366	var base_header: EntryHeader = undefined;	406	var base_header: EntryHeader = undefined;
367	var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;	407	var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;
368	defer delta_offsets.deinit(odb.allocator);	408	defer delta_offsets.deinit(odb.allocator);
369	const base_object = while (true) {	409	const base_object = while (true) {
370	if (odb.cache.get(base_offset)) \|base_object\| break base_object;	410	if (odb.cache.get(base_offset)) \|base_object\| break base_object;
371		411
372	base_header = try EntryHeader.read(odb.format, odb.pack_file.deprecatedReader());	412	base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
373	switch (base_header) {	413	switch (base_header) {
374	.ofs_delta => \|ofs_delta\| {	414	.ofs_delta => \|ofs_delta\| {
375	try delta_offsets.append(odb.allocator, base_offset);	415	try delta_offsets.append(odb.allocator, base_offset);
...	@@ -379,10 +419,10 @@ const Odb = struct {	...	@@ -379,10 +419,10 @@ const Odb = struct {
379	.ref_delta => \|ref_delta\| {	419	.ref_delta => \|ref_delta\| {
380	try delta_offsets.append(odb.allocator, base_offset);	420	try delta_offsets.append(odb.allocator, base_offset);
381	try odb.seekOid(ref_delta.base_object);	421	try odb.seekOid(ref_delta.base_object);
382	base_offset = try odb.pack_file.getPos();	422	base_offset = odb.pack_file.logicalPos();
383	},	423	},
384	else => {	424	else => {
385	const base_data = try readObjectRaw(odb.allocator, odb.pack_file.deprecatedReader(), base_header.uncompressedLength());	425	const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
386	errdefer odb.allocator.free(base_data);	426	errdefer odb.allocator.free(base_data);
387	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };	427	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
388	try odb.cache.put(odb.allocator, base_offset, base_object);	428	try odb.cache.put(odb.allocator, base_offset, base_object);
...	@@ -412,7 +452,7 @@ const Odb = struct {	...	@@ -412,7 +452,7 @@ const Odb = struct {
412	const found_index = while (start_index < end_index) {	452	const found_index = while (start_index < end_index) {
413	const mid_index = start_index + (end_index - start_index) / 2;	453	const mid_index = start_index + (end_index - start_index) / 2;
414	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);	454	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
415	const mid_oid = try Oid.readBytes(odb.format, odb.index_file.deprecatedReader());	455	const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
416	switch (mem.order(u8, mid_oid.slice(), oid.slice())) {	456	switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
417	.lt => start_index = mid_index + 1,	457	.lt => start_index = mid_index + 1,
418	.gt => end_index = mid_index,	458	.gt => end_index = mid_index,
...	@@ -423,12 +463,12 @@ const Odb = struct {	...	@@ -423,12 +463,12 @@ const Odb = struct {
423	const n_objects = odb.index_header.fan_out_table[255];	463	const n_objects = odb.index_header.fan_out_table[255];
424	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);	464	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
425	try odb.index_file.seekTo(offset_values_start + found_index * 4);	465	try odb.index_file.seekTo(offset_values_start + found_index * 4);
426	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.deprecatedReader().readInt(u32, .big));	466	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
427	const pack_offset = pack_offset: {	467	const pack_offset = pack_offset: {
428	if (l1_offset.big) {	468	if (l1_offset.big) {
429	const l2_offset_values_start = offset_values_start + n_objects * 4;	469	const l2_offset_values_start = offset_values_start + n_objects * 4;
430	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);	470	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
431	break :pack_offset try odb.index_file.deprecatedReader().readInt(u64, .big);	471	break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
432	} else {	472	} else {
433	break :pack_offset l1_offset.value;	473	break :pack_offset l1_offset.value;
434	}	474	}
...	@@ -1080,18 +1120,18 @@ const PackHeader = struct {	...	@@ -1080,18 +1120,18 @@ const PackHeader = struct {
1080	const signature = "PACK";	1120	const signature = "PACK";
1081	const supported_version = 2;	1121	const supported_version = 2;
1082		1122
1083	fn read(reader: anytype) !PackHeader {	1123	fn read(reader: *std.Io.Reader) !PackHeader {
1084	const actual_signature = reader.readBytesNoEof(4) catch \|e\| switch (e) {	1124	const actual_signature = reader.take(4) catch \|e\| switch (e) {
1085	error.EndOfStream => return error.InvalidHeader,	1125	error.EndOfStream => return error.InvalidHeader,
1086	else => \|other\| return other,	1126	else => \|other\| return other,
1087	};	1127	};
1088	if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;	1128	if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
1089	const version = reader.readInt(u32, .big) catch \|e\| switch (e) {	1129	const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1090	error.EndOfStream => return error.InvalidHeader,	1130	error.EndOfStream => return error.InvalidHeader,
1091	else => \|other\| return other,	1131	else => \|other\| return other,
1092	};	1132	};
1093	if (version != supported_version) return error.UnsupportedVersion;	1133	if (version != supported_version) return error.UnsupportedVersion;
1094	const total_objects = reader.readInt(u32, .big) catch \|e\| switch (e) {	1134	const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1095	error.EndOfStream => return error.InvalidHeader,	1135	error.EndOfStream => return error.InvalidHeader,
1096	else => \|other\| return other,	1136	else => \|other\| return other,
1097	};	1137	};
...	@@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) {	...	@@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) {
1143	};	1183	};
1144	}	1184	}
1145		1185
1146	fn read(format: Oid.Format, reader: anytype) !EntryHeader {	1186	fn read(format: Oid.Format, reader: *std.Io.Reader) !EntryHeader {
1147	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };	1187	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
1148	const initial: InitialByte = @bitCast(reader.readByte() catch \|e\| switch (e) {	1188	const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
1149	error.EndOfStream => return error.InvalidFormat,	1189	error.EndOfStream => return error.InvalidFormat,
1150	else => \|other\| return other,	1190	else => \|other\| return other,
1151	});	1191	});
1152	const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;	1192	const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
1153	var uncompressed_length: u64 = initial.len;	1193	var uncompressed_length: u64 = initial.len;
1154	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;	1194	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
1155	const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;	1195	const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
...	@@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) {	...	@@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) {
1172	}	1212	}
1173	};	1213	};
1174		1214
1175	fn readSizeVarInt(r: anytype) !u64 {	1215	fn readOffsetVarInt(r: *std.Io.Reader) !u64 {
1176	const Byte = packed struct { value: u7, has_next: bool };
1177	var b: Byte = @bitCast(try r.readByte());
1178	var value: u64 = b.value;
1179	var shift: u6 = 0;
1180	while (b.has_next) {
1181	b = @bitCast(try r.readByte());
1182	shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
1183	value \|= @as(u64, b.value) << shift;
1184	}
1185	return value;
1186	}
1187
1188	fn readOffsetVarInt(r: anytype) !u64 {
1189	const Byte = packed struct { value: u7, has_next: bool };	1216	const Byte = packed struct { value: u7, has_next: bool };
1190	var b: Byte = @bitCast(try r.readByte());	1217	var b: Byte = @bitCast(try r.takeByte());
1191	var value: u64 = b.value;	1218	var value: u64 = b.value;
1192	while (b.has_next) {	1219	while (b.has_next) {
1193	b = @bitCast(try r.readByte());	1220	b = @bitCast(try r.takeByte());
1194	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;	1221	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
1195	value \|= b.value;	1222	value \|= b.value;
1196	}	1223	}
...	@@ -1204,19 +1231,12 @@ const IndexHeader = struct {	...	@@ -1204,19 +1231,12 @@ const IndexHeader = struct {
1204	const supported_version = 2;	1231	const supported_version = 2;
1205	const size = 4 + 4 + @sizeOf([256]u32);	1232	const size = 4 + 4 + @sizeOf([256]u32);
1206		1233
1207	fn read(reader: anytype) !IndexHeader {	1234	fn read(index_header: IndexHeader, reader: std.Io.Reader) !void {
1208	var header_bytes = try reader.readBytesNoEof(size);	1235	const sig = try reader.take(4);
1209	if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;	1236	if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
1210	const version = mem.readInt(u32, header_bytes[4..8], .big);	1237	const version = try reader.takeInt(u32, .big);
1211	if (version != supported_version) return error.UnsupportedVersion;	1238	if (version != supported_version) return error.UnsupportedVersion;
1212		1239	try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
1213	var fan_out_table: [256]u32 = undefined;
1214	var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
1215	const fan_out_table_reader = fan_out_table_stream.reader();
1216	for (&fan_out_table) \|*entry\| {
1217	entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable;
1218	}
1219	return .{ .fan_out_table = fan_out_table };
1220	}	1240	}
1221	};	1241	};
1222		1242
...	@@ -1227,7 +1247,12 @@ const IndexEntry = struct {	...	@@ -1227,7 +1247,12 @@ const IndexEntry = struct {
1227		1247
1228	/// Writes out a version 2 index for the given packfile, as documented in	1248	/// Writes out a version 2 index for the given packfile, as documented in
1229	/// [pack-format](https://git-scm.com/docs/pack-format).	1249	/// [pack-format](https://git-scm.com/docs/pack-format).
1230	pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void {	1250	pub fn indexPack(
		1251	allocator: Allocator,
		1252	format: Oid.Format,
		1253	pack: *std.fs.File.Reader,
		1254	index_writer: *std.fs.File.Writer,
		1255	) !void {
1231	try pack.seekTo(0);	1256	try pack.seekTo(0);
1232		1257
1233	var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;	1258	var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
...	@@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in	...	@@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1280	}	1305	}
1281	@memset(fan_out_table[fan_out_index..], count);	1306	@memset(fan_out_table[fan_out_index..], count);
1282		1307
1283	var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format));	1308	var index_hashed_writer = std.Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
1284	const writer = index_hashed_writer.writer();	1309	const writer = &index_hashed_writer.writer;
1285	try writer.writeAll(IndexHeader.signature);	1310	try writer.writeAll(IndexHeader.signature);
1286	try writer.writeInt(u32, IndexHeader.supported_version, .big);	1311	try writer.writeInt(u32, IndexHeader.supported_version, .big);
1287	for (fan_out_table) \|fan_out_entry\| {	1312	for (fan_out_table) \|fan_out_entry\| {
...	@@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in	...	@@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1314		1339
1315	try writer.writeAll(pack_checksum.slice());	1340	try writer.writeAll(pack_checksum.slice());
1316	const index_checksum = index_hashed_writer.hasher.finalResult();	1341	const index_checksum = index_hashed_writer.hasher.finalResult();
1317	try index_writer.writeAll(index_checksum.slice());	1342	try index_writer.interface.writeAll(index_checksum.slice());
		1343	try index_writer.end();
1318	}	1344	}
1319		1345
1320	/// Performs the first pass over the packfile data for index construction.	1346	/// Performs the first pass over the packfile data for index construction.
...	@@ -1324,68 +1350,51 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in	...	@@ -1324,68 +1350,51 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1324	fn indexPackFirstPass(	1350	fn indexPackFirstPass(
1325	allocator: Allocator,	1351	allocator: Allocator,
1326	format: Oid.Format,	1352	format: Oid.Format,
1327	pack: std.fs.File,	1353	pack: *std.fs.File.Reader,
1328	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),	1354	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1329	pending_deltas: *std.ArrayListUnmanaged(IndexEntry),	1355	pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1330	) !Oid {	1356	) !Oid {
1331	var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());	1357	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1332	var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());	1358	var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
1333	var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));	1359	var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
1334	const pack_reader = pack_hashed_reader.reader();	1360
1335		1361	const pack_header = try PackHeader.read(&pack_hashed.reader);
1336	const pack_header = try PackHeader.read(pack_reader);	1362
1337		1363	for (0..pack_header.total_objects) \|_\| {
1338	var current_entry: u32 = 0;	1364	const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
1339	while (current_entry < pack_header.total_objects) : (current_entry += 1) {	1365	const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
1340	const entry_offset = pack_counting_reader.bytes_read;
1341	var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
1342	const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
1343	switch (entry_header) {	1366	switch (entry_header) {
1344	.commit, .tree, .blob, .tag => \|object\| {	1367	.commit, .tree, .blob, .tag => \|object\| {
1345	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());	1368	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
1346	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1369	var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
1347	var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));	1370	const oid_hasher_w = oid_hasher.writer();
1348	const entry_writer = entry_hashed_writer.writer();
1349	// The object header is not included in the pack data but is	1371	// The object header is not included in the pack data but is
1350	// part of the object's ID	1372	// part of the object's ID
1351	try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });	1373	try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
1352	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	1374	const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
1353	try fifo.pump(entry_counting_reader.reader(), entry_writer);	1375	if (n != object.uncompressed_length) return error.InvalidObject;
1354	if (entry_counting_reader.bytes_read != object.uncompressed_length) {	1376	const oid = oid_hasher.final();
1355	return error.InvalidObject;	1377	if (!skip_checksums) @compileError("TODO");
1356	}
1357	const oid = entry_hashed_writer.hasher.finalResult();
1358	try index_entries.put(allocator, oid, .{	1378	try index_entries.put(allocator, oid, .{
1359	.offset = entry_offset,	1379	.offset = entry_offset,
1360	.crc32 = entry_crc32_reader.hasher.final(),	1380	.crc32 = 0,
1361	});	1381	});
1362	},	1382	},
1363	inline .ofs_delta, .ref_delta => \|delta\| {	1383	inline .ofs_delta, .ref_delta => \|delta\| {
1364	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());	1384	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
1365	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1385	const n = try entry_decompress.reader.discardRemaining();
1366	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	1386	if (n != delta.uncompressed_length) return error.InvalidObject;
1367	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);	1387	if (!skip_checksums) @compileError("TODO");
1368	if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1369	return error.InvalidObject;
1370	}
1371	try pending_deltas.append(allocator, .{	1388	try pending_deltas.append(allocator, .{
1372	.offset = entry_offset,	1389	.offset = entry_offset,
1373	.crc32 = entry_crc32_reader.hasher.final(),	1390	.crc32 = 0,
1374	});	1391	});
1375	},	1392	},
1376	}	1393	}
1377	}	1394	}
1378		1395
1379	const pack_checksum = pack_hashed_reader.hasher.finalResult();	1396	if (!skip_checksums) @compileError("TODO");
1380	const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader());	1397	return pack_hashed.hasher.finalResult();
1381	if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {
1382	return error.CorruptedPack;
1383	}
1384	_ = pack_reader.readByte() catch \|e\| switch (e) {
1385	error.EndOfStream => return pack_checksum,
1386	else => \|other\| return other,
1387	};
1388	return error.InvalidFormat;
1389	}	1398	}
1390		1399
1391	/// Attempts to determine the final object ID of the given deltified object.	1400	/// Attempts to determine the final object ID of the given deltified object.
...	@@ -1394,7 +1403,7 @@ fn indexPackFirstPass(	...	@@ -1394,7 +1403,7 @@ fn indexPackFirstPass(
1394	fn indexPackHashDelta(	1403	fn indexPackHashDelta(
1395	allocator: Allocator,	1404	allocator: Allocator,
1396	format: Oid.Format,	1405	format: Oid.Format,
1397	pack: std.fs.File,	1406	pack: *std.fs.File.Reader,
1398	delta: IndexEntry,	1407	delta: IndexEntry,
1399	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),	1408	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1400	cache: *ObjectCache,	1409	cache: *ObjectCache,
...	@@ -1408,7 +1417,7 @@ fn indexPackHashDelta(	...	@@ -1408,7 +1417,7 @@ fn indexPackHashDelta(
1408	if (cache.get(base_offset)) \|base_object\| break base_object;	1417	if (cache.get(base_offset)) \|base_object\| break base_object;
1409		1418
1410	try pack.seekTo(base_offset);	1419	try pack.seekTo(base_offset);
1411	base_header = try EntryHeader.read(format, pack.deprecatedReader());	1420	base_header = try EntryHeader.read(format, &pack.interface);
1412	switch (base_header) {	1421	switch (base_header) {
1413	.ofs_delta => \|ofs_delta\| {	1422	.ofs_delta => \|ofs_delta\| {
1414	try delta_offsets.append(allocator, base_offset);	1423	try delta_offsets.append(allocator, base_offset);
...	@@ -1419,7 +1428,7 @@ fn indexPackHashDelta(	...	@@ -1419,7 +1428,7 @@ fn indexPackHashDelta(
1419	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;	1428	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1420	},	1429	},
1421	else => {	1430	else => {
1422	const base_data = try readObjectRaw(allocator, pack.deprecatedReader(), base_header.uncompressedLength());	1431	const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
1423	errdefer allocator.free(base_data);	1432	errdefer allocator.free(base_data);
1424	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };	1433	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1425	try cache.put(allocator, base_offset, base_object);	1434	try cache.put(allocator, base_offset, base_object);
...	@@ -1430,11 +1439,13 @@ fn indexPackHashDelta(	...	@@ -1430,11 +1439,13 @@ fn indexPackHashDelta(
1430		1439
1431	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);	1440	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1432		1441
1433	var entry_hasher: Oid.Hasher = .init(format);	1442	var entry_hasher_buffer: [64]u8 = undefined;
1434	var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);	1443	var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
1435	try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });	1444	const entry_hasher_w = entry_hasher.writer();
1436	entry_hasher.update(base_data);	1445	// Writes to hashers cannot fail.
1437	return entry_hasher.finalResult();	1446	entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
		1447	entry_hasher_w.writeAll(base_data) catch unreachable;
		1448	return entry_hasher.final();
1438	}	1449	}
1439		1450
1440	/// Resolves a chain of deltas, returning the final base object data. `pack` is	1451	/// Resolves a chain of deltas, returning the final base object data. `pack` is
...	@@ -1444,7 +1455,7 @@ fn indexPackHashDelta(	...	@@ -1444,7 +1455,7 @@ fn indexPackHashDelta(
1444	fn resolveDeltaChain(	1455	fn resolveDeltaChain(
1445	allocator: Allocator,	1456	allocator: Allocator,
1446	format: Oid.Format,	1457	format: Oid.Format,
1447	pack: std.fs.File,	1458	pack: *std.fs.File.Reader,
1448	base_object: Object,	1459	base_object: Object,
1449	delta_offsets: []const u64,	1460	delta_offsets: []const u64,
1450	cache: *ObjectCache,	1461	cache: *ObjectCache,
...	@@ -1456,21 +1467,19 @@ fn resolveDeltaChain(	...	@@ -1456,21 +1467,19 @@ fn resolveDeltaChain(
1456		1467
1457	const delta_offset = delta_offsets[i];	1468	const delta_offset = delta_offsets[i];
1458	try pack.seekTo(delta_offset);	1469	try pack.seekTo(delta_offset);
1459	const delta_header = try EntryHeader.read(format, pack.deprecatedReader());	1470	const delta_header = try EntryHeader.read(format, &pack.interface);
1460	const delta_data = try readObjectRaw(allocator, pack.deprecatedReader(), delta_header.uncompressedLength());	1471	const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
1461	defer allocator.free(delta_data);	1472	defer allocator.free(delta_data);
1462	var delta_stream = std.io.fixedBufferStream(delta_data);	1473	var delta_reader: std.Io.Reader = .fixed(delta_data);
1463	const delta_reader = delta_stream.reader();	1474	_ = try delta_reader.takeLeb128(u64); // base object size
1464	_ = try readSizeVarInt(delta_reader); // base object size	1475	const expanded_size = try delta_reader.takeLeb128(u64);
1465	const expanded_size = try readSizeVarInt(delta_reader);
1466		1476
1467	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;	1477	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1468	const expanded_data = try allocator.alloc(u8, expanded_alloc_size);	1478	const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1469	errdefer allocator.free(expanded_data);	1479	errdefer allocator.free(expanded_data);
1470	var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);	1480	var expanded_delta_stream: std.Io.Writer = .fixed(expanded_data);
1471	var base_stream = std.io.fixedBufferStream(base_data);	1481	try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
1472	try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());	1482	if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
1473	if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
1474		1483
1475	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });	1484	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1476	base_data = expanded_data;	1485	base_data = expanded_data;
...	@@ -1481,28 +1490,23 @@ fn resolveDeltaChain(	...	@@ -1481,28 +1490,23 @@ fn resolveDeltaChain(
1481	/// Reads the complete contents of an object from `reader`. This function may	1490	/// Reads the complete contents of an object from `reader`. This function may
1482	/// read more bytes than required from `reader`, so the reader position after	1491	/// read more bytes than required from `reader`, so the reader position after
1483	/// returning is not reliable.	1492	/// returning is not reliable.
1484	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {	1493	fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 {
1485	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;	1494	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1486	var buffered_reader = std.io.bufferedReader(reader);	1495	var aw: std.Io.Writer.Allocating = .init(allocator);
1487	var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());	1496	try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
1488	const data = try allocator.alloc(u8, alloc_size);	1497	defer aw.deinit();
1489	errdefer allocator.free(data);	1498	var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
1490	try decompress_stream.reader().readNoEof(data);	1499	try decompress.reader.streamExact(&aw.writer, alloc_size);
1491	_ = decompress_stream.reader().readByte() catch \|e\| switch (e) {	1500	return aw.toOwnedSlice();
1492	error.EndOfStream => return data,
1493	else => \|other\| return other,
1494	};
1495	return error.InvalidFormat;
1496	}	1501	}
1497		1502
1498	/// Expands delta data from `delta_reader` to `writer`. `base_object` must	1503	/// Expands delta data from `delta_reader` to `writer`.
1499	/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
1500	///	1504	///
1501	/// The format of the delta data is documented in	1505	/// The format of the delta data is documented in
1502	/// [pack-format](https://git-scm.com/docs/pack-format).	1506	/// [pack-format](https://git-scm.com/docs/pack-format).
1503	fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {	1507	fn expandDelta(base_object: []const u8, delta_reader: std.Io.Reader, writer: std.Io.Writer) !void {
1504	while (true) {	1508	while (true) {
1505	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch \|e\| switch (e) {	1509	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
1506	error.EndOfStream => return,	1510	error.EndOfStream => return,
1507	else => \|other\| return other,	1511	else => \|other\| return other,
1508	});	1512	});
...	@@ -1517,27 +1521,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo	...	@@ -1517,27 +1521,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo
1517	size3: bool,	1521	size3: bool,
1518	} = @bitCast(inst.value);	1522	} = @bitCast(inst.value);
1519	const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{	1523	const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1520	.offset1 = if (available.offset1) try delta_reader.readByte() else 0,	1524	.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
1521	.offset2 = if (available.offset2) try delta_reader.readByte() else 0,	1525	.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
1522	.offset3 = if (available.offset3) try delta_reader.readByte() else 0,	1526	.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
1523	.offset4 = if (available.offset4) try delta_reader.readByte() else 0,	1527	.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
1524	};	1528	};
1525	const offset: u32 = @bitCast(offset_parts);	1529	const base_offset: u32 = @bitCast(offset_parts);
1526	const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{	1530	const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1527	.size1 = if (available.size1) try delta_reader.readByte() else 0,	1531	.size1 = if (available.size1) try delta_reader.takeByte() else 0,
1528	.size2 = if (available.size2) try delta_reader.readByte() else 0,	1532	.size2 = if (available.size2) try delta_reader.takeByte() else 0,
1529	.size3 = if (available.size3) try delta_reader.readByte() else 0,	1533	.size3 = if (available.size3) try delta_reader.takeByte() else 0,
1530	};	1534	};
1531	var size: u24 = @bitCast(size_parts);	1535	var size: u24 = @bitCast(size_parts);
1532	if (size == 0) size = 0x10000;	1536	if (size == 0) size = 0x10000;
1533	try base_object.seekTo(offset);	1537	try writer.writeAll(base_object[base_offset..][0..size]);
1534	var copy_reader = std.io.limitedReader(base_object.reader(), size);
1535	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1536	try fifo.pump(copy_reader.reader(), writer);
1537	} else if (inst.value != 0) {	1538	} else if (inst.value != 0) {
1538	var data_reader = std.io.limitedReader(delta_reader, inst.value);	1539	try delta_reader.streamExact(writer, inst.value);
1539	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1540	try fifo.pump(data_reader.reader(), writer);
1541	} else {	1540	} else {
1542	return error.InvalidDeltaInstruction;	1541	return error.InvalidDeltaInstruction;
1543	}	1542	}
...	@@ -1567,23 +1566,32 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void	...	@@ -1567,23 +1566,32 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
1567	defer pack_file.close();	1566	defer pack_file.close();
1568	try pack_file.writeAll(testrepo_pack);	1567	try pack_file.writeAll(testrepo_pack);
1569		1568
		1569	var pack_file_buffer: [2000]u8 = undefined;
		1570	var pack_file_reader = pack_file.reader(&pack_file_buffer);
		1571
1570	var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });	1572	var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1571	defer index_file.close();	1573	defer index_file.close();
1572	try indexPack(testing.allocator, format, pack_file, index_file.deprecatedWriter());	1574	var index_file_buffer: [2000]u8 = undefined;
		1575	var index_file_writer = index_file.writer(&index_file_buffer);
		1576	try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
1573		1577
1574	// Arbitrary size limit on files read while checking the repository contents	1578	// Arbitrary size limit on files read while checking the repository contents
1575	// (all files in the test repo are known to be smaller than this)	1579	// (all files in the test repo are known to be smaller than this)
1576	const max_file_size = 8192;	1580	const max_file_size = 8192;
1577		1581
1578	const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);	1582	if (!skip_checksums) {
1579	defer testing.allocator.free(index_file_data);	1583	const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1580	// testrepo.idx is generated by Git. The index created by this file should	1584	defer testing.allocator.free(index_file_data);
1581	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify	1585	// testrepo.idx is generated by Git. The index created by this file should
1582	// this.	1586	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1583	const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");	1587	// this.
1584	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);	1588	const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
		1589	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
		1590	}
1585		1591
1586	var repository = try Repository.init(testing.allocator, format, pack_file, index_file);	1592	var index_file_reader = index_file.reader(&index_file_buffer);
		1593	var repository: Repository = undefined;
		1594	try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
1587	defer repository.deinit();	1595	defer repository.deinit();
1588		1596
1589	var worktree = testing.tmpDir(.{ .iterate = true });	1597	var worktree = testing.tmpDir(.{ .iterate = true });
...	@@ -1653,6 +1661,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void	...	@@ -1653,6 +1661,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
1653	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);	1661	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1654	}	1662	}
1655		1663
		1664	/// Checksum calculation is useful for troubleshooting and debugging, but it's
		1665	/// redundant since the package manager already does content hashing at the
		1666	/// end. Let's save time by not doing that work, but, I left a cookie crumb
		1667	/// trail here if you want to restore the functionality for tinkering purposes.
		1668	const skip_checksums = true;
		1669
1656	test "SHA-1 packfile indexing and checkout" {	1670	test "SHA-1 packfile indexing and checkout" {
1657	try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");	1671	try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
1658	}	1672	}
...	@@ -1676,6 +1690,9 @@ pub fn main() !void {	...	@@ -1676,6 +1690,9 @@ pub fn main() !void {
1676		1690
1677	var pack_file = try std.fs.cwd().openFile(args[2], .{});	1691	var pack_file = try std.fs.cwd().openFile(args[2], .{});
1678	defer pack_file.close();	1692	defer pack_file.close();
		1693	var pack_file_buffer: [4096]u8 = undefined;
		1694	var pack_file_reader = pack_file.reader(&pack_file_buffer);
		1695
1679	const commit = try Oid.parse(format, args[3]);	1696	const commit = try Oid.parse(format, args[3]);
1680	var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});	1697	var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});
1681	defer worktree.close();	1698	defer worktree.close();
...	@@ -1687,11 +1704,11 @@ pub fn main() !void {	...	@@ -1687,11 +1704,11 @@ pub fn main() !void {
1687	var index_file = try git_dir.createFile("idx", .{ .read = true });	1704	var index_file = try git_dir.createFile("idx", .{ .read = true });
1688	defer index_file.close();	1705	defer index_file.close();
1689	var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());	1706	var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
1690	try indexPack(allocator, format, pack_file, index_buffered_writer.writer());	1707	try indexPack(allocator, format, &pack_file_reader, index_buffered_writer.writer());
1691	try index_buffered_writer.flush();	1708	try index_buffered_writer.flush();
1692		1709
1693	std.debug.print("Starting checkout...\n", .{});	1710	std.debug.print("Starting checkout...\n", .{});
1694	var repository = try Repository.init(allocator, format, pack_file, index_file);	1711	var repository = try Repository.init(allocator, format, &pack_file_reader, index_file);
1695	defer repository.deinit();	1712	defer repository.deinit();
1696	var diagnostics: Diagnostics = .{ .allocator = allocator };	1713	var diagnostics: Diagnostics = .{ .allocator = allocator };
1697	defer diagnostics.deinit();	1714	defer diagnostics.deinit();
...	@@ -1701,58 +1718,3 @@ pub fn main() !void {	...	@@ -1701,58 +1718,3 @@ pub fn main() !void {
1701	std.debug.print("Diagnostic: {}\n", .{err});	1718	std.debug.print("Diagnostic: {}\n", .{err});
1702	}	1719	}
1703	}	1720	}
1704
1705	/// Deprecated
1706	fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
1707	return .{ .child_reader = reader, .hasher = hasher };
1708	}
1709
1710	/// Deprecated
1711	fn HashedReader(ReaderType: type, HasherType: type) type {
1712	return struct {
1713	child_reader: ReaderType,
1714	hasher: HasherType,
1715
1716	pub const Error = ReaderType.Error;
1717	pub const Reader = std.io.GenericReader(*@This(), Error, read);
1718
1719	pub fn read(self: *@This(), buf: []u8) Error!usize {
1720	const amt = try self.child_reader.read(buf);
1721	self.hasher.update(buf[0..amt]);
1722	return amt;
1723	}
1724
1725	pub fn reader(self: *@This()) Reader {
1726	return .{ .context = self };
1727	}
1728	};
1729	}
1730
1731	/// Deprecated
1732	pub fn HashedWriter(WriterType: type, HasherType: type) type {
1733	return struct {
1734	child_writer: WriterType,
1735	hasher: HasherType,
1736
1737	pub const Error = WriterType.Error;
1738	pub const Writer = std.io.GenericWriter(*@This(), Error, write);
1739
1740	pub fn write(self: *@This(), buf: []const u8) Error!usize {
1741	const amt = try self.child_writer.write(buf);
1742	self.hasher.update(buf[0..amt]);
1743	return amt;
1744	}
1745
1746	pub fn writer(self: *@This()) Writer {
1747	return .{ .context = self };
1748	}
1749	};
1750	}
1751
1752	/// Deprecated
1753	pub fn hashedWriter(
1754	writer: anytype,
1755	hasher: anytype,
1756	) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1757	return .{ .child_writer = writer, .hasher = hasher };
1758	}

src/link/Dwarf.zig+11-9

 @@ -142,8 +142,8 @@ const DebugInfo = struct {
             &abbrev_code_buf,
             debug_info.section.off(dwarf) + unit_ptr.off + unit_ptr.header_len + entry_ptr.off,
         ) != abbrev_code_buf.len) return error.InputOutput;
-        var abbrev_code_fbs = std.io.fixedBufferStream(&abbrev_code_buf);
+        var abbrev_code_reader: std.Io.Reader = .fixed(&abbrev_code_buf);
-        return @enumFromInt(std.leb.readUleb128(@typeInfo(AbbrevCode).@"enum".tag_type, abbrev_code_fbs.reader()) catch unreachable);
+        return @enumFromInt(abbrev_code_reader.takeLeb128(@typeInfo(AbbrevCode).@"enum".tag_type) catch unreachable);
     }
     const trailer_bytes = 1 + 1;
 @@ -2077,7 +2077,7 @@ pub const WipNav = struct {
                     .generic_decl_const,
                     .generic_decl_func,
                     => true,
-                    else => unreachable,
+                    else => |t| std.debug.panic("bad decl abbrev code: {t}", .{t}),
                 };
             if (parent_type.getCaptures(zcu).len == 0) {
                 if (was_generic_decl) try dwarf.freeCommonEntry(wip_nav.unit, decl_gop.value_ptr.*);
 @@ -6021,15 +6021,17 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 {
 }
 fn uleb128Bytes(value: anytype) u32 {
-    var cw = std.io.countingWriter(std.io.null_writer);
+    var trash_buffer: [64]u8 = undefined;
-    try uleb128(cw.writer(), value);
+    var d: std.Io.Writer.Discarding = .init(&trash_buffer);
-    return @intCast(cw.bytes_written);
+    d.writer.writeUleb128(value) catch unreachable;
+    return @intCast(d.count + d.writer.end);
 }
 fn sleb128Bytes(value: anytype) u32 {
-    var cw = std.io.countingWriter(std.io.null_writer);
+    var trash_buffer: [64]u8 = undefined;
-    try sleb128(cw.writer(), value);
+    var d: std.Io.Writer.Discarding = .init(&trash_buffer);
-    return @intCast(cw.bytes_written);
+    d.writer.writeSleb128(value) catch unreachable;
+    return @intCast(d.count + d.writer.end);
 }
 /// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional

src/link/Elf/Object.zig+7-8

...	@@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: Object, elf_file: Elf, atom_index: Atom.Index	...	@@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: Object, elf_file: Elf, atom_index: Atom.Index
1198	const chdr = @as(align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).;	1198	const chdr = @as(align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).;
1199	switch (chdr.ch_type) {	1199	switch (chdr.ch_type) {
1200	.ZLIB => {	1200	.ZLIB => {
1201	var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);	1201	var stream: std.Io.Reader = .fixed(data[@sizeOf(elf.Elf64_Chdr)..]);
1202	var zlib_stream = std.compress.zlib.decompressor(stream.reader());	1202	var zlib_stream: std.compress.flate.Decompress = .init(&stream, .zlib, &.{});
1203	const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;	1203	const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
1204	const decomp = try gpa.alloc(u8, size);	1204	var aw: std.Io.Writer.Allocating = .init(gpa);
1205	const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;	1205	try aw.ensureUnusedCapacity(size);
1206	if (nread != decomp.len) {	1206	defer aw.deinit();
1207	return error.InputOutput;	1207	_ = try zlib_stream.reader.streamRemaining(&aw.writer);
1208	}	1208	return aw.toOwnedSlice();
1209	return decomp;
1210	},	1209	},
1211	else => @panic("TODO unhandled compression scheme"),	1210	else => @panic("TODO unhandled compression scheme"),
1212	}	1211	}

test/incremental/change_panic_handler-1

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	#target=x86_64-linux-selfhosted
2	#target=x86_64-linux-cbe	1	#target=x86_64-linux-cbe
3	#target=x86_64-windows-cbe	2	#target=x86_64-windows-cbe
4	#update=initial version	3	#update=initial version

test/incremental/change_panic_handler_explicit-1

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	#target=x86_64-linux-selfhosted
2	#target=x86_64-linux-cbe	1	#target=x86_64-linux-cbe
3	#target=x86_64-windows-cbe	2	#target=x86_64-windows-cbe
4	#update=initial version	3	#update=initial version

test/incremental/type_becomes_comptime_only-1

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	#target=x86_64-linux-selfhosted
2	#target=x86_64-linux-cbe	1	#target=x86_64-linux-cbe
3	#target=x86_64-windows-cbe	2	#target=x86_64-windows-cbe
4	#target=wasm32-wasi-selfhosted	3	#target=wasm32-wasi-selfhosted

author	Andrew Kelley <andrew@ziglang.org> 2025-08-01 16:34:43-07:00
committer	GitHub <noreply@github.com> 2025-08-01 16:34:43-07:00
log	742956865c0c55b6650d7b89d923b63ea11cde4f
tree	b4c4b1502c23f084b357fb7224d721188505e5db
parent	dcc3e6e1dd224f1719b0ad9ef6d8d9dc0ed497ec
parent	a6f7927764fde807cb077fb4385be0729035e1db
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #24614 from ziglang/flate