Commit 742956865c0c55b6650d7b89d923b63ea11cde4f | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-08-01 16:34:43-07:00
committer	GitHub <noreply@github.com> 2025-08-01 16:34:43-07:00
log	742956865c0c55b6650d7b89d923b63ea11cde4f
tree	b4c4b1502c23f084b357fb7224d721188505e5db
parent	dcc3e6e1dd224f1719b0ad9ef6d8d9dc0ed497ec
parent	a6f7927764fde807cb077fb4385be0729035e1db
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #24614 from ziglang/flate

std.compress.flate: rework decompression and delete compression

101 files changed, 3745 insertions(+), 7229 deletions(-)

lib
- std
src
- Package
  - Fetch.zig
  - Fetch
    git.zig
- link
  - Dwarf.zig
  - Elf
    Object.zig
test
- incremental

lib/std/Io.zig-12

...	...	@@ -438,8 +438,6 @@ pub fn GenericWriter(
438	438	pub const AnyReader = @import("Io/DeprecatedReader.zig");
439	439	/// Deprecated in favor of `Writer`.
440	440	pub const AnyWriter = @import("Io/DeprecatedWriter.zig");
441		/// Deprecated in favor of `File.Reader` and `File.Writer`.
442		pub const SeekableStream = @import("Io/seekable_stream.zig").SeekableStream;
443	441	/// Deprecated in favor of `Writer`.
444	442	pub const BufferedWriter = @import("Io/buffered_writer.zig").BufferedWriter;
445	443	/// Deprecated in favor of `Writer`.
...	...	@@ -467,12 +465,6 @@ pub const CountingReader = @import("Io/counting_reader.zig").CountingReader;
467	465	/// Deprecated with no replacement; inefficient pattern
468	466	pub const countingReader = @import("Io/counting_reader.zig").countingReader;
469	467
470		pub const BitReader = @import("Io/bit_reader.zig").BitReader;
471		pub const bitReader = @import("Io/bit_reader.zig").bitReader;
472
473		pub const BitWriter = @import("Io/bit_writer.zig").BitWriter;
474		pub const bitWriter = @import("Io/bit_writer.zig").bitWriter;
475
476	468	pub const tty = @import("Io/tty.zig");
477	469
478	470	/// Deprecated in favor of `Writer.Discarding`.
...	...	@@ -948,16 +940,12 @@ pub fn PollFiles(comptime StreamEnum: type) type {
948	940
949	941	test {
950	942	_ = Reader;
951		_ = Reader.Limited;
952	943	_ = Writer;
953		_ = BitReader;
954		_ = BitWriter;
955	944	_ = BufferedReader;
956	945	_ = BufferedWriter;
957	946	_ = CountingWriter;
958	947	_ = CountingReader;
959	948	_ = FixedBufferStream;
960		_ = SeekableStream;
961	949	_ = tty;
962	950	_ = @import("Io/test.zig");
963	951	}

lib/std/Io/Reader.zig+48-44

...	...	@@ -74,6 +74,10 @@ pub const VTable = struct {
74	74	///
75	75	/// `data` may not contain an alias to `Reader.buffer`.
76	76	///
	77	/// `data` is mutable because the implementation may to temporarily modify
	78	/// the fields in order to handle partial reads. Implementations must
	79	/// restore the original value before returning.
	80	///
77	81	/// Implementations may ignore `data`, writing directly to `Reader.buffer`,
78	82	/// modifying `seek` and `end` accordingly, and returning 0 from this
79	83	/// function. Implementations are encouraged to take advantage of this if
...	...	@@ -81,7 +85,7 @@ pub const VTable = struct {
81	85	///
82	86	/// The default implementation calls `stream` with either `data[0]` or
83	87	/// `Reader.buffer`, whichever is bigger.
84		readVec: const fn (r: Reader, data: []const []u8) Error!usize = defaultReadVec,
	88	readVec: const fn (r: Reader, data: [][]u8) Error!usize = defaultReadVec,
85	89
86	90	/// Ensures `capacity` more data can be buffered without rebasing.
87	91	///
...	...	@@ -262,8 +266,7 @@ pub fn streamRemaining(r: Reader, w: Writer) StreamRemainingError!usize {
262	266	/// number of bytes discarded.
263	267	pub fn discardRemaining(r: *Reader) ShortError!usize {
264	268	var offset: usize = r.end - r.seek;
265		r.seek = 0;
266		r.end = 0;
	269	r.seek = r.end;
267	270	while (true) {
268	271	offset += r.vtable.discard(r, .unlimited) catch \|err\| switch (err) {
269	272	error.EndOfStream => return offset,
...	...	@@ -417,7 +420,7 @@ pub fn readVec(r: *Reader, data: [][]u8) Error!usize {
417	420	}
418	421
419	422	/// Writes to `Reader.buffer` or `data`, whichever has larger capacity.
420		pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize {
	423	pub fn defaultReadVec(r: *Reader, data: [][]u8) Error!usize {
421	424	assert(r.seek == r.end);
422	425	r.seek = 0;
423	426	r.end = 0;
...	...	@@ -438,23 +441,6 @@ pub fn defaultReadVec(r: *Reader, data: []const []u8) Error!usize {
438	441	return 0;
439	442	}
440	443
441		/// Always writes to `Reader.buffer` and returns 0.
442		pub fn indirectReadVec(r: *Reader, data: []const []u8) Error!usize {
443		_ = data;
444		assert(r.seek == r.end);
445		var writer: Writer = .{
446		.buffer = r.buffer,
447		.end = r.end,
448		.vtable = &.{ .drain = Writer.fixedDrain },
449		};
450		const limit: Limit = .limited(writer.buffer.len - writer.end);
451		r.end += r.vtable.stream(r, &writer, limit) catch \|err\| switch (err) {
452		error.WriteFailed => unreachable,
453		else => \|e\| return e,
454		};
455		return 0;
456		}
457
458	444	pub fn buffered(r: *Reader) []u8 {
459	445	return r.buffer[r.seek..r.end];
460	446	}
...	...	@@ -463,8 +449,8 @@ pub fn bufferedLen(r: *const Reader) usize {
463	449	return r.end - r.seek;
464	450	}
465	451
466		pub fn hashed(r: *Reader, hasher: anytype) Hashed(@TypeOf(hasher)) {
467		return .{ .in = r, .hasher = hasher };
	452	pub fn hashed(r: *Reader, hasher: anytype, buffer: []u8) Hashed(@TypeOf(hasher)) {
	453	return .init(r, hasher, buffer);
468	454	}
469	455
470	456	pub fn readVecAll(r: *Reader, data: [][]u8) Error!void {
...	...	@@ -539,8 +525,7 @@ pub fn toss(r: *Reader, n: usize) void {
539	525
540	526	/// Equivalent to `toss(r.bufferedLen())`.
541	527	pub fn tossBuffered(r: *Reader) void {
542		r.seek = 0;
543		r.end = 0;
	528	r.seek = r.end;
544	529	}
545	530
546	531	/// Equivalent to `peek` followed by `toss`.
...	...	@@ -627,8 +612,7 @@ pub fn discardShort(r: *Reader, n: usize) ShortError!usize {
627	612	return n;
628	613	}
629	614	var remaining = n - (r.end - r.seek);
630		r.end = 0;
631		r.seek = 0;
	615	r.seek = r.end;
632	616	while (true) {
633	617	const discard_len = r.vtable.discard(r, .limited(remaining)) catch \|err\| switch (err) {
634	618	error.EndOfStream => return n - remaining,
...	...	@@ -1678,7 +1662,7 @@ fn endingStream(r: Reader, w: Writer, limit: Limit) StreamError!usize {
1678	1662	return error.EndOfStream;
1679	1663	}
1680	1664
1681		fn endingReadVec(r: *Reader, data: []const []u8) Error!usize {
	1665	fn endingReadVec(r: *Reader, data: [][]u8) Error!usize {
1682	1666	_ = r;
1683	1667	_ = data;
1684	1668	return error.EndOfStream;
...	...	@@ -1709,6 +1693,15 @@ fn failingDiscard(r: *Reader, limit: Limit) Error!usize {
1709	1693	return error.ReadFailed;
1710	1694	}
1711	1695
	1696	pub fn adaptToOldInterface(r: *Reader) std.Io.AnyReader {
	1697	return .{ .context = r, .readFn = derpRead };
	1698	}
	1699
	1700	fn derpRead(context: *const anyopaque, buffer: []u8) anyerror!usize {
	1701	const r: *Reader = @constCast(@alignCast(@ptrCast(context)));
	1702	return r.readSliceShort(buffer);
	1703	}
	1704
1712	1705	test "readAlloc when the backing reader provides one byte at a time" {
1713	1706	const str = "This is a test";
1714	1707	var tiny_buffer: [1]u8 = undefined;
...	...	@@ -1772,15 +1765,16 @@ pub fn Hashed(comptime Hasher: type) type {
1772	1765	return struct {
1773	1766	in: *Reader,
1774	1767	hasher: Hasher,
1775		interface: Reader,
	1768	reader: Reader,
1776	1769
1777	1770	pub fn init(in: *Reader, hasher: Hasher, buffer: []u8) @This() {
1778	1771	return .{
1779	1772	.in = in,
1780	1773	.hasher = hasher,
1781		.interface = .{
	1774	.reader = .{
1782	1775	.vtable = &.{
1783		.read = @This().read,
	1776	.stream = @This().stream,
	1777	.readVec = @This().readVec,
1784	1778	.discard = @This().discard,
1785	1779	},
1786	1780	.buffer = buffer,
...	...	@@ -1790,33 +1784,39 @@ pub fn Hashed(comptime Hasher: type) type {
1790	1784	};
1791	1785	}
1792	1786
1793		fn read(r: Reader, w: Writer, limit: Limit) StreamError!usize {
1794		const this: *@This() = @alignCast(@fieldParentPtr("interface", r));
1795		const data = w.writableVector(limit);
	1787	fn stream(r: Reader, w: Writer, limit: Limit) StreamError!usize {
	1788	const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
	1789	const data = limit.slice(try w.writableSliceGreedy(1));
	1790	var vec: [1][]u8 = .{data};
	1791	const n = try this.in.readVec(&vec);
	1792	this.hasher.update(data[0..n]);
	1793	w.advance(n);
	1794	return n;
	1795	}
	1796
	1797	fn readVec(r: *Reader, data: [][]u8) Error!usize {
	1798	const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
1796	1799	const n = try this.in.readVec(data);
1797		const result = w.advanceVector(n);
1798	1800	var remaining: usize = n;
1799	1801	for (data) \|slice\| {
1800	1802	if (remaining < slice.len) {
1801	1803	this.hasher.update(slice[0..remaining]);
1802		return result;
	1804	return n;
1803	1805	} else {
1804	1806	remaining -= slice.len;
1805	1807	this.hasher.update(slice);
1806	1808	}
1807	1809	}
1808	1810	assert(remaining == 0);
1809		return result;
	1811	return n;
1810	1812	}
1811	1813
1812	1814	fn discard(r: *Reader, limit: Limit) Error!usize {
1813		const this: *@This() = @alignCast(@fieldParentPtr("interface", r));
1814		var w = this.hasher.writer(&.{});
1815		const n = this.in.stream(&w, limit) catch \|err\| switch (err) {
1816		error.WriteFailed => unreachable,
1817		else => \|e\| return e,
1818		};
1819		return n;
	1815	const this: *@This() = @alignCast(@fieldParentPtr("reader", r));
	1816	const peeked = limit.slice(try this.in.peekGreedy(1));
	1817	this.hasher.update(peeked);
	1818	this.in.toss(peeked.len);
	1819	return peeked.len;
1820	1820	}
1821	1821	};
1822	1822	}
...	...	@@ -1874,3 +1874,7 @@ pub fn writableVectorWsa(
1874	1874	}
1875	1875	return .{ i, n };
1876	1876	}
	1877
	1878	test {
	1879	_ = Limited;
	1880	}

lib/std/Io/Writer.zig+57-2

...	...	@@ -2266,7 +2266,7 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2266	2266	const pattern = data[data.len - 1];
2267	2267	const dest = w.buffer[w.end..];
2268	2268	switch (pattern.len) {
2269		0 => return w.end,
	2269	0 => return 0,
2270	2270	1 => {
2271	2271	assert(splat >= dest.len);
2272	2272	@memset(dest, pattern[0]);
...	...	@@ -2286,6 +2286,13 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2286	2286	}
2287	2287	}
2288	2288
	2289	pub fn unreachableDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usize {
	2290	_ = w;
	2291	_ = data;
	2292	_ = splat;
	2293	unreachable;
	2294	}
	2295
2289	2296	/// Provides a `Writer` implementation based on calling `Hasher.update`, sending
2290	2297	/// all data also to an underlying `Writer`.
2291	2298	///
...	...	@@ -2296,6 +2303,8 @@ pub fn fixedDrain(w: *Writer, data: []const []const u8, splat: usize) Error!usiz
2296	2303	/// generic. A better solution will involve creating a writer for each hash
2297	2304	/// function, where the splat buffer can be tailored to the hash implementation
2298	2305	/// details.
	2306	///
	2307	/// Contrast with `Hashing` which terminates the stream pipeline.
2299	2308	pub fn Hashed(comptime Hasher: type) type {
2300	2309	return struct {
2301	2310	out: *Writer,
...	...	@@ -2341,7 +2350,7 @@ pub fn Hashed(comptime Hasher: type) type {
2341	2350	this.hasher.update(slice);
2342	2351	}
2343	2352	const pattern = data[data.len - 1];
2344		assert(remaining == splat * pattern.len);
	2353	assert(remaining <= splat * pattern.len);
2345	2354	switch (pattern.len) {
2346	2355	0 => {
2347	2356	assert(remaining == 0);
...	...	@@ -2368,6 +2377,52 @@ pub fn Hashed(comptime Hasher: type) type {
2368	2377	};
2369	2378	}
2370	2379
	2380	/// Provides a `Writer` implementation based on calling `Hasher.update`,
	2381	/// discarding all data.
	2382	///
	2383	/// This implementation makes suboptimal buffering decisions due to being
	2384	/// generic. A better solution will involve creating a writer for each hash
	2385	/// function, where the splat buffer can be tailored to the hash implementation
	2386	/// details.
	2387	///
	2388	/// The total number of bytes written is stored in `hasher`.
	2389	///
	2390	/// Contrast with `Hashed` which also passes the data to an underlying stream.
	2391	pub fn Hashing(comptime Hasher: type) type {
	2392	return struct {
	2393	hasher: Hasher,
	2394	writer: Writer,
	2395
	2396	pub fn init(buffer: []u8) @This() {
	2397	return .initHasher(.init(.{}), buffer);
	2398	}
	2399
	2400	pub fn initHasher(hasher: Hasher, buffer: []u8) @This() {
	2401	return .{
	2402	.hasher = hasher,
	2403	.writer = .{
	2404	.buffer = buffer,
	2405	.vtable = &.{ .drain = @This().drain },
	2406	},
	2407	};
	2408	}
	2409
	2410	fn drain(w: *Writer, data: []const []const u8, splat: usize) Error!usize {
	2411	const this: *@This() = @alignCast(@fieldParentPtr("writer", w));
	2412	const hasher = &this.hasher;
	2413	hasher.update(w.buffered());
	2414	w.end = 0;
	2415	var n: usize = 0;
	2416	for (data[0 .. data.len - 1]) \|slice\| {
	2417	hasher.update(slice);
	2418	n += slice.len;
	2419	}
	2420	for (0..splat) \|_\| hasher.update(data[data.len - 1]);
	2421	return n + splat * data[data.len - 1].len;
	2422	}
	2423	};
	2424	}
	2425
2371	2426	/// Maintains `Writer` state such that it writes to the unused capacity of an
2372	2427	/// array list, filling it up completely before making a call through the
2373	2428	/// vtable, causing a resize. Consequently, the same, optimized, non-generic

lib/std/Io/bit_reader.zig deleted-238

...	...	@@ -1,238 +0,0 @@
1		const std = @import("../std.zig");
2
3		//General note on endianess:
4		//Big endian is packed starting in the most significant part of the byte and subsequent
5		// bytes contain less significant bits. Thus we always take bits from the high
6		// end and place them below existing bits in our output.
7		//Little endian is packed starting in the least significant part of the byte and
8		// subsequent bytes contain more significant bits. Thus we always take bits from
9		// the low end and place them above existing bits in our output.
10		//Regardless of endianess, within any given byte the bits are always in most
11		// to least significant order.
12		//Also regardless of endianess, the buffer always aligns bits to the low end
13		// of the byte.
14
15		/// Creates a bit reader which allows for reading bits from an underlying standard reader
16		pub fn BitReader(comptime endian: std.builtin.Endian, comptime Reader: type) type {
17		return struct {
18		reader: Reader,
19		bits: u8 = 0,
20		count: u4 = 0,
21
22		const low_bit_mask = [9]u8{
23		0b00000000,
24		0b00000001,
25		0b00000011,
26		0b00000111,
27		0b00001111,
28		0b00011111,
29		0b00111111,
30		0b01111111,
31		0b11111111,
32		};
33
34		fn Bits(comptime T: type) type {
35		return struct {
36		T,
37		u16,
38		};
39		}
40
41		fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
42		const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
43		return .{
44		@bitCast(@as(UT, @intCast(out))),
45		num,
46		};
47		}
48
49		/// Reads `bits` bits from the reader and returns a specified type
50		/// containing them in the least significant end, returning an error if the
51		/// specified number of bits could not be read.
52		pub fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
53		const b, const c = try self.readBitsTuple(T, num);
54		if (c < num) return error.EndOfStream;
55		return b;
56		}
57
58		/// Reads `bits` bits from the reader and returns a specified type
59		/// containing them in the least significant end. The number of bits successfully
60		/// read is placed in `out_bits`, as reaching the end of the stream is not an error.
61		pub fn readBits(self: @This(), comptime T: type, num: u16, out_bits: u16) !T {
62		const b, const c = try self.readBitsTuple(T, num);
63		out_bits.* = c;
64		return b;
65		}
66
67		/// Reads `bits` bits from the reader and returns a tuple of the specified type
68		/// containing them in the least significant end, and the number of bits successfully
69		/// read. Reaching the end of the stream is not an error.
70		pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
71		const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
72		const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with <u8
73
74		//dump any bits in our buffer first
75		if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
76
77		var out_count: u16 = self.count;
78		var out: U = self.removeBits(self.count);
79
80		//grab all the full bytes we need and put their
81		//bits where they belong
82		const full_bytes_left = (num - out_count) / 8;
83
84		for (0..full_bytes_left) \|_\| {
85		const byte = self.reader.readByte() catch \|err\| switch (err) {
86		error.EndOfStream => return initBits(T, out, out_count),
87		else => \|e\| return e,
88		};
89
90		switch (endian) {
91		.big => {
92		if (U == u8) out = 0 else out <<= 8; //shifting u8 by 8 is illegal in Zig
93		out \|= byte;
94		},
95		.little => {
96		const pos = @as(U, byte) << @intCast(out_count);
97		out \|= pos;
98		},
99		}
100		out_count += 8;
101		}
102
103		const bits_left = num - out_count;
104		const keep = 8 - bits_left;
105
106		if (bits_left == 0) return initBits(T, out, out_count);
107
108		const final_byte = self.reader.readByte() catch \|err\| switch (err) {
109		error.EndOfStream => return initBits(T, out, out_count),
110		else => \|e\| return e,
111		};
112
113		switch (endian) {
114		.big => {
115		out <<= @intCast(bits_left);
116		out \|= final_byte >> @intCast(keep);
117		self.bits = final_byte & low_bit_mask[keep];
118		},
119		.little => {
120		const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
121		out \|= pos;
122		self.bits = final_byte >> @intCast(bits_left);
123		},
124		}
125
126		self.count = @intCast(keep);
127		return initBits(T, out, num);
128		}
129
130		//convenience function for removing bits from
131		//the appropriate part of the buffer based on
132		//endianess.
133		fn removeBits(self: *@This(), num: u4) u8 {
134		if (num == 8) {
135		self.count = 0;
136		return self.bits;
137		}
138
139		const keep = self.count - num;
140		const bits = switch (endian) {
141		.big => self.bits >> @intCast(keep),
142		.little => self.bits & low_bit_mask[num],
143		};
144		switch (endian) {
145		.big => self.bits &= low_bit_mask[keep],
146		.little => self.bits >>= @intCast(num),
147		}
148
149		self.count = keep;
150		return bits;
151		}
152
153		pub fn alignToByte(self: *@This()) void {
154		self.bits = 0;
155		self.count = 0;
156		}
157		};
158		}
159
160		pub fn bitReader(comptime endian: std.builtin.Endian, reader: anytype) BitReader(endian, @TypeOf(reader)) {
161		return .{ .reader = reader };
162		}
163
164		///////////////////////////////
165
166		test "api coverage" {
167		const mem_be = [_]u8{ 0b11001101, 0b00001011 };
168		const mem_le = [_]u8{ 0b00011101, 0b10010101 };
169
170		var mem_in_be = std.io.fixedBufferStream(&mem_be);
171		var bit_stream_be = bitReader(.big, mem_in_be.reader());
172
173		var out_bits: u16 = undefined;
174
175		const expect = std.testing.expect;
176		const expectError = std.testing.expectError;
177
178		try expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits));
179		try expect(out_bits == 1);
180		try expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits));
181		try expect(out_bits == 2);
182		try expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits));
183		try expect(out_bits == 3);
184		try expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits));
185		try expect(out_bits == 4);
186		try expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits));
187		try expect(out_bits == 5);
188		try expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits));
189		try expect(out_bits == 1);
190
191		mem_in_be.pos = 0;
192		bit_stream_be.count = 0;
193		try expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits));
194		try expect(out_bits == 15);
195
196		mem_in_be.pos = 0;
197		bit_stream_be.count = 0;
198		try expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits));
199		try expect(out_bits == 16);
200
201		_ = try bit_stream_be.readBits(u0, 0, &out_bits);
202
203		try expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits));
204		try expect(out_bits == 0);
205		try expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1));
206
207		var mem_in_le = std.io.fixedBufferStream(&mem_le);
208		var bit_stream_le = bitReader(.little, mem_in_le.reader());
209
210		try expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits));
211		try expect(out_bits == 1);
212		try expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits));
213		try expect(out_bits == 2);
214		try expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits));
215		try expect(out_bits == 3);
216		try expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits));
217		try expect(out_bits == 4);
218		try expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits));
219		try expect(out_bits == 5);
220		try expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits));
221		try expect(out_bits == 1);
222
223		mem_in_le.pos = 0;
224		bit_stream_le.count = 0;
225		try expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits));
226		try expect(out_bits == 15);
227
228		mem_in_le.pos = 0;
229		bit_stream_le.count = 0;
230		try expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits));
231		try expect(out_bits == 16);
232
233		_ = try bit_stream_le.readBits(u0, 0, &out_bits);
234
235		try expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits));
236		try expect(out_bits == 0);
237		try expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1));
238		}

lib/std/Io/bit_writer.zig deleted-179

...	...	@@ -1,179 +0,0 @@
1		const std = @import("../std.zig");
2
3		//General note on endianess:
4		//Big endian is packed starting in the most significant part of the byte and subsequent
5		// bytes contain less significant bits. Thus we write out bits from the high end
6		// of our input first.
7		//Little endian is packed starting in the least significant part of the byte and
8		// subsequent bytes contain more significant bits. Thus we write out bits from
9		// the low end of our input first.
10		//Regardless of endianess, within any given byte the bits are always in most
11		// to least significant order.
12		//Also regardless of endianess, the buffer always aligns bits to the low end
13		// of the byte.
14
15		/// Creates a bit writer which allows for writing bits to an underlying standard writer
16		pub fn BitWriter(comptime endian: std.builtin.Endian, comptime Writer: type) type {
17		return struct {
18		writer: Writer,
19		bits: u8 = 0,
20		count: u4 = 0,
21
22		const low_bit_mask = [9]u8{
23		0b00000000,
24		0b00000001,
25		0b00000011,
26		0b00000111,
27		0b00001111,
28		0b00011111,
29		0b00111111,
30		0b01111111,
31		0b11111111,
32		};
33
34		/// Write the specified number of bits to the writer from the least significant bits of
35		/// the specified value. Bits will only be written to the writer when there
36		/// are enough to fill a byte.
37		pub fn writeBits(self: *@This(), value: anytype, num: u16) !void {
38		const T = @TypeOf(value);
39		const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
40		const U = if (@bitSizeOf(T) < 8) u8 else UT; //<u8 is a pain to work with
41
42		var in: U = @as(UT, @bitCast(value));
43		var in_count: u16 = num;
44
45		if (self.count > 0) {
46		//if we can't fill the buffer, add what we have
47		const bits_free = 8 - self.count;
48		if (num < bits_free) {
49		self.addBits(@truncate(in), @intCast(num));
50		return;
51		}
52
53		//finish filling the buffer and flush it
54		if (num == bits_free) {
55		self.addBits(@truncate(in), @intCast(num));
56		return self.flushBits();
57		}
58
59		switch (endian) {
60		.big => {
61		const bits = in >> @intCast(in_count - bits_free);
62		self.addBits(@truncate(bits), bits_free);
63		},
64		.little => {
65		self.addBits(@truncate(in), bits_free);
66		in >>= @intCast(bits_free);
67		},
68		}
69		in_count -= bits_free;
70		try self.flushBits();
71		}
72
73		//write full bytes while we can
74		const full_bytes_left = in_count / 8;
75		for (0..full_bytes_left) \|_\| {
76		switch (endian) {
77		.big => {
78		const bits = in >> @intCast(in_count - 8);
79		try self.writer.writeByte(@truncate(bits));
80		},
81		.little => {
82		try self.writer.writeByte(@truncate(in));
83		if (U == u8) in = 0 else in >>= 8;
84		},
85		}
86		in_count -= 8;
87		}
88
89		//save the remaining bits in the buffer
90		self.addBits(@truncate(in), @intCast(in_count));
91		}
92
93		//convenience funciton for adding bits to the buffer
94		//in the appropriate position based on endianess
95		fn addBits(self: *@This(), bits: u8, num: u4) void {
96		if (num == 8) self.bits = bits else switch (endian) {
97		.big => {
98		self.bits <<= @intCast(num);
99		self.bits \|= bits & low_bit_mask[num];
100		},
101		.little => {
102		const pos = bits << @intCast(self.count);
103		self.bits \|= pos;
104		},
105		}
106		self.count += num;
107		}
108
109		/// Flush any remaining bits to the writer, filling
110		/// unused bits with 0s.
111		pub fn flushBits(self: *@This()) !void {
112		if (self.count == 0) return;
113		if (endian == .big) self.bits <<= @intCast(8 - self.count);
114		try self.writer.writeByte(self.bits);
115		self.bits = 0;
116		self.count = 0;
117		}
118		};
119		}
120
121		pub fn bitWriter(comptime endian: std.builtin.Endian, writer: anytype) BitWriter(endian, @TypeOf(writer)) {
122		return .{ .writer = writer };
123		}
124
125		///////////////////////////////
126
127		test "api coverage" {
128		var mem_be = [_]u8{0} ** 2;
129		var mem_le = [_]u8{0} ** 2;
130
131		var mem_out_be = std.io.fixedBufferStream(&mem_be);
132		var bit_stream_be = bitWriter(.big, mem_out_be.writer());
133
134		const testing = std.testing;
135
136		try bit_stream_be.writeBits(@as(u2, 1), 1);
137		try bit_stream_be.writeBits(@as(u5, 2), 2);
138		try bit_stream_be.writeBits(@as(u128, 3), 3);
139		try bit_stream_be.writeBits(@as(u8, 4), 4);
140		try bit_stream_be.writeBits(@as(u9, 5), 5);
141		try bit_stream_be.writeBits(@as(u1, 1), 1);
142
143		try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011);
144
145		mem_out_be.pos = 0;
146
147		try bit_stream_be.writeBits(@as(u15, 0b110011010000101), 15);
148		try bit_stream_be.flushBits();
149		try testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010);
150
151		mem_out_be.pos = 0;
152		try bit_stream_be.writeBits(@as(u32, 0b110011010000101), 16);
153		try testing.expect(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101);
154
155		try bit_stream_be.writeBits(@as(u0, 0), 0);
156
157		var mem_out_le = std.io.fixedBufferStream(&mem_le);
158		var bit_stream_le = bitWriter(.little, mem_out_le.writer());
159
160		try bit_stream_le.writeBits(@as(u2, 1), 1);
161		try bit_stream_le.writeBits(@as(u5, 2), 2);
162		try bit_stream_le.writeBits(@as(u128, 3), 3);
163		try bit_stream_le.writeBits(@as(u8, 4), 4);
164		try bit_stream_le.writeBits(@as(u9, 5), 5);
165		try bit_stream_le.writeBits(@as(u1, 1), 1);
166
167		try testing.expect(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101);
168
169		mem_out_le.pos = 0;
170		try bit_stream_le.writeBits(@as(u15, 0b110011010000101), 15);
171		try bit_stream_le.flushBits();
172		try testing.expect(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110);
173
174		mem_out_le.pos = 0;
175		try bit_stream_le.writeBits(@as(u32, 0b1100110100001011), 16);
176		try testing.expect(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101);
177
178		try bit_stream_le.writeBits(@as(u0, 0), 0);
179		}

lib/std/Io/fixed_buffer_stream.zig+1-16

...	...	@@ -4,8 +4,7 @@ const testing = std.testing;
4	4	const mem = std.mem;
5	5	const assert = std.debug.assert;
6	6
7		/// This turns a byte buffer into an `io.GenericWriter`, `io.GenericReader`, or `io.SeekableStream`.
8		/// If the supplied byte buffer is const, then `io.GenericWriter` is not available.
	7	/// Deprecated in favor of `std.Io.Reader.fixed` and `std.Io.Writer.fixed`.
9	8	pub fn FixedBufferStream(comptime Buffer: type) type {
10	9	return struct {
11	10	/// `Buffer` is either a `[]u8` or `[]const u8`.
...	...	@@ -20,16 +19,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type {
20	19	pub const Reader = io.GenericReader(*Self, ReadError, read);
21	20	pub const Writer = io.GenericWriter(*Self, WriteError, write);
22	21
23		pub const SeekableStream = io.SeekableStream(
24		*Self,
25		SeekError,
26		GetSeekPosError,
27		seekTo,
28		seekBy,
29		getPos,
30		getEndPos,
31		);
32
33	22	const Self = @This();
34	23
35	24	pub fn reader(self: *Self) Reader {
...	...	@@ -40,10 +29,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type {
40	29	return .{ .context = self };
41	30	}
42	31
43		pub fn seekableStream(self: *Self) SeekableStream {
44		return .{ .context = self };
45		}
46
47	32	pub fn read(self: *Self, dest: []u8) ReadError!usize {
48	33	const size = @min(dest.len, self.buffer.len - self.pos);
49	34	const end = self.pos + size;

lib/std/Io/seekable_stream.zig deleted-35

...	...	@@ -1,35 +0,0 @@
1		const std = @import("../std.zig");
2
3		pub fn SeekableStream(
4		comptime Context: type,
5		comptime SeekErrorType: type,
6		comptime GetSeekPosErrorType: type,
7		comptime seekToFn: fn (context: Context, pos: u64) SeekErrorType!void,
8		comptime seekByFn: fn (context: Context, pos: i64) SeekErrorType!void,
9		comptime getPosFn: fn (context: Context) GetSeekPosErrorType!u64,
10		comptime getEndPosFn: fn (context: Context) GetSeekPosErrorType!u64,
11		) type {
12		return struct {
13		context: Context,
14
15		const Self = @This();
16		pub const SeekError = SeekErrorType;
17		pub const GetSeekPosError = GetSeekPosErrorType;
18
19		pub fn seekTo(self: Self, pos: u64) SeekError!void {
20		return seekToFn(self.context, pos);
21		}
22
23		pub fn seekBy(self: Self, amt: i64) SeekError!void {
24		return seekByFn(self.context, amt);
25		}
26
27		pub fn getEndPos(self: Self) GetSeekPosError!u64 {
28		return getEndPosFn(self.context);
29		}
30
31		pub fn getPos(self: Self) GetSeekPosError!u64 {
32		return getPosFn(self.context);
33		}
34		};
35		}

lib/std/Io/test.zig-45

...	...	@@ -57,51 +57,6 @@ test "write a file, read it, then delete it" {
57	57	try tmp.dir.deleteFile(tmp_file_name);
58	58	}
59	59
60		test "BitStreams with File Stream" {
61		var tmp = tmpDir(.{});
62		defer tmp.cleanup();
63
64		const tmp_file_name = "temp_test_file.txt";
65		{
66		var file = try tmp.dir.createFile(tmp_file_name, .{});
67		defer file.close();
68
69		var bit_stream = io.bitWriter(native_endian, file.deprecatedWriter());
70
71		try bit_stream.writeBits(@as(u2, 1), 1);
72		try bit_stream.writeBits(@as(u5, 2), 2);
73		try bit_stream.writeBits(@as(u128, 3), 3);
74		try bit_stream.writeBits(@as(u8, 4), 4);
75		try bit_stream.writeBits(@as(u9, 5), 5);
76		try bit_stream.writeBits(@as(u1, 1), 1);
77		try bit_stream.flushBits();
78		}
79		{
80		var file = try tmp.dir.openFile(tmp_file_name, .{});
81		defer file.close();
82
83		var bit_stream = io.bitReader(native_endian, file.deprecatedReader());
84
85		var out_bits: u16 = undefined;
86
87		try expect(1 == try bit_stream.readBits(u2, 1, &out_bits));
88		try expect(out_bits == 1);
89		try expect(2 == try bit_stream.readBits(u5, 2, &out_bits));
90		try expect(out_bits == 2);
91		try expect(3 == try bit_stream.readBits(u128, 3, &out_bits));
92		try expect(out_bits == 3);
93		try expect(4 == try bit_stream.readBits(u8, 4, &out_bits));
94		try expect(out_bits == 4);
95		try expect(5 == try bit_stream.readBits(u9, 5, &out_bits));
96		try expect(out_bits == 5);
97		try expect(1 == try bit_stream.readBits(u1, 1, &out_bits));
98		try expect(out_bits == 1);
99
100		try expectError(error.EndOfStream, bit_stream.readBitsNoEof(u1, 1));
101		}
102		try tmp.dir.deleteFile(tmp_file_name);
103		}
104
105	60	test "File seek ops" {
106	61	var tmp = tmpDir(.{});
107	62	defer tmp.cleanup();

lib/std/compress.zig+1-4

...	...	@@ -1,8 +1,7 @@
1	1	//! Compression algorithms.
2	2
	3	/// gzip and zlib are here.
3	4	pub const flate = @import("compress/flate.zig");
4		pub const gzip = @import("compress/gzip.zig");
5		pub const zlib = @import("compress/zlib.zig");
6	5	pub const lzma = @import("compress/lzma.zig");
7	6	pub const lzma2 = @import("compress/lzma2.zig");
8	7	pub const xz = @import("compress/xz.zig");
...	...	@@ -14,6 +13,4 @@ test {
14	13	_ = lzma2;
15	14	_ = xz;
16	15	_ = zstd;
17		_ = gzip;
18		_ = zlib;
19	16	}

lib/std/compress/flate.zig+151-448

...	...	@@ -1,477 +1,180 @@
1		/// Deflate is a lossless data compression file format that uses a combination
2		/// of LZ77 and Huffman coding.
3		pub const deflate = @import("flate/deflate.zig");
4
5		/// Inflate is the decoding process that takes a Deflate bitstream for
6		/// decompression and correctly produces the original full-size data or file.
7		pub const inflate = @import("flate/inflate.zig");
8
9		/// Decompress compressed data from reader and write plain data to the writer.
10		pub fn decompress(reader: anytype, writer: anytype) !void {
11		try inflate.decompress(.raw, reader, writer);
12		}
13
14		/// Decompressor type
15		pub fn Decompressor(comptime ReaderType: type) type {
16		return inflate.Decompressor(.raw, ReaderType);
17		}
18
19		/// Create Decompressor which will read compressed data from reader.
20		pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21		return inflate.decompressor(.raw, reader);
22		}
23
24		/// Compression level, trades between speed and compression size.
25		pub const Options = deflate.Options;
	1	const std = @import("../std.zig");
26	2
27		/// Compress plain data from reader and write compressed data to the writer.
28		pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
29		try deflate.compress(.raw, reader, writer, options);
30		}
	3	/// When decompressing, the output buffer is used as the history window, so
	4	/// less than this may result in failure to decompress streams that were
	5	/// compressed with a larger window.
	6	pub const max_window_len = history_len * 2;
31	7
32		/// Compressor type
33		pub fn Compressor(comptime WriterType: type) type {
34		return deflate.Compressor(.raw, WriterType);
35		}
	8	pub const history_len = 32768;
36	9
37		/// Create Compressor which outputs compressed data to the writer.
38		pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
39		return try deflate.compressor(.raw, writer, options);
40		}
41
42		/// Huffman only compression. Without Lempel-Ziv match searching. Faster
43		/// compression, less memory requirements but bigger compressed sizes.
44		pub const huffman = struct {
45		pub fn compress(reader: anytype, writer: anytype) !void {
46		try deflate.huffman.compress(.raw, reader, writer);
	10	/// Deflate is a lossless data compression file format that uses a combination
	11	/// of LZ77 and Huffman coding.
	12	pub const Compress = @import("flate/Compress.zig");
	13
	14	/// Inflate is the decoding process that consumes a Deflate bitstream and
	15	/// produces the original full-size data.
	16	pub const Decompress = @import("flate/Decompress.zig");
	17
	18	/// Compression without Lempel-Ziv match searching. Faster compression, less
	19	/// memory requirements but bigger compressed sizes.
	20	pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
	21
	22	/// Container of the deflate bit stream body. Container adds header before
	23	/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
	24	/// no footer, raw bit stream).
	25	///
	26	/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
	27	/// addler 32 checksum.
	28	///
	29	/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
	30	/// crc32 checksum and 4 bytes of uncompressed data length.
	31	///
	32	/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
	33	/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
	34	pub const Container = enum {
	35	raw, // no header or footer
	36	gzip, // gzip header and footer
	37	zlib, // zlib header and footer
	38
	39	pub fn size(w: Container) usize {
	40	return headerSize(w) + footerSize(w);
47	41	}
48	42
49		pub fn Compressor(comptime WriterType: type) type {
50		return deflate.huffman.Compressor(.raw, WriterType);
	43	pub fn headerSize(w: Container) usize {
	44	return header(w).len;
51	45	}
52	46
53		pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
54		return deflate.huffman.compressor(.raw, writer);
	47	pub fn footerSize(w: Container) usize {
	48	return switch (w) {
	49	.gzip => 8,
	50	.zlib => 4,
	51	.raw => 0,
	52	};
55	53	}
56		};
57	54
58		// No compression store only. Compressed size is slightly bigger than plain.
59		pub const store = struct {
60		pub fn compress(reader: anytype, writer: anytype) !void {
61		try deflate.store.compress(.raw, reader, writer);
62		}
	55	pub const list = [_]Container{ .raw, .gzip, .zlib };
63	56
64		pub fn Compressor(comptime WriterType: type) type {
65		return deflate.store.Compressor(.raw, WriterType);
66		}
	57	pub const Error = error{
	58	BadGzipHeader,
	59	BadZlibHeader,
	60	WrongGzipChecksum,
	61	WrongGzipSize,
	62	WrongZlibChecksum,
	63	};
67	64
68		pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
69		return deflate.store.compressor(.raw, writer);
	65	pub fn header(container: Container) []const u8 {
	66	return switch (container) {
	67	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
	68	// - ID1 (IDentification 1), always 0x1f
	69	// - ID2 (IDentification 2), always 0x8b
	70	// - CM (Compression Method), always 8 = deflate
	71	// - FLG (Flags), all set to 0
	72	// - 4 bytes, MTIME (Modification time), not used, all set to zero
	73	// - XFL (eXtra FLags), all set to zero
	74	// - OS (Operating System), 03 = Unix
	75	.gzip => &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 },
	76	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
	77	// 1st byte:
	78	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
	79	// - The next four bits is the CM (compression method), which is 8 for deflate.
	80	// 2nd byte:
	81	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
	82	// - The next bit, FDICT, is set if a dictionary is given.
	83	// - The final five FCHECK bits form a mod-31 checksum.
	84	//
	85	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
	86	.zlib => &[_]u8{ 0x78, 0b10_0_11100 },
	87	.raw => &.{},
	88	};
70	89	}
71		};
72
73		/// Container defines header/footer around deflate bit stream. Gzip and zlib
74		/// compression algorithms are containers around deflate bit stream body.
75		const Container = @import("flate/container.zig").Container;
76		const std = @import("std");
77		const testing = std.testing;
78		const fixedBufferStream = std.io.fixedBufferStream;
79		const print = std.debug.print;
80		const builtin = @import("builtin");
81
82		test {
83		_ = deflate;
84		_ = inflate;
85		}
86	90
87		test "compress/decompress" {
88		var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
89		var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
90
91		const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
92		const cases = [_]struct {
93		data: []const u8, // uncompressed content
94		// compressed data sizes per level 4-9
95		gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
96		huffman_only_size: usize = 0,
97		store_size: usize = 0,
98		}{
99		.{
100		.data = @embedFile("flate/testdata/rfc1951.txt"),
101		.gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
102		.huffman_only_size = 20287,
103		.store_size = 36967,
104		},
105		.{
106		.data = @embedFile("flate/testdata/fuzz/roundtrip1.input"),
107		.gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
108		.huffman_only_size = 393,
109		.store_size = 393,
110		},
111		.{
112		.data = @embedFile("flate/testdata/fuzz/roundtrip2.input"),
113		.gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
114		.huffman_only_size = 394,
115		.store_size = 394,
	91	pub const Hasher = union(Container) {
	92	raw: void,
	93	gzip: struct {
	94	crc: std.hash.Crc32 = .init(),
	95	count: u32 = 0,
116	96	},
117		.{
118		.data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"),
119		.gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },
120		.huffman_only_size = 498,
121		.store_size = 747,
122		},
123		};
124
125		for (cases, 0..) \|case, case_no\| { // for each case
126		const data = case.data;
127
128		for (levels, 0..) \|level, i\| { // for each compression level
129
130		inline for (Container.list) \|container\| { // for each wrapping
131		var compressed_size: usize = if (case.gzip_sizes[i] > 0)
132		case.gzip_sizes[i] - Container.gzip.size() + container.size()
133		else
134		0;
135
136		// compress original stream to compressed stream
137		{
138		var original = fixedBufferStream(data);
139		var compressed = fixedBufferStream(&cmp_buf);
140		try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });
141		if (compressed_size == 0) {
142		if (container == .gzip)
143		print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
144		compressed_size = compressed.pos;
145		}
146		try testing.expectEqual(compressed_size, compressed.pos);
147		}
148		// decompress compressed stream to decompressed stream
149		{
150		var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
151		var decompressed = fixedBufferStream(&dcm_buf);
152		try inflate.decompress(container, compressed.reader(), decompressed.writer());
153		try testing.expectEqualSlices(u8, data, decompressed.getWritten());
154		}
155
156		// compressor writer interface
157		{
158		var compressed = fixedBufferStream(&cmp_buf);
159		var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });
160		var cmp_wrt = cmp.writer();
161		try cmp_wrt.writeAll(data);
162		try cmp.finish();
163
164		try testing.expectEqual(compressed_size, compressed.pos);
165		}
166		// decompressor reader interface
167		{
168		var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
169		var dcm = inflate.decompressor(container, compressed.reader());
170		var dcm_rdr = dcm.reader();
171		const n = try dcm_rdr.readAll(&dcm_buf);
172		try testing.expectEqual(data.len, n);
173		try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
174		}
175		}
	97	zlib: std.hash.Adler32,
	98
	99	pub fn init(containter: Container) Hasher {
	100	return switch (containter) {
	101	.gzip => .{ .gzip = .{} },
	102	.zlib => .{ .zlib = .{} },
	103	.raw => .raw,
	104	};
176	105	}
177		// huffman only compression
178		{
179		inline for (Container.list) \|container\| { // for each wrapping
180		var compressed_size: usize = if (case.huffman_only_size > 0)
181		case.huffman_only_size - Container.gzip.size() + container.size()
182		else
183		0;
184	106
185		// compress original stream to compressed stream
186		{
187		var original = fixedBufferStream(data);
188		var compressed = fixedBufferStream(&cmp_buf);
189		var cmp = try deflate.huffman.compressor(container, compressed.writer());
190		try cmp.compress(original.reader());
191		try cmp.finish();
192		if (compressed_size == 0) {
193		if (container == .gzip)
194		print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
195		compressed_size = compressed.pos;
196		}
197		try testing.expectEqual(compressed_size, compressed.pos);
198		}
199		// decompress compressed stream to decompressed stream
200		{
201		var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
202		var decompressed = fixedBufferStream(&dcm_buf);
203		try inflate.decompress(container, compressed.reader(), decompressed.writer());
204		try testing.expectEqualSlices(u8, data, decompressed.getWritten());
205		}
206		}
	107	pub fn container(h: Hasher) Container {
	108	return h;
207	109	}
208	110
209		// store only
210		{
211		inline for (Container.list) \|container\| { // for each wrapping
212		var compressed_size: usize = if (case.store_size > 0)
213		case.store_size - Container.gzip.size() + container.size()
214		else
215		0;
216
217		// compress original stream to compressed stream
218		{
219		var original = fixedBufferStream(data);
220		var compressed = fixedBufferStream(&cmp_buf);
221		var cmp = try deflate.store.compressor(container, compressed.writer());
222		try cmp.compress(original.reader());
223		try cmp.finish();
224		if (compressed_size == 0) {
225		if (container == .gzip)
226		print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
227		compressed_size = compressed.pos;
228		}
229
230		try testing.expectEqual(compressed_size, compressed.pos);
231		}
232		// decompress compressed stream to decompressed stream
233		{
234		var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
235		var decompressed = fixedBufferStream(&dcm_buf);
236		try inflate.decompress(container, compressed.reader(), decompressed.writer());
237		try testing.expectEqualSlices(u8, data, decompressed.getWritten());
238		}
	111	pub fn update(h: *Hasher, buf: []const u8) void {
	112	switch (h.*) {
	113	.raw => {},
	114	.gzip => \|*gzip\| {
	115	gzip.update(buf);
	116	gzip.count +%= buf.len;
	117	},
	118	.zlib => \|*zlib\| {
	119	zlib.update(buf);
	120	},
	121	inline .gzip, .zlib => \|*x\| x.update(buf),
239	122	}
240	123	}
241		}
242		}
243
244		fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void {
245		var in = fixedBufferStream(compressed);
246		var out = std.ArrayList(u8).init(testing.allocator);
247		defer out.deinit();
248
249		try inflate.decompress(container, in.reader(), out.writer());
250		try testing.expectEqualSlices(u8, expected_plain, out.items);
251		}
252
253		test "don't read past deflate stream's end" {
254		try testDecompress(.zlib, &[_]u8{
255		0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
256		0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
257		0x83, 0x95, 0x0b, 0xf5,
258		}, &[_]u8{
259		0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
260		0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
261		0x00, 0x00, 0xff, 0xff, 0xff,
262		});
263		}
264
265		test "zlib header" {
266		// Truncated header
267		try testing.expectError(
268		error.EndOfStream,
269		testDecompress(.zlib, &[_]u8{0x78}, ""),
270		);
271		// Wrong CM
272		try testing.expectError(
273		error.BadZlibHeader,
274		testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
275		);
276		// Wrong CINFO
277		try testing.expectError(
278		error.BadZlibHeader,
279		testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
280		);
281		// Wrong checksum
282		try testing.expectError(
283		error.WrongZlibChecksum,
284		testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
285		);
286		// Truncated checksum
287		try testing.expectError(
288		error.EndOfStream,
289		testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
290		);
291		}
292
293		test "gzip header" {
294		// Truncated header
295		try testing.expectError(
296		error.EndOfStream,
297		testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
298		);
299		// Wrong CM
300		try testing.expectError(
301		error.BadGzipHeader,
302		testDecompress(.gzip, &[_]u8{
303		0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
304		0x00, 0x03,
305		}, undefined),
306		);
307
308		// Wrong checksum
309		try testing.expectError(
310		error.WrongGzipChecksum,
311		testDecompress(.gzip, &[_]u8{
312		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
313		0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
314		0x00, 0x00, 0x00, 0x00,
315		}, undefined),
316		);
317		// Truncated checksum
318		try testing.expectError(
319		error.EndOfStream,
320		testDecompress(.gzip, &[_]u8{
321		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
322		0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
323		}, undefined),
324		);
325		// Wrong initial size
326		try testing.expectError(
327		error.WrongGzipSize,
328		testDecompress(.gzip, &[_]u8{
329		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
330		0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
331		0x00, 0x00, 0x00, 0x01,
332		}, undefined),
333		);
334		// Truncated initial size field
335		try testing.expectError(
336		error.EndOfStream,
337		testDecompress(.gzip, &[_]u8{
338		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
339		0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
340		0x00, 0x00, 0x00,
341		}, undefined),
342		);
343
344		try testDecompress(.gzip, &[_]u8{
345		// GZIP header
346		0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
347		// header.FHCRC (should cover entire header)
348		0x99, 0xd6,
349		// GZIP data
350		0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
351		}, "");
352		}
353
354		test "public interface" {
355		const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
356
357		// deflate final stored block, header + plain (stored) data
358		const deflate_block = [_]u8{
359		0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
360		} ++ plain_data;
361
362		// gzip header/footer + deflate block
363		const gzip_data =
364		[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
365		deflate_block ++
366		[_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
367
368		// zlib header/footer + deflate block
369		const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
370		deflate_block ++
371		[_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
372
373		const gzip = @import("gzip.zig");
374		const zlib = @import("zlib.zig");
375		const flate = @This();
376
377		try testInterface(gzip, &gzip_data, &plain_data);
378		try testInterface(zlib, &zlib_data, &plain_data);
379		try testInterface(flate, &deflate_block, &plain_data);
380		}
381	124
382		fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void {
383		var buffer1: [64]u8 = undefined;
384		var buffer2: [64]u8 = undefined;
385
386		var compressed = fixedBufferStream(&buffer1);
387		var plain = fixedBufferStream(&buffer2);
388
389		// decompress
390		{
391		var in = fixedBufferStream(gzip_data);
392		try pkg.decompress(in.reader(), plain.writer());
393		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
394		}
395		plain.reset();
396		compressed.reset();
397
398		// compress/decompress
399		{
400		var in = fixedBufferStream(plain_data);
401		try pkg.compress(in.reader(), compressed.writer(), .{});
402		compressed.reset();
403		try pkg.decompress(compressed.reader(), plain.writer());
404		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
405		}
406		plain.reset();
407		compressed.reset();
408
409		// compressor/decompressor
410		{
411		var in = fixedBufferStream(plain_data);
412		var cmp = try pkg.compressor(compressed.writer(), .{});
413		try cmp.compress(in.reader());
414		try cmp.finish();
415
416		compressed.reset();
417		var dcp = pkg.decompressor(compressed.reader());
418		try dcp.decompress(plain.writer());
419		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
420		}
421		plain.reset();
422		compressed.reset();
423
424		// huffman
425		{
426		// huffman compress/decompress
427		{
428		var in = fixedBufferStream(plain_data);
429		try pkg.huffman.compress(in.reader(), compressed.writer());
430		compressed.reset();
431		try pkg.decompress(compressed.reader(), plain.writer());
432		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
	125	pub fn writeFooter(hasher: Hasher, writer: std.Io.Writer) std.Io.Writer.Error!void {
	126	var bits: [4]u8 = undefined;
	127	switch (hasher.*) {
	128	.gzip => \|*gzip\| {
	129	// GZIP 8 bytes footer
	130	// - 4 bytes, CRC32 (CRC-32)
	131	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
	132	std.mem.writeInt(u32, &bits, gzip.final(), .little);
	133	try writer.writeAll(&bits);
	134
	135	std.mem.writeInt(u32, &bits, gzip.bytes_read, .little);
	136	try writer.writeAll(&bits);
	137	},
	138	.zlib => \|*zlib\| {
	139	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
	140	// 4 bytes of ADLER32 (Adler-32 checksum)
	141	// Checksum value of the uncompressed data (excluding any
	142	// dictionary data) computed according to Adler-32
	143	// algorithm.
	144	std.mem.writeInt(u32, &bits, zlib.final, .big);
	145	try writer.writeAll(&bits);
	146	},
	147	.raw => {},
	148	}
433	149	}
434		plain.reset();
435		compressed.reset();
	150	};
436	151
437		// huffman compressor/decompressor
438		{
439		var in = fixedBufferStream(plain_data);
440		var cmp = try pkg.huffman.compressor(compressed.writer());
441		try cmp.compress(in.reader());
442		try cmp.finish();
	152	pub const Metadata = union(Container) {
	153	raw: void,
	154	gzip: struct {
	155	crc: u32 = 0,
	156	count: u32 = 0,
	157	},
	158	zlib: struct {
	159	adler: u32 = 0,
	160	},
443	161
444		compressed.reset();
445		try pkg.decompress(compressed.reader(), plain.writer());
446		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
	162	pub fn init(containter: Container) Metadata {
	163	return switch (containter) {
	164	.gzip => .{ .gzip = .{} },
	165	.zlib => .{ .zlib = .{} },
	166	.raw => .raw,
	167	};
447	168	}
448		}
449		plain.reset();
450		compressed.reset();
451	169
452		// store
453		{
454		// store compress/decompress
455		{
456		var in = fixedBufferStream(plain_data);
457		try pkg.store.compress(in.reader(), compressed.writer());
458		compressed.reset();
459		try pkg.decompress(compressed.reader(), plain.writer());
460		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
	170	pub fn container(m: Metadata) Container {
	171	return m;
461	172	}
462		plain.reset();
463		compressed.reset();
464
465		// store compressor/decompressor
466		{
467		var in = fixedBufferStream(plain_data);
468		var cmp = try pkg.store.compressor(compressed.writer());
469		try cmp.compress(in.reader());
470		try cmp.finish();
	173	};
	174	};
471	175
472		compressed.reset();
473		try pkg.decompress(compressed.reader(), plain.writer());
474		try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
475		}
476		}
	176	test {
	177	_ = HuffmanEncoder;
	178	_ = Compress;
	179	_ = Decompress;
477	180	}

lib/std/compress/flate/BlockWriter.zig created+592

...	...	@@ -0,0 +1,592 @@
	1	//! Accepts list of tokens, decides what is best block type to write. What block
	2	//! type will provide best compression. Writes header and body of the block.
	3	const std = @import("std");
	4	const io = std.io;
	5	const assert = std.debug.assert;
	6	const Writer = std.io.Writer;
	7
	8	const BlockWriter = @This();
	9	const flate = @import("../flate.zig");
	10	const Compress = flate.Compress;
	11	const HuffmanEncoder = flate.HuffmanEncoder;
	12	const Token = @import("Token.zig");
	13
	14	const codegen_order = HuffmanEncoder.codegen_order;
	15	const end_code_mark = 255;
	16
	17	output: *Writer,
	18
	19	codegen_freq: [HuffmanEncoder.codegen_code_count]u16,
	20	literal_freq: [HuffmanEncoder.max_num_lit]u16,
	21	distance_freq: [HuffmanEncoder.distance_code_count]u16,
	22	codegen: [HuffmanEncoder.max_num_lit + HuffmanEncoder.distance_code_count + 1]u8,
	23	literal_encoding: HuffmanEncoder,
	24	distance_encoding: HuffmanEncoder,
	25	codegen_encoding: HuffmanEncoder,
	26	fixed_literal_encoding: HuffmanEncoder,
	27	fixed_distance_encoding: HuffmanEncoder,
	28	huff_distance: HuffmanEncoder,
	29
	30	fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,
	31	fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
	32	distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
	33
	34	pub fn init(output: *Writer) BlockWriter {
	35	return .{
	36	.output = output,
	37	.codegen_freq = undefined,
	38	.literal_freq = undefined,
	39	.distance_freq = undefined,
	40	.codegen = undefined,
	41	.literal_encoding = undefined,
	42	.distance_encoding = undefined,
	43	.codegen_encoding = undefined,
	44	.fixed_literal_encoding = undefined,
	45	.fixed_distance_encoding = undefined,
	46	.huff_distance = undefined,
	47	.fixed_literal_codes = undefined,
	48	.fixed_distance_codes = undefined,
	49	.distance_codes = undefined,
	50	};
	51	}
	52
	53	pub fn initBuffers(bw: *BlockWriter) void {
	54	bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);
	55	bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);
	56	bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);
	57	}
	58
	59	/// Flush intrenal bit buffer to the writer.
	60	/// Should be called only when bit stream is at byte boundary.
	61	///
	62	/// That is after final block; when last byte could be incomplete or
	63	/// after stored block; which is aligned to the byte boundary (it has x
	64	/// padding bits after first 3 bits).
	65	pub fn flush(self: *BlockWriter) Writer.Error!void {
	66	try self.bit_writer.flush();
	67	}
	68
	69	fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void {
	70	try self.bit_writer.writeBits(c.code, c.len);
	71	}
	72
	73	/// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
	74	/// the literal and distance lengths arrays (which are concatenated into a single
	75	/// array). This method generates that run-length encoding.
	76	///
	77	/// The result is written into the codegen array, and the frequencies
	78	/// of each code is written into the codegen_freq array.
	79	/// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
	80	/// information. Code bad_code is an end marker
	81	///
	82	/// num_literals: The number of literals in literal_encoding
	83	/// num_distances: The number of distances in distance_encoding
	84	/// lit_enc: The literal encoder to use
	85	/// dist_enc: The distance encoder to use
	86	fn generateCodegen(
	87	self: *BlockWriter,
	88	num_literals: u32,
	89	num_distances: u32,
	90	lit_enc: *Compress.LiteralEncoder,
	91	dist_enc: *Compress.DistanceEncoder,
	92	) void {
	93	for (self.codegen_freq, 0..) \|_, i\| {
	94	self.codegen_freq[i] = 0;
	95	}
	96
	97	// Note that we are using codegen both as a temporary variable for holding
	98	// a copy of the frequencies, and as the place where we put the result.
	99	// This is fine because the output is always shorter than the input used
	100	// so far.
	101	var codegen = &self.codegen; // cache
	102	// Copy the concatenated code sizes to codegen. Put a marker at the end.
	103	var cgnl = codegen[0..num_literals];
	104	for (cgnl, 0..) \|_, i\| {
	105	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
	106	}
	107
	108	cgnl = codegen[num_literals .. num_literals + num_distances];
	109	for (cgnl, 0..) \|_, i\| {
	110	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
	111	}
	112	codegen[num_literals + num_distances] = end_code_mark;
	113
	114	var size = codegen[0];
	115	var count: i32 = 1;
	116	var out_index: u32 = 0;
	117	var in_index: u32 = 1;
	118	while (size != end_code_mark) : (in_index += 1) {
	119	// INVARIANT: We have seen "count" copies of size that have not yet
	120	// had output generated for them.
	121	const next_size = codegen[in_index];
	122	if (next_size == size) {
	123	count += 1;
	124	continue;
	125	}
	126	// We need to generate codegen indicating "count" of size.
	127	if (size != 0) {
	128	codegen[out_index] = size;
	129	out_index += 1;
	130	self.codegen_freq[size] += 1;
	131	count -= 1;
	132	while (count >= 3) {
	133	var n: i32 = 6;
	134	if (n > count) {
	135	n = count;
	136	}
	137	codegen[out_index] = 16;
	138	out_index += 1;
	139	codegen[out_index] = @as(u8, @intCast(n - 3));
	140	out_index += 1;
	141	self.codegen_freq[16] += 1;
	142	count -= n;
	143	}
	144	} else {
	145	while (count >= 11) {
	146	var n: i32 = 138;
	147	if (n > count) {
	148	n = count;
	149	}
	150	codegen[out_index] = 18;
	151	out_index += 1;
	152	codegen[out_index] = @as(u8, @intCast(n - 11));
	153	out_index += 1;
	154	self.codegen_freq[18] += 1;
	155	count -= n;
	156	}
	157	if (count >= 3) {
	158	// 3 <= count <= 10
	159	codegen[out_index] = 17;
	160	out_index += 1;
	161	codegen[out_index] = @as(u8, @intCast(count - 3));
	162	out_index += 1;
	163	self.codegen_freq[17] += 1;
	164	count = 0;
	165	}
	166	}
	167	count -= 1;
	168	while (count >= 0) : (count -= 1) {
	169	codegen[out_index] = size;
	170	out_index += 1;
	171	self.codegen_freq[size] += 1;
	172	}
	173	// Set up invariant for next time through the loop.
	174	size = next_size;
	175	count = 1;
	176	}
	177	// Marker indicating the end of the codegen.
	178	codegen[out_index] = end_code_mark;
	179	}
	180
	181	const DynamicSize = struct {
	182	size: u32,
	183	num_codegens: u32,
	184	};
	185
	186	/// dynamicSize returns the size of dynamically encoded data in bits.
	187	fn dynamicSize(
	188	self: *BlockWriter,
	189	lit_enc: *Compress.LiteralEncoder, // literal encoder
	190	dist_enc: *Compress.DistanceEncoder, // distance encoder
	191	extra_bits: u32,
	192	) DynamicSize {
	193	var num_codegens = self.codegen_freq.len;
	194	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
	195	num_codegens -= 1;
	196	}
	197	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
	198	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
	199	self.codegen_freq[16] * 2 +
	200	self.codegen_freq[17] * 3 +
	201	self.codegen_freq[18] * 7;
	202	const size = header +
	203	lit_enc.bitLength(&self.literal_freq) +
	204	dist_enc.bitLength(&self.distance_freq) +
	205	extra_bits;
	206
	207	return DynamicSize{
	208	.size = @as(u32, @intCast(size)),
	209	.num_codegens = @as(u32, @intCast(num_codegens)),
	210	};
	211	}
	212
	213	/// fixedSize returns the size of dynamically encoded data in bits.
	214	fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 {
	215	return 3 +
	216	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
	217	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
	218	extra_bits;
	219	}
	220
	221	const StoredSize = struct {
	222	size: u32,
	223	storable: bool,
	224	};
	225
	226	/// storedSizeFits calculates the stored size, including header.
	227	/// The function returns the size in bits and whether the block
	228	/// fits inside a single block.
	229	fn storedSizeFits(in: ?[]const u8) StoredSize {
	230	if (in == null) {
	231	return .{ .size = 0, .storable = false };
	232	}
	233	if (in.?.len <= HuffmanEncoder.max_store_block_size) {
	234	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
	235	}
	236	return .{ .size = 0, .storable = false };
	237	}
	238
	239	/// Write the header of a dynamic Huffman block to the output stream.
	240	///
	241	/// num_literals: The number of literals specified in codegen
	242	/// num_distances: The number of distances specified in codegen
	243	/// num_codegens: The number of codegens used in codegen
	244	/// eof: Is it the end-of-file? (end of stream)
	245	fn dynamicHeader(
	246	self: *BlockWriter,
	247	num_literals: u32,
	248	num_distances: u32,
	249	num_codegens: u32,
	250	eof: bool,
	251	) Writer.Error!void {
	252	const first_bits: u32 = if (eof) 5 else 4;
	253	try self.bit_writer.writeBits(first_bits, 3);
	254	try self.bit_writer.writeBits(num_literals - 257, 5);
	255	try self.bit_writer.writeBits(num_distances - 1, 5);
	256	try self.bit_writer.writeBits(num_codegens - 4, 4);
	257
	258	var i: u32 = 0;
	259	while (i < num_codegens) : (i += 1) {
	260	const value = self.codegen_encoding.codes[codegen_order[i]].len;
	261	try self.bit_writer.writeBits(value, 3);
	262	}
	263
	264	i = 0;
	265	while (true) {
	266	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
	267	i += 1;
	268	if (code_word == end_code_mark) {
	269	break;
	270	}
	271	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
	272
	273	switch (code_word) {
	274	16 => {
	275	try self.bit_writer.writeBits(self.codegen[i], 2);
	276	i += 1;
	277	},
	278	17 => {
	279	try self.bit_writer.writeBits(self.codegen[i], 3);
	280	i += 1;
	281	},
	282	18 => {
	283	try self.bit_writer.writeBits(self.codegen[i], 7);
	284	i += 1;
	285	},
	286	else => {},
	287	}
	288	}
	289	}
	290
	291	fn storedHeader(self: *BlockWriter, length: usize, eof: bool) Writer.Error!void {
	292	assert(length <= 65535);
	293	const flag: u32 = if (eof) 1 else 0;
	294	try self.bit_writer.writeBits(flag, 3);
	295	try self.flush();
	296	const l: u16 = @intCast(length);
	297	try self.bit_writer.writeBits(l, 16);
	298	try self.bit_writer.writeBits(~l, 16);
	299	}
	300
	301	fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void {
	302	// Indicate that we are a fixed Huffman block
	303	var value: u32 = 2;
	304	if (eof) {
	305	value = 3;
	306	}
	307	try self.bit_writer.writeBits(value, 3);
	308	}
	309
	310	/// Write a block of tokens with the smallest encoding. Will choose block type.
	311	/// The original input can be supplied, and if the huffman encoded data
	312	/// is larger than the original bytes, the data will be written as a
	313	/// stored block.
	314	/// If the input is null, the tokens will always be Huffman encoded.
	315	pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void {
	316	const lit_and_dist = self.indexTokens(tokens);
	317	const num_literals = lit_and_dist.num_literals;
	318	const num_distances = lit_and_dist.num_distances;
	319
	320	var extra_bits: u32 = 0;
	321	const ret = storedSizeFits(input);
	322	const stored_size = ret.size;
	323	const storable = ret.storable;
	324
	325	if (storable) {
	326	// We only bother calculating the costs of the extra bits required by
	327	// the length of distance fields (which will be the same for both fixed
	328	// and dynamic encoding), if we need to compare those two encodings
	329	// against stored encoding.
	330	var length_code: u16 = Token.length_codes_start + 8;
	331	while (length_code < num_literals) : (length_code += 1) {
	332	// First eight length codes have extra size = 0.
	333	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
	334	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
	335	}
	336	var distance_code: u16 = 4;
	337	while (distance_code < num_distances) : (distance_code += 1) {
	338	// First four distance codes have extra size = 0.
	339	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
	340	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
	341	}
	342	}
	343
	344	// Figure out smallest code.
	345	// Fixed Huffman baseline.
	346	var literal_encoding = &self.fixed_literal_encoding;
	347	var distance_encoding = &self.fixed_distance_encoding;
	348	var size = self.fixedSize(extra_bits);
	349
	350	// Dynamic Huffman?
	351	var num_codegens: u32 = 0;
	352
	353	// Generate codegen and codegenFrequencies, which indicates how to encode
	354	// the literal_encoding and the distance_encoding.
	355	self.generateCodegen(
	356	num_literals,
	357	num_distances,
	358	&self.literal_encoding,
	359	&self.distance_encoding,
	360	);
	361	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
	362	const dynamic_size = self.dynamicSize(
	363	&self.literal_encoding,
	364	&self.distance_encoding,
	365	extra_bits,
	366	);
	367	const dyn_size = dynamic_size.size;
	368	num_codegens = dynamic_size.num_codegens;
	369
	370	if (dyn_size < size) {
	371	size = dyn_size;
	372	literal_encoding = &self.literal_encoding;
	373	distance_encoding = &self.distance_encoding;
	374	}
	375
	376	// Stored bytes?
	377	if (storable and stored_size < size) {
	378	try self.storedBlock(input.?, eof);
	379	return;
	380	}
	381
	382	// Huffman.
	383	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
	384	try self.fixedHeader(eof);
	385	} else {
	386	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
	387	}
	388
	389	// Write the tokens.
	390	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
	391	}
	392
	393	pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
	394	try self.storedHeader(input.len, eof);
	395	try self.bit_writer.writeBytes(input);
	396	}
	397
	398	/// writeBlockDynamic encodes a block using a dynamic Huffman table.
	399	/// This should be used if the symbols used have a disproportionate
	400	/// histogram distribution.
	401	/// If input is supplied and the compression savings are below 1/16th of the
	402	/// input size the block is stored.
	403	fn dynamicBlock(
	404	self: *BlockWriter,
	405	tokens: []const Token,
	406	eof: bool,
	407	input: ?[]const u8,
	408	) Writer.Error!void {
	409	const total_tokens = self.indexTokens(tokens);
	410	const num_literals = total_tokens.num_literals;
	411	const num_distances = total_tokens.num_distances;
	412
	413	// Generate codegen and codegenFrequencies, which indicates how to encode
	414	// the literal_encoding and the distance_encoding.
	415	self.generateCodegen(
	416	num_literals,
	417	num_distances,
	418	&self.literal_encoding,
	419	&self.distance_encoding,
	420	);
	421	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
	422	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
	423	const size = dynamic_size.size;
	424	const num_codegens = dynamic_size.num_codegens;
	425
	426	// Store bytes, if we don't get a reasonable improvement.
	427
	428	const stored_size = storedSizeFits(input);
	429	const ssize = stored_size.size;
	430	const storable = stored_size.storable;
	431	if (storable and ssize < (size + (size >> 4))) {
	432	try self.storedBlock(input.?, eof);
	433	return;
	434	}
	435
	436	// Write Huffman table.
	437	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
	438
	439	// Write the tokens.
	440	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
	441	}
	442
	443	const TotalIndexedTokens = struct {
	444	num_literals: u32,
	445	num_distances: u32,
	446	};
	447
	448	/// Indexes a slice of tokens followed by an end_block_marker, and updates
	449	/// literal_freq and distance_freq, and generates literal_encoding
	450	/// and distance_encoding.
	451	/// The number of literal and distance tokens is returned.
	452	fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
	453	var num_literals: u32 = 0;
	454	var num_distances: u32 = 0;
	455
	456	for (self.literal_freq, 0..) \|_, i\| {
	457	self.literal_freq[i] = 0;
	458	}
	459	for (self.distance_freq, 0..) \|_, i\| {
	460	self.distance_freq[i] = 0;
	461	}
	462
	463	for (tokens) \|t\| {
	464	if (t.kind == Token.Kind.literal) {
	465	self.literal_freq[t.literal()] += 1;
	466	continue;
	467	}
	468	self.literal_freq[t.lengthCode()] += 1;
	469	self.distance_freq[t.distanceCode()] += 1;
	470	}
	471	// add end_block_marker token at the end
	472	self.literal_freq[HuffmanEncoder.end_block_marker] += 1;
	473
	474	// get the number of literals
	475	num_literals = @as(u32, @intCast(self.literal_freq.len));
	476	while (self.literal_freq[num_literals - 1] == 0) {
	477	num_literals -= 1;
	478	}
	479	// get the number of distances
	480	num_distances = @as(u32, @intCast(self.distance_freq.len));
	481	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
	482	num_distances -= 1;
	483	}
	484	if (num_distances == 0) {
	485	// We haven't found a single match. If we want to go with the dynamic encoding,
	486	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
	487	self.distance_freq[0] = 1;
	488	num_distances = 1;
	489	}
	490	self.literal_encoding.generate(&self.literal_freq, 15);
	491	self.distance_encoding.generate(&self.distance_freq, 15);
	492	return TotalIndexedTokens{
	493	.num_literals = num_literals,
	494	.num_distances = num_distances,
	495	};
	496	}
	497
	498	/// Writes a slice of tokens to the output followed by and end_block_marker.
	499	/// codes for literal and distance encoding must be supplied.
	500	fn writeTokens(
	501	self: *BlockWriter,
	502	tokens: []const Token,
	503	le_codes: []Compress.HuffCode,
	504	oe_codes: []Compress.HuffCode,
	505	) Writer.Error!void {
	506	for (tokens) \|t\| {
	507	if (t.kind == Token.Kind.literal) {
	508	try self.writeCode(le_codes[t.literal()]);
	509	continue;
	510	}
	511
	512	// Write the length
	513	const le = t.lengthEncoding();
	514	try self.writeCode(le_codes[le.code]);
	515	if (le.extra_bits > 0) {
	516	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
	517	}
	518
	519	// Write the distance
	520	const oe = t.distanceEncoding();
	521	try self.writeCode(oe_codes[oe.code]);
	522	if (oe.extra_bits > 0) {
	523	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
	524	}
	525	}
	526	// add end_block_marker at the end
	527	try self.writeCode(le_codes[HuffmanEncoder.end_block_marker]);
	528	}
	529
	530	/// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
	531	/// if the results only gains very little from compression.
	532	pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
	533	// Add everything as literals
	534	histogram(input, &self.literal_freq);
	535
	536	self.literal_freq[HuffmanEncoder.end_block_marker] = 1;
	537
	538	const num_literals = HuffmanEncoder.end_block_marker + 1;
	539	self.distance_freq[0] = 1;
	540	const num_distances = 1;
	541
	542	self.literal_encoding.generate(&self.literal_freq, 15);
	543
	544	// Figure out smallest code.
	545	// Always use dynamic Huffman or Store
	546	var num_codegens: u32 = 0;
	547
	548	// Generate codegen and codegenFrequencies, which indicates how to encode
	549	// the literal_encoding and the distance_encoding.
	550	self.generateCodegen(
	551	num_literals,
	552	num_distances,
	553	&self.literal_encoding,
	554	&self.huff_distance,
	555	);
	556	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
	557	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
	558	const size = dynamic_size.size;
	559	num_codegens = dynamic_size.num_codegens;
	560
	561	// Store bytes, if we don't get a reasonable improvement.
	562	const stored_size_ret = storedSizeFits(input);
	563	const ssize = stored_size_ret.size;
	564	const storable = stored_size_ret.storable;
	565
	566	if (storable and ssize < (size + (size >> 4))) {
	567	try self.storedBlock(input, eof);
	568	return;
	569	}
	570
	571	// Huffman.
	572	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
	573	const encoding = self.literal_encoding.codes[0..257];
	574
	575	for (input) \|t\| {
	576	const c = encoding[t];
	577	try self.bit_writer.writeBits(c.code, c.len);
	578	}
	579	try self.writeCode(encoding[HuffmanEncoder.end_block_marker]);
	580	}
	581
	582	fn histogram(b: []const u8, h: *[286]u16) void {
	583	// Clear histogram
	584	for (h, 0..) \|_, i\| {
	585	h[i] = 0;
	586	}
	587
	588	var lh = h.*[0..256];
	589	for (b) \|t\| {
	590	lh[t] += 1;
	591	}
	592	}

lib/std/compress/flate/CircularBuffer.zig deleted-240

...	...	@@ -1,240 +0,0 @@
1		//! 64K buffer of uncompressed data created in inflate (decompression). Has enough
2		//! history to support writing match<length, distance>; copying length of bytes
3		//! from the position distance backward from current.
4		//!
5		//! Reads can return less than available bytes if they are spread across
6		//! different circles. So reads should repeat until get required number of bytes
7		//! or until returned slice is zero length.
8		//!
9		//! Note on deflate limits:
10		//! * non-compressible block is limited to 65,535 bytes.
11		//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12		//!
13		//! Whole non-compressed block can be written without overlap. We always have
14		//! history of up to 64K, more then 32K needed.
15		//!
16		const std = @import("std");
17		const assert = std.debug.assert;
18		const testing = std.testing;
19
20		const consts = @import("consts.zig").match;
21
22		const mask = 0xffff; // 64K - 1
23		const buffer_len = mask + 1; // 64K buffer
24
25		const Self = @This();
26
27		buffer: [buffer_len]u8 = undefined,
28		wp: usize = 0, // write position
29		rp: usize = 0, // read position
30
31		fn writeAll(self: *Self, buf: []const u8) void {
32		for (buf) \|c\| self.write(c);
33		}
34
35		/// Write literal.
36		pub fn write(self: *Self, b: u8) void {
37		assert(self.wp - self.rp < mask);
38		self.buffer[self.wp & mask] = b;
39		self.wp += 1;
40		}
41
42		/// Write match (back-reference to the same data slice) starting at `distance`
43		/// back from current write position, and `length` of bytes.
44		pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45		if (self.wp < distance or
46		length < consts.base_length or length > consts.max_length or
47		distance < consts.min_distance or distance > consts.max_distance)
48		{
49		return error.InvalidMatch;
50		}
51		assert(self.wp - self.rp < mask);
52
53		var from: usize = self.wp - distance & mask;
54		const from_end: usize = from + length;
55		var to: usize = self.wp & mask;
56		const to_end: usize = to + length;
57
58		self.wp += length;
59
60		// Fast path using memcpy
61		if (from_end < buffer_len and to_end < buffer_len) // start and end at the same circle
62		{
63		var cur_len = distance;
64		var remaining_len = length;
65		while (cur_len < remaining_len) {
66		@memcpy(self.buffer[to..][0..cur_len], self.buffer[from..][0..cur_len]);
67		to += cur_len;
68		remaining_len -= cur_len;
69		cur_len = cur_len * 2;
70		}
71		@memcpy(self.buffer[to..][0..remaining_len], self.buffer[from..][0..remaining_len]);
72		return;
73		}
74
75		// Slow byte by byte
76		while (to < to_end) {
77		self.buffer[to & mask] = self.buffer[from & mask];
78		to += 1;
79		from += 1;
80		}
81		}
82
83		/// Returns writable part of the internal buffer of size `n` at most. Advances
84		/// write pointer, assumes that returned buffer will be filled with data.
85		pub fn getWritable(self: *Self, n: usize) []u8 {
86		const wp = self.wp & mask;
87		const len = @min(n, buffer_len - wp);
88		self.wp += len;
89		return self.buffer[wp .. wp + len];
90		}
91
92		/// Read available data. Can return part of the available data if it is
93		/// spread across two circles. So read until this returns zero length.
94		pub fn read(self: *Self) []const u8 {
95		return self.readAtMost(buffer_len);
96		}
97
98		/// Read part of available data. Can return less than max even if there are
99		/// more than max decoded data.
100		pub fn readAtMost(self: *Self, limit: usize) []const u8 {
101		const rb = self.readBlock(if (limit == 0) buffer_len else limit);
102		defer self.rp += rb.len;
103		return self.buffer[rb.head..rb.tail];
104		}
105
106		const ReadBlock = struct {
107		head: usize,
108		tail: usize,
109		len: usize,
110		};
111
112		/// Returns position of continuous read block data.
113		fn readBlock(self: *Self, max: usize) ReadBlock {
114		const r = self.rp & mask;
115		const w = self.wp & mask;
116		const n = @min(
117		max,
118		if (w >= r) w - r else buffer_len - r,
119		);
120		return .{
121		.head = r,
122		.tail = r + n,
123		.len = n,
124		};
125		}
126
127		/// Number of free bytes for write.
128		pub fn free(self: *Self) usize {
129		return buffer_len - (self.wp - self.rp);
130		}
131
132		/// Full if largest match can't fit. 258 is largest match length. That much
133		/// bytes can be produced in single decode step.
134		pub fn full(self: *Self) bool {
135		return self.free() < 258 + 1;
136		}
137
138		// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
139		test writeMatch {
140		var cb: Self = .{};
141
142		cb.writeAll("a salad; ");
143		try cb.writeMatch(5, 9);
144		try cb.writeMatch(3, 3);
145
146		try testing.expectEqualStrings("a salad; a salsal", cb.read());
147		}
148
149		test "writeMatch overlap" {
150		var cb: Self = .{};
151
152		cb.writeAll("a b c ");
153		try cb.writeMatch(8, 4);
154		cb.write('d');
155
156		try testing.expectEqualStrings("a b c b c b c d", cb.read());
157		}
158
159		test readAtMost {
160		var cb: Self = .{};
161
162		cb.writeAll("0123456789");
163		try cb.writeMatch(50, 10);
164
165		try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
166		for (0..6) \|i\| {
167		try testing.expectEqual(i * 10, cb.rp);
168		try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
169		}
170		try testing.expectEqualStrings("", cb.readAtMost(10));
171		try testing.expectEqualStrings("", cb.read());
172		}
173
174		test Self {
175		var cb: Self = .{};
176
177		const data = "0123456789abcdef" ** (1024 / 16);
178		cb.writeAll(data);
179		try testing.expectEqual(@as(usize, 0), cb.rp);
180		try testing.expectEqual(@as(usize, 1024), cb.wp);
181		try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
182
183		for (0..62 * 4) \|_\|
184		try cb.writeMatch(256, 1024); // write 62K
185
186		try testing.expectEqual(@as(usize, 0), cb.rp);
187		try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
188		try testing.expectEqual(@as(usize, 1024), cb.free());
189
190		cb.writeAll(data[0..200]);
191		_ = cb.readAtMost(1024); // make some space
192		cb.writeAll(data); // overflows write position
193		try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
194		try testing.expectEqual(@as(usize, 1024), cb.rp);
195		try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
196
197		const rb = cb.readBlock(Self.buffer_len);
198		try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
199		try testing.expectEqual(@as(usize, 1024), rb.head);
200		try testing.expectEqual(@as(usize, 65536), rb.tail);
201
202		try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
203		try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
204		try testing.expectEqual(@as(usize, 65536), cb.rp);
205		try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
206
207		try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
208		}
209
210		test "write overlap" {
211		var cb: Self = .{};
212		cb.wp = cb.buffer.len - 15;
213		cb.rp = cb.wp;
214
215		cb.writeAll("0123456789");
216		cb.writeAll("abcdefghij");
217
218		try testing.expectEqual(cb.buffer.len + 5, cb.wp);
219		try testing.expectEqual(cb.buffer.len - 15, cb.rp);
220
221		try testing.expectEqualStrings("0123456789abcde", cb.read());
222		try testing.expectEqualStrings("fghij", cb.read());
223
224		try testing.expect(cb.wp == cb.rp);
225		}
226
227		test "writeMatch/read overlap" {
228		var cb: Self = .{};
229		cb.wp = cb.buffer.len - 15;
230		cb.rp = cb.wp;
231
232		cb.writeAll("0123456789");
233		try cb.writeMatch(15, 5);
234
235		try testing.expectEqualStrings("012345678956789", cb.read());
236		try testing.expectEqualStrings("5678956789", cb.read());
237
238		try cb.writeMatch(20, 25);
239		try testing.expectEqualStrings("01234567895678956789", cb.read());
240		}

lib/std/compress/flate/Compress.zig created+332

...	...	@@ -0,0 +1,332 @@
	1	//! Default compression algorithm. Has two steps: tokenization and token
	2	//! encoding.
	3	//!
	4	//! Tokenization takes uncompressed input stream and produces list of tokens.
	5	//! Each token can be literal (byte of data) or match (backrefernce to previous
	6	//! data with length and distance). Tokenization accumulators 32K tokens, when
	7	//! full or `flush` is called tokens are passed to the `block_writer`. Level
	8	//! defines how hard (how slow) it tries to find match.
	9	//!
	10	//! Block writer will decide which type of deflate block to write (stored, fixed,
	11	//! dynamic) and encode tokens to the output byte stream. Client has to call
	12	//! `finish` to write block with the final bit set.
	13	//!
	14	//! Container defines type of header and footer which can be gzip, zlib or raw.
	15	//! They all share same deflate body. Raw has no header or footer just deflate
	16	//! body.
	17	//!
	18	//! Compression algorithm explained in rfc-1951 (slightly edited for this case):
	19	//!
	20	//! The compressor uses a chained hash table `lookup` to find duplicated
	21	//! strings, using a hash function that operates on 4-byte sequences. At any
	22	//! given point during compression, let XYZW be the next 4 input bytes
	23	//! (lookahead) to be examined (not necessarily all different, of course).
	24	//! First, the compressor examines the hash chain for XYZW. If the chain is
	25	//! empty, the compressor simply writes out X as a literal byte and advances
	26	//! one byte in the input. If the hash chain is not empty, indicating that the
	27	//! sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
	28	//! hash function value) has occurred recently, the compressor compares all
	29	//! strings on the XYZW hash chain with the actual input data sequence
	30	//! starting at the current point, and selects the longest match.
	31	//!
	32	//! To improve overall compression, the compressor defers the selection of
	33	//! matches ("lazy matching"): after a match of length N has been found, the
	34	//! compressor searches for a longer match starting at the next input byte. If
	35	//! it finds a longer match, it truncates the previous match to a length of
	36	//! one (thus producing a single literal byte) and then emits the longer
	37	//! match. Otherwise, it emits the original match, and, as described above,
	38	//! advances N bytes before continuing.
	39	//!
	40	//!
	41	//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
	42
	43	const builtin = @import("builtin");
	44	const std = @import("std");
	45	const assert = std.debug.assert;
	46	const testing = std.testing;
	47	const expect = testing.expect;
	48	const mem = std.mem;
	49	const math = std.math;
	50	const Writer = std.Io.Writer;
	51
	52	const Compress = @This();
	53	const Token = @import("Token.zig");
	54	const BlockWriter = @import("BlockWriter.zig");
	55	const flate = @import("../flate.zig");
	56	const Container = flate.Container;
	57	const Lookup = @import("Lookup.zig");
	58	const HuffmanEncoder = flate.HuffmanEncoder;
	59	const LiteralNode = HuffmanEncoder.LiteralNode;
	60
	61	lookup: Lookup = .{},
	62	tokens: Tokens = .{},
	63	block_writer: BlockWriter,
	64	level: LevelArgs,
	65	hasher: Container.Hasher,
	66	writer: Writer,
	67	state: State,
	68
	69	// Match and literal at the previous position.
	70	// Used for lazy match finding in processWindow.
	71	prev_match: ?Token = null,
	72	prev_literal: ?u8 = null,
	73
	74	pub const State = enum { header, middle, ended };
	75
	76	/// Trades between speed and compression size.
	77	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
	78	/// levels 1-3 are using different algorithm to perform faster but with less
	79	/// compression. That is not implemented here.
	80	pub const Level = enum(u4) {
	81	level_4 = 4,
	82	level_5 = 5,
	83	level_6 = 6,
	84	level_7 = 7,
	85	level_8 = 8,
	86	level_9 = 9,
	87
	88	fast = 0xb,
	89	default = 0xc,
	90	best = 0xd,
	91	};
	92
	93	/// Number of tokens to accumulate in deflate before starting block encoding.
	94	///
	95	/// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
	96	/// 8 and max 9 that gives 14 or 15 bits.
	97	pub const n_tokens = 1 << 15;
	98
	99	/// Algorithm knobs for each level.
	100	const LevelArgs = struct {
	101	good: u16, // Do less lookups if we already have match of this length.
	102	nice: u16, // Stop looking for better match if we found match with at least this length.
	103	lazy: u16, // Don't do lazy match find if got match with at least this length.
	104	chain: u16, // How many lookups for previous match to perform.
	105
	106	pub fn get(level: Level) LevelArgs {
	107	return switch (level) {
	108	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
	109	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
	110	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
	111	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
	112	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
	113	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
	114	};
	115	}
	116	};
	117
	118	pub const Options = struct {
	119	level: Level = .default,
	120	container: Container = .raw,
	121	};
	122
	123	pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {
	124	return .{
	125	.block_writer = .init(output),
	126	.level = .get(options.level),
	127	.hasher = .init(options.container),
	128	.state = .header,
	129	.writer = .{
	130	.buffer = buffer,
	131	.vtable = &.{ .drain = drain },
	132	},
	133	};
	134	}
	135
	136	// Tokens store
	137	const Tokens = struct {
	138	list: [n_tokens]Token = undefined,
	139	pos: usize = 0,
	140
	141	fn add(self: *Tokens, t: Token) void {
	142	self.list[self.pos] = t;
	143	self.pos += 1;
	144	}
	145
	146	fn full(self: *Tokens) bool {
	147	return self.pos == self.list.len;
	148	}
	149
	150	fn reset(self: *Tokens) void {
	151	self.pos = 0;
	152	}
	153
	154	fn tokens(self: *Tokens) []const Token {
	155	return self.list[0..self.pos];
	156	}
	157	};
	158
	159	fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {
	160	_ = data;
	161	_ = splat;
	162	const c: *Compress = @fieldParentPtr("writer", me);
	163	const out = c.block_writer.output;
	164	switch (c.state) {
	165	.header => {
	166	c.state = .middle;
	167	const header = c.hasher.container().header();
	168	try out.writeAll(header);
	169	return header.len;
	170	},
	171	.middle => {},
	172	.ended => unreachable,
	173	}
	174
	175	const buffered = me.buffered();
	176	const min_lookahead = Token.min_length + Token.max_length;
	177	const history_plus_lookahead_len = flate.history_len + min_lookahead;
	178	if (buffered.len < history_plus_lookahead_len) return 0;
	179	const lookahead = buffered[flate.history_len..];
	180
	181	// TODO tokenize
	182	_ = lookahead;
	183	//c.hasher.update(lookahead[0..n]);
	184	@panic("TODO");
	185	}
	186
	187	pub fn end(c: *Compress) !void {
	188	try endUnflushed(c);
	189	const out = c.block_writer.output;
	190	try out.flush();
	191	}
	192
	193	pub fn endUnflushed(c: *Compress) !void {
	194	while (c.writer.end != 0) _ = try drain(&c.writer, &.{""}, 1);
	195	c.state = .ended;
	196
	197	const out = c.block_writer.output;
	198
	199	// TODO flush tokens
	200
	201	switch (c.hasher) {
	202	.gzip => \|*gzip\| {
	203	// GZIP 8 bytes footer
	204	// - 4 bytes, CRC32 (CRC-32)
	205	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
	206	const footer = try out.writableArray(8);
	207	std.mem.writeInt(u32, footer[0..4], gzip.crc.final(), .little);
	208	std.mem.writeInt(u32, footer[4..8], @truncate(gzip.count), .little);
	209	},
	210	.zlib => \|*zlib\| {
	211	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
	212	// 4 bytes of ADLER32 (Adler-32 checksum)
	213	// Checksum value of the uncompressed data (excluding any
	214	// dictionary data) computed according to Adler-32
	215	// algorithm.
	216	std.mem.writeInt(u32, try out.writableArray(4), zlib.adler, .big);
	217	},
	218	.raw => {},
	219	}
	220	}
	221
	222	pub const Simple = struct {
	223	/// Note that store blocks are limited to 65535 bytes.
	224	buffer: []u8,
	225	wp: usize,
	226	block_writer: BlockWriter,
	227	hasher: Container.Hasher,
	228	strategy: Strategy,
	229
	230	pub const Strategy = enum { huffman, store };
	231
	232	pub fn init(output: *Writer, buffer: []u8, container: Container, strategy: Strategy) !Simple {
	233	const header = container.header();
	234	try output.writeAll(header);
	235	return .{
	236	.buffer = buffer,
	237	.wp = 0,
	238	.block_writer = .init(output),
	239	.hasher = .init(container),
	240	.strategy = strategy,
	241	};
	242	}
	243
	244	pub fn flush(self: *Simple) !void {
	245	try self.flushBuffer(false);
	246	try self.block_writer.storedBlock("", false);
	247	try self.block_writer.flush();
	248	}
	249
	250	pub fn finish(self: *Simple) !void {
	251	try self.flushBuffer(true);
	252	try self.block_writer.flush();
	253	try self.hasher.container().writeFooter(&self.hasher, self.block_writer.output);
	254	}
	255
	256	fn flushBuffer(self: *Simple, final: bool) !void {
	257	const buf = self.buffer[0..self.wp];
	258	switch (self.strategy) {
	259	.huffman => try self.block_writer.huffmanBlock(buf, final),
	260	.store => try self.block_writer.storedBlock(buf, final),
	261	}
	262	self.wp = 0;
	263	}
	264	};
	265
	266	test "generate a Huffman code from an array of frequencies" {
	267	var freqs: [19]u16 = [_]u16{
	268	8, // 0
	269	1, // 1
	270	1, // 2
	271	2, // 3
	272	5, // 4
	273	10, // 5
	274	9, // 6
	275	1, // 7
	276	0, // 8
	277	0, // 9
	278	0, // 10
	279	0, // 11
	280	0, // 12
	281	0, // 13
	282	0, // 14
	283	0, // 15
	284	1, // 16
	285	3, // 17
	286	5, // 18
	287	};
	288
	289	var codes: [19]HuffmanEncoder.Code = undefined;
	290	var enc: HuffmanEncoder = .{
	291	.codes = &codes,
	292	.freq_cache = undefined,
	293	.bit_count = undefined,
	294	.lns = undefined,
	295	.lfs = undefined,
	296	};
	297	enc.generate(freqs[0..], 7);
	298
	299	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
	300
	301	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
	302	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
	303	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
	304	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
	305	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
	306	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
	307	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
	308	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
	309	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
	310	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
	311	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
	312	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
	313	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
	314	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
	315	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
	316	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
	317	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
	318	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
	319	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
	320
	321	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
	322	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
	323	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
	324	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
	325	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
	326	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
	327	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
	328	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
	329	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
	330	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
	331	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
	332	}

lib/std/compress/flate/Decompress.zig created+1270

...	...	@@ -0,0 +1,1270 @@
	1	const std = @import("../../std.zig");
	2	const assert = std.debug.assert;
	3	const flate = std.compress.flate;
	4	const testing = std.testing;
	5	const Writer = std.Io.Writer;
	6	const Reader = std.Io.Reader;
	7	const Container = flate.Container;
	8
	9	const Decompress = @This();
	10	const Token = @import("Token.zig");
	11
	12	input: *Reader,
	13	next_bits: Bits,
	14	remaining_bits: std.math.Log2Int(Bits),
	15
	16	reader: Reader,
	17
	18	container_metadata: Container.Metadata,
	19
	20	lit_dec: LiteralDecoder,
	21	dst_dec: DistanceDecoder,
	22
	23	final_block: bool,
	24	state: State,
	25
	26	err: ?Error,
	27
	28	/// TODO: change this to usize
	29	const Bits = u64;
	30
	31	const BlockType = enum(u2) {
	32	stored = 0,
	33	fixed = 1,
	34	dynamic = 2,
	35	};
	36
	37	const State = union(enum) {
	38	protocol_header,
	39	block_header,
	40	stored_block: u16,
	41	fixed_block,
	42	dynamic_block,
	43	dynamic_block_literal: u8,
	44	dynamic_block_match: u16,
	45	protocol_footer,
	46	end,
	47	};
	48
	49	pub const Error = Container.Error \|\| error{
	50	InvalidCode,
	51	InvalidMatch,
	52	WrongStoredBlockNlen,
	53	InvalidDynamicBlockHeader,
	54	ReadFailed,
	55	OversubscribedHuffmanTree,
	56	IncompleteHuffmanTree,
	57	MissingEndOfBlockCode,
	58	EndOfStream,
	59	};
	60
	61	const direct_vtable: Reader.VTable = .{
	62	.stream = streamDirect,
	63	.rebase = rebaseFallible,
	64	.discard = discard,
	65	.readVec = readVec,
	66	};
	67
	68	const indirect_vtable: Reader.VTable = .{
	69	.stream = streamIndirect,
	70	.rebase = rebaseFallible,
	71	.discard = discardIndirect,
	72	.readVec = readVec,
	73	};
	74
	75	pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress {
	76	return .{
	77	.reader = .{
	78	.vtable = if (buffer.len == 0) &direct_vtable else &indirect_vtable,
	79	.buffer = buffer,
	80	.seek = 0,
	81	.end = 0,
	82	},
	83	.input = input,
	84	.next_bits = 0,
	85	.remaining_bits = 0,
	86	.container_metadata = .init(container),
	87	.lit_dec = .{},
	88	.dst_dec = .{},
	89	.final_block = false,
	90	.state = .protocol_header,
	91	.err = null,
	92	};
	93	}
	94
	95	fn rebaseFallible(r: *Reader, capacity: usize) Reader.RebaseError!void {
	96	rebase(r, capacity);
	97	}
	98
	99	fn rebase(r: *Reader, capacity: usize) void {
	100	assert(capacity <= r.buffer.len - flate.history_len);
	101	assert(r.end + capacity > r.buffer.len);
	102	const discard_n = r.end - flate.history_len;
	103	const keep = r.buffer[discard_n..r.end];
	104	@memmove(r.buffer[0..keep.len], keep);
	105	assert(keep.len != 0);
	106	r.end = keep.len;
	107	r.seek -= discard_n;
	108	}
	109
	110	/// This could be improved so that when an amount is discarded that includes an
	111	/// entire frame, skip decoding that frame.
	112	fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	113	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
	114	var writer: Writer = .{
	115	.vtable = &.{
	116	.drain = std.Io.Writer.Discarding.drain,
	117	.sendFile = std.Io.Writer.Discarding.sendFile,
	118	},
	119	.buffer = r.buffer,
	120	.end = r.end,
	121	};
	122	defer {
	123	assert(writer.end != 0);
	124	r.end = writer.end;
	125	r.seek = r.end;
	126	}
	127	const n = r.stream(&writer, limit) catch \|err\| switch (err) {
	128	error.WriteFailed => unreachable,
	129	error.ReadFailed => return error.ReadFailed,
	130	error.EndOfStream => return error.EndOfStream,
	131	};
	132	assert(n <= @intFromEnum(limit));
	133	return n;
	134	}
	135
	136	fn discardIndirect(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	137	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	138	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
	139	var writer: Writer = .{
	140	.buffer = r.buffer,
	141	.end = r.end,
	142	.vtable = &.{ .drain = Writer.unreachableDrain },
	143	};
	144	{
	145	defer r.end = writer.end;
	146	_ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
	147	error.WriteFailed => unreachable,
	148	else => \|e\| return e,
	149	};
	150	}
	151	const n = limit.minInt(r.end - r.seek);
	152	r.seek += n;
	153	return n;
	154	}
	155
	156	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
	157	_ = data;
	158	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	159	return streamIndirectInner(d);
	160	}
	161
	162	fn streamIndirectInner(d: *Decompress) Reader.Error!usize {
	163	const r = &d.reader;
	164	if (r.end + flate.history_len > r.buffer.len) rebase(r, flate.history_len);
	165	var writer: Writer = .{
	166	.buffer = r.buffer,
	167	.end = r.end,
	168	.vtable = &.{ .drain = Writer.unreachableDrain },
	169	};
	170	defer r.end = writer.end;
	171	_ = streamFallible(d, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
	172	error.WriteFailed => unreachable,
	173	else => \|e\| return e,
	174	};
	175	return 0;
	176	}
	177
	178	fn decodeLength(self: *Decompress, code: u8) !u16 {
	179	if (code > 28) return error.InvalidCode;
	180	const ml = Token.matchLength(code);
	181	return if (ml.extra_bits == 0) // 0 - 5 extra bits
	182	ml.base
	183	else
	184	ml.base + try self.takeBitsRuntime(ml.extra_bits);
	185	}
	186
	187	fn decodeDistance(self: *Decompress, code: u8) !u16 {
	188	if (code > 29) return error.InvalidCode;
	189	const md = Token.matchDistance(code);
	190	return if (md.extra_bits == 0) // 0 - 13 extra bits
	191	md.base
	192	else
	193	md.base + try self.takeBitsRuntime(md.extra_bits);
	194	}
	195
	196	// Decode code length symbol to code length. Writes decoded length into
	197	// lens slice starting at position pos. Returns number of positions
	198	// advanced.
	199	fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usize {
	200	if (pos >= lens.len)
	201	return error.InvalidDynamicBlockHeader;
	202
	203	switch (code) {
	204	0...15 => {
	205	// Represent code lengths of 0 - 15
	206	lens[pos] = @intCast(code);
	207	return 1;
	208	},
	209	16 => {
	210	// Copy the previous code length 3 - 6 times.
	211	// The next 2 bits indicate repeat length
	212	const n: u8 = @as(u8, try self.takeBits(u2)) + 3;
	213	if (pos == 0 or pos + n > lens.len)
	214	return error.InvalidDynamicBlockHeader;
	215	for (0..n) \|i\| {
	216	lens[pos + i] = lens[pos + i - 1];
	217	}
	218	return n;
	219	},
	220	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
	221	17 => return @as(u8, try self.takeBits(u3)) + 3,
	222	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
	223	18 => return @as(u8, try self.takeBits(u7)) + 11,
	224	else => return error.InvalidDynamicBlockHeader,
	225	}
	226	}
	227
	228	fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
	229	// Maximum code len is 15 bits.
	230	const sym = try decoder.find(@bitReverse(try self.peekBits(u15)));
	231	try self.tossBits(sym.code_bits);
	232	return sym;
	233	}
	234
	235	fn streamDirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	236	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	237	return streamFallible(d, w, limit);
	238	}
	239
	240	fn streamIndirect(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	241	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	242	_ = limit;
	243	_ = w;
	244	return streamIndirectInner(d);
	245	}
	246
	247	fn streamFallible(d: Decompress, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	248	return streamInner(d, w, limit) catch \|err\| switch (err) {
	249	error.EndOfStream => {
	250	if (d.state == .end) {
	251	return error.EndOfStream;
	252	} else {
	253	d.err = error.EndOfStream;
	254	return error.ReadFailed;
	255	}
	256	},
	257	error.WriteFailed => return error.WriteFailed,
	258	else => \|e\| {
	259	// In the event of an error, state is unmodified so that it can be
	260	// better used to diagnose the failure.
	261	d.err = e;
	262	return error.ReadFailed;
	263	},
	264	};
	265	}
	266
	267	fn streamInner(d: Decompress, w: Writer, limit: std.Io.Limit) (Error \|\| Reader.StreamError)!usize {
	268	var remaining = @intFromEnum(limit);
	269	const in = d.input;
	270	sw: switch (d.state) {
	271	.protocol_header => switch (d.container_metadata.container()) {
	272	.gzip => {
	273	const Header = extern struct {
	274	magic: u16 align(1),
	275	method: u8,
	276	flags: packed struct(u8) {
	277	text: bool,
	278	hcrc: bool,
	279	extra: bool,
	280	name: bool,
	281	comment: bool,
	282	reserved: u3,
	283	},
	284	mtime: u32 align(1),
	285	xfl: u8,
	286	os: u8,
	287	};
	288	const header = try in.takeStruct(Header, .little);
	289	if (header.magic != 0x8b1f or header.method != 0x08)
	290	return error.BadGzipHeader;
	291	if (header.flags.extra) {
	292	const extra_len = try in.takeInt(u16, .little);
	293	try in.discardAll(extra_len);
	294	}
	295	if (header.flags.name) {
	296	_ = try in.discardDelimiterInclusive(0);
	297	}
	298	if (header.flags.comment) {
	299	_ = try in.discardDelimiterInclusive(0);
	300	}
	301	if (header.flags.hcrc) {
	302	try in.discardAll(2);
	303	}
	304	continue :sw .block_header;
	305	},
	306	.zlib => {
	307	const header = try in.takeArray(2);
	308	const cmf: packed struct(u8) { cm: u4, cinfo: u4 } = @bitCast(header[0]);
	309	if (cmf.cm != 8 or cmf.cinfo > 7) return error.BadZlibHeader;
	310	continue :sw .block_header;
	311	},
	312	.raw => continue :sw .block_header,
	313	},
	314	.block_header => {
	315	d.final_block = (try d.takeBits(u1)) != 0;
	316	const block_type: BlockType = @enumFromInt(try d.takeBits(u2));
	317	switch (block_type) {
	318	.stored => {
	319	d.alignBitsDiscarding();
	320	// everything after this is byte aligned in stored block
	321	const len = try in.takeInt(u16, .little);
	322	const nlen = try in.takeInt(u16, .little);
	323	if (len != ~nlen) return error.WrongStoredBlockNlen;
	324	continue :sw .{ .stored_block = len };
	325	},
	326	.fixed => continue :sw .fixed_block,
	327	.dynamic => {
	328	const hlit: u16 = @as(u16, try d.takeBits(u5)) + 257; // number of ll code entries present - 257
	329	const hdist: u16 = @as(u16, try d.takeBits(u5)) + 1; // number of distance code entries - 1
	330	const hclen: u8 = @as(u8, try d.takeBits(u4)) + 4; // hclen + 4 code lengths are encoded
	331
	332	if (hlit > 286 or hdist > 30)
	333	return error.InvalidDynamicBlockHeader;
	334
	335	// lengths for code lengths
	336	var cl_lens: [19]u4 = @splat(0);
	337	for (flate.HuffmanEncoder.codegen_order[0..hclen]) \|i\| {
	338	cl_lens[i] = try d.takeBits(u3);
	339	}
	340	var cl_dec: CodegenDecoder = .{};
	341	try cl_dec.generate(&cl_lens);
	342
	343	// decoded code lengths
	344	var dec_lens: [286 + 30]u4 = @splat(0);
	345	var pos: usize = 0;
	346	while (pos < hlit + hdist) {
	347	const peeked = @bitReverse(try d.peekBits(u7));
	348	const sym = try cl_dec.find(peeked);
	349	try d.tossBits(sym.code_bits);
	350	pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos);
	351	}
	352	if (pos > hlit + hdist) {
	353	return error.InvalidDynamicBlockHeader;
	354	}
	355
	356	// literal code lengths to literal decoder
	357	try d.lit_dec.generate(dec_lens[0..hlit]);
	358
	359	// distance code lengths to distance decoder
	360	try d.dst_dec.generate(dec_lens[hlit..][0..hdist]);
	361
	362	continue :sw .dynamic_block;
	363	},
	364	}
	365	},
	366	.stored_block => \|remaining_len\| {
	367	const out = try w.writableSliceGreedyPreserve(flate.history_len, 1);
	368	var limited_out: [1][]u8 = .{limit.min(.limited(remaining_len)).slice(out)};
	369	const n = try d.input.readVec(&limited_out);
	370	if (remaining_len - n == 0) {
	371	d.state = if (d.final_block) .protocol_footer else .block_header;
	372	} else {
	373	d.state = .{ .stored_block = @intCast(remaining_len - n) };
	374	}
	375	w.advance(n);
	376	return n;
	377	},
	378	.fixed_block => {
	379	while (remaining > 0) {
	380	const code = try d.readFixedCode();
	381	switch (code) {
	382	0...255 => {
	383	try w.writeBytePreserve(flate.history_len, @intCast(code));
	384	remaining -= 1;
	385	},
	386	256 => {
	387	d.state = if (d.final_block) .protocol_footer else .block_header;
	388	return @intFromEnum(limit) - remaining;
	389	},
	390	257...285 => {
	391	// Handles fixed block non literal (length) code.
	392	// Length code is followed by 5 bits of distance code.
	393	const length = try d.decodeLength(@intCast(code - 257));
	394	const distance = try d.decodeDistance(@bitReverse(try d.takeBits(u5)));
	395	try writeMatch(w, length, distance);
	396	remaining -= length;
	397	},
	398	else => return error.InvalidCode,
	399	}
	400	}
	401	d.state = .fixed_block;
	402	return @intFromEnum(limit) - remaining;
	403	},
	404	.dynamic_block => {
	405	// In larger archives most blocks are usually dynamic, so
	406	// decompression performance depends on this logic.
	407	var sym = try d.decodeSymbol(&d.lit_dec);
	408	sym: switch (sym.kind) {
	409	.literal => {
	410	if (remaining != 0) {
	411	@branchHint(.likely);
	412	remaining -= 1;
	413	try w.writeBytePreserve(flate.history_len, sym.symbol);
	414	sym = try d.decodeSymbol(&d.lit_dec);
	415	continue :sym sym.kind;
	416	} else {
	417	d.state = .{ .dynamic_block_literal = sym.symbol };
	418	return @intFromEnum(limit) - remaining;
	419	}
	420	},
	421	.match => {
	422	// Decode match backreference <length, distance>
	423	const length = try d.decodeLength(sym.symbol);
	424	continue :sw .{ .dynamic_block_match = length };
	425	},
	426	.end_of_block => {
	427	d.state = if (d.final_block) .protocol_footer else .block_header;
	428	continue :sw d.state;
	429	},
	430	}
	431	},
	432	.dynamic_block_literal => \|symbol\| {
	433	assert(remaining != 0);
	434	remaining -= 1;
	435	try w.writeBytePreserve(flate.history_len, symbol);
	436	continue :sw .dynamic_block;
	437	},
	438	.dynamic_block_match => \|length\| {
	439	if (remaining >= length) {
	440	@branchHint(.likely);
	441	remaining -= length;
	442	const dsm = try d.decodeSymbol(&d.dst_dec);
	443	const distance = try d.decodeDistance(dsm.symbol);
	444	try writeMatch(w, length, distance);
	445	continue :sw .dynamic_block;
	446	} else {
	447	d.state = .{ .dynamic_block_match = length };
	448	return @intFromEnum(limit) - remaining;
	449	}
	450	},
	451	.protocol_footer => {
	452	switch (d.container_metadata) {
	453	.gzip => \|*gzip\| {
	454	d.alignBitsDiscarding();
	455	gzip.* = .{
	456	.crc = try in.takeInt(u32, .little),
	457	.count = try in.takeInt(u32, .little),
	458	};
	459	},
	460	.zlib => \|*zlib\| {
	461	d.alignBitsDiscarding();
	462	zlib.* = .{
	463	.adler = try in.takeInt(u32, .little),
	464	};
	465	},
	466	.raw => {
	467	d.alignBitsPreserving();
	468	},
	469	}
	470	d.state = .end;
	471	return @intFromEnum(limit) - remaining;
	472	},
	473	.end => return error.EndOfStream,
	474	}
	475	}
	476
	477	/// Write match (back-reference to the same data slice) starting at `distance`
	478	/// back from current write position, and `length` of bytes.
	479	fn writeMatch(w: *Writer, length: u16, distance: u16) !void {
	480	if (w.end < distance) return error.InvalidMatch;
	481	if (length < Token.base_length) return error.InvalidMatch;
	482	if (length > Token.max_length) return error.InvalidMatch;
	483	if (distance < Token.min_distance) return error.InvalidMatch;
	484	if (distance > Token.max_distance) return error.InvalidMatch;
	485
	486	// This is not a @memmove; it intentionally repeats patterns caused by
	487	// iterating one byte at a time.
	488	const dest = try w.writableSlicePreserve(flate.history_len, length);
	489	const end = dest.ptr - w.buffer.ptr;
	490	const src = w.buffer[end - distance ..][0..length];
	491	for (dest, src) \|d, s\| d. = s;
	492	}
	493
	494	fn takeBits(d: *Decompress, comptime U: type) !U {
	495	const remaining_bits = d.remaining_bits;
	496	const next_bits = d.next_bits;
	497	if (remaining_bits >= @bitSizeOf(U)) {
	498	const u: U = @truncate(next_bits);
	499	d.next_bits = next_bits >> @bitSizeOf(U);
	500	d.remaining_bits = remaining_bits - @bitSizeOf(U);
	501	return u;
	502	}
	503	const in = d.input;
	504	const next_int = in.takeInt(Bits, .little) catch \|err\| switch (err) {
	505	error.ReadFailed => return error.ReadFailed,
	506	error.EndOfStream => return takeBitsEnding(d, U),
	507	};
	508	const needed_bits = @bitSizeOf(U) - remaining_bits;
	509	const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
	510	d.next_bits = next_int >> needed_bits;
	511	d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits));
	512	return u;
	513	}
	514
	515	fn takeBitsEnding(d: *Decompress, comptime U: type) !U {
	516	const remaining_bits = d.remaining_bits;
	517	const next_bits = d.next_bits;
	518	const in = d.input;
	519	const n = in.bufferedLen();
	520	assert(n < @sizeOf(Bits));
	521	const needed_bits = @bitSizeOf(U) - remaining_bits;
	522	if (n * 8 < needed_bits) return error.EndOfStream;
	523	const next_int = in.takeVarInt(Bits, .little, n) catch \|err\| switch (err) {
	524	error.ReadFailed => return error.ReadFailed,
	525	error.EndOfStream => unreachable,
	526	};
	527	const u: U = @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
	528	d.next_bits = next_int >> needed_bits;
	529	d.remaining_bits = @intCast(n * 8 - @as(usize, needed_bits));
	530	return u;
	531	}
	532
	533	fn peekBits(d: *Decompress, comptime U: type) !U {
	534	const remaining_bits = d.remaining_bits;
	535	const next_bits = d.next_bits;
	536	if (remaining_bits >= @bitSizeOf(U)) return @truncate(next_bits);
	537	const in = d.input;
	538	const next_int = in.peekInt(Bits, .little) catch \|err\| switch (err) {
	539	error.ReadFailed => return error.ReadFailed,
	540	error.EndOfStream => return peekBitsEnding(d, U),
	541	};
	542	const needed_bits = @bitSizeOf(U) - remaining_bits;
	543	return @intCast(((next_int & ((@as(Bits, 1) << needed_bits) - 1)) << remaining_bits) \| next_bits);
	544	}
	545
	546	fn peekBitsEnding(d: *Decompress, comptime U: type) !U {
	547	const remaining_bits = d.remaining_bits;
	548	const next_bits = d.next_bits;
	549	const in = d.input;
	550	var u: Bits = 0;
	551	var remaining_needed_bits = @bitSizeOf(U) - remaining_bits;
	552	var i: usize = 0;
	553	while (remaining_needed_bits >= 8) {
	554	const byte = try specialPeek(in, next_bits, i);
	555	u \|= @as(Bits, byte) << @intCast(i * 8);
	556	remaining_needed_bits -= 8;
	557	i += 1;
	558	}
	559	if (remaining_needed_bits != 0) {
	560	const byte = try specialPeek(in, next_bits, i);
	561	u \|= @as(Bits, byte) << @intCast((i * 8) + remaining_needed_bits);
	562	}
	563	return @truncate((u << remaining_bits) \| next_bits);
	564	}
	565
	566	/// If there is any unconsumed data, handles EndOfStream by pretending there
	567	/// are zeroes afterwards.
	568	fn specialPeek(in: *Reader, next_bits: Bits, i: usize) Reader.Error!u8 {
	569	const peeked = in.peek(i + 1) catch \|err\| switch (err) {
	570	error.ReadFailed => return error.ReadFailed,
	571	error.EndOfStream => if (next_bits == 0 and i == 0) return error.EndOfStream else return 0,
	572	};
	573	return peeked[i];
	574	}
	575
	576	fn tossBits(d: *Decompress, n: u4) !void {
	577	const remaining_bits = d.remaining_bits;
	578	const next_bits = d.next_bits;
	579	if (remaining_bits >= n) {
	580	d.next_bits = next_bits >> n;
	581	d.remaining_bits = remaining_bits - n;
	582	} else {
	583	const in = d.input;
	584	const next_int = in.takeInt(Bits, .little) catch \|err\| switch (err) {
	585	error.ReadFailed => return error.ReadFailed,
	586	error.EndOfStream => return tossBitsEnding(d, n),
	587	};
	588	const needed_bits = n - remaining_bits;
	589	d.next_bits = next_int >> needed_bits;
	590	d.remaining_bits = @intCast(@bitSizeOf(Bits) - @as(usize, needed_bits));
	591	}
	592	}
	593
	594	fn tossBitsEnding(d: *Decompress, n: u4) !void {
	595	const remaining_bits = d.remaining_bits;
	596	const in = d.input;
	597	const buffered_n = in.bufferedLen();
	598	if (buffered_n == 0) return error.EndOfStream;
	599	assert(buffered_n < @sizeOf(Bits));
	600	const needed_bits = n - remaining_bits;
	601	const next_int = in.takeVarInt(Bits, .little, buffered_n) catch \|err\| switch (err) {
	602	error.ReadFailed => return error.ReadFailed,
	603	error.EndOfStream => unreachable,
	604	};
	605	d.next_bits = next_int >> needed_bits;
	606	d.remaining_bits = @intCast(@as(usize, n) * 8 -\| @as(usize, needed_bits));
	607	}
	608
	609	fn takeBitsRuntime(d: *Decompress, n: u4) !u16 {
	610	const x = try peekBits(d, u16);
	611	const mask: u16 = (@as(u16, 1) << n) - 1;
	612	const u: u16 = @as(u16, @truncate(x)) & mask;
	613	try tossBits(d, n);
	614	return u;
	615	}
	616
	617	fn alignBitsDiscarding(d: *Decompress) void {
	618	const remaining_bits = d.remaining_bits;
	619	if (remaining_bits == 0) return;
	620	const n_bytes = remaining_bits / 8;
	621	const in = d.input;
	622	in.seek -= n_bytes;
	623	d.remaining_bits = 0;
	624	d.next_bits = 0;
	625	}
	626
	627	fn alignBitsPreserving(d: *Decompress) void {
	628	const remaining_bits: usize = d.remaining_bits;
	629	if (remaining_bits == 0) return;
	630	const n_bytes = (remaining_bits + 7) / 8;
	631	const in = d.input;
	632	in.seek -= n_bytes;
	633	d.remaining_bits = 0;
	634	d.next_bits = 0;
	635	}
	636
	637	/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
	638	/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
	639	/// Lit Value Bits Codes
	640	/// --------- ---- -----
	641	/// 0 - 143 8 00110000 through
	642	/// 10111111
	643	/// 144 - 255 9 110010000 through
	644	/// 111111111
	645	/// 256 - 279 7 0000000 through
	646	/// 0010111
	647	/// 280 - 287 8 11000000 through
	648	/// 11000111
	649	fn readFixedCode(d: *Decompress) !u16 {
	650	const code7 = @bitReverse(try d.takeBits(u7));
	651	return switch (code7) {
	652	0...0b0010_111 => @as(u16, code7) + 256,
	653	0b0010_111 + 1...0b1011_111 => (@as(u16, code7) << 1) + @as(u16, try d.takeBits(u1)) - 0b0011_0000,
	654	0b1011_111 + 1...0b1100_011 => (@as(u16, code7 - 0b1100000) << 1) + try d.takeBits(u1) + 280,
	655	else => (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, @bitReverse(try d.takeBits(u2))) + 144,
	656	};
	657	}
	658
	659	pub const Symbol = packed struct {
	660	pub const Kind = enum(u2) {
	661	literal,
	662	end_of_block,
	663	match,
	664	};
	665
	666	symbol: u8 = 0, // symbol from alphabet
	667	code_bits: u4 = 0, // number of bits in code 0-15
	668	kind: Kind = .literal,
	669
	670	code: u16 = 0, // huffman code of the symbol
	671	next: u16 = 0, // pointer to the next symbol in linked list
	672	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
	673
	674	// Sorting less than function.
	675	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
	676	if (a.code_bits == b.code_bits) {
	677	if (a.kind == b.kind) {
	678	return a.symbol < b.symbol;
	679	}
	680	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
	681	}
	682	return a.code_bits < b.code_bits;
	683	}
	684	};
	685
	686	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
	687	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
	688	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
	689
	690	/// Creates huffman tree codes from list of code lengths (in `build`).
	691	///
	692	/// `find` then finds symbol for code bits. Code can be any length between 1 and
	693	/// 15 bits. When calling `find` we don't know how many bits will be used to
	694	/// find symbol. When symbol is returned it has code_bits field which defines
	695	/// how much we should advance in bit stream.
	696	///
	697	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
	698	/// many times in this table; 32K places for 286 (at most) symbols.
	699	/// Small lookup table is optimization for faster search.
	700	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
	701	/// with difference that we here use statically allocated arrays.
	702	///
	703	fn HuffmanDecoder(
	704	comptime alphabet_size: u16,
	705	comptime max_code_bits: u4,
	706	comptime lookup_bits: u4,
	707	) type {
	708	const lookup_shift = max_code_bits - lookup_bits;
	709
	710	return struct {
	711	// all symbols in alaphabet, sorted by code_len, symbol
	712	symbols: [alphabet_size]Symbol = undefined,
	713	// lookup table code -> symbol
	714	lookup: [1 << lookup_bits]Symbol = undefined,
	715
	716	const Self = @This();
	717
	718	/// Generates symbols and lookup tables from list of code lens for each symbol.
	719	pub fn generate(self: *Self, lens: []const u4) !void {
	720	try checkCompleteness(lens);
	721
	722	// init alphabet with code_bits
	723	for (self.symbols, 0..) \|_, i\| {
	724	const cb: u4 = if (i < lens.len) lens[i] else 0;
	725	self.symbols[i] = if (i < 256)
	726	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
	727	else if (i == 256)
	728	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
	729	else
	730	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
	731	}
	732	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
	733
	734	// reset lookup table
	735	for (0..self.lookup.len) \|i\| {
	736	self.lookup[i] = .{};
	737	}
	738
	739	// assign code to symbols
	740	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
	741	var code: u16 = 0;
	742	var idx: u16 = 0;
	743	for (&self.symbols, 0..) \|*sym, pos\| {
	744	if (sym.code_bits == 0) continue; // skip unused
	745	sym.code = code;
	746
	747	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
	748	const next_idx = next_code >> lookup_shift;
	749
	750	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
	751	if (sym.code_bits <= lookup_bits) {
	752	// fill small lookup table
	753	for (idx..next_idx) \|j\|
	754	self.lookup[j] = sym.*;
	755	} else {
	756	// insert into linked table starting at root
	757	const root = &self.lookup[idx];
	758	const root_next = root.next;
	759	root.next = @intCast(pos);
	760	sym.next = root_next;
	761	}
	762
	763	idx = next_idx;
	764	code = next_code;
	765	}
	766	}
	767
	768	/// Given the list of code lengths check that it represents a canonical
	769	/// Huffman code for n symbols.
	770	///
	771	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
	772	fn checkCompleteness(lens: []const u4) !void {
	773	if (alphabet_size == 286)
	774	if (lens[256] == 0) return error.MissingEndOfBlockCode;
	775
	776	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
	777	var max: usize = 0;
	778	for (lens) \|n\| {
	779	if (n == 0) continue;
	780	if (n > max) max = n;
	781	count[n] += 1;
	782	}
	783	if (max == 0) // empty tree
	784	return;
	785
	786	// check for an over-subscribed or incomplete set of lengths
	787	var left: usize = 1; // one possible code of zero length
	788	for (1..count.len) \|len\| {
	789	left <<= 1; // one more bit, double codes left
	790	if (count[len] > left)
	791	return error.OversubscribedHuffmanTree;
	792	left -= count[len]; // deduct count from possible codes
	793	}
	794	if (left > 0) { // left > 0 means incomplete
	795	// incomplete code ok only for single length 1 code
	796	if (max_code_bits > 7 and max == count[0] + count[1]) return;
	797	return error.IncompleteHuffmanTree;
	798	}
	799	}
	800
	801	/// Finds symbol for lookup table code.
	802	pub fn find(self: *Self, code: u16) !Symbol {
	803	// try to find in lookup table
	804	const idx = code >> lookup_shift;
	805	const sym = self.lookup[idx];
	806	if (sym.code_bits != 0) return sym;
	807	// if not use linked list of symbols with same prefix
	808	return self.findLinked(code, sym.next);
	809	}
	810
	811	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
	812	var pos = start;
	813	while (pos > 0) {
	814	const sym = self.symbols[pos];
	815	const shift = max_code_bits - sym.code_bits;
	816	// compare code_bits number of upper bits
	817	if ((code ^ sym.code) >> shift == 0) return sym;
	818	pos = sym.next;
	819	}
	820	return error.InvalidCode;
	821	}
	822	};
	823	}
	824
	825	test "init/find" {
	826	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
	827	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
	828	var h: CodegenDecoder = .{};
	829	try h.generate(&code_lens);
	830
	831	const expected = [_]struct {
	832	sym: Symbol,
	833	code: u16,
	834	}{
	835	.{
	836	.code = 0b00_00000,
	837	.sym = .{ .symbol = 3, .code_bits = 2 },
	838	},
	839	.{
	840	.code = 0b01_00000,
	841	.sym = .{ .symbol = 18, .code_bits = 2 },
	842	},
	843	.{
	844	.code = 0b100_0000,
	845	.sym = .{ .symbol = 1, .code_bits = 3 },
	846	},
	847	.{
	848	.code = 0b101_0000,
	849	.sym = .{ .symbol = 4, .code_bits = 3 },
	850	},
	851	.{
	852	.code = 0b110_0000,
	853	.sym = .{ .symbol = 17, .code_bits = 3 },
	854	},
	855	.{
	856	.code = 0b1110_000,
	857	.sym = .{ .symbol = 0, .code_bits = 4 },
	858	},
	859	.{
	860	.code = 0b1111_000,
	861	.sym = .{ .symbol = 16, .code_bits = 4 },
	862	},
	863	};
	864
	865	// unused symbols
	866	for (0..12) \|i\| {
	867	try testing.expectEqual(0, h.symbols[i].code_bits);
	868	}
	869	// used, from index 12
	870	for (expected, 12..) \|e, i\| {
	871	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
	872	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
	873	const sym_from_code = try h.find(e.code);
	874	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
	875	}
	876
	877	// All possible codes for each symbol.
	878	// Lookup table has 126 elements, to cover all possible 7 bit codes.
	879	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
	880	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
	881
	882	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
	883	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
	884
	885	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
	886	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
	887
	888	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
	889	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
	890
	891	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
	892	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
	893
	894	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
	895	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
	896
	897	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
	898	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
	899	}
	900
	901	test "encode/decode literals" {
	902	var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;
	903	for (1..286) \|j\| { // for all different number of codes
	904	var enc: flate.HuffmanEncoder = .{
	905	.codes = &codes,
	906	.freq_cache = undefined,
	907	.bit_count = undefined,
	908	.lns = undefined,
	909	.lfs = undefined,
	910	};
	911	// create frequencies
	912	var freq = [_]u16{0} ** 286;
	913	freq[256] = 1; // ensure we have end of block code
	914	for (&freq, 1..) \|*f, i\| {
	915	if (i % j == 0)
	916	f.* = @intCast(i);
	917	}
	918
	919	// encoder from frequencies
	920	enc.generate(&freq, 15);
	921
	922	// get code_lens from encoder
	923	var code_lens = [_]u4{0} ** 286;
	924	for (code_lens, 0..) \|_, i\| {
	925	code_lens[i] = @intCast(enc.codes[i].len);
	926	}
	927	// generate decoder from code lens
	928	var dec: LiteralDecoder = .{};
	929	try dec.generate(&code_lens);
	930
	931	// expect decoder code to match original encoder code
	932	for (dec.symbols) \|s\| {
	933	if (s.code_bits == 0) continue;
	934	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
	935	const symbol: u16 = switch (s.kind) {
	936	.literal => s.symbol,
	937	.end_of_block => 256,
	938	.match => @as(u16, s.symbol) + 257,
	939	};
	940
	941	const c = enc.codes[symbol];
	942	try testing.expect(c.code == c_code);
	943	}
	944
	945	// find each symbol by code
	946	for (enc.codes) \|c\| {
	947	if (c.len == 0) continue;
	948
	949	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
	950	const s = try dec.find(s_code);
	951	try testing.expect(s.code == s_code);
	952	try testing.expect(s.code_bits == c.len);
	953	}
	954	}
	955	}
	956
	957	test "non compressed block (type 0)" {
	958	try testDecompress(.raw, &[_]u8{
	959	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
	960	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
	961	}, "Hello world\n");
	962	}
	963
	964	test "fixed code block (type 1)" {
	965	try testDecompress(.raw, &[_]u8{
	966	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
	967	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
	968	}, "Hello world\n");
	969	}
	970
	971	test "dynamic block (type 2)" {
	972	try testDecompress(.raw, &[_]u8{
	973	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
	974	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
	975	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
	976	}, "ABCDEABCD ABCDEABCD");
	977	}
	978
	979	test "gzip non compressed block (type 0)" {
	980	try testDecompress(.gzip, &[_]u8{
	981	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
	982	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
	983	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
	984	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
	985	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
	986	}, "Hello world\n");
	987	}
	988
	989	test "gzip fixed code block (type 1)" {
	990	try testDecompress(.gzip, &[_]u8{
	991	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
	992	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
	993	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
	994	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
	995	}, "Hello world\n");
	996	}
	997
	998	test "gzip dynamic block (type 2)" {
	999	try testDecompress(.gzip, &[_]u8{
	1000	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
	1001	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
	1002	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
	1003	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
	1004	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
	1005	}, "ABCDEABCD ABCDEABCD");
	1006	}
	1007
	1008	test "gzip header with name" {
	1009	try testDecompress(.gzip, &[_]u8{
	1010	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
	1011	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
	1012	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
	1013	}, "Hello world\n");
	1014	}
	1015
	1016	test "zlib decompress non compressed block (type 0)" {
	1017	try testDecompress(.zlib, &[_]u8{
	1018	0x78, 0b10_0_11100, // zlib header (2 bytes)
	1019	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
	1020	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
	1021	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
	1022	}, "Hello world\n");
	1023	}
	1024
	1025	test "failing end-of-stream" {
	1026	try testFailure(.raw, @embedFile("testdata/fuzz/end-of-stream.input"), error.EndOfStream);
	1027	}
	1028	test "failing invalid-distance" {
	1029	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-distance.input"), error.InvalidMatch);
	1030	}
	1031	test "failing invalid-tree01" {
	1032	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree01.input"), error.IncompleteHuffmanTree);
	1033	}
	1034	test "failing invalid-tree02" {
	1035	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree02.input"), error.EndOfStream);
	1036	}
	1037	test "failing invalid-tree03" {
	1038	try testFailure(.raw, @embedFile("testdata/fuzz/invalid-tree03.input"), error.IncompleteHuffmanTree);
	1039	}
	1040	test "failing lengths-overflow" {
	1041	try testFailure(.raw, @embedFile("testdata/fuzz/lengths-overflow.input"), error.InvalidDynamicBlockHeader);
	1042	}
	1043	test "failing out-of-codes" {
	1044	try testFailure(.raw, @embedFile("testdata/fuzz/out-of-codes.input"), error.InvalidCode);
	1045	}
	1046	test "failing puff01" {
	1047	try testFailure(.raw, @embedFile("testdata/fuzz/puff01.input"), error.WrongStoredBlockNlen);
	1048	}
	1049	test "failing puff02" {
	1050	try testFailure(.raw, @embedFile("testdata/fuzz/puff02.input"), error.EndOfStream);
	1051	}
	1052	test "failing puff04" {
	1053	try testFailure(.raw, @embedFile("testdata/fuzz/puff04.input"), error.InvalidCode);
	1054	}
	1055	test "failing puff05" {
	1056	try testFailure(.raw, @embedFile("testdata/fuzz/puff05.input"), error.EndOfStream);
	1057	}
	1058	test "failing puff06" {
	1059	try testFailure(.raw, @embedFile("testdata/fuzz/puff06.input"), error.EndOfStream);
	1060	}
	1061	test "failing puff08" {
	1062	try testFailure(.raw, @embedFile("testdata/fuzz/puff08.input"), error.InvalidCode);
	1063	}
	1064	test "failing puff10" {
	1065	try testFailure(.raw, @embedFile("testdata/fuzz/puff10.input"), error.InvalidCode);
	1066	}
	1067	test "failing puff11" {
	1068	try testFailure(.raw, @embedFile("testdata/fuzz/puff11.input"), error.EndOfStream);
	1069	}
	1070	test "failing puff12" {
	1071	try testFailure(.raw, @embedFile("testdata/fuzz/puff12.input"), error.InvalidDynamicBlockHeader);
	1072	}
	1073	test "failing puff13" {
	1074	try testFailure(.raw, @embedFile("testdata/fuzz/puff13.input"), error.IncompleteHuffmanTree);
	1075	}
	1076	test "failing puff14" {
	1077	try testFailure(.raw, @embedFile("testdata/fuzz/puff14.input"), error.EndOfStream);
	1078	}
	1079	test "failing puff15" {
	1080	try testFailure(.raw, @embedFile("testdata/fuzz/puff15.input"), error.IncompleteHuffmanTree);
	1081	}
	1082	test "failing puff16" {
	1083	try testFailure(.raw, @embedFile("testdata/fuzz/puff16.input"), error.InvalidDynamicBlockHeader);
	1084	}
	1085	test "failing puff17" {
	1086	try testFailure(.raw, @embedFile("testdata/fuzz/puff17.input"), error.MissingEndOfBlockCode);
	1087	}
	1088	test "failing fuzz1" {
	1089	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz1.input"), error.InvalidDynamicBlockHeader);
	1090	}
	1091	test "failing fuzz2" {
	1092	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz2.input"), error.InvalidDynamicBlockHeader);
	1093	}
	1094	test "failing fuzz3" {
	1095	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz3.input"), error.InvalidMatch);
	1096	}
	1097	test "failing fuzz4" {
	1098	try testFailure(.raw, @embedFile("testdata/fuzz/fuzz4.input"), error.OversubscribedHuffmanTree);
	1099	}
	1100	test "failing puff18" {
	1101	try testFailure(.raw, @embedFile("testdata/fuzz/puff18.input"), error.OversubscribedHuffmanTree);
	1102	}
	1103	test "failing puff19" {
	1104	try testFailure(.raw, @embedFile("testdata/fuzz/puff19.input"), error.OversubscribedHuffmanTree);
	1105	}
	1106	test "failing puff20" {
	1107	try testFailure(.raw, @embedFile("testdata/fuzz/puff20.input"), error.OversubscribedHuffmanTree);
	1108	}
	1109	test "failing puff21" {
	1110	try testFailure(.raw, @embedFile("testdata/fuzz/puff21.input"), error.OversubscribedHuffmanTree);
	1111	}
	1112	test "failing puff22" {
	1113	try testFailure(.raw, @embedFile("testdata/fuzz/puff22.input"), error.OversubscribedHuffmanTree);
	1114	}
	1115	test "failing puff23" {
	1116	try testFailure(.raw, @embedFile("testdata/fuzz/puff23.input"), error.OversubscribedHuffmanTree);
	1117	}
	1118	test "failing puff24" {
	1119	try testFailure(.raw, @embedFile("testdata/fuzz/puff24.input"), error.IncompleteHuffmanTree);
	1120	}
	1121	test "failing puff25" {
	1122	try testFailure(.raw, @embedFile("testdata/fuzz/puff25.input"), error.OversubscribedHuffmanTree);
	1123	}
	1124	test "failing puff26" {
	1125	try testFailure(.raw, @embedFile("testdata/fuzz/puff26.input"), error.InvalidDynamicBlockHeader);
	1126	}
	1127	test "failing puff27" {
	1128	try testFailure(.raw, @embedFile("testdata/fuzz/puff27.input"), error.InvalidDynamicBlockHeader);
	1129	}
	1130
	1131	test "deflate-stream" {
	1132	try testDecompress(
	1133	.raw,
	1134	@embedFile("testdata/fuzz/deflate-stream.input"),
	1135	@embedFile("testdata/fuzz/deflate-stream.expect"),
	1136	);
	1137	}
	1138
	1139	test "empty-distance-alphabet01" {
	1140	try testFailure(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet01.input"), error.EndOfStream);
	1141	}
	1142
	1143	test "empty-distance-alphabet02" {
	1144	try testDecompress(.raw, @embedFile("testdata/fuzz/empty-distance-alphabet02.input"), "");
	1145	}
	1146
	1147	test "puff03" {
	1148	try testDecompress(.raw, @embedFile("testdata/fuzz/puff03.input"), &.{0xa});
	1149	}
	1150
	1151	test "puff09" {
	1152	try testDecompress(.raw, @embedFile("testdata/fuzz/puff09.input"), "P");
	1153	}
	1154
	1155	test "bug 18966" {
	1156	try testDecompress(
	1157	.gzip,
	1158	@embedFile("testdata/fuzz/bug_18966.input"),
	1159	@embedFile("testdata/fuzz/bug_18966.expect"),
	1160	);
	1161	}
	1162
	1163	test "reading into empty buffer" {
	1164	// Inspired by https://github.com/ziglang/zig/issues/19895
	1165	const input = &[_]u8{
	1166	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
	1167	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
	1168	};
	1169	var in: Reader = .fixed(input);
	1170	var decomp: Decompress = .init(&in, .raw, &.{});
	1171	const r = &decomp.reader;
	1172	var bufs: [1][]u8 = .{&.{}};
	1173	try testing.expectEqual(0, try r.readVec(&bufs));
	1174	}
	1175
	1176	test "zlib header" {
	1177	// Truncated header
	1178	try testFailure(.zlib, &[_]u8{0x78}, error.EndOfStream);
	1179
	1180	// Wrong CM
	1181	try testFailure(.zlib, &[_]u8{ 0x79, 0x94 }, error.BadZlibHeader);
	1182
	1183	// Wrong CINFO
	1184	try testFailure(.zlib, &[_]u8{ 0x88, 0x98 }, error.BadZlibHeader);
	1185
	1186	// Truncated checksum
	1187	try testFailure(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, error.EndOfStream);
	1188	}
	1189
	1190	test "gzip header" {
	1191	// Truncated header
	1192	try testFailure(.gzip, &[_]u8{ 0x1f, 0x8B }, error.EndOfStream);
	1193
	1194	// Wrong CM
	1195	try testFailure(.gzip, &[_]u8{
	1196	0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
	1197	0x00, 0x03,
	1198	}, error.BadGzipHeader);
	1199
	1200	// Truncated checksum
	1201	try testFailure(.gzip, &[_]u8{
	1202	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
	1203	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
	1204	}, error.EndOfStream);
	1205
	1206	// Truncated initial size field
	1207	try testFailure(.gzip, &[_]u8{
	1208	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
	1209	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
	1210	0x00, 0x00, 0x00,
	1211	}, error.EndOfStream);
	1212
	1213	try testDecompress(.gzip, &[_]u8{
	1214	// GZIP header
	1215	0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
	1216	// header.FHCRC (should cover entire header)
	1217	0x99, 0xd6,
	1218	// GZIP data
	1219	0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	1220	}, "");
	1221	}
	1222
	1223	test "zlib should not overshoot" {
	1224	// Compressed zlib data with extra 4 bytes at the end.
	1225	const data = [_]u8{
	1226	0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
	1227	0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
	1228	0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
	1229	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
	1230	};
	1231
	1232	var reader: std.Io.Reader = .fixed(&data);
	1233
	1234	var decompress_buffer: [flate.max_window_len]u8 = undefined;
	1235	var decompress: Decompress = .init(&reader, .zlib, &decompress_buffer);
	1236	var out: [128]u8 = undefined;
	1237
	1238	{
	1239	const n = try decompress.reader.readSliceShort(&out);
	1240	try std.testing.expectEqual(46, n);
	1241	try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
	1242	}
	1243
	1244	// 4 bytes after compressed chunk are available in reader.
	1245	const n = try reader.readSliceShort(&out);
	1246	try std.testing.expectEqual(n, 4);
	1247	try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
	1248	}
	1249
	1250	fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !void {
	1251	var reader: Reader = .fixed(in);
	1252	var aw: Writer.Allocating = .init(testing.allocator);
	1253	try aw.ensureUnusedCapacity(flate.history_len);
	1254	defer aw.deinit();
	1255
	1256	var decompress: Decompress = .init(&reader, container, &.{});
	1257	try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer));
	1258	try testing.expectEqual(expected_err, decompress.err orelse return error.TestFailed);
	1259	}
	1260
	1261	fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
	1262	var in: std.Io.Reader = .fixed(compressed);
	1263	var aw: std.Io.Writer.Allocating = .init(testing.allocator);
	1264	try aw.ensureUnusedCapacity(flate.history_len);
	1265	defer aw.deinit();
	1266
	1267	var decompress: Decompress = .init(&in, container, &.{});
	1268	_ = try decompress.reader.streamRemaining(&aw.writer);
	1269	try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
	1270	}

lib/std/compress/flate/HuffmanEncoder.zig created+463

...	...	@@ -0,0 +1,463 @@
	1	const HuffmanEncoder = @This();
	2	const std = @import("std");
	3	const assert = std.debug.assert;
	4	const testing = std.testing;
	5
	6	codes: []Code,
	7	// Reusable buffer with the longest possible frequency table.
	8	freq_cache: [max_num_frequencies + 1]LiteralNode,
	9	bit_count: [17]u32,
	10	lns: []LiteralNode, // sorted by literal, stored to avoid repeated allocation in generate
	11	lfs: []LiteralNode, // sorted by frequency, stored to avoid repeated allocation in generate
	12
	13	pub const LiteralNode = struct {
	14	literal: u16,
	15	freq: u16,
	16
	17	pub fn max() LiteralNode {
	18	return .{
	19	.literal = std.math.maxInt(u16),
	20	.freq = std.math.maxInt(u16),
	21	};
	22	}
	23	};
	24
	25	pub const Code = struct {
	26	code: u16 = 0,
	27	len: u16 = 0,
	28	};
	29
	30	/// The odd order in which the codegen code sizes are written.
	31	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
	32	/// The number of codegen codes.
	33	pub const codegen_code_count = 19;
	34
	35	/// The largest distance code.
	36	pub const distance_code_count = 30;
	37
	38	/// Maximum number of literals.
	39	pub const max_num_lit = 286;
	40
	41	/// Max number of frequencies used for a Huffman Code
	42	/// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
	43	/// The largest of these is max_num_lit.
	44	pub const max_num_frequencies = max_num_lit;
	45
	46	/// Biggest block size for uncompressed block.
	47	pub const max_store_block_size = 65535;
	48	/// The special code used to mark the end of a block.
	49	pub const end_block_marker = 256;
	50
	51	/// Update this Huffman Code object to be the minimum code for the specified frequency count.
	52	///
	53	/// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
	54	/// max_bits The maximum number of bits to use for any literal.
	55	pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
	56	var list = self.freq_cache[0 .. freq.len + 1];
	57	// Number of non-zero literals
	58	var count: u32 = 0;
	59	// Set list to be the set of all non-zero literals and their frequencies
	60	for (freq, 0..) \|f, i\| {
	61	if (f != 0) {
	62	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
	63	count += 1;
	64	} else {
	65	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
	66	self.codes[i].len = 0;
	67	}
	68	}
	69	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
	70
	71	list = list[0..count];
	72	if (count <= 2) {
	73	// Handle the small cases here, because they are awkward for the general case code. With
	74	// two or fewer literals, everything has bit length 1.
	75	for (list, 0..) \|node, i\| {
	76	// "list" is in order of increasing literal value.
	77	self.codes[node.literal] = .{
	78	.code = @intCast(i),
	79	.len = 1,
	80	};
	81	}
	82	return;
	83	}
	84	self.lfs = list;
	85	std.mem.sort(LiteralNode, self.lfs, {}, byFreq);
	86
	87	// Get the number of literals for each bit count
	88	const bit_count = self.bitCounts(list, max_bits);
	89	// And do the assignment
	90	self.assignEncodingAndSize(bit_count, list);
	91	}
	92
	93	pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
	94	var total: u32 = 0;
	95	for (freq, 0..) \|f, i\| {
	96	if (f != 0) {
	97	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
	98	}
	99	}
	100	return total;
	101	}
	102
	103	/// Return the number of literals assigned to each bit size in the Huffman encoding
	104	///
	105	/// This method is only called when list.len >= 3
	106	/// The cases of 0, 1, and 2 literals are handled by special case code.
	107	///
	108	/// list: An array of the literals with non-zero frequencies
	109	/// and their associated frequencies. The array is in order of increasing
	110	/// frequency, and has as its last element a special element with frequency
	111	/// `math.maxInt(i32)`
	112	///
	113	/// max_bits: The maximum number of bits that should be used to encode any literal.
	114	/// Must be less than 16.
	115	///
	116	/// Returns an integer array in which array[i] indicates the number of literals
	117	/// that should be encoded in i bits.
	118	fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
	119	var max_bits = max_bits_to_use;
	120	const n = list.len;
	121	const max_bits_limit = 16;
	122
	123	assert(max_bits < max_bits_limit);
	124
	125	// The tree can't have greater depth than n - 1, no matter what. This
	126	// saves a little bit of work in some small cases
	127	max_bits = @min(max_bits, n - 1);
	128
	129	// Create information about each of the levels.
	130	// A bogus "Level 0" whose sole purpose is so that
	131	// level1.prev.needed == 0. This makes level1.next_pair_freq
	132	// be a legitimate value that never gets chosen.
	133	var levels: [max_bits_limit]LevelInfo = std.mem.zeroes([max_bits_limit]LevelInfo);
	134	// leaf_counts[i] counts the number of literals at the left
	135	// of ancestors of the rightmost node at level i.
	136	// leaf_counts[i][j] is the number of literals at the left
	137	// of the level j ancestor.
	138	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(@splat(0));
	139
	140	{
	141	var level = @as(u32, 1);
	142	while (level <= max_bits) : (level += 1) {
	143	// For every level, the first two items are the first two characters.
	144	// We initialize the levels as if we had already figured this out.
	145	levels[level] = LevelInfo{
	146	.level = level,
	147	.last_freq = list[1].freq,
	148	.next_char_freq = list[2].freq,
	149	.next_pair_freq = list[0].freq + list[1].freq,
	150	.needed = 0,
	151	};
	152	leaf_counts[level][level] = 2;
	153	if (level == 1) {
	154	levels[level].next_pair_freq = std.math.maxInt(i32);
	155	}
	156	}
	157	}
	158
	159	// We need a total of 2*n - 2 items at top level and have already generated 2.
	160	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
	161
	162	{
	163	var level = max_bits;
	164	while (true) {
	165	var l = &levels[level];
	166	if (l.next_pair_freq == std.math.maxInt(i32) and l.next_char_freq == std.math.maxInt(i32)) {
	167	// We've run out of both leaves and pairs.
	168	// End all calculations for this level.
	169	// To make sure we never come back to this level or any lower level,
	170	// set next_pair_freq impossibly large.
	171	l.needed = 0;
	172	levels[level + 1].next_pair_freq = std.math.maxInt(i32);
	173	level += 1;
	174	continue;
	175	}
	176
	177	const prev_freq = l.last_freq;
	178	if (l.next_char_freq < l.next_pair_freq) {
	179	// The next item on this row is a leaf node.
	180	const next = leaf_counts[level][level] + 1;
	181	l.last_freq = l.next_char_freq;
	182	// Lower leaf_counts are the same of the previous node.
	183	leaf_counts[level][level] = next;
	184	if (next >= list.len) {
	185	l.next_char_freq = LiteralNode.max().freq;
	186	} else {
	187	l.next_char_freq = list[next].freq;
	188	}
	189	} else {
	190	// The next item on this row is a pair from the previous row.
	191	// next_pair_freq isn't valid until we generate two
	192	// more values in the level below
	193	l.last_freq = l.next_pair_freq;
	194	// Take leaf counts from the lower level, except counts[level] remains the same.
	195	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
	196	levels[l.level - 1].needed = 2;
	197	}
	198
	199	l.needed -= 1;
	200	if (l.needed == 0) {
	201	// We've done everything we need to do for this level.
	202	// Continue calculating one level up. Fill in next_pair_freq
	203	// of that level with the sum of the two nodes we've just calculated on
	204	// this level.
	205	if (l.level == max_bits) {
	206	// All done!
	207	break;
	208	}
	209	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
	210	level += 1;
	211	} else {
	212	// If we stole from below, move down temporarily to replenish it.
	213	while (levels[level - 1].needed > 0) {
	214	level -= 1;
	215	if (level == 0) {
	216	break;
	217	}
	218	}
	219	}
	220	}
	221	}
	222
	223	// Somethings is wrong if at the end, the top level is null or hasn't used
	224	// all of the leaves.
	225	assert(leaf_counts[max_bits][max_bits] == n);
	226
	227	var bit_count = self.bit_count[0 .. max_bits + 1];
	228	var bits: u32 = 1;
	229	const counts = &leaf_counts[max_bits];
	230	{
	231	var level = max_bits;
	232	while (level > 0) : (level -= 1) {
	233	// counts[level] gives the number of literals requiring at least "bits"
	234	// bits to encode.
	235	bit_count[bits] = counts[level] - counts[level - 1];
	236	bits += 1;
	237	if (level == 0) {
	238	break;
	239	}
	240	}
	241	}
	242	return bit_count;
	243	}
	244
	245	/// Look at the leaves and assign them a bit count and an encoding as specified
	246	/// in RFC 1951 3.2.2
	247	fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
	248	var code = @as(u16, 0);
	249	var list = list_arg;
	250
	251	for (bit_count, 0..) \|bits, n\| {
	252	code <<= 1;
	253	if (n == 0 or bits == 0) {
	254	continue;
	255	}
	256	// The literals list[list.len-bits] .. list[list.len-bits]
	257	// are encoded using "bits" bits, and get the values
	258	// code, code + 1, .... The code values are
	259	// assigned in literal order (not frequency order).
	260	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
	261
	262	self.lns = chunk;
	263	std.mem.sort(LiteralNode, self.lns, {}, byLiteral);
	264
	265	for (chunk) \|node\| {
	266	self.codes[node.literal] = .{
	267	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
	268	.len = @as(u16, @intCast(n)),
	269	};
	270	code += 1;
	271	}
	272	list = list[0 .. list.len - @as(u32, @intCast(bits))];
	273	}
	274	}
	275
	276	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
	277	_ = context;
	278	if (a.freq == b.freq) {
	279	return a.literal < b.literal;
	280	}
	281	return a.freq < b.freq;
	282	}
	283
	284	/// Describes the state of the constructed tree for a given depth.
	285	const LevelInfo = struct {
	286	/// Our level. for better printing
	287	level: u32,
	288	/// The frequency of the last node at this level
	289	last_freq: u32,
	290	/// The frequency of the next character to add to this level
	291	next_char_freq: u32,
	292	/// The frequency of the next pair (from level below) to add to this level.
	293	/// Only valid if the "needed" value of the next lower level is 0.
	294	next_pair_freq: u32,
	295	/// The number of chains remaining to generate for this level before moving
	296	/// up to the next level
	297	needed: u32,
	298	};
	299
	300	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
	301	_ = context;
	302	return a.literal < b.literal;
	303	}
	304
	305	/// Reverse bit-by-bit a N-bit code.
	306	fn bitReverse(comptime T: type, value: T, n: usize) T {
	307	const r = @bitReverse(value);
	308	return r >> @as(std.math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
	309	}
	310
	311	test bitReverse {
	312	const ReverseBitsTest = struct {
	313	in: u16,
	314	bit_count: u5,
	315	out: u16,
	316	};
	317
	318	const reverse_bits_tests = [_]ReverseBitsTest{
	319	.{ .in = 1, .bit_count = 1, .out = 1 },
	320	.{ .in = 1, .bit_count = 2, .out = 2 },
	321	.{ .in = 1, .bit_count = 3, .out = 4 },
	322	.{ .in = 1, .bit_count = 4, .out = 8 },
	323	.{ .in = 1, .bit_count = 5, .out = 16 },
	324	.{ .in = 17, .bit_count = 5, .out = 17 },
	325	.{ .in = 257, .bit_count = 9, .out = 257 },
	326	.{ .in = 29, .bit_count = 5, .out = 23 },
	327	};
	328
	329	for (reverse_bits_tests) \|h\| {
	330	const v = bitReverse(u16, h.in, h.bit_count);
	331	try std.testing.expectEqual(h.out, v);
	332	}
	333	}
	334
	335	/// Generates a HuffmanCode corresponding to the fixed literal table
	336	pub fn fixedLiteralEncoder(codes: *[max_num_frequencies]Code) HuffmanEncoder {
	337	var h: HuffmanEncoder = undefined;
	338	h.codes = codes;
	339	var ch: u16 = 0;
	340
	341	while (ch < max_num_frequencies) : (ch += 1) {
	342	var bits: u16 = undefined;
	343	var size: u16 = undefined;
	344	switch (ch) {
	345	0...143 => {
	346	// size 8, 000110000 .. 10111111
	347	bits = ch + 48;
	348	size = 8;
	349	},
	350	144...255 => {
	351	// size 9, 110010000 .. 111111111
	352	bits = ch + 400 - 144;
	353	size = 9;
	354	},
	355	256...279 => {
	356	// size 7, 0000000 .. 0010111
	357	bits = ch - 256;
	358	size = 7;
	359	},
	360	else => {
	361	// size 8, 11000000 .. 11000111
	362	bits = ch + 192 - 280;
	363	size = 8;
	364	},
	365	}
	366	h.codes[ch] = .{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
	367	}
	368	return h;
	369	}
	370
	371	pub fn fixedDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
	372	var h: HuffmanEncoder = undefined;
	373	h.codes = codes;
	374	for (h.codes, 0..) \|_, ch\| {
	375	h.codes[ch] = .{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
	376	}
	377	return h;
	378	}
	379
	380	pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
	381	var distance_freq: [distance_code_count]u16 = @splat(0);
	382	distance_freq[0] = 1;
	383	// huff_distance is a static distance encoder used for huffman only encoding.
	384	// It can be reused since we will not be encoding distance values.
	385	var h: HuffmanEncoder = .{};
	386	h.codes = codes;
	387	h.generate(distance_freq[0..], 15);
	388	return h;
	389	}
	390
	391	test "generate a Huffman code for the fixed literal table specific to Deflate" {
	392	var codes: [max_num_frequencies]Code = undefined;
	393	const enc: HuffmanEncoder = .fixedLiteralEncoder(&codes);
	394	for (enc.codes) \|c\| {
	395	switch (c.len) {
	396	7 => {
	397	const v = @bitReverse(@as(u7, @intCast(c.code)));
	398	try testing.expect(v <= 0b0010111);
	399	},
	400	8 => {
	401	const v = @bitReverse(@as(u8, @intCast(c.code)));
	402	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
	403	(v >= 0b11000000 and v <= 11000111));
	404	},
	405	9 => {
	406	const v = @bitReverse(@as(u9, @intCast(c.code)));
	407	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
	408	},
	409	else => unreachable,
	410	}
	411	}
	412	}
	413
	414	test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
	415	var codes: [distance_code_count]Code = undefined;
	416	const enc = fixedDistanceEncoder(&codes);
	417	for (enc.codes) \|c\| {
	418	const v = @bitReverse(@as(u5, @intCast(c.code)));
	419	try testing.expect(v <= 29);
	420	try testing.expect(c.len == 5);
	421	}
	422	}
	423
	424	pub const fixed_codes = [_]u8{
	425	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
	426	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
	427	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
	428	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
	429	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
	430	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
	431	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
	432	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
	433	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
	434	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
	435	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
	436	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
	437	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
	438	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
	439	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
	440	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
	441	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
	442	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
	443	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
	444	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
	445	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
	446	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
	447	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
	448	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
	449	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
	450	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
	451	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
	452	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
	453	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
	454	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
	455	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
	456	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
	457	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
	458	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
	459	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
	460	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
	461	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
	462	0b10100011,
	463	};

lib/std/compress/flate/Lookup.zig+21-16

...	...	@@ -5,22 +5,27 @@
5	5	const std = @import("std");
6	6	const testing = std.testing;
7	7	const expect = testing.expect;
8		const consts = @import("consts.zig");
	8	const flate = @import("../flate.zig");
	9	const Token = @import("Token.zig");
9	10
10		const Self = @This();
	11	const Lookup = @This();
11	12
12	13	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13		const chain_len = 2 * consts.history.len;
	14	const chain_len = 2 * flate.history_len;
	15
	16	pub const bits = 15;
	17	pub const len = 1 << bits;
	18	pub const shift = 32 - bits;
14	19
15	20	// Maps hash => first position
16		head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,
	21	head: [len]u16 = [_]u16{0} ** len,
17	22	// Maps position => previous positions for the same hash value
18	23	chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
19	24
20	25	// Calculates hash of the 4 bytes from data.
21	26	// Inserts `pos` position of that hash in the lookup tables.
22	27	// Returns previous location with the same hash value.
23		pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
	28	pub fn add(self: *Lookup, data: []const u8, pos: u16) u16 {
24	29	if (data.len < 4) return 0;
25	30	const h = hash(data[0..4]);
26	31	return self.set(h, pos);
...	...	@@ -28,11 +33,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
28	33
29	34	// Returns previous location with the same hash value given the current
30	35	// position.
31		pub fn prev(self: *Self, pos: u16) u16 {
	36	pub fn prev(self: *Lookup, pos: u16) u16 {
32	37	return self.chain[pos];
33	38	}
34	39
35		fn set(self: *Self, h: u32, pos: u16) u16 {
	40	fn set(self: *Lookup, h: u32, pos: u16) u16 {
36	41	const p = self.head[h];
37	42	self.head[h] = pos;
38	43	self.chain[pos] = p;
...	...	@@ -40,7 +45,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 {
40	45	}
41	46
42	47	// Slide all positions in head and chain for `n`
43		pub fn slide(self: *Self, n: u16) void {
	48	pub fn slide(self: *Lookup, n: u16) void {
44	49	for (&self.head) \|*v\| {
45	50	v.* -\|= n;
46	51	}
...	...	@@ -52,8 +57,8 @@ pub fn slide(self: *Self, n: u16) void {
52	57
53	58	// Add `len` 4 bytes hashes from `data` into lookup.
54	59	// Position of the first byte is `pos`.
55		pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
56		if (len == 0 or data.len < consts.match.min_length) {
	60	pub fn bulkAdd(self: *Lookup, data: []const u8, length: u16, pos: u16) void {
	61	if (length == 0 or data.len < Token.min_length) {
57	62	return;
58	63	}
59	64	var hb =
...	...	@@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
64	69	_ = self.set(hashu(hb), pos);
65	70
66	71	var i = pos;
67		for (4..@min(len + 3, data.len)) \|j\| {
	72	for (4..@min(length + 3, data.len)) \|j\| {
68	73	hb = (hb << 8) \| @as(u32, data[j]);
69	74	i += 1;
70	75	_ = self.set(hashu(hb), i);
...	...	@@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 {
80	85	}
81	86
82	87	fn hashu(v: u32) u32 {
83		return @intCast((v *% prime4) >> consts.lookup.shift);
	88	return @intCast((v *% prime4) >> shift);
84	89	}
85	90
86	91	test add {
...	...	@@ -91,7 +96,7 @@ test add {
91	96	0x01, 0x02, 0x03,
92	97	};
93	98
94		var h: Self = .{};
	99	var h: Lookup = .{};
95	100	for (data, 0..) \|_, i\| {
96	101	const p = h.add(data[i..], @intCast(i));
97	102	if (i >= 8 and i < 24) {
...	...	@@ -101,7 +106,7 @@ test add {
101	106	}
102	107	}
103	108
104		const v = Self.hash(data[2 .. 2 + 4]);
	109	const v = Lookup.hash(data[2 .. 2 + 4]);
105	110	try expect(h.head[v] == 2 + 16);
106	111	try expect(h.chain[2 + 16] == 2 + 8);
107	112	try expect(h.chain[2 + 8] == 2);
...	...	@@ -111,13 +116,13 @@ test bulkAdd {
111	116	const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
112	117
113	118	// one by one
114		var h: Self = .{};
	119	var h: Lookup = .{};
115	120	for (data, 0..) \|_, i\| {
116	121	_ = h.add(data[i..], @intCast(i));
117	122	}
118	123
119	124	// in bulk
120		var bh: Self = .{};
	125	var bh: Lookup = .{};
121	126	bh.bulkAdd(data, data.len, 0);
122	127
123	128	try testing.expectEqualSlices(u16, &h.head, &bh.head);

lib/std/compress/flate/SlidingWindow.zig deleted-160

...	...	@@ -1,160 +0,0 @@
1		//! Used in deflate (compression), holds uncompressed data form which Tokens are
2		//! produces. In combination with Lookup it is used to find matches in history data.
3		//!
4		const std = @import("std");
5		const consts = @import("consts.zig");
6
7		const expect = testing.expect;
8		const assert = std.debug.assert;
9		const testing = std.testing;
10
11		const hist_len = consts.history.len;
12		const buffer_len = 2 * hist_len;
13		const min_lookahead = consts.match.min_length + consts.match.max_length;
14		const max_rp = buffer_len - min_lookahead;
15
16		const Self = @This();
17
18		buffer: [buffer_len]u8 = undefined,
19		wp: usize = 0, // write position
20		rp: usize = 0, // read position
21		fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23		/// Returns number of bytes written, or 0 if buffer is full and need to slide.
24		pub fn write(self: *Self, buf: []const u8) usize {
25		if (self.rp >= max_rp) return 0; // need to slide
26
27		const n = @min(buf.len, buffer_len - self.wp);
28		@memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29		self.wp += n;
30		return n;
31		}
32
33		/// Slide buffer for hist_len.
34		/// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35		/// Returns number of bytes removed.
36		pub fn slide(self: *Self) u16 {
37		assert(self.rp >= max_rp and self.wp >= self.rp);
38		const n = self.wp - hist_len;
39		@memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40		self.rp -= hist_len;
41		self.wp -= hist_len;
42		self.fp -= hist_len;
43		return @intCast(n);
44		}
45
46		/// Data from the current position (read position). Those part of the buffer is
47		/// not converted to tokens yet.
48		fn lookahead(self: *Self) []const u8 {
49		assert(self.wp >= self.rp);
50		return self.buffer[self.rp..self.wp];
51		}
52
53		/// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54		/// preserved otherwise preserves enough data for the longest match. Returns
55		/// null if there is not enough data.
56		pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57		const min: usize = if (should_flush) 0 else min_lookahead;
58		const lh = self.lookahead();
59		return if (lh.len > min) lh else null;
60		}
61
62		/// Advances read position, shrinks lookahead.
63		pub fn advance(self: *Self, n: u16) void {
64		assert(self.wp >= self.rp + n);
65		self.rp += n;
66		}
67
68		/// Returns writable part of the buffer, where new uncompressed data can be
69		/// written.
70		pub fn writable(self: *Self) []u8 {
71		return self.buffer[self.wp..];
72		}
73
74		/// Notification of what part of writable buffer is filled with data.
75		pub fn written(self: *Self, n: usize) void {
76		self.wp += n;
77		}
78
79		/// Finds match length between previous and current position.
80		/// Used in hot path!
81		pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82		const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83		// lookahead buffers from previous and current positions
84		const prev_lh = self.buffer[prev_pos..][0..max_len];
85		const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87		// If we already have match (min_len > 0),
88		// test the first byte above previous len a[min_len] != b[min_len]
89		// and then all the bytes from that position to zero.
90		// That is likely positions to find difference than looping from first bytes.
91		var i: usize = min_len;
92		if (i > 0) {
93		if (max_len <= i) return 0;
94		while (true) {
95		if (prev_lh[i] != curr_lh[i]) return 0;
96		if (i == 0) break;
97		i -= 1;
98		}
99		i = min_len;
100		}
101		while (i < max_len) : (i += 1)
102		if (prev_lh[i] != curr_lh[i]) break;
103		return if (i >= consts.match.min_length) @intCast(i) else 0;
104		}
105
106		/// Current position of non-compressed data. Data before rp are already converted
107		/// to tokens.
108		pub fn pos(self: *Self) u16 {
109		return @intCast(self.rp);
110		}
111
112		/// Notification that token list is cleared.
113		pub fn flush(self: *Self) void {
114		self.fp = @intCast(self.rp);
115		}
116
117		/// Part of the buffer since last flush or null if there was slide in between (so
118		/// fp becomes negative).
119		pub fn tokensBuffer(self: *Self) ?[]const u8 {
120		assert(self.fp <= self.rp);
121		if (self.fp < 0) return null;
122		return self.buffer[@intCast(self.fp)..self.rp];
123		}
124
125		test match {
126		const data = "Blah blah blah blah blah!";
127		var win: Self = .{};
128		try expect(win.write(data) == data.len);
129		try expect(win.wp == data.len);
130		try expect(win.rp == 0);
131
132		// length between l symbols
133		try expect(win.match(1, 6, 0) == 18);
134		try expect(win.match(1, 11, 0) == 13);
135		try expect(win.match(1, 16, 0) == 8);
136		try expect(win.match(1, 21, 0) == 0);
137
138		// position 15 = "blah blah!"
139		// position 20 = "blah!"
140		try expect(win.match(15, 20, 0) == 4);
141		try expect(win.match(15, 20, 3) == 4);
142		try expect(win.match(15, 20, 4) == 0);
143		}
144
145		test slide {
146		var win: Self = .{};
147		win.wp = Self.buffer_len - 11;
148		win.rp = Self.buffer_len - 111;
149		win.buffer[win.rp] = 0xab;
150		try expect(win.lookahead().len == 100);
151		try expect(win.tokensBuffer().?.len == win.rp);
152
153		const n = win.slide();
154		try expect(n == 32757);
155		try expect(win.buffer[win.rp] == 0xab);
156		try expect(win.rp == Self.hist_len - 111);
157		try expect(win.wp == Self.hist_len - 11);
158		try expect(win.lookahead().len == 100);
159		try expect(win.tokensBuffer() == null);
160		}

lib/std/compress/flate/Token.zig+13-7

...	...	@@ -6,7 +6,6 @@ const std = @import("std");
6	6	const assert = std.debug.assert;
7	7	const print = std.debug.print;
8	8	const expect = std.testing.expect;
9		const consts = @import("consts.zig").match;
10	9
11	10	const Token = @This();
12	11
...	...	@@ -21,16 +20,23 @@ dist: u15 = 0,
21	20	len_lit: u8 = 0,
22	21	kind: Kind = .literal,
23	22
	23	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
	24	pub const min_length = 4; // min length used in this algorithm
	25	pub const max_length = 258;
	26
	27	pub const min_distance = 1;
	28	pub const max_distance = std.compress.flate.history_len;
	29
24	30	pub fn literal(t: Token) u8 {
25	31	return t.len_lit;
26	32	}
27	33
28	34	pub fn distance(t: Token) u16 {
29		return @as(u16, t.dist) + consts.min_distance;
	35	return @as(u16, t.dist) + min_distance;
30	36	}
31	37
32	38	pub fn length(t: Token) u16 {
33		return @as(u16, t.len_lit) + consts.base_length;
	39	return @as(u16, t.len_lit) + base_length;
34	40	}
35	41
36	42	pub fn initLiteral(lit: u8) Token {
...	...	@@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token {
40	46	// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
41	47	// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
42	48	pub fn initMatch(dist: u16, len: u16) Token {
43		assert(len >= consts.min_length and len <= consts.max_length);
44		assert(dist >= consts.min_distance and dist <= consts.max_distance);
	49	assert(len >= min_length and len <= max_length);
	50	assert(dist >= min_distance and dist <= max_distance);
45	51	return .{
46	52	.kind = .match,
47		.dist = @intCast(dist - consts.min_distance),
48		.len_lit = @intCast(len - consts.base_length),
	53	.dist = @intCast(dist - min_distance),
	54	.len_lit = @intCast(len - base_length),
49	55	};
50	56	}
51	57

lib/std/compress/flate/bit_reader.zig deleted-422

...	...	@@ -1,422 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const testing = std.testing;
4
5		pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) {
6		return BitReader(T, @TypeOf(reader)).init(reader);
7		}
8
9		pub fn BitReader64(comptime ReaderType: type) type {
10		return BitReader(u64, ReaderType);
11		}
12
13		pub fn BitReader32(comptime ReaderType: type) type {
14		return BitReader(u32, ReaderType);
15		}
16
17		/// Bit reader used during inflate (decompression). Has internal buffer of 64
18		/// bits which shifts right after bits are consumed. Uses forward_reader to fill
19		/// that internal buffer when needed.
20		///
21		/// readF is the core function. Supports few different ways of getting bits
22		/// controlled by flags. In hot path we try to avoid checking whether we need to
23		/// fill buffer from forward_reader by calling fill in advance and readF with
24		/// buffered flag set.
25		///
26		pub fn BitReader(comptime T: type, comptime ReaderType: type) type {
27		assert(T == u32 or T == u64);
28		const t_bytes: usize = @sizeOf(T);
29		const Tshift = if (T == u64) u6 else u5;
30
31		return struct {
32		// Underlying reader used for filling internal bits buffer
33		forward_reader: ReaderType = undefined,
34		// Internal buffer of 64 bits
35		bits: T = 0,
36		// Number of bits in the buffer
37		nbits: u32 = 0,
38
39		const Self = @This();
40
41		pub const Error = ReaderType.Error \|\| error{EndOfStream};
42
43		pub fn init(rdr: ReaderType) Self {
44		var self = Self{ .forward_reader = rdr };
45		self.fill(1) catch {};
46		return self;
47		}
48
49		/// Try to have `nice` bits are available in buffer. Reads from
50		/// forward reader if there is no `nice` bits in buffer. Returns error
51		/// if end of forward stream is reached and internal buffer is empty.
52		/// It will not error if less than `nice` bits are in buffer, only when
53		/// all bits are exhausted. During inflate we usually know what is the
54		/// maximum bits for the next step but usually that step will need less
55		/// bits to decode. So `nice` is not hard limit, it will just try to have
56		/// that number of bits available. If end of forward stream is reached
57		/// it may be some extra zero bits in buffer.
58		pub inline fn fill(self: *Self, nice: u6) !void {
59		if (self.nbits >= nice and nice != 0) {
60		return; // We have enough bits
61		}
62		// Read more bits from forward reader
63
64		// Number of empty bytes in bits, round nbits to whole bytes.
65		const empty_bytes =
66		@as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
67		(self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
68
69		var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
70		const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
71		if (bytes_read > 0) {
72		const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
73		self.bits \|= u << @as(Tshift, @intCast(self.nbits));
74		self.nbits += 8 * @as(u8, @intCast(bytes_read));
75		return;
76		}
77
78		if (self.nbits == 0)
79		return error.EndOfStream;
80		}
81
82		/// Read exactly buf.len bytes into buf.
83		pub fn readAll(self: *Self, buf: []u8) !void {
84		assert(self.alignBits() == 0); // internal bits must be at byte boundary
85
86		// First read from internal bits buffer.
87		var n: usize = 0;
88		while (self.nbits > 0 and n < buf.len) {
89		buf[n] = try self.readF(u8, flag.buffered);
90		n += 1;
91		}
92		// Then use forward reader for all other bytes.
93		try self.forward_reader.readNoEof(buf[n..]);
94		}
95
96		pub const flag = struct {
97		pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer
98		pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before
99		pub const reverse: u3 = 0b100; // bit reverse read bits
100		};
101
102		/// Alias for readF(U, 0).
103		pub fn read(self: *Self, comptime U: type) !U {
104		return self.readF(U, 0);
105		}
106
107		/// Alias for readF with flag.peak set.
108		pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U {
109		return self.readF(U, how \| flag.peek);
110		}
111
112		/// Read with flags provided.
113		pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
114		if (U == T) {
115		assert(how == 0);
116		assert(self.alignBits() == 0);
117		try self.fill(@bitSizeOf(T));
118		if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
119		const v = self.bits;
120		self.nbits = 0;
121		self.bits = 0;
122		return v;
123		}
124		const n: Tshift = @bitSizeOf(U);
125		switch (how) {
126		0 => { // `normal` read
127		try self.fill(n); // ensure that there are n bits in the buffer
128		const u: U = @truncate(self.bits); // get n bits
129		try self.shift(n); // advance buffer for n
130		return u;
131		},
132		(flag.peek) => { // no shift, leave bits in the buffer
133		try self.fill(n);
134		return @truncate(self.bits);
135		},
136		flag.buffered => { // no fill, assume that buffer has enough bits
137		const u: U = @truncate(self.bits);
138		try self.shift(n);
139		return u;
140		},
141		(flag.reverse) => { // same as 0 with bit reverse
142		try self.fill(n);
143		const u: U = @truncate(self.bits);
144		try self.shift(n);
145		return @bitReverse(u);
146		},
147		(flag.peek \| flag.reverse) => {
148		try self.fill(n);
149		return @bitReverse(@as(U, @truncate(self.bits)));
150		},
151		(flag.buffered \| flag.reverse) => {
152		const u: U = @truncate(self.bits);
153		try self.shift(n);
154		return @bitReverse(u);
155		},
156		(flag.peek \| flag.buffered) => {
157		return @truncate(self.bits);
158		},
159		(flag.peek \| flag.buffered \| flag.reverse) => {
160		return @bitReverse(@as(U, @truncate(self.bits)));
161		},
162		}
163		}
164
165		/// Read n number of bits.
166		/// Only buffered flag can be used in how.
167		pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
168		switch (how) {
169		0 => {
170		try self.fill(n);
171		},
172		flag.buffered => {},
173		else => unreachable,
174		}
175		const mask: u16 = (@as(u16, 1) << n) - 1;
176		const u: u16 = @as(u16, @truncate(self.bits)) & mask;
177		try self.shift(n);
178		return u;
179		}
180
181		/// Advance buffer for n bits.
182		pub fn shift(self: *Self, n: Tshift) !void {
183		if (n > self.nbits) return error.EndOfStream;
184		self.bits >>= n;
185		self.nbits -= n;
186		}
187
188		/// Skip n bytes.
189		pub fn skipBytes(self: *Self, n: u16) !void {
190		for (0..n) \|_\| {
191		try self.fill(8);
192		try self.shift(8);
193		}
194		}
195
196		// Number of bits to align stream to the byte boundary.
197		fn alignBits(self: *Self) u3 {
198		return @intCast(self.nbits & 0x7);
199		}
200
201		/// Align stream to the byte boundary.
202		pub fn alignToByte(self: *Self) void {
203		const ab = self.alignBits();
204		if (ab > 0) self.shift(ab) catch unreachable;
205		}
206
207		/// Skip zero terminated string.
208		pub fn skipStringZ(self: *Self) !void {
209		while (true) {
210		if (try self.readF(u8, 0) == 0) break;
211		}
212		}
213
214		/// Read deflate fixed fixed code.
215		/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
216		/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
217		/// Lit Value Bits Codes
218		/// --------- ---- -----
219		/// 0 - 143 8 00110000 through
220		/// 10111111
221		/// 144 - 255 9 110010000 through
222		/// 111111111
223		/// 256 - 279 7 0000000 through
224		/// 0010111
225		/// 280 - 287 8 11000000 through
226		/// 11000111
227		pub fn readFixedCode(self: *Self) !u16 {
228		try self.fill(7 + 2);
229		const code7 = try self.readF(u7, flag.buffered \| flag.reverse);
230		if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
231		return @as(u16, code7) + 256;
232		} else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
233		return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000;
234		} else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
235		return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280;
236		} else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
237		return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered \| flag.reverse)) + 144;
238		}
239		}
240		};
241		}
242
243		test "readF" {
244		var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
245		var br = bitReader(u64, fbs.reader());
246		const F = BitReader64(@TypeOf(fbs.reader())).flag;
247
248		try testing.expectEqual(@as(u8, 48), br.nbits);
249		try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
250
251		try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
252		try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
253		try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
254		try testing.expectEqual(@as(u3, 5), br.alignBits());
255
256		try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110);
257		try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110);
258		try br.shift(9);
259		try testing.expectEqual(@as(u8, 36), br.nbits);
260		try testing.expectEqual(@as(u3, 4), br.alignBits());
261
262		try testing.expect(try br.readF(u4, 0) == 0b0100);
263		try testing.expectEqual(@as(u8, 32), br.nbits);
264		try testing.expectEqual(@as(u3, 0), br.alignBits());
265
266		try br.shift(1);
267		try testing.expectEqual(@as(u3, 7), br.alignBits());
268		try br.shift(1);
269		try testing.expectEqual(@as(u3, 6), br.alignBits());
270		br.alignToByte();
271		try testing.expectEqual(@as(u3, 0), br.alignBits());
272
273		try testing.expectEqual(@as(u64, 0xc9), br.bits);
274		try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
275		try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
276		}
277
278		test "read block type 1 data" {
279		inline for ([_]type{ u64, u32 }) \|T\| {
280		const data = [_]u8{
281		0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
282		0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
283		0x0c, 0x01, 0x02, 0x03, //
284		0xaa, 0xbb, 0xcc, 0xdd,
285		};
286		var fbs = std.io.fixedBufferStream(&data);
287		var br = bitReader(T, fbs.reader());
288		const F = BitReader(T, @TypeOf(fbs.reader())).flag;
289
290		try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
291		try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
292
293		for ("Hello world\n") \|c\| {
294		try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
295		}
296		try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
297		br.alignToByte();
298		try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
299		try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
300		try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
301		}
302		}
303
304		test "shift/fill" {
305		const data = [_]u8{
306		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
307		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308		};
309		var fbs = std.io.fixedBufferStream(&data);
310		var br = bitReader(u64, fbs.reader());
311
312		try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
313		try br.shift(8);
314		try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
315		try br.fill(60); // fill with 1 byte
316		try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
317		try br.shift(8 * 4 + 4);
318		try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
319
320		try br.fill(60); // fill with 4 bytes (shift by 4)
321		try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
322		try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
323
324		try br.shift(@intCast(br.nbits)); // clear buffer
325		try br.fill(8); // refill with the rest of the bytes
326		try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
327		}
328
329		test "readAll" {
330		inline for ([_]type{ u64, u32 }) \|T\| {
331		const data = [_]u8{
332		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
333		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
334		};
335		var fbs = std.io.fixedBufferStream(&data);
336		var br = bitReader(T, fbs.reader());
337
338		switch (T) {
339		u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
340		u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
341		else => unreachable,
342		}
343
344		var out: [16]u8 = undefined;
345		try br.readAll(out[0..]);
346		try testing.expect(br.nbits == 0);
347		try testing.expect(br.bits == 0);
348
349		try testing.expectEqualSlices(u8, data[0..16], &out);
350		}
351		}
352
353		test "readFixedCode" {
354		inline for ([_]type{ u64, u32 }) \|T\| {
355		const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
356
357		var fbs = std.io.fixedBufferStream(&fixed_codes);
358		var rdr = bitReader(T, fbs.reader());
359
360		for (0..286) \|c\| {
361		try testing.expectEqual(c, try rdr.readFixedCode());
362		}
363		try testing.expect(rdr.nbits == 0);
364		}
365		}
366
367		test "u32 leaves no bits on u32 reads" {
368		const data = [_]u8{
369		0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
370		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
371		};
372		var fbs = std.io.fixedBufferStream(&data);
373		var br = bitReader(u32, fbs.reader());
374
375		_ = try br.read(u3);
376		try testing.expectEqual(29, br.nbits);
377		br.alignToByte();
378		try testing.expectEqual(24, br.nbits);
379		try testing.expectEqual(0x04_03_02_01, try br.read(u32));
380		try testing.expectEqual(0, br.nbits);
381		try testing.expectEqual(0x08_07_06_05, try br.read(u32));
382		try testing.expectEqual(0, br.nbits);
383
384		_ = try br.read(u9);
385		try testing.expectEqual(23, br.nbits);
386		br.alignToByte();
387		try testing.expectEqual(16, br.nbits);
388		try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
389		try testing.expectEqual(0, br.nbits);
390		}
391
392		test "u64 need fill after alignToByte" {
393		const data = [_]u8{
394		0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
395		0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
396		};
397
398		// without fill
399		var fbs = std.io.fixedBufferStream(&data);
400		var br = bitReader(u64, fbs.reader());
401		_ = try br.read(u23);
402		try testing.expectEqual(41, br.nbits);
403		br.alignToByte();
404		try testing.expectEqual(40, br.nbits);
405		try testing.expectEqual(0x06_05_04_03, try br.read(u32));
406		try testing.expectEqual(8, br.nbits);
407		try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
408		try testing.expectEqual(32, br.nbits);
409
410		// fill after align ensures all bits filled
411		fbs.reset();
412		br = bitReader(u64, fbs.reader());
413		_ = try br.read(u23);
414		try testing.expectEqual(41, br.nbits);
415		br.alignToByte();
416		try br.fill(0);
417		try testing.expectEqual(64, br.nbits);
418		try testing.expectEqual(0x06_05_04_03, try br.read(u32));
419		try testing.expectEqual(32, br.nbits);
420		try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
421		try testing.expectEqual(0, br.nbits);
422		}

lib/std/compress/flate/bit_writer.zig deleted-99

...	...	@@ -1,99 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3
4		/// Bit writer for use in deflate (compression).
5		///
6		/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7		/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8		/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9		///
10		pub fn BitWriter(comptime WriterType: type) type {
11		// buffer_flush_size indicates the buffer size
12		// after which bytes are flushed to the writer.
13		// Should preferably be a multiple of 6, since
14		// we accumulate 6 bytes between writes to the buffer.
15		const buffer_flush_size = 240;
16
17		// buffer_size is the actual output byte buffer size.
18		// It must have additional headroom for a flush
19		// which can contain up to 8 bytes.
20		const buffer_size = buffer_flush_size + 8;
21
22		return struct {
23		inner_writer: WriterType,
24
25		// Data waiting to be written is bytes[0 .. nbytes]
26		// and then the low nbits of bits. Data is always written
27		// sequentially into the bytes array.
28		bits: u64 = 0,
29		nbits: u32 = 0, // number of bits
30		bytes: [buffer_size]u8 = undefined,
31		nbytes: u32 = 0, // number of bytes
32
33		const Self = @This();
34
35		pub const Error = WriterType.Error \|\| error{UnfinishedBits};
36
37		pub fn init(writer: WriterType) Self {
38		return .{ .inner_writer = writer };
39		}
40
41		pub fn setWriter(self: *Self, new_writer: WriterType) void {
42		//assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43		self.inner_writer = new_writer;
44		}
45
46		pub fn flush(self: *Self) Error!void {
47		var n = self.nbytes;
48		while (self.nbits != 0) {
49		self.bytes[n] = @as(u8, @truncate(self.bits));
50		self.bits >>= 8;
51		if (self.nbits > 8) { // Avoid underflow
52		self.nbits -= 8;
53		} else {
54		self.nbits = 0;
55		}
56		n += 1;
57		}
58		self.bits = 0;
59		_ = try self.inner_writer.write(self.bytes[0..n]);
60		self.nbytes = 0;
61		}
62
63		pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64		self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65		self.nbits += nb;
66		if (self.nbits < 48)
67		return;
68
69		var n = self.nbytes;
70		std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71		n += 6;
72		if (n >= buffer_flush_size) {
73		_ = try self.inner_writer.write(self.bytes[0..n]);
74		n = 0;
75		}
76		self.nbytes = n;
77		self.bits >>= 48;
78		self.nbits -= 48;
79		}
80
81		pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82		var n = self.nbytes;
83		if (self.nbits & 7 != 0) {
84		return error.UnfinishedBits;
85		}
86		while (self.nbits != 0) {
87		self.bytes[n] = @as(u8, @truncate(self.bits));
88		self.bits >>= 8;
89		self.nbits -= 8;
90		n += 1;
91		}
92		if (n != 0) {
93		_ = try self.inner_writer.write(self.bytes[0..n]);
94		}
95		self.nbytes = 0;
96		_ = try self.inner_writer.write(bytes);
97		}
98		};
99		}

lib/std/compress/flate/block_writer.zig deleted-706

...	...	@@ -1,706 +0,0 @@
1		const std = @import("std");
2		const io = std.io;
3		const assert = std.debug.assert;
4
5		const hc = @import("huffman_encoder.zig");
6		const consts = @import("consts.zig").huffman;
7		const Token = @import("Token.zig");
8		const BitWriter = @import("bit_writer.zig").BitWriter;
9
10		pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11		return BlockWriter(@TypeOf(writer)).init(writer);
12		}
13
14		/// Accepts list of tokens, decides what is best block type to write. What block
15		/// type will provide best compression. Writes header and body of the block.
16		///
17		pub fn BlockWriter(comptime WriterType: type) type {
18		const BitWriterType = BitWriter(WriterType);
19		return struct {
20		const codegen_order = consts.codegen_order;
21		const end_code_mark = 255;
22		const Self = @This();
23
24		pub const Error = BitWriterType.Error;
25		bit_writer: BitWriterType,
26
27		codegen_freq: [consts.codegen_code_count]u16 = undefined,
28		literal_freq: [consts.max_num_lit]u16 = undefined,
29		distance_freq: [consts.distance_code_count]u16 = undefined,
30		codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31		literal_encoding: hc.LiteralEncoder = .{},
32		distance_encoding: hc.DistanceEncoder = .{},
33		codegen_encoding: hc.CodegenEncoder = .{},
34		fixed_literal_encoding: hc.LiteralEncoder,
35		fixed_distance_encoding: hc.DistanceEncoder,
36		huff_distance: hc.DistanceEncoder,
37
38		pub fn init(writer: WriterType) Self {
39		return .{
40		.bit_writer = BitWriterType.init(writer),
41		.fixed_literal_encoding = hc.fixedLiteralEncoder(),
42		.fixed_distance_encoding = hc.fixedDistanceEncoder(),
43		.huff_distance = hc.huffmanDistanceEncoder(),
44		};
45		}
46
47		/// Flush intrenal bit buffer to the writer.
48		/// Should be called only when bit stream is at byte boundary.
49		///
50		/// That is after final block; when last byte could be incomplete or
51		/// after stored block; which is aligned to the byte boundary (it has x
52		/// padding bits after first 3 bits).
53		pub fn flush(self: *Self) Error!void {
54		try self.bit_writer.flush();
55		}
56
57		pub fn setWriter(self: *Self, new_writer: WriterType) void {
58		self.bit_writer.setWriter(new_writer);
59		}
60
61		fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62		try self.bit_writer.writeBits(c.code, c.len);
63		}
64
65		// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66		// the literal and distance lengths arrays (which are concatenated into a single
67		// array). This method generates that run-length encoding.
68		//
69		// The result is written into the codegen array, and the frequencies
70		// of each code is written into the codegen_freq array.
71		// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72		// information. Code bad_code is an end marker
73		//
74		// num_literals: The number of literals in literal_encoding
75		// num_distances: The number of distances in distance_encoding
76		// lit_enc: The literal encoder to use
77		// dist_enc: The distance encoder to use
78		fn generateCodegen(
79		self: *Self,
80		num_literals: u32,
81		num_distances: u32,
82		lit_enc: *hc.LiteralEncoder,
83		dist_enc: *hc.DistanceEncoder,
84		) void {
85		for (self.codegen_freq, 0..) \|_, i\| {
86		self.codegen_freq[i] = 0;
87		}
88
89		// Note that we are using codegen both as a temporary variable for holding
90		// a copy of the frequencies, and as the place where we put the result.
91		// This is fine because the output is always shorter than the input used
92		// so far.
93		var codegen = &self.codegen; // cache
94		// Copy the concatenated code sizes to codegen. Put a marker at the end.
95		var cgnl = codegen[0..num_literals];
96		for (cgnl, 0..) \|_, i\| {
97		cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98		}
99
100		cgnl = codegen[num_literals .. num_literals + num_distances];
101		for (cgnl, 0..) \|_, i\| {
102		cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103		}
104		codegen[num_literals + num_distances] = end_code_mark;
105
106		var size = codegen[0];
107		var count: i32 = 1;
108		var out_index: u32 = 0;
109		var in_index: u32 = 1;
110		while (size != end_code_mark) : (in_index += 1) {
111		// INVARIANT: We have seen "count" copies of size that have not yet
112		// had output generated for them.
113		const next_size = codegen[in_index];
114		if (next_size == size) {
115		count += 1;
116		continue;
117		}
118		// We need to generate codegen indicating "count" of size.
119		if (size != 0) {
120		codegen[out_index] = size;
121		out_index += 1;
122		self.codegen_freq[size] += 1;
123		count -= 1;
124		while (count >= 3) {
125		var n: i32 = 6;
126		if (n > count) {
127		n = count;
128		}
129		codegen[out_index] = 16;
130		out_index += 1;
131		codegen[out_index] = @as(u8, @intCast(n - 3));
132		out_index += 1;
133		self.codegen_freq[16] += 1;
134		count -= n;
135		}
136		} else {
137		while (count >= 11) {
138		var n: i32 = 138;
139		if (n > count) {
140		n = count;
141		}
142		codegen[out_index] = 18;
143		out_index += 1;
144		codegen[out_index] = @as(u8, @intCast(n - 11));
145		out_index += 1;
146		self.codegen_freq[18] += 1;
147		count -= n;
148		}
149		if (count >= 3) {
150		// 3 <= count <= 10
151		codegen[out_index] = 17;
152		out_index += 1;
153		codegen[out_index] = @as(u8, @intCast(count - 3));
154		out_index += 1;
155		self.codegen_freq[17] += 1;
156		count = 0;
157		}
158		}
159		count -= 1;
160		while (count >= 0) : (count -= 1) {
161		codegen[out_index] = size;
162		out_index += 1;
163		self.codegen_freq[size] += 1;
164		}
165		// Set up invariant for next time through the loop.
166		size = next_size;
167		count = 1;
168		}
169		// Marker indicating the end of the codegen.
170		codegen[out_index] = end_code_mark;
171		}
172
173		const DynamicSize = struct {
174		size: u32,
175		num_codegens: u32,
176		};
177
178		// dynamicSize returns the size of dynamically encoded data in bits.
179		fn dynamicSize(
180		self: *Self,
181		lit_enc: *hc.LiteralEncoder, // literal encoder
182		dist_enc: *hc.DistanceEncoder, // distance encoder
183		extra_bits: u32,
184		) DynamicSize {
185		var num_codegens = self.codegen_freq.len;
186		while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187		num_codegens -= 1;
188		}
189		const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190		self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191		self.codegen_freq[16] * 2 +
192		self.codegen_freq[17] * 3 +
193		self.codegen_freq[18] * 7;
194		const size = header +
195		lit_enc.bitLength(&self.literal_freq) +
196		dist_enc.bitLength(&self.distance_freq) +
197		extra_bits;
198
199		return DynamicSize{
200		.size = @as(u32, @intCast(size)),
201		.num_codegens = @as(u32, @intCast(num_codegens)),
202		};
203		}
204
205		// fixedSize returns the size of dynamically encoded data in bits.
206		fn fixedSize(self: *Self, extra_bits: u32) u32 {
207		return 3 +
208		self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209		self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210		extra_bits;
211		}
212
213		const StoredSize = struct {
214		size: u32,
215		storable: bool,
216		};
217
218		// storedSizeFits calculates the stored size, including header.
219		// The function returns the size in bits and whether the block
220		// fits inside a single block.
221		fn storedSizeFits(in: ?[]const u8) StoredSize {
222		if (in == null) {
223		return .{ .size = 0, .storable = false };
224		}
225		if (in.?.len <= consts.max_store_block_size) {
226		return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227		}
228		return .{ .size = 0, .storable = false };
229		}
230
231		// Write the header of a dynamic Huffman block to the output stream.
232		//
233		// num_literals: The number of literals specified in codegen
234		// num_distances: The number of distances specified in codegen
235		// num_codegens: The number of codegens used in codegen
236		// eof: Is it the end-of-file? (end of stream)
237		fn dynamicHeader(
238		self: *Self,
239		num_literals: u32,
240		num_distances: u32,
241		num_codegens: u32,
242		eof: bool,
243		) Error!void {
244		const first_bits: u32 = if (eof) 5 else 4;
245		try self.bit_writer.writeBits(first_bits, 3);
246		try self.bit_writer.writeBits(num_literals - 257, 5);
247		try self.bit_writer.writeBits(num_distances - 1, 5);
248		try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250		var i: u32 = 0;
251		while (i < num_codegens) : (i += 1) {
252		const value = self.codegen_encoding.codes[codegen_order[i]].len;
253		try self.bit_writer.writeBits(value, 3);
254		}
255
256		i = 0;
257		while (true) {
258		const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259		i += 1;
260		if (code_word == end_code_mark) {
261		break;
262		}
263		try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265		switch (code_word) {
266		16 => {
267		try self.bit_writer.writeBits(self.codegen[i], 2);
268		i += 1;
269		},
270		17 => {
271		try self.bit_writer.writeBits(self.codegen[i], 3);
272		i += 1;
273		},
274		18 => {
275		try self.bit_writer.writeBits(self.codegen[i], 7);
276		i += 1;
277		},
278		else => {},
279		}
280		}
281		}
282
283		fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284		assert(length <= 65535);
285		const flag: u32 = if (eof) 1 else 0;
286		try self.bit_writer.writeBits(flag, 3);
287		try self.flush();
288		const l: u16 = @intCast(length);
289		try self.bit_writer.writeBits(l, 16);
290		try self.bit_writer.writeBits(~l, 16);
291		}
292
293		fn fixedHeader(self: *Self, eof: bool) Error!void {
294		// Indicate that we are a fixed Huffman block
295		var value: u32 = 2;
296		if (eof) {
297		value = 3;
298		}
299		try self.bit_writer.writeBits(value, 3);
300		}
301
302		// Write a block of tokens with the smallest encoding. Will choose block type.
303		// The original input can be supplied, and if the huffman encoded data
304		// is larger than the original bytes, the data will be written as a
305		// stored block.
306		// If the input is null, the tokens will always be Huffman encoded.
307		pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308		const lit_and_dist = self.indexTokens(tokens);
309		const num_literals = lit_and_dist.num_literals;
310		const num_distances = lit_and_dist.num_distances;
311
312		var extra_bits: u32 = 0;
313		const ret = storedSizeFits(input);
314		const stored_size = ret.size;
315		const storable = ret.storable;
316
317		if (storable) {
318		// We only bother calculating the costs of the extra bits required by
319		// the length of distance fields (which will be the same for both fixed
320		// and dynamic encoding), if we need to compare those two encodings
321		// against stored encoding.
322		var length_code: u16 = Token.length_codes_start + 8;
323		while (length_code < num_literals) : (length_code += 1) {
324		// First eight length codes have extra size = 0.
325		extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326		@as(u32, @intCast(Token.lengthExtraBits(length_code)));
327		}
328		var distance_code: u16 = 4;
329		while (distance_code < num_distances) : (distance_code += 1) {
330		// First four distance codes have extra size = 0.
331		extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332		@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333		}
334		}
335
336		// Figure out smallest code.
337		// Fixed Huffman baseline.
338		var literal_encoding = &self.fixed_literal_encoding;
339		var distance_encoding = &self.fixed_distance_encoding;
340		var size = self.fixedSize(extra_bits);
341
342		// Dynamic Huffman?
343		var num_codegens: u32 = 0;
344
345		// Generate codegen and codegenFrequencies, which indicates how to encode
346		// the literal_encoding and the distance_encoding.
347		self.generateCodegen(
348		num_literals,
349		num_distances,
350		&self.literal_encoding,
351		&self.distance_encoding,
352		);
353		self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354		const dynamic_size = self.dynamicSize(
355		&self.literal_encoding,
356		&self.distance_encoding,
357		extra_bits,
358		);
359		const dyn_size = dynamic_size.size;
360		num_codegens = dynamic_size.num_codegens;
361
362		if (dyn_size < size) {
363		size = dyn_size;
364		literal_encoding = &self.literal_encoding;
365		distance_encoding = &self.distance_encoding;
366		}
367
368		// Stored bytes?
369		if (storable and stored_size < size) {
370		try self.storedBlock(input.?, eof);
371		return;
372		}
373
374		// Huffman.
375		if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376		try self.fixedHeader(eof);
377		} else {
378		try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379		}
380
381		// Write the tokens.
382		try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383		}
384
385		pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386		try self.storedHeader(input.len, eof);
387		try self.bit_writer.writeBytes(input);
388		}
389
390		// writeBlockDynamic encodes a block using a dynamic Huffman table.
391		// This should be used if the symbols used have a disproportionate
392		// histogram distribution.
393		// If input is supplied and the compression savings are below 1/16th of the
394		// input size the block is stored.
395		fn dynamicBlock(
396		self: *Self,
397		tokens: []const Token,
398		eof: bool,
399		input: ?[]const u8,
400		) Error!void {
401		const total_tokens = self.indexTokens(tokens);
402		const num_literals = total_tokens.num_literals;
403		const num_distances = total_tokens.num_distances;
404
405		// Generate codegen and codegenFrequencies, which indicates how to encode
406		// the literal_encoding and the distance_encoding.
407		self.generateCodegen(
408		num_literals,
409		num_distances,
410		&self.literal_encoding,
411		&self.distance_encoding,
412		);
413		self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414		const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415		const size = dynamic_size.size;
416		const num_codegens = dynamic_size.num_codegens;
417
418		// Store bytes, if we don't get a reasonable improvement.
419
420		const stored_size = storedSizeFits(input);
421		const ssize = stored_size.size;
422		const storable = stored_size.storable;
423		if (storable and ssize < (size + (size >> 4))) {
424		try self.storedBlock(input.?, eof);
425		return;
426		}
427
428		// Write Huffman table.
429		try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431		// Write the tokens.
432		try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433		}
434
435		const TotalIndexedTokens = struct {
436		num_literals: u32,
437		num_distances: u32,
438		};
439
440		// Indexes a slice of tokens followed by an end_block_marker, and updates
441		// literal_freq and distance_freq, and generates literal_encoding
442		// and distance_encoding.
443		// The number of literal and distance tokens is returned.
444		fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445		var num_literals: u32 = 0;
446		var num_distances: u32 = 0;
447
448		for (self.literal_freq, 0..) \|_, i\| {
449		self.literal_freq[i] = 0;
450		}
451		for (self.distance_freq, 0..) \|_, i\| {
452		self.distance_freq[i] = 0;
453		}
454
455		for (tokens) \|t\| {
456		if (t.kind == Token.Kind.literal) {
457		self.literal_freq[t.literal()] += 1;
458		continue;
459		}
460		self.literal_freq[t.lengthCode()] += 1;
461		self.distance_freq[t.distanceCode()] += 1;
462		}
463		// add end_block_marker token at the end
464		self.literal_freq[consts.end_block_marker] += 1;
465
466		// get the number of literals
467		num_literals = @as(u32, @intCast(self.literal_freq.len));
468		while (self.literal_freq[num_literals - 1] == 0) {
469		num_literals -= 1;
470		}
471		// get the number of distances
472		num_distances = @as(u32, @intCast(self.distance_freq.len));
473		while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474		num_distances -= 1;
475		}
476		if (num_distances == 0) {
477		// We haven't found a single match. If we want to go with the dynamic encoding,
478		// we should count at least one distance to be sure that the distance huffman tree could be encoded.
479		self.distance_freq[0] = 1;
480		num_distances = 1;
481		}
482		self.literal_encoding.generate(&self.literal_freq, 15);
483		self.distance_encoding.generate(&self.distance_freq, 15);
484		return TotalIndexedTokens{
485		.num_literals = num_literals,
486		.num_distances = num_distances,
487		};
488		}
489
490		// Writes a slice of tokens to the output followed by and end_block_marker.
491		// codes for literal and distance encoding must be supplied.
492		fn writeTokens(
493		self: *Self,
494		tokens: []const Token,
495		le_codes: []hc.HuffCode,
496		oe_codes: []hc.HuffCode,
497		) Error!void {
498		for (tokens) \|t\| {
499		if (t.kind == Token.Kind.literal) {
500		try self.writeCode(le_codes[t.literal()]);
501		continue;
502		}
503
504		// Write the length
505		const le = t.lengthEncoding();
506		try self.writeCode(le_codes[le.code]);
507		if (le.extra_bits > 0) {
508		try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509		}
510
511		// Write the distance
512		const oe = t.distanceEncoding();
513		try self.writeCode(oe_codes[oe.code]);
514		if (oe.extra_bits > 0) {
515		try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516		}
517		}
518		// add end_block_marker at the end
519		try self.writeCode(le_codes[consts.end_block_marker]);
520		}
521
522		// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523		// if the results only gains very little from compression.
524		pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525		// Add everything as literals
526		histogram(input, &self.literal_freq);
527
528		self.literal_freq[consts.end_block_marker] = 1;
529
530		const num_literals = consts.end_block_marker + 1;
531		self.distance_freq[0] = 1;
532		const num_distances = 1;
533
534		self.literal_encoding.generate(&self.literal_freq, 15);
535
536		// Figure out smallest code.
537		// Always use dynamic Huffman or Store
538		var num_codegens: u32 = 0;
539
540		// Generate codegen and codegenFrequencies, which indicates how to encode
541		// the literal_encoding and the distance_encoding.
542		self.generateCodegen(
543		num_literals,
544		num_distances,
545		&self.literal_encoding,
546		&self.huff_distance,
547		);
548		self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549		const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550		const size = dynamic_size.size;
551		num_codegens = dynamic_size.num_codegens;
552
553		// Store bytes, if we don't get a reasonable improvement.
554		const stored_size_ret = storedSizeFits(input);
555		const ssize = stored_size_ret.size;
556		const storable = stored_size_ret.storable;
557
558		if (storable and ssize < (size + (size >> 4))) {
559		try self.storedBlock(input, eof);
560		return;
561		}
562
563		// Huffman.
564		try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565		const encoding = self.literal_encoding.codes[0..257];
566
567		for (input) \|t\| {
568		const c = encoding[t];
569		try self.bit_writer.writeBits(c.code, c.len);
570		}
571		try self.writeCode(encoding[consts.end_block_marker]);
572		}
573
574		// histogram accumulates a histogram of b in h.
575		fn histogram(b: []const u8, h: *[286]u16) void {
576		// Clear histogram
577		for (h, 0..) \|_, i\| {
578		h[i] = 0;
579		}
580
581		var lh = h.*[0..256];
582		for (b) \|t\| {
583		lh[t] += 1;
584		}
585		}
586		};
587		}
588
589		// tests
590		const expect = std.testing.expect;
591		const fmt = std.fmt;
592		const testing = std.testing;
593		const ArrayList = std.ArrayList;
594
595		const TestCase = @import("testdata/block_writer.zig").TestCase;
596		const testCases = @import("testdata/block_writer.zig").testCases;
597
598		// tests if the writeBlock encoding has changed.
599		test "write" {
600		inline for (0..testCases.len) \|i\| {
601		try testBlock(testCases[i], .write_block);
602		}
603		}
604
605		// tests if the writeBlockDynamic encoding has changed.
606		test "dynamicBlock" {
607		inline for (0..testCases.len) \|i\| {
608		try testBlock(testCases[i], .write_dyn_block);
609		}
610		}
611
612		test "huffmanBlock" {
613		inline for (0..testCases.len) \|i\| {
614		try testBlock(testCases[i], .write_huffman_block);
615		}
616		try testBlock(.{
617		.tokens = &[_]Token{},
618		.input = "huffman-rand-max.input",
619		.want = "huffman-rand-max.{s}.expect",
620		}, .write_huffman_block);
621		}
622
623		const TestFn = enum {
624		write_block,
625		write_dyn_block, // write dynamic block
626		write_huffman_block,
627
628		fn to_s(self: TestFn) []const u8 {
629		return switch (self) {
630		.write_block => "wb",
631		.write_dyn_block => "dyn",
632		.write_huffman_block => "huff",
633		};
634		}
635
636		fn write(
637		comptime self: TestFn,
638		bw: anytype,
639		tok: []const Token,
640		input: ?[]const u8,
641		final: bool,
642		) !void {
643		switch (self) {
644		.write_block => try bw.write(tok, final, input),
645		.write_dyn_block => try bw.dynamicBlock(tok, final, input),
646		.write_huffman_block => try bw.huffmanBlock(input.?, final),
647		}
648		try bw.flush();
649		}
650		};
651
652		// testBlock tests a block against its references
653		//
654		// size
655		// 64K [file-name].input - input non compressed file
656		// 8.1K [file-name].golden -
657		// 78 [file-name].dyn.expect - output with writeBlockDynamic
658		// 78 [file-name].wb.expect - output with writeBlock
659		// 8.1K [file-name].huff.expect - output with writeBlockHuff
660		// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661		// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662		//
663		// wb - writeBlock
664		// dyn - writeBlockDynamic
665		// huff - writeBlockHuff
666		//
667		fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668		if (tc.input.len != 0 and tc.want.len != 0) {
669		const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670		const input = @embedFile("testdata/block_writer/" ++ tc.input);
671		const want = @embedFile("testdata/block_writer/" ++ want_name);
672		try testWriteBlock(tfn, input, want, tc.tokens);
673		}
674
675		if (tfn == .write_huffman_block) {
676		return;
677		}
678
679		const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680		const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681		try testWriteBlock(tfn, null, want, tc.tokens);
682		}
683
684		// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685		fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686		var buf = ArrayList(u8).init(testing.allocator);
687		var bw = blockWriter(buf.writer());
688		try tfn.write(&bw, tokens, input, false);
689		var got = buf.items;
690		try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691		try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692		//
693		// Test if the writer produces the same output after reset.
694		buf.deinit();
695		buf = ArrayList(u8).init(testing.allocator);
696		defer buf.deinit();
697		bw.setWriter(buf.writer());
698
699		try tfn.write(&bw, tokens, input, true);
700		try bw.flush();
701		got = buf.items;
702
703		try expect(got[0] & 1 == 1); // bfinal is set
704		buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705		try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706		}

lib/std/compress/flate/consts.zig deleted-49

...	...	@@ -1,49 +0,0 @@
1		pub const deflate = struct {
2		// Number of tokens to accumulate in deflate before starting block encoding.
3		//
4		// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5		// 8 and max 9 that gives 14 or 15 bits.
6		pub const tokens = 1 << 15;
7		};
8
9		pub const match = struct {
10		pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11		pub const min_length = 4; // min length used in this algorithm
12		pub const max_length = 258;
13
14		pub const min_distance = 1;
15		pub const max_distance = 32768;
16		};
17
18		pub const history = struct {
19		pub const len = match.max_distance;
20		};
21
22		pub const lookup = struct {
23		pub const bits = 15;
24		pub const len = 1 << bits;
25		pub const shift = 32 - bits;
26		};
27
28		pub const huffman = struct {
29		// The odd order in which the codegen code sizes are written.
30		pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31		// The number of codegen codes.
32		pub const codegen_code_count = 19;
33
34		// The largest distance code.
35		pub const distance_code_count = 30;
36
37		// Maximum number of literals.
38		pub const max_num_lit = 286;
39
40		// Max number of frequencies used for a Huffman Code
41		// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42		// The largest of these is max_num_lit.
43		pub const max_num_frequencies = max_num_lit;
44
45		// Biggest block size for uncompressed block.
46		pub const max_store_block_size = 65535;
47		// The special code used to mark the end of a block.
48		pub const end_block_marker = 256;
49		};

lib/std/compress/flate/container.zig deleted-208

...	...	@@ -1,208 +0,0 @@
1		//! Container of the deflate bit stream body. Container adds header before
2		//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
3		//! no footer, raw bit stream).
4		//!
5		//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
6		//! addler 32 checksum.
7		//!
8		//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
9		//! crc32 checksum and 4 bytes of uncompressed data length.
10		//!
11		//!
12		//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
13		//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
14		//!
15
16		const std = @import("std");
17
18		pub const Container = enum {
19		raw, // no header or footer
20		gzip, // gzip header and footer
21		zlib, // zlib header and footer
22
23		pub fn size(w: Container) usize {
24		return headerSize(w) + footerSize(w);
25		}
26
27		pub fn headerSize(w: Container) usize {
28		return switch (w) {
29		.gzip => 10,
30		.zlib => 2,
31		.raw => 0,
32		};
33		}
34
35		pub fn footerSize(w: Container) usize {
36		return switch (w) {
37		.gzip => 8,
38		.zlib => 4,
39		.raw => 0,
40		};
41		}
42
43		pub const list = [_]Container{ .raw, .gzip, .zlib };
44
45		pub const Error = error{
46		BadGzipHeader,
47		BadZlibHeader,
48		WrongGzipChecksum,
49		WrongGzipSize,
50		WrongZlibChecksum,
51		};
52
53		pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
54		switch (wrap) {
55		.gzip => {
56		// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
57		// - ID1 (IDentification 1), always 0x1f
58		// - ID2 (IDentification 2), always 0x8b
59		// - CM (Compression Method), always 8 = deflate
60		// - FLG (Flags), all set to 0
61		// - 4 bytes, MTIME (Modification time), not used, all set to zero
62		// - XFL (eXtra FLags), all set to zero
63		// - OS (Operating System), 03 = Unix
64		const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
65		try writer.writeAll(&gzipHeader);
66		},
67		.zlib => {
68		// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
69		// 1st byte:
70		// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
71		// - The next four bits is the CM (compression method), which is 8 for deflate.
72		// 2nd byte:
73		// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
74		// - The next bit, FDICT, is set if a dictionary is given.
75		// - The final five FCHECK bits form a mod-31 checksum.
76		//
77		// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
78		const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
79		try writer.writeAll(&zlibHeader);
80		},
81		.raw => {},
82		}
83		}
84
85		pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
86		var bits: [4]u8 = undefined;
87		switch (wrap) {
88		.gzip => {
89		// GZIP 8 bytes footer
90		// - 4 bytes, CRC32 (CRC-32)
91		// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
92		std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
93		try writer.writeAll(&bits);
94
95		std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
96		try writer.writeAll(&bits);
97		},
98		.zlib => {
99		// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
100		// 4 bytes of ADLER32 (Adler-32 checksum)
101		// Checksum value of the uncompressed data (excluding any
102		// dictionary data) computed according to Adler-32
103		// algorithm.
104		std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
105		try writer.writeAll(&bits);
106		},
107		.raw => {},
108		}
109		}
110
111		pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
112		switch (wrap) {
113		.gzip => try parseGzipHeader(reader),
114		.zlib => try parseZlibHeader(reader),
115		.raw => {},
116		}
117		}
118
119		fn parseGzipHeader(reader: anytype) !void {
120		const magic1 = try reader.read(u8);
121		const magic2 = try reader.read(u8);
122		const method = try reader.read(u8);
123		const flags = try reader.read(u8);
124		try reader.skipBytes(6); // mtime(4), xflags, os
125		if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
126		return error.BadGzipHeader;
127		// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
128		if (flags != 0) {
129		if (flags & 0b0000_0100 != 0) { // FEXTRA
130		const extra_len = try reader.read(u16);
131		try reader.skipBytes(extra_len);
132		}
133		if (flags & 0b0000_1000 != 0) { // FNAME
134		try reader.skipStringZ();
135		}
136		if (flags & 0b0001_0000 != 0) { // FCOMMENT
137		try reader.skipStringZ();
138		}
139		if (flags & 0b0000_0010 != 0) { // FHCRC
140		try reader.skipBytes(2);
141		}
142		}
143		}
144
145		fn parseZlibHeader(reader: anytype) !void {
146		const cm = try reader.read(u4);
147		const cinfo = try reader.read(u4);
148		_ = try reader.read(u8);
149		if (cm != 8 or cinfo > 7) {
150		return error.BadZlibHeader;
151		}
152		}
153
154		pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155		switch (wrap) {
156		.gzip => {
157		try reader.fill(0);
158		if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
159		if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
160		},
161		.zlib => {
162		const chksum: u32 = @byteSwap(hasher.chksum());
163		if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
164		},
165		.raw => {},
166		}
167		}
168
169		pub fn Hasher(comptime wrap: Container) type {
170		const HasherType = switch (wrap) {
171		.gzip => std.hash.Crc32,
172		.zlib => std.hash.Adler32,
173		.raw => struct {
174		pub fn init() @This() {
175		return .{};
176		}
177		},
178		};
179
180		return struct {
181		hasher: HasherType = HasherType.init(),
182		bytes: usize = 0,
183
184		const Self = @This();
185
186		pub fn update(self: *Self, buf: []const u8) void {
187		switch (wrap) {
188		.raw => {},
189		else => {
190		self.hasher.update(buf);
191		self.bytes += buf.len;
192		},
193		}
194		}
195
196		pub fn chksum(self: *Self) u32 {
197		switch (wrap) {
198		.raw => return 0,
199		else => return self.hasher.final(),
200		}
201		}
202
203		pub fn bytesRead(self: *Self) u32 {
204		return @truncate(self.bytes);
205		}
206		};
207		}
208		};

lib/std/compress/flate/deflate.zig deleted-744

...	...	@@ -1,744 +0,0 @@
1		const std = @import("std");
2		const io = std.io;
3		const assert = std.debug.assert;
4		const testing = std.testing;
5		const expect = testing.expect;
6		const print = std.debug.print;
7
8		const Token = @import("Token.zig");
9		const consts = @import("consts.zig");
10		const BlockWriter = @import("block_writer.zig").BlockWriter;
11		const Container = @import("container.zig").Container;
12		const SlidingWindow = @import("SlidingWindow.zig");
13		const Lookup = @import("Lookup.zig");
14
15		pub const Options = struct {
16		level: Level = .default,
17		};
18
19		/// Trades between speed and compression size.
20		/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21		/// levels 1-3 are using different algorithm to perform faster but with less
22		/// compression. That is not implemented here.
23		pub const Level = enum(u4) {
24		// zig fmt: off
25		fast = 0xb, level_4 = 4,
26		level_5 = 5,
27		default = 0xc, level_6 = 6,
28		level_7 = 7,
29		level_8 = 8,
30		best = 0xd, level_9 = 9,
31		// zig fmt: on
32		};
33
34		/// Algorithm knobs for each level.
35		const LevelArgs = struct {
36		good: u16, // Do less lookups if we already have match of this length.
37		nice: u16, // Stop looking for better match if we found match with at least this length.
38		lazy: u16, // Don't do lazy match find if got match with at least this length.
39		chain: u16, // How many lookups for previous match to perform.
40
41		pub fn get(level: Level) LevelArgs {
42		// zig fmt: off
43		return switch (level) {
44		.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45		.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46		.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47		.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48		.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49		.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50		};
51		// zig fmt: on
52		}
53		};
54
55		/// Compress plain data from reader into compressed stream written to writer.
56		pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57		var c = try compressor(container, writer, options);
58		try c.compress(reader);
59		try c.finish();
60		}
61
62		/// Create compressor for writer type.
63		pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64		container,
65		@TypeOf(writer),
66		) {
67		return try Compressor(container, @TypeOf(writer)).init(writer, options);
68		}
69
70		/// Compressor type.
71		pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72		const TokenWriterType = BlockWriter(WriterType);
73		return Deflate(container, WriterType, TokenWriterType);
74		}
75
76		/// Default compression algorithm. Has two steps: tokenization and token
77		/// encoding.
78		///
79		/// Tokenization takes uncompressed input stream and produces list of tokens.
80		/// Each token can be literal (byte of data) or match (backrefernce to previous
81		/// data with length and distance). Tokenization accumulators 32K tokens, when
82		/// full or `flush` is called tokens are passed to the `block_writer`. Level
83		/// defines how hard (how slow) it tries to find match.
84		///
85		/// Block writer will decide which type of deflate block to write (stored, fixed,
86		/// dynamic) and encode tokens to the output byte stream. Client has to call
87		/// `finish` to write block with the final bit set.
88		///
89		/// Container defines type of header and footer which can be gzip, zlib or raw.
90		/// They all share same deflate body. Raw has no header or footer just deflate
91		/// body.
92		///
93		/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
94		///
95		/// The compressor uses a chained hash table `lookup` to find duplicated
96		/// strings, using a hash function that operates on 4-byte sequences. At any
97		/// given point during compression, let XYZW be the next 4 input bytes
98		/// (lookahead) to be examined (not necessarily all different, of course).
99		/// First, the compressor examines the hash chain for XYZW. If the chain is
100		/// empty, the compressor simply writes out X as a literal byte and advances
101		/// one byte in the input. If the hash chain is not empty, indicating that the
102		/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
103		/// hash function value) has occurred recently, the compressor compares all
104		/// strings on the XYZW hash chain with the actual input data sequence
105		/// starting at the current point, and selects the longest match.
106		///
107		/// To improve overall compression, the compressor defers the selection of
108		/// matches ("lazy matching"): after a match of length N has been found, the
109		/// compressor searches for a longer match starting at the next input byte. If
110		/// it finds a longer match, it truncates the previous match to a length of
111		/// one (thus producing a single literal byte) and then emits the longer
112		/// match. Otherwise, it emits the original match, and, as described above,
113		/// advances N bytes before continuing.
114		///
115		///
116		/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
117		///
118		/// Deflate function accepts BlockWriterType so we can change that in test to test
119		/// just tokenization part.
120		///
121		fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
122		return struct {
123		lookup: Lookup = .{},
124		win: SlidingWindow = .{},
125		tokens: Tokens = .{},
126		wrt: WriterType,
127		block_writer: BlockWriterType,
128		level: LevelArgs,
129		hasher: container.Hasher() = .{},
130
131		// Match and literal at the previous position.
132		// Used for lazy match finding in processWindow.
133		prev_match: ?Token = null,
134		prev_literal: ?u8 = null,
135
136		const Self = @This();
137
138		pub fn init(wrt: WriterType, options: Options) !Self {
139		const self = Self{
140		.wrt = wrt,
141		.block_writer = BlockWriterType.init(wrt),
142		.level = LevelArgs.get(options.level),
143		};
144		try container.writeHeader(self.wrt);
145		return self;
146		}
147
148		const FlushOption = enum { none, flush, final };
149
150		// Process data in window and create tokens. If token buffer is full
151		// flush tokens to the token writer. In the case of `flush` or `final`
152		// option it will process all data from the window. In the `none` case
153		// it will preserve some data for the next match.
154		fn tokenize(self: *Self, flush_opt: FlushOption) !void {
155		// flush - process all data from window
156		const should_flush = (flush_opt != .none);
157
158		// While there is data in active lookahead buffer.
159		while (self.win.activeLookahead(should_flush)) \|lh\| {
160		var step: u16 = 1; // 1 in the case of literal, match length otherwise
161		const pos: u16 = self.win.pos();
162		const literal = lh[0]; // literal at current position
163		const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
164
165		// Try to find match at least min_len long.
166		if (self.findMatch(pos, lh, min_len)) \|match\| {
167		// Found better match than previous.
168		try self.addPrevLiteral();
169
170		// Is found match length good enough?
171		if (match.length() >= self.level.lazy) {
172		// Don't try to lazy find better match, use this.
173		step = try self.addMatch(match);
174		} else {
175		// Store this match.
176		self.prev_literal = literal;
177		self.prev_match = match;
178		}
179		} else {
180		// There is no better match at current pos then it was previous.
181		// Write previous match or literal.
182		if (self.prev_match) \|m\| {
183		// Write match from previous position.
184		step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
185		} else {
186		// No match at previous position.
187		// Write previous literal if any, and remember this literal.
188		try self.addPrevLiteral();
189		self.prev_literal = literal;
190		}
191		}
192		// Advance window and add hashes.
193		self.windowAdvance(step, lh, pos);
194		}
195
196		if (should_flush) {
197		// In the case of flushing, last few lookahead buffers were smaller then min match len.
198		// So only last literal can be unwritten.
199		assert(self.prev_match == null);
200		try self.addPrevLiteral();
201		self.prev_literal = null;
202
203		try self.flushTokens(flush_opt);
204		}
205		}
206
207		fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
208		// current position is already added in findMatch
209		self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
210		self.win.advance(step);
211		}
212
213		// Add previous literal (if any) to the tokens list.
214		fn addPrevLiteral(self: *Self) !void {
215		if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
216		}
217
218		// Add match to the tokens list, reset prev pointers.
219		// Returns length of the added match.
220		fn addMatch(self: *Self, m: Token) !u16 {
221		try self.addToken(m);
222		self.prev_literal = null;
223		self.prev_match = null;
224		return m.length();
225		}
226
227		fn addToken(self: *Self, token: Token) !void {
228		self.tokens.add(token);
229		if (self.tokens.full()) try self.flushTokens(.none);
230		}
231
232		// Finds largest match in the history window with the data at current pos.
233		fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
234		var len: u16 = min_len;
235		// Previous location with the same hash (same 4 bytes).
236		var prev_pos = self.lookup.add(lh, pos);
237		// Last found match.
238		var match: ?Token = null;
239
240		// How much back-references to try, performance knob.
241		var chain: usize = self.level.chain;
242		if (len >= self.level.good) {
243		// If we've got a match that's good enough, only look in 1/4 the chain.
244		chain >>= 2;
245		}
246
247		// Hot path loop!
248		while (prev_pos > 0 and chain > 0) : (chain -= 1) {
249		const distance = pos - prev_pos;
250		if (distance > consts.match.max_distance)
251		break;
252
253		const new_len = self.win.match(prev_pos, pos, len);
254		if (new_len > len) {
255		match = Token.initMatch(@intCast(distance), new_len);
256		if (new_len >= self.level.nice) {
257		// The match is good enough that we don't try to find a better one.
258		return match;
259		}
260		len = new_len;
261		}
262		prev_pos = self.lookup.prev(prev_pos);
263		}
264
265		return match;
266		}
267
268		fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
269		// Pass tokens to the token writer
270		try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
271		// Stored block ensures byte alignment.
272		// It has 3 bits (final, block_type) and then padding until byte boundary.
273		// After that everything is aligned to the boundary in the stored block.
274		// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
275		// Last 4 bytes are byte aligned.
276		if (flush_opt == .flush) {
277		try self.block_writer.storedBlock("", false);
278		}
279		if (flush_opt != .none) {
280		// Safe to call only when byte aligned or it is OK to add
281		// padding bits (on last byte of the final block).
282		try self.block_writer.flush();
283		}
284		// Reset internal tokens store.
285		self.tokens.reset();
286		// Notify win that tokens are flushed.
287		self.win.flush();
288		}
289
290		// Slide win and if needed lookup tables.
291		fn slide(self: *Self) void {
292		const n = self.win.slide();
293		self.lookup.slide(n);
294		}
295
296		/// Compresses as much data as possible, stops when the reader becomes
297		/// empty. It will introduce some output latency (reading input without
298		/// producing all output) because some data are still in internal
299		/// buffers.
300		///
301		/// It is up to the caller to call flush (if needed) or finish (required)
302		/// when is need to output any pending data or complete stream.
303		///
304		pub fn compress(self: *Self, reader: anytype) !void {
305		while (true) {
306		// Fill window from reader
307		const buf = self.win.writable();
308		if (buf.len == 0) {
309		try self.tokenize(.none);
310		self.slide();
311		continue;
312		}
313		const n = try reader.readAll(buf);
314		self.hasher.update(buf[0..n]);
315		self.win.written(n);
316		// Process window
317		try self.tokenize(.none);
318		// Exit when no more data in reader
319		if (n < buf.len) break;
320		}
321		}
322
323		/// Flushes internal buffers to the output writer. Outputs empty stored
324		/// block to sync bit stream to the byte boundary, so that the
325		/// decompressor can get all input data available so far.
326		///
327		/// It is useful mainly in compressed network protocols, to ensure that
328		/// deflate bit stream can be used as byte stream. May degrade
329		/// compression so it should be used only when necessary.
330		///
331		/// Completes the current deflate block and follows it with an empty
332		/// stored block that is three zero bits plus filler bits to the next
333		/// byte, followed by four bytes (00 00 ff ff).
334		///
335		pub fn flush(self: *Self) !void {
336		try self.tokenize(.flush);
337		}
338
339		/// Completes deflate bit stream by writing any pending data as deflate
340		/// final deflate block. HAS to be called once all data are written to
341		/// the compressor as a signal that next block has to have final bit
342		/// set.
343		///
344		pub fn finish(self: *Self) !void {
345		try self.tokenize(.final);
346		try container.writeFooter(&self.hasher, self.wrt);
347		}
348
349		/// Use another writer while preserving history. Most probably flush
350		/// should be called on old writer before setting new.
351		pub fn setWriter(self: *Self, new_writer: WriterType) void {
352		self.block_writer.setWriter(new_writer);
353		self.wrt = new_writer;
354		}
355
356		// Writer interface
357
358		pub const Writer = io.GenericWriter(*Self, Error, write);
359		pub const Error = BlockWriterType.Error;
360
361		/// Write `input` of uncompressed data.
362		/// See compress.
363		pub fn write(self: *Self, input: []const u8) !usize {
364		var fbs = io.fixedBufferStream(input);
365		try self.compress(fbs.reader());
366		return input.len;
367		}
368
369		pub fn writer(self: *Self) Writer {
370		return .{ .context = self };
371		}
372		};
373		}
374
375		// Tokens store
376		const Tokens = struct {
377		list: [consts.deflate.tokens]Token = undefined,
378		pos: usize = 0,
379
380		fn add(self: *Tokens, t: Token) void {
381		self.list[self.pos] = t;
382		self.pos += 1;
383		}
384
385		fn full(self: *Tokens) bool {
386		return self.pos == self.list.len;
387		}
388
389		fn reset(self: *Tokens) void {
390		self.pos = 0;
391		}
392
393		fn tokens(self: *Tokens) []const Token {
394		return self.list[0..self.pos];
395		}
396		};
397
398		/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
399		/// only performs Huffman entropy encoding. Results in faster compression, much
400		/// less memory requirements during compression but bigger compressed sizes.
401		pub const huffman = struct {
402		pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
403		var c = try huffman.compressor(container, writer);
404		try c.compress(reader);
405		try c.finish();
406		}
407
408		pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
409		return SimpleCompressor(.huffman, container, WriterType);
410		}
411
412		pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
413		return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
414		}
415		};
416
417		/// Creates store blocks only. Data are not compressed only packed into deflate
418		/// store blocks. That adds 9 bytes of header for each block. Max stored block
419		/// size is 64K. Block is emitted when flush is called on on finish.
420		pub const store = struct {
421		pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
422		var c = try store.compressor(container, writer);
423		try c.compress(reader);
424		try c.finish();
425		}
426
427		pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
428		return SimpleCompressor(.store, container, WriterType);
429		}
430
431		pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
432		return try store.Compressor(container, @TypeOf(writer)).init(writer);
433		}
434		};
435
436		const SimpleCompressorKind = enum {
437		huffman,
438		store,
439		};
440
441		fn simpleCompressor(
442		comptime kind: SimpleCompressorKind,
443		comptime container: Container,
444		writer: anytype,
445		) !SimpleCompressor(kind, container, @TypeOf(writer)) {
446		return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
447		}
448
449		fn SimpleCompressor(
450		comptime kind: SimpleCompressorKind,
451		comptime container: Container,
452		comptime WriterType: type,
453		) type {
454		const BlockWriterType = BlockWriter(WriterType);
455		return struct {
456		buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
457		wp: usize = 0,
458
459		wrt: WriterType,
460		block_writer: BlockWriterType,
461		hasher: container.Hasher() = .{},
462
463		const Self = @This();
464
465		pub fn init(wrt: WriterType) !Self {
466		const self = Self{
467		.wrt = wrt,
468		.block_writer = BlockWriterType.init(wrt),
469		};
470		try container.writeHeader(self.wrt);
471		return self;
472		}
473
474		pub fn flush(self: *Self) !void {
475		try self.flushBuffer(false);
476		try self.block_writer.storedBlock("", false);
477		try self.block_writer.flush();
478		}
479
480		pub fn finish(self: *Self) !void {
481		try self.flushBuffer(true);
482		try self.block_writer.flush();
483		try container.writeFooter(&self.hasher, self.wrt);
484		}
485
486		fn flushBuffer(self: *Self, final: bool) !void {
487		const buf = self.buffer[0..self.wp];
488		switch (kind) {
489		.huffman => try self.block_writer.huffmanBlock(buf, final),
490		.store => try self.block_writer.storedBlock(buf, final),
491		}
492		self.wp = 0;
493		}
494
495		// Writes all data from the input reader of uncompressed data.
496		// It is up to the caller to call flush or finish if there is need to
497		// output compressed blocks.
498		pub fn compress(self: *Self, reader: anytype) !void {
499		while (true) {
500		// read from rdr into buffer
501		const buf = self.buffer[self.wp..];
502		if (buf.len == 0) {
503		try self.flushBuffer(false);
504		continue;
505		}
506		const n = try reader.readAll(buf);
507		self.hasher.update(buf[0..n]);
508		self.wp += n;
509		if (n < buf.len) break; // no more data in reader
510		}
511		}
512
513		// Writer interface
514
515		pub const Writer = io.GenericWriter(*Self, Error, write);
516		pub const Error = BlockWriterType.Error;
517
518		// Write `input` of uncompressed data.
519		pub fn write(self: *Self, input: []const u8) !usize {
520		var fbs = io.fixedBufferStream(input);
521		try self.compress(fbs.reader());
522		return input.len;
523		}
524
525		pub fn writer(self: *Self) Writer {
526		return .{ .context = self };
527		}
528		};
529		}
530
531		const builtin = @import("builtin");
532
533		test "tokenization" {
534		const L = Token.initLiteral;
535		const M = Token.initMatch;
536
537		const cases = [_]struct {
538		data: []const u8,
539		tokens: []const Token,
540		}{
541		.{
542		.data = "Blah blah blah blah blah!",
543		.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
544		},
545		.{
546		.data = "ABCDEABCD ABCDEABCD",
547		.tokens = &[_]Token{
548		L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
549		L('A'), M(10, 8),
550		},
551		},
552		};
553
554		for (cases) \|c\| {
555		inline for (Container.list) \|container\| { // for each wrapping
556
557		var cw = io.countingWriter(io.null_writer);
558		const cww = cw.writer();
559		var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
560
561		_ = try df.write(c.data);
562		try df.flush();
563
564		// df.token_writer.show();
565		try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
566		try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
567
568		try testing.expectEqual(container.headerSize(), cw.bytes_written);
569		try df.finish();
570		try testing.expectEqual(container.size(), cw.bytes_written);
571		}
572		}
573		}
574
575		// Tests that tokens written are equal to expected token list.
576		const TestTokenWriter = struct {
577		const Self = @This();
578
579		pos: usize = 0,
580		actual: [128]Token = undefined,
581
582		pub fn init(_: anytype) Self {
583		return .{};
584		}
585		pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
586		for (tokens) \|t\| {
587		self.actual[self.pos] = t;
588		self.pos += 1;
589		}
590		}
591
592		pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
593
594		pub fn get(self: *Self) []Token {
595		return self.actual[0..self.pos];
596		}
597
598		pub fn show(self: *Self) void {
599		print("\n", .{});
600		for (self.get()) \|t\| {
601		t.show();
602		}
603		}
604
605		pub fn flush(_: *Self) !void {}
606		};
607
608		test "file tokenization" {
609		const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
610		const cases = [_]struct {
611		data: []const u8, // uncompressed content
612		// expected number of tokens producet in deflate tokenization
613		tokens_count: [levels.len]usize = .{0} ** levels.len,
614		}{
615		.{
616		.data = @embedFile("testdata/rfc1951.txt"),
617		.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
618		},
619
620		.{
621		.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
622		.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
623		},
624		.{
625		.data = @embedFile("testdata/block_writer/huffman-pi.input"),
626		.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
627		},
628		.{
629		.data = @embedFile("testdata/block_writer/huffman-text.input"),
630		.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
631		},
632		.{
633		.data = @embedFile("testdata/fuzz/roundtrip1.input"),
634		.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
635		},
636		.{
637		.data = @embedFile("testdata/fuzz/roundtrip2.input"),
638		.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
639		},
640		};
641
642		for (cases) \|case\| { // for each case
643		const data = case.data;
644
645		for (levels, 0..) \|level, i\| { // for each compression level
646		var original = io.fixedBufferStream(data);
647
648		// buffer for decompressed data
649		var al = std.ArrayList(u8).init(testing.allocator);
650		defer al.deinit();
651		const writer = al.writer();
652
653		// create compressor
654		const WriterType = @TypeOf(writer);
655		const TokenWriter = TokenDecoder(@TypeOf(writer));
656		var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
657
658		// Stream uncompressed `original` data to the compressor. It will
659		// produce tokens list and pass that list to the TokenDecoder. This
660		// TokenDecoder uses CircularBuffer from inflate to convert list of
661		// tokens back to the uncompressed stream.
662		try cmp.compress(original.reader());
663		try cmp.flush();
664		const expected_count = case.tokens_count[i];
665		const actual = cmp.block_writer.tokens_count;
666		if (expected_count == 0) {
667		print("actual token count {d}\n", .{actual});
668		} else {
669		try testing.expectEqual(expected_count, actual);
670		}
671
672		try testing.expectEqual(data.len, al.items.len);
673		try testing.expectEqualSlices(u8, data, al.items);
674		}
675		}
676		}
677
678		fn TokenDecoder(comptime WriterType: type) type {
679		return struct {
680		const CircularBuffer = @import("CircularBuffer.zig");
681		hist: CircularBuffer = .{},
682		wrt: WriterType,
683		tokens_count: usize = 0,
684
685		const Self = @This();
686
687		pub fn init(wrt: WriterType) Self {
688		return .{ .wrt = wrt };
689		}
690
691		pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
692		self.tokens_count += tokens.len;
693		for (tokens) \|t\| {
694		switch (t.kind) {
695		.literal => self.hist.write(t.literal()),
696		.match => try self.hist.writeMatch(t.length(), t.distance()),
697		}
698		if (self.hist.free() < 285) try self.flushWin();
699		}
700		try self.flushWin();
701		}
702
703		pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
704
705		fn flushWin(self: *Self) !void {
706		while (true) {
707		const buf = self.hist.read();
708		if (buf.len == 0) break;
709		try self.wrt.writeAll(buf);
710		}
711		}
712
713		pub fn flush(_: *Self) !void {}
714		};
715		}
716
717		test "store simple compressor" {
718		const data = "Hello world!";
719		const expected = [_]u8{
720		0x1, // block type 0, final bit set
721		0xc, 0x0, // len = 12
722		0xf3, 0xff, // ~len
723		'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
724		//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
725		};
726
727		var fbs = std.io.fixedBufferStream(data);
728		var al = std.ArrayList(u8).init(testing.allocator);
729		defer al.deinit();
730
731		var cmp = try store.compressor(.raw, al.writer());
732		try cmp.compress(fbs.reader());
733		try cmp.finish();
734		try testing.expectEqualSlices(u8, &expected, al.items);
735
736		fbs.reset();
737		try al.resize(0);
738
739		// huffman only compresoor will also emit store block for this small sample
740		var hc = try huffman.compressor(.raw, al.writer());
741		try hc.compress(fbs.reader());
742		try hc.finish();
743		try testing.expectEqualSlices(u8, &expected, al.items);
744		}

lib/std/compress/flate/huffman_decoder.zig deleted-302

...	...	@@ -1,302 +0,0 @@
1		const std = @import("std");
2		const testing = std.testing;
3
4		pub const Symbol = packed struct {
5		pub const Kind = enum(u2) {
6		literal,
7		end_of_block,
8		match,
9		};
10
11		symbol: u8 = 0, // symbol from alphabet
12		code_bits: u4 = 0, // number of bits in code 0-15
13		kind: Kind = .literal,
14
15		code: u16 = 0, // huffman code of the symbol
16		next: u16 = 0, // pointer to the next symbol in linked list
17		// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19		// Sorting less than function.
20		pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21		if (a.code_bits == b.code_bits) {
22		if (a.kind == b.kind) {
23		return a.symbol < b.symbol;
24		}
25		return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26		}
27		return a.code_bits < b.code_bits;
28		}
29		};
30
31		pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32		pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33		pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35		pub const Error = error{
36		InvalidCode,
37		OversubscribedHuffmanTree,
38		IncompleteHuffmanTree,
39		MissingEndOfBlockCode,
40		};
41
42		/// Creates huffman tree codes from list of code lengths (in `build`).
43		///
44		/// `find` then finds symbol for code bits. Code can be any length between 1 and
45		/// 15 bits. When calling `find` we don't know how many bits will be used to
46		/// find symbol. When symbol is returned it has code_bits field which defines
47		/// how much we should advance in bit stream.
48		///
49		/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50		/// many times in this table; 32K places for 286 (at most) symbols.
51		/// Small lookup table is optimization for faster search.
52		/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53		/// with difference that we here use statically allocated arrays.
54		///
55		fn HuffmanDecoder(
56		comptime alphabet_size: u16,
57		comptime max_code_bits: u4,
58		comptime lookup_bits: u4,
59		) type {
60		const lookup_shift = max_code_bits - lookup_bits;
61
62		return struct {
63		// all symbols in alaphabet, sorted by code_len, symbol
64		symbols: [alphabet_size]Symbol = undefined,
65		// lookup table code -> symbol
66		lookup: [1 << lookup_bits]Symbol = undefined,
67
68		const Self = @This();
69
70		/// Generates symbols and lookup tables from list of code lens for each symbol.
71		pub fn generate(self: *Self, lens: []const u4) !void {
72		try checkCompleteness(lens);
73
74		// init alphabet with code_bits
75		for (self.symbols, 0..) \|_, i\| {
76		const cb: u4 = if (i < lens.len) lens[i] else 0;
77		self.symbols[i] = if (i < 256)
78		.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79		else if (i == 256)
80		.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81		else
82		.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83		}
84		std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86		// reset lookup table
87		for (0..self.lookup.len) \|i\| {
88		self.lookup[i] = .{};
89		}
90
91		// assign code to symbols
92		// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93		var code: u16 = 0;
94		var idx: u16 = 0;
95		for (&self.symbols, 0..) \|*sym, pos\| {
96		if (sym.code_bits == 0) continue; // skip unused
97		sym.code = code;
98
99		const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
100		const next_idx = next_code >> lookup_shift;
101
102		if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
103		if (sym.code_bits <= lookup_bits) {
104		// fill small lookup table
105		for (idx..next_idx) \|j\|
106		self.lookup[j] = sym.*;
107		} else {
108		// insert into linked table starting at root
109		const root = &self.lookup[idx];
110		const root_next = root.next;
111		root.next = @intCast(pos);
112		sym.next = root_next;
113		}
114
115		idx = next_idx;
116		code = next_code;
117		}
118		}
119
120		/// Given the list of code lengths check that it represents a canonical
121		/// Huffman code for n symbols.
122		///
123		/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
124		fn checkCompleteness(lens: []const u4) !void {
125		if (alphabet_size == 286)
126		if (lens[256] == 0) return error.MissingEndOfBlockCode;
127
128		var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
129		var max: usize = 0;
130		for (lens) \|n\| {
131		if (n == 0) continue;
132		if (n > max) max = n;
133		count[n] += 1;
134		}
135		if (max == 0) // empty tree
136		return;
137
138		// check for an over-subscribed or incomplete set of lengths
139		var left: usize = 1; // one possible code of zero length
140		for (1..count.len) \|len\| {
141		left <<= 1; // one more bit, double codes left
142		if (count[len] > left)
143		return error.OversubscribedHuffmanTree;
144		left -= count[len]; // deduct count from possible codes
145		}
146		if (left > 0) { // left > 0 means incomplete
147		// incomplete code ok only for single length 1 code
148		if (max_code_bits > 7 and max == count[0] + count[1]) return;
149		return error.IncompleteHuffmanTree;
150		}
151		}
152
153		/// Finds symbol for lookup table code.
154		pub fn find(self: *Self, code: u16) !Symbol {
155		// try to find in lookup table
156		const idx = code >> lookup_shift;
157		const sym = self.lookup[idx];
158		if (sym.code_bits != 0) return sym;
159		// if not use linked list of symbols with same prefix
160		return self.findLinked(code, sym.next);
161		}
162
163		inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
164		var pos = start;
165		while (pos > 0) {
166		const sym = self.symbols[pos];
167		const shift = max_code_bits - sym.code_bits;
168		// compare code_bits number of upper bits
169		if ((code ^ sym.code) >> shift == 0) return sym;
170		pos = sym.next;
171		}
172		return error.InvalidCode;
173		}
174		};
175		}
176
177		test "init/find" {
178		// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
179		const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
180		var h: CodegenDecoder = .{};
181		try h.generate(&code_lens);
182
183		const expected = [_]struct {
184		sym: Symbol,
185		code: u16,
186		}{
187		.{
188		.code = 0b00_00000,
189		.sym = .{ .symbol = 3, .code_bits = 2 },
190		},
191		.{
192		.code = 0b01_00000,
193		.sym = .{ .symbol = 18, .code_bits = 2 },
194		},
195		.{
196		.code = 0b100_0000,
197		.sym = .{ .symbol = 1, .code_bits = 3 },
198		},
199		.{
200		.code = 0b101_0000,
201		.sym = .{ .symbol = 4, .code_bits = 3 },
202		},
203		.{
204		.code = 0b110_0000,
205		.sym = .{ .symbol = 17, .code_bits = 3 },
206		},
207		.{
208		.code = 0b1110_000,
209		.sym = .{ .symbol = 0, .code_bits = 4 },
210		},
211		.{
212		.code = 0b1111_000,
213		.sym = .{ .symbol = 16, .code_bits = 4 },
214		},
215		};
216
217		// unused symbols
218		for (0..12) \|i\| {
219		try testing.expectEqual(0, h.symbols[i].code_bits);
220		}
221		// used, from index 12
222		for (expected, 12..) \|e, i\| {
223		try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
224		try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
225		const sym_from_code = try h.find(e.code);
226		try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
227		}
228
229		// All possible codes for each symbol.
230		// Lookup table has 126 elements, to cover all possible 7 bit codes.
231		for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
232		try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
233
234		for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
235		try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
236
237		for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
238		try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
239
240		for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
241		try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
242
243		for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
244		try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
245
246		for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
247		try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
248
249		for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
250		try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
251		}
252
253		test "encode/decode literals" {
254		const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
255
256		for (1..286) \|j\| { // for all different number of codes
257		var enc: LiteralEncoder = .{};
258		// create frequencies
259		var freq = [_]u16{0} ** 286;
260		freq[256] = 1; // ensure we have end of block code
261		for (&freq, 1..) \|*f, i\| {
262		if (i % j == 0)
263		f.* = @intCast(i);
264		}
265
266		// encoder from frequencies
267		enc.generate(&freq, 15);
268
269		// get code_lens from encoder
270		var code_lens = [_]u4{0} ** 286;
271		for (code_lens, 0..) \|_, i\| {
272		code_lens[i] = @intCast(enc.codes[i].len);
273		}
274		// generate decoder from code lens
275		var dec: LiteralDecoder = .{};
276		try dec.generate(&code_lens);
277
278		// expect decoder code to match original encoder code
279		for (dec.symbols) \|s\| {
280		if (s.code_bits == 0) continue;
281		const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
282		const symbol: u16 = switch (s.kind) {
283		.literal => s.symbol,
284		.end_of_block => 256,
285		.match => @as(u16, s.symbol) + 257,
286		};
287
288		const c = enc.codes[symbol];
289		try testing.expect(c.code == c_code);
290		}
291
292		// find each symbol by code
293		for (enc.codes) \|c\| {
294		if (c.len == 0) continue;
295
296		const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
297		const s = try dec.find(s_code);
298		try testing.expect(s.code == s_code);
299		try testing.expect(s.code_bits == c.len);
300		}
301		}
302		}

lib/std/compress/flate/huffman_encoder.zig deleted-536

...	...	@@ -1,536 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const math = std.math;
4		const mem = std.mem;
5		const sort = std.sort;
6		const testing = std.testing;
7
8		const consts = @import("consts.zig").huffman;
9
10		const LiteralNode = struct {
11		literal: u16,
12		freq: u16,
13		};
14
15		// Describes the state of the constructed tree for a given depth.
16		const LevelInfo = struct {
17		// Our level. for better printing
18		level: u32,
19
20		// The frequency of the last node at this level
21		last_freq: u32,
22
23		// The frequency of the next character to add to this level
24		next_char_freq: u32,
25
26		// The frequency of the next pair (from level below) to add to this level.
27		// Only valid if the "needed" value of the next lower level is 0.
28		next_pair_freq: u32,
29
30		// The number of chains remaining to generate for this level before moving
31		// up to the next level
32		needed: u32,
33		};
34
35		// hcode is a huffman code with a bit code and bit length.
36		pub const HuffCode = struct {
37		code: u16 = 0,
38		len: u16 = 0,
39
40		// set sets the code and length of an hcode.
41		fn set(self: *HuffCode, code: u16, length: u16) void {
42		self.len = length;
43		self.code = code;
44		}
45		};
46
47		pub fn HuffmanEncoder(comptime size: usize) type {
48		return struct {
49		codes: [size]HuffCode = undefined,
50		// Reusable buffer with the longest possible frequency table.
51		freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52		bit_count: [17]u32 = undefined,
53		lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54		lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56		const Self = @This();
57
58		// Update this Huffman Code object to be the minimum code for the specified frequency count.
59		//
60		// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61		// max_bits The maximum number of bits to use for any literal.
62		pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63		var list = self.freq_cache[0 .. freq.len + 1];
64		// Number of non-zero literals
65		var count: u32 = 0;
66		// Set list to be the set of all non-zero literals and their frequencies
67		for (freq, 0..) \|f, i\| {
68		if (f != 0) {
69		list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70		count += 1;
71		} else {
72		list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73		self.codes[i].len = 0;
74		}
75		}
76		list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78		list = list[0..count];
79		if (count <= 2) {
80		// Handle the small cases here, because they are awkward for the general case code. With
81		// two or fewer literals, everything has bit length 1.
82		for (list, 0..) \|node, i\| {
83		// "list" is in order of increasing literal value.
84		self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85		}
86		return;
87		}
88		self.lfs = list;
89		mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91		// Get the number of literals for each bit count
92		const bit_count = self.bitCounts(list, max_bits);
93		// And do the assignment
94		self.assignEncodingAndSize(bit_count, list);
95		}
96
97		pub fn bitLength(self: *Self, freq: []u16) u32 {
98		var total: u32 = 0;
99		for (freq, 0..) \|f, i\| {
100		if (f != 0) {
101		total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102		}
103		}
104		return total;
105		}
106
107		// Return the number of literals assigned to each bit size in the Huffman encoding
108		//
109		// This method is only called when list.len >= 3
110		// The cases of 0, 1, and 2 literals are handled by special case code.
111		//
112		// list: An array of the literals with non-zero frequencies
113		// and their associated frequencies. The array is in order of increasing
114		// frequency, and has as its last element a special element with frequency
115		// std.math.maxInt(i32)
116		//
117		// max_bits: The maximum number of bits that should be used to encode any literal.
118		// Must be less than 16.
119		//
120		// Returns an integer array in which array[i] indicates the number of literals
121		// that should be encoded in i bits.
122		fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123		var max_bits = max_bits_to_use;
124		const n = list.len;
125		const max_bits_limit = 16;
126
127		assert(max_bits < max_bits_limit);
128
129		// The tree can't have greater depth than n - 1, no matter what. This
130		// saves a little bit of work in some small cases
131		max_bits = @min(max_bits, n - 1);
132
133		// Create information about each of the levels.
134		// A bogus "Level 0" whose sole purpose is so that
135		// level1.prev.needed == 0. This makes level1.next_pair_freq
136		// be a legitimate value that never gets chosen.
137		var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138		// leaf_counts[i] counts the number of literals at the left
139		// of ancestors of the rightmost node at level i.
140		// leaf_counts[i][j] is the number of literals at the left
141		// of the level j ancestor.
142		var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144		{
145		var level = @as(u32, 1);
146		while (level <= max_bits) : (level += 1) {
147		// For every level, the first two items are the first two characters.
148		// We initialize the levels as if we had already figured this out.
149		levels[level] = LevelInfo{
150		.level = level,
151		.last_freq = list[1].freq,
152		.next_char_freq = list[2].freq,
153		.next_pair_freq = list[0].freq + list[1].freq,
154		.needed = 0,
155		};
156		leaf_counts[level][level] = 2;
157		if (level == 1) {
158		levels[level].next_pair_freq = math.maxInt(i32);
159		}
160		}
161		}
162
163		// We need a total of 2*n - 2 items at top level and have already generated 2.
164		levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166		{
167		var level = max_bits;
168		while (true) {
169		var l = &levels[level];
170		if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171		// We've run out of both leaves and pairs.
172		// End all calculations for this level.
173		// To make sure we never come back to this level or any lower level,
174		// set next_pair_freq impossibly large.
175		l.needed = 0;
176		levels[level + 1].next_pair_freq = math.maxInt(i32);
177		level += 1;
178		continue;
179		}
180
181		const prev_freq = l.last_freq;
182		if (l.next_char_freq < l.next_pair_freq) {
183		// The next item on this row is a leaf node.
184		const next = leaf_counts[level][level] + 1;
185		l.last_freq = l.next_char_freq;
186		// Lower leaf_counts are the same of the previous node.
187		leaf_counts[level][level] = next;
188		if (next >= list.len) {
189		l.next_char_freq = maxNode().freq;
190		} else {
191		l.next_char_freq = list[next].freq;
192		}
193		} else {
194		// The next item on this row is a pair from the previous row.
195		// next_pair_freq isn't valid until we generate two
196		// more values in the level below
197		l.last_freq = l.next_pair_freq;
198		// Take leaf counts from the lower level, except counts[level] remains the same.
199		@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200		levels[l.level - 1].needed = 2;
201		}
202
203		l.needed -= 1;
204		if (l.needed == 0) {
205		// We've done everything we need to do for this level.
206		// Continue calculating one level up. Fill in next_pair_freq
207		// of that level with the sum of the two nodes we've just calculated on
208		// this level.
209		if (l.level == max_bits) {
210		// All done!
211		break;
212		}
213		levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214		level += 1;
215		} else {
216		// If we stole from below, move down temporarily to replenish it.
217		while (levels[level - 1].needed > 0) {
218		level -= 1;
219		if (level == 0) {
220		break;
221		}
222		}
223		}
224		}
225		}
226
227		// Somethings is wrong if at the end, the top level is null or hasn't used
228		// all of the leaves.
229		assert(leaf_counts[max_bits][max_bits] == n);
230
231		var bit_count = self.bit_count[0 .. max_bits + 1];
232		var bits: u32 = 1;
233		const counts = &leaf_counts[max_bits];
234		{
235		var level = max_bits;
236		while (level > 0) : (level -= 1) {
237		// counts[level] gives the number of literals requiring at least "bits"
238		// bits to encode.
239		bit_count[bits] = counts[level] - counts[level - 1];
240		bits += 1;
241		if (level == 0) {
242		break;
243		}
244		}
245		}
246		return bit_count;
247		}
248
249		// Look at the leaves and assign them a bit count and an encoding as specified
250		// in RFC 1951 3.2.2
251		fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252		var code = @as(u16, 0);
253		var list = list_arg;
254
255		for (bit_count, 0..) \|bits, n\| {
256		code <<= 1;
257		if (n == 0 or bits == 0) {
258		continue;
259		}
260		// The literals list[list.len-bits] .. list[list.len-bits]
261		// are encoded using "bits" bits, and get the values
262		// code, code + 1, .... The code values are
263		// assigned in literal order (not frequency order).
264		const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266		self.lns = chunk;
267		mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269		for (chunk) \|node\| {
270		self.codes[node.literal] = HuffCode{
271		.code = bitReverse(u16, code, @as(u5, @intCast(n))),
272		.len = @as(u16, @intCast(n)),
273		};
274		code += 1;
275		}
276		list = list[0 .. list.len - @as(u32, @intCast(bits))];
277		}
278		}
279		};
280		}
281
282		fn maxNode() LiteralNode {
283		return LiteralNode{
284		.literal = math.maxInt(u16),
285		.freq = math.maxInt(u16),
286		};
287		}
288
289		pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290		return .{};
291		}
292
293		pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294		pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295		pub const CodegenEncoder = HuffmanEncoder(19);
296
297		// Generates a HuffmanCode corresponding to the fixed literal table
298		pub fn fixedLiteralEncoder() LiteralEncoder {
299		var h: LiteralEncoder = undefined;
300		var ch: u16 = 0;
301
302		while (ch < consts.max_num_frequencies) : (ch += 1) {
303		var bits: u16 = undefined;
304		var size: u16 = undefined;
305		switch (ch) {
306		0...143 => {
307		// size 8, 000110000 .. 10111111
308		bits = ch + 48;
309		size = 8;
310		},
311		144...255 => {
312		// size 9, 110010000 .. 111111111
313		bits = ch + 400 - 144;
314		size = 9;
315		},
316		256...279 => {
317		// size 7, 0000000 .. 0010111
318		bits = ch - 256;
319		size = 7;
320		},
321		else => {
322		// size 8, 11000000 .. 11000111
323		bits = ch + 192 - 280;
324		size = 8;
325		},
326		}
327		h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328		}
329		return h;
330		}
331
332		pub fn fixedDistanceEncoder() DistanceEncoder {
333		var h: DistanceEncoder = undefined;
334		for (h.codes, 0..) \|_, ch\| {
335		h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336		}
337		return h;
338		}
339
340		pub fn huffmanDistanceEncoder() DistanceEncoder {
341		var distance_freq = [1]u16{0} ** consts.distance_code_count;
342		distance_freq[0] = 1;
343		// huff_distance is a static distance encoder used for huffman only encoding.
344		// It can be reused since we will not be encoding distance values.
345		var h: DistanceEncoder = .{};
346		h.generate(distance_freq[0..], 15);
347		return h;
348		}
349
350		fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351		_ = context;
352		return a.literal < b.literal;
353		}
354
355		fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356		_ = context;
357		if (a.freq == b.freq) {
358		return a.literal < b.literal;
359		}
360		return a.freq < b.freq;
361		}
362
363		test "generate a Huffman code from an array of frequencies" {
364		var freqs: [19]u16 = [_]u16{
365		8, // 0
366		1, // 1
367		1, // 2
368		2, // 3
369		5, // 4
370		10, // 5
371		9, // 6
372		1, // 7
373		0, // 8
374		0, // 9
375		0, // 10
376		0, // 11
377		0, // 12
378		0, // 13
379		0, // 14
380		0, // 15
381		1, // 16
382		3, // 17
383		5, // 18
384		};
385
386		var enc = huffmanEncoder(19);
387		enc.generate(freqs[0..], 7);
388
389		try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391		try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392		try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393		try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394		try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395		try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396		try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397		try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398		try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399		try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400		try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401		try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402		try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403		try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404		try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405		try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406		try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407		try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408		try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409		try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411		try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412		try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413		try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414		try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415		try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416		try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417		try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418		try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419		try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420		try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421		try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422		}
423
424		test "generate a Huffman code for the fixed literal table specific to Deflate" {
425		const enc = fixedLiteralEncoder();
426		for (enc.codes) \|c\| {
427		switch (c.len) {
428		7 => {
429		const v = @bitReverse(@as(u7, @intCast(c.code)));
430		try testing.expect(v <= 0b0010111);
431		},
432		8 => {
433		const v = @bitReverse(@as(u8, @intCast(c.code)));
434		try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435		(v >= 0b11000000 and v <= 11000111));
436		},
437		9 => {
438		const v = @bitReverse(@as(u9, @intCast(c.code)));
439		try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440		},
441		else => unreachable,
442		}
443		}
444		}
445
446		test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447		const enc = fixedDistanceEncoder();
448		for (enc.codes) \|c\| {
449		const v = @bitReverse(@as(u5, @intCast(c.code)));
450		try testing.expect(v <= 29);
451		try testing.expect(c.len == 5);
452		}
453		}
454
455		// Reverse bit-by-bit a N-bit code.
456		fn bitReverse(comptime T: type, value: T, n: usize) T {
457		const r = @bitReverse(value);
458		return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
459		}
460
461		test bitReverse {
462		const ReverseBitsTest = struct {
463		in: u16,
464		bit_count: u5,
465		out: u16,
466		};
467
468		const reverse_bits_tests = [_]ReverseBitsTest{
469		.{ .in = 1, .bit_count = 1, .out = 1 },
470		.{ .in = 1, .bit_count = 2, .out = 2 },
471		.{ .in = 1, .bit_count = 3, .out = 4 },
472		.{ .in = 1, .bit_count = 4, .out = 8 },
473		.{ .in = 1, .bit_count = 5, .out = 16 },
474		.{ .in = 17, .bit_count = 5, .out = 17 },
475		.{ .in = 257, .bit_count = 9, .out = 257 },
476		.{ .in = 29, .bit_count = 5, .out = 23 },
477		};
478
479		for (reverse_bits_tests) \|h\| {
480		const v = bitReverse(u16, h.in, h.bit_count);
481		try std.testing.expectEqual(h.out, v);
482		}
483		}
484
485		test "fixedLiteralEncoder codes" {
486		var al = std.ArrayList(u8).init(testing.allocator);
487		defer al.deinit();
488		var bw = std.io.bitWriter(.little, al.writer());
489
490		const f = fixedLiteralEncoder();
491		for (f.codes) \|c\| {
492		try bw.writeBits(c.code, c.len);
493		}
494		try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495		}
496
497		pub const fixed_codes = [_]u8{
498		0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499		0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500		0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501		0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502		0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503		0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504		0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505		0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506		0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507		0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508		0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509		0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510		0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511		0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512		0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513		0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514		0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515		0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516		0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517		0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518		0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519		0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520		0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521		0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522		0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523		0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524		0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525		0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526		0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527		0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528		0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529		0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530		0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531		0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532		0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533		0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534		0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535		0b10100011,
536		};

lib/std/compress/flate/inflate.zig deleted-570

...	...	@@ -1,570 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const testing = std.testing;
4
5		const hfd = @import("huffman_decoder.zig");
6		const BitReader = @import("bit_reader.zig").BitReader;
7		const CircularBuffer = @import("CircularBuffer.zig");
8		const Container = @import("container.zig").Container;
9		const Token = @import("Token.zig");
10		const codegen_order = @import("consts.zig").huffman.codegen_order;
11
12		/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
13		/// `writer` stream.
14		pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
15		var d = decompressor(container, reader);
16		try d.decompress(writer);
17		}
18
19		/// Inflate decompressor for the reader type.
20		pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) {
21		return Decompressor(container, @TypeOf(reader)).init(reader);
22		}
23
24		pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type {
25		// zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
26		// gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
27		// For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
28		const lookahead: type = if (container == .zlib) u32 else u64;
29		return Inflate(container, lookahead, ReaderType);
30		}
31
32		/// Inflate decompresses deflate bit stream. Reads compressed data from reader
33		/// provided in init. Decompressed data are stored in internal hist buffer and
34		/// can be accesses iterable `next` or reader interface.
35		///
36		/// Container defines header/footer wrapper around deflate bit stream. Can be
37		/// gzip or zlib.
38		///
39		/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
40		/// * stored, non compressed, max 64k in size
41		/// * fixed, huffman codes are predefined
42		/// * dynamic, huffman code tables are encoded at the block start
43		///
44		/// `step` function runs decoder until internal `hist` buffer is full. Client
45		/// than needs to read that data in order to proceed with decoding.
46		///
47		/// Allocates 74.5K of internal buffers, most important are:
48		/// * 64K for history (CircularBuffer)
49		/// * ~10K huffman decoders (Literal and DistanceDecoder)
50		///
51		pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type {
52		assert(LookaheadType == u32 or LookaheadType == u64);
53		const BitReaderType = BitReader(LookaheadType, ReaderType);
54
55		return struct {
56		//const BitReaderType = BitReader(ReaderType);
57		const F = BitReaderType.flag;
58
59		bits: BitReaderType = .{},
60		hist: CircularBuffer = .{},
61		// Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
62		hasher: container.Hasher() = .{},
63
64		// dynamic block huffman code decoders
65		lit_dec: hfd.LiteralDecoder = .{}, // literals
66		dst_dec: hfd.DistanceDecoder = .{}, // distances
67
68		// current read state
69		bfinal: u1 = 0,
70		block_type: u2 = 0b11,
71		state: ReadState = .protocol_header,
72
73		const ReadState = enum {
74		protocol_header,
75		block_header,
76		block,
77		protocol_footer,
78		end,
79		};
80
81		const Self = @This();
82
83		pub const Error = BitReaderType.Error \|\| Container.Error \|\| hfd.Error \|\| error{
84		InvalidCode,
85		InvalidMatch,
86		InvalidBlockType,
87		WrongStoredBlockNlen,
88		InvalidDynamicBlockHeader,
89		};
90
91		pub fn init(rt: ReaderType) Self {
92		return .{ .bits = BitReaderType.init(rt) };
93		}
94
95		fn blockHeader(self: *Self) !void {
96		self.bfinal = try self.bits.read(u1);
97		self.block_type = try self.bits.read(u2);
98		}
99
100		fn storedBlock(self: *Self) !bool {
101		self.bits.alignToByte(); // skip padding until byte boundary
102		// everything after this is byte aligned in stored block
103		var len = try self.bits.read(u16);
104		const nlen = try self.bits.read(u16);
105		if (len != ~nlen) return error.WrongStoredBlockNlen;
106
107		while (len > 0) {
108		const buf = self.hist.getWritable(len);
109		try self.bits.readAll(buf);
110		len -= @intCast(buf.len);
111		}
112		return true;
113		}
114
115		fn fixedBlock(self: *Self) !bool {
116		while (!self.hist.full()) {
117		const code = try self.bits.readFixedCode();
118		switch (code) {
119		0...255 => self.hist.write(@intCast(code)),
120		256 => return true, // end of block
121		257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
122		else => return error.InvalidCode,
123		}
124		}
125		return false;
126		}
127
128		// Handles fixed block non literal (length) code.
129		// Length code is followed by 5 bits of distance code.
130		fn fixedDistanceCode(self: *Self, code: u8) !void {
131		try self.bits.fill(5 + 5 + 13);
132		const length = try self.decodeLength(code);
133		const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered \| F.reverse));
134		try self.hist.writeMatch(length, distance);
135		}
136
137		inline fn decodeLength(self: *Self, code: u8) !u16 {
138		if (code > 28) return error.InvalidCode;
139		const ml = Token.matchLength(code);
140		return if (ml.extra_bits == 0) // 0 - 5 extra bits
141		ml.base
142		else
143		ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
144		}
145
146		fn decodeDistance(self: *Self, code: u8) !u16 {
147		if (code > 29) return error.InvalidCode;
148		const md = Token.matchDistance(code);
149		return if (md.extra_bits == 0) // 0 - 13 extra bits
150		md.base
151		else
152		md.base + try self.bits.readN(md.extra_bits, F.buffered);
153		}
154
155		fn dynamicBlockHeader(self: *Self) !void {
156		const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
157		const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
158		const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
159
160		if (hlit > 286 or hdist > 30)
161		return error.InvalidDynamicBlockHeader;
162
163		// lengths for code lengths
164		var cl_lens = [_]u4{0} ** 19;
165		for (0..hclen) \|i\| {
166		cl_lens[codegen_order[i]] = try self.bits.read(u3);
167		}
168		var cl_dec: hfd.CodegenDecoder = .{};
169		try cl_dec.generate(&cl_lens);
170
171		// decoded code lengths
172		var dec_lens = [_]u4{0} ** (286 + 30);
173		var pos: usize = 0;
174		while (pos < hlit + hdist) {
175		const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176		try self.bits.shift(sym.code_bits);
177		pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
178		}
179		if (pos > hlit + hdist) {
180		return error.InvalidDynamicBlockHeader;
181		}
182
183		// literal code lengths to literal decoder
184		try self.lit_dec.generate(dec_lens[0..hlit]);
185
186		// distance code lengths to distance decoder
187		try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
188		}
189
190		// Decode code length symbol to code length. Writes decoded length into
191		// lens slice starting at position pos. Returns number of positions
192		// advanced.
193		fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
194		if (pos >= lens.len)
195		return error.InvalidDynamicBlockHeader;
196
197		switch (code) {
198		0...15 => {
199		// Represent code lengths of 0 - 15
200		lens[pos] = @intCast(code);
201		return 1;
202		},
203		16 => {
204		// Copy the previous code length 3 - 6 times.
205		// The next 2 bits indicate repeat length
206		const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
207		if (pos == 0 or pos + n > lens.len)
208		return error.InvalidDynamicBlockHeader;
209		for (0..n) \|i\| {
210		lens[pos + i] = lens[pos + i - 1];
211		}
212		return n;
213		},
214		// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
215		17 => return @as(u8, try self.bits.read(u3)) + 3,
216		// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
217		18 => return @as(u8, try self.bits.read(u7)) + 11,
218		else => return error.InvalidDynamicBlockHeader,
219		}
220		}
221
222		// In larger archives most blocks are usually dynamic, so decompression
223		// performance depends on this function.
224		fn dynamicBlock(self: *Self) !bool {
225		// Hot path loop!
226		while (!self.hist.full()) {
227		try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
228		const sym = try self.decodeSymbol(&self.lit_dec);
229
230		switch (sym.kind) {
231		.literal => self.hist.write(sym.symbol),
232		.match => { // Decode match backreference <length, distance>
233		// fill so we can use buffered reads
234		if (LookaheadType == u32)
235		try self.bits.fill(5 + 15)
236		else
237		try self.bits.fill(5 + 15 + 13);
238		const length = try self.decodeLength(sym.symbol);
239		const dsm = try self.decodeSymbol(&self.dst_dec);
240		if (LookaheadType == u32) try self.bits.fill(13);
241		const distance = try self.decodeDistance(dsm.symbol);
242		try self.hist.writeMatch(length, distance);
243		},
244		.end_of_block => return true,
245		}
246		}
247		return false;
248		}
249
250		// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
251		// decoder to find symbol for that code. We then know how many bits is
252		// used. Shift bit reader for that much bits, those bits are used. And
253		// return symbol.
254		fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
255		const sym = try decoder.find(try self.bits.peekF(u15, F.buffered \| F.reverse));
256		try self.bits.shift(sym.code_bits);
257		return sym;
258		}
259
260		fn step(self: *Self) !void {
261		switch (self.state) {
262		.protocol_header => {
263		try container.parseHeader(&self.bits);
264		self.state = .block_header;
265		},
266		.block_header => {
267		try self.blockHeader();
268		self.state = .block;
269		if (self.block_type == 2) try self.dynamicBlockHeader();
270		},
271		.block => {
272		const done = switch (self.block_type) {
273		0 => try self.storedBlock(),
274		1 => try self.fixedBlock(),
275		2 => try self.dynamicBlock(),
276		else => return error.InvalidBlockType,
277		};
278		if (done) {
279		self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
280		}
281		},
282		.protocol_footer => {
283		self.bits.alignToByte();
284		try container.parseFooter(&self.hasher, &self.bits);
285		self.state = .end;
286		},
287		.end => {},
288		}
289		}
290
291		/// Replaces the inner reader with new reader.
292		pub fn setReader(self: *Self, new_reader: ReaderType) void {
293		self.bits.forward_reader = new_reader;
294		if (self.state == .end or self.state == .protocol_footer) {
295		self.state = .protocol_header;
296		}
297		}
298
299		// Reads all compressed data from the internal reader and outputs plain
300		// (uncompressed) data to the provided writer.
301		pub fn decompress(self: *Self, writer: anytype) !void {
302		while (try self.next()) \|buf\| {
303		try writer.writeAll(buf);
304		}
305		}
306
307		/// Returns the number of bytes that have been read from the internal
308		/// reader but not yet consumed by the decompressor.
309		pub fn unreadBytes(self: Self) usize {
310		// There can be no error here: the denominator is not zero, and
311		// overflow is not possible since the type is unsigned.
312		return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
313		}
314
315		// Iterator interface
316
317		/// Can be used in iterator like loop without memcpy to another buffer:
318		/// while (try inflate.next()) \|buf\| { ... }
319		pub fn next(self: *Self) Error!?[]const u8 {
320		const out = try self.get(0);
321		if (out.len == 0) return null;
322		return out;
323		}
324
325		/// Returns decompressed data from internal sliding window buffer.
326		/// Returned buffer can be any length between 0 and `limit` bytes. 0
327		/// returned bytes means end of stream reached. With limit=0 returns as
328		/// much data it can. It newer will be more than 65536 bytes, which is
329		/// size of internal buffer.
330		pub fn get(self: *Self, limit: usize) Error![]const u8 {
331		while (true) {
332		const out = self.hist.readAtMost(limit);
333		if (out.len > 0) {
334		self.hasher.update(out);
335		return out;
336		}
337		if (self.state == .end) return out;
338		try self.step();
339		}
340		}
341
342		// Reader interface
343
344		pub const Reader = std.io.GenericReader(*Self, Error, read);
345
346		/// Returns the number of bytes read. It may be less than buffer.len.
347		/// If the number of bytes read is 0, it means end of stream.
348		/// End of stream is not an error condition.
349		pub fn read(self: *Self, buffer: []u8) Error!usize {
350		if (buffer.len == 0) return 0;
351		const out = try self.get(buffer.len);
352		@memcpy(buffer[0..out.len], out);
353		return out.len;
354		}
355
356		pub fn reader(self: *Self) Reader {
357		return .{ .context = self };
358		}
359		};
360		}
361
362		test "decompress" {
363		const cases = [_]struct {
364		in: []const u8,
365		out: []const u8,
366		}{
367		// non compressed block (type 0)
368		.{
369		.in = &[_]u8{
370		0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
371		'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
372		},
373		.out = "Hello world\n",
374		},
375		// fixed code block (type 1)
376		.{
377		.in = &[_]u8{
378		0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
379		0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
380		},
381		.out = "Hello world\n",
382		},
383		// dynamic block (type 2)
384		.{
385		.in = &[_]u8{
386		0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
387		0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
388		0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
389		},
390		.out = "ABCDEABCD ABCDEABCD",
391		},
392		};
393		for (cases) \|c\| {
394		var fb = std.io.fixedBufferStream(c.in);
395		var al = std.ArrayList(u8).init(testing.allocator);
396		defer al.deinit();
397
398		try decompress(.raw, fb.reader(), al.writer());
399		try testing.expectEqualStrings(c.out, al.items);
400		}
401		}
402
403		test "gzip decompress" {
404		const cases = [_]struct {
405		in: []const u8,
406		out: []const u8,
407		}{
408		// non compressed block (type 0)
409		.{
410		.in = &[_]u8{
411		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
412		0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
413		'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
414		0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
415		0x0c, 0x00, 0x00, 0x00, // gzip footer: size
416		},
417		.out = "Hello world\n",
418		},
419		// fixed code block (type 1)
420		.{
421		.in = &[_]u8{
422		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
423		0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
424		0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
425		0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
426		},
427		.out = "Hello world\n",
428		},
429		// dynamic block (type 2)
430		.{
431		.in = &[_]u8{
432		0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
433		0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
434		0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
435		0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
436		0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
437		},
438		.out = "ABCDEABCD ABCDEABCD",
439		},
440		// gzip header with name
441		.{
442		.in = &[_]u8{
443		0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
444		0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
445		0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
446		},
447		.out = "Hello world\n",
448		},
449		};
450		for (cases) \|c\| {
451		var fb = std.io.fixedBufferStream(c.in);
452		var al = std.ArrayList(u8).init(testing.allocator);
453		defer al.deinit();
454
455		try decompress(.gzip, fb.reader(), al.writer());
456		try testing.expectEqualStrings(c.out, al.items);
457		}
458		}
459
460		test "zlib decompress" {
461		const cases = [_]struct {
462		in: []const u8,
463		out: []const u8,
464		}{
465		// non compressed block (type 0)
466		.{
467		.in = &[_]u8{
468		0x78, 0b10_0_11100, // zlib header (2 bytes)
469		0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
470		'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
471		0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
472		},
473		.out = "Hello world\n",
474		},
475		};
476		for (cases) \|c\| {
477		var fb = std.io.fixedBufferStream(c.in);
478		var al = std.ArrayList(u8).init(testing.allocator);
479		defer al.deinit();
480
481		try decompress(.zlib, fb.reader(), al.writer());
482		try testing.expectEqualStrings(c.out, al.items);
483		}
484		}
485
486		test "fuzzing tests" {
487		const cases = [_]struct {
488		input: []const u8,
489		out: []const u8 = "",
490		err: ?anyerror = null,
491		}{
492		.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
493		.{ .input = "empty-distance-alphabet01" },
494		.{ .input = "empty-distance-alphabet02" },
495		.{ .input = "end-of-stream", .err = error.EndOfStream },
496		.{ .input = "invalid-distance", .err = error.InvalidMatch },
497		.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
498		.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
499		.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
500		.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
501		.{ .input = "out-of-codes", .err = error.InvalidCode },
502		.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
503		.{ .input = "puff02", .err = error.EndOfStream },
504		.{ .input = "puff03", .out = &[_]u8{0xa} },
505		.{ .input = "puff04", .err = error.InvalidCode },
506		.{ .input = "puff05", .err = error.EndOfStream },
507		.{ .input = "puff06", .err = error.EndOfStream },
508		.{ .input = "puff08", .err = error.InvalidCode },
509		.{ .input = "puff09", .out = "P" },
510		.{ .input = "puff10", .err = error.InvalidCode },
511		.{ .input = "puff11", .err = error.InvalidMatch },
512		.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
513		.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
514		.{ .input = "puff14", .err = error.EndOfStream },
515		.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
516		.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
517		.{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
518		.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
519		.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
520		.{ .input = "fuzz3", .err = error.InvalidMatch },
521		.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
522		.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
523		.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
524		.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
525		.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
526		.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
527		.{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
528		.{ .input = "puff24", .err = error.IncompleteHuffmanTree },
529		.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
530		.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
531		.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
532		};
533
534		inline for (cases, 0..) \|c, case_no\| {
535		var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
536		var out = std.ArrayList(u8).init(testing.allocator);
537		defer out.deinit();
538		errdefer std.debug.print("test case failed {}\n", .{case_no});
539
540		if (c.err) \|expected_err\| {
541		try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
542		} else {
543		try decompress(.raw, in.reader(), out.writer());
544		try testing.expectEqualStrings(c.out, out.items);
545		}
546		}
547		}
548
549		test "bug 18966" {
550		const input = @embedFile("testdata/fuzz/bug_18966.input");
551		const expect = @embedFile("testdata/fuzz/bug_18966.expect");
552
553		var in = std.io.fixedBufferStream(input);
554		var out = std.ArrayList(u8).init(testing.allocator);
555		defer out.deinit();
556
557		try decompress(.gzip, in.reader(), out.writer());
558		try testing.expectEqualStrings(expect, out.items);
559		}
560
561		test "bug 19895" {
562		const input = &[_]u8{
563		0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
564		'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
565		};
566		var in = std.io.fixedBufferStream(input);
567		var decomp = decompressor(.raw, in.reader());
568		var buf: [0]u8 = undefined;
569		try testing.expectEqual(0, try decomp.read(&buf));
570		}

lib/std/compress/flate/testdata/block_writer.zig deleted-606

...	...	@@ -1,606 +0,0 @@
1		const Token = @import("../Token.zig");
2
3		pub const TestCase = struct {
4		tokens: []const Token,
5		input: []const u8 = "", // File name of input data matching the tokens.
6		want: []const u8 = "", // File name of data with the expected output with input available.
7		want_no_input: []const u8 = "", // File name of the expected output when no input is available.
8		};
9
10		pub const testCases = blk: {
11		@setEvalBranchQuota(4096 * 2);
12
13		const L = Token.initLiteral;
14		const M = Token.initMatch;
15		const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests
16
17		break :blk &[_]TestCase{
18		TestCase{
19		.input = "huffman-null-max.input",
20		.want = "huffman-null-max.{s}.expect",
21		.want_no_input = "huffman-null-max.{s}.expect-noinput",
22		.tokens = &[_]Token{
23		L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
24		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
25		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
26		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
27		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
28		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
29		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
30		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
31		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
32		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
33		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
34		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
35		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0),
36		},
37		},
38		TestCase{
39		.input = "huffman-pi.input",
40		.want = "huffman-pi.{s}.expect",
41		.want_no_input = "huffman-pi.{s}.expect-noinput",
42		.tokens = &[_]Token{
43		L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'),
44		L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'),
45		L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'),
46		L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'),
47		L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'),
48		L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'),
49		L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'),
50		L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'),
51		L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'),
52		L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'),
53		L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'),
54		L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'),
55		L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'),
56		L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'),
57		L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'),
58		L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'),
59		L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'),
60		L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'),
61		L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'),
62		L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4),
63		L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'),
64		L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'),
65		L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'),
66		L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'),
67		L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'),
68		L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'),
69		L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4),
70		L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'),
71		L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'),
72		L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'),
73		L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'),
74		L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'),
75		L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4),
76		L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'),
77		L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'),
78		L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'),
79		L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'),
80		L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'),
81		L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'),
82		L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'),
83		L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'),
84		L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'),
85		L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'),
86		L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'),
87		L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'),
88		L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'),
89		L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'),
90		L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'),
91		L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'),
92		L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'),
93		L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'),
94		L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'),
95		M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'),
96		L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'),
97		L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'),
98		L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'),
99		L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'),
100		L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'),
101		L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'),
102		L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'),
103		L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'),
104		L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'),
105		L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'),
106		L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'),
107		L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'),
108		L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'),
109		L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'),
110		L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'),
111		L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'),
112		L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'),
113		M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'),
114		L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'),
115		L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'),
116		L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'),
117		L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'),
118		L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'),
119		L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'),
120		L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'),
121		L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'),
122		L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'),
123		L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'),
124		L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'),
125		L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'),
126		L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'),
127		L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'),
128		L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'),
129		L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'),
130		L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'),
131		L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'),
132		L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'),
133		M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'),
134		L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4),
135		M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'),
136		L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'),
137		L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'),
138		L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'),
139		L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'),
140		L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'),
141		L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'),
142		L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'),
143		L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4),
144		M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'),
145		L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'),
146		L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'),
147		L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'),
148		L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'),
149		M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'),
150		L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'),
151		L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'),
152		L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'),
153		L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'),
154		L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'),
155		L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'),
156		L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'),
157		L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'),
158		L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'),
159		L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'),
160		L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'),
161		L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'),
162		L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'),
163		L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'),
164		L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'),
165		L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'),
166		M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'),
167		L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'),
168		L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'),
169		M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'),
170		L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'),
171		L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'),
172		L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'),
173		L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'),
174		L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'),
175		L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'),
176		L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'),
177		M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5),
178		M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4),
179		L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'),
180		L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4),
181		L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6),
182		L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'),
183		L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'),
184		M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'),
185		L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4),
186		L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'),
187		L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'),
188		L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'),
189		L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'),
190		M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'),
191		L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'),
192		L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4),
193		L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'),
194		L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'),
195		L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'),
196		M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'),
197		L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'),
198		L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'),
199		M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'),
200		L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'),
201		L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4),
202		L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'),
203		M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'),
204		L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4),
205		L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4),
206		L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'),
207		L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'),
208		M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4),
209		L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'),
210		L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'),
211		L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'),
212		M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'),
213		L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'),
214		L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4),
215		L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'),
216		L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'),
217		L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4),
218		L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'),
219		L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'),
220		L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'),
221		L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'),
222		M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'),
223		L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'),
224		M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4),
225		L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'),
226		M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'),
227		L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'),
228		L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'),
229		L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'),
230		L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'),
231		M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'),
232		L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'),
233		L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'),
234		L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'),
235		L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'),
236		L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'),
237		L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'),
238		L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'),
239		L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'),
240		L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'),
241		M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'),
242		M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'),
243		L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'),
244		L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'),
245		M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'),
246		L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4),
247		L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'),
248		L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'),
249		L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'),
250		L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'),
251		L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4),
252		L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'),
253		L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'),
254		L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'),
255		L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4),
256		L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'),
257		M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'),
258		L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'),
259		L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'),
260		L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'),
261		L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'),
262		L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4),
263		L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'),
264		M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'),
265		L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'),
266		M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'),
267		L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'),
268		L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'),
269		L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'),
270		M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'),
271		L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'),
272		L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4),
273		L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'),
274		L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'),
275		L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'),
276		L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4),
277		L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'),
278		L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'),
279		L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'),
280		M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'),
281		L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'),
282		L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'),
283		M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'),
284		M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'),
285		L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'),
286		M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'),
287		L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4),
288		L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4),
289		L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'),
290		M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'),
291		L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'),
292		L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4),
293		L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'),
294		L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'),
295		M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'),
296		L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'),
297		L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'),
298		L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4),
299		L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4),
300		M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'),
301		L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4),
302		L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'),
303		L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'),
304		L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'),
305		L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'),
306		M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'),
307		L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'),
308		L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'),
309		L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'),
310		L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'),
311		L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'),
312		L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'),
313		L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'),
314		L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5),
315		M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4),
316		L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'),
317		M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'),
318		L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'),
319		L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'),
320		L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'),
321		L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'),
322		L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'),
323		M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4),
324		L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'),
325		L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'),
326		L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'),
327		L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'),
328		M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'),
329		L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4),
330		M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'),
331		L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'),
332		L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4),
333		L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4),
334		M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'),
335		L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'),
336		L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'),
337		L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'),
338		L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4),
339		L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4),
340		L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4),
341		L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'),
342		M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'),
343		L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'),
344		L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'),
345		L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'),
346		L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'),
347		M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4),
348		M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'),
349		L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'),
350		M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'),
351		L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'),
352		M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'),
353		M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'),
354		L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'),
355		L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'),
356		L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'),
357		L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4),
358		L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'),
359		M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'),
360		L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4),
361		M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'),
362		M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4),
363		L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'),
364		L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'),
365		L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'),
366		},
367		},
368		TestCase{
369		.input = "huffman-rand-1k.input",
370		.want = "huffman-rand-1k.{s}.expect",
371		.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
372		.tokens = &[_]Token{
373		L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d),
374		L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b),
375		L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4),
376		L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde),
377		L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14),
378		L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c),
379		L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5),
380		L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20),
381		L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4),
382		L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
383		L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0),
384		L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda),
385		L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb),
386		L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43),
387		L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95),
388		L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe),
389		L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1),
390		L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9),
391		L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66),
392		L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1),
393		L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47),
394		L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5),
395		L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8),
396		L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba),
397		L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb),
398		L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62),
399		L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2),
400		L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a),
401		L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b),
402		L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7),
403		L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba),
404		L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5),
405		L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14),
406		L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67),
407		L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70),
408		L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68),
409		L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1),
410		L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa),
411		L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab),
412		L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69),
413		L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37),
414		L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff),
415		L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3),
416		L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c),
417		L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1),
418		L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67),
419		L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8),
420		L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e),
421		L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6),
422		L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69),
423		L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e),
424		L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75),
425		L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0),
426		L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99),
427		L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55),
428		L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34),
429		L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a),
430		L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f),
431		L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8),
432		L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2),
433		L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb),
434		L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48),
435		L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e),
436		L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee),
437		L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90),
438		L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3),
439		L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0),
440		L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba),
441		L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef),
442		L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75),
443		L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75),
444		L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a),
445		},
446		},
447		TestCase{
448		.input = "huffman-rand-limit.input",
449		.want = "huffman-rand-limit.{s}.expect",
450		.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
451		.tokens = &[_]Token{
452		L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80),
453		L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde),
454		L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73),
455		L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15),
456		L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9),
457		L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa),
458		L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70),
459		L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59),
460		L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46),
461		L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b),
462		L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
463		L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f),
464		L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4),
465		L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa),
466		},
467		},
468		TestCase{
469		.input = "huffman-shifts.input",
470		.want = "huffman-shifts.{s}.expect",
471		.want_no_input = "huffman-shifts.{s}.expect-noinput",
472		.tokens = &[_]Token{
473		L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
474		M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
475		M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258),
476		M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256),
477		},
478		},
479		TestCase{
480		.input = "huffman-text-shift.input",
481		.want = "huffman-text-shift.{s}.expect",
482		.want_no_input = "huffman-text-shift.{s}.expect-noinput",
483		.tokens = &[_]Token{
484		L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'),
485		L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'),
486		L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'),
487		L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'),
488		L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'),
489		L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'),
490		L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'),
491		L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'),
492		L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'),
493		L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'),
494		L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'),
495		L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'),
496		L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'),
497		L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'),
498		L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'),
499		M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'),
500		L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'),
501		L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'),
502		L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'),
503		L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'),
504		L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'),
505		L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'),
506		L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'),
507		L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'),
508		L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa),
509		L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'),
510		L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'),
511		L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'),
512		L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'),
513		L('&'), L('/'), L('?'), L('"'),
514		},
515		},
516		TestCase{
517		.input = "huffman-text.input",
518		.want = "huffman-text.{s}.expect",
519		.want_no_input = "huffman-text.{s}.expect-noinput",
520		.tokens = &[_]Token{
521		L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'),
522		L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'),
523		L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'),
524		L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4),
525		L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'),
526		L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'),
527		L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '),
528		L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'),
529		L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'),
530		L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'),
531		L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'),
532		L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'),
533		L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'),
534		L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'),
535		L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L(''), L(''),
536		L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5),
537		M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'),
538		M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'),
539		L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'),
540		L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'),
541		L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'),
542		L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'),
543		L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'),
544		L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6),
545		L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '),
546		L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('),
547		M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'),
548		L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'),
549		L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa),
550		L('}'), L(0xa),
551		},
552		},
553		TestCase{
554		.input = "huffman-zero.input",
555		.want = "huffman-zero.{s}.expect",
556		.want_no_input = "huffman-zero.{s}.expect-noinput",
557		.tokens = &[_]Token{ L(0x30), ml, M(1, 49) },
558		},
559		TestCase{
560		.input = "",
561		.want = "",
562		.want_no_input = "null-long-match.{s}.expect-noinput",
563		.tokens = &[_]Token{
564		L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
565		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
566		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
567		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
568		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
569		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
570		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
571		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
572		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
573		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
574		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
575		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
576		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
577		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
578		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
579		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
580		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
581		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
582		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
583		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
584		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
585		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
586		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
587		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
588		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
589		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
590		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
591		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
592		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
593		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
594		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
595		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
596		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
597		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
598		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
599		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
600		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
601		ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
602		ml, ml, ml, M(1, 8),
603		},
604		},
605		};
606		};

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.input deleted-1

...	...	@@ -1 +0,0 @@
1		3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
		\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input deleted-4

...	...	@@ -1,4 +0,0 @@
1		aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2		��vH
3		��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4		�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.input deleted-2

...	...	@@ -1,2 +0,0 @@
1		101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2		232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
		\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input deleted-14

...	...	@@ -1,14 +0,0 @@
1		//Copyright2009ThGoAuthor.Allrightrrvd.
2		//UofthiourccodigovrndbyBSD-tyl
3		//licnthtcnbfoundinthLICENSEfil.
4
5		pckgmin
6
7		import"o"
8
9		funcmin(){
10		vrb=mk([]byt,65535)
11		f,_:=o.Crt("huffmn-null-mx.in")
12		f.Writ(b)
13		}
14		ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
		\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.input deleted-14

...	...	@@ -1,14 +0,0 @@
1		// zig v0.10.0
2		// create a file filled with 0x00
3		const std = @import("std");
4
5		pub fn main() !void {
6		var b = [1]u8{0} ** 65535;
7		const f = try std.fs.cwd().createFile(
8		"huffman-null-max.in",
9		.{ .read = true },
10		);
11		defer f.close();
12
13		_ = try f.writeAll(b[0..]);
14		}

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.input deleted-1

...	...	@@ -1 +0,0 @@
1		00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
		\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect and /dev/null differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput and /dev/null differ

lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput deleted

Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput and /dev/null differ

lib/std/compress/gzip.zig deleted-66

...	...	@@ -1,66 +0,0 @@
1		const deflate = @import("flate/deflate.zig");
2		const inflate = @import("flate/inflate.zig");
3
4		/// Decompress compressed data from reader and write plain data to the writer.
5		pub fn decompress(reader: anytype, writer: anytype) !void {
6		try inflate.decompress(.gzip, reader, writer);
7		}
8
9		/// Decompressor type
10		pub fn Decompressor(comptime ReaderType: type) type {
11		return inflate.Decompressor(.gzip, ReaderType);
12		}
13
14		/// Create Decompressor which will read compressed data from reader.
15		pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16		return inflate.decompressor(.gzip, reader);
17		}
18
19		/// Compression level, trades between speed and compression size.
20		pub const Options = deflate.Options;
21
22		/// Compress plain data from reader and write compressed data to the writer.
23		pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24		try deflate.compress(.gzip, reader, writer, options);
25		}
26
27		/// Compressor type
28		pub fn Compressor(comptime WriterType: type) type {
29		return deflate.Compressor(.gzip, WriterType);
30		}
31
32		/// Create Compressor which outputs compressed data to the writer.
33		pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34		return try deflate.compressor(.gzip, writer, options);
35		}
36
37		/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38		/// compression, less memory requirements but bigger compressed sizes.
39		pub const huffman = struct {
40		pub fn compress(reader: anytype, writer: anytype) !void {
41		try deflate.huffman.compress(.gzip, reader, writer);
42		}
43
44		pub fn Compressor(comptime WriterType: type) type {
45		return deflate.huffman.Compressor(.gzip, WriterType);
46		}
47
48		pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49		return deflate.huffman.compressor(.gzip, writer);
50		}
51		};
52
53		// No compression store only. Compressed size is slightly bigger than plain.
54		pub const store = struct {
55		pub fn compress(reader: anytype, writer: anytype) !void {
56		try deflate.store.compress(.gzip, reader, writer);
57		}
58
59		pub fn Compressor(comptime WriterType: type) type {
60		return deflate.store.Compressor(.gzip, WriterType);
61		}
62
63		pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64		return deflate.store.compressor(.gzip, writer);
65		}
66		};

lib/std/compress/zlib.zig deleted-101

...	...	@@ -1,101 +0,0 @@
1		const deflate = @import("flate/deflate.zig");
2		const inflate = @import("flate/inflate.zig");
3
4		/// Decompress compressed data from reader and write plain data to the writer.
5		pub fn decompress(reader: anytype, writer: anytype) !void {
6		try inflate.decompress(.zlib, reader, writer);
7		}
8
9		/// Decompressor type
10		pub fn Decompressor(comptime ReaderType: type) type {
11		return inflate.Decompressor(.zlib, ReaderType);
12		}
13
14		/// Create Decompressor which will read compressed data from reader.
15		pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16		return inflate.decompressor(.zlib, reader);
17		}
18
19		/// Compression level, trades between speed and compression size.
20		pub const Options = deflate.Options;
21
22		/// Compress plain data from reader and write compressed data to the writer.
23		pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24		try deflate.compress(.zlib, reader, writer, options);
25		}
26
27		/// Compressor type
28		pub fn Compressor(comptime WriterType: type) type {
29		return deflate.Compressor(.zlib, WriterType);
30		}
31
32		/// Create Compressor which outputs compressed data to the writer.
33		pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34		return try deflate.compressor(.zlib, writer, options);
35		}
36
37		/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38		/// compression, less memory requirements but bigger compressed sizes.
39		pub const huffman = struct {
40		pub fn compress(reader: anytype, writer: anytype) !void {
41		try deflate.huffman.compress(.zlib, reader, writer);
42		}
43
44		pub fn Compressor(comptime WriterType: type) type {
45		return deflate.huffman.Compressor(.zlib, WriterType);
46		}
47
48		pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49		return deflate.huffman.compressor(.zlib, writer);
50		}
51		};
52
53		// No compression store only. Compressed size is slightly bigger than plain.
54		pub const store = struct {
55		pub fn compress(reader: anytype, writer: anytype) !void {
56		try deflate.store.compress(.zlib, reader, writer);
57		}
58
59		pub fn Compressor(comptime WriterType: type) type {
60		return deflate.store.Compressor(.zlib, WriterType);
61		}
62
63		pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64		return deflate.store.compressor(.zlib, writer);
65		}
66		};
67
68		test "should not overshoot" {
69		const std = @import("std");
70
71		// Compressed zlib data with extra 4 bytes at the end.
72		const data = [_]u8{
73		0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
74		0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
75		0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
76		0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
77		};
78
79		var stream = std.io.fixedBufferStream(data[0..]);
80		const reader = stream.reader();
81
82		var dcp = decompressor(reader);
83		var out: [128]u8 = undefined;
84
85		// Decompress
86		var n = try dcp.reader().readAll(out[0..]);
87
88		// Expected decompressed data
89		try std.testing.expectEqual(46, n);
90		try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
91
92		// Decompressor don't overshoot underlying reader.
93		// It is leaving it at the end of compressed data chunk.
94		try std.testing.expectEqual(data.len - 4, stream.getPos());
95		try std.testing.expectEqual(0, dcp.unreadBytes());
96
97		// 4 bytes after compressed chunk are available in reader.
98		n = try reader.readAll(out[0..]);
99		try std.testing.expectEqual(n, 4);
100		try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
101		}

lib/std/compress/zstd/Decompress.zig+36-5

...	...	@@ -89,7 +89,7 @@ pub fn init(input: *Reader, buffer: []u8, options: Options) Decompress {
89	89	.stream = stream,
90	90	.rebase = rebase,
91	91	.discard = discard,
92		.readVec = Reader.indirectReadVec,
	92	.readVec = readVec,
93	93	},
94	94	.buffer = buffer,
95	95	.seek = 0,
...	...	@@ -109,10 +109,24 @@ fn rebase(r: *Reader, capacity: usize) Reader.RebaseError!void {
109	109	r.seek -= discard_n;
110	110	}
111	111
112		fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
113		r.rebase(zstd.block_size_max) catch unreachable;
114		var d: Writer.Discarding = .init(r.buffer);
115		const n = r.stream(&d.writer, limit) catch \|err\| switch (err) {
	112	/// This could be improved so that when an amount is discarded that includes an
	113	/// entire frame, skip decoding that frame.
	114	fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
	115	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	116	r.rebase(d.window_len) catch unreachable;
	117	var writer: Writer = .{
	118	.vtable = &.{
	119	.drain = std.Io.Writer.Discarding.drain,
	120	.sendFile = std.Io.Writer.Discarding.sendFile,
	121	},
	122	.buffer = r.buffer,
	123	.end = r.end,
	124	};
	125	defer {
	126	r.end = writer.end;
	127	r.seek = r.end;
	128	}
	129	const n = r.stream(&writer, limit) catch \|err\| switch (err) {
116	130	error.WriteFailed => unreachable,
117	131	error.ReadFailed => return error.ReadFailed,
118	132	error.EndOfStream => return error.EndOfStream,
...	...	@@ -121,6 +135,23 @@ fn discard(r: *Reader, limit: Limit) Reader.Error!usize {
121	135	return n;
122	136	}
123	137
	138	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
	139	_ = data;
	140	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	141	assert(r.seek == r.end);
	142	r.rebase(d.window_len) catch unreachable;
	143	var writer: Writer = .{
	144	.buffer = r.buffer,
	145	.end = r.end,
	146	.vtable = &.{ .drain = Writer.fixedDrain },
	147	};
	148	r.end += r.vtable.stream(r, &writer, .limited(writer.buffer.len - writer.end)) catch \|err\| switch (err) {
	149	error.WriteFailed => unreachable,
	150	else => \|e\| return e,
	151	};
	152	return 0;
	153	}
	154
124	155	fn stream(r: Reader, w: Writer, limit: Limit) Reader.StreamError!usize {
125	156	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
126	157	const in = d.input;

lib/std/debug/Dwarf.zig+19-13

...	...	@@ -2019,10 +2019,14 @@ pub fn compactUnwindToDwarfRegNumber(unwind_reg_number: u3) !u8 {
2019	2019	/// This function is to make it handy to comment out the return and make it
2020	2020	/// into a crash when working on this file.
2021	2021	pub fn bad() error{InvalidDebugInfo} {
2022		if (debug_debug_mode) @panic("bad dwarf");
	2022	invalidDebugInfoDetected();
2023	2023	return error.InvalidDebugInfo;
2024	2024	}
2025	2025
	2026	fn invalidDebugInfoDetected() void {
	2027	if (debug_debug_mode) @panic("bad dwarf");
	2028	}
	2029
2026	2030	fn missing() error{MissingDebugInfo} {
2027	2031	if (debug_debug_mode) @panic("missing dwarf");
2028	2032	return error.MissingDebugInfo;
...	...	@@ -2235,21 +2239,23 @@ pub const ElfModule = struct {
2235	2239
2236	2240	const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size);
2237	2241	sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: {
2238		var section_stream = std.io.fixedBufferStream(section_bytes);
2239		const section_reader = section_stream.reader();
2240		const chdr = section_reader.readStruct(elf.Chdr) catch continue;
	2242	var section_reader: std.Io.Reader = .fixed(section_bytes);
	2243	const chdr = section_reader.takeStruct(elf.Chdr, endian) catch continue;
2241	2244	if (chdr.ch_type != .ZLIB) continue;
2242	2245
2243		var zlib_stream = std.compress.zlib.decompressor(section_reader);
2244
2245		const decompressed_section = try gpa.alloc(u8, chdr.ch_size);
2246		errdefer gpa.free(decompressed_section);
2247
2248		const read = zlib_stream.reader().readAll(decompressed_section) catch continue;
2249		assert(read == decompressed_section.len);
2250
	2246	var decompress: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});
	2247	var decompressed_section: std.ArrayListUnmanaged(u8) = .empty;
	2248	defer decompressed_section.deinit(gpa);
	2249	decompress.reader.appendRemainingUnlimited(gpa, null, &decompressed_section, std.compress.flate.history_len) catch {
	2250	invalidDebugInfoDetected();
	2251	continue;
	2252	};
	2253	if (chdr.ch_size != decompressed_section.items.len) {
	2254	invalidDebugInfoDetected();
	2255	continue;
	2256	}
2251	2257	break :blk .{
2252		.data = decompressed_section,
	2258	.data = try decompressed_section.toOwnedSlice(gpa),
2253	2259	.virtual_address = shdr.sh_addr,
2254	2260	.owned = true,
2255	2261	};

lib/std/fs/File.zig+1-17

...	...	@@ -1105,22 +1105,6 @@ pub fn deprecatedWriter(file: File) DeprecatedWriter {
1105	1105	return .{ .context = file };
1106	1106	}
1107	1107
1108		/// Deprecated in favor of `Reader` and `Writer`.
1109		pub const SeekableStream = io.SeekableStream(
1110		File,
1111		SeekError,
1112		GetSeekPosError,
1113		seekTo,
1114		seekBy,
1115		getPos,
1116		getEndPos,
1117		);
1118
1119		/// Deprecated in favor of `Reader` and `Writer`.
1120		pub fn seekableStream(file: File) SeekableStream {
1121		return .{ .context = file };
1122		}
1123
1124	1108	/// Memoizes key information about a file handle such as:
1125	1109	/// * The size from calling stat, or the error that occurred therein.
1126	1110	/// * The current seek position.
...	...	@@ -1321,7 +1305,7 @@ pub const Reader = struct {
1321	1305	}
1322	1306	}
1323	1307
1324		fn readVec(io_reader: *std.Io.Reader, data: []const []u8) std.Io.Reader.Error!usize {
	1308	fn readVec(io_reader: *std.Io.Reader, data: [][]u8) std.Io.Reader.Error!usize {
1325	1309	const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader));
1326	1310	switch (r.mode) {
1327	1311	.positional, .positional_reading => {

lib/std/hash.zig+2-3

...	...	@@ -1,5 +1,4 @@
1		const adler = @import("hash/adler.zig");
2		pub const Adler32 = adler.Adler32;
	1	pub const Adler32 = @import("hash/Adler32.zig");
3	2
4	3	const auto_hash = @import("hash/auto_hash.zig");
5	4	pub const autoHash = auto_hash.autoHash;
...	...	@@ -116,7 +115,7 @@ test int {
116	115	}
117	116
118	117	test {
119		_ = adler;
	118	_ = Adler32;
120	119	_ = auto_hash;
121	120	_ = crc;
122	121	_ = fnv;

lib/std/hash/Adler32.zig created+117

...	...	@@ -0,0 +1,117 @@
	1	//! https://tools.ietf.org/html/rfc1950#section-9
	2	//! https://github.com/madler/zlib/blob/master/adler32.c
	3
	4	const Adler32 = @This();
	5	const std = @import("std");
	6	const testing = std.testing;
	7
	8	adler: u32 = 1,
	9
	10	pub fn permute(state: u32, input: []const u8) u32 {
	11	const base = 65521;
	12	const nmax = 5552;
	13
	14	var s1 = state & 0xffff;
	15	var s2 = (state >> 16) & 0xffff;
	16
	17	if (input.len == 1) {
	18	s1 +%= input[0];
	19	if (s1 >= base) {
	20	s1 -= base;
	21	}
	22	s2 +%= s1;
	23	if (s2 >= base) {
	24	s2 -= base;
	25	}
	26	} else if (input.len < 16) {
	27	for (input) \|b\| {
	28	s1 +%= b;
	29	s2 +%= s1;
	30	}
	31	if (s1 >= base) {
	32	s1 -= base;
	33	}
	34
	35	s2 %= base;
	36	} else {
	37	const n = nmax / 16; // note: 16 \| nmax
	38
	39	var i: usize = 0;
	40
	41	while (i + nmax <= input.len) {
	42	var rounds: usize = 0;
	43	while (rounds < n) : (rounds += 1) {
	44	comptime var j: usize = 0;
	45	inline while (j < 16) : (j += 1) {
	46	s1 +%= input[i + j];
	47	s2 +%= s1;
	48	}
	49	i += 16;
	50	}
	51
	52	s1 %= base;
	53	s2 %= base;
	54	}
	55
	56	if (i < input.len) {
	57	while (i + 16 <= input.len) : (i += 16) {
	58	comptime var j: usize = 0;
	59	inline while (j < 16) : (j += 1) {
	60	s1 +%= input[i + j];
	61	s2 +%= s1;
	62	}
	63	}
	64	while (i < input.len) : (i += 1) {
	65	s1 +%= input[i];
	66	s2 +%= s1;
	67	}
	68
	69	s1 %= base;
	70	s2 %= base;
	71	}
	72	}
	73
	74	return s1 \| (s2 << 16);
	75	}
	76
	77	pub fn update(a: *Adler32, input: []const u8) void {
	78	a.adler = permute(a.adler, input);
	79	}
	80
	81	pub fn hash(input: []const u8) u32 {
	82	return permute(1, input);
	83	}
	84
	85	test "sanity" {
	86	try testing.expectEqual(@as(u32, 0x620062), hash("a"));
	87	try testing.expectEqual(@as(u32, 0xbc002ed), hash("example"));
	88	}
	89
	90	test "long" {
	91	const long1 = [_]u8{1} ** 1024;
	92	try testing.expectEqual(@as(u32, 0x06780401), hash(long1[0..]));
	93
	94	const long2 = [_]u8{1} ** 1025;
	95	try testing.expectEqual(@as(u32, 0x0a7a0402), hash(long2[0..]));
	96	}
	97
	98	test "very long" {
	99	const long = [_]u8{1} ** 5553;
	100	try testing.expectEqual(@as(u32, 0x707f15b2), hash(long[0..]));
	101	}
	102
	103	test "very long with variation" {
	104	const long = comptime blk: {
	105	@setEvalBranchQuota(7000);
	106	var result: [6000]u8 = undefined;
	107
	108	var i: usize = 0;
	109	while (i < result.len) : (i += 1) {
	110	result[i] = @as(u8, @truncate(i));
	111	}
	112
	113	break :blk result;
	114	};
	115
	116	try testing.expectEqual(@as(u32, 0x5af38d6e), hash(long[0..]));
	117	}

lib/std/hash/adler.zig deleted-134

...	...	@@ -1,134 +0,0 @@
1		// Adler32 checksum.
2		//
3		// https://tools.ietf.org/html/rfc1950#section-9
4		// https://github.com/madler/zlib/blob/master/adler32.c
5
6		const std = @import("std");
7		const testing = std.testing;
8
9		pub const Adler32 = struct {
10		const base = 65521;
11		const nmax = 5552;
12
13		adler: u32,
14
15		pub fn init() Adler32 {
16		return Adler32{ .adler = 1 };
17		}
18
19		// This fast variant is taken from zlib. It reduces the required modulos and unrolls longer
20		// buffer inputs and should be much quicker.
21		pub fn update(self: *Adler32, input: []const u8) void {
22		var s1 = self.adler & 0xffff;
23		var s2 = (self.adler >> 16) & 0xffff;
24
25		if (input.len == 1) {
26		s1 +%= input[0];
27		if (s1 >= base) {
28		s1 -= base;
29		}
30		s2 +%= s1;
31		if (s2 >= base) {
32		s2 -= base;
33		}
34		} else if (input.len < 16) {
35		for (input) \|b\| {
36		s1 +%= b;
37		s2 +%= s1;
38		}
39		if (s1 >= base) {
40		s1 -= base;
41		}
42
43		s2 %= base;
44		} else {
45		const n = nmax / 16; // note: 16 \| nmax
46
47		var i: usize = 0;
48
49		while (i + nmax <= input.len) {
50		var rounds: usize = 0;
51		while (rounds < n) : (rounds += 1) {
52		comptime var j: usize = 0;
53		inline while (j < 16) : (j += 1) {
54		s1 +%= input[i + j];
55		s2 +%= s1;
56		}
57		i += 16;
58		}
59
60		s1 %= base;
61		s2 %= base;
62		}
63
64		if (i < input.len) {
65		while (i + 16 <= input.len) : (i += 16) {
66		comptime var j: usize = 0;
67		inline while (j < 16) : (j += 1) {
68		s1 +%= input[i + j];
69		s2 +%= s1;
70		}
71		}
72		while (i < input.len) : (i += 1) {
73		s1 +%= input[i];
74		s2 +%= s1;
75		}
76
77		s1 %= base;
78		s2 %= base;
79		}
80		}
81
82		self.adler = s1 \| (s2 << 16);
83		}
84
85		pub fn final(self: *Adler32) u32 {
86		return self.adler;
87		}
88
89		pub fn hash(input: []const u8) u32 {
90		var c = Adler32.init();
91		c.update(input);
92		return c.final();
93		}
94		};
95
96		test "adler32 sanity" {
97		try testing.expectEqual(@as(u32, 0x620062), Adler32.hash("a"));
98		try testing.expectEqual(@as(u32, 0xbc002ed), Adler32.hash("example"));
99		}
100
101		test "adler32 long" {
102		const long1 = [_]u8{1} ** 1024;
103		try testing.expectEqual(@as(u32, 0x06780401), Adler32.hash(long1[0..]));
104
105		const long2 = [_]u8{1} ** 1025;
106		try testing.expectEqual(@as(u32, 0x0a7a0402), Adler32.hash(long2[0..]));
107		}
108
109		test "adler32 very long" {
110		const long = [_]u8{1} ** 5553;
111		try testing.expectEqual(@as(u32, 0x707f15b2), Adler32.hash(long[0..]));
112		}
113
114		test "adler32 very long with variation" {
115		const long = comptime blk: {
116		@setEvalBranchQuota(7000);
117		var result: [6000]u8 = undefined;
118
119		var i: usize = 0;
120		while (i < result.len) : (i += 1) {
121		result[i] = @as(u8, @truncate(i));
122		}
123
124		break :blk result;
125		};
126
127		try testing.expectEqual(@as(u32, 0x5af38d6e), std.hash.Adler32.hash(long[0..]));
128		}
129
130		const verify = @import("verify.zig");
131
132		test "adler32 iterative" {
133		try verify.iterativeApi(Adler32);
134		}

lib/std/hash/verify.zig+1-1

...	...	@@ -45,7 +45,7 @@ pub fn smhasher(comptime hash_fn: anytype) u32 {
45	45
46	46	pub fn iterativeApi(comptime Hash: anytype) !void {
47	47	// Sum(1..32) = 528
48		var buf: [528]u8 = [_]u8{0} ** 528;
	48	var buf: [528]u8 = @splat(0);
49	49	var len: usize = 0;
50	50	const seed = 0;
51	51

lib/std/http/Client.zig+9-15

...	...	@@ -405,13 +405,8 @@ pub const RequestTransfer = union(enum) {
405	405
406	406	/// The decompressor for response messages.
407	407	pub const Compression = union(enum) {
408		pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
409		pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
410		// https://github.com/ziglang/zig/issues/18937
411		//pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
412
413		deflate: DeflateDecompressor,
414		gzip: GzipDecompressor,
	408	//deflate: std.compress.flate.Decompress,
	409	//gzip: std.compress.flate.Decompress,
415	410	// https://github.com/ziglang/zig/issues/18937
416	411	//zstd: ZstdDecompressor,
417	412	none: void,
...	...	@@ -1079,12 +1074,10 @@ pub const Request = struct {
1079	1074	switch (req.response.transfer_compression) {
1080	1075	.identity => req.response.compression = .none,
1081	1076	.compress, .@"x-compress" => return error.CompressionUnsupported,
1082		.deflate => req.response.compression = .{
1083		.deflate = std.compress.zlib.decompressor(req.transferReader()),
1084		},
1085		.gzip, .@"x-gzip" => req.response.compression = .{
1086		.gzip = std.compress.gzip.decompressor(req.transferReader()),
1087		},
	1077	// I'm about to upstream my http.Client rewrite
	1078	.deflate => return error.CompressionUnsupported,
	1079	// I'm about to upstream my http.Client rewrite
	1080	.gzip, .@"x-gzip" => return error.CompressionUnsupported,
1088	1081	// https://github.com/ziglang/zig/issues/18937
1089	1082	//.zstd => req.response.compression = .{
1090	1083	// .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),
...	...	@@ -1110,8 +1103,9 @@ pub const Request = struct {
1110	1103	/// Reads data from the response body. Must be called after `wait`.
1111	1104	pub fn read(req: *Request, buffer: []u8) ReadError!usize {
1112	1105	const out_index = switch (req.response.compression) {
1113		.deflate => \|*deflate\| deflate.read(buffer) catch return error.DecompressionFailure,
1114		.gzip => \|*gzip\| gzip.read(buffer) catch return error.DecompressionFailure,
	1106	// I'm about to upstream my http client rewrite
	1107	//.deflate => \|*deflate\| deflate.readSlice(buffer) catch return error.DecompressionFailure,
	1108	//.gzip => \|*gzip\| gzip.read(buffer) catch return error.DecompressionFailure,
1115	1109	// https://github.com/ziglang/zig/issues/18937
1116	1110	//.zstd => \|*zstd\| zstd.read(buffer) catch return error.DecompressionFailure,
1117	1111	else => try req.transferRead(buffer),

lib/std/http/Server.zig+2-2

...	...	@@ -130,8 +130,8 @@ pub const Request = struct {
130	130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131	131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132	132
133		deflate: DeflateDecompressor,
134		gzip: GzipDecompressor,
	133	deflate: std.compress.flate.Decompress,
	134	gzip: std.compress.flate.Decompress,
135	135	zstd: std.compress.zstd.Decompress,
136	136	none: void,
137	137	};

lib/std/net.zig+3-2

...	...	@@ -1973,12 +1973,13 @@ pub const Stream = struct {
1973	1973
1974	1974	fn stream(io_r: Io.Reader, io_w: Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize {
1975	1975	const dest = limit.slice(try io_w.writableSliceGreedy(1));
1976		const n = try readVec(io_r, &.{dest});
	1976	var bufs: [1][]u8 = .{dest};
	1977	const n = try readVec(io_r, &bufs);
1977	1978	io_w.advance(n);
1978	1979	return n;
1979	1980	}
1980	1981
1981		fn readVec(io_r: *std.Io.Reader, data: []const []u8) Io.Reader.Error!usize {
	1982	fn readVec(io_r: *std.Io.Reader, data: [][]u8) Io.Reader.Error!usize {
1982	1983	const r: *Reader = @alignCast(@fieldParentPtr("interface_state", io_r));
1983	1984	var iovecs: [max_buffers_len]windows.ws2_32.WSABUF = undefined;
1984	1985	const bufs_n, const data_size = try io_r.writableVectorWsa(&iovecs, data);

lib/std/zip.zig+392-535

...	...	@@ -5,11 +5,11 @@
5	5
6	6	const builtin = @import("builtin");
7	7	const std = @import("std");
8		const testing = std.testing;
9
10		pub const testutil = @import("zip/test.zig");
11		const File = testutil.File;
12		const FileStore = testutil.FileStore;
	8	const File = std.fs.File;
	9	const is_le = builtin.target.cpu.arch.endian() == .little;
	10	const Writer = std.Io.Writer;
	11	const Reader = std.Io.Reader;
	12	const flate = std.compress.flate;
13	13
14	14	pub const CompressionMethod = enum(u16) {
15	15	store = 0,
...	...	@@ -95,102 +95,116 @@ pub const EndRecord = extern struct {
95	95	central_directory_size: u32 align(1),
96	96	central_directory_offset: u32 align(1),
97	97	comment_len: u16 align(1),
	98
98	99	pub fn need_zip64(self: EndRecord) bool {
99	100	return isMaxInt(self.record_count_disk) or
100	101	isMaxInt(self.record_count_total) or
101	102	isMaxInt(self.central_directory_size) or
102	103	isMaxInt(self.central_directory_offset);
103	104	}
104		};
105	105
106		/// Find and return the end record for the given seekable zip stream.
107		/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and
108		/// its context must also have a `.reader()` method that returns an instance of
109		/// `std.io.GenericReader`.
110		pub fn findEndRecord(seekable_stream: anytype, stream_len: u64) !EndRecord {
111		var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
112		const record_len_max = @min(stream_len, buf.len);
113		var loaded_len: u32 = 0;
114
115		var comment_len: u16 = 0;
116		while (true) {
117		const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
118		if (record_len > record_len_max)
119		return error.ZipNoEndRecord;
120
121		if (record_len > loaded_len) {
122		const new_loaded_len = @min(loaded_len + 300, record_len_max);
123		const read_len = new_loaded_len - loaded_len;
124
125		try seekable_stream.seekTo(stream_len - @as(u64, new_loaded_len));
126		const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
127		const len = try (if (@TypeOf(seekable_stream.context) == std.fs.File) seekable_stream.context.deprecatedReader() else seekable_stream.context.reader()).readAll(read_buf);
128		if (len != read_len)
129		return error.ZipTruncated;
130		loaded_len = new_loaded_len;
131		}
	106	pub const FindBufferError = error{ ZipNoEndRecord, ZipTruncated };
132	107
133		const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];
134		if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and
135		std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)
136		{
137		const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);
138		if (builtin.target.cpu.arch.endian() != .little) {
139		std.mem.byteSwapAllFields(@TypeOf(record.*), record);
	108	/// TODO audit this logic
	109	pub fn findBuffer(buffer: []const u8) FindBufferError!EndRecord {
	110	const pos = std.mem.lastIndexOf(u8, buffer, &end_record_sig) orelse return error.ZipNoEndRecord;
	111	if (pos + @sizeOf(EndRecord) > buffer.len) return error.EndOfStream;
	112	const record_ptr: *EndRecord = @ptrCast(buffer[pos..][0..@sizeOf(EndRecord)]);
	113	var record = record_ptr.*;
	114	if (!is_le) std.mem.byteSwapAllFields(EndRecord, &record);
	115	return record;
	116	}
	117
	118	pub const FindFileError = File.GetEndPosError \|\| File.SeekError \|\| File.ReadError \|\| error{
	119	ZipNoEndRecord,
	120	EndOfStream,
	121	ReadFailed,
	122	};
	123
	124	pub fn findFile(fr: *File.Reader) FindFileError!EndRecord {
	125	const end_pos = try fr.getSize();
	126
	127	var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
	128	const record_len_max = @min(end_pos, buf.len);
	129	var loaded_len: u32 = 0;
	130	var comment_len: u16 = 0;
	131	while (true) {
	132	const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
	133	if (record_len > record_len_max)
	134	return error.ZipNoEndRecord;
	135
	136	if (record_len > loaded_len) {
	137	const new_loaded_len = @min(loaded_len + 300, record_len_max);
	138	const read_len = new_loaded_len - loaded_len;
	139
	140	try fr.seekTo(end_pos - @as(u64, new_loaded_len));
	141	const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
	142	fr.interface.readSliceAll(read_buf) catch \|err\| switch (err) {
	143	error.ReadFailed => return fr.err.?,
	144	error.EndOfStream => return error.EndOfStream,
	145	};
	146	loaded_len = new_loaded_len;
	147	}
	148
	149	const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];
	150	if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and
	151	std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)
	152	{
	153	const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);
	154	if (!is_le) std.mem.byteSwapAllFields(EndRecord, record);
	155	return record.*;
140	156	}
141		return record.*;
	157
	158	if (comment_len == std.math.maxInt(u16))
	159	return error.ZipNoEndRecord;
	160	comment_len += 1;
142	161	}
	162	}
	163	};
143	164
144		if (comment_len == std.math.maxInt(u16))
145		return error.ZipNoEndRecord;
146		comment_len += 1;
	165	pub const Decompress = struct {
	166	interface: Reader,
	167	state: union {
	168	inflate: flate.Decompress,
	169	store: *Reader,
	170	},
	171
	172	pub fn init(reader: *Reader, method: CompressionMethod, buffer: []u8) Reader {
	173	return switch (method) {
	174	.store => .{
	175	.state = .{ .store = reader },
	176	.interface = .{
	177	.context = undefined,
	178	.vtable = &.{ .stream = streamStore },
	179	.buffer = buffer,
	180	.end = 0,
	181	.seek = 0,
	182	},
	183	},
	184	.deflate => .{
	185	.state = .{ .inflate = .init(reader, .raw) },
	186	.interface = .{
	187	.context = undefined,
	188	.vtable = &.{ .stream = streamDeflate },
	189	.buffer = buffer,
	190	.end = 0,
	191	.seek = 0,
	192	},
	193	},
	194	else => unreachable,
	195	};
147	196	}
148		}
149	197
150		/// Decompresses the given data from `reader` into `writer`. Stops early if more
151		/// than `uncompressed_size` bytes are processed and verifies that exactly that
152		/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data.
153		/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method.
154		pub fn decompress(
155		method: CompressionMethod,
156		uncompressed_size: u64,
157		reader: anytype,
158		writer: anytype,
159		) !u32 {
160		var hash = std.hash.Crc32.init();
161
162		var total_uncompressed: u64 = 0;
163		switch (method) {
164		.store => {
165		var buf: [4096]u8 = undefined;
166		while (true) {
167		const len = try reader.read(&buf);
168		if (len == 0) break;
169		try writer.writeAll(buf[0..len]);
170		hash.update(buf[0..len]);
171		total_uncompressed += @intCast(len);
172		}
173		},
174		.deflate => {
175		var br = std.io.bufferedReader(reader);
176		var decompressor = std.compress.flate.decompressor(br.reader());
177		while (try decompressor.next()) \|chunk\| {
178		try writer.writeAll(chunk);
179		hash.update(chunk);
180		total_uncompressed += @intCast(chunk.len);
181		if (total_uncompressed > uncompressed_size)
182		return error.ZipUncompressSizeTooSmall;
183		}
184		if (br.end != br.start)
185		return error.ZipDeflateTruncated;
186		},
187		_ => return error.UnsupportedCompressionMethod,
	198	fn streamStore(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
	199	const d: *Decompress = @fieldParentPtr("interface", r);
	200	return d.store.read(w, limit);
188	201	}
189		if (total_uncompressed != uncompressed_size)
190		return error.ZipUncompressSizeMismatch;
191	202
192		return hash.final();
193		}
	203	fn streamDeflate(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
	204	const d: *Decompress = @fieldParentPtr("interface", r);
	205	return flate.Decompress.read(&d.inflate, w, limit);
	206	}
	207	};
194	208
195	209	fn isBadFilename(filename: []const u8) bool {
196	210	if (filename.len == 0 or filename[0] == '/')
...	...	@@ -253,319 +267,337 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data
253	267	}
254	268	}
255	269
256		pub fn Iterator(comptime SeekableStream: type) type {
257		return struct {
258		stream: SeekableStream,
	270	pub const Iterator = struct {
	271	input: *File.Reader,
259	272
260		cd_record_count: u64,
261		cd_zip_offset: u64,
262		cd_size: u64,
	273	cd_record_count: u64,
	274	cd_zip_offset: u64,
	275	cd_size: u64,
263	276
264		cd_record_index: u64 = 0,
265		cd_record_offset: u64 = 0,
	277	cd_record_index: u64 = 0,
	278	cd_record_offset: u64 = 0,
266	279
267		const Self = @This();
	280	pub fn init(input: *File.Reader) !Iterator {
	281	const end_record = try EndRecord.findFile(input);
268	282
269		pub fn init(stream: SeekableStream) !Self {
270		const stream_len = try stream.getEndPos();
271
272		const end_record = try findEndRecord(stream, stream_len);
273
274		if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
275		return error.ZipDiskRecordCountTooLarge;
276
277		if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
278		return error.ZipMultiDiskUnsupported;
	283	if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
	284	return error.ZipDiskRecordCountTooLarge;
279	285
280		{
281		const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);
282		if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
283		return error.ZipMultiDiskUnsupported;
284		}
	286	if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
	287	return error.ZipMultiDiskUnsupported;
285	288
286		var result = Self{
287		.stream = stream,
288		.cd_record_count = end_record.record_count_total,
289		.cd_zip_offset = end_record.central_directory_offset,
290		.cd_size = end_record.central_directory_size,
291		};
292		if (!end_record.need_zip64()) return result;
293
294		const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
295		if (locator_end_offset > stream_len)
296		return error.ZipTruncated;
297		try stream.seekTo(stream_len - locator_end_offset);
298		const locator = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndLocator64, .little);
299		if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
300		return error.ZipBadLocatorSig;
301		if (locator.zip64_disk_count != 0)
302		return error.ZipUnsupportedZip64DiskCount;
303		if (locator.total_disk_count != 1)
	289	{
	290	const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);
	291	if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
304	292	return error.ZipMultiDiskUnsupported;
	293	}
305	294
306		try stream.seekTo(locator.record_file_offset);
307
308		const record64 = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndRecord64, .little);
309
310		if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
311		return error.ZipBadEndRecord64Sig;
312
313		if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
314		return error.ZipEndRecord64SizeTooSmall;
315		if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
316		return error.ZipEndRecord64UnhandledExtraData;
	295	var result: Iterator = .{
	296	.input = input,
	297	.cd_record_count = end_record.record_count_total,
	298	.cd_zip_offset = end_record.central_directory_offset,
	299	.cd_size = end_record.central_directory_size,
	300	};
	301	if (!end_record.need_zip64()) return result;
317	302
318		if (record64.version_needed_to_extract > 45)
319		return error.ZipUnsupportedVersion;
	303	const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
	304	const stream_len = try input.getSize();
320	305
321		{
322		const is_multidisk = record64.disk_number != 0 or
323		record64.central_directory_disk_number != 0 or
324		record64.record_count_disk != record64.record_count_total;
325		if (is_multidisk)
326		return error.ZipMultiDiskUnsupported;
327		}
	306	if (locator_end_offset > stream_len)
	307	return error.ZipTruncated;
	308	try input.seekTo(stream_len - locator_end_offset);
	309	const locator = input.interface.takeStruct(EndLocator64, .little) catch \|err\| switch (err) {
	310	error.ReadFailed => return input.err.?,
	311	error.EndOfStream => return error.EndOfStream,
	312	};
	313	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
	314	return error.ZipBadLocatorSig;
	315	if (locator.zip64_disk_count != 0)
	316	return error.ZipUnsupportedZip64DiskCount;
	317	if (locator.total_disk_count != 1)
	318	return error.ZipMultiDiskUnsupported;
	319
	320	try input.seekTo(locator.record_file_offset);
	321
	322	const record64 = input.interface.takeStruct(EndRecord64, .little) catch \|err\| switch (err) {
	323	error.ReadFailed => return input.err.?,
	324	error.EndOfStream => return error.EndOfStream,
	325	};
328	326
329		if (isMaxInt(end_record.record_count_total)) {
330		result.cd_record_count = record64.record_count_total;
331		} else if (end_record.record_count_total != record64.record_count_total)
332		return error.Zip64RecordCountTotalMismatch;
	327	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
	328	return error.ZipBadEndRecord64Sig;
333	329
334		if (isMaxInt(end_record.central_directory_offset)) {
335		result.cd_zip_offset = record64.central_directory_offset;
336		} else if (end_record.central_directory_offset != record64.central_directory_offset)
337		return error.Zip64CentralDirectoryOffsetMismatch;
	330	if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
	331	return error.ZipEndRecord64SizeTooSmall;
	332	if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
	333	return error.ZipEndRecord64UnhandledExtraData;
338	334
339		if (isMaxInt(end_record.central_directory_size)) {
340		result.cd_size = record64.central_directory_size;
341		} else if (end_record.central_directory_size != record64.central_directory_size)
342		return error.Zip64CentralDirectorySizeMismatch;
	335	if (record64.version_needed_to_extract > 45)
	336	return error.ZipUnsupportedVersion;
343	337
344		return result;
	338	{
	339	const is_multidisk = record64.disk_number != 0 or
	340	record64.central_directory_disk_number != 0 or
	341	record64.record_count_disk != record64.record_count_total;
	342	if (is_multidisk)
	343	return error.ZipMultiDiskUnsupported;
345	344	}
346	345
347		pub fn next(self: *Self) !?Entry {
348		if (self.cd_record_index == self.cd_record_count) {
349		if (self.cd_record_offset != self.cd_size)
350		return if (self.cd_size > self.cd_record_offset)
351		error.ZipCdOversized
352		else
353		error.ZipCdUndersized;
	346	if (isMaxInt(end_record.record_count_total)) {
	347	result.cd_record_count = record64.record_count_total;
	348	} else if (end_record.record_count_total != record64.record_count_total)
	349	return error.Zip64RecordCountTotalMismatch;
354	350
355		return null;
356		}
	351	if (isMaxInt(end_record.central_directory_offset)) {
	352	result.cd_zip_offset = record64.central_directory_offset;
	353	} else if (end_record.central_directory_offset != record64.central_directory_offset)
	354	return error.Zip64CentralDirectoryOffsetMismatch;
357	355
358		const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
359		try self.stream.seekTo(header_zip_offset);
360		const header = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readStructEndian(CentralDirectoryFileHeader, .little);
361		if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
362		return error.ZipBadCdOffset;
	356	if (isMaxInt(end_record.central_directory_size)) {
	357	result.cd_size = record64.central_directory_size;
	358	} else if (end_record.central_directory_size != record64.central_directory_size)
	359	return error.Zip64CentralDirectorySizeMismatch;
363	360
364		self.cd_record_index += 1;
365		self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;
	361	return result;
	362	}
366	363
367		// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file
368		// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip
369		// has an undocumented version 788 but extracts just fine.
	364	pub fn next(self: *Iterator) !?Entry {
	365	if (self.cd_record_index == self.cd_record_count) {
	366	if (self.cd_record_offset != self.cd_size)
	367	return if (self.cd_size > self.cd_record_offset)
	368	error.ZipCdOversized
	369	else
	370	error.ZipCdUndersized;
370	371
371		if (header.flags.encrypted)
372		return error.ZipEncryptionUnsupported;
373		// TODO: check/verify more flags
374		if (header.disk_number != 0)
375		return error.ZipMultiDiskUnsupported;
	372	return null;
	373	}
376	374
377		var extents: FileExtents = .{
378		.uncompressed_size = header.uncompressed_size,
379		.compressed_size = header.compressed_size,
380		.local_file_header_offset = header.local_file_header_offset,
381		};
	375	const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
	376	const input = self.input;
	377	try input.seekTo(header_zip_offset);
	378	const header = input.interface.takeStruct(CentralDirectoryFileHeader, .little) catch \|err\| switch (err) {
	379	error.ReadFailed => return input.err.?,
	380	error.EndOfStream => return error.EndOfStream,
	381	};
	382	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
	383	return error.ZipBadCdOffset;
	384
	385	self.cd_record_index += 1;
	386	self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;
	387
	388	// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file
	389	// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip
	390	// has an undocumented version 788 but extracts just fine.
	391
	392	if (header.flags.encrypted)
	393	return error.ZipEncryptionUnsupported;
	394	// TODO: check/verify more flags
	395	if (header.disk_number != 0)
	396	return error.ZipMultiDiskUnsupported;
	397
	398	var extents: FileExtents = .{
	399	.uncompressed_size = header.uncompressed_size,
	400	.compressed_size = header.compressed_size,
	401	.local_file_header_offset = header.local_file_header_offset,
	402	};
382	403
383		if (header.extra_len > 0) {
384		var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
385		const extra = extra_buf[0..header.extra_len];
	404	if (header.extra_len > 0) {
	405	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
	406	const extra = extra_buf[0..header.extra_len];
386	407
387		{
388		try self.stream.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
389		const len = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readAll(extra);
390		if (len != extra.len)
391		return error.ZipTruncated;
392		}
	408	try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
	409	input.interface.readSliceAll(extra) catch \|err\| switch (err) {
	410	error.ReadFailed => return input.err.?,
	411	error.EndOfStream => return error.EndOfStream,
	412	};
393	413
394		var extra_offset: usize = 0;
395		while (extra_offset + 4 <= extra.len) {
396		const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
397		const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
398		const end = extra_offset + 4 + data_size;
399		if (end > extra.len)
400		return error.ZipBadExtraFieldSize;
401		const data = extra[extra_offset + 4 .. end];
402		switch (@as(ExtraHeader, @enumFromInt(header_id))) {
403		.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),
404		else => {}, // ignore
405		}
406		extra_offset = end;
	414	var extra_offset: usize = 0;
	415	while (extra_offset + 4 <= extra.len) {
	416	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
	417	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
	418	const end = extra_offset + 4 + data_size;
	419	if (end > extra.len)
	420	return error.ZipBadExtraFieldSize;
	421	const data = extra[extra_offset + 4 .. end];
	422	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
	423	.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),
	424	else => {}, // ignore
407	425	}
	426	extra_offset = end;
408	427	}
409
410		return .{
411		.version_needed_to_extract = header.version_needed_to_extract,
412		.flags = header.flags,
413		.compression_method = header.compression_method,
414		.last_modification_time = header.last_modification_time,
415		.last_modification_date = header.last_modification_date,
416		.header_zip_offset = header_zip_offset,
417		.crc32 = header.crc32,
418		.filename_len = header.filename_len,
419		.compressed_size = extents.compressed_size,
420		.uncompressed_size = extents.uncompressed_size,
421		.file_offset = extents.local_file_header_offset,
422		};
423	428	}
424	429
425		pub const Entry = struct {
426		version_needed_to_extract: u16,
427		flags: GeneralPurposeFlags,
428		compression_method: CompressionMethod,
429		last_modification_time: u16,
430		last_modification_date: u16,
431		header_zip_offset: u64,
432		crc32: u32,
433		filename_len: u32,
434		compressed_size: u64,
435		uncompressed_size: u64,
436		file_offset: u64,
437
438		pub fn extract(
439		self: Entry,
440		stream: SeekableStream,
441		options: ExtractOptions,
442		filename_buf: []u8,
443		dest: std.fs.Dir,
444		) !u32 {
445		if (filename_buf.len < self.filename_len)
446		return error.ZipInsufficientBuffer;
447		const filename = filename_buf[0..self.filename_len];
	430	return .{
	431	.version_needed_to_extract = header.version_needed_to_extract,
	432	.flags = header.flags,
	433	.compression_method = header.compression_method,
	434	.last_modification_time = header.last_modification_time,
	435	.last_modification_date = header.last_modification_date,
	436	.header_zip_offset = header_zip_offset,
	437	.crc32 = header.crc32,
	438	.filename_len = header.filename_len,
	439	.compressed_size = extents.compressed_size,
	440	.uncompressed_size = extents.uncompressed_size,
	441	.file_offset = extents.local_file_header_offset,
	442	};
	443	}
448	444
	445	pub const Entry = struct {
	446	version_needed_to_extract: u16,
	447	flags: GeneralPurposeFlags,
	448	compression_method: CompressionMethod,
	449	last_modification_time: u16,
	450	last_modification_date: u16,
	451	header_zip_offset: u64,
	452	crc32: u32,
	453	filename_len: u32,
	454	compressed_size: u64,
	455	uncompressed_size: u64,
	456	file_offset: u64,
	457
	458	pub fn extract(
	459	self: Entry,
	460	stream: *File.Reader,
	461	options: ExtractOptions,
	462	filename_buf: []u8,
	463	dest: std.fs.Dir,
	464	) !void {
	465	if (filename_buf.len < self.filename_len)
	466	return error.ZipInsufficientBuffer;
	467	switch (self.compression_method) {
	468	.store, .deflate => {},
	469	else => return error.UnsupportedCompressionMethod,
	470	}
	471	const filename = filename_buf[0..self.filename_len];
	472	{
449	473	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
	474	try stream.interface.readSliceAll(filename);
	475	}
450	476
451		{
452		const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(filename);
453		if (len != filename.len)
454		return error.ZipBadFileOffset;
455		}
	477	const local_data_header_offset: u64 = local_data_header_offset: {
	478	const local_header = blk: {
	479	try stream.seekTo(self.file_offset);
	480	break :blk try stream.interface.takeStruct(LocalFileHeader, .little);
	481	};
	482	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
	483	return error.ZipBadFileOffset;
	484	if (local_header.version_needed_to_extract != self.version_needed_to_extract)
	485	return error.ZipMismatchVersionNeeded;
	486	if (local_header.last_modification_time != self.last_modification_time)
	487	return error.ZipMismatchModTime;
	488	if (local_header.last_modification_date != self.last_modification_date)
	489	return error.ZipMismatchModDate;
	490
	491	if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
	492	return error.ZipMismatchFlags;
	493	if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
	494	return error.ZipMismatchCrc32;
	495	var extents: FileExtents = .{
	496	.uncompressed_size = local_header.uncompressed_size,
	497	.compressed_size = local_header.compressed_size,
	498	.local_file_header_offset = 0,
	499	};
	500	if (local_header.extra_len > 0) {
	501	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
	502	const extra = extra_buf[0..local_header.extra_len];
456	503
457		const local_data_header_offset: u64 = local_data_header_offset: {
458		const local_header = blk: {
459		try stream.seekTo(self.file_offset);
460		break :blk try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(LocalFileHeader, .little);
461		};
462		if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
463		return error.ZipBadFileOffset;
464		if (local_header.version_needed_to_extract != self.version_needed_to_extract)
465		return error.ZipMismatchVersionNeeded;
466		if (local_header.last_modification_time != self.last_modification_time)
467		return error.ZipMismatchModTime;
468		if (local_header.last_modification_date != self.last_modification_date)
469		return error.ZipMismatchModDate;
470
471		if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
472		return error.ZipMismatchFlags;
473		if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
474		return error.ZipMismatchCrc32;
475		var extents: FileExtents = .{
476		.uncompressed_size = local_header.uncompressed_size,
477		.compressed_size = local_header.compressed_size,
478		.local_file_header_offset = 0,
479		};
480		if (local_header.extra_len > 0) {
481		var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
482		const extra = extra_buf[0..local_header.extra_len];
483
484		{
485		try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
486		const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(extra);
487		if (len != extra.len)
488		return error.ZipTruncated;
489		}
	504	{
	505	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
	506	try stream.interface.readSliceAll(extra);
	507	}
490	508
491		var extra_offset: usize = 0;
492		while (extra_offset + 4 <= local_header.extra_len) {
493		const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
494		const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
495		const end = extra_offset + 4 + data_size;
496		if (end > local_header.extra_len)
497		return error.ZipBadExtraFieldSize;
498		const data = extra[extra_offset + 4 .. end];
499		switch (@as(ExtraHeader, @enumFromInt(header_id))) {
500		.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),
501		else => {}, // ignore
502		}
503		extra_offset = end;
	509	var extra_offset: usize = 0;
	510	while (extra_offset + 4 <= local_header.extra_len) {
	511	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
	512	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
	513	const end = extra_offset + 4 + data_size;
	514	if (end > local_header.extra_len)
	515	return error.ZipBadExtraFieldSize;
	516	const data = extra[extra_offset + 4 .. end];
	517	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
	518	.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),
	519	else => {}, // ignore
504	520	}
	521	extra_offset = end;
505	522	}
	523	}
506	524
507		if (extents.compressed_size != 0 and
508		extents.compressed_size != self.compressed_size)
509		return error.ZipMismatchCompLen;
510		if (extents.uncompressed_size != 0 and
511		extents.uncompressed_size != self.uncompressed_size)
512		return error.ZipMismatchUncompLen;
	525	if (extents.compressed_size != 0 and
	526	extents.compressed_size != self.compressed_size)
	527	return error.ZipMismatchCompLen;
	528	if (extents.uncompressed_size != 0 and
	529	extents.uncompressed_size != self.uncompressed_size)
	530	return error.ZipMismatchUncompLen;
513	531
514		if (local_header.filename_len != self.filename_len)
515		return error.ZipMismatchFilenameLen;
	532	if (local_header.filename_len != self.filename_len)
	533	return error.ZipMismatchFilenameLen;
516	534
517		break :local_data_header_offset @as(u64, local_header.filename_len) +
518		@as(u64, local_header.extra_len);
519		};
	535	break :local_data_header_offset @as(u64, local_header.filename_len) +
	536	@as(u64, local_header.extra_len);
	537	};
520	538
521		if (isBadFilename(filename))
522		return error.ZipBadFilename;
	539	if (isBadFilename(filename))
	540	return error.ZipBadFilename;
523	541
524		if (options.allow_backslashes) {
525		std.mem.replaceScalar(u8, filename, '\\', '/');
526		} else {
527		if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|
528		return error.ZipFilenameHasBackslash;
529		}
	542	if (options.allow_backslashes) {
	543	std.mem.replaceScalar(u8, filename, '\\', '/');
	544	} else {
	545	if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|
	546	return error.ZipFilenameHasBackslash;
	547	}
530	548
531		// All entries that end in '/' are directories
532		if (filename[filename.len - 1] == '/') {
533		if (self.uncompressed_size != 0)
534		return error.ZipBadDirectorySize;
535		try dest.makePath(filename[0 .. filename.len - 1]);
536		return std.hash.Crc32.hash(&.{});
537		}
	549	// All entries that end in '/' are directories
	550	if (filename[filename.len - 1] == '/') {
	551	if (self.uncompressed_size != 0)
	552	return error.ZipBadDirectorySize;
	553	try dest.makePath(filename[0 .. filename.len - 1]);
	554	return;
	555	}
538	556
539		const out_file = blk: {
540		if (std.fs.path.dirname(filename)) \|dirname\| {
541		var parent_dir = try dest.makeOpenPath(dirname, .{});
542		defer parent_dir.close();
	557	const out_file = blk: {
	558	if (std.fs.path.dirname(filename)) \|dirname\| {
	559	var parent_dir = try dest.makeOpenPath(dirname, .{});
	560	defer parent_dir.close();
543	561
544		const basename = std.fs.path.basename(filename);
545		break :blk try parent_dir.createFile(basename, .{ .exclusive = true });
546		}
547		break :blk try dest.createFile(filename, .{ .exclusive = true });
548		};
549		defer out_file.close();
550		const local_data_file_offset: u64 =
551		@as(u64, self.file_offset) +
552		@as(u64, @sizeOf(LocalFileHeader)) +
553		local_data_header_offset;
554		try stream.seekTo(local_data_file_offset);
555		var limited_reader = std.io.limitedReader((if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()), self.compressed_size);
556		const crc = try decompress(
557		self.compression_method,
558		self.uncompressed_size,
559		limited_reader.reader(),
560		out_file.deprecatedWriter(),
561		);
562		if (limited_reader.bytes_left != 0)
563		return error.ZipDecompressTruncated;
564		return crc;
	562	const basename = std.fs.path.basename(filename);
	563	break :blk try parent_dir.createFile(basename, .{ .exclusive = true });
	564	}
	565	break :blk try dest.createFile(filename, .{ .exclusive = true });
	566	};
	567	defer out_file.close();
	568	var out_file_buffer: [1024]u8 = undefined;
	569	var file_writer = out_file.writer(&out_file_buffer);
	570	const local_data_file_offset: u64 =
	571	@as(u64, self.file_offset) +
	572	@as(u64, @sizeOf(LocalFileHeader)) +
	573	local_data_header_offset;
	574	try stream.seekTo(local_data_file_offset);
	575
	576	// TODO limit based on self.compressed_size
	577
	578	switch (self.compression_method) {
	579	.store => {
	580	stream.interface.streamExact64(&file_writer.interface, self.uncompressed_size) catch \|err\| switch (err) {
	581	error.ReadFailed => return stream.err.?,
	582	error.WriteFailed => return file_writer.err.?,
	583	error.EndOfStream => return error.ZipDecompressTruncated,
	584	};
	585	},
	586	.deflate => {
	587	var flate_buffer: [flate.max_window_len]u8 = undefined;
	588	var decompress: flate.Decompress = .init(&stream.interface, .raw, &flate_buffer);
	589	decompress.reader.streamExact64(&file_writer.interface, self.uncompressed_size) catch \|err\| switch (err) {
	590	error.ReadFailed => return stream.err.?,
	591	error.WriteFailed => return file_writer.err orelse decompress.err.?,
	592	error.EndOfStream => return error.ZipDecompressTruncated,
	593	};
	594	},
	595	else => return error.UnsupportedCompressionMethod,
565	596	}
566		};
	597	try file_writer.end();
	598	}
567	599	};
568		}
	600	};
569	601
570	602	// returns true if `filename` starts with `root` followed by a forward slash
571	603	fn filenameInRoot(filename: []const u8, root: []const u8) bool {
...	...	@@ -610,196 +642,21 @@ pub const ExtractOptions = struct {
610	642	/// Allow filenames within the zip to use backslashes. Back slashes are normalized
611	643	/// to forward slashes before forwarding them to platform APIs.
612	644	allow_backslashes: bool = false,
613
614	645	diagnostics: ?*Diagnostics = null,
	646	verify_checksums: bool = false,
615	647	};
616	648
617		/// Extract the zipped files inside `seekable_stream` to the given `dest` directory.
618		/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and
619		/// its context must also have a `.reader()` method that returns an instance of
620		/// `std.io.GenericReader`.
621		pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptions) !void {
622		const SeekableStream = @TypeOf(seekable_stream);
623		var iter = try Iterator(SeekableStream).init(seekable_stream);
	649	/// Extract the zipped files to the given `dest` directory.
	650	pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void {
	651	if (options.verify_checksums) @panic("TODO unimplemented");
	652
	653	var iter = try Iterator.init(fr);
624	654
625	655	var filename_buf: [std.fs.max_path_bytes]u8 = undefined;
626	656	while (try iter.next()) \|entry\| {
627		const crc32 = try entry.extract(seekable_stream, options, &filename_buf, dest);
628		if (crc32 != entry.crc32)
629		return error.ZipCrcMismatch;
	657	try entry.extract(fr, options, &filename_buf, dest);
630	658	if (options.diagnostics) \|d\| {
631	659	try d.nextFilename(filename_buf[0..entry.filename_len]);
632	660	}
633	661	}
634	662	}
635
636		fn testZip(options: ExtractOptions, comptime files: []const File, write_opt: testutil.WriteZipOptions) !void {
637		var store: [files.len]FileStore = undefined;
638		try testZipWithStore(options, files, write_opt, &store);
639		}
640		fn testZipWithStore(
641		options: ExtractOptions,
642		test_files: []const File,
643		write_opt: testutil.WriteZipOptions,
644		store: []FileStore,
645		) !void {
646		var zip_buf: [4096]u8 = undefined;
647		var fbs = try testutil.makeZipWithStore(&zip_buf, test_files, write_opt, store);
648
649		var tmp = testing.tmpDir(.{ .no_follow = true });
650		defer tmp.cleanup();
651		try extract(tmp.dir, fbs.seekableStream(), options);
652		try testutil.expectFiles(test_files, tmp.dir, .{});
653		}
654		fn testZipError(expected_error: anyerror, file: File, options: ExtractOptions) !void {
655		var zip_buf: [4096]u8 = undefined;
656		var store: [1]FileStore = undefined;
657		var fbs = try testutil.makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store);
658		var tmp = testing.tmpDir(.{ .no_follow = true });
659		defer tmp.cleanup();
660		try testing.expectError(expected_error, extract(tmp.dir, fbs.seekableStream(), options));
661		}
662
663		test "zip one file" {
664		try testZip(.{}, &[_]File{
665		.{ .name = "onefile.txt", .content = "Just a single file\n", .compression = .store },
666		}, .{});
667		}
668		test "zip multiple files" {
669		try testZip(.{ .allow_backslashes = true }, &[_]File{
670		.{ .name = "foo", .content = "a foo file\n", .compression = .store },
671		.{ .name = "subdir/bar", .content = "bar is this right?\nanother newline\n", .compression = .store },
672		.{ .name = "subdir\\whoa", .content = "you can do backslashes", .compression = .store },
673		.{ .name = "subdir/another/baz", .content = "bazzy mc bazzerson", .compression = .store },
674		}, .{});
675		}
676		test "zip deflated" {
677		try testZip(.{}, &[_]File{
678		.{ .name = "deflateme", .content = "This is a deflated file.\nIt should be smaller in the Zip file1\n", .compression = .deflate },
679		// TODO: re-enable this if/when we add support for deflate64
680		//.{ .name = "deflateme64", .content = "The 64k version of deflate!\n", .compression = .deflate64 },
681		.{ .name = "raw", .content = "Not all files need to be deflated in the same Zip.\n", .compression = .store },
682		}, .{});
683		}
684		test "zip verify filenames" {
685		// no empty filenames
686		try testZipError(error.ZipBadFilename, .{ .name = "", .content = "", .compression = .store }, .{});
687		// no absolute paths
688		try testZipError(error.ZipBadFilename, .{ .name = "/", .content = "", .compression = .store }, .{});
689		try testZipError(error.ZipBadFilename, .{ .name = "/foo", .content = "", .compression = .store }, .{});
690		try testZipError(error.ZipBadFilename, .{ .name = "/foo/bar", .content = "", .compression = .store }, .{});
691		// no '..' components
692		try testZipError(error.ZipBadFilename, .{ .name = "..", .content = "", .compression = .store }, .{});
693		try testZipError(error.ZipBadFilename, .{ .name = "foo/..", .content = "", .compression = .store }, .{});
694		try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/..", .content = "", .compression = .store }, .{});
695		try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/../", .content = "", .compression = .store }, .{});
696		// no backslashes
697		try testZipError(error.ZipFilenameHasBackslash, .{ .name = "foo\\bar", .content = "", .compression = .store }, .{});
698		}
699
700		test "zip64" {
701		const test_files = [_]File{
702		.{ .name = "fram", .content = "fram foo fro fraba", .compression = .store },
703		.{ .name = "subdir/barro", .content = "aljdk;jal;jfd;lajkf", .compression = .store },
704		};
705
706		try testZip(.{}, &test_files, .{
707		.end = .{
708		.zip64 = .{},
709		.record_count_disk = std.math.maxInt(u16), // trigger zip64
710		},
711		});
712		try testZip(.{}, &test_files, .{
713		.end = .{
714		.zip64 = .{},
715		.record_count_total = std.math.maxInt(u16), // trigger zip64
716		},
717		});
718		try testZip(.{}, &test_files, .{
719		.end = .{
720		.zip64 = .{},
721		.record_count_disk = std.math.maxInt(u16), // trigger zip64
722		.record_count_total = std.math.maxInt(u16), // trigger zip64
723		},
724		});
725		try testZip(.{}, &test_files, .{
726		.end = .{
727		.zip64 = .{},
728		.central_directory_size = std.math.maxInt(u32), // trigger zip64
729		},
730		});
731		try testZip(.{}, &test_files, .{
732		.end = .{
733		.zip64 = .{},
734		.central_directory_offset = std.math.maxInt(u32), // trigger zip64
735		},
736		});
737		try testZip(.{}, &test_files, .{
738		.end = .{
739		.zip64 = .{},
740		.central_directory_offset = std.math.maxInt(u32), // trigger zip64
741		},
742		.local_header = .{
743		.zip64 = .{ // trigger local header zip64
744		.data_size = 16,
745		},
746		.compressed_size = std.math.maxInt(u32),
747		.uncompressed_size = std.math.maxInt(u32),
748		.extra_len = 20,
749		},
750		});
751		}
752
753		test "bad zip files" {
754		var tmp = testing.tmpDir(.{ .no_follow = true });
755		defer tmp.cleanup();
756		var zip_buf: [4096]u8 = undefined;
757
758		const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }};
759
760		{
761		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
762		try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
763		}
764		{
765		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } });
766		try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
767		}
768		{
769		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
770		try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
771		}
772		{
773		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } });
774		try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
775		}
776		{
777		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
778		try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
779		}
780		{
781		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } });
782		try testing.expectError(error.ZipDiskRecordCountTooLarge, extract(tmp.dir, fbs.seekableStream(), .{}));
783		}
784		{
785		var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } });
786		try testing.expectError(error.ZipCdOversized, extract(tmp.dir, fbs.seekableStream(), .{}));
787		}
788		{
789		var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } });
790		try testing.expectError(error.ZipCdUndersized, extract(tmp.dir, fbs.seekableStream(), .{}));
791		}
792		{
793		var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
794		try testing.expectError(error.ZipBadCdOffset, extract(tmp.dir, fbs.seekableStream(), .{}));
795		}
796		{
797		var fbs = try testutil.makeZip(&zip_buf, &file_a, .{
798		.end = .{
799		.zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } },
800		.central_directory_size = std.math.maxInt(u32), // trigger 64
801		},
802		});
803		try testing.expectError(error.ZipBadLocatorSig, extract(tmp.dir, fbs.seekableStream(), .{}));
804		}
805		}

lib/std/zip/test.zig deleted-298

...	...	@@ -1,298 +0,0 @@
1		const std = @import("std");
2		const testing = std.testing;
3		const zip = @import("../zip.zig");
4		const maxInt = std.math.maxInt;
5
6		pub const File = struct {
7		name: []const u8,
8		content: []const u8,
9		compression: zip.CompressionMethod,
10		};
11
12		pub fn expectFiles(
13		test_files: []const File,
14		dir: std.fs.Dir,
15		opt: struct {
16		strip_prefix: ?[]const u8 = null,
17		},
18		) !void {
19		for (test_files) \|test_file\| {
20		var normalized_sub_path_buf: [std.fs.max_path_bytes]u8 = undefined;
21
22		const name = blk: {
23		if (opt.strip_prefix) \|strip_prefix\| {
24		try testing.expect(test_file.name.len >= strip_prefix.len);
25		try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]);
26		break :blk test_file.name[strip_prefix.len..];
27		}
28		break :blk test_file.name;
29		};
30		const normalized_sub_path = normalized_sub_path_buf[0..name.len];
31		@memcpy(normalized_sub_path, name);
32		std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
33		var file = try dir.openFile(normalized_sub_path, .{});
34		defer file.close();
35		var content_buf: [4096]u8 = undefined;
36		const n = try file.deprecatedReader().readAll(&content_buf);
37		try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
38		}
39		}
40
41		// Used to store any data from writing a file to the zip archive that's needed
42		// when writing the corresponding central directory record.
43		pub const FileStore = struct {
44		compression: zip.CompressionMethod,
45		file_offset: u64,
46		crc32: u32,
47		compressed_size: u32,
48		uncompressed_size: usize,
49		};
50
51		pub fn makeZip(
52		buf: []u8,
53		comptime files: []const File,
54		options: WriteZipOptions,
55		) !std.io.FixedBufferStream([]u8) {
56		var store: [files.len]FileStore = undefined;
57		return try makeZipWithStore(buf, files, options, &store);
58		}
59
60		pub fn makeZipWithStore(
61		buf: []u8,
62		files: []const File,
63		options: WriteZipOptions,
64		store: []FileStore,
65		) !std.io.FixedBufferStream([]u8) {
66		var fbs = std.io.fixedBufferStream(buf);
67		try writeZip(fbs.writer(), files, store, options);
68		return std.io.fixedBufferStream(buf[0..fbs.pos]);
69		}
70
71		pub const WriteZipOptions = struct {
72		end: ?EndRecordOptions = null,
73		local_header: ?LocalHeaderOptions = null,
74		};
75		pub const LocalHeaderOptions = struct {
76		zip64: ?LocalHeaderZip64Options = null,
77		compressed_size: ?u32 = null,
78		uncompressed_size: ?u32 = null,
79		extra_len: ?u16 = null,
80		};
81		pub const LocalHeaderZip64Options = struct {
82		data_size: ?u16 = null,
83		};
84		pub const EndRecordOptions = struct {
85		zip64: ?Zip64Options = null,
86		sig: ?[4]u8 = null,
87		disk_number: ?u16 = null,
88		central_directory_disk_number: ?u16 = null,
89		record_count_disk: ?u16 = null,
90		record_count_total: ?u16 = null,
91		central_directory_size: ?u32 = null,
92		central_directory_offset: ?u32 = null,
93		comment_len: ?u16 = null,
94		comment: ?[]const u8 = null,
95		};
96		pub const Zip64Options = struct {
97		locator_sig: ?[4]u8 = null,
98		locator_zip64_disk_count: ?u32 = null,
99		locator_record_file_offset: ?u64 = null,
100		locator_total_disk_count: ?u32 = null,
101		//record_size: ?u64 = null,
102		central_directory_size: ?u64 = null,
103		};
104
105		pub fn writeZip(
106		writer: anytype,
107		files: []const File,
108		store: []FileStore,
109		options: WriteZipOptions,
110		) !void {
111		if (store.len < files.len) return error.FileStoreTooSmall;
112		var zipper = initZipper(writer);
113		for (files, 0..) \|file, i\| {
114		store[i] = try zipper.writeFile(.{
115		.name = file.name,
116		.content = file.content,
117		.compression = file.compression,
118		.write_options = options,
119		});
120		}
121		for (files, 0..) \|file, i\| {
122		try zipper.writeCentralRecord(store[i], .{
123		.name = file.name,
124		});
125		}
126		try zipper.writeEndRecord(if (options.end) \|e\| e else .{});
127		}
128
129		pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
130		return .{ .counting_writer = std.io.countingWriter(writer) };
131		}
132
133		/// Provides methods to format and write the contents of a zip archive
134		/// to the underlying Writer.
135		pub fn Zipper(comptime Writer: type) type {
136		return struct {
137		counting_writer: std.io.CountingWriter(Writer),
138		central_count: u64 = 0,
139		first_central_offset: ?u64 = null,
140		last_central_limit: ?u64 = null,
141
142		const Self = @This();
143
144		pub fn writeFile(
145		self: *Self,
146		opt: struct {
147		name: []const u8,
148		content: []const u8,
149		compression: zip.CompressionMethod,
150		write_options: WriteZipOptions,
151		},
152		) !FileStore {
153		const writer = self.counting_writer.writer();
154
155		const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
156		const crc32 = std.hash.Crc32.hash(opt.content);
157
158		const header_options = opt.write_options.local_header;
159		{
160		var compressed_size: u32 = 0;
161		var uncompressed_size: u32 = 0;
162		var extra_len: u16 = 0;
163		if (header_options) \|hdr_options\| {
164		compressed_size = if (hdr_options.compressed_size) \|size\| size else 0;
165		uncompressed_size = if (hdr_options.uncompressed_size) \|size\| size else @intCast(opt.content.len);
166		extra_len = if (hdr_options.extra_len) \|len\| len else 0;
167		}
168		const hdr: zip.LocalFileHeader = .{
169		.signature = zip.local_file_header_sig,
170		.version_needed_to_extract = 10,
171		.flags = .{ .encrypted = false, ._ = 0 },
172		.compression_method = opt.compression,
173		.last_modification_time = 0,
174		.last_modification_date = 0,
175		.crc32 = crc32,
176		.compressed_size = compressed_size,
177		.uncompressed_size = uncompressed_size,
178		.filename_len = @intCast(opt.name.len),
179		.extra_len = extra_len,
180		};
181		try writer.writeStructEndian(hdr, .little);
182		}
183		try writer.writeAll(opt.name);
184
185		if (header_options) \|hdr\| {
186		if (hdr.zip64) \|options\| {
187		try writer.writeInt(u16, 0x0001, .little);
188		const data_size = if (options.data_size) \|size\| size else 8;
189		try writer.writeInt(u16, data_size, .little);
190		try writer.writeInt(u64, 0, .little);
191		try writer.writeInt(u64, @intCast(opt.content.len), .little);
192		}
193		}
194
195		var compressed_size: u32 = undefined;
196		switch (opt.compression) {
197		.store => {
198		try writer.writeAll(opt.content);
199		compressed_size = @intCast(opt.content.len);
200		},
201		.deflate => {
202		const offset = self.counting_writer.bytes_written;
203		var fbs = std.io.fixedBufferStream(opt.content);
204		try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205		std.debug.assert(fbs.pos == opt.content.len);
206		compressed_size = @intCast(self.counting_writer.bytes_written - offset);
207		},
208		else => unreachable,
209		}
210		return .{
211		.compression = opt.compression,
212		.file_offset = file_offset,
213		.crc32 = crc32,
214		.compressed_size = compressed_size,
215		.uncompressed_size = opt.content.len,
216		};
217		}
218
219		pub fn writeCentralRecord(
220		self: *Self,
221		store: FileStore,
222		opt: struct {
223		name: []const u8,
224		version_needed_to_extract: u16 = 10,
225		},
226		) !void {
227		if (self.first_central_offset == null) {
228		self.first_central_offset = self.counting_writer.bytes_written;
229		}
230		self.central_count += 1;
231
232		const hdr: zip.CentralDirectoryFileHeader = .{
233		.signature = zip.central_file_header_sig,
234		.version_made_by = 0,
235		.version_needed_to_extract = opt.version_needed_to_extract,
236		.flags = .{ .encrypted = false, ._ = 0 },
237		.compression_method = store.compression,
238		.last_modification_time = 0,
239		.last_modification_date = 0,
240		.crc32 = store.crc32,
241		.compressed_size = store.compressed_size,
242		.uncompressed_size = @intCast(store.uncompressed_size),
243		.filename_len = @intCast(opt.name.len),
244		.extra_len = 0,
245		.comment_len = 0,
246		.disk_number = 0,
247		.internal_file_attributes = 0,
248		.external_file_attributes = 0,
249		.local_file_header_offset = @intCast(store.file_offset),
250		};
251		try self.counting_writer.writer().writeStructEndian(hdr, .little);
252		try self.counting_writer.writer().writeAll(opt.name);
253		self.last_central_limit = self.counting_writer.bytes_written;
254		}
255
256		pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
257		const cd_offset = self.first_central_offset orelse 0;
258		const cd_end = self.last_central_limit orelse 0;
259
260		if (opt.zip64) \|zip64\| {
261		const end64_off = cd_end;
262		const fixed: zip.EndRecord64 = .{
263		.signature = zip.end_record64_sig,
264		.end_record_size = @sizeOf(zip.EndRecord64) - 12,
265		.version_made_by = 0,
266		.version_needed_to_extract = 45,
267		.disk_number = 0,
268		.central_directory_disk_number = 0,
269		.record_count_disk = @intCast(self.central_count),
270		.record_count_total = @intCast(self.central_count),
271		.central_directory_size = @intCast(cd_end - cd_offset),
272		.central_directory_offset = @intCast(cd_offset),
273		};
274		try self.counting_writer.writer().writeStructEndian(fixed, .little);
275		const locator: zip.EndLocator64 = .{
276		.signature = if (zip64.locator_sig) \|s\| s else zip.end_locator64_sig,
277		.zip64_disk_count = if (zip64.locator_zip64_disk_count) \|c\| c else 0,
278		.record_file_offset = if (zip64.locator_record_file_offset) \|o\| o else @intCast(end64_off),
279		.total_disk_count = if (zip64.locator_total_disk_count) \|c\| c else 1,
280		};
281		try self.counting_writer.writer().writeStructEndian(locator, .little);
282		}
283		const hdr: zip.EndRecord = .{
284		.signature = if (opt.sig) \|s\| s else zip.end_record_sig,
285		.disk_number = if (opt.disk_number) \|n\| n else 0,
286		.central_directory_disk_number = if (opt.central_directory_disk_number) \|n\| n else 0,
287		.record_count_disk = if (opt.record_count_disk) \|c\| c else @intCast(self.central_count),
288		.record_count_total = if (opt.record_count_total) \|c\| c else @intCast(self.central_count),
289		.central_directory_size = if (opt.central_directory_size) \|s\| s else @intCast(cd_end - cd_offset),
290		.central_directory_offset = if (opt.central_directory_offset) \|o\| o else @intCast(cd_offset),
291		.comment_len = if (opt.comment_len) \|l\| l else (if (opt.comment) \|c\| @as(u16, @intCast(c.len)) else 0),
292		};
293		try self.counting_writer.writer().writeStructEndian(hdr, .little);
294		if (opt.comment) \|c\|
295		try self.counting_writer.writer().writeAll(c);
296		}
297		};
298		}

src/Package/Fetch.zig+19-78

...	...	@@ -1203,12 +1203,11 @@ fn unpackResource(
1203	1203	return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1204	1204	},
1205	1205	.@"tar.gz" => {
1206		const reader = resource.reader();
1207		var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
1208		var dcp = std.compress.gzip.decompressor(br.reader());
1209		var adapter_buffer: [1024]u8 = undefined;
1210		var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
1211		return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
	1206	var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
	1207	var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
	1208	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1209	var decompress: std.compress.flate.Decompress = .init(&adapter.new_interface, .gzip, &flate_buffer);
	1210	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1212	1211	},
1213	1212	.@"tar.xz" => {
1214	1213	const gpa = f.arena.child_allocator;
...	...	@@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1352	1351	));
1353	1352	defer zip_file.close();
1354	1353
1355		std.zip.extract(out_dir, zip_file.seekableStream(), .{
	1354	var zip_file_buffer: [1024]u8 = undefined;
	1355	var zip_file_reader = zip_file.reader(&zip_file_buffer);
	1356
	1357	std.zip.extract(out_dir, &zip_file_reader, .{
1356	1358	.allow_backslashes = true,
1357	1359	.diagnostics = &diagnostics,
1358	1360	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(
...	...	@@ -1384,23 +1386,28 @@ fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource.Git) anyerror!U
1384	1386	defer pack_dir.close();
1385	1387	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1386	1388	defer pack_file.close();
1387		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
	1389	var pack_file_buffer: [4096]u8 = undefined;
	1390	var fifo = std.fifo.LinearFifo(u8, .{ .Slice = {} }).init(&pack_file_buffer);
1388	1391	try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());
1389	1392
	1393	var pack_file_reader = pack_file.reader(&pack_file_buffer);
	1394
1390	1395	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
1391	1396	defer index_file.close();
	1397	var index_file_buffer: [2000]u8 = undefined;
	1398	var index_file_writer = index_file.writer(&index_file_buffer);
1392	1399	{
1393	1400	const index_prog_node = f.prog_node.start("Index pack", 0);
1394	1401	defer index_prog_node.end();
1395		var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
1396		try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer());
1397		try index_buffered_writer.flush();
	1402	try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
1398	1403	}
1399	1404
1400	1405	{
	1406	var index_file_reader = index_file.reader(&index_file_buffer);
1401	1407	const checkout_prog_node = f.prog_node.start("Checkout", 0);
1402	1408	defer checkout_prog_node.end();
1403		var repository = try git.Repository.init(gpa, object_format, pack_file, index_file);
	1409	var repository: git.Repository = undefined;
	1410	try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
1404	1411	defer repository.deinit();
1405	1412	var diagnostics: git.Diagnostics = .{ .allocator = arena };
1406	1413	try repository.checkout(out_dir, resource.want_oid, &diagnostics);
...	...	@@ -2071,72 +2078,6 @@ const UnpackResult = struct {
2071	2078	}
2072	2079	};
2073	2080
2074		test "zip" {
2075		const gpa = std.testing.allocator;
2076		var tmp = std.testing.tmpDir(.{});
2077		defer tmp.cleanup();
2078
2079		const test_files = [_]std.zip.testutil.File{
2080		.{ .name = "foo", .content = "this is just foo\n", .compression = .store },
2081		.{ .name = "bar", .content = "another file\n", .compression = .deflate },
2082		};
2083		{
2084		var zip_file = try tmp.dir.createFile("test.zip", .{});
2085		defer zip_file.close();
2086		var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2087		var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2088		try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2089		try bw.flush();
2090		}
2091
2092		const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2093		defer gpa.free(zip_path);
2094
2095		var fb: TestFetchBuilder = undefined;
2096		var fetch = try fb.build(gpa, tmp.dir, zip_path);
2097		defer fb.deinit();
2098
2099		try fetch.run();
2100
2101		var out = try fb.packageDir();
2102		defer out.close();
2103
2104		try std.zip.testutil.expectFiles(&test_files, out, .{});
2105		}
2106
2107		test "zip with one root folder" {
2108		const gpa = std.testing.allocator;
2109		var tmp = std.testing.tmpDir(.{});
2110		defer tmp.cleanup();
2111
2112		const test_files = [_]std.zip.testutil.File{
2113		.{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store },
2114		.{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store },
2115		};
2116		{
2117		var zip_file = try tmp.dir.createFile("test.zip", .{});
2118		defer zip_file.close();
2119		var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2120		var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2121		try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2122		try bw.flush();
2123		}
2124
2125		const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2126		defer gpa.free(zip_path);
2127
2128		var fb: TestFetchBuilder = undefined;
2129		var fetch = try fb.build(gpa, tmp.dir, zip_path);
2130		defer fb.deinit();
2131
2132		try fetch.run();
2133
2134		var out = try fb.packageDir();
2135		defer out.close();
2136
2137		try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" });
2138		}
2139
2140	2081	test "tarball with duplicate paths" {
2141	2082	// This tarball has duplicate path 'dir1/file1' to simulate case sensitve
2142	2083	// file system on any file sytstem.

src/Package/Fetch/git.zig+177-215

...	...	@@ -66,6 +66,33 @@ pub const Oid = union(Format) {
66	66	}
67	67	};
68	68
	69	const Hashing = union(Format) {
	70	sha1: std.Io.Writer.Hashing(Sha1),
	71	sha256: std.Io.Writer.Hashing(Sha256),
	72
	73	fn init(oid_format: Format, buffer: []u8) Hashing {
	74	return switch (oid_format) {
	75	.sha1 => .{ .sha1 = .init(buffer) },
	76	.sha256 => .{ .sha256 = .init(buffer) },
	77	};
	78	}
	79
	80	fn writer(h: @This()) std.Io.Writer {
	81	return switch (h.*) {
	82	inline else => \|*inner\| &inner.writer,
	83	};
	84	}
	85
	86	fn final(h: *@This()) Oid {
	87	switch (h.*) {
	88	inline else => \|*inner, tag\| {
	89	inner.writer.flush() catch unreachable; // hashers cannot fail
	90	return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult());
	91	},
	92	}
	93	}
	94	};
	95
69	96	pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
70	97	assert(bytes.len == oid_format.byteLength());
71	98	return switch (oid_format) {
...	...	@@ -73,9 +100,9 @@ pub const Oid = union(Format) {
73	100	};
74	101	}
75	102
76		pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid {
	103	pub fn readBytes(oid_format: Format, reader: *std.Io.Reader) !Oid {
77	104	return switch (oid_format) {
78		inline else => \|tag\| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),
	105	inline else => \|tag\| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*),
79	106	};
80	107	}
81	108
...	...	@@ -166,8 +193,15 @@ pub const Diagnostics = struct {
166	193	pub const Repository = struct {
167	194	odb: Odb,
168	195
169		pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
170		return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };
	196	pub fn init(
	197	repo: *Repository,
	198	allocator: Allocator,
	199	format: Oid.Format,
	200	pack_file: *std.fs.File.Reader,
	201	index_file: *std.fs.File.Reader,
	202	) !void {
	203	repo.* = .{ .odb = undefined };
	204	try repo.odb.init(allocator, format, pack_file, index_file);
171	205	}
172	206
173	207	pub fn deinit(repository: *Repository) void {
...	...	@@ -335,24 +369,30 @@ pub const Repository = struct {
335	369	/// [pack-format](https://git-scm.com/docs/pack-format).
336	370	const Odb = struct {
337	371	format: Oid.Format,
338		pack_file: std.fs.File,
	372	pack_file: *std.fs.File.Reader,
339	373	index_header: IndexHeader,
340		index_file: std.fs.File,
	374	index_file: *std.fs.File.Reader,
341	375	cache: ObjectCache = .{},
342	376	allocator: Allocator,
343	377
344	378	/// Initializes the database from open pack and index files.
345		fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
	379	fn init(
	380	odb: *Odb,
	381	allocator: Allocator,
	382	format: Oid.Format,
	383	pack_file: *std.fs.File.Reader,
	384	index_file: *std.fs.File.Reader,
	385	) !void {
346	386	try pack_file.seekTo(0);
347	387	try index_file.seekTo(0);
348		const index_header = try IndexHeader.read(index_file.deprecatedReader());
349		return .{
	388	odb.* = .{
350	389	.format = format,
351	390	.pack_file = pack_file,
352		.index_header = index_header,
	391	.index_header = undefined,
353	392	.index_file = index_file,
354	393	.allocator = allocator,
355	394	};
	395	try odb.index_header.read(&index_file.interface);
356	396	}
357	397
358	398	fn deinit(odb: *Odb) void {
...	...	@@ -362,14 +402,14 @@ const Odb = struct {
362	402
363	403	/// Reads the object at the current position in the database.
364	404	fn readObject(odb: *Odb) !Object {
365		var base_offset = try odb.pack_file.getPos();
	405	var base_offset = odb.pack_file.logicalPos();
366	406	var base_header: EntryHeader = undefined;
367	407	var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;
368	408	defer delta_offsets.deinit(odb.allocator);
369	409	const base_object = while (true) {
370	410	if (odb.cache.get(base_offset)) \|base_object\| break base_object;
371	411
372		base_header = try EntryHeader.read(odb.format, odb.pack_file.deprecatedReader());
	412	base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface);
373	413	switch (base_header) {
374	414	.ofs_delta => \|ofs_delta\| {
375	415	try delta_offsets.append(odb.allocator, base_offset);
...	...	@@ -379,10 +419,10 @@ const Odb = struct {
379	419	.ref_delta => \|ref_delta\| {
380	420	try delta_offsets.append(odb.allocator, base_offset);
381	421	try odb.seekOid(ref_delta.base_object);
382		base_offset = try odb.pack_file.getPos();
	422	base_offset = odb.pack_file.logicalPos();
383	423	},
384	424	else => {
385		const base_data = try readObjectRaw(odb.allocator, odb.pack_file.deprecatedReader(), base_header.uncompressedLength());
	425	const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
386	426	errdefer odb.allocator.free(base_data);
387	427	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
388	428	try odb.cache.put(odb.allocator, base_offset, base_object);
...	...	@@ -412,7 +452,7 @@ const Odb = struct {
412	452	const found_index = while (start_index < end_index) {
413	453	const mid_index = start_index + (end_index - start_index) / 2;
414	454	try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
415		const mid_oid = try Oid.readBytes(odb.format, odb.index_file.deprecatedReader());
	455	const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface);
416	456	switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
417	457	.lt => start_index = mid_index + 1,
418	458	.gt => end_index = mid_index,
...	...	@@ -423,12 +463,12 @@ const Odb = struct {
423	463	const n_objects = odb.index_header.fan_out_table[255];
424	464	const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
425	465	try odb.index_file.seekTo(offset_values_start + found_index * 4);
426		const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.deprecatedReader().readInt(u32, .big));
	466	const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big));
427	467	const pack_offset = pack_offset: {
428	468	if (l1_offset.big) {
429	469	const l2_offset_values_start = offset_values_start + n_objects * 4;
430	470	try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
431		break :pack_offset try odb.index_file.deprecatedReader().readInt(u64, .big);
	471	break :pack_offset try odb.index_file.interface.takeInt(u64, .big);
432	472	} else {
433	473	break :pack_offset l1_offset.value;
434	474	}
...	...	@@ -1080,18 +1120,18 @@ const PackHeader = struct {
1080	1120	const signature = "PACK";
1081	1121	const supported_version = 2;
1082	1122
1083		fn read(reader: anytype) !PackHeader {
1084		const actual_signature = reader.readBytesNoEof(4) catch \|e\| switch (e) {
	1123	fn read(reader: *std.Io.Reader) !PackHeader {
	1124	const actual_signature = reader.take(4) catch \|e\| switch (e) {
1085	1125	error.EndOfStream => return error.InvalidHeader,
1086	1126	else => \|other\| return other,
1087	1127	};
1088		if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
1089		const version = reader.readInt(u32, .big) catch \|e\| switch (e) {
	1128	if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
	1129	const version = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1090	1130	error.EndOfStream => return error.InvalidHeader,
1091	1131	else => \|other\| return other,
1092	1132	};
1093	1133	if (version != supported_version) return error.UnsupportedVersion;
1094		const total_objects = reader.readInt(u32, .big) catch \|e\| switch (e) {
	1134	const total_objects = reader.takeInt(u32, .big) catch \|e\| switch (e) {
1095	1135	error.EndOfStream => return error.InvalidHeader,
1096	1136	else => \|other\| return other,
1097	1137	};
...	...	@@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) {
1143	1183	};
1144	1184	}
1145	1185
1146		fn read(format: Oid.Format, reader: anytype) !EntryHeader {
	1186	fn read(format: Oid.Format, reader: *std.Io.Reader) !EntryHeader {
1147	1187	const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
1148		const initial: InitialByte = @bitCast(reader.readByte() catch \|e\| switch (e) {
	1188	const initial: InitialByte = @bitCast(reader.takeByte() catch \|e\| switch (e) {
1149	1189	error.EndOfStream => return error.InvalidFormat,
1150	1190	else => \|other\| return other,
1151	1191	});
1152		const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
	1192	const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0;
1153	1193	var uncompressed_length: u64 = initial.len;
1154	1194	uncompressed_length \|= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
1155	1195	const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat;
...	...	@@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) {
1172	1212	}
1173	1213	};
1174	1214
1175		fn readSizeVarInt(r: anytype) !u64 {
1176		const Byte = packed struct { value: u7, has_next: bool };
1177		var b: Byte = @bitCast(try r.readByte());
1178		var value: u64 = b.value;
1179		var shift: u6 = 0;
1180		while (b.has_next) {
1181		b = @bitCast(try r.readByte());
1182		shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
1183		value \|= @as(u64, b.value) << shift;
1184		}
1185		return value;
1186		}
1187
1188		fn readOffsetVarInt(r: anytype) !u64 {
	1215	fn readOffsetVarInt(r: *std.Io.Reader) !u64 {
1189	1216	const Byte = packed struct { value: u7, has_next: bool };
1190		var b: Byte = @bitCast(try r.readByte());
	1217	var b: Byte = @bitCast(try r.takeByte());
1191	1218	var value: u64 = b.value;
1192	1219	while (b.has_next) {
1193		b = @bitCast(try r.readByte());
	1220	b = @bitCast(try r.takeByte());
1194	1221	value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
1195	1222	value \|= b.value;
1196	1223	}
...	...	@@ -1204,19 +1231,12 @@ const IndexHeader = struct {
1204	1231	const supported_version = 2;
1205	1232	const size = 4 + 4 + @sizeOf([256]u32);
1206	1233
1207		fn read(reader: anytype) !IndexHeader {
1208		var header_bytes = try reader.readBytesNoEof(size);
1209		if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
1210		const version = mem.readInt(u32, header_bytes[4..8], .big);
	1234	fn read(index_header: IndexHeader, reader: std.Io.Reader) !void {
	1235	const sig = try reader.take(4);
	1236	if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
	1237	const version = try reader.takeInt(u32, .big);
1211	1238	if (version != supported_version) return error.UnsupportedVersion;
1212
1213		var fan_out_table: [256]u32 = undefined;
1214		var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
1215		const fan_out_table_reader = fan_out_table_stream.reader();
1216		for (&fan_out_table) \|*entry\| {
1217		entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable;
1218		}
1219		return .{ .fan_out_table = fan_out_table };
	1239	try reader.readSliceEndian(u32, &index_header.fan_out_table, .big);
1220	1240	}
1221	1241	};
1222	1242
...	...	@@ -1227,7 +1247,12 @@ const IndexEntry = struct {
1227	1247
1228	1248	/// Writes out a version 2 index for the given packfile, as documented in
1229	1249	/// [pack-format](https://git-scm.com/docs/pack-format).
1230		pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void {
	1250	pub fn indexPack(
	1251	allocator: Allocator,
	1252	format: Oid.Format,
	1253	pack: *std.fs.File.Reader,
	1254	index_writer: *std.fs.File.Writer,
	1255	) !void {
1231	1256	try pack.seekTo(0);
1232	1257
1233	1258	var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
...	...	@@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1280	1305	}
1281	1306	@memset(fan_out_table[fan_out_index..], count);
1282	1307
1283		var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format));
1284		const writer = index_hashed_writer.writer();
	1308	var index_hashed_writer = std.Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{});
	1309	const writer = &index_hashed_writer.writer;
1285	1310	try writer.writeAll(IndexHeader.signature);
1286	1311	try writer.writeInt(u32, IndexHeader.supported_version, .big);
1287	1312	for (fan_out_table) \|fan_out_entry\| {
...	...	@@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1314	1339
1315	1340	try writer.writeAll(pack_checksum.slice());
1316	1341	const index_checksum = index_hashed_writer.hasher.finalResult();
1317		try index_writer.writeAll(index_checksum.slice());
	1342	try index_writer.interface.writeAll(index_checksum.slice());
	1343	try index_writer.end();
1318	1344	}
1319	1345
1320	1346	/// Performs the first pass over the packfile data for index construction.
...	...	@@ -1324,68 +1350,51 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1324	1350	fn indexPackFirstPass(
1325	1351	allocator: Allocator,
1326	1352	format: Oid.Format,
1327		pack: std.fs.File,
	1353	pack: *std.fs.File.Reader,
1328	1354	index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1329	1355	pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1330	1356	) !Oid {
1331		var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());
1332		var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1333		var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1334		const pack_reader = pack_hashed_reader.reader();
1335
1336		const pack_header = try PackHeader.read(pack_reader);
1337
1338		var current_entry: u32 = 0;
1339		while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1340		const entry_offset = pack_counting_reader.bytes_read;
1341		var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
1342		const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
	1357	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
	1358	var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
	1359	var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer);
	1360
	1361	const pack_header = try PackHeader.read(&pack_hashed.reader);
	1362
	1363	for (0..pack_header.total_objects) \|_\| {
	1364	const entry_offset = pack.logicalPos() - pack_hashed.reader.bufferedLen();
	1365	const entry_header = try EntryHeader.read(format, &pack_hashed.reader);
1343	1366	switch (entry_header) {
1344	1367	.commit, .tree, .blob, .tag => \|object\| {
1345		var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1346		var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1347		var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1348		const entry_writer = entry_hashed_writer.writer();
	1368	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &.{});
	1369	var oid_hasher: Oid.Hashing = .init(format, &flate_buffer);
	1370	const oid_hasher_w = oid_hasher.writer();
1349	1371	// The object header is not included in the pack data but is
1350	1372	// part of the object's ID
1351		try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });
1352		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1353		try fifo.pump(entry_counting_reader.reader(), entry_writer);
1354		if (entry_counting_reader.bytes_read != object.uncompressed_length) {
1355		return error.InvalidObject;
1356		}
1357		const oid = entry_hashed_writer.hasher.finalResult();
	1373	try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length });
	1374	const n = try entry_decompress.reader.streamRemaining(oid_hasher_w);
	1375	if (n != object.uncompressed_length) return error.InvalidObject;
	1376	const oid = oid_hasher.final();
	1377	if (!skip_checksums) @compileError("TODO");
1358	1378	try index_entries.put(allocator, oid, .{
1359	1379	.offset = entry_offset,
1360		.crc32 = entry_crc32_reader.hasher.final(),
	1380	.crc32 = 0,
1361	1381	});
1362	1382	},
1363	1383	inline .ofs_delta, .ref_delta => \|delta\| {
1364		var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1365		var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1366		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1367		try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1368		if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1369		return error.InvalidObject;
1370		}
	1384	var entry_decompress: std.compress.flate.Decompress = .init(&pack_hashed.reader, .zlib, &flate_buffer);
	1385	const n = try entry_decompress.reader.discardRemaining();
	1386	if (n != delta.uncompressed_length) return error.InvalidObject;
	1387	if (!skip_checksums) @compileError("TODO");
1371	1388	try pending_deltas.append(allocator, .{
1372	1389	.offset = entry_offset,
1373		.crc32 = entry_crc32_reader.hasher.final(),
	1390	.crc32 = 0,
1374	1391	});
1375	1392	},
1376	1393	}
1377	1394	}
1378	1395
1379		const pack_checksum = pack_hashed_reader.hasher.finalResult();
1380		const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader());
1381		if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {
1382		return error.CorruptedPack;
1383		}
1384		_ = pack_reader.readByte() catch \|e\| switch (e) {
1385		error.EndOfStream => return pack_checksum,
1386		else => \|other\| return other,
1387		};
1388		return error.InvalidFormat;
	1396	if (!skip_checksums) @compileError("TODO");
	1397	return pack_hashed.hasher.finalResult();
1389	1398	}
1390	1399
1391	1400	/// Attempts to determine the final object ID of the given deltified object.
...	...	@@ -1394,7 +1403,7 @@ fn indexPackFirstPass(
1394	1403	fn indexPackHashDelta(
1395	1404	allocator: Allocator,
1396	1405	format: Oid.Format,
1397		pack: std.fs.File,
	1406	pack: *std.fs.File.Reader,
1398	1407	delta: IndexEntry,
1399	1408	index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1400	1409	cache: *ObjectCache,
...	...	@@ -1408,7 +1417,7 @@ fn indexPackHashDelta(
1408	1417	if (cache.get(base_offset)) \|base_object\| break base_object;
1409	1418
1410	1419	try pack.seekTo(base_offset);
1411		base_header = try EntryHeader.read(format, pack.deprecatedReader());
	1420	base_header = try EntryHeader.read(format, &pack.interface);
1412	1421	switch (base_header) {
1413	1422	.ofs_delta => \|ofs_delta\| {
1414	1423	try delta_offsets.append(allocator, base_offset);
...	...	@@ -1419,7 +1428,7 @@ fn indexPackHashDelta(
1419	1428	base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1420	1429	},
1421	1430	else => {
1422		const base_data = try readObjectRaw(allocator, pack.deprecatedReader(), base_header.uncompressedLength());
	1431	const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
1423	1432	errdefer allocator.free(base_data);
1424	1433	const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1425	1434	try cache.put(allocator, base_offset, base_object);
...	...	@@ -1430,11 +1439,13 @@ fn indexPackHashDelta(
1430	1439
1431	1440	const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
1432	1441
1433		var entry_hasher: Oid.Hasher = .init(format);
1434		var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1435		try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1436		entry_hasher.update(base_data);
1437		return entry_hasher.finalResult();
	1442	var entry_hasher_buffer: [64]u8 = undefined;
	1443	var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer);
	1444	const entry_hasher_w = entry_hasher.writer();
	1445	// Writes to hashers cannot fail.
	1446	entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable;
	1447	entry_hasher_w.writeAll(base_data) catch unreachable;
	1448	return entry_hasher.final();
1438	1449	}
1439	1450
1440	1451	/// Resolves a chain of deltas, returning the final base object data. `pack` is
...	...	@@ -1444,7 +1455,7 @@ fn indexPackHashDelta(
1444	1455	fn resolveDeltaChain(
1445	1456	allocator: Allocator,
1446	1457	format: Oid.Format,
1447		pack: std.fs.File,
	1458	pack: *std.fs.File.Reader,
1448	1459	base_object: Object,
1449	1460	delta_offsets: []const u64,
1450	1461	cache: *ObjectCache,
...	...	@@ -1456,21 +1467,19 @@ fn resolveDeltaChain(
1456	1467
1457	1468	const delta_offset = delta_offsets[i];
1458	1469	try pack.seekTo(delta_offset);
1459		const delta_header = try EntryHeader.read(format, pack.deprecatedReader());
1460		const delta_data = try readObjectRaw(allocator, pack.deprecatedReader(), delta_header.uncompressedLength());
	1470	const delta_header = try EntryHeader.read(format, &pack.interface);
	1471	const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
1461	1472	defer allocator.free(delta_data);
1462		var delta_stream = std.io.fixedBufferStream(delta_data);
1463		const delta_reader = delta_stream.reader();
1464		_ = try readSizeVarInt(delta_reader); // base object size
1465		const expanded_size = try readSizeVarInt(delta_reader);
	1473	var delta_reader: std.Io.Reader = .fixed(delta_data);
	1474	_ = try delta_reader.takeLeb128(u64); // base object size
	1475	const expanded_size = try delta_reader.takeLeb128(u64);
1466	1476
1467	1477	const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1468	1478	const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1469	1479	errdefer allocator.free(expanded_data);
1470		var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
1471		var base_stream = std.io.fixedBufferStream(base_data);
1472		try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
1473		if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
	1480	var expanded_delta_stream: std.Io.Writer = .fixed(expanded_data);
	1481	try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
	1482	if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
1474	1483
1475	1484	try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1476	1485	base_data = expanded_data;
...	...	@@ -1481,28 +1490,23 @@ fn resolveDeltaChain(
1481	1490	/// Reads the complete contents of an object from `reader`. This function may
1482	1491	/// read more bytes than required from `reader`, so the reader position after
1483	1492	/// returning is not reliable.
1484		fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
	1493	fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 {
1485	1494	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1486		var buffered_reader = std.io.bufferedReader(reader);
1487		var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());
1488		const data = try allocator.alloc(u8, alloc_size);
1489		errdefer allocator.free(data);
1490		try decompress_stream.reader().readNoEof(data);
1491		_ = decompress_stream.reader().readByte() catch \|e\| switch (e) {
1492		error.EndOfStream => return data,
1493		else => \|other\| return other,
1494		};
1495		return error.InvalidFormat;
	1495	var aw: std.Io.Writer.Allocating = .init(allocator);
	1496	try aw.ensureTotalCapacity(alloc_size + std.compress.flate.max_window_len);
	1497	defer aw.deinit();
	1498	var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
	1499	try decompress.reader.streamExact(&aw.writer, alloc_size);
	1500	return aw.toOwnedSlice();
1496	1501	}
1497	1502
1498		/// Expands delta data from `delta_reader` to `writer`. `base_object` must
1499		/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
	1503	/// Expands delta data from `delta_reader` to `writer`.
1500	1504	///
1501	1505	/// The format of the delta data is documented in
1502	1506	/// [pack-format](https://git-scm.com/docs/pack-format).
1503		fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {
	1507	fn expandDelta(base_object: []const u8, delta_reader: std.Io.Reader, writer: std.Io.Writer) !void {
1504	1508	while (true) {
1505		const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch \|e\| switch (e) {
	1509	const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch \|e\| switch (e) {
1506	1510	error.EndOfStream => return,
1507	1511	else => \|other\| return other,
1508	1512	});
...	...	@@ -1517,27 +1521,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo
1517	1521	size3: bool,
1518	1522	} = @bitCast(inst.value);
1519	1523	const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1520		.offset1 = if (available.offset1) try delta_reader.readByte() else 0,
1521		.offset2 = if (available.offset2) try delta_reader.readByte() else 0,
1522		.offset3 = if (available.offset3) try delta_reader.readByte() else 0,
1523		.offset4 = if (available.offset4) try delta_reader.readByte() else 0,
	1524	.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
	1525	.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
	1526	.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
	1527	.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
1524	1528	};
1525		const offset: u32 = @bitCast(offset_parts);
	1529	const base_offset: u32 = @bitCast(offset_parts);
1526	1530	const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1527		.size1 = if (available.size1) try delta_reader.readByte() else 0,
1528		.size2 = if (available.size2) try delta_reader.readByte() else 0,
1529		.size3 = if (available.size3) try delta_reader.readByte() else 0,
	1531	.size1 = if (available.size1) try delta_reader.takeByte() else 0,
	1532	.size2 = if (available.size2) try delta_reader.takeByte() else 0,
	1533	.size3 = if (available.size3) try delta_reader.takeByte() else 0,
1530	1534	};
1531	1535	var size: u24 = @bitCast(size_parts);
1532	1536	if (size == 0) size = 0x10000;
1533		try base_object.seekTo(offset);
1534		var copy_reader = std.io.limitedReader(base_object.reader(), size);
1535		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1536		try fifo.pump(copy_reader.reader(), writer);
	1537	try writer.writeAll(base_object[base_offset..][0..size]);
1537	1538	} else if (inst.value != 0) {
1538		var data_reader = std.io.limitedReader(delta_reader, inst.value);
1539		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1540		try fifo.pump(data_reader.reader(), writer);
	1539	try delta_reader.streamExact(writer, inst.value);
1541	1540	} else {
1542	1541	return error.InvalidDeltaInstruction;
1543	1542	}
...	...	@@ -1567,23 +1566,32 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
1567	1566	defer pack_file.close();
1568	1567	try pack_file.writeAll(testrepo_pack);
1569	1568
	1569	var pack_file_buffer: [2000]u8 = undefined;
	1570	var pack_file_reader = pack_file.reader(&pack_file_buffer);
	1571
1570	1572	var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1571	1573	defer index_file.close();
1572		try indexPack(testing.allocator, format, pack_file, index_file.deprecatedWriter());
	1574	var index_file_buffer: [2000]u8 = undefined;
	1575	var index_file_writer = index_file.writer(&index_file_buffer);
	1576	try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer);
1573	1577
1574	1578	// Arbitrary size limit on files read while checking the repository contents
1575	1579	// (all files in the test repo are known to be smaller than this)
1576	1580	const max_file_size = 8192;
1577	1581
1578		const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1579		defer testing.allocator.free(index_file_data);
1580		// testrepo.idx is generated by Git. The index created by this file should
1581		// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1582		// this.
1583		const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
1584		try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
	1582	if (!skip_checksums) {
	1583	const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
	1584	defer testing.allocator.free(index_file_data);
	1585	// testrepo.idx is generated by Git. The index created by this file should
	1586	// match it exactly. Running `git verify-pack -v testrepo.pack` can verify
	1587	// this.
	1588	const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
	1589	try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
	1590	}
1585	1591
1586		var repository = try Repository.init(testing.allocator, format, pack_file, index_file);
	1592	var index_file_reader = index_file.reader(&index_file_buffer);
	1593	var repository: Repository = undefined;
	1594	try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader);
1587	1595	defer repository.deinit();
1588	1596
1589	1597	var worktree = testing.tmpDir(.{ .iterate = true });
...	...	@@ -1653,6 +1661,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
1653	1661	try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1654	1662	}
1655	1663
	1664	/// Checksum calculation is useful for troubleshooting and debugging, but it's
	1665	/// redundant since the package manager already does content hashing at the
	1666	/// end. Let's save time by not doing that work, but, I left a cookie crumb
	1667	/// trail here if you want to restore the functionality for tinkering purposes.
	1668	const skip_checksums = true;
	1669
1656	1670	test "SHA-1 packfile indexing and checkout" {
1657	1671	try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
1658	1672	}
...	...	@@ -1676,6 +1690,9 @@ pub fn main() !void {
1676	1690
1677	1691	var pack_file = try std.fs.cwd().openFile(args[2], .{});
1678	1692	defer pack_file.close();
	1693	var pack_file_buffer: [4096]u8 = undefined;
	1694	var pack_file_reader = pack_file.reader(&pack_file_buffer);
	1695
1679	1696	const commit = try Oid.parse(format, args[3]);
1680	1697	var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});
1681	1698	defer worktree.close();
...	...	@@ -1687,11 +1704,11 @@ pub fn main() !void {
1687	1704	var index_file = try git_dir.createFile("idx", .{ .read = true });
1688	1705	defer index_file.close();
1689	1706	var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
1690		try indexPack(allocator, format, pack_file, index_buffered_writer.writer());
	1707	try indexPack(allocator, format, &pack_file_reader, index_buffered_writer.writer());
1691	1708	try index_buffered_writer.flush();
1692	1709
1693	1710	std.debug.print("Starting checkout...\n", .{});
1694		var repository = try Repository.init(allocator, format, pack_file, index_file);
	1711	var repository = try Repository.init(allocator, format, &pack_file_reader, index_file);
1695	1712	defer repository.deinit();
1696	1713	var diagnostics: Diagnostics = .{ .allocator = allocator };
1697	1714	defer diagnostics.deinit();
...	...	@@ -1701,58 +1718,3 @@ pub fn main() !void {
1701	1718	std.debug.print("Diagnostic: {}\n", .{err});
1702	1719	}
1703	1720	}
1704
1705		/// Deprecated
1706		fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
1707		return .{ .child_reader = reader, .hasher = hasher };
1708		}
1709
1710		/// Deprecated
1711		fn HashedReader(ReaderType: type, HasherType: type) type {
1712		return struct {
1713		child_reader: ReaderType,
1714		hasher: HasherType,
1715
1716		pub const Error = ReaderType.Error;
1717		pub const Reader = std.io.GenericReader(*@This(), Error, read);
1718
1719		pub fn read(self: *@This(), buf: []u8) Error!usize {
1720		const amt = try self.child_reader.read(buf);
1721		self.hasher.update(buf[0..amt]);
1722		return amt;
1723		}
1724
1725		pub fn reader(self: *@This()) Reader {
1726		return .{ .context = self };
1727		}
1728		};
1729		}
1730
1731		/// Deprecated
1732		pub fn HashedWriter(WriterType: type, HasherType: type) type {
1733		return struct {
1734		child_writer: WriterType,
1735		hasher: HasherType,
1736
1737		pub const Error = WriterType.Error;
1738		pub const Writer = std.io.GenericWriter(*@This(), Error, write);
1739
1740		pub fn write(self: *@This(), buf: []const u8) Error!usize {
1741		const amt = try self.child_writer.write(buf);
1742		self.hasher.update(buf[0..amt]);
1743		return amt;
1744		}
1745
1746		pub fn writer(self: *@This()) Writer {
1747		return .{ .context = self };
1748		}
1749		};
1750		}
1751
1752		/// Deprecated
1753		pub fn hashedWriter(
1754		writer: anytype,
1755		hasher: anytype,
1756		) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1757		return .{ .child_writer = writer, .hasher = hasher };
1758		}

src/link/Dwarf.zig+11-9

...	...	@@ -142,8 +142,8 @@ const DebugInfo = struct {
142	142	&abbrev_code_buf,
143	143	debug_info.section.off(dwarf) + unit_ptr.off + unit_ptr.header_len + entry_ptr.off,
144	144	) != abbrev_code_buf.len) return error.InputOutput;
145		var abbrev_code_fbs = std.io.fixedBufferStream(&abbrev_code_buf);
146		return @enumFromInt(std.leb.readUleb128(@typeInfo(AbbrevCode).@"enum".tag_type, abbrev_code_fbs.reader()) catch unreachable);
	145	var abbrev_code_reader: std.Io.Reader = .fixed(&abbrev_code_buf);
	146	return @enumFromInt(abbrev_code_reader.takeLeb128(@typeInfo(AbbrevCode).@"enum".tag_type) catch unreachable);
147	147	}
148	148
149	149	const trailer_bytes = 1 + 1;
...	...	@@ -2077,7 +2077,7 @@ pub const WipNav = struct {
2077	2077	.generic_decl_const,
2078	2078	.generic_decl_func,
2079	2079	=> true,
2080		else => unreachable,
	2080	else => \|t\| std.debug.panic("bad decl abbrev code: {t}", .{t}),
2081	2081	};
2082	2082	if (parent_type.getCaptures(zcu).len == 0) {
2083	2083	if (was_generic_decl) try dwarf.freeCommonEntry(wip_nav.unit, decl_gop.value_ptr.*);
...	...	@@ -6021,15 +6021,17 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 {
6021	6021	}
6022	6022
6023	6023	fn uleb128Bytes(value: anytype) u32 {
6024		var cw = std.io.countingWriter(std.io.null_writer);
6025		try uleb128(cw.writer(), value);
6026		return @intCast(cw.bytes_written);
	6024	var trash_buffer: [64]u8 = undefined;
	6025	var d: std.Io.Writer.Discarding = .init(&trash_buffer);
	6026	d.writer.writeUleb128(value) catch unreachable;
	6027	return @intCast(d.count + d.writer.end);
6027	6028	}
6028	6029
6029	6030	fn sleb128Bytes(value: anytype) u32 {
6030		var cw = std.io.countingWriter(std.io.null_writer);
6031		try sleb128(cw.writer(), value);
6032		return @intCast(cw.bytes_written);
	6031	var trash_buffer: [64]u8 = undefined;
	6032	var d: std.Io.Writer.Discarding = .init(&trash_buffer);
	6033	d.writer.writeSleb128(value) catch unreachable;
	6034	return @intCast(d.count + d.writer.end);
6033	6035	}
6034	6036
6035	6037	/// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional

src/link/Elf/Object.zig+7-8

...	...	@@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: Object, elf_file: Elf, atom_index: Atom.Index
1198	1198	const chdr = @as(align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).;
1199	1199	switch (chdr.ch_type) {
1200	1200	.ZLIB => {
1201		var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);
1202		var zlib_stream = std.compress.zlib.decompressor(stream.reader());
	1201	var stream: std.Io.Reader = .fixed(data[@sizeOf(elf.Elf64_Chdr)..]);
	1202	var zlib_stream: std.compress.flate.Decompress = .init(&stream, .zlib, &.{});
1203	1203	const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
1204		const decomp = try gpa.alloc(u8, size);
1205		const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;
1206		if (nread != decomp.len) {
1207		return error.InputOutput;
1208		}
1209		return decomp;
	1204	var aw: std.Io.Writer.Allocating = .init(gpa);
	1205	try aw.ensureUnusedCapacity(size);
	1206	defer aw.deinit();
	1207	_ = try zlib_stream.reader.streamRemaining(&aw.writer);
	1208	return aw.toOwnedSlice();
1210	1209	},
1211	1210	else => @panic("TODO unhandled compression scheme"),
1212	1211	}

test/incremental/change_panic_handler-1

...	...	@@ -1,4 +1,3 @@
1		#target=x86_64-linux-selfhosted
2	1	#target=x86_64-linux-cbe
3	2	#target=x86_64-windows-cbe
4	3	#update=initial version

test/incremental/change_panic_handler_explicit-1

...	...	@@ -1,4 +1,3 @@
1		#target=x86_64-linux-selfhosted
2	1	#target=x86_64-linux-cbe
3	2	#target=x86_64-windows-cbe
4	3	#update=initial version

test/incremental/type_becomes_comptime_only-1

...	...	@@ -1,4 +1,3 @@
1		#target=x86_64-linux-selfhosted
2	1	#target=x86_64-linux-cbe
3	2	#target=x86_64-windows-cbe
4	3	#target=wasm32-wasi-selfhosted