Commit 83513ade3591de673e9ac4824fe974cd8f90c847 | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-07-25 22:10:29-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
log	83513ade3591de673e9ac4824fe974cd8f90c847
tree	5a581e31b4b004061673ac1e44e669e1808be3bd
parent	a2d21d63270ebb5eec0d437f7726c261455da66b

std.compress: rework flate to new I/O API

24 files changed, 3613 insertions(+), 4956 deletions(-)

lib/std/compress.zig+1-4

...	@@ -1,8 +1,7 @@	...	@@ -1,8 +1,7 @@
1	//! Compression algorithms.	1	//! Compression algorithms.
2		2
		3	/// gzip and zlib are here.
3	pub const flate = @import("compress/flate.zig");	4	pub const flate = @import("compress/flate.zig");
4	pub const gzip = @import("compress/gzip.zig");
5	pub const zlib = @import("compress/zlib.zig");
6	pub const lzma = @import("compress/lzma.zig");	5	pub const lzma = @import("compress/lzma.zig");
7	pub const lzma2 = @import("compress/lzma2.zig");	6	pub const lzma2 = @import("compress/lzma2.zig");
8	pub const xz = @import("compress/xz.zig");	7	pub const xz = @import("compress/xz.zig");
...	@@ -14,6 +13,4 @@ test {	...	@@ -14,6 +13,4 @@ test {
14	_ = lzma2;	13	_ = lzma2;
15	_ = xz;	14	_ = xz;
16	_ = zstd;	15	_ = zstd;
17	_ = gzip;
18	_ = zlib;
19	}	16	}

lib/std/compress/flate.zig+341-178

...	@@ -1,94 +1,189 @@	...	@@ -1,94 +1,189 @@
1	/// Deflate is a lossless data compression file format that uses a combination	1	const builtin = @import("builtin");
2	/// of LZ77 and Huffman coding.	2	const std = @import("../std.zig");
3	pub const deflate = @import("flate/deflate.zig");	3	const testing = std.testing;
4		4	const Writer = std.io.Writer;
5	/// Inflate is the decoding process that takes a Deflate bitstream for	5
6	/// decompression and correctly produces the original full-size data or file.	6	/// Container of the deflate bit stream body. Container adds header before
7	pub const inflate = @import("flate/inflate.zig");	7	/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
8		8	/// no footer, raw bit stream).
9	/// Decompress compressed data from reader and write plain data to the writer.	9	///
10	pub fn decompress(reader: anytype, writer: anytype) !void {	10	/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
11	try inflate.decompress(.raw, reader, writer);	11	/// addler 32 checksum.
12	}	12	///
		13	/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
		14	/// crc32 checksum and 4 bytes of uncompressed data length.
		15	///
		16	///
		17	/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
		18	/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
		19	pub const Container = enum {
		20	raw, // no header or footer
		21	gzip, // gzip header and footer
		22	zlib, // zlib header and footer
		23
		24	pub fn size(w: Container) usize {
		25	return headerSize(w) + footerSize(w);
		26	}
13		27
14	/// Decompressor type	28	pub fn headerSize(w: Container) usize {
15	pub fn Decompressor(comptime ReaderType: type) type {	29	return header(w).len;
16	return inflate.Decompressor(.raw, ReaderType);	30	}
17	}
18		31
19	/// Create Decompressor which will read compressed data from reader.	32	pub fn footerSize(w: Container) usize {
20	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {	33	return switch (w) {
21	return inflate.decompressor(.raw, reader);	34	.gzip => 8,
22	}	35	.zlib => 4,
		36	.raw => 0,
		37	};
		38	}
23		39
24	/// Compression level, trades between speed and compression size.	40	pub const list = [_]Container{ .raw, .gzip, .zlib };
25	pub const Options = deflate.Options;
26		41
27	/// Compress plain data from reader and write compressed data to the writer.	42	pub const Error = error{
28	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {	43	BadGzipHeader,
29	try deflate.compress(.raw, reader, writer, options);	44	BadZlibHeader,
30	}	45	WrongGzipChecksum,
		46	WrongGzipSize,
		47	WrongZlibChecksum,
		48	};
31		49
32	/// Compressor type	50	pub fn header(container: Container) []const u8 {
33	pub fn Compressor(comptime WriterType: type) type {	51	return switch (container) {
34	return deflate.Compressor(.raw, WriterType);	52	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
35	}	53	// - ID1 (IDentification 1), always 0x1f
		54	// - ID2 (IDentification 2), always 0x8b
		55	// - CM (Compression Method), always 8 = deflate
		56	// - FLG (Flags), all set to 0
		57	// - 4 bytes, MTIME (Modification time), not used, all set to zero
		58	// - XFL (eXtra FLags), all set to zero
		59	// - OS (Operating System), 03 = Unix
		60	.gzip => &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 },
		61	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
		62	// 1st byte:
		63	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
		64	// - The next four bits is the CM (compression method), which is 8 for deflate.
		65	// 2nd byte:
		66	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
		67	// - The next bit, FDICT, is set if a dictionary is given.
		68	// - The final five FCHECK bits form a mod-31 checksum.
		69	//
		70	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
		71	.zlib => &[_]u8{ 0x78, 0b10_0_11100 },
		72	.raw => &.{},
		73	};
		74	}
36		75
37	/// Create Compressor which outputs compressed data to the writer.	76	pub const Hasher = union(Container) {
38	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {	77	raw: void,
39	return try deflate.compressor(.raw, writer, options);	78	gzip: struct {
40	}	79	crc: std.hash.Crc32 = .init(),
		80	count: usize = 0,
		81	},
		82	zlib: std.hash.Adler32,
		83
		84	pub fn init(containter: Container) Hasher {
		85	return switch (containter) {
		86	.gzip => .{ .gzip = .{} },
		87	.zlib => .{ .zlib = .init() },
		88	.raw => .raw,
		89	};
		90	}
41		91
42	/// Huffman only compression. Without Lempel-Ziv match searching. Faster	92	pub fn container(h: Hasher) Container {
43	/// compression, less memory requirements but bigger compressed sizes.	93	return h;
44	pub const huffman = struct {	94	}
45	pub fn compress(reader: anytype, writer: anytype) !void {
46	try deflate.huffman.compress(.raw, reader, writer);
47	}
48		95
49	pub fn Compressor(comptime WriterType: type) type {	96	pub fn update(h: *Hasher, buf: []const u8) void {
50	return deflate.huffman.Compressor(.raw, WriterType);	97	switch (h.*) {
51	}	98	.raw => {},
		99	.gzip => \|*gzip\| {
		100	gzip.update(buf);
		101	gzip.count += buf.len;
		102	},
		103	.zlib => \|*zlib\| {
		104	zlib.update(buf);
		105	},
		106	inline .gzip, .zlib => \|*x\| x.update(buf),
		107	}
		108	}
52		109
53	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {	110	pub fn writeFooter(hasher: Hasher, writer: Writer) Writer.Error!void {
54	return deflate.huffman.compressor(.raw, writer);	111	var bits: [4]u8 = undefined;
55	}	112	switch (hasher.*) {
		113	.gzip => \|*gzip\| {
		114	// GZIP 8 bytes footer
		115	// - 4 bytes, CRC32 (CRC-32)
		116	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
		117	std.mem.writeInt(u32, &bits, gzip.final(), .little);
		118	try writer.writeAll(&bits);
		119
		120	std.mem.writeInt(u32, &bits, gzip.bytes_read, .little);
		121	try writer.writeAll(&bits);
		122	},
		123	.zlib => \|*zlib\| {
		124	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
		125	// 4 bytes of ADLER32 (Adler-32 checksum)
		126	// Checksum value of the uncompressed data (excluding any
		127	// dictionary data) computed according to Adler-32
		128	// algorithm.
		129	std.mem.writeInt(u32, &bits, zlib.final, .big);
		130	try writer.writeAll(&bits);
		131	},
		132	.raw => {},
		133	}
		134	}
		135	};
56	};	136	};
57		137
58	// No compression store only. Compressed size is slightly bigger than plain.	138	/// When decompressing, the output buffer is used as the history window, so
59	pub const store = struct {	139	/// less than this may result in failure to decompress streams that were
60	pub fn compress(reader: anytype, writer: anytype) !void {	140	/// compressed with a larger window.
61	try deflate.store.compress(.raw, reader, writer);	141	pub const max_window_len = 1 << 16;
62	}
63		142
64	pub fn Compressor(comptime WriterType: type) type {	143	/// Deflate is a lossless data compression file format that uses a combination
65	return deflate.store.Compressor(.raw, WriterType);	144	/// of LZ77 and Huffman coding.
66	}	145	pub const Compress = @import("flate/Compress.zig");
67		146
68	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {	147	/// Inflate is the decoding process that takes a Deflate bitstream for
69	return deflate.store.compressor(.raw, writer);	148	/// decompression and correctly produces the original full-size data or file.
70	}	149	pub const Decompress = @import("flate/Decompress.zig");
71	};
72		150
73	/// Container defines header/footer around deflate bit stream. Gzip and zlib	151	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
74	/// compression algorithms are containers around deflate bit stream body.	152	/// compression, less memory requirements but bigger compressed sizes.
75	const Container = @import("flate/container.zig").Container;	153	pub const huffman = struct {
76	const std = @import("std");	154	// The odd order in which the codegen code sizes are written.
77	const testing = std.testing;	155	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
78	const fixedBufferStream = std.io.fixedBufferStream;	156	// The number of codegen codes.
79	const print = std.debug.print;	157	pub const codegen_code_count = 19;
80	const builtin = @import("builtin");	158
		159	// The largest distance code.
		160	pub const distance_code_count = 30;
		161
		162	// Maximum number of literals.
		163	pub const max_num_lit = 286;
		164
		165	// Max number of frequencies used for a Huffman Code
		166	// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
		167	// The largest of these is max_num_lit.
		168	pub const max_num_frequencies = max_num_lit;
		169
		170	// Biggest block size for uncompressed block.
		171	pub const max_store_block_size = 65535;
		172	// The special code used to mark the end of a block.
		173	pub const end_block_marker = 256;
		174	};
81		175
82	test {	176	test {
83	_ = deflate;	177	_ = Compress;
84	_ = inflate;	178	_ = Decompress;
85	}	179	}
86		180
87	test "compress/decompress" {	181	test "compress/decompress" {
		182	const print = std.debug.print;
88	var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer	183	var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
89	var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer	184	var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
90		185
91	const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };	186	const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
92	const cases = [_]struct {	187	const cases = [_]struct {
93	data: []const u8, // uncompressed content	188	data: []const u8, // uncompressed content
94	// compressed data sizes per level 4-9	189	// compressed data sizes per level 4-9
...	@@ -135,28 +230,34 @@ test "compress/decompress" {	...	@@ -135,28 +230,34 @@ test "compress/decompress" {
135		230
136	// compress original stream to compressed stream	231	// compress original stream to compressed stream
137	{	232	{
138	var original = fixedBufferStream(data);	233	var original: std.io.Reader = .fixed(data);
139	var compressed = fixedBufferStream(&cmp_buf);	234	var compressed: Writer = .fixed(&cmp_buf);
140	try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });	235	var compress: Compress = .init(&original, &.{}, .{ .container = .raw, .level = level });
		236	const n = try compress.reader.streamRemaining(&compressed);
141	if (compressed_size == 0) {	237	if (compressed_size == 0) {
142	if (container == .gzip)	238	if (container == .gzip)
143	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });	239	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
144	compressed_size = compressed.pos;	240	compressed_size = compressed.end;
145	}	241	}
146	try testing.expectEqual(compressed_size, compressed.pos);	242	try testing.expectEqual(compressed_size, n);
		243	try testing.expectEqual(compressed_size, compressed.end);
147	}	244	}
148	// decompress compressed stream to decompressed stream	245	// decompress compressed stream to decompressed stream
149	{	246	{
150	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);	247	var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
151	var decompressed = fixedBufferStream(&dcm_buf);	248	var decompressed: Writer = .fixed(&dcm_buf);
152	try inflate.decompress(container, compressed.reader(), decompressed.writer());	249	var decompress: Decompress = .init(&compressed, container, &.{});
153	try testing.expectEqualSlices(u8, data, decompressed.getWritten());	250	_ = try decompress.reader.streamRemaining(&decompressed);
		251	try testing.expectEqualSlices(u8, data, decompressed.buffered());
154	}	252	}
155		253
156	// compressor writer interface	254	// compressor writer interface
157	{	255	{
158	var compressed = fixedBufferStream(&cmp_buf);	256	var compressed: Writer = .fixed(&cmp_buf);
159	var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });	257	var cmp = try Compress.init(&compressed, &.{}, .{
		258	.level = level,
		259	.container = container,
		260	});
160	var cmp_wrt = cmp.writer();	261	var cmp_wrt = cmp.writer();
161	try cmp_wrt.writeAll(data);	262	try cmp_wrt.writeAll(data);
162	try cmp.finish();	263	try cmp.finish();
...	@@ -165,10 +266,9 @@ test "compress/decompress" {	...	@@ -165,10 +266,9 @@ test "compress/decompress" {
165	}	266	}
166	// decompressor reader interface	267	// decompressor reader interface
167	{	268	{
168	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);	269	var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
169	var dcm = inflate.decompressor(container, compressed.reader());	270	var decompress: Decompress = .init(&compressed, container, &.{});
170	var dcm_rdr = dcm.reader();	271	const n = try decompress.reader.readSliceShort(&dcm_buf);
171	const n = try dcm_rdr.readAll(&dcm_buf);
172	try testing.expectEqual(data.len, n);	272	try testing.expectEqual(data.len, n);
173	try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);	273	try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
174	}	274	}
...	@@ -184,9 +284,9 @@ test "compress/decompress" {	...	@@ -184,9 +284,9 @@ test "compress/decompress" {
184		284
185	// compress original stream to compressed stream	285	// compress original stream to compressed stream
186	{	286	{
187	var original = fixedBufferStream(data);	287	var original: std.io.Reader = .fixed(data);
188	var compressed = fixedBufferStream(&cmp_buf);	288	var compressed: Writer = .fixed(&cmp_buf);
189	var cmp = try deflate.huffman.compressor(container, compressed.writer());	289	var cmp = try Compress.Huffman.init(container, &compressed);
190	try cmp.compress(original.reader());	290	try cmp.compress(original.reader());
191	try cmp.finish();	291	try cmp.finish();
192	if (compressed_size == 0) {	292	if (compressed_size == 0) {
...	@@ -198,10 +298,11 @@ test "compress/decompress" {	...	@@ -198,10 +298,11 @@ test "compress/decompress" {
198	}	298	}
199	// decompress compressed stream to decompressed stream	299	// decompress compressed stream to decompressed stream
200	{	300	{
201	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);	301	var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
202	var decompressed = fixedBufferStream(&dcm_buf);	302	var decompress: Decompress = .init(&compressed, container, &.{});
203	try inflate.decompress(container, compressed.reader(), decompressed.writer());	303	var decompressed: Writer = .fixed(&dcm_buf);
204	try testing.expectEqualSlices(u8, data, decompressed.getWritten());	304	_ = try decompress.reader.streamRemaining(&decompressed);
		305	try testing.expectEqualSlices(u8, data, decompressed.buffered());
205	}	306	}
206	}	307	}
207	}	308	}
...	@@ -216,9 +317,9 @@ test "compress/decompress" {	...	@@ -216,9 +317,9 @@ test "compress/decompress" {
216		317
217	// compress original stream to compressed stream	318	// compress original stream to compressed stream
218	{	319	{
219	var original = fixedBufferStream(data);	320	var original: std.io.Reader = .fixed(data);
220	var compressed = fixedBufferStream(&cmp_buf);	321	var compressed: Writer = .fixed(&cmp_buf);
221	var cmp = try deflate.store.compressor(container, compressed.writer());	322	var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed);
222	try cmp.compress(original.reader());	323	try cmp.compress(original.reader());
223	try cmp.finish();	324	try cmp.finish();
224	if (compressed_size == 0) {	325	if (compressed_size == 0) {
...	@@ -231,23 +332,25 @@ test "compress/decompress" {	...	@@ -231,23 +332,25 @@ test "compress/decompress" {
231	}	332	}
232	// decompress compressed stream to decompressed stream	333	// decompress compressed stream to decompressed stream
233	{	334	{
234	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);	335	var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
235	var decompressed = fixedBufferStream(&dcm_buf);	336	var decompress: Decompress = .init(&compressed, container, &.{});
236	try inflate.decompress(container, compressed.reader(), decompressed.writer());	337	var decompressed: Writer = .fixed(&dcm_buf);
237	try testing.expectEqualSlices(u8, data, decompressed.getWritten());	338	_ = try decompress.reader.streamRemaining(&decompressed);
		339	try testing.expectEqualSlices(u8, data, decompressed.buffered());
238	}	340	}
239	}	341	}
240	}	342	}
241	}	343	}
242	}	344	}
243		345
244	fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void {	346	fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
245	var in = fixedBufferStream(compressed);	347	var in: std.io.Reader = .fixed(compressed);
246	var out = std.ArrayList(u8).init(testing.allocator);	348	var aw: std.io.Writer.Allocating = .init(testing.allocator);
247	defer out.deinit();	349	defer aw.deinit();
248		350
249	try inflate.decompress(container, in.reader(), out.writer());	351	var decompress: Decompress = .init(&in, container, &.{});
250	try testing.expectEqualSlices(u8, expected_plain, out.items);	352	_ = try decompress.reader.streamRemaining(&aw.writer);
		353	try testing.expectEqualSlices(u8, expected_plain, aw.items);
251	}	354	}
252		355
253	test "don't read past deflate stream's end" {	356	test "don't read past deflate stream's end" {
...	@@ -352,126 +455,186 @@ test "gzip header" {	...	@@ -352,126 +455,186 @@ test "gzip header" {
352	}	455	}
353		456
354	test "public interface" {	457	test "public interface" {
355	const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };	458	const plain_data_buf = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
356		459
357	// deflate final stored block, header + plain (stored) data	460	// deflate final stored block, header + plain (stored) data
358	const deflate_block = [_]u8{	461	const deflate_block = [_]u8{
359	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen	462	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
360	} ++ plain_data;	463	} ++ plain_data_buf;
361		464
362	// gzip header/footer + deflate block	465	const plain_data: []const u8 = &plain_data_buf;
363	const gzip_data =	466	const gzip_data: []const u8 = &deflate_block;
364	[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)	467
365	deflate_block ++	468	//// gzip header/footer + deflate block
366	[_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)	469	//const gzip_data =
367		470	// [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
368	// zlib header/footer + deflate block	471	// deflate_block ++
369	const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}	472	// [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
370	deflate_block ++	473
371	[_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum	474	//// zlib header/footer + deflate block
372		475	//const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
373	const gzip = @import("gzip.zig");	476	// deflate_block ++
374	const zlib = @import("zlib.zig");	477	// [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
375	const flate = @This();	478
376		479	// TODO
377	try testInterface(gzip, &gzip_data, &plain_data);	480	//const gzip = @import("gzip.zig");
378	try testInterface(zlib, &zlib_data, &plain_data);	481	//const zlib = @import("zlib.zig");
379	try testInterface(flate, &deflate_block, &plain_data);
380	}
381		482
382	fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void {
383	var buffer1: [64]u8 = undefined;	483	var buffer1: [64]u8 = undefined;
384	var buffer2: [64]u8 = undefined;	484	var buffer2: [64]u8 = undefined;
385		485
386	var compressed = fixedBufferStream(&buffer1);	486	// TODO These used to be functions, need to migrate the tests
387	var plain = fixedBufferStream(&buffer2);	487	const decompress = void;
		488	const compress = void;
		489	const store = void;
388		490
389	// decompress	491	// decompress
390	{	492	{
391	var in = fixedBufferStream(gzip_data);	493	var plain: Writer = .fixed(&buffer2);
392	try pkg.decompress(in.reader(), plain.writer());	494
393	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	495	var in: std.io.Reader = .fixed(gzip_data);
		496	try decompress(&in, &plain);
		497	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
394	}	498	}
395	plain.reset();
396	compressed.reset();
397		499
398	// compress/decompress	500	// compress/decompress
399	{	501	{
400	var in = fixedBufferStream(plain_data);	502	var plain: Writer = .fixed(&buffer2);
401	try pkg.compress(in.reader(), compressed.writer(), .{});	503	var compressed: Writer = .fixed(&buffer1);
402	compressed.reset();	504
403	try pkg.decompress(compressed.reader(), plain.writer());	505	var in: std.io.Reader = .fixed(plain_data);
404	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	506	try compress(&in, &compressed, .{});
		507
		508	var r: std.io.Reader = .fixed(&buffer1);
		509	try decompress(&r, &plain);
		510	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
405	}	511	}
406	plain.reset();
407	compressed.reset();
408		512
409	// compressor/decompressor	513	// compressor/decompressor
410	{	514	{
411	var in = fixedBufferStream(plain_data);	515	var plain: Writer = .fixed(&buffer2);
412	var cmp = try pkg.compressor(compressed.writer(), .{});	516	var compressed: Writer = .fixed(&buffer1);
413	try cmp.compress(in.reader());	517
		518	var in: std.io.Reader = .fixed(plain_data);
		519	var cmp = try Compress(&compressed, .{});
		520	try cmp.compress(&in);
414	try cmp.finish();	521	try cmp.finish();
415		522
416	compressed.reset();	523	var r: std.io.Reader = .fixed(&buffer1);
417	var dcp = pkg.decompressor(compressed.reader());	524	var dcp = Decompress(&r);
418	try dcp.decompress(plain.writer());	525	try dcp.decompress(&plain);
419	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	526	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
420	}	527	}
421	plain.reset();
422	compressed.reset();
423		528
424	// huffman	529	// huffman
425	{	530	{
426	// huffman compress/decompress	531	// huffman compress/decompress
427	{	532	{
428	var in = fixedBufferStream(plain_data);	533	var plain: Writer = .fixed(&buffer2);
429	try pkg.huffman.compress(in.reader(), compressed.writer());	534	var compressed: Writer = .fixed(&buffer1);
430	compressed.reset();	535
431	try pkg.decompress(compressed.reader(), plain.writer());	536	var in: std.io.Reader = .fixed(plain_data);
432	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	537	try huffman.compress(&in, &compressed);
		538
		539	var r: std.io.Reader = .fixed(&buffer1);
		540	try decompress(&r, &plain);
		541	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
433	}	542	}
434	plain.reset();
435	compressed.reset();
436		543
437	// huffman compressor/decompressor	544	// huffman compressor/decompressor
438	{	545	{
439	var in = fixedBufferStream(plain_data);	546	var plain: Writer = .fixed(&buffer2);
440	var cmp = try pkg.huffman.compressor(compressed.writer());	547	var compressed: Writer = .fixed(&buffer1);
441	try cmp.compress(in.reader());	548
		549	var in: std.io.Reader = .fixed(plain_data);
		550	var cmp = try huffman.Compressor(&compressed);
		551	try cmp.compress(&in);
442	try cmp.finish();	552	try cmp.finish();
443		553
444	compressed.reset();	554	var r: std.io.Reader = .fixed(&buffer1);
445	try pkg.decompress(compressed.reader(), plain.writer());	555	try decompress(&r, &plain);
446	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	556	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
447	}	557	}
448	}	558	}
449	plain.reset();
450	compressed.reset();
451		559
452	// store	560	// store
453	{	561	{
454	// store compress/decompress	562	// store compress/decompress
455	{	563	{
456	var in = fixedBufferStream(plain_data);	564	var plain: Writer = .fixed(&buffer2);
457	try pkg.store.compress(in.reader(), compressed.writer());	565	var compressed: Writer = .fixed(&buffer1);
458	compressed.reset();	566
459	try pkg.decompress(compressed.reader(), plain.writer());	567	var in: std.io.Reader = .fixed(plain_data);
460	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	568	try store.compress(&in, &compressed);
		569
		570	var r: std.io.Reader = .fixed(&buffer1);
		571	try decompress(&r, &plain);
		572	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
461	}	573	}
462	plain.reset();
463	compressed.reset();
464		574
465	// store compressor/decompressor	575	// store compressor/decompressor
466	{	576	{
467	var in = fixedBufferStream(plain_data);	577	var plain: Writer = .fixed(&buffer2);
468	var cmp = try pkg.store.compressor(compressed.writer());	578	var compressed: Writer = .fixed(&buffer1);
469	try cmp.compress(in.reader());	579
		580	var in: std.io.Reader = .fixed(plain_data);
		581	var cmp = try store.compressor(&compressed);
		582	try cmp.compress(&in);
470	try cmp.finish();	583	try cmp.finish();
471		584
472	compressed.reset();	585	var r: std.io.Reader = .fixed(&buffer1);
473	try pkg.decompress(compressed.reader(), plain.writer());	586	try decompress(&r, &plain);
474	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());	587	try testing.expectEqualSlices(u8, plain_data, plain.buffered());
475	}	588	}
476	}	589	}
477	}	590	}
		591
		592	pub const match = struct {
		593	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
		594	pub const min_length = 4; // min length used in this algorithm
		595	pub const max_length = 258;
		596
		597	pub const min_distance = 1;
		598	pub const max_distance = 32768;
		599	};
		600
		601	pub const history_len = match.max_distance;
		602
		603	pub const lookup = struct {
		604	pub const bits = 15;
		605	pub const len = 1 << bits;
		606	pub const shift = 32 - bits;
		607	};
		608
		609	test "zlib should not overshoot" {
		610	// Compressed zlib data with extra 4 bytes at the end.
		611	const data = [_]u8{
		612	0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
		613	0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
		614	0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
		615	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
		616	};
		617
		618	var stream: std.io.Reader = .fixed(&data);
		619	const reader = stream.reader();
		620
		621	var dcp = Decompress.init(reader);
		622	var out: [128]u8 = undefined;
		623
		624	// Decompress
		625	var n = try dcp.reader().readAll(out[0..]);
		626
		627	// Expected decompressed data
		628	try std.testing.expectEqual(46, n);
		629	try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
		630
		631	// Decompressor don't overshoot underlying reader.
		632	// It is leaving it at the end of compressed data chunk.
		633	try std.testing.expectEqual(data.len - 4, stream.getPos());
		634	try std.testing.expectEqual(0, dcp.unreadBytes());
		635
		636	// 4 bytes after compressed chunk are available in reader.
		637	n = try reader.readAll(out[0..]);
		638	try std.testing.expectEqual(n, 4);
		639	try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
		640	}

lib/std/compress/flate/BlockWriter.zig created+696

...	@@ -0,0 +1,696 @@
1	//! Accepts list of tokens, decides what is best block type to write. What block
2	//! type will provide best compression. Writes header and body of the block.
3	const std = @import("std");
4	const io = std.io;
5	const assert = std.debug.assert;
6	const Writer = std.io.Writer;
7
8	const BlockWriter = @This();
9	const flate = @import("../flate.zig");
10	const Compress = flate.Compress;
11	const huffman = flate.huffman;
12	const Token = @import("Token.zig");
13
14	const codegen_order = huffman.codegen_order;
15	const end_code_mark = 255;
16
17	output: *Writer,
18
19	codegen_freq: [huffman.codegen_code_count]u16 = undefined,
20	literal_freq: [huffman.max_num_lit]u16 = undefined,
21	distance_freq: [huffman.distance_code_count]u16 = undefined,
22	codegen: [huffman.max_num_lit + huffman.distance_code_count + 1]u8 = undefined,
23	literal_encoding: Compress.LiteralEncoder = .{},
24	distance_encoding: Compress.DistanceEncoder = .{},
25	codegen_encoding: Compress.CodegenEncoder = .{},
26	fixed_literal_encoding: Compress.LiteralEncoder,
27	fixed_distance_encoding: Compress.DistanceEncoder,
28	huff_distance: Compress.DistanceEncoder,
29
30	pub fn init(output: *Writer) BlockWriter {
31	return .{
32	.output = output,
33	.fixed_literal_encoding = Compress.fixedLiteralEncoder(),
34	.fixed_distance_encoding = Compress.fixedDistanceEncoder(),
35	.huff_distance = Compress.huffmanDistanceEncoder(),
36	};
37	}
38
39	/// Flush intrenal bit buffer to the writer.
40	/// Should be called only when bit stream is at byte boundary.
41	///
42	/// That is after final block; when last byte could be incomplete or
43	/// after stored block; which is aligned to the byte boundary (it has x
44	/// padding bits after first 3 bits).
45	pub fn flush(self: *BlockWriter) Writer.Error!void {
46	try self.bit_writer.flush();
47	}
48
49	pub fn setWriter(self: BlockWriter, new_writer: Writer) void {
50	self.bit_writer.setWriter(new_writer);
51	}
52
53	fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void {
54	try self.bit_writer.writeBits(c.code, c.len);
55	}
56
57	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
58	// the literal and distance lengths arrays (which are concatenated into a single
59	// array). This method generates that run-length encoding.
60	//
61	// The result is written into the codegen array, and the frequencies
62	// of each code is written into the codegen_freq array.
63	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
64	// information. Code bad_code is an end marker
65	//
66	// num_literals: The number of literals in literal_encoding
67	// num_distances: The number of distances in distance_encoding
68	// lit_enc: The literal encoder to use
69	// dist_enc: The distance encoder to use
70	fn generateCodegen(
71	self: *BlockWriter,
72	num_literals: u32,
73	num_distances: u32,
74	lit_enc: *Compress.LiteralEncoder,
75	dist_enc: *Compress.DistanceEncoder,
76	) void {
77	for (self.codegen_freq, 0..) \|_, i\| {
78	self.codegen_freq[i] = 0;
79	}
80
81	// Note that we are using codegen both as a temporary variable for holding
82	// a copy of the frequencies, and as the place where we put the result.
83	// This is fine because the output is always shorter than the input used
84	// so far.
85	var codegen = &self.codegen; // cache
86	// Copy the concatenated code sizes to codegen. Put a marker at the end.
87	var cgnl = codegen[0..num_literals];
88	for (cgnl, 0..) \|_, i\| {
89	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
90	}
91
92	cgnl = codegen[num_literals .. num_literals + num_distances];
93	for (cgnl, 0..) \|_, i\| {
94	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
95	}
96	codegen[num_literals + num_distances] = end_code_mark;
97
98	var size = codegen[0];
99	var count: i32 = 1;
100	var out_index: u32 = 0;
101	var in_index: u32 = 1;
102	while (size != end_code_mark) : (in_index += 1) {
103	// INVARIANT: We have seen "count" copies of size that have not yet
104	// had output generated for them.
105	const next_size = codegen[in_index];
106	if (next_size == size) {
107	count += 1;
108	continue;
109	}
110	// We need to generate codegen indicating "count" of size.
111	if (size != 0) {
112	codegen[out_index] = size;
113	out_index += 1;
114	self.codegen_freq[size] += 1;
115	count -= 1;
116	while (count >= 3) {
117	var n: i32 = 6;
118	if (n > count) {
119	n = count;
120	}
121	codegen[out_index] = 16;
122	out_index += 1;
123	codegen[out_index] = @as(u8, @intCast(n - 3));
124	out_index += 1;
125	self.codegen_freq[16] += 1;
126	count -= n;
127	}
128	} else {
129	while (count >= 11) {
130	var n: i32 = 138;
131	if (n > count) {
132	n = count;
133	}
134	codegen[out_index] = 18;
135	out_index += 1;
136	codegen[out_index] = @as(u8, @intCast(n - 11));
137	out_index += 1;
138	self.codegen_freq[18] += 1;
139	count -= n;
140	}
141	if (count >= 3) {
142	// 3 <= count <= 10
143	codegen[out_index] = 17;
144	out_index += 1;
145	codegen[out_index] = @as(u8, @intCast(count - 3));
146	out_index += 1;
147	self.codegen_freq[17] += 1;
148	count = 0;
149	}
150	}
151	count -= 1;
152	while (count >= 0) : (count -= 1) {
153	codegen[out_index] = size;
154	out_index += 1;
155	self.codegen_freq[size] += 1;
156	}
157	// Set up invariant for next time through the loop.
158	size = next_size;
159	count = 1;
160	}
161	// Marker indicating the end of the codegen.
162	codegen[out_index] = end_code_mark;
163	}
164
165	const DynamicSize = struct {
166	size: u32,
167	num_codegens: u32,
168	};
169
170	// dynamicSize returns the size of dynamically encoded data in bits.
171	fn dynamicSize(
172	self: *BlockWriter,
173	lit_enc: *Compress.LiteralEncoder, // literal encoder
174	dist_enc: *Compress.DistanceEncoder, // distance encoder
175	extra_bits: u32,
176	) DynamicSize {
177	var num_codegens = self.codegen_freq.len;
178	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
179	num_codegens -= 1;
180	}
181	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
182	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
183	self.codegen_freq[16] * 2 +
184	self.codegen_freq[17] * 3 +
185	self.codegen_freq[18] * 7;
186	const size = header +
187	lit_enc.bitLength(&self.literal_freq) +
188	dist_enc.bitLength(&self.distance_freq) +
189	extra_bits;
190
191	return DynamicSize{
192	.size = @as(u32, @intCast(size)),
193	.num_codegens = @as(u32, @intCast(num_codegens)),
194	};
195	}
196
197	// fixedSize returns the size of dynamically encoded data in bits.
198	fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 {
199	return 3 +
200	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
201	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
202	extra_bits;
203	}
204
205	const StoredSize = struct {
206	size: u32,
207	storable: bool,
208	};
209
210	// storedSizeFits calculates the stored size, including header.
211	// The function returns the size in bits and whether the block
212	// fits inside a single block.
213	fn storedSizeFits(in: ?[]const u8) StoredSize {
214	if (in == null) {
215	return .{ .size = 0, .storable = false };
216	}
217	if (in.?.len <= huffman.max_store_block_size) {
218	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
219	}
220	return .{ .size = 0, .storable = false };
221	}
222
223	// Write the header of a dynamic Huffman block to the output stream.
224	//
225	// num_literals: The number of literals specified in codegen
226	// num_distances: The number of distances specified in codegen
227	// num_codegens: The number of codegens used in codegen
228	// eof: Is it the end-of-file? (end of stream)
229	fn dynamicHeader(
230	self: *BlockWriter,
231	num_literals: u32,
232	num_distances: u32,
233	num_codegens: u32,
234	eof: bool,
235	) Writer.Error!void {
236	const first_bits: u32 = if (eof) 5 else 4;
237	try self.bit_writer.writeBits(first_bits, 3);
238	try self.bit_writer.writeBits(num_literals - 257, 5);
239	try self.bit_writer.writeBits(num_distances - 1, 5);
240	try self.bit_writer.writeBits(num_codegens - 4, 4);
241
242	var i: u32 = 0;
243	while (i < num_codegens) : (i += 1) {
244	const value = self.codegen_encoding.codes[codegen_order[i]].len;
245	try self.bit_writer.writeBits(value, 3);
246	}
247
248	i = 0;
249	while (true) {
250	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
251	i += 1;
252	if (code_word == end_code_mark) {
253	break;
254	}
255	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
256
257	switch (code_word) {
258	16 => {
259	try self.bit_writer.writeBits(self.codegen[i], 2);
260	i += 1;
261	},
262	17 => {
263	try self.bit_writer.writeBits(self.codegen[i], 3);
264	i += 1;
265	},
266	18 => {
267	try self.bit_writer.writeBits(self.codegen[i], 7);
268	i += 1;
269	},
270	else => {},
271	}
272	}
273	}
274
275	fn storedHeader(self: *BlockWriter, length: usize, eof: bool) Writer.Error!void {
276	assert(length <= 65535);
277	const flag: u32 = if (eof) 1 else 0;
278	try self.bit_writer.writeBits(flag, 3);
279	try self.flush();
280	const l: u16 = @intCast(length);
281	try self.bit_writer.writeBits(l, 16);
282	try self.bit_writer.writeBits(~l, 16);
283	}
284
285	fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void {
286	// Indicate that we are a fixed Huffman block
287	var value: u32 = 2;
288	if (eof) {
289	value = 3;
290	}
291	try self.bit_writer.writeBits(value, 3);
292	}
293
294	// Write a block of tokens with the smallest encoding. Will choose block type.
295	// The original input can be supplied, and if the huffman encoded data
296	// is larger than the original bytes, the data will be written as a
297	// stored block.
298	// If the input is null, the tokens will always be Huffman encoded.
299	pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void {
300	const lit_and_dist = self.indexTokens(tokens);
301	const num_literals = lit_and_dist.num_literals;
302	const num_distances = lit_and_dist.num_distances;
303
304	var extra_bits: u32 = 0;
305	const ret = storedSizeFits(input);
306	const stored_size = ret.size;
307	const storable = ret.storable;
308
309	if (storable) {
310	// We only bother calculating the costs of the extra bits required by
311	// the length of distance fields (which will be the same for both fixed
312	// and dynamic encoding), if we need to compare those two encodings
313	// against stored encoding.
314	var length_code: u16 = Token.length_codes_start + 8;
315	while (length_code < num_literals) : (length_code += 1) {
316	// First eight length codes have extra size = 0.
317	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
318	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
319	}
320	var distance_code: u16 = 4;
321	while (distance_code < num_distances) : (distance_code += 1) {
322	// First four distance codes have extra size = 0.
323	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
324	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
325	}
326	}
327
328	// Figure out smallest code.
329	// Fixed Huffman baseline.
330	var literal_encoding = &self.fixed_literal_encoding;
331	var distance_encoding = &self.fixed_distance_encoding;
332	var size = self.fixedSize(extra_bits);
333
334	// Dynamic Huffman?
335	var num_codegens: u32 = 0;
336
337	// Generate codegen and codegenFrequencies, which indicates how to encode
338	// the literal_encoding and the distance_encoding.
339	self.generateCodegen(
340	num_literals,
341	num_distances,
342	&self.literal_encoding,
343	&self.distance_encoding,
344	);
345	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
346	const dynamic_size = self.dynamicSize(
347	&self.literal_encoding,
348	&self.distance_encoding,
349	extra_bits,
350	);
351	const dyn_size = dynamic_size.size;
352	num_codegens = dynamic_size.num_codegens;
353
354	if (dyn_size < size) {
355	size = dyn_size;
356	literal_encoding = &self.literal_encoding;
357	distance_encoding = &self.distance_encoding;
358	}
359
360	// Stored bytes?
361	if (storable and stored_size < size) {
362	try self.storedBlock(input.?, eof);
363	return;
364	}
365
366	// Huffman.
367	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
368	try self.fixedHeader(eof);
369	} else {
370	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
371	}
372
373	// Write the tokens.
374	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
375	}
376
377	pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
378	try self.storedHeader(input.len, eof);
379	try self.bit_writer.writeBytes(input);
380	}
381
382	// writeBlockDynamic encodes a block using a dynamic Huffman table.
383	// This should be used if the symbols used have a disproportionate
384	// histogram distribution.
385	// If input is supplied and the compression savings are below 1/16th of the
386	// input size the block is stored.
387	fn dynamicBlock(
388	self: *BlockWriter,
389	tokens: []const Token,
390	eof: bool,
391	input: ?[]const u8,
392	) Writer.Error!void {
393	const total_tokens = self.indexTokens(tokens);
394	const num_literals = total_tokens.num_literals;
395	const num_distances = total_tokens.num_distances;
396
397	// Generate codegen and codegenFrequencies, which indicates how to encode
398	// the literal_encoding and the distance_encoding.
399	self.generateCodegen(
400	num_literals,
401	num_distances,
402	&self.literal_encoding,
403	&self.distance_encoding,
404	);
405	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
406	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
407	const size = dynamic_size.size;
408	const num_codegens = dynamic_size.num_codegens;
409
410	// Store bytes, if we don't get a reasonable improvement.
411
412	const stored_size = storedSizeFits(input);
413	const ssize = stored_size.size;
414	const storable = stored_size.storable;
415	if (storable and ssize < (size + (size >> 4))) {
416	try self.storedBlock(input.?, eof);
417	return;
418	}
419
420	// Write Huffman table.
421	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
422
423	// Write the tokens.
424	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
425	}
426
427	const TotalIndexedTokens = struct {
428	num_literals: u32,
429	num_distances: u32,
430	};
431
432	// Indexes a slice of tokens followed by an end_block_marker, and updates
433	// literal_freq and distance_freq, and generates literal_encoding
434	// and distance_encoding.
435	// The number of literal and distance tokens is returned.
436	fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
437	var num_literals: u32 = 0;
438	var num_distances: u32 = 0;
439
440	for (self.literal_freq, 0..) \|_, i\| {
441	self.literal_freq[i] = 0;
442	}
443	for (self.distance_freq, 0..) \|_, i\| {
444	self.distance_freq[i] = 0;
445	}
446
447	for (tokens) \|t\| {
448	if (t.kind == Token.Kind.literal) {
449	self.literal_freq[t.literal()] += 1;
450	continue;
451	}
452	self.literal_freq[t.lengthCode()] += 1;
453	self.distance_freq[t.distanceCode()] += 1;
454	}
455	// add end_block_marker token at the end
456	self.literal_freq[huffman.end_block_marker] += 1;
457
458	// get the number of literals
459	num_literals = @as(u32, @intCast(self.literal_freq.len));
460	while (self.literal_freq[num_literals - 1] == 0) {
461	num_literals -= 1;
462	}
463	// get the number of distances
464	num_distances = @as(u32, @intCast(self.distance_freq.len));
465	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
466	num_distances -= 1;
467	}
468	if (num_distances == 0) {
469	// We haven't found a single match. If we want to go with the dynamic encoding,
470	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
471	self.distance_freq[0] = 1;
472	num_distances = 1;
473	}
474	self.literal_encoding.generate(&self.literal_freq, 15);
475	self.distance_encoding.generate(&self.distance_freq, 15);
476	return TotalIndexedTokens{
477	.num_literals = num_literals,
478	.num_distances = num_distances,
479	};
480	}
481
482	// Writes a slice of tokens to the output followed by and end_block_marker.
483	// codes for literal and distance encoding must be supplied.
484	fn writeTokens(
485	self: *BlockWriter,
486	tokens: []const Token,
487	le_codes: []Compress.HuffCode,
488	oe_codes: []Compress.HuffCode,
489	) Writer.Error!void {
490	for (tokens) \|t\| {
491	if (t.kind == Token.Kind.literal) {
492	try self.writeCode(le_codes[t.literal()]);
493	continue;
494	}
495
496	// Write the length
497	const le = t.lengthEncoding();
498	try self.writeCode(le_codes[le.code]);
499	if (le.extra_bits > 0) {
500	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
501	}
502
503	// Write the distance
504	const oe = t.distanceEncoding();
505	try self.writeCode(oe_codes[oe.code]);
506	if (oe.extra_bits > 0) {
507	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
508	}
509	}
510	// add end_block_marker at the end
511	try self.writeCode(le_codes[huffman.end_block_marker]);
512	}
513
514	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
515	// if the results only gains very little from compression.
516	pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
517	// Add everything as literals
518	histogram(input, &self.literal_freq);
519
520	self.literal_freq[huffman.end_block_marker] = 1;
521
522	const num_literals = huffman.end_block_marker + 1;
523	self.distance_freq[0] = 1;
524	const num_distances = 1;
525
526	self.literal_encoding.generate(&self.literal_freq, 15);
527
528	// Figure out smallest code.
529	// Always use dynamic Huffman or Store
530	var num_codegens: u32 = 0;
531
532	// Generate codegen and codegenFrequencies, which indicates how to encode
533	// the literal_encoding and the distance_encoding.
534	self.generateCodegen(
535	num_literals,
536	num_distances,
537	&self.literal_encoding,
538	&self.huff_distance,
539	);
540	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
541	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
542	const size = dynamic_size.size;
543	num_codegens = dynamic_size.num_codegens;
544
545	// Store bytes, if we don't get a reasonable improvement.
546	const stored_size_ret = storedSizeFits(input);
547	const ssize = stored_size_ret.size;
548	const storable = stored_size_ret.storable;
549
550	if (storable and ssize < (size + (size >> 4))) {
551	try self.storedBlock(input, eof);
552	return;
553	}
554
555	// Huffman.
556	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
557	const encoding = self.literal_encoding.codes[0..257];
558
559	for (input) \|t\| {
560	const c = encoding[t];
561	try self.bit_writer.writeBits(c.code, c.len);
562	}
563	try self.writeCode(encoding[huffman.end_block_marker]);
564	}
565
566	// histogram accumulates a histogram of b in h.
567	fn histogram(b: []const u8, h: *[286]u16) void {
568	// Clear histogram
569	for (h, 0..) \|_, i\| {
570	h[i] = 0;
571	}
572
573	var lh = h.*[0..256];
574	for (b) \|t\| {
575	lh[t] += 1;
576	}
577	}
578
579	// tests
580	const expect = std.testing.expect;
581	const fmt = std.fmt;
582	const testing = std.testing;
583	const ArrayList = std.ArrayList;
584
585	const TestCase = @import("testdata/block_writer.zig").TestCase;
586	const testCases = @import("testdata/block_writer.zig").testCases;
587
588	// tests if the writeBlock encoding has changed.
589	test "write" {
590	inline for (0..testCases.len) \|i\| {
591	try testBlock(testCases[i], .write_block);
592	}
593	}
594
595	// tests if the writeBlockDynamic encoding has changed.
596	test "dynamicBlock" {
597	inline for (0..testCases.len) \|i\| {
598	try testBlock(testCases[i], .write_dyn_block);
599	}
600	}
601
602	test "huffmanBlock" {
603	inline for (0..testCases.len) \|i\| {
604	try testBlock(testCases[i], .write_huffman_block);
605	}
606	try testBlock(.{
607	.tokens = &[_]Token{},
608	.input = "huffman-rand-max.input",
609	.want = "huffman-rand-max.{s}.expect",
610	}, .write_huffman_block);
611	}
612
613	const TestFn = enum {
614	write_block,
615	write_dyn_block, // write dynamic block
616	write_huffman_block,
617
618	fn to_s(self: TestFn) []const u8 {
619	return switch (self) {
620	.write_block => "wb",
621	.write_dyn_block => "dyn",
622	.write_huffman_block => "huff",
623	};
624	}
625
626	fn write(
627	comptime self: TestFn,
628	bw: anytype,
629	tok: []const Token,
630	input: ?[]const u8,
631	final: bool,
632	) !void {
633	switch (self) {
634	.write_block => try bw.write(tok, final, input),
635	.write_dyn_block => try bw.dynamicBlock(tok, final, input),
636	.write_huffman_block => try bw.huffmanBlock(input.?, final),
637	}
638	try bw.flush();
639	}
640	};
641
642	// testBlock tests a block against its references
643	//
644	// size
645	// 64K [file-name].input - input non compressed file
646	// 8.1K [file-name].golden -
647	// 78 [file-name].dyn.expect - output with writeBlockDynamic
648	// 78 [file-name].wb.expect - output with writeBlock
649	// 8.1K [file-name].huff.expect - output with writeBlockHuff
650	// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
651	// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
652	//
653	// wb - writeBlock
654	// dyn - writeBlockDynamic
655	// huff - writeBlockHuff
656	//
657	fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
658	if (tc.input.len != 0 and tc.want.len != 0) {
659	const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
660	const input = @embedFile("testdata/block_writer/" ++ tc.input);
661	const want = @embedFile("testdata/block_writer/" ++ want_name);
662	try testWriteBlock(tfn, input, want, tc.tokens);
663	}
664
665	if (tfn == .write_huffman_block) {
666	return;
667	}
668
669	const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
670	const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
671	try testWriteBlock(tfn, null, want, tc.tokens);
672	}
673
674	// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
675	fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
676	var buf = ArrayList(u8).init(testing.allocator);
677	var bw: BlockWriter = .init(buf.writer());
678	try tfn.write(&bw, tokens, input, false);
679	var got = buf.items;
680	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
681	try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
682	//
683	// Test if the writer produces the same output after reset.
684	buf.deinit();
685	buf = ArrayList(u8).init(testing.allocator);
686	defer buf.deinit();
687	bw.setWriter(buf.writer());
688
689	try tfn.write(&bw, tokens, input, true);
690	try bw.flush();
691	got = buf.items;
692
693	try expect(got[0] & 1 == 1); // bfinal is set
694	buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
695	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
696	}

lib/std/compress/flate/CircularBuffer.zig deleted-240

...	@@ -1,240 +0,0 @@
1	//! 64K buffer of uncompressed data created in inflate (decompression). Has enough
2	//! history to support writing match<length, distance>; copying length of bytes
3	//! from the position distance backward from current.
4	//!
5	//! Reads can return less than available bytes if they are spread across
6	//! different circles. So reads should repeat until get required number of bytes
7	//! or until returned slice is zero length.
8	//!
9	//! Note on deflate limits:
10	//! * non-compressible block is limited to 65,535 bytes.
11	//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12	//!
13	//! Whole non-compressed block can be written without overlap. We always have
14	//! history of up to 64K, more then 32K needed.
15	//!
16	const std = @import("std");
17	const assert = std.debug.assert;
18	const testing = std.testing;
19
20	const consts = @import("consts.zig").match;
21
22	const mask = 0xffff; // 64K - 1
23	const buffer_len = mask + 1; // 64K buffer
24
25	const Self = @This();
26
27	buffer: [buffer_len]u8 = undefined,
28	wp: usize = 0, // write position
29	rp: usize = 0, // read position
30
31	fn writeAll(self: *Self, buf: []const u8) void {
32	for (buf) \|c\| self.write(c);
33	}
34
35	/// Write literal.
36	pub fn write(self: *Self, b: u8) void {
37	assert(self.wp - self.rp < mask);
38	self.buffer[self.wp & mask] = b;
39	self.wp += 1;
40	}
41
42	/// Write match (back-reference to the same data slice) starting at `distance`
43	/// back from current write position, and `length` of bytes.
44	pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45	if (self.wp < distance or
46	length < consts.base_length or length > consts.max_length or
47	distance < consts.min_distance or distance > consts.max_distance)
48	{
49	return error.InvalidMatch;
50	}
51	assert(self.wp - self.rp < mask);
52
53	var from: usize = self.wp - distance & mask;
54	const from_end: usize = from + length;
55	var to: usize = self.wp & mask;
56	const to_end: usize = to + length;
57
58	self.wp += length;
59
60	// Fast path using memcpy
61	if (from_end < buffer_len and to_end < buffer_len) // start and end at the same circle
62	{
63	var cur_len = distance;
64	var remaining_len = length;
65	while (cur_len < remaining_len) {
66	@memcpy(self.buffer[to..][0..cur_len], self.buffer[from..][0..cur_len]);
67	to += cur_len;
68	remaining_len -= cur_len;
69	cur_len = cur_len * 2;
70	}
71	@memcpy(self.buffer[to..][0..remaining_len], self.buffer[from..][0..remaining_len]);
72	return;
73	}
74
75	// Slow byte by byte
76	while (to < to_end) {
77	self.buffer[to & mask] = self.buffer[from & mask];
78	to += 1;
79	from += 1;
80	}
81	}
82
83	/// Returns writable part of the internal buffer of size `n` at most. Advances
84	/// write pointer, assumes that returned buffer will be filled with data.
85	pub fn getWritable(self: *Self, n: usize) []u8 {
86	const wp = self.wp & mask;
87	const len = @min(n, buffer_len - wp);
88	self.wp += len;
89	return self.buffer[wp .. wp + len];
90	}
91
92	/// Read available data. Can return part of the available data if it is
93	/// spread across two circles. So read until this returns zero length.
94	pub fn read(self: *Self) []const u8 {
95	return self.readAtMost(buffer_len);
96	}
97
98	/// Read part of available data. Can return less than max even if there are
99	/// more than max decoded data.
100	pub fn readAtMost(self: *Self, limit: usize) []const u8 {
101	const rb = self.readBlock(if (limit == 0) buffer_len else limit);
102	defer self.rp += rb.len;
103	return self.buffer[rb.head..rb.tail];
104	}
105
106	const ReadBlock = struct {
107	head: usize,
108	tail: usize,
109	len: usize,
110	};
111
112	/// Returns position of continuous read block data.
113	fn readBlock(self: *Self, max: usize) ReadBlock {
114	const r = self.rp & mask;
115	const w = self.wp & mask;
116	const n = @min(
117	max,
118	if (w >= r) w - r else buffer_len - r,
119	);
120	return .{
121	.head = r,
122	.tail = r + n,
123	.len = n,
124	};
125	}
126
127	/// Number of free bytes for write.
128	pub fn free(self: *Self) usize {
129	return buffer_len - (self.wp - self.rp);
130	}
131
132	/// Full if largest match can't fit. 258 is largest match length. That much
133	/// bytes can be produced in single decode step.
134	pub fn full(self: *Self) bool {
135	return self.free() < 258 + 1;
136	}
137
138	// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
139	test writeMatch {
140	var cb: Self = .{};
141
142	cb.writeAll("a salad; ");
143	try cb.writeMatch(5, 9);
144	try cb.writeMatch(3, 3);
145
146	try testing.expectEqualStrings("a salad; a salsal", cb.read());
147	}
148
149	test "writeMatch overlap" {
150	var cb: Self = .{};
151
152	cb.writeAll("a b c ");
153	try cb.writeMatch(8, 4);
154	cb.write('d');
155
156	try testing.expectEqualStrings("a b c b c b c d", cb.read());
157	}
158
159	test readAtMost {
160	var cb: Self = .{};
161
162	cb.writeAll("0123456789");
163	try cb.writeMatch(50, 10);
164
165	try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
166	for (0..6) \|i\| {
167	try testing.expectEqual(i * 10, cb.rp);
168	try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
169	}
170	try testing.expectEqualStrings("", cb.readAtMost(10));
171	try testing.expectEqualStrings("", cb.read());
172	}
173
174	test Self {
175	var cb: Self = .{};
176
177	const data = "0123456789abcdef" ** (1024 / 16);
178	cb.writeAll(data);
179	try testing.expectEqual(@as(usize, 0), cb.rp);
180	try testing.expectEqual(@as(usize, 1024), cb.wp);
181	try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
182
183	for (0..62 * 4) \|_\|
184	try cb.writeMatch(256, 1024); // write 62K
185
186	try testing.expectEqual(@as(usize, 0), cb.rp);
187	try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
188	try testing.expectEqual(@as(usize, 1024), cb.free());
189
190	cb.writeAll(data[0..200]);
191	_ = cb.readAtMost(1024); // make some space
192	cb.writeAll(data); // overflows write position
193	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
194	try testing.expectEqual(@as(usize, 1024), cb.rp);
195	try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
196
197	const rb = cb.readBlock(Self.buffer_len);
198	try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
199	try testing.expectEqual(@as(usize, 1024), rb.head);
200	try testing.expectEqual(@as(usize, 65536), rb.tail);
201
202	try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
203	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
204	try testing.expectEqual(@as(usize, 65536), cb.rp);
205	try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
206
207	try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
208	}
209
210	test "write overlap" {
211	var cb: Self = .{};
212	cb.wp = cb.buffer.len - 15;
213	cb.rp = cb.wp;
214
215	cb.writeAll("0123456789");
216	cb.writeAll("abcdefghij");
217
218	try testing.expectEqual(cb.buffer.len + 5, cb.wp);
219	try testing.expectEqual(cb.buffer.len - 15, cb.rp);
220
221	try testing.expectEqualStrings("0123456789abcde", cb.read());
222	try testing.expectEqualStrings("fghij", cb.read());
223
224	try testing.expect(cb.wp == cb.rp);
225	}
226
227	test "writeMatch/read overlap" {
228	var cb: Self = .{};
229	cb.wp = cb.buffer.len - 15;
230	cb.rp = cb.wp;
231
232	cb.writeAll("0123456789");
233	try cb.writeMatch(15, 5);
234
235	try testing.expectEqualStrings("012345678956789", cb.read());
236	try testing.expectEqualStrings("5678956789", cb.read());
237
238	try cb.writeMatch(20, 25);
239	try testing.expectEqualStrings("01234567895678956789", cb.read());
240	}

lib/std/compress/flate/Compress.zig created+1264

...	@@ -0,0 +1,1264 @@
1	//! Default compression algorithm. Has two steps: tokenization and token
2	//! encoding.
3	//!
4	//! Tokenization takes uncompressed input stream and produces list of tokens.
5	//! Each token can be literal (byte of data) or match (backrefernce to previous
6	//! data with length and distance). Tokenization accumulators 32K tokens, when
7	//! full or `flush` is called tokens are passed to the `block_writer`. Level
8	//! defines how hard (how slow) it tries to find match.
9	//!
10	//! Block writer will decide which type of deflate block to write (stored, fixed,
11	//! dynamic) and encode tokens to the output byte stream. Client has to call
12	//! `finish` to write block with the final bit set.
13	//!
14	//! Container defines type of header and footer which can be gzip, zlib or raw.
15	//! They all share same deflate body. Raw has no header or footer just deflate
16	//! body.
17	//!
18	//! Compression algorithm explained in rfc-1951 (slightly edited for this case):
19	//!
20	//! The compressor uses a chained hash table `lookup` to find duplicated
21	//! strings, using a hash function that operates on 4-byte sequences. At any
22	//! given point during compression, let XYZW be the next 4 input bytes
23	//! (lookahead) to be examined (not necessarily all different, of course).
24	//! First, the compressor examines the hash chain for XYZW. If the chain is
25	//! empty, the compressor simply writes out X as a literal byte and advances
26	//! one byte in the input. If the hash chain is not empty, indicating that the
27	//! sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
28	//! hash function value) has occurred recently, the compressor compares all
29	//! strings on the XYZW hash chain with the actual input data sequence
30	//! starting at the current point, and selects the longest match.
31	//!
32	//! To improve overall compression, the compressor defers the selection of
33	//! matches ("lazy matching"): after a match of length N has been found, the
34	//! compressor searches for a longer match starting at the next input byte. If
35	//! it finds a longer match, it truncates the previous match to a length of
36	//! one (thus producing a single literal byte) and then emits the longer
37	//! match. Otherwise, it emits the original match, and, as described above,
38	//! advances N bytes before continuing.
39	//!
40	//!
41	//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
42	const builtin = @import("builtin");
43	const std = @import("std");
44	const assert = std.debug.assert;
45	const testing = std.testing;
46	const expect = testing.expect;
47	const mem = std.mem;
48	const math = std.math;
49	const Writer = std.Io.Writer;
50	const Reader = std.Io.Reader;
51
52	const Compress = @This();
53	const Token = @import("Token.zig");
54	const BlockWriter = @import("BlockWriter.zig");
55	const flate = @import("../flate.zig");
56	const Container = flate.Container;
57	const Lookup = @import("Lookup.zig");
58	const huffman = flate.huffman;
59
60	lookup: Lookup = .{},
61	tokens: Tokens = .{},
62	/// Asserted to have a buffer capacity of at least `flate.max_window_len`.
63	input: *Reader,
64	block_writer: BlockWriter,
65	level: LevelArgs,
66	hasher: Container.Hasher,
67	reader: Reader,
68
69	// Match and literal at the previous position.
70	// Used for lazy match finding in processWindow.
71	prev_match: ?Token = null,
72	prev_literal: ?u8 = null,
73
74	/// Trades between speed and compression size.
75	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
76	/// levels 1-3 are using different algorithm to perform faster but with less
77	/// compression. That is not implemented here.
78	pub const Level = enum(u4) {
79	level_4 = 4,
80	level_5 = 5,
81	level_6 = 6,
82	level_7 = 7,
83	level_8 = 8,
84	level_9 = 9,
85
86	fast = 0xb,
87	default = 0xc,
88	best = 0xd,
89	};
90
91	/// Number of tokens to accumulate in deflate before starting block encoding.
92	///
93	/// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
94	/// 8 and max 9 that gives 14 or 15 bits.
95	pub const n_tokens = 1 << 15;
96
97	/// Algorithm knobs for each level.
98	const LevelArgs = struct {
99	good: u16, // Do less lookups if we already have match of this length.
100	nice: u16, // Stop looking for better match if we found match with at least this length.
101	lazy: u16, // Don't do lazy match find if got match with at least this length.
102	chain: u16, // How many lookups for previous match to perform.
103
104	pub fn get(level: Level) LevelArgs {
105	return switch (level) {
106	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
107	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
108	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
109	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
110	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
111	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
112	};
113	}
114	};
115
116	pub const Options = struct {
117	level: Level = .default,
118	container: Container = .raw,
119	};
120
121	pub fn init(input: *Reader, buffer: []u8, options: Options) Compress {
122	return .{
123	.input = input,
124	.block_writer = undefined,
125	.level = .get(options.level),
126	.hasher = .init(options.container),
127	.state = .header,
128	.reader = .{
129	.buffer = buffer,
130	.stream = stream,
131	},
132	};
133	}
134
135	const FlushOption = enum { none, flush, final };
136
137	/// Process data in window and create tokens. If token buffer is full
138	/// flush tokens to the token writer.
139	///
140	/// Returns number of bytes consumed from `lh`.
141	fn tokenizeSlice(c: Compress, bw: Writer, limit: std.Io.Limit, lh: []const u8) !usize {
142	_ = bw;
143	_ = limit;
144	if (true) @panic("TODO");
145	var step: u16 = 1; // 1 in the case of literal, match length otherwise
146	const pos: u16 = c.win.pos();
147	const literal = lh[0]; // literal at current position
148	const min_len: u16 = if (c.prev_match) \|m\| m.length() else 0;
149
150	// Try to find match at least min_len long.
151	if (c.findMatch(pos, lh, min_len)) \|match\| {
152	// Found better match than previous.
153	try c.addPrevLiteral();
154
155	// Is found match length good enough?
156	if (match.length() >= c.level.lazy) {
157	// Don't try to lazy find better match, use this.
158	step = try c.addMatch(match);
159	} else {
160	// Store this match.
161	c.prev_literal = literal;
162	c.prev_match = match;
163	}
164	} else {
165	// There is no better match at current pos then it was previous.
166	// Write previous match or literal.
167	if (c.prev_match) \|m\| {
168	// Write match from previous position.
169	step = try c.addMatch(m) - 1; // we already advanced 1 from previous position
170	} else {
171	// No match at previous position.
172	// Write previous literal if any, and remember this literal.
173	try c.addPrevLiteral();
174	c.prev_literal = literal;
175	}
176	}
177	// Advance window and add hashes.
178	c.windowAdvance(step, lh, pos);
179	}
180
181	fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void {
182	// current position is already added in findMatch
183	self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
184	self.win.advance(step);
185	}
186
187	// Add previous literal (if any) to the tokens list.
188	fn addPrevLiteral(self: *Compress) !void {
189	if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
190	}
191
192	// Add match to the tokens list, reset prev pointers.
193	// Returns length of the added match.
194	fn addMatch(self: *Compress, m: Token) !u16 {
195	try self.addToken(m);
196	self.prev_literal = null;
197	self.prev_match = null;
198	return m.length();
199	}
200
201	fn addToken(self: *Compress, token: Token) !void {
202	self.tokens.add(token);
203	if (self.tokens.full()) try self.flushTokens(.none);
204	}
205
206	// Finds largest match in the history window with the data at current pos.
207	fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token {
208	var len: u16 = min_len;
209	// Previous location with the same hash (same 4 bytes).
210	var prev_pos = self.lookup.add(lh, pos);
211	// Last found match.
212	var match: ?Token = null;
213
214	// How much back-references to try, performance knob.
215	var chain: usize = self.level.chain;
216	if (len >= self.level.good) {
217	// If we've got a match that's good enough, only look in 1/4 the chain.
218	chain >>= 2;
219	}
220
221	// Hot path loop!
222	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
223	const distance = pos - prev_pos;
224	if (distance > flate.match.max_distance)
225	break;
226
227	const new_len = self.win.match(prev_pos, pos, len);
228	if (new_len > len) {
229	match = Token.initMatch(@intCast(distance), new_len);
230	if (new_len >= self.level.nice) {
231	// The match is good enough that we don't try to find a better one.
232	return match;
233	}
234	len = new_len;
235	}
236	prev_pos = self.lookup.prev(prev_pos);
237	}
238
239	return match;
240	}
241
242	fn flushTokens(self: *Compress, flush_opt: FlushOption) !void {
243	// Pass tokens to the token writer
244	try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
245	// Stored block ensures byte alignment.
246	// It has 3 bits (final, block_type) and then padding until byte boundary.
247	// After that everything is aligned to the boundary in the stored block.
248	// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
249	// Last 4 bytes are byte aligned.
250	if (flush_opt == .flush) {
251	try self.block_writer.storedBlock("", false);
252	}
253	if (flush_opt != .none) {
254	// Safe to call only when byte aligned or it is OK to add
255	// padding bits (on last byte of the final block).
256	try self.block_writer.flush();
257	}
258	// Reset internal tokens store.
259	self.tokens.reset();
260	// Notify win that tokens are flushed.
261	self.win.flush();
262	}
263
264	// Slide win and if needed lookup tables.
265	fn slide(self: *Compress) void {
266	const n = self.win.slide();
267	self.lookup.slide(n);
268	}
269
270	/// Flushes internal buffers to the output writer. Outputs empty stored
271	/// block to sync bit stream to the byte boundary, so that the
272	/// decompressor can get all input data available so far.
273	///
274	/// It is useful mainly in compressed network protocols, to ensure that
275	/// deflate bit stream can be used as byte stream. May degrade
276	/// compression so it should be used only when necessary.
277	///
278	/// Completes the current deflate block and follows it with an empty
279	/// stored block that is three zero bits plus filler bits to the next
280	/// byte, followed by four bytes (00 00 ff ff).
281	///
282	pub fn flush(c: *Compress) !void {
283	try c.tokenize(.flush);
284	}
285
286	/// Completes deflate bit stream by writing any pending data as deflate
287	/// final deflate block. HAS to be called once all data are written to
288	/// the compressor as a signal that next block has to have final bit
289	/// set.
290	///
291	pub fn finish(c: *Compress) !void {
292	_ = c;
293	@panic("TODO");
294	}
295
296	/// Use another writer while preserving history. Most probably flush
297	/// should be called on old writer before setting new.
298	pub fn setWriter(self: Compress, new_writer: Writer) void {
299	self.block_writer.setWriter(new_writer);
300	self.output = new_writer;
301	}
302
303	// Tokens store
304	const Tokens = struct {
305	list: [n_tokens]Token = undefined,
306	pos: usize = 0,
307
308	fn add(self: *Tokens, t: Token) void {
309	self.list[self.pos] = t;
310	self.pos += 1;
311	}
312
313	fn full(self: *Tokens) bool {
314	return self.pos == self.list.len;
315	}
316
317	fn reset(self: *Tokens) void {
318	self.pos = 0;
319	}
320
321	fn tokens(self: *Tokens) []const Token {
322	return self.list[0..self.pos];
323	}
324	};
325
326	/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
327	/// only performs Huffman entropy encoding. Results in faster compression, much
328	/// less memory requirements during compression but bigger compressed sizes.
329	pub const Huffman = SimpleCompressor(.huffman, .raw);
330
331	/// Creates store blocks only. Data are not compressed only packed into deflate
332	/// store blocks. That adds 9 bytes of header for each block. Max stored block
333	/// size is 64K. Block is emitted when flush is called on on finish.
334	pub const store = struct {
335	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
336	return SimpleCompressor(.store, container, WriterType);
337	}
338
339	pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
340	return try store.Compressor(container, @TypeOf(writer)).init(writer);
341	}
342	};
343
344	const SimpleCompressorKind = enum {
345	huffman,
346	store,
347	};
348
349	fn simpleCompressor(
350	comptime kind: SimpleCompressorKind,
351	comptime container: Container,
352	writer: anytype,
353	) !SimpleCompressor(kind, container, @TypeOf(writer)) {
354	return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
355	}
356
357	fn SimpleCompressor(
358	comptime kind: SimpleCompressorKind,
359	comptime container: Container,
360	comptime WriterType: type,
361	) type {
362	const BlockWriterType = BlockWriter(WriterType);
363	return struct {
364	buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
365	wp: usize = 0,
366
367	output: WriterType,
368	block_writer: BlockWriterType,
369	hasher: container.Hasher() = .{},
370
371	const Self = @This();
372
373	pub fn init(output: WriterType) !Self {
374	const self = Self{
375	.output = output,
376	.block_writer = BlockWriterType.init(output),
377	};
378	try container.writeHeader(self.output);
379	return self;
380	}
381
382	pub fn flush(self: *Self) !void {
383	try self.flushBuffer(false);
384	try self.block_writer.storedBlock("", false);
385	try self.block_writer.flush();
386	}
387
388	pub fn finish(self: *Self) !void {
389	try self.flushBuffer(true);
390	try self.block_writer.flush();
391	try container.writeFooter(&self.hasher, self.output);
392	}
393
394	fn flushBuffer(self: *Self, final: bool) !void {
395	const buf = self.buffer[0..self.wp];
396	switch (kind) {
397	.huffman => try self.block_writer.huffmanBlock(buf, final),
398	.store => try self.block_writer.storedBlock(buf, final),
399	}
400	self.wp = 0;
401	}
402	};
403	}
404
405	const LiteralNode = struct {
406	literal: u16,
407	freq: u16,
408	};
409
410	// Describes the state of the constructed tree for a given depth.
411	const LevelInfo = struct {
412	// Our level. for better printing
413	level: u32,
414
415	// The frequency of the last node at this level
416	last_freq: u32,
417
418	// The frequency of the next character to add to this level
419	next_char_freq: u32,
420
421	// The frequency of the next pair (from level below) to add to this level.
422	// Only valid if the "needed" value of the next lower level is 0.
423	next_pair_freq: u32,
424
425	// The number of chains remaining to generate for this level before moving
426	// up to the next level
427	needed: u32,
428	};
429
430	// hcode is a huffman code with a bit code and bit length.
431	pub const HuffCode = struct {
432	code: u16 = 0,
433	len: u16 = 0,
434
435	// set sets the code and length of an hcode.
436	fn set(self: *HuffCode, code: u16, length: u16) void {
437	self.len = length;
438	self.code = code;
439	}
440	};
441
442	pub fn HuffmanEncoder(comptime size: usize) type {
443	return struct {
444	codes: [size]HuffCode = undefined,
445	// Reusable buffer with the longest possible frequency table.
446	freq_cache: [huffman.max_num_frequencies + 1]LiteralNode = undefined,
447	bit_count: [17]u32 = undefined,
448	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
449	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
450
451	const Self = @This();
452
453	// Update this Huffman Code object to be the minimum code for the specified frequency count.
454	//
455	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
456	// max_bits The maximum number of bits to use for any literal.
457	pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
458	var list = self.freq_cache[0 .. freq.len + 1];
459	// Number of non-zero literals
460	var count: u32 = 0;
461	// Set list to be the set of all non-zero literals and their frequencies
462	for (freq, 0..) \|f, i\| {
463	if (f != 0) {
464	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
465	count += 1;
466	} else {
467	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
468	self.codes[i].len = 0;
469	}
470	}
471	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
472
473	list = list[0..count];
474	if (count <= 2) {
475	// Handle the small cases here, because they are awkward for the general case code. With
476	// two or fewer literals, everything has bit length 1.
477	for (list, 0..) \|node, i\| {
478	// "list" is in order of increasing literal value.
479	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
480	}
481	return;
482	}
483	self.lfs = list;
484	mem.sort(LiteralNode, self.lfs, {}, byFreq);
485
486	// Get the number of literals for each bit count
487	const bit_count = self.bitCounts(list, max_bits);
488	// And do the assignment
489	self.assignEncodingAndSize(bit_count, list);
490	}
491
492	pub fn bitLength(self: *Self, freq: []u16) u32 {
493	var total: u32 = 0;
494	for (freq, 0..) \|f, i\| {
495	if (f != 0) {
496	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
497	}
498	}
499	return total;
500	}
501
502	// Return the number of literals assigned to each bit size in the Huffman encoding
503	//
504	// This method is only called when list.len >= 3
505	// The cases of 0, 1, and 2 literals are handled by special case code.
506	//
507	// list: An array of the literals with non-zero frequencies
508	// and their associated frequencies. The array is in order of increasing
509	// frequency, and has as its last element a special element with frequency
510	// `math.maxInt(i32)`
511	//
512	// max_bits: The maximum number of bits that should be used to encode any literal.
513	// Must be less than 16.
514	//
515	// Returns an integer array in which array[i] indicates the number of literals
516	// that should be encoded in i bits.
517	fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
518	var max_bits = max_bits_to_use;
519	const n = list.len;
520	const max_bits_limit = 16;
521
522	assert(max_bits < max_bits_limit);
523
524	// The tree can't have greater depth than n - 1, no matter what. This
525	// saves a little bit of work in some small cases
526	max_bits = @min(max_bits, n - 1);
527
528	// Create information about each of the levels.
529	// A bogus "Level 0" whose sole purpose is so that
530	// level1.prev.needed == 0. This makes level1.next_pair_freq
531	// be a legitimate value that never gets chosen.
532	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
533	// leaf_counts[i] counts the number of literals at the left
534	// of ancestors of the rightmost node at level i.
535	// leaf_counts[i][j] is the number of literals at the left
536	// of the level j ancestor.
537	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
538
539	{
540	var level = @as(u32, 1);
541	while (level <= max_bits) : (level += 1) {
542	// For every level, the first two items are the first two characters.
543	// We initialize the levels as if we had already figured this out.
544	levels[level] = LevelInfo{
545	.level = level,
546	.last_freq = list[1].freq,
547	.next_char_freq = list[2].freq,
548	.next_pair_freq = list[0].freq + list[1].freq,
549	.needed = 0,
550	};
551	leaf_counts[level][level] = 2;
552	if (level == 1) {
553	levels[level].next_pair_freq = math.maxInt(i32);
554	}
555	}
556	}
557
558	// We need a total of 2*n - 2 items at top level and have already generated 2.
559	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
560
561	{
562	var level = max_bits;
563	while (true) {
564	var l = &levels[level];
565	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
566	// We've run out of both leaves and pairs.
567	// End all calculations for this level.
568	// To make sure we never come back to this level or any lower level,
569	// set next_pair_freq impossibly large.
570	l.needed = 0;
571	levels[level + 1].next_pair_freq = math.maxInt(i32);
572	level += 1;
573	continue;
574	}
575
576	const prev_freq = l.last_freq;
577	if (l.next_char_freq < l.next_pair_freq) {
578	// The next item on this row is a leaf node.
579	const next = leaf_counts[level][level] + 1;
580	l.last_freq = l.next_char_freq;
581	// Lower leaf_counts are the same of the previous node.
582	leaf_counts[level][level] = next;
583	if (next >= list.len) {
584	l.next_char_freq = maxNode().freq;
585	} else {
586	l.next_char_freq = list[next].freq;
587	}
588	} else {
589	// The next item on this row is a pair from the previous row.
590	// next_pair_freq isn't valid until we generate two
591	// more values in the level below
592	l.last_freq = l.next_pair_freq;
593	// Take leaf counts from the lower level, except counts[level] remains the same.
594	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
595	levels[l.level - 1].needed = 2;
596	}
597
598	l.needed -= 1;
599	if (l.needed == 0) {
600	// We've done everything we need to do for this level.
601	// Continue calculating one level up. Fill in next_pair_freq
602	// of that level with the sum of the two nodes we've just calculated on
603	// this level.
604	if (l.level == max_bits) {
605	// All done!
606	break;
607	}
608	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
609	level += 1;
610	} else {
611	// If we stole from below, move down temporarily to replenish it.
612	while (levels[level - 1].needed > 0) {
613	level -= 1;
614	if (level == 0) {
615	break;
616	}
617	}
618	}
619	}
620	}
621
622	// Somethings is wrong if at the end, the top level is null or hasn't used
623	// all of the leaves.
624	assert(leaf_counts[max_bits][max_bits] == n);
625
626	var bit_count = self.bit_count[0 .. max_bits + 1];
627	var bits: u32 = 1;
628	const counts = &leaf_counts[max_bits];
629	{
630	var level = max_bits;
631	while (level > 0) : (level -= 1) {
632	// counts[level] gives the number of literals requiring at least "bits"
633	// bits to encode.
634	bit_count[bits] = counts[level] - counts[level - 1];
635	bits += 1;
636	if (level == 0) {
637	break;
638	}
639	}
640	}
641	return bit_count;
642	}
643
644	// Look at the leaves and assign them a bit count and an encoding as specified
645	// in RFC 1951 3.2.2
646	fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
647	var code = @as(u16, 0);
648	var list = list_arg;
649
650	for (bit_count, 0..) \|bits, n\| {
651	code <<= 1;
652	if (n == 0 or bits == 0) {
653	continue;
654	}
655	// The literals list[list.len-bits] .. list[list.len-bits]
656	// are encoded using "bits" bits, and get the values
657	// code, code + 1, .... The code values are
658	// assigned in literal order (not frequency order).
659	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
660
661	self.lns = chunk;
662	mem.sort(LiteralNode, self.lns, {}, byLiteral);
663
664	for (chunk) \|node\| {
665	self.codes[node.literal] = HuffCode{
666	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
667	.len = @as(u16, @intCast(n)),
668	};
669	code += 1;
670	}
671	list = list[0 .. list.len - @as(u32, @intCast(bits))];
672	}
673	}
674	};
675	}
676
677	fn maxNode() LiteralNode {
678	return LiteralNode{
679	.literal = math.maxInt(u16),
680	.freq = math.maxInt(u16),
681	};
682	}
683
684	pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
685	return .{};
686	}
687
688	pub const LiteralEncoder = HuffmanEncoder(huffman.max_num_frequencies);
689	pub const DistanceEncoder = HuffmanEncoder(huffman.distance_code_count);
690	pub const CodegenEncoder = HuffmanEncoder(19);
691
692	// Generates a HuffmanCode corresponding to the fixed literal table
693	pub fn fixedLiteralEncoder() LiteralEncoder {
694	var h: LiteralEncoder = undefined;
695	var ch: u16 = 0;
696
697	while (ch < huffman.max_num_frequencies) : (ch += 1) {
698	var bits: u16 = undefined;
699	var size: u16 = undefined;
700	switch (ch) {
701	0...143 => {
702	// size 8, 000110000 .. 10111111
703	bits = ch + 48;
704	size = 8;
705	},
706	144...255 => {
707	// size 9, 110010000 .. 111111111
708	bits = ch + 400 - 144;
709	size = 9;
710	},
711	256...279 => {
712	// size 7, 0000000 .. 0010111
713	bits = ch - 256;
714	size = 7;
715	},
716	else => {
717	// size 8, 11000000 .. 11000111
718	bits = ch + 192 - 280;
719	size = 8;
720	},
721	}
722	h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
723	}
724	return h;
725	}
726
727	pub fn fixedDistanceEncoder() DistanceEncoder {
728	var h: DistanceEncoder = undefined;
729	for (h.codes, 0..) \|_, ch\| {
730	h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
731	}
732	return h;
733	}
734
735	pub fn huffmanDistanceEncoder() DistanceEncoder {
736	var distance_freq = [1]u16{0} ** huffman.distance_code_count;
737	distance_freq[0] = 1;
738	// huff_distance is a static distance encoder used for huffman only encoding.
739	// It can be reused since we will not be encoding distance values.
740	var h: DistanceEncoder = .{};
741	h.generate(distance_freq[0..], 15);
742	return h;
743	}
744
745	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
746	_ = context;
747	return a.literal < b.literal;
748	}
749
750	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
751	_ = context;
752	if (a.freq == b.freq) {
753	return a.literal < b.literal;
754	}
755	return a.freq < b.freq;
756	}
757
758	fn stream(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
759	const c: *Compress = @fieldParentPtr("reader", r);
760	switch (c.state) {
761	.header => \|i\| {
762	const header = c.hasher.container().header();
763	const n = try w.write(header[i..]);
764	if (header.len - i - n == 0) {
765	c.state = .middle;
766	} else {
767	c.state.header += n;
768	}
769	return n;
770	},
771	.middle => {
772	c.input.fillMore() catch \|err\| switch (err) {
773	error.EndOfStream => {
774	c.state = .final;
775	return 0;
776	},
777	else => \|e\| return e,
778	};
779	const buffer_contents = c.input.buffered();
780	const min_lookahead = flate.match.min_length + flate.match.max_length;
781	const history_plus_lookahead_len = flate.history_len + min_lookahead;
782	if (buffer_contents.len < history_plus_lookahead_len) return 0;
783	const lookahead = buffer_contents[flate.history_len..];
784	const start = w.count;
785	const n = try c.tokenizeSlice(w, limit, lookahead) catch \|err\| switch (err) {
786	error.WriteFailed => return error.WriteFailed,
787	};
788	c.hasher.update(lookahead[0..n]);
789	c.input.toss(n);
790	return w.count - start;
791	},
792	.final => {
793	const buffer_contents = c.input.buffered();
794	const start = w.count;
795	const n = c.tokenizeSlice(w, limit, buffer_contents) catch \|err\| switch (err) {
796	error.WriteFailed => return error.WriteFailed,
797	};
798	if (buffer_contents.len - n == 0) {
799	c.hasher.update(buffer_contents);
800	c.input.tossAll();
801	{
802	// In the case of flushing, last few lookahead buffers were
803	// smaller than min match len, so only last literal can be
804	// unwritten.
805	assert(c.prev_match == null);
806	try c.addPrevLiteral();
807	c.prev_literal = null;
808
809	try c.flushTokens(.final);
810	}
811	switch (c.hasher) {
812	.gzip => \|*gzip\| {
813	// GZIP 8 bytes footer
814	// - 4 bytes, CRC32 (CRC-32)
815	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
816	comptime assert(c.footer_buffer.len == 8);
817	std.mem.writeInt(u32, c.footer_buffer[0..4], gzip.final(), .little);
818	std.mem.writeInt(u32, c.footer_buffer[4..8], gzip.bytes_read, .little);
819	c.state = .{ .footer = 0 };
820	},
821	.zlib => \|*zlib\| {
822	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
823	// 4 bytes of ADLER32 (Adler-32 checksum)
824	// Checksum value of the uncompressed data (excluding any
825	// dictionary data) computed according to Adler-32
826	// algorithm.
827	comptime assert(c.footer_buffer.len == 8);
828	std.mem.writeInt(u32, c.footer_buffer[4..8], zlib.final, .big);
829	c.state = .{ .footer = 4 };
830	},
831	.raw => {
832	c.state = .ended;
833	},
834	}
835	}
836	return w.count - start;
837	},
838	.ended => return error.EndOfStream,
839	.footer => \|i\| {
840	const remaining = c.footer_buffer[i..];
841	const n = try w.write(limit.slice(remaining));
842	c.state = if (n == remaining) .ended else .{ .footer = i - n };
843	return n;
844	},
845	}
846	}
847
848	test "generate a Huffman code from an array of frequencies" {
849	var freqs: [19]u16 = [_]u16{
850	8, // 0
851	1, // 1
852	1, // 2
853	2, // 3
854	5, // 4
855	10, // 5
856	9, // 6
857	1, // 7
858	0, // 8
859	0, // 9
860	0, // 10
861	0, // 11
862	0, // 12
863	0, // 13
864	0, // 14
865	0, // 15
866	1, // 16
867	3, // 17
868	5, // 18
869	};
870
871	var enc = huffmanEncoder(19);
872	enc.generate(freqs[0..], 7);
873
874	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
875
876	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
877	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
878	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
879	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
880	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
881	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
882	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
883	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
884	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
885	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
886	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
887	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
888	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
889	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
890	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
891	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
892	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
893	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
894	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
895
896	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
897	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
898	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
899	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
900	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
901	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
902	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
903	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
904	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
905	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
906	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
907	}
908
909	test "generate a Huffman code for the fixed literal table specific to Deflate" {
910	const enc = fixedLiteralEncoder();
911	for (enc.codes) \|c\| {
912	switch (c.len) {
913	7 => {
914	const v = @bitReverse(@as(u7, @intCast(c.code)));
915	try testing.expect(v <= 0b0010111);
916	},
917	8 => {
918	const v = @bitReverse(@as(u8, @intCast(c.code)));
919	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
920	(v >= 0b11000000 and v <= 11000111));
921	},
922	9 => {
923	const v = @bitReverse(@as(u9, @intCast(c.code)));
924	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
925	},
926	else => unreachable,
927	}
928	}
929	}
930
931	test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
932	const enc = fixedDistanceEncoder();
933	for (enc.codes) \|c\| {
934	const v = @bitReverse(@as(u5, @intCast(c.code)));
935	try testing.expect(v <= 29);
936	try testing.expect(c.len == 5);
937	}
938	}
939
940	// Reverse bit-by-bit a N-bit code.
941	fn bitReverse(comptime T: type, value: T, n: usize) T {
942	const r = @bitReverse(value);
943	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
944	}
945
946	test bitReverse {
947	const ReverseBitsTest = struct {
948	in: u16,
949	bit_count: u5,
950	out: u16,
951	};
952
953	const reverse_bits_tests = [_]ReverseBitsTest{
954	.{ .in = 1, .bit_count = 1, .out = 1 },
955	.{ .in = 1, .bit_count = 2, .out = 2 },
956	.{ .in = 1, .bit_count = 3, .out = 4 },
957	.{ .in = 1, .bit_count = 4, .out = 8 },
958	.{ .in = 1, .bit_count = 5, .out = 16 },
959	.{ .in = 17, .bit_count = 5, .out = 17 },
960	.{ .in = 257, .bit_count = 9, .out = 257 },
961	.{ .in = 29, .bit_count = 5, .out = 23 },
962	};
963
964	for (reverse_bits_tests) \|h\| {
965	const v = bitReverse(u16, h.in, h.bit_count);
966	try std.testing.expectEqual(h.out, v);
967	}
968	}
969
970	test "fixedLiteralEncoder codes" {
971	var al = std.ArrayList(u8).init(testing.allocator);
972	defer al.deinit();
973	var bw = std.Io.bitWriter(.little, al.writer());
974
975	const f = fixedLiteralEncoder();
976	for (f.codes) \|c\| {
977	try bw.writeBits(c.code, c.len);
978	}
979	try testing.expectEqualSlices(u8, &fixed_codes, al.items);
980	}
981
982	pub const fixed_codes = [_]u8{
983	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
984	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
985	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
986	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
987	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
988	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
989	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
990	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
991	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
992	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
993	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
994	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
995	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
996	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
997	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
998	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
999	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
1000	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
1001	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
1002	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
1003	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
1004	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
1005	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
1006	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
1007	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
1008	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
1009	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
1010	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
1011	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
1012	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
1013	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
1014	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
1015	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
1016	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
1017	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
1018	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
1019	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
1020	0b10100011,
1021	};
1022
1023	test "tokenization" {
1024	const L = Token.initLiteral;
1025	const M = Token.initMatch;
1026
1027	const cases = [_]struct {
1028	data: []const u8,
1029	tokens: []const Token,
1030	}{
1031	.{
1032	.data = "Blah blah blah blah blah!",
1033	.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
1034	},
1035	.{
1036	.data = "ABCDEABCD ABCDEABCD",
1037	.tokens = &[_]Token{
1038	L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
1039	L('A'), M(10, 8),
1040	},
1041	},
1042	};
1043
1044	for (cases) \|c\| {
1045	inline for (Container.list) \|container\| { // for each wrapping
1046
1047	var cw = std.Io.countingWriter(std.Io.null_writer);
1048	const cww = cw.writer();
1049	var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
1050
1051	_ = try df.write(c.data);
1052	try df.flush();
1053
1054	// df.token_writer.show();
1055	try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
1056	try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
1057
1058	try testing.expectEqual(container.headerSize(), cw.bytes_written);
1059	try df.finish();
1060	try testing.expectEqual(container.size(), cw.bytes_written);
1061	}
1062	}
1063	}
1064
1065	// Tests that tokens written are equal to expected token list.
1066	const TestTokenWriter = struct {
1067	const Self = @This();
1068
1069	pos: usize = 0,
1070	actual: [128]Token = undefined,
1071
1072	pub fn init(_: anytype) Self {
1073	return .{};
1074	}
1075	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
1076	for (tokens) \|t\| {
1077	self.actual[self.pos] = t;
1078	self.pos += 1;
1079	}
1080	}
1081
1082	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
1083
1084	pub fn get(self: *Self) []Token {
1085	return self.actual[0..self.pos];
1086	}
1087
1088	pub fn show(self: *Self) void {
1089	std.debug.print("\n", .{});
1090	for (self.get()) \|t\| {
1091	t.show();
1092	}
1093	}
1094
1095	pub fn flush(_: *Self) !void {}
1096	};
1097
1098	test "file tokenization" {
1099	const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
1100	const cases = [_]struct {
1101	data: []const u8, // uncompressed content
1102	// expected number of tokens producet in deflate tokenization
1103	tokens_count: [levels.len]usize = .{0} ** levels.len,
1104	}{
1105	.{
1106	.data = @embedFile("testdata/rfc1951.txt"),
1107	.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
1108	},
1109
1110	.{
1111	.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
1112	.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
1113	},
1114	.{
1115	.data = @embedFile("testdata/block_writer/huffman-pi.input"),
1116	.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
1117	},
1118	.{
1119	.data = @embedFile("testdata/block_writer/huffman-text.input"),
1120	.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
1121	},
1122	.{
1123	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
1124	.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
1125	},
1126	.{
1127	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
1128	.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
1129	},
1130	};
1131
1132	for (cases) \|case\| { // for each case
1133	const data = case.data;
1134
1135	for (levels, 0..) \|level, i\| { // for each compression level
1136	var original: Reader = .fixed(data);
1137
1138	// buffer for decompressed data
1139	var al = std.ArrayList(u8).init(testing.allocator);
1140	defer al.deinit();
1141	const writer = al.writer();
1142
1143	// create compressor
1144	const WriterType = @TypeOf(writer);
1145	const TokenWriter = TokenDecoder(@TypeOf(writer));
1146	var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
1147
1148	// Stream uncompressed `original` data to the compressor. It will
1149	// produce tokens list and pass that list to the TokenDecoder. This
1150	// TokenDecoder uses CircularBuffer from inflate to convert list of
1151	// tokens back to the uncompressed stream.
1152	try cmp.compress(original.reader());
1153	try cmp.flush();
1154	const expected_count = case.tokens_count[i];
1155	const actual = cmp.block_writer.tokens_count;
1156	if (expected_count == 0) {
1157	std.debug.print("actual token count {d}\n", .{actual});
1158	} else {
1159	try testing.expectEqual(expected_count, actual);
1160	}
1161
1162	try testing.expectEqual(data.len, al.items.len);
1163	try testing.expectEqualSlices(u8, data, al.items);
1164	}
1165	}
1166	}
1167
1168	const TokenDecoder = struct {
1169	output: *Writer,
1170	tokens_count: usize,
1171
1172	pub fn init(output: *Writer) TokenDecoder {
1173	return .{
1174	.output = output,
1175	.tokens_count = 0,
1176	};
1177	}
1178
1179	pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void {
1180	self.tokens_count += tokens.len;
1181	for (tokens) \|t\| {
1182	switch (t.kind) {
1183	.literal => self.hist.write(t.literal()),
1184	.match => try self.hist.writeMatch(t.length(), t.distance()),
1185	}
1186	if (self.hist.free() < 285) try self.flushWin();
1187	}
1188	try self.flushWin();
1189	}
1190
1191	fn flushWin(self: *TokenDecoder) !void {
1192	while (true) {
1193	const buf = self.hist.read();
1194	if (buf.len == 0) break;
1195	try self.output.writeAll(buf);
1196	}
1197	}
1198	};
1199
1200	test "store simple compressor" {
1201	const data = "Hello world!";
1202	const expected = [_]u8{
1203	0x1, // block type 0, final bit set
1204	0xc, 0x0, // len = 12
1205	0xf3, 0xff, // ~len
1206	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
1207	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
1208	};
1209
1210	var fbs: Reader = .fixed(data);
1211	var al = std.ArrayList(u8).init(testing.allocator);
1212	defer al.deinit();
1213
1214	var cmp = try store.compressor(.raw, al.writer());
1215	try cmp.compress(&fbs);
1216	try cmp.finish();
1217	try testing.expectEqualSlices(u8, &expected, al.items);
1218
1219	fbs = .fixed(data);
1220	try al.resize(0);
1221
1222	// huffman only compresoor will also emit store block for this small sample
1223	var hc = try huffman.compressor(.raw, al.writer());
1224	try hc.compress(&fbs);
1225	try hc.finish();
1226	try testing.expectEqualSlices(u8, &expected, al.items);
1227	}
1228
1229	test "sliding window match" {
1230	const data = "Blah blah blah blah blah!";
1231	var win: Writer = .{};
1232	try expect(win.write(data) == data.len);
1233	try expect(win.wp == data.len);
1234	try expect(win.rp == 0);
1235
1236	// length between l symbols
1237	try expect(win.match(1, 6, 0) == 18);
1238	try expect(win.match(1, 11, 0) == 13);
1239	try expect(win.match(1, 16, 0) == 8);
1240	try expect(win.match(1, 21, 0) == 0);
1241
1242	// position 15 = "blah blah!"
1243	// position 20 = "blah!"
1244	try expect(win.match(15, 20, 0) == 4);
1245	try expect(win.match(15, 20, 3) == 4);
1246	try expect(win.match(15, 20, 4) == 0);
1247	}
1248
1249	test "sliding window slide" {
1250	var win: Writer = .{};
1251	win.wp = Writer.buffer_len - 11;
1252	win.rp = Writer.buffer_len - 111;
1253	win.buffer[win.rp] = 0xab;
1254	try expect(win.lookahead().len == 100);
1255	try expect(win.tokensBuffer().?.len == win.rp);
1256
1257	const n = win.slide();
1258	try expect(n == 32757);
1259	try expect(win.buffer[win.rp] == 0xab);
1260	try expect(win.rp == Writer.hist_len - 111);
1261	try expect(win.wp == Writer.hist_len - 11);
1262	try expect(win.lookahead().len == 100);
1263	try expect(win.tokensBuffer() == null);
1264	}

lib/std/compress/flate/Decompress.zig created+894

...	@@ -0,0 +1,894 @@
1	const std = @import("../../std.zig");
2	const flate = std.compress.flate;
3	const Container = flate.Container;
4	const Token = @import("Token.zig");
5	const testing = std.testing;
6	const Decompress = @This();
7	const Writer = std.io.Writer;
8	const Reader = std.io.Reader;
9
10	input: *Reader,
11	reader: Reader,
12	/// Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
13	hasher: Container.Hasher,
14
15	lit_dec: LiteralDecoder,
16	dst_dec: DistanceDecoder,
17
18	final_block: bool,
19	state: State,
20
21	read_err: ?Error,
22
23	const BlockType = enum(u2) {
24	stored = 0,
25	fixed = 1,
26	dynamic = 2,
27	};
28
29	const State = union(enum) {
30	protocol_header,
31	block_header,
32	stored_block: u16,
33	fixed_block,
34	dynamic_block,
35	protocol_footer,
36	end,
37	};
38
39	pub const Error = Container.Error \|\| error{
40	InvalidCode,
41	InvalidMatch,
42	InvalidBlockType,
43	WrongStoredBlockNlen,
44	InvalidDynamicBlockHeader,
45	EndOfStream,
46	ReadFailed,
47	OversubscribedHuffmanTree,
48	IncompleteHuffmanTree,
49	MissingEndOfBlockCode,
50	};
51
52	pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress {
53	return .{
54	.reader = .{
55	// TODO populate discard so that when an amount is discarded that
56	// includes an entire frame, skip decoding that frame.
57	.vtable = &.{ .stream = stream },
58	.buffer = buffer,
59	.seek = 0,
60	.end = 0,
61	},
62	.input = input,
63	.hasher = .init(container),
64	.lit_dec = .{},
65	.dst_dec = .{},
66	.final_block = false,
67	.state = .protocol_header,
68	.read_err = null,
69	};
70	}
71
72	fn decodeLength(self: *Decompress, code: u8) !u16 {
73	if (code > 28) return error.InvalidCode;
74	const ml = Token.matchLength(code);
75	return if (ml.extra_bits == 0) // 0 - 5 extra bits
76	ml.base
77	else
78	ml.base + try self.takeNBitsBuffered(ml.extra_bits);
79	}
80
81	fn decodeDistance(self: *Decompress, code: u8) !u16 {
82	if (code > 29) return error.InvalidCode;
83	const md = Token.matchDistance(code);
84	return if (md.extra_bits == 0) // 0 - 13 extra bits
85	md.base
86	else
87	md.base + try self.takeNBitsBuffered(md.extra_bits);
88	}
89
90	// Decode code length symbol to code length. Writes decoded length into
91	// lens slice starting at position pos. Returns number of positions
92	// advanced.
93	fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usize {
94	if (pos >= lens.len)
95	return error.InvalidDynamicBlockHeader;
96
97	switch (code) {
98	0...15 => {
99	// Represent code lengths of 0 - 15
100	lens[pos] = @intCast(code);
101	return 1;
102	},
103	16 => {
104	// Copy the previous code length 3 - 6 times.
105	// The next 2 bits indicate repeat length
106	const n: u8 = @as(u8, try self.takeBits(u2)) + 3;
107	if (pos == 0 or pos + n > lens.len)
108	return error.InvalidDynamicBlockHeader;
109	for (0..n) \|i\| {
110	lens[pos + i] = lens[pos + i - 1];
111	}
112	return n;
113	},
114	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
115	17 => return @as(u8, try self.takeBits(u3)) + 3,
116	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
117	18 => return @as(u8, try self.takeBits(u7)) + 11,
118	else => return error.InvalidDynamicBlockHeader,
119	}
120	}
121
122	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
123	// decoder to find symbol for that code. We then know how many bits is
124	// used. Shift bit reader for that much bits, those bits are used. And
125	// return symbol.
126	fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
127	const sym = try decoder.find(try self.peekBitsReverseBuffered(u15));
128	try self.shiftBits(sym.code_bits);
129	return sym;
130	}
131
132	pub fn stream(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
133	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
134	return readInner(d, w, limit) catch \|err\| switch (err) {
135	error.EndOfStream => return error.EndOfStream,
136	error.WriteFailed => return error.WriteFailed,
137	else => \|e\| {
138	// In the event of an error, state is unmodified so that it can be
139	// better used to diagnose the failure.
140	d.read_err = e;
141	return error.ReadFailed;
142	},
143	};
144	}
145
146	fn readInner(d: Decompress, w: Writer, limit: std.io.Limit) (Error \|\| Reader.StreamError)!usize {
147	const in = d.input;
148	sw: switch (d.state) {
149	.protocol_header => switch (d.hasher.container()) {
150	.gzip => {
151	const Header = extern struct {
152	magic: u16 align(1),
153	method: u8,
154	flags: packed struct(u8) {
155	text: bool,
156	hcrc: bool,
157	extra: bool,
158	name: bool,
159	comment: bool,
160	reserved: u3,
161	},
162	mtime: u32 align(1),
163	xfl: u8,
164	os: u8,
165	};
166	const header = try in.takeStruct(Header, .little);
167	if (header.magic != 0x8b1f or header.method != 0x08)
168	return error.BadGzipHeader;
169	if (header.flags.extra) {
170	const extra_len = try in.takeInt(u16, .little);
171	try in.discardAll(extra_len);
172	}
173	if (header.flags.name) {
174	_ = try in.discardDelimiterInclusive(0);
175	}
176	if (header.flags.comment) {
177	_ = try in.discardDelimiterInclusive(0);
178	}
179	if (header.flags.hcrc) {
180	try in.discardAll(2);
181	}
182	continue :sw .block_header;
183	},
184	.zlib => {
185	const Header = extern struct {
186	cmf: packed struct(u8) {
187	cm: u4,
188	cinfo: u4,
189	},
190	flg: u8,
191	};
192	const header = try in.takeStruct(Header);
193	if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader;
194	continue :sw .block_header;
195	},
196	.raw => continue :sw .block_header,
197	},
198	.block_header => {
199	d.final_block = (try d.takeBits(u1)) != 0;
200	const block_type = try d.takeBits(BlockType);
201	switch (block_type) {
202	.stored => {
203	d.alignBitsToByte(); // skip padding until byte boundary
204	// everything after this is byte aligned in stored block
205	const len = try in.takeInt(u16, .little);
206	const nlen = try in.takeInt(u16, .little);
207	if (len != ~nlen) return error.WrongStoredBlockNlen;
208	continue :sw .{ .stored_block = len };
209	},
210	.fixed => continue :sw .fixed_block,
211	.dynamic => {
212	const hlit: u16 = @as(u16, try d.takeBits(u5)) + 257; // number of ll code entries present - 257
213	const hdist: u16 = @as(u16, try d.takeBits(u5)) + 1; // number of distance code entries - 1
214	const hclen: u8 = @as(u8, try d.takeBits(u4)) + 4; // hclen + 4 code lengths are encoded
215
216	if (hlit > 286 or hdist > 30)
217	return error.InvalidDynamicBlockHeader;
218
219	// lengths for code lengths
220	var cl_lens = [_]u4{0} ** 19;
221	for (0..hclen) \|i\| {
222	cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3);
223	}
224	var cl_dec: CodegenDecoder = .{};
225	try cl_dec.generate(&cl_lens);
226
227	// decoded code lengths
228	var dec_lens = [_]u4{0} ** (286 + 30);
229	var pos: usize = 0;
230	while (pos < hlit + hdist) {
231	const sym = try cl_dec.find(try d.peekBitsReverse(u7));
232	try d.shiftBits(sym.code_bits);
233	pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos);
234	}
235	if (pos > hlit + hdist) {
236	return error.InvalidDynamicBlockHeader;
237	}
238
239	// literal code lengths to literal decoder
240	try d.lit_dec.generate(dec_lens[0..hlit]);
241
242	// distance code lengths to distance decoder
243	try d.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
244
245	continue :sw .dynamic_block;
246	},
247	}
248	},
249	.stored_block => \|remaining_len\| {
250	const out = try w.writableSliceGreedyPreserve(flate.history_len, 1);
251	const limited_out = limit.min(.limited(remaining_len)).slice(out);
252	const n = try d.input.readVec(&.{limited_out});
253	if (remaining_len - n == 0) {
254	d.state = if (d.final_block) .protocol_footer else .block_header;
255	} else {
256	d.state = .{ .stored_block = @intCast(remaining_len - n) };
257	}
258	w.advance(n);
259	return n;
260	},
261	.fixed_block => {
262	const start = w.count;
263	while (@intFromEnum(limit) > w.count - start) {
264	const code = try d.readFixedCode();
265	switch (code) {
266	0...255 => try w.writeBytePreserve(flate.history_len, @intCast(code)),
267	256 => {
268	d.state = if (d.final_block) .protocol_footer else .block_header;
269	return w.count - start;
270	},
271	257...285 => {
272	// Handles fixed block non literal (length) code.
273	// Length code is followed by 5 bits of distance code.
274	const length = try d.decodeLength(@intCast(code - 257));
275	const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5));
276	try writeMatch(w, length, distance);
277	},
278	else => return error.InvalidCode,
279	}
280	}
281	d.state = .fixed_block;
282	return w.count - start;
283	},
284	.dynamic_block => {
285	// In larger archives most blocks are usually dynamic, so decompression
286	// performance depends on this logic.
287	const start = w.count;
288	while (@intFromEnum(limit) > w.count - start) {
289	const sym = try d.decodeSymbol(&d.lit_dec);
290
291	switch (sym.kind) {
292	.literal => try w.writeBytePreserve(flate.history_len, sym.symbol),
293	.match => {
294	// Decode match backreference <length, distance>
295	const length = try d.decodeLength(sym.symbol);
296	const dsm = try d.decodeSymbol(&d.dst_dec);
297	const distance = try d.decodeDistance(dsm.symbol);
298	try writeMatch(w, length, distance);
299	},
300	.end_of_block => {
301	d.state = if (d.final_block) .protocol_footer else .block_header;
302	return w.count - start;
303	},
304	}
305	}
306	d.state = .dynamic_block;
307	return w.count - start;
308	},
309	.protocol_footer => {
310	d.alignBitsToByte();
311	switch (d.hasher) {
312	.gzip => \|*gzip\| {
313	if (try in.takeInt(u32, .little) != gzip.crc.final()) return error.WrongGzipChecksum;
314	if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize;
315	},
316	.zlib => \|*zlib\| {
317	const chksum: u32 = @byteSwap(zlib.final());
318	if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum;
319	},
320	.raw => {},
321	}
322	d.state = .end;
323	return 0;
324	},
325	.end => return error.EndOfStream,
326	}
327	}
328
329	/// Write match (back-reference to the same data slice) starting at `distance`
330	/// back from current write position, and `length` of bytes.
331	fn writeMatch(bw: *Writer, length: u16, distance: u16) !void {
332	_ = bw;
333	_ = length;
334	_ = distance;
335	@panic("TODO");
336	}
337
338	fn takeBits(d: *Decompress, comptime T: type) !T {
339	_ = d;
340	@panic("TODO");
341	}
342
343	fn takeBitsReverseBuffered(d: *Decompress, comptime T: type) !T {
344	_ = d;
345	@panic("TODO");
346	}
347
348	fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 {
349	_ = d;
350	_ = n;
351	@panic("TODO");
352	}
353
354	fn peekBitsReverse(d: *Decompress, comptime T: type) !T {
355	_ = d;
356	@panic("TODO");
357	}
358
359	fn peekBitsReverseBuffered(d: *Decompress, comptime T: type) !T {
360	_ = d;
361	@panic("TODO");
362	}
363
364	fn alignBitsToByte(d: *Decompress) void {
365	_ = d;
366	@panic("TODO");
367	}
368
369	fn shiftBits(d: *Decompress, n: u6) !void {
370	_ = d;
371	_ = n;
372	@panic("TODO");
373	}
374
375	fn readFixedCode(d: *Decompress) !u16 {
376	_ = d;
377	@panic("TODO");
378	}
379
380	pub const Symbol = packed struct {
381	pub const Kind = enum(u2) {
382	literal,
383	end_of_block,
384	match,
385	};
386
387	symbol: u8 = 0, // symbol from alphabet
388	code_bits: u4 = 0, // number of bits in code 0-15
389	kind: Kind = .literal,
390
391	code: u16 = 0, // huffman code of the symbol
392	next: u16 = 0, // pointer to the next symbol in linked list
393	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
394
395	// Sorting less than function.
396	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
397	if (a.code_bits == b.code_bits) {
398	if (a.kind == b.kind) {
399	return a.symbol < b.symbol;
400	}
401	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
402	}
403	return a.code_bits < b.code_bits;
404	}
405	};
406
407	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
408	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
409	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
410
411	/// Creates huffman tree codes from list of code lengths (in `build`).
412	///
413	/// `find` then finds symbol for code bits. Code can be any length between 1 and
414	/// 15 bits. When calling `find` we don't know how many bits will be used to
415	/// find symbol. When symbol is returned it has code_bits field which defines
416	/// how much we should advance in bit stream.
417	///
418	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
419	/// many times in this table; 32K places for 286 (at most) symbols.
420	/// Small lookup table is optimization for faster search.
421	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
422	/// with difference that we here use statically allocated arrays.
423	///
424	fn HuffmanDecoder(
425	comptime alphabet_size: u16,
426	comptime max_code_bits: u4,
427	comptime lookup_bits: u4,
428	) type {
429	const lookup_shift = max_code_bits - lookup_bits;
430
431	return struct {
432	// all symbols in alaphabet, sorted by code_len, symbol
433	symbols: [alphabet_size]Symbol = undefined,
434	// lookup table code -> symbol
435	lookup: [1 << lookup_bits]Symbol = undefined,
436
437	const Self = @This();
438
439	/// Generates symbols and lookup tables from list of code lens for each symbol.
440	pub fn generate(self: *Self, lens: []const u4) !void {
441	try checkCompleteness(lens);
442
443	// init alphabet with code_bits
444	for (self.symbols, 0..) \|_, i\| {
445	const cb: u4 = if (i < lens.len) lens[i] else 0;
446	self.symbols[i] = if (i < 256)
447	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
448	else if (i == 256)
449	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
450	else
451	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
452	}
453	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
454
455	// reset lookup table
456	for (0..self.lookup.len) \|i\| {
457	self.lookup[i] = .{};
458	}
459
460	// assign code to symbols
461	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
462	var code: u16 = 0;
463	var idx: u16 = 0;
464	for (&self.symbols, 0..) \|*sym, pos\| {
465	if (sym.code_bits == 0) continue; // skip unused
466	sym.code = code;
467
468	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
469	const next_idx = next_code >> lookup_shift;
470
471	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
472	if (sym.code_bits <= lookup_bits) {
473	// fill small lookup table
474	for (idx..next_idx) \|j\|
475	self.lookup[j] = sym.*;
476	} else {
477	// insert into linked table starting at root
478	const root = &self.lookup[idx];
479	const root_next = root.next;
480	root.next = @intCast(pos);
481	sym.next = root_next;
482	}
483
484	idx = next_idx;
485	code = next_code;
486	}
487	}
488
489	/// Given the list of code lengths check that it represents a canonical
490	/// Huffman code for n symbols.
491	///
492	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
493	fn checkCompleteness(lens: []const u4) !void {
494	if (alphabet_size == 286)
495	if (lens[256] == 0) return error.MissingEndOfBlockCode;
496
497	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
498	var max: usize = 0;
499	for (lens) \|n\| {
500	if (n == 0) continue;
501	if (n > max) max = n;
502	count[n] += 1;
503	}
504	if (max == 0) // empty tree
505	return;
506
507	// check for an over-subscribed or incomplete set of lengths
508	var left: usize = 1; // one possible code of zero length
509	for (1..count.len) \|len\| {
510	left <<= 1; // one more bit, double codes left
511	if (count[len] > left)
512	return error.OversubscribedHuffmanTree;
513	left -= count[len]; // deduct count from possible codes
514	}
515	if (left > 0) { // left > 0 means incomplete
516	// incomplete code ok only for single length 1 code
517	if (max_code_bits > 7 and max == count[0] + count[1]) return;
518	return error.IncompleteHuffmanTree;
519	}
520	}
521
522	/// Finds symbol for lookup table code.
523	pub fn find(self: *Self, code: u16) !Symbol {
524	// try to find in lookup table
525	const idx = code >> lookup_shift;
526	const sym = self.lookup[idx];
527	if (sym.code_bits != 0) return sym;
528	// if not use linked list of symbols with same prefix
529	return self.findLinked(code, sym.next);
530	}
531
532	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
533	var pos = start;
534	while (pos > 0) {
535	const sym = self.symbols[pos];
536	const shift = max_code_bits - sym.code_bits;
537	// compare code_bits number of upper bits
538	if ((code ^ sym.code) >> shift == 0) return sym;
539	pos = sym.next;
540	}
541	return error.InvalidCode;
542	}
543	};
544	}
545
546	test "init/find" {
547	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
548	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
549	var h: CodegenDecoder = .{};
550	try h.generate(&code_lens);
551
552	const expected = [_]struct {
553	sym: Symbol,
554	code: u16,
555	}{
556	.{
557	.code = 0b00_00000,
558	.sym = .{ .symbol = 3, .code_bits = 2 },
559	},
560	.{
561	.code = 0b01_00000,
562	.sym = .{ .symbol = 18, .code_bits = 2 },
563	},
564	.{
565	.code = 0b100_0000,
566	.sym = .{ .symbol = 1, .code_bits = 3 },
567	},
568	.{
569	.code = 0b101_0000,
570	.sym = .{ .symbol = 4, .code_bits = 3 },
571	},
572	.{
573	.code = 0b110_0000,
574	.sym = .{ .symbol = 17, .code_bits = 3 },
575	},
576	.{
577	.code = 0b1110_000,
578	.sym = .{ .symbol = 0, .code_bits = 4 },
579	},
580	.{
581	.code = 0b1111_000,
582	.sym = .{ .symbol = 16, .code_bits = 4 },
583	},
584	};
585
586	// unused symbols
587	for (0..12) \|i\| {
588	try testing.expectEqual(0, h.symbols[i].code_bits);
589	}
590	// used, from index 12
591	for (expected, 12..) \|e, i\| {
592	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
593	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
594	const sym_from_code = try h.find(e.code);
595	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
596	}
597
598	// All possible codes for each symbol.
599	// Lookup table has 126 elements, to cover all possible 7 bit codes.
600	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
601	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
602
603	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
604	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
605
606	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
607	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
608
609	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
610	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
611
612	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
613	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
614
615	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
616	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
617
618	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
619	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
620	}
621
622	test "encode/decode literals" {
623	const LiteralEncoder = std.compress.flate.Compress.LiteralEncoder;
624
625	for (1..286) \|j\| { // for all different number of codes
626	var enc: LiteralEncoder = .{};
627	// create frequencies
628	var freq = [_]u16{0} ** 286;
629	freq[256] = 1; // ensure we have end of block code
630	for (&freq, 1..) \|*f, i\| {
631	if (i % j == 0)
632	f.* = @intCast(i);
633	}
634
635	// encoder from frequencies
636	enc.generate(&freq, 15);
637
638	// get code_lens from encoder
639	var code_lens = [_]u4{0} ** 286;
640	for (code_lens, 0..) \|_, i\| {
641	code_lens[i] = @intCast(enc.codes[i].len);
642	}
643	// generate decoder from code lens
644	var dec: LiteralDecoder = .{};
645	try dec.generate(&code_lens);
646
647	// expect decoder code to match original encoder code
648	for (dec.symbols) \|s\| {
649	if (s.code_bits == 0) continue;
650	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
651	const symbol: u16 = switch (s.kind) {
652	.literal => s.symbol,
653	.end_of_block => 256,
654	.match => @as(u16, s.symbol) + 257,
655	};
656
657	const c = enc.codes[symbol];
658	try testing.expect(c.code == c_code);
659	}
660
661	// find each symbol by code
662	for (enc.codes) \|c\| {
663	if (c.len == 0) continue;
664
665	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
666	const s = try dec.find(s_code);
667	try testing.expect(s.code == s_code);
668	try testing.expect(s.code_bits == c.len);
669	}
670	}
671	}
672
673	test "decompress" {
674	const cases = [_]struct {
675	in: []const u8,
676	out: []const u8,
677	}{
678	// non compressed block (type 0)
679	.{
680	.in = &[_]u8{
681	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
682	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
683	},
684	.out = "Hello world\n",
685	},
686	// fixed code block (type 1)
687	.{
688	.in = &[_]u8{
689	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
690	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
691	},
692	.out = "Hello world\n",
693	},
694	// dynamic block (type 2)
695	.{
696	.in = &[_]u8{
697	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
698	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
699	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
700	},
701	.out = "ABCDEABCD ABCDEABCD",
702	},
703	};
704	for (cases) \|c\| {
705	var fb: Reader = .fixed(c.in);
706	var aw: Writer.Allocating = .init(testing.allocator);
707	defer aw.deinit();
708
709	var decompress: Decompress = .init(&fb, .raw, &.{});
710	const r = &decompress.reader;
711	_ = try r.streamRemaining(&aw.writer);
712	try testing.expectEqualStrings(c.out, aw.getWritten());
713	}
714	}
715
716	test "gzip decompress" {
717	const cases = [_]struct {
718	in: []const u8,
719	out: []const u8,
720	}{
721	// non compressed block (type 0)
722	.{
723	.in = &[_]u8{
724	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
725	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
726	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
727	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
728	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
729	},
730	.out = "Hello world\n",
731	},
732	// fixed code block (type 1)
733	.{
734	.in = &[_]u8{
735	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
736	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
737	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
738	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
739	},
740	.out = "Hello world\n",
741	},
742	// dynamic block (type 2)
743	.{
744	.in = &[_]u8{
745	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
746	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
747	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
748	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
749	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
750	},
751	.out = "ABCDEABCD ABCDEABCD",
752	},
753	// gzip header with name
754	.{
755	.in = &[_]u8{
756	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
757	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
758	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
759	},
760	.out = "Hello world\n",
761	},
762	};
763	for (cases) \|c\| {
764	var fb: Reader = .fixed(c.in);
765	var aw: Writer.Allocating = .init(testing.allocator);
766	defer aw.deinit();
767
768	var decompress: Decompress = .init(&fb, .gzip, &.{});
769	const r = &decompress.reader;
770	_ = try r.streamRemaining(&aw.writer);
771	try testing.expectEqualStrings(c.out, aw.getWritten());
772	}
773	}
774
775	test "zlib decompress" {
776	const cases = [_]struct {
777	in: []const u8,
778	out: []const u8,
779	}{
780	// non compressed block (type 0)
781	.{
782	.in = &[_]u8{
783	0x78, 0b10_0_11100, // zlib header (2 bytes)
784	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
785	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
786	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
787	},
788	.out = "Hello world\n",
789	},
790	};
791	for (cases) \|c\| {
792	var fb: Reader = .fixed(c.in);
793	var aw: Writer.Allocating = .init(testing.allocator);
794	defer aw.deinit();
795
796	var decompress: Decompress = .init(&fb, .zlib, &.{});
797	const r = &decompress.reader;
798	_ = try r.streamRemaining(&aw.writer);
799	try testing.expectEqualStrings(c.out, aw.getWritten());
800	}
801	}
802
803	test "fuzzing tests" {
804	const cases = [_]struct {
805	input: []const u8,
806	out: []const u8 = "",
807	err: ?anyerror = null,
808	}{
809	.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
810	.{ .input = "empty-distance-alphabet01" },
811	.{ .input = "empty-distance-alphabet02" },
812	.{ .input = "end-of-stream", .err = error.EndOfStream },
813	.{ .input = "invalid-distance", .err = error.InvalidMatch },
814	.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
815	.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
816	.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
817	.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
818	.{ .input = "out-of-codes", .err = error.InvalidCode },
819	.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
820	.{ .input = "puff02", .err = error.EndOfStream },
821	.{ .input = "puff03", .out = &[_]u8{0xa} },
822	.{ .input = "puff04", .err = error.InvalidCode },
823	.{ .input = "puff05", .err = error.EndOfStream },
824	.{ .input = "puff06", .err = error.EndOfStream },
825	.{ .input = "puff08", .err = error.InvalidCode },
826	.{ .input = "puff09", .out = "P" },
827	.{ .input = "puff10", .err = error.InvalidCode },
828	.{ .input = "puff11", .err = error.InvalidMatch },
829	.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
830	.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
831	.{ .input = "puff14", .err = error.EndOfStream },
832	.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
833	.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
834	.{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
835	.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
836	.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
837	.{ .input = "fuzz3", .err = error.InvalidMatch },
838	.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
839	.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
840	.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
841	.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
842	.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
843	.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
844	.{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
845	.{ .input = "puff24", .err = error.IncompleteHuffmanTree },
846	.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
847	.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
848	.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
849	};
850
851	inline for (cases, 0..) \|c, case_no\| {
852	var in: Reader = .fixed(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
853	var aw: Writer.Allocating = .init(testing.allocator);
854	defer aw.deinit();
855	errdefer std.debug.print("test case failed {}\n", .{case_no});
856
857	var decompress: Decompress = .init(&in, .raw, &.{});
858	const r = &decompress.reader;
859	if (c.err) \|expected_err\| {
860	try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer));
861	try testing.expectError(expected_err, decompress.read_err.?);
862	} else {
863	_ = try r.streamRemaining(&aw.writer);
864	try testing.expectEqualStrings(c.out, aw.getWritten());
865	}
866	}
867	}
868
869	test "bug 18966" {
870	const input = @embedFile("testdata/fuzz/bug_18966.input");
871	const expect = @embedFile("testdata/fuzz/bug_18966.expect");
872
873	var in: Reader = .fixed(input);
874	var aw: Writer.Allocating = .init(testing.allocator);
875	defer aw.deinit();
876
877	var decompress: Decompress = .init(&in, .gzip, &.{});
878	const r = &decompress.reader;
879	_ = try r.streamRemaining(&aw.writer);
880	try testing.expectEqualStrings(expect, aw.getWritten());
881	}
882
883	test "reading into empty buffer" {
884	// Inspired by https://github.com/ziglang/zig/issues/19895
885	const input = &[_]u8{
886	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
887	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
888	};
889	var in: Reader = .fixed(input);
890	var decomp: Decompress = .init(&in, .raw, &.{});
891	const r = &decomp.reader;
892	var buf: [0]u8 = undefined;
893	try testing.expectEqual(0, try r.readVec(&.{&buf}));
894	}

lib/std/compress/flate/Lookup.zig+15-15

...	@@ -5,22 +5,22 @@	...	@@ -5,22 +5,22 @@
5	const std = @import("std");	5	const std = @import("std");
6	const testing = std.testing;	6	const testing = std.testing;
7	const expect = testing.expect;	7	const expect = testing.expect;
8	const consts = @import("consts.zig");	8	const flate = @import("../flate.zig");
9		9
10	const Self = @This();	10	const Lookup = @This();
11		11
12	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761	12	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13	const chain_len = 2 * consts.history.len;	13	const chain_len = 2 * flate.history_len;
14		14
15	// Maps hash => first position	15	// Maps hash => first position
16	head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,	16	head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len,
17	// Maps position => previous positions for the same hash value	17	// Maps position => previous positions for the same hash value
18	chain: [chain_len]u16 = [_]u16{0} ** (chain_len),	18	chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
19		19
20	// Calculates hash of the 4 bytes from data.	20	// Calculates hash of the 4 bytes from data.
21	// Inserts `pos` position of that hash in the lookup tables.	21	// Inserts `pos` position of that hash in the lookup tables.
22	// Returns previous location with the same hash value.	22	// Returns previous location with the same hash value.
23	pub fn add(self: *Self, data: []const u8, pos: u16) u16 {	23	pub fn add(self: *Lookup, data: []const u8, pos: u16) u16 {
24	if (data.len < 4) return 0;	24	if (data.len < 4) return 0;
25	const h = hash(data[0..4]);	25	const h = hash(data[0..4]);
26	return self.set(h, pos);	26	return self.set(h, pos);
...	@@ -28,11 +28,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 {	...	@@ -28,11 +28,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
28		28
29	// Returns previous location with the same hash value given the current	29	// Returns previous location with the same hash value given the current
30	// position.	30	// position.
31	pub fn prev(self: *Self, pos: u16) u16 {	31	pub fn prev(self: *Lookup, pos: u16) u16 {
32	return self.chain[pos];	32	return self.chain[pos];
33	}	33	}
34		34
35	fn set(self: *Self, h: u32, pos: u16) u16 {	35	fn set(self: *Lookup, h: u32, pos: u16) u16 {
36	const p = self.head[h];	36	const p = self.head[h];
37	self.head[h] = pos;	37	self.head[h] = pos;
38	self.chain[pos] = p;	38	self.chain[pos] = p;
...	@@ -40,7 +40,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 {	...	@@ -40,7 +40,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 {
40	}	40	}
41		41
42	// Slide all positions in head and chain for `n`	42	// Slide all positions in head and chain for `n`
43	pub fn slide(self: *Self, n: u16) void {	43	pub fn slide(self: *Lookup, n: u16) void {
44	for (&self.head) \|*v\| {	44	for (&self.head) \|*v\| {
45	v.* -\|= n;	45	v.* -\|= n;
46	}	46	}
...	@@ -52,8 +52,8 @@ pub fn slide(self: *Self, n: u16) void {	...	@@ -52,8 +52,8 @@ pub fn slide(self: *Self, n: u16) void {
52		52
53	// Add `len` 4 bytes hashes from `data` into lookup.	53	// Add `len` 4 bytes hashes from `data` into lookup.
54	// Position of the first byte is `pos`.	54	// Position of the first byte is `pos`.
55	pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {	55	pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {
56	if (len == 0 or data.len < consts.match.min_length) {	56	if (len == 0 or data.len < flate.match.min_length) {
57	return;	57	return;
58	}	58	}
59	var hb =	59	var hb =
...	@@ -80,7 +80,7 @@ fn hash(b: *const [4]u8) u32 {	...	@@ -80,7 +80,7 @@ fn hash(b: *const [4]u8) u32 {
80	}	80	}
81		81
82	fn hashu(v: u32) u32 {	82	fn hashu(v: u32) u32 {
83	return @intCast((v *% prime4) >> consts.lookup.shift);	83	return @intCast((v *% prime4) >> flate.lookup.shift);
84	}	84	}
85		85
86	test add {	86	test add {
...	@@ -91,7 +91,7 @@ test add {	...	@@ -91,7 +91,7 @@ test add {
91	0x01, 0x02, 0x03,	91	0x01, 0x02, 0x03,
92	};	92	};
93		93
94	var h: Self = .{};	94	var h: Lookup = .{};
95	for (data, 0..) \|_, i\| {	95	for (data, 0..) \|_, i\| {
96	const p = h.add(data[i..], @intCast(i));	96	const p = h.add(data[i..], @intCast(i));
97	if (i >= 8 and i < 24) {	97	if (i >= 8 and i < 24) {
...	@@ -101,7 +101,7 @@ test add {	...	@@ -101,7 +101,7 @@ test add {
101	}	101	}
102	}	102	}
103		103
104	const v = Self.hash(data[2 .. 2 + 4]);	104	const v = Lookup.hash(data[2 .. 2 + 4]);
105	try expect(h.head[v] == 2 + 16);	105	try expect(h.head[v] == 2 + 16);
106	try expect(h.chain[2 + 16] == 2 + 8);	106	try expect(h.chain[2 + 16] == 2 + 8);
107	try expect(h.chain[2 + 8] == 2);	107	try expect(h.chain[2 + 8] == 2);
...	@@ -111,13 +111,13 @@ test bulkAdd {	...	@@ -111,13 +111,13 @@ test bulkAdd {
111	const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";	111	const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
112		112
113	// one by one	113	// one by one
114	var h: Self = .{};	114	var h: Lookup = .{};
115	for (data, 0..) \|_, i\| {	115	for (data, 0..) \|_, i\| {
116	_ = h.add(data[i..], @intCast(i));	116	_ = h.add(data[i..], @intCast(i));
117	}	117	}
118		118
119	// in bulk	119	// in bulk
120	var bh: Self = .{};	120	var bh: Lookup = .{};
121	bh.bulkAdd(data, data.len, 0);	121	bh.bulkAdd(data, data.len, 0);
122		122
123	try testing.expectEqualSlices(u16, &h.head, &bh.head);	123	try testing.expectEqualSlices(u16, &h.head, &bh.head);

lib/std/compress/flate/SlidingWindow.zig deleted-160

...	@@ -1,160 +0,0 @@
1	//! Used in deflate (compression), holds uncompressed data form which Tokens are
2	//! produces. In combination with Lookup it is used to find matches in history data.
3	//!
4	const std = @import("std");
5	const consts = @import("consts.zig");
6
7	const expect = testing.expect;
8	const assert = std.debug.assert;
9	const testing = std.testing;
10
11	const hist_len = consts.history.len;
12	const buffer_len = 2 * hist_len;
13	const min_lookahead = consts.match.min_length + consts.match.max_length;
14	const max_rp = buffer_len - min_lookahead;
15
16	const Self = @This();
17
18	buffer: [buffer_len]u8 = undefined,
19	wp: usize = 0, // write position
20	rp: usize = 0, // read position
21	fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23	/// Returns number of bytes written, or 0 if buffer is full and need to slide.
24	pub fn write(self: *Self, buf: []const u8) usize {
25	if (self.rp >= max_rp) return 0; // need to slide
26
27	const n = @min(buf.len, buffer_len - self.wp);
28	@memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29	self.wp += n;
30	return n;
31	}
32
33	/// Slide buffer for hist_len.
34	/// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35	/// Returns number of bytes removed.
36	pub fn slide(self: *Self) u16 {
37	assert(self.rp >= max_rp and self.wp >= self.rp);
38	const n = self.wp - hist_len;
39	@memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40	self.rp -= hist_len;
41	self.wp -= hist_len;
42	self.fp -= hist_len;
43	return @intCast(n);
44	}
45
46	/// Data from the current position (read position). Those part of the buffer is
47	/// not converted to tokens yet.
48	fn lookahead(self: *Self) []const u8 {
49	assert(self.wp >= self.rp);
50	return self.buffer[self.rp..self.wp];
51	}
52
53	/// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54	/// preserved otherwise preserves enough data for the longest match. Returns
55	/// null if there is not enough data.
56	pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57	const min: usize = if (should_flush) 0 else min_lookahead;
58	const lh = self.lookahead();
59	return if (lh.len > min) lh else null;
60	}
61
62	/// Advances read position, shrinks lookahead.
63	pub fn advance(self: *Self, n: u16) void {
64	assert(self.wp >= self.rp + n);
65	self.rp += n;
66	}
67
68	/// Returns writable part of the buffer, where new uncompressed data can be
69	/// written.
70	pub fn writable(self: *Self) []u8 {
71	return self.buffer[self.wp..];
72	}
73
74	/// Notification of what part of writable buffer is filled with data.
75	pub fn written(self: *Self, n: usize) void {
76	self.wp += n;
77	}
78
79	/// Finds match length between previous and current position.
80	/// Used in hot path!
81	pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82	const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83	// lookahead buffers from previous and current positions
84	const prev_lh = self.buffer[prev_pos..][0..max_len];
85	const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87	// If we already have match (min_len > 0),
88	// test the first byte above previous len a[min_len] != b[min_len]
89	// and then all the bytes from that position to zero.
90	// That is likely positions to find difference than looping from first bytes.
91	var i: usize = min_len;
92	if (i > 0) {
93	if (max_len <= i) return 0;
94	while (true) {
95	if (prev_lh[i] != curr_lh[i]) return 0;
96	if (i == 0) break;
97	i -= 1;
98	}
99	i = min_len;
100	}
101	while (i < max_len) : (i += 1)
102	if (prev_lh[i] != curr_lh[i]) break;
103	return if (i >= consts.match.min_length) @intCast(i) else 0;
104	}
105
106	/// Current position of non-compressed data. Data before rp are already converted
107	/// to tokens.
108	pub fn pos(self: *Self) u16 {
109	return @intCast(self.rp);
110	}
111
112	/// Notification that token list is cleared.
113	pub fn flush(self: *Self) void {
114	self.fp = @intCast(self.rp);
115	}
116
117	/// Part of the buffer since last flush or null if there was slide in between (so
118	/// fp becomes negative).
119	pub fn tokensBuffer(self: *Self) ?[]const u8 {
120	assert(self.fp <= self.rp);
121	if (self.fp < 0) return null;
122	return self.buffer[@intCast(self.fp)..self.rp];
123	}
124
125	test match {
126	const data = "Blah blah blah blah blah!";
127	var win: Self = .{};
128	try expect(win.write(data) == data.len);
129	try expect(win.wp == data.len);
130	try expect(win.rp == 0);
131
132	// length between l symbols
133	try expect(win.match(1, 6, 0) == 18);
134	try expect(win.match(1, 11, 0) == 13);
135	try expect(win.match(1, 16, 0) == 8);
136	try expect(win.match(1, 21, 0) == 0);
137
138	// position 15 = "blah blah!"
139	// position 20 = "blah!"
140	try expect(win.match(15, 20, 0) == 4);
141	try expect(win.match(15, 20, 3) == 4);
142	try expect(win.match(15, 20, 4) == 0);
143	}
144
145	test slide {
146	var win: Self = .{};
147	win.wp = Self.buffer_len - 11;
148	win.rp = Self.buffer_len - 111;
149	win.buffer[win.rp] = 0xab;
150	try expect(win.lookahead().len == 100);
151	try expect(win.tokensBuffer().?.len == win.rp);
152
153	const n = win.slide();
154	try expect(n == 32757);
155	try expect(win.buffer[win.rp] == 0xab);
156	try expect(win.rp == Self.hist_len - 111);
157	try expect(win.wp == Self.hist_len - 11);
158	try expect(win.lookahead().len == 100);
159	try expect(win.tokensBuffer() == null);
160	}

lib/std/compress/flate/Token.zig+7-7

...	@@ -6,7 +6,7 @@ const std = @import("std");	...	@@ -6,7 +6,7 @@ const std = @import("std");
6	const assert = std.debug.assert;	6	const assert = std.debug.assert;
7	const print = std.debug.print;	7	const print = std.debug.print;
8	const expect = std.testing.expect;	8	const expect = std.testing.expect;
9	const consts = @import("consts.zig").match;	9	const match = std.compress.flate.match;
10		10
11	const Token = @This();	11	const Token = @This();
12		12
...	@@ -26,11 +26,11 @@ pub fn literal(t: Token) u8 {	...	@@ -26,11 +26,11 @@ pub fn literal(t: Token) u8 {
26	}	26	}
27		27
28	pub fn distance(t: Token) u16 {	28	pub fn distance(t: Token) u16 {
29	return @as(u16, t.dist) + consts.min_distance;	29	return @as(u16, t.dist) + match.min_distance;
30	}	30	}
31		31
32	pub fn length(t: Token) u16 {	32	pub fn length(t: Token) u16 {
33	return @as(u16, t.len_lit) + consts.base_length;	33	return @as(u16, t.len_lit) + match.base_length;
34	}	34	}
35		35
36	pub fn initLiteral(lit: u8) Token {	36	pub fn initLiteral(lit: u8) Token {
...	@@ -40,12 +40,12 @@ pub fn initLiteral(lit: u8) Token {	...	@@ -40,12 +40,12 @@ pub fn initLiteral(lit: u8) Token {
40	// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)	40	// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
41	// length range 3 - 258, stored in len_lit as 0 - 255 (u8)	41	// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
42	pub fn initMatch(dist: u16, len: u16) Token {	42	pub fn initMatch(dist: u16, len: u16) Token {
43	assert(len >= consts.min_length and len <= consts.max_length);	43	assert(len >= match.min_length and len <= match.max_length);
44	assert(dist >= consts.min_distance and dist <= consts.max_distance);	44	assert(dist >= match.min_distance and dist <= match.max_distance);
45	return .{	45	return .{
46	.kind = .match,	46	.kind = .match,
47	.dist = @intCast(dist - consts.min_distance),	47	.dist = @intCast(dist - match.min_distance),
48	.len_lit = @intCast(len - consts.base_length),	48	.len_lit = @intCast(len - match.base_length),
49	};	49	};
50	}	50	}
51		51

lib/std/compress/flate/bit_reader.zig deleted-422

...	@@ -1,422 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) {
6	return BitReader(T, @TypeOf(reader)).init(reader);
7	}
8
9	pub fn BitReader64(comptime ReaderType: type) type {
10	return BitReader(u64, ReaderType);
11	}
12
13	pub fn BitReader32(comptime ReaderType: type) type {
14	return BitReader(u32, ReaderType);
15	}
16
17	/// Bit reader used during inflate (decompression). Has internal buffer of 64
18	/// bits which shifts right after bits are consumed. Uses forward_reader to fill
19	/// that internal buffer when needed.
20	///
21	/// readF is the core function. Supports few different ways of getting bits
22	/// controlled by flags. In hot path we try to avoid checking whether we need to
23	/// fill buffer from forward_reader by calling fill in advance and readF with
24	/// buffered flag set.
25	///
26	pub fn BitReader(comptime T: type, comptime ReaderType: type) type {
27	assert(T == u32 or T == u64);
28	const t_bytes: usize = @sizeOf(T);
29	const Tshift = if (T == u64) u6 else u5;
30
31	return struct {
32	// Underlying reader used for filling internal bits buffer
33	forward_reader: ReaderType = undefined,
34	// Internal buffer of 64 bits
35	bits: T = 0,
36	// Number of bits in the buffer
37	nbits: u32 = 0,
38
39	const Self = @This();
40
41	pub const Error = ReaderType.Error \|\| error{EndOfStream};
42
43	pub fn init(rdr: ReaderType) Self {
44	var self = Self{ .forward_reader = rdr };
45	self.fill(1) catch {};
46	return self;
47	}
48
49	/// Try to have `nice` bits are available in buffer. Reads from
50	/// forward reader if there is no `nice` bits in buffer. Returns error
51	/// if end of forward stream is reached and internal buffer is empty.
52	/// It will not error if less than `nice` bits are in buffer, only when
53	/// all bits are exhausted. During inflate we usually know what is the
54	/// maximum bits for the next step but usually that step will need less
55	/// bits to decode. So `nice` is not hard limit, it will just try to have
56	/// that number of bits available. If end of forward stream is reached
57	/// it may be some extra zero bits in buffer.
58	pub inline fn fill(self: *Self, nice: u6) !void {
59	if (self.nbits >= nice and nice != 0) {
60	return; // We have enough bits
61	}
62	// Read more bits from forward reader
63
64	// Number of empty bytes in bits, round nbits to whole bytes.
65	const empty_bytes =
66	@as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
67	(self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
68
69	var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
70	const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
71	if (bytes_read > 0) {
72	const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
73	self.bits \|= u << @as(Tshift, @intCast(self.nbits));
74	self.nbits += 8 * @as(u8, @intCast(bytes_read));
75	return;
76	}
77
78	if (self.nbits == 0)
79	return error.EndOfStream;
80	}
81
82	/// Read exactly buf.len bytes into buf.
83	pub fn readAll(self: *Self, buf: []u8) !void {
84	assert(self.alignBits() == 0); // internal bits must be at byte boundary
85
86	// First read from internal bits buffer.
87	var n: usize = 0;
88	while (self.nbits > 0 and n < buf.len) {
89	buf[n] = try self.readF(u8, flag.buffered);
90	n += 1;
91	}
92	// Then use forward reader for all other bytes.
93	try self.forward_reader.readNoEof(buf[n..]);
94	}
95
96	pub const flag = struct {
97	pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer
98	pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before
99	pub const reverse: u3 = 0b100; // bit reverse read bits
100	};
101
102	/// Alias for readF(U, 0).
103	pub fn read(self: *Self, comptime U: type) !U {
104	return self.readF(U, 0);
105	}
106
107	/// Alias for readF with flag.peak set.
108	pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U {
109	return self.readF(U, how \| flag.peek);
110	}
111
112	/// Read with flags provided.
113	pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
114	if (U == T) {
115	assert(how == 0);
116	assert(self.alignBits() == 0);
117	try self.fill(@bitSizeOf(T));
118	if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
119	const v = self.bits;
120	self.nbits = 0;
121	self.bits = 0;
122	return v;
123	}
124	const n: Tshift = @bitSizeOf(U);
125	switch (how) {
126	0 => { // `normal` read
127	try self.fill(n); // ensure that there are n bits in the buffer
128	const u: U = @truncate(self.bits); // get n bits
129	try self.shift(n); // advance buffer for n
130	return u;
131	},
132	(flag.peek) => { // no shift, leave bits in the buffer
133	try self.fill(n);
134	return @truncate(self.bits);
135	},
136	flag.buffered => { // no fill, assume that buffer has enough bits
137	const u: U = @truncate(self.bits);
138	try self.shift(n);
139	return u;
140	},
141	(flag.reverse) => { // same as 0 with bit reverse
142	try self.fill(n);
143	const u: U = @truncate(self.bits);
144	try self.shift(n);
145	return @bitReverse(u);
146	},
147	(flag.peek \| flag.reverse) => {
148	try self.fill(n);
149	return @bitReverse(@as(U, @truncate(self.bits)));
150	},
151	(flag.buffered \| flag.reverse) => {
152	const u: U = @truncate(self.bits);
153	try self.shift(n);
154	return @bitReverse(u);
155	},
156	(flag.peek \| flag.buffered) => {
157	return @truncate(self.bits);
158	},
159	(flag.peek \| flag.buffered \| flag.reverse) => {
160	return @bitReverse(@as(U, @truncate(self.bits)));
161	},
162	}
163	}
164
165	/// Read n number of bits.
166	/// Only buffered flag can be used in how.
167	pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
168	switch (how) {
169	0 => {
170	try self.fill(n);
171	},
172	flag.buffered => {},
173	else => unreachable,
174	}
175	const mask: u16 = (@as(u16, 1) << n) - 1;
176	const u: u16 = @as(u16, @truncate(self.bits)) & mask;
177	try self.shift(n);
178	return u;
179	}
180
181	/// Advance buffer for n bits.
182	pub fn shift(self: *Self, n: Tshift) !void {
183	if (n > self.nbits) return error.EndOfStream;
184	self.bits >>= n;
185	self.nbits -= n;
186	}
187
188	/// Skip n bytes.
189	pub fn skipBytes(self: *Self, n: u16) !void {
190	for (0..n) \|_\| {
191	try self.fill(8);
192	try self.shift(8);
193	}
194	}
195
196	// Number of bits to align stream to the byte boundary.
197	fn alignBits(self: *Self) u3 {
198	return @intCast(self.nbits & 0x7);
199	}
200
201	/// Align stream to the byte boundary.
202	pub fn alignToByte(self: *Self) void {
203	const ab = self.alignBits();
204	if (ab > 0) self.shift(ab) catch unreachable;
205	}
206
207	/// Skip zero terminated string.
208	pub fn skipStringZ(self: *Self) !void {
209	while (true) {
210	if (try self.readF(u8, 0) == 0) break;
211	}
212	}
213
214	/// Read deflate fixed fixed code.
215	/// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
216	/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
217	/// Lit Value Bits Codes
218	/// --------- ---- -----
219	/// 0 - 143 8 00110000 through
220	/// 10111111
221	/// 144 - 255 9 110010000 through
222	/// 111111111
223	/// 256 - 279 7 0000000 through
224	/// 0010111
225	/// 280 - 287 8 11000000 through
226	/// 11000111
227	pub fn readFixedCode(self: *Self) !u16 {
228	try self.fill(7 + 2);
229	const code7 = try self.readF(u7, flag.buffered \| flag.reverse);
230	if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
231	return @as(u16, code7) + 256;
232	} else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
233	return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000;
234	} else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
235	return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280;
236	} else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
237	return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered \| flag.reverse)) + 144;
238	}
239	}
240	};
241	}
242
243	test "readF" {
244	var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
245	var br = bitReader(u64, fbs.reader());
246	const F = BitReader64(@TypeOf(fbs.reader())).flag;
247
248	try testing.expectEqual(@as(u8, 48), br.nbits);
249	try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
250
251	try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
252	try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
253	try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
254	try testing.expectEqual(@as(u3, 5), br.alignBits());
255
256	try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110);
257	try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110);
258	try br.shift(9);
259	try testing.expectEqual(@as(u8, 36), br.nbits);
260	try testing.expectEqual(@as(u3, 4), br.alignBits());
261
262	try testing.expect(try br.readF(u4, 0) == 0b0100);
263	try testing.expectEqual(@as(u8, 32), br.nbits);
264	try testing.expectEqual(@as(u3, 0), br.alignBits());
265
266	try br.shift(1);
267	try testing.expectEqual(@as(u3, 7), br.alignBits());
268	try br.shift(1);
269	try testing.expectEqual(@as(u3, 6), br.alignBits());
270	br.alignToByte();
271	try testing.expectEqual(@as(u3, 0), br.alignBits());
272
273	try testing.expectEqual(@as(u64, 0xc9), br.bits);
274	try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
275	try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
276	}
277
278	test "read block type 1 data" {
279	inline for ([_]type{ u64, u32 }) \|T\| {
280	const data = [_]u8{
281	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
282	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
283	0x0c, 0x01, 0x02, 0x03, //
284	0xaa, 0xbb, 0xcc, 0xdd,
285	};
286	var fbs = std.io.fixedBufferStream(&data);
287	var br = bitReader(T, fbs.reader());
288	const F = BitReader(T, @TypeOf(fbs.reader())).flag;
289
290	try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
291	try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
292
293	for ("Hello world\n") \|c\| {
294	try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
295	}
296	try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
297	br.alignToByte();
298	try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
299	try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
300	try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
301	}
302	}
303
304	test "shift/fill" {
305	const data = [_]u8{
306	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
307	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308	};
309	var fbs = std.io.fixedBufferStream(&data);
310	var br = bitReader(u64, fbs.reader());
311
312	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
313	try br.shift(8);
314	try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
315	try br.fill(60); // fill with 1 byte
316	try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
317	try br.shift(8 * 4 + 4);
318	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
319
320	try br.fill(60); // fill with 4 bytes (shift by 4)
321	try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
322	try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
323
324	try br.shift(@intCast(br.nbits)); // clear buffer
325	try br.fill(8); // refill with the rest of the bytes
326	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
327	}
328
329	test "readAll" {
330	inline for ([_]type{ u64, u32 }) \|T\| {
331	const data = [_]u8{
332	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
333	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
334	};
335	var fbs = std.io.fixedBufferStream(&data);
336	var br = bitReader(T, fbs.reader());
337
338	switch (T) {
339	u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
340	u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
341	else => unreachable,
342	}
343
344	var out: [16]u8 = undefined;
345	try br.readAll(out[0..]);
346	try testing.expect(br.nbits == 0);
347	try testing.expect(br.bits == 0);
348
349	try testing.expectEqualSlices(u8, data[0..16], &out);
350	}
351	}
352
353	test "readFixedCode" {
354	inline for ([_]type{ u64, u32 }) \|T\| {
355	const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
356
357	var fbs = std.io.fixedBufferStream(&fixed_codes);
358	var rdr = bitReader(T, fbs.reader());
359
360	for (0..286) \|c\| {
361	try testing.expectEqual(c, try rdr.readFixedCode());
362	}
363	try testing.expect(rdr.nbits == 0);
364	}
365	}
366
367	test "u32 leaves no bits on u32 reads" {
368	const data = [_]u8{
369	0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
370	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
371	};
372	var fbs = std.io.fixedBufferStream(&data);
373	var br = bitReader(u32, fbs.reader());
374
375	_ = try br.read(u3);
376	try testing.expectEqual(29, br.nbits);
377	br.alignToByte();
378	try testing.expectEqual(24, br.nbits);
379	try testing.expectEqual(0x04_03_02_01, try br.read(u32));
380	try testing.expectEqual(0, br.nbits);
381	try testing.expectEqual(0x08_07_06_05, try br.read(u32));
382	try testing.expectEqual(0, br.nbits);
383
384	_ = try br.read(u9);
385	try testing.expectEqual(23, br.nbits);
386	br.alignToByte();
387	try testing.expectEqual(16, br.nbits);
388	try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
389	try testing.expectEqual(0, br.nbits);
390	}
391
392	test "u64 need fill after alignToByte" {
393	const data = [_]u8{
394	0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
395	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
396	};
397
398	// without fill
399	var fbs = std.io.fixedBufferStream(&data);
400	var br = bitReader(u64, fbs.reader());
401	_ = try br.read(u23);
402	try testing.expectEqual(41, br.nbits);
403	br.alignToByte();
404	try testing.expectEqual(40, br.nbits);
405	try testing.expectEqual(0x06_05_04_03, try br.read(u32));
406	try testing.expectEqual(8, br.nbits);
407	try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
408	try testing.expectEqual(32, br.nbits);
409
410	// fill after align ensures all bits filled
411	fbs.reset();
412	br = bitReader(u64, fbs.reader());
413	_ = try br.read(u23);
414	try testing.expectEqual(41, br.nbits);
415	br.alignToByte();
416	try br.fill(0);
417	try testing.expectEqual(64, br.nbits);
418	try testing.expectEqual(0x06_05_04_03, try br.read(u32));
419	try testing.expectEqual(32, br.nbits);
420	try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
421	try testing.expectEqual(0, br.nbits);
422	}

lib/std/compress/flate/bit_writer.zig deleted-99

...	@@ -1,99 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	/// Bit writer for use in deflate (compression).
5	///
6	/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7	/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8	/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9	///
10	pub fn BitWriter(comptime WriterType: type) type {
11	// buffer_flush_size indicates the buffer size
12	// after which bytes are flushed to the writer.
13	// Should preferably be a multiple of 6, since
14	// we accumulate 6 bytes between writes to the buffer.
15	const buffer_flush_size = 240;
16
17	// buffer_size is the actual output byte buffer size.
18	// It must have additional headroom for a flush
19	// which can contain up to 8 bytes.
20	const buffer_size = buffer_flush_size + 8;
21
22	return struct {
23	inner_writer: WriterType,
24
25	// Data waiting to be written is bytes[0 .. nbytes]
26	// and then the low nbits of bits. Data is always written
27	// sequentially into the bytes array.
28	bits: u64 = 0,
29	nbits: u32 = 0, // number of bits
30	bytes: [buffer_size]u8 = undefined,
31	nbytes: u32 = 0, // number of bytes
32
33	const Self = @This();
34
35	pub const Error = WriterType.Error \|\| error{UnfinishedBits};
36
37	pub fn init(writer: WriterType) Self {
38	return .{ .inner_writer = writer };
39	}
40
41	pub fn setWriter(self: *Self, new_writer: WriterType) void {
42	//assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43	self.inner_writer = new_writer;
44	}
45
46	pub fn flush(self: *Self) Error!void {
47	var n = self.nbytes;
48	while (self.nbits != 0) {
49	self.bytes[n] = @as(u8, @truncate(self.bits));
50	self.bits >>= 8;
51	if (self.nbits > 8) { // Avoid underflow
52	self.nbits -= 8;
53	} else {
54	self.nbits = 0;
55	}
56	n += 1;
57	}
58	self.bits = 0;
59	_ = try self.inner_writer.write(self.bytes[0..n]);
60	self.nbytes = 0;
61	}
62
63	pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64	self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65	self.nbits += nb;
66	if (self.nbits < 48)
67	return;
68
69	var n = self.nbytes;
70	std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71	n += 6;
72	if (n >= buffer_flush_size) {
73	_ = try self.inner_writer.write(self.bytes[0..n]);
74	n = 0;
75	}
76	self.nbytes = n;
77	self.bits >>= 48;
78	self.nbits -= 48;
79	}
80
81	pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82	var n = self.nbytes;
83	if (self.nbits & 7 != 0) {
84	return error.UnfinishedBits;
85	}
86	while (self.nbits != 0) {
87	self.bytes[n] = @as(u8, @truncate(self.bits));
88	self.bits >>= 8;
89	self.nbits -= 8;
90	n += 1;
91	}
92	if (n != 0) {
93	_ = try self.inner_writer.write(self.bytes[0..n]);
94	}
95	self.nbytes = 0;
96	_ = try self.inner_writer.write(bytes);
97	}
98	};
99	}

lib/std/compress/flate/block_writer.zig deleted-706

...	@@ -1,706 +0,0 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4
5	const hc = @import("huffman_encoder.zig");
6	const consts = @import("consts.zig").huffman;
7	const Token = @import("Token.zig");
8	const BitWriter = @import("bit_writer.zig").BitWriter;
9
10	pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11	return BlockWriter(@TypeOf(writer)).init(writer);
12	}
13
14	/// Accepts list of tokens, decides what is best block type to write. What block
15	/// type will provide best compression. Writes header and body of the block.
16	///
17	pub fn BlockWriter(comptime WriterType: type) type {
18	const BitWriterType = BitWriter(WriterType);
19	return struct {
20	const codegen_order = consts.codegen_order;
21	const end_code_mark = 255;
22	const Self = @This();
23
24	pub const Error = BitWriterType.Error;
25	bit_writer: BitWriterType,
26
27	codegen_freq: [consts.codegen_code_count]u16 = undefined,
28	literal_freq: [consts.max_num_lit]u16 = undefined,
29	distance_freq: [consts.distance_code_count]u16 = undefined,
30	codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31	literal_encoding: hc.LiteralEncoder = .{},
32	distance_encoding: hc.DistanceEncoder = .{},
33	codegen_encoding: hc.CodegenEncoder = .{},
34	fixed_literal_encoding: hc.LiteralEncoder,
35	fixed_distance_encoding: hc.DistanceEncoder,
36	huff_distance: hc.DistanceEncoder,
37
38	pub fn init(writer: WriterType) Self {
39	return .{
40	.bit_writer = BitWriterType.init(writer),
41	.fixed_literal_encoding = hc.fixedLiteralEncoder(),
42	.fixed_distance_encoding = hc.fixedDistanceEncoder(),
43	.huff_distance = hc.huffmanDistanceEncoder(),
44	};
45	}
46
47	/// Flush intrenal bit buffer to the writer.
48	/// Should be called only when bit stream is at byte boundary.
49	///
50	/// That is after final block; when last byte could be incomplete or
51	/// after stored block; which is aligned to the byte boundary (it has x
52	/// padding bits after first 3 bits).
53	pub fn flush(self: *Self) Error!void {
54	try self.bit_writer.flush();
55	}
56
57	pub fn setWriter(self: *Self, new_writer: WriterType) void {
58	self.bit_writer.setWriter(new_writer);
59	}
60
61	fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62	try self.bit_writer.writeBits(c.code, c.len);
63	}
64
65	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66	// the literal and distance lengths arrays (which are concatenated into a single
67	// array). This method generates that run-length encoding.
68	//
69	// The result is written into the codegen array, and the frequencies
70	// of each code is written into the codegen_freq array.
71	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72	// information. Code bad_code is an end marker
73	//
74	// num_literals: The number of literals in literal_encoding
75	// num_distances: The number of distances in distance_encoding
76	// lit_enc: The literal encoder to use
77	// dist_enc: The distance encoder to use
78	fn generateCodegen(
79	self: *Self,
80	num_literals: u32,
81	num_distances: u32,
82	lit_enc: *hc.LiteralEncoder,
83	dist_enc: *hc.DistanceEncoder,
84	) void {
85	for (self.codegen_freq, 0..) \|_, i\| {
86	self.codegen_freq[i] = 0;
87	}
88
89	// Note that we are using codegen both as a temporary variable for holding
90	// a copy of the frequencies, and as the place where we put the result.
91	// This is fine because the output is always shorter than the input used
92	// so far.
93	var codegen = &self.codegen; // cache
94	// Copy the concatenated code sizes to codegen. Put a marker at the end.
95	var cgnl = codegen[0..num_literals];
96	for (cgnl, 0..) \|_, i\| {
97	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98	}
99
100	cgnl = codegen[num_literals .. num_literals + num_distances];
101	for (cgnl, 0..) \|_, i\| {
102	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103	}
104	codegen[num_literals + num_distances] = end_code_mark;
105
106	var size = codegen[0];
107	var count: i32 = 1;
108	var out_index: u32 = 0;
109	var in_index: u32 = 1;
110	while (size != end_code_mark) : (in_index += 1) {
111	// INVARIANT: We have seen "count" copies of size that have not yet
112	// had output generated for them.
113	const next_size = codegen[in_index];
114	if (next_size == size) {
115	count += 1;
116	continue;
117	}
118	// We need to generate codegen indicating "count" of size.
119	if (size != 0) {
120	codegen[out_index] = size;
121	out_index += 1;
122	self.codegen_freq[size] += 1;
123	count -= 1;
124	while (count >= 3) {
125	var n: i32 = 6;
126	if (n > count) {
127	n = count;
128	}
129	codegen[out_index] = 16;
130	out_index += 1;
131	codegen[out_index] = @as(u8, @intCast(n - 3));
132	out_index += 1;
133	self.codegen_freq[16] += 1;
134	count -= n;
135	}
136	} else {
137	while (count >= 11) {
138	var n: i32 = 138;
139	if (n > count) {
140	n = count;
141	}
142	codegen[out_index] = 18;
143	out_index += 1;
144	codegen[out_index] = @as(u8, @intCast(n - 11));
145	out_index += 1;
146	self.codegen_freq[18] += 1;
147	count -= n;
148	}
149	if (count >= 3) {
150	// 3 <= count <= 10
151	codegen[out_index] = 17;
152	out_index += 1;
153	codegen[out_index] = @as(u8, @intCast(count - 3));
154	out_index += 1;
155	self.codegen_freq[17] += 1;
156	count = 0;
157	}
158	}
159	count -= 1;
160	while (count >= 0) : (count -= 1) {
161	codegen[out_index] = size;
162	out_index += 1;
163	self.codegen_freq[size] += 1;
164	}
165	// Set up invariant for next time through the loop.
166	size = next_size;
167	count = 1;
168	}
169	// Marker indicating the end of the codegen.
170	codegen[out_index] = end_code_mark;
171	}
172
173	const DynamicSize = struct {
174	size: u32,
175	num_codegens: u32,
176	};
177
178	// dynamicSize returns the size of dynamically encoded data in bits.
179	fn dynamicSize(
180	self: *Self,
181	lit_enc: *hc.LiteralEncoder, // literal encoder
182	dist_enc: *hc.DistanceEncoder, // distance encoder
183	extra_bits: u32,
184	) DynamicSize {
185	var num_codegens = self.codegen_freq.len;
186	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187	num_codegens -= 1;
188	}
189	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191	self.codegen_freq[16] * 2 +
192	self.codegen_freq[17] * 3 +
193	self.codegen_freq[18] * 7;
194	const size = header +
195	lit_enc.bitLength(&self.literal_freq) +
196	dist_enc.bitLength(&self.distance_freq) +
197	extra_bits;
198
199	return DynamicSize{
200	.size = @as(u32, @intCast(size)),
201	.num_codegens = @as(u32, @intCast(num_codegens)),
202	};
203	}
204
205	// fixedSize returns the size of dynamically encoded data in bits.
206	fn fixedSize(self: *Self, extra_bits: u32) u32 {
207	return 3 +
208	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210	extra_bits;
211	}
212
213	const StoredSize = struct {
214	size: u32,
215	storable: bool,
216	};
217
218	// storedSizeFits calculates the stored size, including header.
219	// The function returns the size in bits and whether the block
220	// fits inside a single block.
221	fn storedSizeFits(in: ?[]const u8) StoredSize {
222	if (in == null) {
223	return .{ .size = 0, .storable = false };
224	}
225	if (in.?.len <= consts.max_store_block_size) {
226	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227	}
228	return .{ .size = 0, .storable = false };
229	}
230
231	// Write the header of a dynamic Huffman block to the output stream.
232	//
233	// num_literals: The number of literals specified in codegen
234	// num_distances: The number of distances specified in codegen
235	// num_codegens: The number of codegens used in codegen
236	// eof: Is it the end-of-file? (end of stream)
237	fn dynamicHeader(
238	self: *Self,
239	num_literals: u32,
240	num_distances: u32,
241	num_codegens: u32,
242	eof: bool,
243	) Error!void {
244	const first_bits: u32 = if (eof) 5 else 4;
245	try self.bit_writer.writeBits(first_bits, 3);
246	try self.bit_writer.writeBits(num_literals - 257, 5);
247	try self.bit_writer.writeBits(num_distances - 1, 5);
248	try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250	var i: u32 = 0;
251	while (i < num_codegens) : (i += 1) {
252	const value = self.codegen_encoding.codes[codegen_order[i]].len;
253	try self.bit_writer.writeBits(value, 3);
254	}
255
256	i = 0;
257	while (true) {
258	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259	i += 1;
260	if (code_word == end_code_mark) {
261	break;
262	}
263	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265	switch (code_word) {
266	16 => {
267	try self.bit_writer.writeBits(self.codegen[i], 2);
268	i += 1;
269	},
270	17 => {
271	try self.bit_writer.writeBits(self.codegen[i], 3);
272	i += 1;
273	},
274	18 => {
275	try self.bit_writer.writeBits(self.codegen[i], 7);
276	i += 1;
277	},
278	else => {},
279	}
280	}
281	}
282
283	fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284	assert(length <= 65535);
285	const flag: u32 = if (eof) 1 else 0;
286	try self.bit_writer.writeBits(flag, 3);
287	try self.flush();
288	const l: u16 = @intCast(length);
289	try self.bit_writer.writeBits(l, 16);
290	try self.bit_writer.writeBits(~l, 16);
291	}
292
293	fn fixedHeader(self: *Self, eof: bool) Error!void {
294	// Indicate that we are a fixed Huffman block
295	var value: u32 = 2;
296	if (eof) {
297	value = 3;
298	}
299	try self.bit_writer.writeBits(value, 3);
300	}
301
302	// Write a block of tokens with the smallest encoding. Will choose block type.
303	// The original input can be supplied, and if the huffman encoded data
304	// is larger than the original bytes, the data will be written as a
305	// stored block.
306	// If the input is null, the tokens will always be Huffman encoded.
307	pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308	const lit_and_dist = self.indexTokens(tokens);
309	const num_literals = lit_and_dist.num_literals;
310	const num_distances = lit_and_dist.num_distances;
311
312	var extra_bits: u32 = 0;
313	const ret = storedSizeFits(input);
314	const stored_size = ret.size;
315	const storable = ret.storable;
316
317	if (storable) {
318	// We only bother calculating the costs of the extra bits required by
319	// the length of distance fields (which will be the same for both fixed
320	// and dynamic encoding), if we need to compare those two encodings
321	// against stored encoding.
322	var length_code: u16 = Token.length_codes_start + 8;
323	while (length_code < num_literals) : (length_code += 1) {
324	// First eight length codes have extra size = 0.
325	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
327	}
328	var distance_code: u16 = 4;
329	while (distance_code < num_distances) : (distance_code += 1) {
330	// First four distance codes have extra size = 0.
331	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333	}
334	}
335
336	// Figure out smallest code.
337	// Fixed Huffman baseline.
338	var literal_encoding = &self.fixed_literal_encoding;
339	var distance_encoding = &self.fixed_distance_encoding;
340	var size = self.fixedSize(extra_bits);
341
342	// Dynamic Huffman?
343	var num_codegens: u32 = 0;
344
345	// Generate codegen and codegenFrequencies, which indicates how to encode
346	// the literal_encoding and the distance_encoding.
347	self.generateCodegen(
348	num_literals,
349	num_distances,
350	&self.literal_encoding,
351	&self.distance_encoding,
352	);
353	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354	const dynamic_size = self.dynamicSize(
355	&self.literal_encoding,
356	&self.distance_encoding,
357	extra_bits,
358	);
359	const dyn_size = dynamic_size.size;
360	num_codegens = dynamic_size.num_codegens;
361
362	if (dyn_size < size) {
363	size = dyn_size;
364	literal_encoding = &self.literal_encoding;
365	distance_encoding = &self.distance_encoding;
366	}
367
368	// Stored bytes?
369	if (storable and stored_size < size) {
370	try self.storedBlock(input.?, eof);
371	return;
372	}
373
374	// Huffman.
375	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376	try self.fixedHeader(eof);
377	} else {
378	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379	}
380
381	// Write the tokens.
382	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383	}
384
385	pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386	try self.storedHeader(input.len, eof);
387	try self.bit_writer.writeBytes(input);
388	}
389
390	// writeBlockDynamic encodes a block using a dynamic Huffman table.
391	// This should be used if the symbols used have a disproportionate
392	// histogram distribution.
393	// If input is supplied and the compression savings are below 1/16th of the
394	// input size the block is stored.
395	fn dynamicBlock(
396	self: *Self,
397	tokens: []const Token,
398	eof: bool,
399	input: ?[]const u8,
400	) Error!void {
401	const total_tokens = self.indexTokens(tokens);
402	const num_literals = total_tokens.num_literals;
403	const num_distances = total_tokens.num_distances;
404
405	// Generate codegen and codegenFrequencies, which indicates how to encode
406	// the literal_encoding and the distance_encoding.
407	self.generateCodegen(
408	num_literals,
409	num_distances,
410	&self.literal_encoding,
411	&self.distance_encoding,
412	);
413	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415	const size = dynamic_size.size;
416	const num_codegens = dynamic_size.num_codegens;
417
418	// Store bytes, if we don't get a reasonable improvement.
419
420	const stored_size = storedSizeFits(input);
421	const ssize = stored_size.size;
422	const storable = stored_size.storable;
423	if (storable and ssize < (size + (size >> 4))) {
424	try self.storedBlock(input.?, eof);
425	return;
426	}
427
428	// Write Huffman table.
429	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431	// Write the tokens.
432	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433	}
434
435	const TotalIndexedTokens = struct {
436	num_literals: u32,
437	num_distances: u32,
438	};
439
440	// Indexes a slice of tokens followed by an end_block_marker, and updates
441	// literal_freq and distance_freq, and generates literal_encoding
442	// and distance_encoding.
443	// The number of literal and distance tokens is returned.
444	fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445	var num_literals: u32 = 0;
446	var num_distances: u32 = 0;
447
448	for (self.literal_freq, 0..) \|_, i\| {
449	self.literal_freq[i] = 0;
450	}
451	for (self.distance_freq, 0..) \|_, i\| {
452	self.distance_freq[i] = 0;
453	}
454
455	for (tokens) \|t\| {
456	if (t.kind == Token.Kind.literal) {
457	self.literal_freq[t.literal()] += 1;
458	continue;
459	}
460	self.literal_freq[t.lengthCode()] += 1;
461	self.distance_freq[t.distanceCode()] += 1;
462	}
463	// add end_block_marker token at the end
464	self.literal_freq[consts.end_block_marker] += 1;
465
466	// get the number of literals
467	num_literals = @as(u32, @intCast(self.literal_freq.len));
468	while (self.literal_freq[num_literals - 1] == 0) {
469	num_literals -= 1;
470	}
471	// get the number of distances
472	num_distances = @as(u32, @intCast(self.distance_freq.len));
473	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474	num_distances -= 1;
475	}
476	if (num_distances == 0) {
477	// We haven't found a single match. If we want to go with the dynamic encoding,
478	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
479	self.distance_freq[0] = 1;
480	num_distances = 1;
481	}
482	self.literal_encoding.generate(&self.literal_freq, 15);
483	self.distance_encoding.generate(&self.distance_freq, 15);
484	return TotalIndexedTokens{
485	.num_literals = num_literals,
486	.num_distances = num_distances,
487	};
488	}
489
490	// Writes a slice of tokens to the output followed by and end_block_marker.
491	// codes for literal and distance encoding must be supplied.
492	fn writeTokens(
493	self: *Self,
494	tokens: []const Token,
495	le_codes: []hc.HuffCode,
496	oe_codes: []hc.HuffCode,
497	) Error!void {
498	for (tokens) \|t\| {
499	if (t.kind == Token.Kind.literal) {
500	try self.writeCode(le_codes[t.literal()]);
501	continue;
502	}
503
504	// Write the length
505	const le = t.lengthEncoding();
506	try self.writeCode(le_codes[le.code]);
507	if (le.extra_bits > 0) {
508	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509	}
510
511	// Write the distance
512	const oe = t.distanceEncoding();
513	try self.writeCode(oe_codes[oe.code]);
514	if (oe.extra_bits > 0) {
515	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516	}
517	}
518	// add end_block_marker at the end
519	try self.writeCode(le_codes[consts.end_block_marker]);
520	}
521
522	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523	// if the results only gains very little from compression.
524	pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525	// Add everything as literals
526	histogram(input, &self.literal_freq);
527
528	self.literal_freq[consts.end_block_marker] = 1;
529
530	const num_literals = consts.end_block_marker + 1;
531	self.distance_freq[0] = 1;
532	const num_distances = 1;
533
534	self.literal_encoding.generate(&self.literal_freq, 15);
535
536	// Figure out smallest code.
537	// Always use dynamic Huffman or Store
538	var num_codegens: u32 = 0;
539
540	// Generate codegen and codegenFrequencies, which indicates how to encode
541	// the literal_encoding and the distance_encoding.
542	self.generateCodegen(
543	num_literals,
544	num_distances,
545	&self.literal_encoding,
546	&self.huff_distance,
547	);
548	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550	const size = dynamic_size.size;
551	num_codegens = dynamic_size.num_codegens;
552
553	// Store bytes, if we don't get a reasonable improvement.
554	const stored_size_ret = storedSizeFits(input);
555	const ssize = stored_size_ret.size;
556	const storable = stored_size_ret.storable;
557
558	if (storable and ssize < (size + (size >> 4))) {
559	try self.storedBlock(input, eof);
560	return;
561	}
562
563	// Huffman.
564	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565	const encoding = self.literal_encoding.codes[0..257];
566
567	for (input) \|t\| {
568	const c = encoding[t];
569	try self.bit_writer.writeBits(c.code, c.len);
570	}
571	try self.writeCode(encoding[consts.end_block_marker]);
572	}
573
574	// histogram accumulates a histogram of b in h.
575	fn histogram(b: []const u8, h: *[286]u16) void {
576	// Clear histogram
577	for (h, 0..) \|_, i\| {
578	h[i] = 0;
579	}
580
581	var lh = h.*[0..256];
582	for (b) \|t\| {
583	lh[t] += 1;
584	}
585	}
586	};
587	}
588
589	// tests
590	const expect = std.testing.expect;
591	const fmt = std.fmt;
592	const testing = std.testing;
593	const ArrayList = std.ArrayList;
594
595	const TestCase = @import("testdata/block_writer.zig").TestCase;
596	const testCases = @import("testdata/block_writer.zig").testCases;
597
598	// tests if the writeBlock encoding has changed.
599	test "write" {
600	inline for (0..testCases.len) \|i\| {
601	try testBlock(testCases[i], .write_block);
602	}
603	}
604
605	// tests if the writeBlockDynamic encoding has changed.
606	test "dynamicBlock" {
607	inline for (0..testCases.len) \|i\| {
608	try testBlock(testCases[i], .write_dyn_block);
609	}
610	}
611
612	test "huffmanBlock" {
613	inline for (0..testCases.len) \|i\| {
614	try testBlock(testCases[i], .write_huffman_block);
615	}
616	try testBlock(.{
617	.tokens = &[_]Token{},
618	.input = "huffman-rand-max.input",
619	.want = "huffman-rand-max.{s}.expect",
620	}, .write_huffman_block);
621	}
622
623	const TestFn = enum {
624	write_block,
625	write_dyn_block, // write dynamic block
626	write_huffman_block,
627
628	fn to_s(self: TestFn) []const u8 {
629	return switch (self) {
630	.write_block => "wb",
631	.write_dyn_block => "dyn",
632	.write_huffman_block => "huff",
633	};
634	}
635
636	fn write(
637	comptime self: TestFn,
638	bw: anytype,
639	tok: []const Token,
640	input: ?[]const u8,
641	final: bool,
642	) !void {
643	switch (self) {
644	.write_block => try bw.write(tok, final, input),
645	.write_dyn_block => try bw.dynamicBlock(tok, final, input),
646	.write_huffman_block => try bw.huffmanBlock(input.?, final),
647	}
648	try bw.flush();
649	}
650	};
651
652	// testBlock tests a block against its references
653	//
654	// size
655	// 64K [file-name].input - input non compressed file
656	// 8.1K [file-name].golden -
657	// 78 [file-name].dyn.expect - output with writeBlockDynamic
658	// 78 [file-name].wb.expect - output with writeBlock
659	// 8.1K [file-name].huff.expect - output with writeBlockHuff
660	// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661	// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662	//
663	// wb - writeBlock
664	// dyn - writeBlockDynamic
665	// huff - writeBlockHuff
666	//
667	fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668	if (tc.input.len != 0 and tc.want.len != 0) {
669	const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670	const input = @embedFile("testdata/block_writer/" ++ tc.input);
671	const want = @embedFile("testdata/block_writer/" ++ want_name);
672	try testWriteBlock(tfn, input, want, tc.tokens);
673	}
674
675	if (tfn == .write_huffman_block) {
676	return;
677	}
678
679	const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680	const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681	try testWriteBlock(tfn, null, want, tc.tokens);
682	}
683
684	// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685	fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686	var buf = ArrayList(u8).init(testing.allocator);
687	var bw = blockWriter(buf.writer());
688	try tfn.write(&bw, tokens, input, false);
689	var got = buf.items;
690	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691	try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692	//
693	// Test if the writer produces the same output after reset.
694	buf.deinit();
695	buf = ArrayList(u8).init(testing.allocator);
696	defer buf.deinit();
697	bw.setWriter(buf.writer());
698
699	try tfn.write(&bw, tokens, input, true);
700	try bw.flush();
701	got = buf.items;
702
703	try expect(got[0] & 1 == 1); // bfinal is set
704	buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706	}

lib/std/compress/flate/consts.zig deleted-49

...	@@ -1,49 +0,0 @@
1	pub const deflate = struct {
2	// Number of tokens to accumulate in deflate before starting block encoding.
3	//
4	// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5	// 8 and max 9 that gives 14 or 15 bits.
6	pub const tokens = 1 << 15;
7	};
8
9	pub const match = struct {
10	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11	pub const min_length = 4; // min length used in this algorithm
12	pub const max_length = 258;
13
14	pub const min_distance = 1;
15	pub const max_distance = 32768;
16	};
17
18	pub const history = struct {
19	pub const len = match.max_distance;
20	};
21
22	pub const lookup = struct {
23	pub const bits = 15;
24	pub const len = 1 << bits;
25	pub const shift = 32 - bits;
26	};
27
28	pub const huffman = struct {
29	// The odd order in which the codegen code sizes are written.
30	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31	// The number of codegen codes.
32	pub const codegen_code_count = 19;
33
34	// The largest distance code.
35	pub const distance_code_count = 30;
36
37	// Maximum number of literals.
38	pub const max_num_lit = 286;
39
40	// Max number of frequencies used for a Huffman Code
41	// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42	// The largest of these is max_num_lit.
43	pub const max_num_frequencies = max_num_lit;
44
45	// Biggest block size for uncompressed block.
46	pub const max_store_block_size = 65535;
47	// The special code used to mark the end of a block.
48	pub const end_block_marker = 256;
49	};

lib/std/compress/flate/container.zig deleted-208

...	@@ -1,208 +0,0 @@
1	//! Container of the deflate bit stream body. Container adds header before
2	//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
3	//! no footer, raw bit stream).
4	//!
5	//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
6	//! addler 32 checksum.
7	//!
8	//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
9	//! crc32 checksum and 4 bytes of uncompressed data length.
10	//!
11	//!
12	//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
13	//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
14	//!
15
16	const std = @import("std");
17
18	pub const Container = enum {
19	raw, // no header or footer
20	gzip, // gzip header and footer
21	zlib, // zlib header and footer
22
23	pub fn size(w: Container) usize {
24	return headerSize(w) + footerSize(w);
25	}
26
27	pub fn headerSize(w: Container) usize {
28	return switch (w) {
29	.gzip => 10,
30	.zlib => 2,
31	.raw => 0,
32	};
33	}
34
35	pub fn footerSize(w: Container) usize {
36	return switch (w) {
37	.gzip => 8,
38	.zlib => 4,
39	.raw => 0,
40	};
41	}
42
43	pub const list = [_]Container{ .raw, .gzip, .zlib };
44
45	pub const Error = error{
46	BadGzipHeader,
47	BadZlibHeader,
48	WrongGzipChecksum,
49	WrongGzipSize,
50	WrongZlibChecksum,
51	};
52
53	pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
54	switch (wrap) {
55	.gzip => {
56	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
57	// - ID1 (IDentification 1), always 0x1f
58	// - ID2 (IDentification 2), always 0x8b
59	// - CM (Compression Method), always 8 = deflate
60	// - FLG (Flags), all set to 0
61	// - 4 bytes, MTIME (Modification time), not used, all set to zero
62	// - XFL (eXtra FLags), all set to zero
63	// - OS (Operating System), 03 = Unix
64	const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
65	try writer.writeAll(&gzipHeader);
66	},
67	.zlib => {
68	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
69	// 1st byte:
70	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
71	// - The next four bits is the CM (compression method), which is 8 for deflate.
72	// 2nd byte:
73	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
74	// - The next bit, FDICT, is set if a dictionary is given.
75	// - The final five FCHECK bits form a mod-31 checksum.
76	//
77	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
78	const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
79	try writer.writeAll(&zlibHeader);
80	},
81	.raw => {},
82	}
83	}
84
85	pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
86	var bits: [4]u8 = undefined;
87	switch (wrap) {
88	.gzip => {
89	// GZIP 8 bytes footer
90	// - 4 bytes, CRC32 (CRC-32)
91	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
92	std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
93	try writer.writeAll(&bits);
94
95	std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
96	try writer.writeAll(&bits);
97	},
98	.zlib => {
99	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
100	// 4 bytes of ADLER32 (Adler-32 checksum)
101	// Checksum value of the uncompressed data (excluding any
102	// dictionary data) computed according to Adler-32
103	// algorithm.
104	std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
105	try writer.writeAll(&bits);
106	},
107	.raw => {},
108	}
109	}
110
111	pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
112	switch (wrap) {
113	.gzip => try parseGzipHeader(reader),
114	.zlib => try parseZlibHeader(reader),
115	.raw => {},
116	}
117	}
118
119	fn parseGzipHeader(reader: anytype) !void {
120	const magic1 = try reader.read(u8);
121	const magic2 = try reader.read(u8);
122	const method = try reader.read(u8);
123	const flags = try reader.read(u8);
124	try reader.skipBytes(6); // mtime(4), xflags, os
125	if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
126	return error.BadGzipHeader;
127	// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
128	if (flags != 0) {
129	if (flags & 0b0000_0100 != 0) { // FEXTRA
130	const extra_len = try reader.read(u16);
131	try reader.skipBytes(extra_len);
132	}
133	if (flags & 0b0000_1000 != 0) { // FNAME
134	try reader.skipStringZ();
135	}
136	if (flags & 0b0001_0000 != 0) { // FCOMMENT
137	try reader.skipStringZ();
138	}
139	if (flags & 0b0000_0010 != 0) { // FHCRC
140	try reader.skipBytes(2);
141	}
142	}
143	}
144
145	fn parseZlibHeader(reader: anytype) !void {
146	const cm = try reader.read(u4);
147	const cinfo = try reader.read(u4);
148	_ = try reader.read(u8);
149	if (cm != 8 or cinfo > 7) {
150	return error.BadZlibHeader;
151	}
152	}
153
154	pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155	switch (wrap) {
156	.gzip => {
157	try reader.fill(0);
158	if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
159	if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
160	},
161	.zlib => {
162	const chksum: u32 = @byteSwap(hasher.chksum());
163	if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
164	},
165	.raw => {},
166	}
167	}
168
169	pub fn Hasher(comptime wrap: Container) type {
170	const HasherType = switch (wrap) {
171	.gzip => std.hash.Crc32,
172	.zlib => std.hash.Adler32,
173	.raw => struct {
174	pub fn init() @This() {
175	return .{};
176	}
177	},
178	};
179
180	return struct {
181	hasher: HasherType = HasherType.init(),
182	bytes: usize = 0,
183
184	const Self = @This();
185
186	pub fn update(self: *Self, buf: []const u8) void {
187	switch (wrap) {
188	.raw => {},
189	else => {
190	self.hasher.update(buf);
191	self.bytes += buf.len;
192	},
193	}
194	}
195
196	pub fn chksum(self: *Self) u32 {
197	switch (wrap) {
198	.raw => return 0,
199	else => return self.hasher.final(),
200	}
201	}
202
203	pub fn bytesRead(self: *Self) u32 {
204	return @truncate(self.bytes);
205	}
206	};
207	}
208	};

lib/std/compress/flate/deflate.zig deleted-744

...	@@ -1,744 +0,0 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4	const testing = std.testing;
5	const expect = testing.expect;
6	const print = std.debug.print;
7
8	const Token = @import("Token.zig");
9	const consts = @import("consts.zig");
10	const BlockWriter = @import("block_writer.zig").BlockWriter;
11	const Container = @import("container.zig").Container;
12	const SlidingWindow = @import("SlidingWindow.zig");
13	const Lookup = @import("Lookup.zig");
14
15	pub const Options = struct {
16	level: Level = .default,
17	};
18
19	/// Trades between speed and compression size.
20	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21	/// levels 1-3 are using different algorithm to perform faster but with less
22	/// compression. That is not implemented here.
23	pub const Level = enum(u4) {
24	// zig fmt: off
25	fast = 0xb, level_4 = 4,
26	level_5 = 5,
27	default = 0xc, level_6 = 6,
28	level_7 = 7,
29	level_8 = 8,
30	best = 0xd, level_9 = 9,
31	// zig fmt: on
32	};
33
34	/// Algorithm knobs for each level.
35	const LevelArgs = struct {
36	good: u16, // Do less lookups if we already have match of this length.
37	nice: u16, // Stop looking for better match if we found match with at least this length.
38	lazy: u16, // Don't do lazy match find if got match with at least this length.
39	chain: u16, // How many lookups for previous match to perform.
40
41	pub fn get(level: Level) LevelArgs {
42	// zig fmt: off
43	return switch (level) {
44	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50	};
51	// zig fmt: on
52	}
53	};
54
55	/// Compress plain data from reader into compressed stream written to writer.
56	pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57	var c = try compressor(container, writer, options);
58	try c.compress(reader);
59	try c.finish();
60	}
61
62	/// Create compressor for writer type.
63	pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64	container,
65	@TypeOf(writer),
66	) {
67	return try Compressor(container, @TypeOf(writer)).init(writer, options);
68	}
69
70	/// Compressor type.
71	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72	const TokenWriterType = BlockWriter(WriterType);
73	return Deflate(container, WriterType, TokenWriterType);
74	}
75
76	/// Default compression algorithm. Has two steps: tokenization and token
77	/// encoding.
78	///
79	/// Tokenization takes uncompressed input stream and produces list of tokens.
80	/// Each token can be literal (byte of data) or match (backrefernce to previous
81	/// data with length and distance). Tokenization accumulators 32K tokens, when
82	/// full or `flush` is called tokens are passed to the `block_writer`. Level
83	/// defines how hard (how slow) it tries to find match.
84	///
85	/// Block writer will decide which type of deflate block to write (stored, fixed,
86	/// dynamic) and encode tokens to the output byte stream. Client has to call
87	/// `finish` to write block with the final bit set.
88	///
89	/// Container defines type of header and footer which can be gzip, zlib or raw.
90	/// They all share same deflate body. Raw has no header or footer just deflate
91	/// body.
92	///
93	/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
94	///
95	/// The compressor uses a chained hash table `lookup` to find duplicated
96	/// strings, using a hash function that operates on 4-byte sequences. At any
97	/// given point during compression, let XYZW be the next 4 input bytes
98	/// (lookahead) to be examined (not necessarily all different, of course).
99	/// First, the compressor examines the hash chain for XYZW. If the chain is
100	/// empty, the compressor simply writes out X as a literal byte and advances
101	/// one byte in the input. If the hash chain is not empty, indicating that the
102	/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
103	/// hash function value) has occurred recently, the compressor compares all
104	/// strings on the XYZW hash chain with the actual input data sequence
105	/// starting at the current point, and selects the longest match.
106	///
107	/// To improve overall compression, the compressor defers the selection of
108	/// matches ("lazy matching"): after a match of length N has been found, the
109	/// compressor searches for a longer match starting at the next input byte. If
110	/// it finds a longer match, it truncates the previous match to a length of
111	/// one (thus producing a single literal byte) and then emits the longer
112	/// match. Otherwise, it emits the original match, and, as described above,
113	/// advances N bytes before continuing.
114	///
115	///
116	/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
117	///
118	/// Deflate function accepts BlockWriterType so we can change that in test to test
119	/// just tokenization part.
120	///
121	fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
122	return struct {
123	lookup: Lookup = .{},
124	win: SlidingWindow = .{},
125	tokens: Tokens = .{},
126	wrt: WriterType,
127	block_writer: BlockWriterType,
128	level: LevelArgs,
129	hasher: container.Hasher() = .{},
130
131	// Match and literal at the previous position.
132	// Used for lazy match finding in processWindow.
133	prev_match: ?Token = null,
134	prev_literal: ?u8 = null,
135
136	const Self = @This();
137
138	pub fn init(wrt: WriterType, options: Options) !Self {
139	const self = Self{
140	.wrt = wrt,
141	.block_writer = BlockWriterType.init(wrt),
142	.level = LevelArgs.get(options.level),
143	};
144	try container.writeHeader(self.wrt);
145	return self;
146	}
147
148	const FlushOption = enum { none, flush, final };
149
150	// Process data in window and create tokens. If token buffer is full
151	// flush tokens to the token writer. In the case of `flush` or `final`
152	// option it will process all data from the window. In the `none` case
153	// it will preserve some data for the next match.
154	fn tokenize(self: *Self, flush_opt: FlushOption) !void {
155	// flush - process all data from window
156	const should_flush = (flush_opt != .none);
157
158	// While there is data in active lookahead buffer.
159	while (self.win.activeLookahead(should_flush)) \|lh\| {
160	var step: u16 = 1; // 1 in the case of literal, match length otherwise
161	const pos: u16 = self.win.pos();
162	const literal = lh[0]; // literal at current position
163	const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
164
165	// Try to find match at least min_len long.
166	if (self.findMatch(pos, lh, min_len)) \|match\| {
167	// Found better match than previous.
168	try self.addPrevLiteral();
169
170	// Is found match length good enough?
171	if (match.length() >= self.level.lazy) {
172	// Don't try to lazy find better match, use this.
173	step = try self.addMatch(match);
174	} else {
175	// Store this match.
176	self.prev_literal = literal;
177	self.prev_match = match;
178	}
179	} else {
180	// There is no better match at current pos then it was previous.
181	// Write previous match or literal.
182	if (self.prev_match) \|m\| {
183	// Write match from previous position.
184	step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
185	} else {
186	// No match at previous position.
187	// Write previous literal if any, and remember this literal.
188	try self.addPrevLiteral();
189	self.prev_literal = literal;
190	}
191	}
192	// Advance window and add hashes.
193	self.windowAdvance(step, lh, pos);
194	}
195
196	if (should_flush) {
197	// In the case of flushing, last few lookahead buffers were smaller then min match len.
198	// So only last literal can be unwritten.
199	assert(self.prev_match == null);
200	try self.addPrevLiteral();
201	self.prev_literal = null;
202
203	try self.flushTokens(flush_opt);
204	}
205	}
206
207	fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
208	// current position is already added in findMatch
209	self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
210	self.win.advance(step);
211	}
212
213	// Add previous literal (if any) to the tokens list.
214	fn addPrevLiteral(self: *Self) !void {
215	if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
216	}
217
218	// Add match to the tokens list, reset prev pointers.
219	// Returns length of the added match.
220	fn addMatch(self: *Self, m: Token) !u16 {
221	try self.addToken(m);
222	self.prev_literal = null;
223	self.prev_match = null;
224	return m.length();
225	}
226
227	fn addToken(self: *Self, token: Token) !void {
228	self.tokens.add(token);
229	if (self.tokens.full()) try self.flushTokens(.none);
230	}
231
232	// Finds largest match in the history window with the data at current pos.
233	fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
234	var len: u16 = min_len;
235	// Previous location with the same hash (same 4 bytes).
236	var prev_pos = self.lookup.add(lh, pos);
237	// Last found match.
238	var match: ?Token = null;
239
240	// How much back-references to try, performance knob.
241	var chain: usize = self.level.chain;
242	if (len >= self.level.good) {
243	// If we've got a match that's good enough, only look in 1/4 the chain.
244	chain >>= 2;
245	}
246
247	// Hot path loop!
248	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
249	const distance = pos - prev_pos;
250	if (distance > consts.match.max_distance)
251	break;
252
253	const new_len = self.win.match(prev_pos, pos, len);
254	if (new_len > len) {
255	match = Token.initMatch(@intCast(distance), new_len);
256	if (new_len >= self.level.nice) {
257	// The match is good enough that we don't try to find a better one.
258	return match;
259	}
260	len = new_len;
261	}
262	prev_pos = self.lookup.prev(prev_pos);
263	}
264
265	return match;
266	}
267
268	fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
269	// Pass tokens to the token writer
270	try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
271	// Stored block ensures byte alignment.
272	// It has 3 bits (final, block_type) and then padding until byte boundary.
273	// After that everything is aligned to the boundary in the stored block.
274	// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
275	// Last 4 bytes are byte aligned.
276	if (flush_opt == .flush) {
277	try self.block_writer.storedBlock("", false);
278	}
279	if (flush_opt != .none) {
280	// Safe to call only when byte aligned or it is OK to add
281	// padding bits (on last byte of the final block).
282	try self.block_writer.flush();
283	}
284	// Reset internal tokens store.
285	self.tokens.reset();
286	// Notify win that tokens are flushed.
287	self.win.flush();
288	}
289
290	// Slide win and if needed lookup tables.
291	fn slide(self: *Self) void {
292	const n = self.win.slide();
293	self.lookup.slide(n);
294	}
295
296	/// Compresses as much data as possible, stops when the reader becomes
297	/// empty. It will introduce some output latency (reading input without
298	/// producing all output) because some data are still in internal
299	/// buffers.
300	///
301	/// It is up to the caller to call flush (if needed) or finish (required)
302	/// when is need to output any pending data or complete stream.
303	///
304	pub fn compress(self: *Self, reader: anytype) !void {
305	while (true) {
306	// Fill window from reader
307	const buf = self.win.writable();
308	if (buf.len == 0) {
309	try self.tokenize(.none);
310	self.slide();
311	continue;
312	}
313	const n = try reader.readAll(buf);
314	self.hasher.update(buf[0..n]);
315	self.win.written(n);
316	// Process window
317	try self.tokenize(.none);
318	// Exit when no more data in reader
319	if (n < buf.len) break;
320	}
321	}
322
323	/// Flushes internal buffers to the output writer. Outputs empty stored
324	/// block to sync bit stream to the byte boundary, so that the
325	/// decompressor can get all input data available so far.
326	///
327	/// It is useful mainly in compressed network protocols, to ensure that
328	/// deflate bit stream can be used as byte stream. May degrade
329	/// compression so it should be used only when necessary.
330	///
331	/// Completes the current deflate block and follows it with an empty
332	/// stored block that is three zero bits plus filler bits to the next
333	/// byte, followed by four bytes (00 00 ff ff).
334	///
335	pub fn flush(self: *Self) !void {
336	try self.tokenize(.flush);
337	}
338
339	/// Completes deflate bit stream by writing any pending data as deflate
340	/// final deflate block. HAS to be called once all data are written to
341	/// the compressor as a signal that next block has to have final bit
342	/// set.
343	///
344	pub fn finish(self: *Self) !void {
345	try self.tokenize(.final);
346	try container.writeFooter(&self.hasher, self.wrt);
347	}
348
349	/// Use another writer while preserving history. Most probably flush
350	/// should be called on old writer before setting new.
351	pub fn setWriter(self: *Self, new_writer: WriterType) void {
352	self.block_writer.setWriter(new_writer);
353	self.wrt = new_writer;
354	}
355
356	// Writer interface
357
358	pub const Writer = io.GenericWriter(*Self, Error, write);
359	pub const Error = BlockWriterType.Error;
360
361	/// Write `input` of uncompressed data.
362	/// See compress.
363	pub fn write(self: *Self, input: []const u8) !usize {
364	var fbs = io.fixedBufferStream(input);
365	try self.compress(fbs.reader());
366	return input.len;
367	}
368
369	pub fn writer(self: *Self) Writer {
370	return .{ .context = self };
371	}
372	};
373	}
374
375	// Tokens store
376	const Tokens = struct {
377	list: [consts.deflate.tokens]Token = undefined,
378	pos: usize = 0,
379
380	fn add(self: *Tokens, t: Token) void {
381	self.list[self.pos] = t;
382	self.pos += 1;
383	}
384
385	fn full(self: *Tokens) bool {
386	return self.pos == self.list.len;
387	}
388
389	fn reset(self: *Tokens) void {
390	self.pos = 0;
391	}
392
393	fn tokens(self: *Tokens) []const Token {
394	return self.list[0..self.pos];
395	}
396	};
397
398	/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
399	/// only performs Huffman entropy encoding. Results in faster compression, much
400	/// less memory requirements during compression but bigger compressed sizes.
401	pub const huffman = struct {
402	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
403	var c = try huffman.compressor(container, writer);
404	try c.compress(reader);
405	try c.finish();
406	}
407
408	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
409	return SimpleCompressor(.huffman, container, WriterType);
410	}
411
412	pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
413	return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
414	}
415	};
416
417	/// Creates store blocks only. Data are not compressed only packed into deflate
418	/// store blocks. That adds 9 bytes of header for each block. Max stored block
419	/// size is 64K. Block is emitted when flush is called on on finish.
420	pub const store = struct {
421	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
422	var c = try store.compressor(container, writer);
423	try c.compress(reader);
424	try c.finish();
425	}
426
427	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
428	return SimpleCompressor(.store, container, WriterType);
429	}
430
431	pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
432	return try store.Compressor(container, @TypeOf(writer)).init(writer);
433	}
434	};
435
436	const SimpleCompressorKind = enum {
437	huffman,
438	store,
439	};
440
441	fn simpleCompressor(
442	comptime kind: SimpleCompressorKind,
443	comptime container: Container,
444	writer: anytype,
445	) !SimpleCompressor(kind, container, @TypeOf(writer)) {
446	return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
447	}
448
449	fn SimpleCompressor(
450	comptime kind: SimpleCompressorKind,
451	comptime container: Container,
452	comptime WriterType: type,
453	) type {
454	const BlockWriterType = BlockWriter(WriterType);
455	return struct {
456	buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
457	wp: usize = 0,
458
459	wrt: WriterType,
460	block_writer: BlockWriterType,
461	hasher: container.Hasher() = .{},
462
463	const Self = @This();
464
465	pub fn init(wrt: WriterType) !Self {
466	const self = Self{
467	.wrt = wrt,
468	.block_writer = BlockWriterType.init(wrt),
469	};
470	try container.writeHeader(self.wrt);
471	return self;
472	}
473
474	pub fn flush(self: *Self) !void {
475	try self.flushBuffer(false);
476	try self.block_writer.storedBlock("", false);
477	try self.block_writer.flush();
478	}
479
480	pub fn finish(self: *Self) !void {
481	try self.flushBuffer(true);
482	try self.block_writer.flush();
483	try container.writeFooter(&self.hasher, self.wrt);
484	}
485
486	fn flushBuffer(self: *Self, final: bool) !void {
487	const buf = self.buffer[0..self.wp];
488	switch (kind) {
489	.huffman => try self.block_writer.huffmanBlock(buf, final),
490	.store => try self.block_writer.storedBlock(buf, final),
491	}
492	self.wp = 0;
493	}
494
495	// Writes all data from the input reader of uncompressed data.
496	// It is up to the caller to call flush or finish if there is need to
497	// output compressed blocks.
498	pub fn compress(self: *Self, reader: anytype) !void {
499	while (true) {
500	// read from rdr into buffer
501	const buf = self.buffer[self.wp..];
502	if (buf.len == 0) {
503	try self.flushBuffer(false);
504	continue;
505	}
506	const n = try reader.readAll(buf);
507	self.hasher.update(buf[0..n]);
508	self.wp += n;
509	if (n < buf.len) break; // no more data in reader
510	}
511	}
512
513	// Writer interface
514
515	pub const Writer = io.GenericWriter(*Self, Error, write);
516	pub const Error = BlockWriterType.Error;
517
518	// Write `input` of uncompressed data.
519	pub fn write(self: *Self, input: []const u8) !usize {
520	var fbs = io.fixedBufferStream(input);
521	try self.compress(fbs.reader());
522	return input.len;
523	}
524
525	pub fn writer(self: *Self) Writer {
526	return .{ .context = self };
527	}
528	};
529	}
530
531	const builtin = @import("builtin");
532
533	test "tokenization" {
534	const L = Token.initLiteral;
535	const M = Token.initMatch;
536
537	const cases = [_]struct {
538	data: []const u8,
539	tokens: []const Token,
540	}{
541	.{
542	.data = "Blah blah blah blah blah!",
543	.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
544	},
545	.{
546	.data = "ABCDEABCD ABCDEABCD",
547	.tokens = &[_]Token{
548	L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
549	L('A'), M(10, 8),
550	},
551	},
552	};
553
554	for (cases) \|c\| {
555	inline for (Container.list) \|container\| { // for each wrapping
556
557	var cw = io.countingWriter(io.null_writer);
558	const cww = cw.writer();
559	var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
560
561	_ = try df.write(c.data);
562	try df.flush();
563
564	// df.token_writer.show();
565	try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
566	try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
567
568	try testing.expectEqual(container.headerSize(), cw.bytes_written);
569	try df.finish();
570	try testing.expectEqual(container.size(), cw.bytes_written);
571	}
572	}
573	}
574
575	// Tests that tokens written are equal to expected token list.
576	const TestTokenWriter = struct {
577	const Self = @This();
578
579	pos: usize = 0,
580	actual: [128]Token = undefined,
581
582	pub fn init(_: anytype) Self {
583	return .{};
584	}
585	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
586	for (tokens) \|t\| {
587	self.actual[self.pos] = t;
588	self.pos += 1;
589	}
590	}
591
592	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
593
594	pub fn get(self: *Self) []Token {
595	return self.actual[0..self.pos];
596	}
597
598	pub fn show(self: *Self) void {
599	print("\n", .{});
600	for (self.get()) \|t\| {
601	t.show();
602	}
603	}
604
605	pub fn flush(_: *Self) !void {}
606	};
607
608	test "file tokenization" {
609	const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
610	const cases = [_]struct {
611	data: []const u8, // uncompressed content
612	// expected number of tokens producet in deflate tokenization
613	tokens_count: [levels.len]usize = .{0} ** levels.len,
614	}{
615	.{
616	.data = @embedFile("testdata/rfc1951.txt"),
617	.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
618	},
619
620	.{
621	.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
622	.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
623	},
624	.{
625	.data = @embedFile("testdata/block_writer/huffman-pi.input"),
626	.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
627	},
628	.{
629	.data = @embedFile("testdata/block_writer/huffman-text.input"),
630	.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
631	},
632	.{
633	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
634	.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
635	},
636	.{
637	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
638	.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
639	},
640	};
641
642	for (cases) \|case\| { // for each case
643	const data = case.data;
644
645	for (levels, 0..) \|level, i\| { // for each compression level
646	var original = io.fixedBufferStream(data);
647
648	// buffer for decompressed data
649	var al = std.ArrayList(u8).init(testing.allocator);
650	defer al.deinit();
651	const writer = al.writer();
652
653	// create compressor
654	const WriterType = @TypeOf(writer);
655	const TokenWriter = TokenDecoder(@TypeOf(writer));
656	var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
657
658	// Stream uncompressed `original` data to the compressor. It will
659	// produce tokens list and pass that list to the TokenDecoder. This
660	// TokenDecoder uses CircularBuffer from inflate to convert list of
661	// tokens back to the uncompressed stream.
662	try cmp.compress(original.reader());
663	try cmp.flush();
664	const expected_count = case.tokens_count[i];
665	const actual = cmp.block_writer.tokens_count;
666	if (expected_count == 0) {
667	print("actual token count {d}\n", .{actual});
668	} else {
669	try testing.expectEqual(expected_count, actual);
670	}
671
672	try testing.expectEqual(data.len, al.items.len);
673	try testing.expectEqualSlices(u8, data, al.items);
674	}
675	}
676	}
677
678	fn TokenDecoder(comptime WriterType: type) type {
679	return struct {
680	const CircularBuffer = @import("CircularBuffer.zig");
681	hist: CircularBuffer = .{},
682	wrt: WriterType,
683	tokens_count: usize = 0,
684
685	const Self = @This();
686
687	pub fn init(wrt: WriterType) Self {
688	return .{ .wrt = wrt };
689	}
690
691	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
692	self.tokens_count += tokens.len;
693	for (tokens) \|t\| {
694	switch (t.kind) {
695	.literal => self.hist.write(t.literal()),
696	.match => try self.hist.writeMatch(t.length(), t.distance()),
697	}
698	if (self.hist.free() < 285) try self.flushWin();
699	}
700	try self.flushWin();
701	}
702
703	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
704
705	fn flushWin(self: *Self) !void {
706	while (true) {
707	const buf = self.hist.read();
708	if (buf.len == 0) break;
709	try self.wrt.writeAll(buf);
710	}
711	}
712
713	pub fn flush(_: *Self) !void {}
714	};
715	}
716
717	test "store simple compressor" {
718	const data = "Hello world!";
719	const expected = [_]u8{
720	0x1, // block type 0, final bit set
721	0xc, 0x0, // len = 12
722	0xf3, 0xff, // ~len
723	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
724	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
725	};
726
727	var fbs = std.io.fixedBufferStream(data);
728	var al = std.ArrayList(u8).init(testing.allocator);
729	defer al.deinit();
730
731	var cmp = try store.compressor(.raw, al.writer());
732	try cmp.compress(fbs.reader());
733	try cmp.finish();
734	try testing.expectEqualSlices(u8, &expected, al.items);
735
736	fbs.reset();
737	try al.resize(0);
738
739	// huffman only compresoor will also emit store block for this small sample
740	var hc = try huffman.compressor(.raw, al.writer());
741	try hc.compress(fbs.reader());
742	try hc.finish();
743	try testing.expectEqualSlices(u8, &expected, al.items);
744	}

lib/std/compress/flate/huffman_decoder.zig deleted-302

...	@@ -1,302 +0,0 @@
1	const std = @import("std");
2	const testing = std.testing;
3
4	pub const Symbol = packed struct {
5	pub const Kind = enum(u2) {
6	literal,
7	end_of_block,
8	match,
9	};
10
11	symbol: u8 = 0, // symbol from alphabet
12	code_bits: u4 = 0, // number of bits in code 0-15
13	kind: Kind = .literal,
14
15	code: u16 = 0, // huffman code of the symbol
16	next: u16 = 0, // pointer to the next symbol in linked list
17	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19	// Sorting less than function.
20	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21	if (a.code_bits == b.code_bits) {
22	if (a.kind == b.kind) {
23	return a.symbol < b.symbol;
24	}
25	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26	}
27	return a.code_bits < b.code_bits;
28	}
29	};
30
31	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35	pub const Error = error{
36	InvalidCode,
37	OversubscribedHuffmanTree,
38	IncompleteHuffmanTree,
39	MissingEndOfBlockCode,
40	};
41
42	/// Creates huffman tree codes from list of code lengths (in `build`).
43	///
44	/// `find` then finds symbol for code bits. Code can be any length between 1 and
45	/// 15 bits. When calling `find` we don't know how many bits will be used to
46	/// find symbol. When symbol is returned it has code_bits field which defines
47	/// how much we should advance in bit stream.
48	///
49	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50	/// many times in this table; 32K places for 286 (at most) symbols.
51	/// Small lookup table is optimization for faster search.
52	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53	/// with difference that we here use statically allocated arrays.
54	///
55	fn HuffmanDecoder(
56	comptime alphabet_size: u16,
57	comptime max_code_bits: u4,
58	comptime lookup_bits: u4,
59	) type {
60	const lookup_shift = max_code_bits - lookup_bits;
61
62	return struct {
63	// all symbols in alaphabet, sorted by code_len, symbol
64	symbols: [alphabet_size]Symbol = undefined,
65	// lookup table code -> symbol
66	lookup: [1 << lookup_bits]Symbol = undefined,
67
68	const Self = @This();
69
70	/// Generates symbols and lookup tables from list of code lens for each symbol.
71	pub fn generate(self: *Self, lens: []const u4) !void {
72	try checkCompleteness(lens);
73
74	// init alphabet with code_bits
75	for (self.symbols, 0..) \|_, i\| {
76	const cb: u4 = if (i < lens.len) lens[i] else 0;
77	self.symbols[i] = if (i < 256)
78	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79	else if (i == 256)
80	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81	else
82	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83	}
84	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86	// reset lookup table
87	for (0..self.lookup.len) \|i\| {
88	self.lookup[i] = .{};
89	}
90
91	// assign code to symbols
92	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93	var code: u16 = 0;
94	var idx: u16 = 0;
95	for (&self.symbols, 0..) \|*sym, pos\| {
96	if (sym.code_bits == 0) continue; // skip unused
97	sym.code = code;
98
99	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
100	const next_idx = next_code >> lookup_shift;
101
102	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
103	if (sym.code_bits <= lookup_bits) {
104	// fill small lookup table
105	for (idx..next_idx) \|j\|
106	self.lookup[j] = sym.*;
107	} else {
108	// insert into linked table starting at root
109	const root = &self.lookup[idx];
110	const root_next = root.next;
111	root.next = @intCast(pos);
112	sym.next = root_next;
113	}
114
115	idx = next_idx;
116	code = next_code;
117	}
118	}
119
120	/// Given the list of code lengths check that it represents a canonical
121	/// Huffman code for n symbols.
122	///
123	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
124	fn checkCompleteness(lens: []const u4) !void {
125	if (alphabet_size == 286)
126	if (lens[256] == 0) return error.MissingEndOfBlockCode;
127
128	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
129	var max: usize = 0;
130	for (lens) \|n\| {
131	if (n == 0) continue;
132	if (n > max) max = n;
133	count[n] += 1;
134	}
135	if (max == 0) // empty tree
136	return;
137
138	// check for an over-subscribed or incomplete set of lengths
139	var left: usize = 1; // one possible code of zero length
140	for (1..count.len) \|len\| {
141	left <<= 1; // one more bit, double codes left
142	if (count[len] > left)
143	return error.OversubscribedHuffmanTree;
144	left -= count[len]; // deduct count from possible codes
145	}
146	if (left > 0) { // left > 0 means incomplete
147	// incomplete code ok only for single length 1 code
148	if (max_code_bits > 7 and max == count[0] + count[1]) return;
149	return error.IncompleteHuffmanTree;
150	}
151	}
152
153	/// Finds symbol for lookup table code.
154	pub fn find(self: *Self, code: u16) !Symbol {
155	// try to find in lookup table
156	const idx = code >> lookup_shift;
157	const sym = self.lookup[idx];
158	if (sym.code_bits != 0) return sym;
159	// if not use linked list of symbols with same prefix
160	return self.findLinked(code, sym.next);
161	}
162
163	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
164	var pos = start;
165	while (pos > 0) {
166	const sym = self.symbols[pos];
167	const shift = max_code_bits - sym.code_bits;
168	// compare code_bits number of upper bits
169	if ((code ^ sym.code) >> shift == 0) return sym;
170	pos = sym.next;
171	}
172	return error.InvalidCode;
173	}
174	};
175	}
176
177	test "init/find" {
178	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
179	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
180	var h: CodegenDecoder = .{};
181	try h.generate(&code_lens);
182
183	const expected = [_]struct {
184	sym: Symbol,
185	code: u16,
186	}{
187	.{
188	.code = 0b00_00000,
189	.sym = .{ .symbol = 3, .code_bits = 2 },
190	},
191	.{
192	.code = 0b01_00000,
193	.sym = .{ .symbol = 18, .code_bits = 2 },
194	},
195	.{
196	.code = 0b100_0000,
197	.sym = .{ .symbol = 1, .code_bits = 3 },
198	},
199	.{
200	.code = 0b101_0000,
201	.sym = .{ .symbol = 4, .code_bits = 3 },
202	},
203	.{
204	.code = 0b110_0000,
205	.sym = .{ .symbol = 17, .code_bits = 3 },
206	},
207	.{
208	.code = 0b1110_000,
209	.sym = .{ .symbol = 0, .code_bits = 4 },
210	},
211	.{
212	.code = 0b1111_000,
213	.sym = .{ .symbol = 16, .code_bits = 4 },
214	},
215	};
216
217	// unused symbols
218	for (0..12) \|i\| {
219	try testing.expectEqual(0, h.symbols[i].code_bits);
220	}
221	// used, from index 12
222	for (expected, 12..) \|e, i\| {
223	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
224	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
225	const sym_from_code = try h.find(e.code);
226	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
227	}
228
229	// All possible codes for each symbol.
230	// Lookup table has 126 elements, to cover all possible 7 bit codes.
231	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
232	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
233
234	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
235	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
236
237	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
238	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
239
240	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
241	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
242
243	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
244	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
245
246	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
247	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
248
249	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
250	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
251	}
252
253	test "encode/decode literals" {
254	const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
255
256	for (1..286) \|j\| { // for all different number of codes
257	var enc: LiteralEncoder = .{};
258	// create frequencies
259	var freq = [_]u16{0} ** 286;
260	freq[256] = 1; // ensure we have end of block code
261	for (&freq, 1..) \|*f, i\| {
262	if (i % j == 0)
263	f.* = @intCast(i);
264	}
265
266	// encoder from frequencies
267	enc.generate(&freq, 15);
268
269	// get code_lens from encoder
270	var code_lens = [_]u4{0} ** 286;
271	for (code_lens, 0..) \|_, i\| {
272	code_lens[i] = @intCast(enc.codes[i].len);
273	}
274	// generate decoder from code lens
275	var dec: LiteralDecoder = .{};
276	try dec.generate(&code_lens);
277
278	// expect decoder code to match original encoder code
279	for (dec.symbols) \|s\| {
280	if (s.code_bits == 0) continue;
281	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
282	const symbol: u16 = switch (s.kind) {
283	.literal => s.symbol,
284	.end_of_block => 256,
285	.match => @as(u16, s.symbol) + 257,
286	};
287
288	const c = enc.codes[symbol];
289	try testing.expect(c.code == c_code);
290	}
291
292	// find each symbol by code
293	for (enc.codes) \|c\| {
294	if (c.len == 0) continue;
295
296	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
297	const s = try dec.find(s_code);
298	try testing.expect(s.code == s_code);
299	try testing.expect(s.code_bits == c.len);
300	}
301	}
302	}

lib/std/compress/flate/huffman_encoder.zig deleted-536

...	@@ -1,536 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const consts = @import("consts.zig").huffman;
9
10	const LiteralNode = struct {
11	literal: u16,
12	freq: u16,
13	};
14
15	// Describes the state of the constructed tree for a given depth.
16	const LevelInfo = struct {
17	// Our level. for better printing
18	level: u32,
19
20	// The frequency of the last node at this level
21	last_freq: u32,
22
23	// The frequency of the next character to add to this level
24	next_char_freq: u32,
25
26	// The frequency of the next pair (from level below) to add to this level.
27	// Only valid if the "needed" value of the next lower level is 0.
28	next_pair_freq: u32,
29
30	// The number of chains remaining to generate for this level before moving
31	// up to the next level
32	needed: u32,
33	};
34
35	// hcode is a huffman code with a bit code and bit length.
36	pub const HuffCode = struct {
37	code: u16 = 0,
38	len: u16 = 0,
39
40	// set sets the code and length of an hcode.
41	fn set(self: *HuffCode, code: u16, length: u16) void {
42	self.len = length;
43	self.code = code;
44	}
45	};
46
47	pub fn HuffmanEncoder(comptime size: usize) type {
48	return struct {
49	codes: [size]HuffCode = undefined,
50	// Reusable buffer with the longest possible frequency table.
51	freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52	bit_count: [17]u32 = undefined,
53	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56	const Self = @This();
57
58	// Update this Huffman Code object to be the minimum code for the specified frequency count.
59	//
60	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61	// max_bits The maximum number of bits to use for any literal.
62	pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63	var list = self.freq_cache[0 .. freq.len + 1];
64	// Number of non-zero literals
65	var count: u32 = 0;
66	// Set list to be the set of all non-zero literals and their frequencies
67	for (freq, 0..) \|f, i\| {
68	if (f != 0) {
69	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70	count += 1;
71	} else {
72	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73	self.codes[i].len = 0;
74	}
75	}
76	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78	list = list[0..count];
79	if (count <= 2) {
80	// Handle the small cases here, because they are awkward for the general case code. With
81	// two or fewer literals, everything has bit length 1.
82	for (list, 0..) \|node, i\| {
83	// "list" is in order of increasing literal value.
84	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85	}
86	return;
87	}
88	self.lfs = list;
89	mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91	// Get the number of literals for each bit count
92	const bit_count = self.bitCounts(list, max_bits);
93	// And do the assignment
94	self.assignEncodingAndSize(bit_count, list);
95	}
96
97	pub fn bitLength(self: *Self, freq: []u16) u32 {
98	var total: u32 = 0;
99	for (freq, 0..) \|f, i\| {
100	if (f != 0) {
101	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102	}
103	}
104	return total;
105	}
106
107	// Return the number of literals assigned to each bit size in the Huffman encoding
108	//
109	// This method is only called when list.len >= 3
110	// The cases of 0, 1, and 2 literals are handled by special case code.
111	//
112	// list: An array of the literals with non-zero frequencies
113	// and their associated frequencies. The array is in order of increasing
114	// frequency, and has as its last element a special element with frequency
115	// std.math.maxInt(i32)
116	//
117	// max_bits: The maximum number of bits that should be used to encode any literal.
118	// Must be less than 16.
119	//
120	// Returns an integer array in which array[i] indicates the number of literals
121	// that should be encoded in i bits.
122	fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123	var max_bits = max_bits_to_use;
124	const n = list.len;
125	const max_bits_limit = 16;
126
127	assert(max_bits < max_bits_limit);
128
129	// The tree can't have greater depth than n - 1, no matter what. This
130	// saves a little bit of work in some small cases
131	max_bits = @min(max_bits, n - 1);
132
133	// Create information about each of the levels.
134	// A bogus "Level 0" whose sole purpose is so that
135	// level1.prev.needed == 0. This makes level1.next_pair_freq
136	// be a legitimate value that never gets chosen.
137	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138	// leaf_counts[i] counts the number of literals at the left
139	// of ancestors of the rightmost node at level i.
140	// leaf_counts[i][j] is the number of literals at the left
141	// of the level j ancestor.
142	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144	{
145	var level = @as(u32, 1);
146	while (level <= max_bits) : (level += 1) {
147	// For every level, the first two items are the first two characters.
148	// We initialize the levels as if we had already figured this out.
149	levels[level] = LevelInfo{
150	.level = level,
151	.last_freq = list[1].freq,
152	.next_char_freq = list[2].freq,
153	.next_pair_freq = list[0].freq + list[1].freq,
154	.needed = 0,
155	};
156	leaf_counts[level][level] = 2;
157	if (level == 1) {
158	levels[level].next_pair_freq = math.maxInt(i32);
159	}
160	}
161	}
162
163	// We need a total of 2*n - 2 items at top level and have already generated 2.
164	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166	{
167	var level = max_bits;
168	while (true) {
169	var l = &levels[level];
170	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171	// We've run out of both leaves and pairs.
172	// End all calculations for this level.
173	// To make sure we never come back to this level or any lower level,
174	// set next_pair_freq impossibly large.
175	l.needed = 0;
176	levels[level + 1].next_pair_freq = math.maxInt(i32);
177	level += 1;
178	continue;
179	}
180
181	const prev_freq = l.last_freq;
182	if (l.next_char_freq < l.next_pair_freq) {
183	// The next item on this row is a leaf node.
184	const next = leaf_counts[level][level] + 1;
185	l.last_freq = l.next_char_freq;
186	// Lower leaf_counts are the same of the previous node.
187	leaf_counts[level][level] = next;
188	if (next >= list.len) {
189	l.next_char_freq = maxNode().freq;
190	} else {
191	l.next_char_freq = list[next].freq;
192	}
193	} else {
194	// The next item on this row is a pair from the previous row.
195	// next_pair_freq isn't valid until we generate two
196	// more values in the level below
197	l.last_freq = l.next_pair_freq;
198	// Take leaf counts from the lower level, except counts[level] remains the same.
199	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200	levels[l.level - 1].needed = 2;
201	}
202
203	l.needed -= 1;
204	if (l.needed == 0) {
205	// We've done everything we need to do for this level.
206	// Continue calculating one level up. Fill in next_pair_freq
207	// of that level with the sum of the two nodes we've just calculated on
208	// this level.
209	if (l.level == max_bits) {
210	// All done!
211	break;
212	}
213	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214	level += 1;
215	} else {
216	// If we stole from below, move down temporarily to replenish it.
217	while (levels[level - 1].needed > 0) {
218	level -= 1;
219	if (level == 0) {
220	break;
221	}
222	}
223	}
224	}
225	}
226
227	// Somethings is wrong if at the end, the top level is null or hasn't used
228	// all of the leaves.
229	assert(leaf_counts[max_bits][max_bits] == n);
230
231	var bit_count = self.bit_count[0 .. max_bits + 1];
232	var bits: u32 = 1;
233	const counts = &leaf_counts[max_bits];
234	{
235	var level = max_bits;
236	while (level > 0) : (level -= 1) {
237	// counts[level] gives the number of literals requiring at least "bits"
238	// bits to encode.
239	bit_count[bits] = counts[level] - counts[level - 1];
240	bits += 1;
241	if (level == 0) {
242	break;
243	}
244	}
245	}
246	return bit_count;
247	}
248
249	// Look at the leaves and assign them a bit count and an encoding as specified
250	// in RFC 1951 3.2.2
251	fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252	var code = @as(u16, 0);
253	var list = list_arg;
254
255	for (bit_count, 0..) \|bits, n\| {
256	code <<= 1;
257	if (n == 0 or bits == 0) {
258	continue;
259	}
260	// The literals list[list.len-bits] .. list[list.len-bits]
261	// are encoded using "bits" bits, and get the values
262	// code, code + 1, .... The code values are
263	// assigned in literal order (not frequency order).
264	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266	self.lns = chunk;
267	mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269	for (chunk) \|node\| {
270	self.codes[node.literal] = HuffCode{
271	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
272	.len = @as(u16, @intCast(n)),
273	};
274	code += 1;
275	}
276	list = list[0 .. list.len - @as(u32, @intCast(bits))];
277	}
278	}
279	};
280	}
281
282	fn maxNode() LiteralNode {
283	return LiteralNode{
284	.literal = math.maxInt(u16),
285	.freq = math.maxInt(u16),
286	};
287	}
288
289	pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290	return .{};
291	}
292
293	pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294	pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295	pub const CodegenEncoder = HuffmanEncoder(19);
296
297	// Generates a HuffmanCode corresponding to the fixed literal table
298	pub fn fixedLiteralEncoder() LiteralEncoder {
299	var h: LiteralEncoder = undefined;
300	var ch: u16 = 0;
301
302	while (ch < consts.max_num_frequencies) : (ch += 1) {
303	var bits: u16 = undefined;
304	var size: u16 = undefined;
305	switch (ch) {
306	0...143 => {
307	// size 8, 000110000 .. 10111111
308	bits = ch + 48;
309	size = 8;
310	},
311	144...255 => {
312	// size 9, 110010000 .. 111111111
313	bits = ch + 400 - 144;
314	size = 9;
315	},
316	256...279 => {
317	// size 7, 0000000 .. 0010111
318	bits = ch - 256;
319	size = 7;
320	},
321	else => {
322	// size 8, 11000000 .. 11000111
323	bits = ch + 192 - 280;
324	size = 8;
325	},
326	}
327	h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328	}
329	return h;
330	}
331
332	pub fn fixedDistanceEncoder() DistanceEncoder {
333	var h: DistanceEncoder = undefined;
334	for (h.codes, 0..) \|_, ch\| {
335	h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336	}
337	return h;
338	}
339
340	pub fn huffmanDistanceEncoder() DistanceEncoder {
341	var distance_freq = [1]u16{0} ** consts.distance_code_count;
342	distance_freq[0] = 1;
343	// huff_distance is a static distance encoder used for huffman only encoding.
344	// It can be reused since we will not be encoding distance values.
345	var h: DistanceEncoder = .{};
346	h.generate(distance_freq[0..], 15);
347	return h;
348	}
349
350	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351	_ = context;
352	return a.literal < b.literal;
353	}
354
355	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356	_ = context;
357	if (a.freq == b.freq) {
358	return a.literal < b.literal;
359	}
360	return a.freq < b.freq;
361	}
362
363	test "generate a Huffman code from an array of frequencies" {
364	var freqs: [19]u16 = [_]u16{
365	8, // 0
366	1, // 1
367	1, // 2
368	2, // 3
369	5, // 4
370	10, // 5
371	9, // 6
372	1, // 7
373	0, // 8
374	0, // 9
375	0, // 10
376	0, // 11
377	0, // 12
378	0, // 13
379	0, // 14
380	0, // 15
381	1, // 16
382	3, // 17
383	5, // 18
384	};
385
386	var enc = huffmanEncoder(19);
387	enc.generate(freqs[0..], 7);
388
389	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422	}
423
424	test "generate a Huffman code for the fixed literal table specific to Deflate" {
425	const enc = fixedLiteralEncoder();
426	for (enc.codes) \|c\| {
427	switch (c.len) {
428	7 => {
429	const v = @bitReverse(@as(u7, @intCast(c.code)));
430	try testing.expect(v <= 0b0010111);
431	},
432	8 => {
433	const v = @bitReverse(@as(u8, @intCast(c.code)));
434	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435	(v >= 0b11000000 and v <= 11000111));
436	},
437	9 => {
438	const v = @bitReverse(@as(u9, @intCast(c.code)));
439	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440	},
441	else => unreachable,
442	}
443	}
444	}
445
446	test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447	const enc = fixedDistanceEncoder();
448	for (enc.codes) \|c\| {
449	const v = @bitReverse(@as(u5, @intCast(c.code)));
450	try testing.expect(v <= 29);
451	try testing.expect(c.len == 5);
452	}
453	}
454
455	// Reverse bit-by-bit a N-bit code.
456	fn bitReverse(comptime T: type, value: T, n: usize) T {
457	const r = @bitReverse(value);
458	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
459	}
460
461	test bitReverse {
462	const ReverseBitsTest = struct {
463	in: u16,
464	bit_count: u5,
465	out: u16,
466	};
467
468	const reverse_bits_tests = [_]ReverseBitsTest{
469	.{ .in = 1, .bit_count = 1, .out = 1 },
470	.{ .in = 1, .bit_count = 2, .out = 2 },
471	.{ .in = 1, .bit_count = 3, .out = 4 },
472	.{ .in = 1, .bit_count = 4, .out = 8 },
473	.{ .in = 1, .bit_count = 5, .out = 16 },
474	.{ .in = 17, .bit_count = 5, .out = 17 },
475	.{ .in = 257, .bit_count = 9, .out = 257 },
476	.{ .in = 29, .bit_count = 5, .out = 23 },
477	};
478
479	for (reverse_bits_tests) \|h\| {
480	const v = bitReverse(u16, h.in, h.bit_count);
481	try std.testing.expectEqual(h.out, v);
482	}
483	}
484
485	test "fixedLiteralEncoder codes" {
486	var al = std.ArrayList(u8).init(testing.allocator);
487	defer al.deinit();
488	var bw = std.io.bitWriter(.little, al.writer());
489
490	const f = fixedLiteralEncoder();
491	for (f.codes) \|c\| {
492	try bw.writeBits(c.code, c.len);
493	}
494	try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495	}
496
497	pub const fixed_codes = [_]u8{
498	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535	0b10100011,
536	};

lib/std/compress/flate/inflate.zig deleted-570

...	@@ -1,570 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	const hfd = @import("huffman_decoder.zig");
6	const BitReader = @import("bit_reader.zig").BitReader;
7	const CircularBuffer = @import("CircularBuffer.zig");
8	const Container = @import("container.zig").Container;
9	const Token = @import("Token.zig");
10	const codegen_order = @import("consts.zig").huffman.codegen_order;
11
12	/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
13	/// `writer` stream.
14	pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
15	var d = decompressor(container, reader);
16	try d.decompress(writer);
17	}
18
19	/// Inflate decompressor for the reader type.
20	pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) {
21	return Decompressor(container, @TypeOf(reader)).init(reader);
22	}
23
24	pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type {
25	// zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
26	// gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
27	// For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
28	const lookahead: type = if (container == .zlib) u32 else u64;
29	return Inflate(container, lookahead, ReaderType);
30	}
31
32	/// Inflate decompresses deflate bit stream. Reads compressed data from reader
33	/// provided in init. Decompressed data are stored in internal hist buffer and
34	/// can be accesses iterable `next` or reader interface.
35	///
36	/// Container defines header/footer wrapper around deflate bit stream. Can be
37	/// gzip or zlib.
38	///
39	/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
40	/// * stored, non compressed, max 64k in size
41	/// * fixed, huffman codes are predefined
42	/// * dynamic, huffman code tables are encoded at the block start
43	///
44	/// `step` function runs decoder until internal `hist` buffer is full. Client
45	/// than needs to read that data in order to proceed with decoding.
46	///
47	/// Allocates 74.5K of internal buffers, most important are:
48	/// * 64K for history (CircularBuffer)
49	/// * ~10K huffman decoders (Literal and DistanceDecoder)
50	///
51	pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type {
52	assert(LookaheadType == u32 or LookaheadType == u64);
53	const BitReaderType = BitReader(LookaheadType, ReaderType);
54
55	return struct {
56	//const BitReaderType = BitReader(ReaderType);
57	const F = BitReaderType.flag;
58
59	bits: BitReaderType = .{},
60	hist: CircularBuffer = .{},
61	// Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
62	hasher: container.Hasher() = .{},
63
64	// dynamic block huffman code decoders
65	lit_dec: hfd.LiteralDecoder = .{}, // literals
66	dst_dec: hfd.DistanceDecoder = .{}, // distances
67
68	// current read state
69	bfinal: u1 = 0,
70	block_type: u2 = 0b11,
71	state: ReadState = .protocol_header,
72
73	const ReadState = enum {
74	protocol_header,
75	block_header,
76	block,
77	protocol_footer,
78	end,
79	};
80
81	const Self = @This();
82
83	pub const Error = BitReaderType.Error \|\| Container.Error \|\| hfd.Error \|\| error{
84	InvalidCode,
85	InvalidMatch,
86	InvalidBlockType,
87	WrongStoredBlockNlen,
88	InvalidDynamicBlockHeader,
89	};
90
91	pub fn init(rt: ReaderType) Self {
92	return .{ .bits = BitReaderType.init(rt) };
93	}
94
95	fn blockHeader(self: *Self) !void {
96	self.bfinal = try self.bits.read(u1);
97	self.block_type = try self.bits.read(u2);
98	}
99
100	fn storedBlock(self: *Self) !bool {
101	self.bits.alignToByte(); // skip padding until byte boundary
102	// everything after this is byte aligned in stored block
103	var len = try self.bits.read(u16);
104	const nlen = try self.bits.read(u16);
105	if (len != ~nlen) return error.WrongStoredBlockNlen;
106
107	while (len > 0) {
108	const buf = self.hist.getWritable(len);
109	try self.bits.readAll(buf);
110	len -= @intCast(buf.len);
111	}
112	return true;
113	}
114
115	fn fixedBlock(self: *Self) !bool {
116	while (!self.hist.full()) {
117	const code = try self.bits.readFixedCode();
118	switch (code) {
119	0...255 => self.hist.write(@intCast(code)),
120	256 => return true, // end of block
121	257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
122	else => return error.InvalidCode,
123	}
124	}
125	return false;
126	}
127
128	// Handles fixed block non literal (length) code.
129	// Length code is followed by 5 bits of distance code.
130	fn fixedDistanceCode(self: *Self, code: u8) !void {
131	try self.bits.fill(5 + 5 + 13);
132	const length = try self.decodeLength(code);
133	const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered \| F.reverse));
134	try self.hist.writeMatch(length, distance);
135	}
136
137	inline fn decodeLength(self: *Self, code: u8) !u16 {
138	if (code > 28) return error.InvalidCode;
139	const ml = Token.matchLength(code);
140	return if (ml.extra_bits == 0) // 0 - 5 extra bits
141	ml.base
142	else
143	ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
144	}
145
146	fn decodeDistance(self: *Self, code: u8) !u16 {
147	if (code > 29) return error.InvalidCode;
148	const md = Token.matchDistance(code);
149	return if (md.extra_bits == 0) // 0 - 13 extra bits
150	md.base
151	else
152	md.base + try self.bits.readN(md.extra_bits, F.buffered);
153	}
154
155	fn dynamicBlockHeader(self: *Self) !void {
156	const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
157	const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
158	const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
159
160	if (hlit > 286 or hdist > 30)
161	return error.InvalidDynamicBlockHeader;
162
163	// lengths for code lengths
164	var cl_lens = [_]u4{0} ** 19;
165	for (0..hclen) \|i\| {
166	cl_lens[codegen_order[i]] = try self.bits.read(u3);
167	}
168	var cl_dec: hfd.CodegenDecoder = .{};
169	try cl_dec.generate(&cl_lens);
170
171	// decoded code lengths
172	var dec_lens = [_]u4{0} ** (286 + 30);
173	var pos: usize = 0;
174	while (pos < hlit + hdist) {
175	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176	try self.bits.shift(sym.code_bits);
177	pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
178	}
179	if (pos > hlit + hdist) {
180	return error.InvalidDynamicBlockHeader;
181	}
182
183	// literal code lengths to literal decoder
184	try self.lit_dec.generate(dec_lens[0..hlit]);
185
186	// distance code lengths to distance decoder
187	try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
188	}
189
190	// Decode code length symbol to code length. Writes decoded length into
191	// lens slice starting at position pos. Returns number of positions
192	// advanced.
193	fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
194	if (pos >= lens.len)
195	return error.InvalidDynamicBlockHeader;
196
197	switch (code) {
198	0...15 => {
199	// Represent code lengths of 0 - 15
200	lens[pos] = @intCast(code);
201	return 1;
202	},
203	16 => {
204	// Copy the previous code length 3 - 6 times.
205	// The next 2 bits indicate repeat length
206	const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
207	if (pos == 0 or pos + n > lens.len)
208	return error.InvalidDynamicBlockHeader;
209	for (0..n) \|i\| {
210	lens[pos + i] = lens[pos + i - 1];
211	}
212	return n;
213	},
214	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
215	17 => return @as(u8, try self.bits.read(u3)) + 3,
216	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
217	18 => return @as(u8, try self.bits.read(u7)) + 11,
218	else => return error.InvalidDynamicBlockHeader,
219	}
220	}
221
222	// In larger archives most blocks are usually dynamic, so decompression
223	// performance depends on this function.
224	fn dynamicBlock(self: *Self) !bool {
225	// Hot path loop!
226	while (!self.hist.full()) {
227	try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
228	const sym = try self.decodeSymbol(&self.lit_dec);
229
230	switch (sym.kind) {
231	.literal => self.hist.write(sym.symbol),
232	.match => { // Decode match backreference <length, distance>
233	// fill so we can use buffered reads
234	if (LookaheadType == u32)
235	try self.bits.fill(5 + 15)
236	else
237	try self.bits.fill(5 + 15 + 13);
238	const length = try self.decodeLength(sym.symbol);
239	const dsm = try self.decodeSymbol(&self.dst_dec);
240	if (LookaheadType == u32) try self.bits.fill(13);
241	const distance = try self.decodeDistance(dsm.symbol);
242	try self.hist.writeMatch(length, distance);
243	},
244	.end_of_block => return true,
245	}
246	}
247	return false;
248	}
249
250	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
251	// decoder to find symbol for that code. We then know how many bits is
252	// used. Shift bit reader for that much bits, those bits are used. And
253	// return symbol.
254	fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
255	const sym = try decoder.find(try self.bits.peekF(u15, F.buffered \| F.reverse));
256	try self.bits.shift(sym.code_bits);
257	return sym;
258	}
259
260	fn step(self: *Self) !void {
261	switch (self.state) {
262	.protocol_header => {
263	try container.parseHeader(&self.bits);
264	self.state = .block_header;
265	},
266	.block_header => {
267	try self.blockHeader();
268	self.state = .block;
269	if (self.block_type == 2) try self.dynamicBlockHeader();
270	},
271	.block => {
272	const done = switch (self.block_type) {
273	0 => try self.storedBlock(),
274	1 => try self.fixedBlock(),
275	2 => try self.dynamicBlock(),
276	else => return error.InvalidBlockType,
277	};
278	if (done) {
279	self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
280	}
281	},
282	.protocol_footer => {
283	self.bits.alignToByte();
284	try container.parseFooter(&self.hasher, &self.bits);
285	self.state = .end;
286	},
287	.end => {},
288	}
289	}
290
291	/// Replaces the inner reader with new reader.
292	pub fn setReader(self: *Self, new_reader: ReaderType) void {
293	self.bits.forward_reader = new_reader;
294	if (self.state == .end or self.state == .protocol_footer) {
295	self.state = .protocol_header;
296	}
297	}
298
299	// Reads all compressed data from the internal reader and outputs plain
300	// (uncompressed) data to the provided writer.
301	pub fn decompress(self: *Self, writer: anytype) !void {
302	while (try self.next()) \|buf\| {
303	try writer.writeAll(buf);
304	}
305	}
306
307	/// Returns the number of bytes that have been read from the internal
308	/// reader but not yet consumed by the decompressor.
309	pub fn unreadBytes(self: Self) usize {
310	// There can be no error here: the denominator is not zero, and
311	// overflow is not possible since the type is unsigned.
312	return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
313	}
314
315	// Iterator interface
316
317	/// Can be used in iterator like loop without memcpy to another buffer:
318	/// while (try inflate.next()) \|buf\| { ... }
319	pub fn next(self: *Self) Error!?[]const u8 {
320	const out = try self.get(0);
321	if (out.len == 0) return null;
322	return out;
323	}
324
325	/// Returns decompressed data from internal sliding window buffer.
326	/// Returned buffer can be any length between 0 and `limit` bytes. 0
327	/// returned bytes means end of stream reached. With limit=0 returns as
328	/// much data it can. It newer will be more than 65536 bytes, which is
329	/// size of internal buffer.
330	pub fn get(self: *Self, limit: usize) Error![]const u8 {
331	while (true) {
332	const out = self.hist.readAtMost(limit);
333	if (out.len > 0) {
334	self.hasher.update(out);
335	return out;
336	}
337	if (self.state == .end) return out;
338	try self.step();
339	}
340	}
341
342	// Reader interface
343
344	pub const Reader = std.io.GenericReader(*Self, Error, read);
345
346	/// Returns the number of bytes read. It may be less than buffer.len.
347	/// If the number of bytes read is 0, it means end of stream.
348	/// End of stream is not an error condition.
349	pub fn read(self: *Self, buffer: []u8) Error!usize {
350	if (buffer.len == 0) return 0;
351	const out = try self.get(buffer.len);
352	@memcpy(buffer[0..out.len], out);
353	return out.len;
354	}
355
356	pub fn reader(self: *Self) Reader {
357	return .{ .context = self };
358	}
359	};
360	}
361
362	test "decompress" {
363	const cases = [_]struct {
364	in: []const u8,
365	out: []const u8,
366	}{
367	// non compressed block (type 0)
368	.{
369	.in = &[_]u8{
370	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
371	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
372	},
373	.out = "Hello world\n",
374	},
375	// fixed code block (type 1)
376	.{
377	.in = &[_]u8{
378	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
379	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
380	},
381	.out = "Hello world\n",
382	},
383	// dynamic block (type 2)
384	.{
385	.in = &[_]u8{
386	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
387	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
388	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
389	},
390	.out = "ABCDEABCD ABCDEABCD",
391	},
392	};
393	for (cases) \|c\| {
394	var fb = std.io.fixedBufferStream(c.in);
395	var al = std.ArrayList(u8).init(testing.allocator);
396	defer al.deinit();
397
398	try decompress(.raw, fb.reader(), al.writer());
399	try testing.expectEqualStrings(c.out, al.items);
400	}
401	}
402
403	test "gzip decompress" {
404	const cases = [_]struct {
405	in: []const u8,
406	out: []const u8,
407	}{
408	// non compressed block (type 0)
409	.{
410	.in = &[_]u8{
411	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
412	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
413	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
414	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
415	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
416	},
417	.out = "Hello world\n",
418	},
419	// fixed code block (type 1)
420	.{
421	.in = &[_]u8{
422	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
423	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
424	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
425	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
426	},
427	.out = "Hello world\n",
428	},
429	// dynamic block (type 2)
430	.{
431	.in = &[_]u8{
432	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
433	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
434	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
435	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
436	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
437	},
438	.out = "ABCDEABCD ABCDEABCD",
439	},
440	// gzip header with name
441	.{
442	.in = &[_]u8{
443	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
444	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
445	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
446	},
447	.out = "Hello world\n",
448	},
449	};
450	for (cases) \|c\| {
451	var fb = std.io.fixedBufferStream(c.in);
452	var al = std.ArrayList(u8).init(testing.allocator);
453	defer al.deinit();
454
455	try decompress(.gzip, fb.reader(), al.writer());
456	try testing.expectEqualStrings(c.out, al.items);
457	}
458	}
459
460	test "zlib decompress" {
461	const cases = [_]struct {
462	in: []const u8,
463	out: []const u8,
464	}{
465	// non compressed block (type 0)
466	.{
467	.in = &[_]u8{
468	0x78, 0b10_0_11100, // zlib header (2 bytes)
469	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
470	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
471	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
472	},
473	.out = "Hello world\n",
474	},
475	};
476	for (cases) \|c\| {
477	var fb = std.io.fixedBufferStream(c.in);
478	var al = std.ArrayList(u8).init(testing.allocator);
479	defer al.deinit();
480
481	try decompress(.zlib, fb.reader(), al.writer());
482	try testing.expectEqualStrings(c.out, al.items);
483	}
484	}
485
486	test "fuzzing tests" {
487	const cases = [_]struct {
488	input: []const u8,
489	out: []const u8 = "",
490	err: ?anyerror = null,
491	}{
492	.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
493	.{ .input = "empty-distance-alphabet01" },
494	.{ .input = "empty-distance-alphabet02" },
495	.{ .input = "end-of-stream", .err = error.EndOfStream },
496	.{ .input = "invalid-distance", .err = error.InvalidMatch },
497	.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
498	.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
499	.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
500	.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
501	.{ .input = "out-of-codes", .err = error.InvalidCode },
502	.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
503	.{ .input = "puff02", .err = error.EndOfStream },
504	.{ .input = "puff03", .out = &[_]u8{0xa} },
505	.{ .input = "puff04", .err = error.InvalidCode },
506	.{ .input = "puff05", .err = error.EndOfStream },
507	.{ .input = "puff06", .err = error.EndOfStream },
508	.{ .input = "puff08", .err = error.InvalidCode },
509	.{ .input = "puff09", .out = "P" },
510	.{ .input = "puff10", .err = error.InvalidCode },
511	.{ .input = "puff11", .err = error.InvalidMatch },
512	.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
513	.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
514	.{ .input = "puff14", .err = error.EndOfStream },
515	.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
516	.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
517	.{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
518	.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
519	.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
520	.{ .input = "fuzz3", .err = error.InvalidMatch },
521	.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
522	.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
523	.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
524	.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
525	.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
526	.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
527	.{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
528	.{ .input = "puff24", .err = error.IncompleteHuffmanTree },
529	.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
530	.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
531	.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
532	};
533
534	inline for (cases, 0..) \|c, case_no\| {
535	var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
536	var out = std.ArrayList(u8).init(testing.allocator);
537	defer out.deinit();
538	errdefer std.debug.print("test case failed {}\n", .{case_no});
539
540	if (c.err) \|expected_err\| {
541	try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
542	} else {
543	try decompress(.raw, in.reader(), out.writer());
544	try testing.expectEqualStrings(c.out, out.items);
545	}
546	}
547	}
548
549	test "bug 18966" {
550	const input = @embedFile("testdata/fuzz/bug_18966.input");
551	const expect = @embedFile("testdata/fuzz/bug_18966.expect");
552
553	var in = std.io.fixedBufferStream(input);
554	var out = std.ArrayList(u8).init(testing.allocator);
555	defer out.deinit();
556
557	try decompress(.gzip, in.reader(), out.writer());
558	try testing.expectEqualStrings(expect, out.items);
559	}
560
561	test "bug 19895" {
562	const input = &[_]u8{
563	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
564	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
565	};
566	var in = std.io.fixedBufferStream(input);
567	var decomp = decompressor(.raw, in.reader());
568	var buf: [0]u8 = undefined;
569	try testing.expectEqual(0, try decomp.read(&buf));
570	}

lib/std/compress/gzip.zig deleted-66

...	@@ -1,66 +0,0 @@
1	const deflate = @import("flate/deflate.zig");
2	const inflate = @import("flate/inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.gzip, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Decompressor(.gzip, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.gzip, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.gzip, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.gzip, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.gzip, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.gzip, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.gzip, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.gzip, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.gzip, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.gzip, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.gzip, writer);
65	}
66	};

lib/std/compress/zlib.zig deleted-101

...	@@ -1,101 +0,0 @@
1	const deflate = @import("flate/deflate.zig");
2	const inflate = @import("flate/inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.zlib, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Decompressor(.zlib, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.zlib, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.zlib, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.zlib, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.zlib, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.zlib, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.zlib, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.zlib, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.zlib, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.zlib, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.zlib, writer);
65	}
66	};
67
68	test "should not overshoot" {
69	const std = @import("std");
70
71	// Compressed zlib data with extra 4 bytes at the end.
72	const data = [_]u8{
73	0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
74	0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
75	0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
76	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
77	};
78
79	var stream = std.io.fixedBufferStream(data[0..]);
80	const reader = stream.reader();
81
82	var dcp = decompressor(reader);
83	var out: [128]u8 = undefined;
84
85	// Decompress
86	var n = try dcp.reader().readAll(out[0..]);
87
88	// Expected decompressed data
89	try std.testing.expectEqual(46, n);
90	try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
91
92	// Decompressor don't overshoot underlying reader.
93	// It is leaving it at the end of compressed data chunk.
94	try std.testing.expectEqual(data.len - 4, stream.getPos());
95	try std.testing.expectEqual(0, dcp.unreadBytes());
96
97	// 4 bytes after compressed chunk are available in reader.
98	n = try reader.readAll(out[0..]);
99	try std.testing.expectEqual(n, 4);
100	try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
101	}

lib/std/debug/Dwarf.zig+5-9

...	@@ -2235,18 +2235,14 @@ pub const ElfModule = struct {	...	@@ -2235,18 +2235,14 @@ pub const ElfModule = struct {
2235		2235
2236	const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size);	2236	const section_bytes = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size);
2237	sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: {	2237	sections[section_index.?] = if ((shdr.sh_flags & elf.SHF_COMPRESSED) > 0) blk: {
2238	var section_stream = std.io.fixedBufferStream(section_bytes);	2238	var section_reader: std.Io.Reader = .fixed(section_bytes);
2239	const section_reader = section_stream.reader();	2239	const chdr = section_reader.takeStruct(elf.Chdr, endian) catch continue;
2240	const chdr = section_reader.readStruct(elf.Chdr) catch continue;
2241	if (chdr.ch_type != .ZLIB) continue;	2240	if (chdr.ch_type != .ZLIB) continue;
2242		2241
2243	var zlib_stream = std.compress.zlib.decompressor(section_reader);	2242	var zlib_stream: std.compress.flate.Decompress = .init(&section_reader, .zlib, &.{});
2244		2243	const decompressed_section = zlib_stream.reader.allocRemaining(gpa, .unlimited) catch continue;
2245	const decompressed_section = try gpa.alloc(u8, chdr.ch_size);
2246	errdefer gpa.free(decompressed_section);	2244	errdefer gpa.free(decompressed_section);
2247		2245	assert(chdr.ch_size == decompressed_section.len);
2248	const read = zlib_stream.reader().readAll(decompressed_section) catch continue;
2249	assert(read == decompressed_section.len);
2250		2246
2251	break :blk .{	2247	break :blk .{
2252	.data = decompressed_section,	2248	.data = decompressed_section,

lib/std/http/Client.zig+2-7

 @@ -405,13 +405,8 @@ pub const RequestTransfer = union(enum) {
 /// The decompressor for response messages.
 pub const Compression = union(enum) {
-    pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
+    deflate: std.compress.flate.Decompress,
-    pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
+    gzip: std.compress.flate.Decompress,
-    // https://github.com/ziglang/zig/issues/18937
-    //pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
-    deflate: DeflateDecompressor,
-    gzip: GzipDecompressor,
     // https://github.com/ziglang/zig/issues/18937
     //zstd: ZstdDecompressor,
     none: void,

lib/std/http/Server.zig+2-2

...	@@ -130,8 +130,8 @@ pub const Request = struct {	...	@@ -130,8 +130,8 @@ pub const Request = struct {
130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);	130	pub const DeflateDecompressor = std.compress.zlib.Decompressor(std.io.AnyReader);
131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);	131	pub const GzipDecompressor = std.compress.gzip.Decompressor(std.io.AnyReader);
132		132
133	deflate: DeflateDecompressor,	133	deflate: std.compress.flate.Decompress,
134	gzip: GzipDecompressor,	134	gzip: std.compress.flate.Decompress,
135	zstd: std.compress.zstd.Decompress,	135	zstd: std.compress.zstd.Decompress,
136	none: void,	136	none: void,
137	};	137	};

lib/std/zip.zig+386-531

...	@@ -5,11 +5,10 @@	...	@@ -5,11 +5,10 @@
5		5
6	const builtin = @import("builtin");	6	const builtin = @import("builtin");
7	const std = @import("std");	7	const std = @import("std");
8	const testing = std.testing;	8	const File = std.fs.File;
9		9	const is_le = builtin.target.cpu.arch.endian() == .little;
10	pub const testutil = @import("zip/test.zig");	10	const Writer = std.io.Writer;
11	const File = testutil.File;	11	const Reader = std.io.Reader;
12	const FileStore = testutil.FileStore;
13		12
14	pub const CompressionMethod = enum(u16) {	13	pub const CompressionMethod = enum(u16) {
15	store = 0,	14	store = 0,
...	@@ -95,102 +94,116 @@ pub const EndRecord = extern struct {	...	@@ -95,102 +94,116 @@ pub const EndRecord = extern struct {
95	central_directory_size: u32 align(1),	94	central_directory_size: u32 align(1),
96	central_directory_offset: u32 align(1),	95	central_directory_offset: u32 align(1),
97	comment_len: u16 align(1),	96	comment_len: u16 align(1),
		97
98	pub fn need_zip64(self: EndRecord) bool {	98	pub fn need_zip64(self: EndRecord) bool {
99	return isMaxInt(self.record_count_disk) or	99	return isMaxInt(self.record_count_disk) or
100	isMaxInt(self.record_count_total) or	100	isMaxInt(self.record_count_total) or
101	isMaxInt(self.central_directory_size) or	101	isMaxInt(self.central_directory_size) or
102	isMaxInt(self.central_directory_offset);	102	isMaxInt(self.central_directory_offset);
103	}	103	}
104	};
105		104
106	/// Find and return the end record for the given seekable zip stream.	105	pub const FindBufferError = error{ ZipNoEndRecord, ZipTruncated };
107	/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and
108	/// its context must also have a `.reader()` method that returns an instance of
109	/// `std.io.GenericReader`.
110	pub fn findEndRecord(seekable_stream: anytype, stream_len: u64) !EndRecord {
111	var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
112	const record_len_max = @min(stream_len, buf.len);
113	var loaded_len: u32 = 0;
114
115	var comment_len: u16 = 0;
116	while (true) {
117	const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
118	if (record_len > record_len_max)
119	return error.ZipNoEndRecord;
120
121	if (record_len > loaded_len) {
122	const new_loaded_len = @min(loaded_len + 300, record_len_max);
123	const read_len = new_loaded_len - loaded_len;
124
125	try seekable_stream.seekTo(stream_len - @as(u64, new_loaded_len));
126	const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
127	const len = try (if (@TypeOf(seekable_stream.context) == std.fs.File) seekable_stream.context.deprecatedReader() else seekable_stream.context.reader()).readAll(read_buf);
128	if (len != read_len)
129	return error.ZipTruncated;
130	loaded_len = new_loaded_len;
131	}
132		106
133	const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];	107	/// TODO audit this logic
134	if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and	108	pub fn findBuffer(buffer: []const u8) FindBufferError!EndRecord {
135	std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)	109	const pos = std.mem.lastIndexOf(u8, buffer, &end_record_sig) orelse return error.ZipNoEndRecord;
136	{	110	if (pos + @sizeOf(EndRecord) > buffer.len) return error.EndOfStream;
137	const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);	111	const record_ptr: *EndRecord = @ptrCast(buffer[pos..][0..@sizeOf(EndRecord)]);
138	if (builtin.target.cpu.arch.endian() != .little) {	112	var record = record_ptr.*;
139	std.mem.byteSwapAllFields(@TypeOf(record.*), record);	113	if (!is_le) std.mem.byteSwapAllFields(EndRecord, &record);
		114	return record;
		115	}
		116
		117	pub const FindFileError = File.GetEndPosError \|\| File.SeekError \|\| File.ReadError \|\| error{
		118	ZipNoEndRecord,
		119	EndOfStream,
		120	};
		121
		122	pub fn findFile(fr: *File.Reader) FindFileError!EndRecord {
		123	const end_pos = try fr.getSize();
		124
		125	var buf: [@sizeOf(EndRecord) + std.math.maxInt(u16)]u8 = undefined;
		126	const record_len_max = @min(end_pos, buf.len);
		127	var loaded_len: u32 = 0;
		128	var comment_len: u16 = 0;
		129	while (true) {
		130	const record_len: u32 = @as(u32, comment_len) + @sizeOf(EndRecord);
		131	if (record_len > record_len_max)
		132	return error.ZipNoEndRecord;
		133
		134	if (record_len > loaded_len) {
		135	const new_loaded_len = @min(loaded_len + 300, record_len_max);
		136	const read_len = new_loaded_len - loaded_len;
		137
		138	try fr.seekTo(end_pos - @as(u64, new_loaded_len));
		139	const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
		140	var br = fr.interface().unbuffered();
		141	br.readSlice(read_buf) catch \|err\| switch (err) {
		142	error.ReadFailed => return fr.err.?,
		143	error.EndOfStream => return error.EndOfStream,
		144	};
		145	loaded_len = new_loaded_len;
		146	}
		147
		148	const record_bytes = buf[buf.len - record_len ..][0..@sizeOf(EndRecord)];
		149	if (std.mem.eql(u8, record_bytes[0..4], &end_record_sig) and
		150	std.mem.readInt(u16, record_bytes[20..22], .little) == comment_len)
		151	{
		152	const record: *align(1) EndRecord = @ptrCast(record_bytes.ptr);
		153	if (!is_le) std.mem.byteSwapAllFields(EndRecord, record);
		154	return record.*;
140	}	155	}
141	return record.*;	156
		157	if (comment_len == std.math.maxInt(u16))
		158	return error.ZipNoEndRecord;
		159	comment_len += 1;
142	}	160	}
		161	}
		162	};
143		163
144	if (comment_len == std.math.maxInt(u16))	164	pub const Decompress = struct {
145	return error.ZipNoEndRecord;	165	interface: Reader,
146	comment_len += 1;	166	state: union {
		167	inflate: std.compress.flate.Decompress,
		168	store: *Reader,
		169	},
		170
		171	pub fn init(reader: *Reader, method: CompressionMethod, buffer: []u8) Reader {
		172	return switch (method) {
		173	.store => .{
		174	.state = .{ .store = reader },
		175	.interface = .{
		176	.context = undefined,
		177	.vtable = &.{ .stream = streamStore },
		178	.buffer = buffer,
		179	.end = 0,
		180	.seek = 0,
		181	},
		182	},
		183	.deflate => .{
		184	.state = .{ .inflate = .init(reader, .raw) },
		185	.interface = .{
		186	.context = undefined,
		187	.vtable = &.{ .stream = streamDeflate },
		188	.buffer = buffer,
		189	.end = 0,
		190	.seek = 0,
		191	},
		192	},
		193	else => unreachable,
		194	};
147	}	195	}
148	}
149		196
150	/// Decompresses the given data from `reader` into `writer`. Stops early if more	197	fn streamStore(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
151	/// than `uncompressed_size` bytes are processed and verifies that exactly that	198	const d: *Decompress = @fieldParentPtr("interface", r);
152	/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data.	199	return d.store.read(w, limit);
153	/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method.
154	pub fn decompress(
155	method: CompressionMethod,
156	uncompressed_size: u64,
157	reader: anytype,
158	writer: anytype,
159	) !u32 {
160	var hash = std.hash.Crc32.init();
161
162	var total_uncompressed: u64 = 0;
163	switch (method) {
164	.store => {
165	var buf: [4096]u8 = undefined;
166	while (true) {
167	const len = try reader.read(&buf);
168	if (len == 0) break;
169	try writer.writeAll(buf[0..len]);
170	hash.update(buf[0..len]);
171	total_uncompressed += @intCast(len);
172	}
173	},
174	.deflate => {
175	var br = std.io.bufferedReader(reader);
176	var decompressor = std.compress.flate.decompressor(br.reader());
177	while (try decompressor.next()) \|chunk\| {
178	try writer.writeAll(chunk);
179	hash.update(chunk);
180	total_uncompressed += @intCast(chunk.len);
181	if (total_uncompressed > uncompressed_size)
182	return error.ZipUncompressSizeTooSmall;
183	}
184	if (br.end != br.start)
185	return error.ZipDeflateTruncated;
186	},
187	_ => return error.UnsupportedCompressionMethod,
188	}	200	}
189	if (total_uncompressed != uncompressed_size)
190	return error.ZipUncompressSizeMismatch;
191		201
192	return hash.final();	202	fn streamDeflate(r: Reader, w: Writer, limit: std.io.Limit) Reader.StreamError!usize {
193	}	203	const d: *Decompress = @fieldParentPtr("interface", r);
		204	return std.compress.flate.Decompress.read(&d.inflate, w, limit);
		205	}
		206	};
194		207
195	fn isBadFilename(filename: []const u8) bool {	208	fn isBadFilename(filename: []const u8) bool {
196	if (filename.len == 0 or filename[0] == '/')	209	if (filename.len == 0 or filename[0] == '/')
...	@@ -253,319 +266,332 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data	...	@@ -253,319 +266,332 @@ fn readZip64FileExtents(comptime T: type, header: T, extents: *FileExtents, data
253	}	266	}
254	}	267	}
255		268
256	pub fn Iterator(comptime SeekableStream: type) type {	269	pub const Iterator = struct {
257	return struct {	270	input: *File.Reader,
258	stream: SeekableStream,
259		271
260	cd_record_count: u64,	272	cd_record_count: u64,
261	cd_zip_offset: u64,	273	cd_zip_offset: u64,
262	cd_size: u64,	274	cd_size: u64,
263		275
264	cd_record_index: u64 = 0,	276	cd_record_index: u64 = 0,
265	cd_record_offset: u64 = 0,	277	cd_record_offset: u64 = 0,
266		278
267	const Self = @This();	279	pub fn init(input: *File.Reader) !Iterator {
		280	const end_record = try EndRecord.findFile(input);
268		281
269	pub fn init(stream: SeekableStream) !Self {	282	if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
270	const stream_len = try stream.getEndPos();	283	return error.ZipDiskRecordCountTooLarge;
271		284
272	const end_record = try findEndRecord(stream, stream_len);	285	if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
273		286	return error.ZipMultiDiskUnsupported;
274	if (!isMaxInt(end_record.record_count_disk) and end_record.record_count_disk > end_record.record_count_total)
275	return error.ZipDiskRecordCountTooLarge;
276
277	if (end_record.disk_number != 0 or end_record.central_directory_disk_number != 0)
278	return error.ZipMultiDiskUnsupported;
279		287
280	{	288	{
281	const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);	289	const counts_valid = !isMaxInt(end_record.record_count_disk) and !isMaxInt(end_record.record_count_total);
282	if (counts_valid and end_record.record_count_disk != end_record.record_count_total)	290	if (counts_valid and end_record.record_count_disk != end_record.record_count_total)
283	return error.ZipMultiDiskUnsupported;
284	}
285
286	var result = Self{
287	.stream = stream,
288	.cd_record_count = end_record.record_count_total,
289	.cd_zip_offset = end_record.central_directory_offset,
290	.cd_size = end_record.central_directory_size,
291	};
292	if (!end_record.need_zip64()) return result;
293
294	const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
295	if (locator_end_offset > stream_len)
296	return error.ZipTruncated;
297	try stream.seekTo(stream_len - locator_end_offset);
298	const locator = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndLocator64, .little);
299	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
300	return error.ZipBadLocatorSig;
301	if (locator.zip64_disk_count != 0)
302	return error.ZipUnsupportedZip64DiskCount;
303	if (locator.total_disk_count != 1)
304	return error.ZipMultiDiskUnsupported;	291	return error.ZipMultiDiskUnsupported;
		292	}
305		293
306	try stream.seekTo(locator.record_file_offset);	294	var result: Iterator = .{
307		295	.input = input,
308	const record64 = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(EndRecord64, .little);	296	.cd_record_count = end_record.record_count_total,
309		297	.cd_zip_offset = end_record.central_directory_offset,
310	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))	298	.cd_size = end_record.central_directory_size,
311	return error.ZipBadEndRecord64Sig;	299	};
312		300	if (!end_record.need_zip64()) return result;
313	if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
314	return error.ZipEndRecord64SizeTooSmall;
315	if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
316	return error.ZipEndRecord64UnhandledExtraData;
317		301
318	if (record64.version_needed_to_extract > 45)	302	const locator_end_offset: u64 = @as(u64, end_record.comment_len) + @sizeOf(EndRecord) + @sizeOf(EndLocator64);
319	return error.ZipUnsupportedVersion;	303	const stream_len = try input.getSize();
320		304
321	{	305	if (locator_end_offset > stream_len)
322	const is_multidisk = record64.disk_number != 0 or	306	return error.ZipTruncated;
323	record64.central_directory_disk_number != 0 or	307	try input.seekTo(stream_len - locator_end_offset);
324	record64.record_count_disk != record64.record_count_total;	308	const locator = input.interface.takeStructEndian(EndLocator64, .little) catch \|err\| switch (err) {
325	if (is_multidisk)	309	error.ReadFailed => return input.err.?,
326	return error.ZipMultiDiskUnsupported;	310	error.EndOfStream => return error.EndOfStream,
327	}	311	};
		312	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
		313	return error.ZipBadLocatorSig;
		314	if (locator.zip64_disk_count != 0)
		315	return error.ZipUnsupportedZip64DiskCount;
		316	if (locator.total_disk_count != 1)
		317	return error.ZipMultiDiskUnsupported;
		318
		319	try input.seekTo(locator.record_file_offset);
		320
		321	const record64 = input.interface.takeStructEndian(EndRecord64, .little) catch \|err\| switch (err) {
		322	error.ReadFailed => return input.err.?,
		323	error.EndOfStream => return error.EndOfStream,
		324	};
328		325
329	if (isMaxInt(end_record.record_count_total)) {	326	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
330	result.cd_record_count = record64.record_count_total;	327	return error.ZipBadEndRecord64Sig;
331	} else if (end_record.record_count_total != record64.record_count_total)
332	return error.Zip64RecordCountTotalMismatch;
333		328
334	if (isMaxInt(end_record.central_directory_offset)) {	329	if (record64.end_record_size < @sizeOf(EndRecord64) - 12)
335	result.cd_zip_offset = record64.central_directory_offset;	330	return error.ZipEndRecord64SizeTooSmall;
336	} else if (end_record.central_directory_offset != record64.central_directory_offset)	331	if (record64.end_record_size > @sizeOf(EndRecord64) - 12)
337	return error.Zip64CentralDirectoryOffsetMismatch;	332	return error.ZipEndRecord64UnhandledExtraData;
338		333
339	if (isMaxInt(end_record.central_directory_size)) {	334	if (record64.version_needed_to_extract > 45)
340	result.cd_size = record64.central_directory_size;	335	return error.ZipUnsupportedVersion;
341	} else if (end_record.central_directory_size != record64.central_directory_size)
342	return error.Zip64CentralDirectorySizeMismatch;
343		336
344	return result;	337	{
		338	const is_multidisk = record64.disk_number != 0 or
		339	record64.central_directory_disk_number != 0 or
		340	record64.record_count_disk != record64.record_count_total;
		341	if (is_multidisk)
		342	return error.ZipMultiDiskUnsupported;
345	}	343	}
346		344
347	pub fn next(self: *Self) !?Entry {	345	if (isMaxInt(end_record.record_count_total)) {
348	if (self.cd_record_index == self.cd_record_count) {	346	result.cd_record_count = record64.record_count_total;
349	if (self.cd_record_offset != self.cd_size)	347	} else if (end_record.record_count_total != record64.record_count_total)
350	return if (self.cd_size > self.cd_record_offset)	348	return error.Zip64RecordCountTotalMismatch;
351	error.ZipCdOversized
352	else
353	error.ZipCdUndersized;
354		349
355	return null;	350	if (isMaxInt(end_record.central_directory_offset)) {
356	}	351	result.cd_zip_offset = record64.central_directory_offset;
		352	} else if (end_record.central_directory_offset != record64.central_directory_offset)
		353	return error.Zip64CentralDirectoryOffsetMismatch;
357		354
358	const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;	355	if (isMaxInt(end_record.central_directory_size)) {
359	try self.stream.seekTo(header_zip_offset);	356	result.cd_size = record64.central_directory_size;
360	const header = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readStructEndian(CentralDirectoryFileHeader, .little);	357	} else if (end_record.central_directory_size != record64.central_directory_size)
361	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))	358	return error.Zip64CentralDirectorySizeMismatch;
362	return error.ZipBadCdOffset;
363		359
364	self.cd_record_index += 1;	360	return result;
365	self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;	361	}
366		362
367	// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file	363	pub fn next(self: *Iterator) !?Entry {
368	// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip	364	if (self.cd_record_index == self.cd_record_count) {
369	// has an undocumented version 788 but extracts just fine.	365	if (self.cd_record_offset != self.cd_size)
		366	return if (self.cd_size > self.cd_record_offset)
		367	error.ZipCdOversized
		368	else
		369	error.ZipCdUndersized;
370		370
371	if (header.flags.encrypted)	371	return null;
372	return error.ZipEncryptionUnsupported;	372	}
373	// TODO: check/verify more flags
374	if (header.disk_number != 0)
375	return error.ZipMultiDiskUnsupported;
376		373
377	var extents: FileExtents = .{	374	const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
378	.uncompressed_size = header.uncompressed_size,	375	const input = self.input;
379	.compressed_size = header.compressed_size,	376	try input.seekTo(header_zip_offset);
380	.local_file_header_offset = header.local_file_header_offset,	377	const header = input.interface.takeStructEndian(CentralDirectoryFileHeader, .little) catch \|err\| switch (err) {
381	};	378	error.ReadFailed => return input.err.?,
		379	error.EndOfStream => return error.EndOfStream,
		380	};
		381	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
		382	return error.ZipBadCdOffset;
		383
		384	self.cd_record_index += 1;
		385	self.cd_record_offset += @sizeOf(CentralDirectoryFileHeader) + header.filename_len + header.extra_len + header.comment_len;
		386
		387	// Note: checking the version_needed_to_extract doesn't seem to be helpful, i.e. the zip file
		388	// at https://github.com/ninja-build/ninja/releases/download/v1.12.0/ninja-linux.zip
		389	// has an undocumented version 788 but extracts just fine.
		390
		391	if (header.flags.encrypted)
		392	return error.ZipEncryptionUnsupported;
		393	// TODO: check/verify more flags
		394	if (header.disk_number != 0)
		395	return error.ZipMultiDiskUnsupported;
		396
		397	var extents: FileExtents = .{
		398	.uncompressed_size = header.uncompressed_size,
		399	.compressed_size = header.compressed_size,
		400	.local_file_header_offset = header.local_file_header_offset,
		401	};
382		402
383	if (header.extra_len > 0) {	403	if (header.extra_len > 0) {
384	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;	404	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
385	const extra = extra_buf[0..header.extra_len];	405	const extra = extra_buf[0..header.extra_len];
386		406
387	{	407	try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
388	try self.stream.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);	408	input.interface.readSlice(extra) catch \|err\| switch (err) {
389	const len = try (if (@TypeOf(self.stream.context) == std.fs.File) self.stream.context.deprecatedReader() else self.stream.context.reader()).readAll(extra);	409	error.ReadFailed => return input.err.?,
390	if (len != extra.len)	410	error.EndOfStream => return error.EndOfStream,
391	return error.ZipTruncated;	411	};
392	}
393		412
394	var extra_offset: usize = 0;	413	var extra_offset: usize = 0;
395	while (extra_offset + 4 <= extra.len) {	414	while (extra_offset + 4 <= extra.len) {
396	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);	415	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
397	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);	416	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
398	const end = extra_offset + 4 + data_size;	417	const end = extra_offset + 4 + data_size;
399	if (end > extra.len)	418	if (end > extra.len)
400	return error.ZipBadExtraFieldSize;	419	return error.ZipBadExtraFieldSize;
401	const data = extra[extra_offset + 4 .. end];	420	const data = extra[extra_offset + 4 .. end];
402	switch (@as(ExtraHeader, @enumFromInt(header_id))) {	421	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
403	.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),	422	.zip64_info => try readZip64FileExtents(CentralDirectoryFileHeader, header, &extents, data),
404	else => {}, // ignore	423	else => {}, // ignore
405	}
406	extra_offset = end;
407	}	424	}
		425	extra_offset = end;
408	}	426	}
409
410	return .{
411	.version_needed_to_extract = header.version_needed_to_extract,
412	.flags = header.flags,
413	.compression_method = header.compression_method,
414	.last_modification_time = header.last_modification_time,
415	.last_modification_date = header.last_modification_date,
416	.header_zip_offset = header_zip_offset,
417	.crc32 = header.crc32,
418	.filename_len = header.filename_len,
419	.compressed_size = extents.compressed_size,
420	.uncompressed_size = extents.uncompressed_size,
421	.file_offset = extents.local_file_header_offset,
422	};
423	}	427	}
424		428
425	pub const Entry = struct {	429	return .{
426	version_needed_to_extract: u16,	430	.version_needed_to_extract = header.version_needed_to_extract,
427	flags: GeneralPurposeFlags,	431	.flags = header.flags,
428	compression_method: CompressionMethod,	432	.compression_method = header.compression_method,
429	last_modification_time: u16,	433	.last_modification_time = header.last_modification_time,
430	last_modification_date: u16,	434	.last_modification_date = header.last_modification_date,
431	header_zip_offset: u64,	435	.header_zip_offset = header_zip_offset,
432	crc32: u32,	436	.crc32 = header.crc32,
433	filename_len: u32,	437	.filename_len = header.filename_len,
434	compressed_size: u64,	438	.compressed_size = extents.compressed_size,
435	uncompressed_size: u64,	439	.uncompressed_size = extents.uncompressed_size,
436	file_offset: u64,	440	.file_offset = extents.local_file_header_offset,
437		441	};
438	pub fn extract(	442	}
439	self: Entry,
440	stream: SeekableStream,
441	options: ExtractOptions,
442	filename_buf: []u8,
443	dest: std.fs.Dir,
444	) !u32 {
445	if (filename_buf.len < self.filename_len)
446	return error.ZipInsufficientBuffer;
447	const filename = filename_buf[0..self.filename_len];
448		443
		444	pub const Entry = struct {
		445	version_needed_to_extract: u16,
		446	flags: GeneralPurposeFlags,
		447	compression_method: CompressionMethod,
		448	last_modification_time: u16,
		449	last_modification_date: u16,
		450	header_zip_offset: u64,
		451	crc32: u32,
		452	filename_len: u32,
		453	compressed_size: u64,
		454	uncompressed_size: u64,
		455	file_offset: u64,
		456
		457	pub fn extract(
		458	self: Entry,
		459	stream: *File.Reader,
		460	options: ExtractOptions,
		461	filename_buf: []u8,
		462	dest: std.fs.Dir,
		463	) !u32 {
		464	if (filename_buf.len < self.filename_len)
		465	return error.ZipInsufficientBuffer;
		466	switch (self.compression_method) {
		467	.store, .deflate => {},
		468	else => return error.UnsupportedCompressionMethod,
		469	}
		470	const filename = filename_buf[0..self.filename_len];
		471	{
449	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));	472	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
		473	try stream.interface.readSlice(filename);
		474	}
450		475
451	{	476	const local_data_header_offset: u64 = local_data_header_offset: {
452	const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(filename);	477	const local_header = blk: {
453	if (len != filename.len)	478	try stream.seekTo(self.file_offset);
454	return error.ZipBadFileOffset;	479	break :blk try stream.interface.takeStructEndian(LocalFileHeader, .little);
455	}	480	};
		481	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
		482	return error.ZipBadFileOffset;
		483	if (local_header.version_needed_to_extract != self.version_needed_to_extract)
		484	return error.ZipMismatchVersionNeeded;
		485	if (local_header.last_modification_time != self.last_modification_time)
		486	return error.ZipMismatchModTime;
		487	if (local_header.last_modification_date != self.last_modification_date)
		488	return error.ZipMismatchModDate;
		489
		490	if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
		491	return error.ZipMismatchFlags;
		492	if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
		493	return error.ZipMismatchCrc32;
		494	var extents: FileExtents = .{
		495	.uncompressed_size = local_header.uncompressed_size,
		496	.compressed_size = local_header.compressed_size,
		497	.local_file_header_offset = 0,
		498	};
		499	if (local_header.extra_len > 0) {
		500	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
		501	const extra = extra_buf[0..local_header.extra_len];
456		502
457	const local_data_header_offset: u64 = local_data_header_offset: {	503	{
458	const local_header = blk: {	504	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
459	try stream.seekTo(self.file_offset);	505	try stream.interface.readSlice(extra);
460	break :blk try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readStructEndian(LocalFileHeader, .little);	506	}
461	};
462	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
463	return error.ZipBadFileOffset;
464	if (local_header.version_needed_to_extract != self.version_needed_to_extract)
465	return error.ZipMismatchVersionNeeded;
466	if (local_header.last_modification_time != self.last_modification_time)
467	return error.ZipMismatchModTime;
468	if (local_header.last_modification_date != self.last_modification_date)
469	return error.ZipMismatchModDate;
470
471	if (@as(u16, @bitCast(local_header.flags)) != @as(u16, @bitCast(self.flags)))
472	return error.ZipMismatchFlags;
473	if (local_header.crc32 != 0 and local_header.crc32 != self.crc32)
474	return error.ZipMismatchCrc32;
475	var extents: FileExtents = .{
476	.uncompressed_size = local_header.uncompressed_size,
477	.compressed_size = local_header.compressed_size,
478	.local_file_header_offset = 0,
479	};
480	if (local_header.extra_len > 0) {
481	var extra_buf: [std.math.maxInt(u16)]u8 = undefined;
482	const extra = extra_buf[0..local_header.extra_len];
483
484	{
485	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
486	const len = try (if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()).readAll(extra);
487	if (len != extra.len)
488	return error.ZipTruncated;
489	}
490		507
491	var extra_offset: usize = 0;	508	var extra_offset: usize = 0;
492	while (extra_offset + 4 <= local_header.extra_len) {	509	while (extra_offset + 4 <= local_header.extra_len) {
493	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);	510	const header_id = std.mem.readInt(u16, extra[extra_offset..][0..2], .little);
494	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);	511	const data_size = std.mem.readInt(u16, extra[extra_offset..][2..4], .little);
495	const end = extra_offset + 4 + data_size;	512	const end = extra_offset + 4 + data_size;
496	if (end > local_header.extra_len)	513	if (end > local_header.extra_len)
497	return error.ZipBadExtraFieldSize;	514	return error.ZipBadExtraFieldSize;
498	const data = extra[extra_offset + 4 .. end];	515	const data = extra[extra_offset + 4 .. end];
499	switch (@as(ExtraHeader, @enumFromInt(header_id))) {	516	switch (@as(ExtraHeader, @enumFromInt(header_id))) {
500	.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),	517	.zip64_info => try readZip64FileExtents(LocalFileHeader, local_header, &extents, data),
501	else => {}, // ignore	518	else => {}, // ignore
502	}
503	extra_offset = end;
504	}	519	}
		520	extra_offset = end;
505	}	521	}
		522	}
506		523
507	if (extents.compressed_size != 0 and	524	if (extents.compressed_size != 0 and
508	extents.compressed_size != self.compressed_size)	525	extents.compressed_size != self.compressed_size)
509	return error.ZipMismatchCompLen;	526	return error.ZipMismatchCompLen;
510	if (extents.uncompressed_size != 0 and	527	if (extents.uncompressed_size != 0 and
511	extents.uncompressed_size != self.uncompressed_size)	528	extents.uncompressed_size != self.uncompressed_size)
512	return error.ZipMismatchUncompLen;	529	return error.ZipMismatchUncompLen;
513		530
514	if (local_header.filename_len != self.filename_len)	531	if (local_header.filename_len != self.filename_len)
515	return error.ZipMismatchFilenameLen;	532	return error.ZipMismatchFilenameLen;
516		533
517	break :local_data_header_offset @as(u64, local_header.filename_len) +	534	break :local_data_header_offset @as(u64, local_header.filename_len) +
518	@as(u64, local_header.extra_len);	535	@as(u64, local_header.extra_len);
519	};	536	};
520		537
521	if (isBadFilename(filename))	538	if (isBadFilename(filename))
522	return error.ZipBadFilename;	539	return error.ZipBadFilename;
523		540
524	if (options.allow_backslashes) {	541	if (options.allow_backslashes) {
525	std.mem.replaceScalar(u8, filename, '\\', '/');	542	std.mem.replaceScalar(u8, filename, '\\', '/');
526	} else {	543	} else {
527	if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|	544	if (std.mem.indexOfScalar(u8, filename, '\\')) \|_\|
528	return error.ZipFilenameHasBackslash;	545	return error.ZipFilenameHasBackslash;
529	}	546	}
530		547
531	// All entries that end in '/' are directories	548	// All entries that end in '/' are directories
532	if (filename[filename.len - 1] == '/') {	549	if (filename[filename.len - 1] == '/') {
533	if (self.uncompressed_size != 0)	550	if (self.uncompressed_size != 0)
534	return error.ZipBadDirectorySize;	551	return error.ZipBadDirectorySize;
535	try dest.makePath(filename[0 .. filename.len - 1]);	552	try dest.makePath(filename[0 .. filename.len - 1]);
536	return std.hash.Crc32.hash(&.{});	553	return std.hash.Crc32.hash(&.{});
537	}	554	}
538		555
539	const out_file = blk: {	556	const out_file = blk: {
540	if (std.fs.path.dirname(filename)) \|dirname\| {	557	if (std.fs.path.dirname(filename)) \|dirname\| {
541	var parent_dir = try dest.makeOpenPath(dirname, .{});	558	var parent_dir = try dest.makeOpenPath(dirname, .{});
542	defer parent_dir.close();	559	defer parent_dir.close();
543		560
544	const basename = std.fs.path.basename(filename);	561	const basename = std.fs.path.basename(filename);
545	break :blk try parent_dir.createFile(basename, .{ .exclusive = true });	562	break :blk try parent_dir.createFile(basename, .{ .exclusive = true });
546	}	563	}
547	break :blk try dest.createFile(filename, .{ .exclusive = true });	564	break :blk try dest.createFile(filename, .{ .exclusive = true });
548	};	565	};
549	defer out_file.close();	566	defer out_file.close();
550	const local_data_file_offset: u64 =	567	var file_writer = out_file.writer();
551	@as(u64, self.file_offset) +	568	var file_bw = file_writer.writer(&.{});
552	@as(u64, @sizeOf(LocalFileHeader)) +	569	const local_data_file_offset: u64 =
553	local_data_header_offset;	570	@as(u64, self.file_offset) +
554	try stream.seekTo(local_data_file_offset);	571	@as(u64, @sizeOf(LocalFileHeader)) +
555	var limited_reader = std.io.limitedReader((if (@TypeOf(stream.context) == std.fs.File) stream.context.deprecatedReader() else stream.context.reader()), self.compressed_size);	572	local_data_header_offset;
556	const crc = try decompress(	573	try stream.seekTo(local_data_file_offset);
557	self.compression_method,	574	var limited_file_reader = stream.interface.limited(.limited(self.compressed_size));
558	self.uncompressed_size,	575	var file_read_buffer: [1000]u8 = undefined;
559	limited_reader.reader(),	576	var decompress_read_buffer: [1000]u8 = undefined;
560	out_file.deprecatedWriter(),	577	var limited_br = limited_file_reader.reader().buffered(&file_read_buffer);
561	);	578	var decompress: Decompress = undefined;
562	if (limited_reader.bytes_left != 0)	579	var decompress_br = decompress.readable(&limited_br, self.compression_method, &decompress_read_buffer);
563	return error.ZipDecompressTruncated;	580	const start_out = file_bw.count;
564	return crc;	581	var hash_writer = file_bw.hashed(std.hash.Crc32.init());
565	}	582	var hash_bw = hash_writer.writer(&.{});
566	};	583	decompress_br.readAll(&hash_bw, .limited(self.uncompressed_size)) catch \|err\| switch (err) {
		584	error.ReadFailed => return stream.err.?,
		585	error.WriteFailed => return file_writer.err.?,
		586	error.EndOfStream => return error.ZipDecompressTruncated,
		587	};
		588	if (limited_file_reader.remaining.nonzero()) return error.ZipDecompressTruncated;
		589	const written = file_bw.count - start_out;
		590	if (written != self.uncompressed_size) return error.ZipUncompressSizeMismatch;
		591	return hash_writer.hasher.final();
		592	}
567	};	593	};
568	}	594	};
569		595
570	// returns true if `filename` starts with `root` followed by a forward slash	596	// returns true if `filename` starts with `root` followed by a forward slash
571	fn filenameInRoot(filename: []const u8, root: []const u8) bool {	597	fn filenameInRoot(filename: []const u8, root: []const u8) bool {
...	@@ -614,17 +640,13 @@ pub const ExtractOptions = struct {	...	@@ -614,17 +640,13 @@ pub const ExtractOptions = struct {
614	diagnostics: ?*Diagnostics = null,	640	diagnostics: ?*Diagnostics = null,
615	};	641	};
616		642
617	/// Extract the zipped files inside `seekable_stream` to the given `dest` directory.	643	/// Extract the zipped files to the given `dest` directory.
618	/// Note that `seekable_stream` must be an instance of `std.io.SeekableStream` and	644	pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void {
619	/// its context must also have a `.reader()` method that returns an instance of	645	var iter = try Iterator.init(fr);
620	/// `std.io.GenericReader`.
621	pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptions) !void {
622	const SeekableStream = @TypeOf(seekable_stream);
623	var iter = try Iterator(SeekableStream).init(seekable_stream);
624		646
625	var filename_buf: [std.fs.max_path_bytes]u8 = undefined;	647	var filename_buf: [std.fs.max_path_bytes]u8 = undefined;
626	while (try iter.next()) \|entry\| {	648	while (try iter.next()) \|entry\| {
627	const crc32 = try entry.extract(seekable_stream, options, &filename_buf, dest);	649	const crc32 = try entry.extract(fr, options, &filename_buf, dest);
628	if (crc32 != entry.crc32)	650	if (crc32 != entry.crc32)
629	return error.ZipCrcMismatch;	651	return error.ZipCrcMismatch;
630	if (options.diagnostics) \|d\| {	652	if (options.diagnostics) \|d\| {
...	@@ -633,173 +655,6 @@ pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptio	...	@@ -633,173 +655,6 @@ pub fn extract(dest: std.fs.Dir, seekable_stream: anytype, options: ExtractOptio
633	}	655	}
634	}	656	}
635		657
636	fn testZip(options: ExtractOptions, comptime files: []const File, write_opt: testutil.WriteZipOptions) !void {	658	test {
637	var store: [files.len]FileStore = undefined;	659	_ = @import("zip/test.zig");
638	try testZipWithStore(options, files, write_opt, &store);
639	}
640	fn testZipWithStore(
641	options: ExtractOptions,
642	test_files: []const File,
643	write_opt: testutil.WriteZipOptions,
644	store: []FileStore,
645	) !void {
646	var zip_buf: [4096]u8 = undefined;
647	var fbs = try testutil.makeZipWithStore(&zip_buf, test_files, write_opt, store);
648
649	var tmp = testing.tmpDir(.{ .no_follow = true });
650	defer tmp.cleanup();
651	try extract(tmp.dir, fbs.seekableStream(), options);
652	try testutil.expectFiles(test_files, tmp.dir, .{});
653	}
654	fn testZipError(expected_error: anyerror, file: File, options: ExtractOptions) !void {
655	var zip_buf: [4096]u8 = undefined;
656	var store: [1]FileStore = undefined;
657	var fbs = try testutil.makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store);
658	var tmp = testing.tmpDir(.{ .no_follow = true });
659	defer tmp.cleanup();
660	try testing.expectError(expected_error, extract(tmp.dir, fbs.seekableStream(), options));
661	}
662
663	test "zip one file" {
664	try testZip(.{}, &[_]File{
665	.{ .name = "onefile.txt", .content = "Just a single file\n", .compression = .store },
666	}, .{});
667	}
668	test "zip multiple files" {
669	try testZip(.{ .allow_backslashes = true }, &[_]File{
670	.{ .name = "foo", .content = "a foo file\n", .compression = .store },
671	.{ .name = "subdir/bar", .content = "bar is this right?\nanother newline\n", .compression = .store },
672	.{ .name = "subdir\\whoa", .content = "you can do backslashes", .compression = .store },
673	.{ .name = "subdir/another/baz", .content = "bazzy mc bazzerson", .compression = .store },
674	}, .{});
675	}
676	test "zip deflated" {
677	try testZip(.{}, &[_]File{
678	.{ .name = "deflateme", .content = "This is a deflated file.\nIt should be smaller in the Zip file1\n", .compression = .deflate },
679	// TODO: re-enable this if/when we add support for deflate64
680	//.{ .name = "deflateme64", .content = "The 64k version of deflate!\n", .compression = .deflate64 },
681	.{ .name = "raw", .content = "Not all files need to be deflated in the same Zip.\n", .compression = .store },
682	}, .{});
683	}
684	test "zip verify filenames" {
685	// no empty filenames
686	try testZipError(error.ZipBadFilename, .{ .name = "", .content = "", .compression = .store }, .{});
687	// no absolute paths
688	try testZipError(error.ZipBadFilename, .{ .name = "/", .content = "", .compression = .store }, .{});
689	try testZipError(error.ZipBadFilename, .{ .name = "/foo", .content = "", .compression = .store }, .{});
690	try testZipError(error.ZipBadFilename, .{ .name = "/foo/bar", .content = "", .compression = .store }, .{});
691	// no '..' components
692	try testZipError(error.ZipBadFilename, .{ .name = "..", .content = "", .compression = .store }, .{});
693	try testZipError(error.ZipBadFilename, .{ .name = "foo/..", .content = "", .compression = .store }, .{});
694	try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/..", .content = "", .compression = .store }, .{});
695	try testZipError(error.ZipBadFilename, .{ .name = "foo/bar/../", .content = "", .compression = .store }, .{});
696	// no backslashes
697	try testZipError(error.ZipFilenameHasBackslash, .{ .name = "foo\\bar", .content = "", .compression = .store }, .{});
698	}
699
700	test "zip64" {
701	const test_files = [_]File{
702	.{ .name = "fram", .content = "fram foo fro fraba", .compression = .store },
703	.{ .name = "subdir/barro", .content = "aljdk;jal;jfd;lajkf", .compression = .store },
704	};
705
706	try testZip(.{}, &test_files, .{
707	.end = .{
708	.zip64 = .{},
709	.record_count_disk = std.math.maxInt(u16), // trigger zip64
710	},
711	});
712	try testZip(.{}, &test_files, .{
713	.end = .{
714	.zip64 = .{},
715	.record_count_total = std.math.maxInt(u16), // trigger zip64
716	},
717	});
718	try testZip(.{}, &test_files, .{
719	.end = .{
720	.zip64 = .{},
721	.record_count_disk = std.math.maxInt(u16), // trigger zip64
722	.record_count_total = std.math.maxInt(u16), // trigger zip64
723	},
724	});
725	try testZip(.{}, &test_files, .{
726	.end = .{
727	.zip64 = .{},
728	.central_directory_size = std.math.maxInt(u32), // trigger zip64
729	},
730	});
731	try testZip(.{}, &test_files, .{
732	.end = .{
733	.zip64 = .{},
734	.central_directory_offset = std.math.maxInt(u32), // trigger zip64
735	},
736	});
737	try testZip(.{}, &test_files, .{
738	.end = .{
739	.zip64 = .{},
740	.central_directory_offset = std.math.maxInt(u32), // trigger zip64
741	},
742	.local_header = .{
743	.zip64 = .{ // trigger local header zip64
744	.data_size = 16,
745	},
746	.compressed_size = std.math.maxInt(u32),
747	.uncompressed_size = std.math.maxInt(u32),
748	.extra_len = 20,
749	},
750	});
751	}
752
753	test "bad zip files" {
754	var tmp = testing.tmpDir(.{ .no_follow = true });
755	defer tmp.cleanup();
756	var zip_buf: [4096]u8 = undefined;
757
758	const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }};
759
760	{
761	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
762	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
763	}
764	{
765	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } });
766	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
767	}
768	{
769	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
770	try testing.expectError(error.ZipNoEndRecord, extract(tmp.dir, fbs.seekableStream(), .{}));
771	}
772	{
773	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } });
774	try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
775	}
776	{
777	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
778	try testing.expectError(error.ZipMultiDiskUnsupported, extract(tmp.dir, fbs.seekableStream(), .{}));
779	}
780	{
781	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } });
782	try testing.expectError(error.ZipDiskRecordCountTooLarge, extract(tmp.dir, fbs.seekableStream(), .{}));
783	}
784	{
785	var fbs = try testutil.makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } });
786	try testing.expectError(error.ZipCdOversized, extract(tmp.dir, fbs.seekableStream(), .{}));
787	}
788	{
789	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } });
790	try testing.expectError(error.ZipCdUndersized, extract(tmp.dir, fbs.seekableStream(), .{}));
791	}
792	{
793	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
794	try testing.expectError(error.ZipBadCdOffset, extract(tmp.dir, fbs.seekableStream(), .{}));
795	}
796	{
797	var fbs = try testutil.makeZip(&zip_buf, &file_a, .{
798	.end = .{
799	.zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } },
800	.central_directory_size = std.math.maxInt(u32), // trigger 64
801	},
802	});
803	try testing.expectError(error.ZipBadLocatorSig, extract(tmp.dir, fbs.seekableStream(), .{}));
804	}
805	}	660	}