Commit 58e60697e2930f4311ae9e744ae1c2877e0b69ed | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-08-25 18:03:48-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
log	58e60697e2930f4311ae9e744ae1c2877e0b69ed
tree	fd84142e826d2252f23eaae9b002ae0d3f43e341
parent	6464e0d4fc9937e154c34567891bae84c63732b9

std.compress.lzma: update for new I/O API

5 files changed, 556 insertions(+), 653 deletions(-)

lib
- std
  - compress
    lzma.zig
    lzma
    test.zig
    lzma2.zig
    xz.zig
    xz
    Decompress.zig

lib/std/compress/lzma.zig+214-232

...	@@ -4,49 +4,34 @@ const mem = std.mem;	...	@@ -4,49 +4,34 @@ const mem = std.mem;
4	const Allocator = std.mem.Allocator;	4	const Allocator = std.mem.Allocator;
5	const assert = std.debug.assert;	5	const assert = std.debug.assert;
6	const ArrayList = std.ArrayList;	6	const ArrayList = std.ArrayList;
		7	const Writer = std.Io.Writer;
		8	const Reader = std.Io.Reader;
7		9
8	pub const RangeDecoder = struct {	10	pub const RangeDecoder = struct {
9	range: u32,	11	range: u32,
10	code: u32,	12	code: u32,
11		13
12	pub fn init(reader: anytype) !RangeDecoder {	14	pub fn init(reader: *Reader) !RangeDecoder {
13	const reserved = try reader.readByte();	15	const reserved = try reader.takeByte();
14	if (reserved != 0) {	16	if (reserved != 0) return error.InvalidRangeCode;
15	return error.CorruptInput;
16	}
17	return RangeDecoder{
18	.range = 0xFFFF_FFFF,
19	.code = try reader.readInt(u32, .big),
20	};
21	}
22
23	pub fn fromParts(
24	range: u32,
25	code: u32,
26	) RangeDecoder {
27	return .{	17	return .{
28	.range = range,	18	.range = 0xFFFF_FFFF,
29	.code = code,	19	.code = try reader.takeInt(u32, .big),
30	};	20	};
31	}	21	}
32		22
33	pub fn set(self: *RangeDecoder, range: u32, code: u32) void {	23	pub fn isFinished(self: RangeDecoder) bool {
34	self.range = range;
35	self.code = code;
36	}
37
38	pub inline fn isFinished(self: RangeDecoder) bool {
39	return self.code == 0;	24	return self.code == 0;
40	}	25	}
41		26
42	inline fn normalize(self: *RangeDecoder, reader: anytype) !void {	27	fn normalize(self: RangeDecoder, reader: Reader) !void {
43	if (self.range < 0x0100_0000) {	28	if (self.range < 0x0100_0000) {
44	self.range <<= 8;	29	self.range <<= 8;
45	self.code = (self.code << 8) ^ @as(u32, try reader.readByte());	30	self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
46	}	31	}
47	}	32	}
48		33
49	inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {	34	fn getBit(self: RangeDecoder, reader: Reader) !bool {
50	self.range >>= 1;	35	self.range >>= 1;
51		36
52	const bit = self.code >= self.range;	37	const bit = self.code >= self.range;
...	@@ -57,7 +42,7 @@ pub const RangeDecoder = struct {	...	@@ -57,7 +42,7 @@ pub const RangeDecoder = struct {
57	return bit;	42	return bit;
58	}	43	}
59		44
60	pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {	45	pub fn get(self: RangeDecoder, reader: Reader, count: usize) !u32 {
61	var result: u32 = 0;	46	var result: u32 = 0;
62	var i: usize = 0;	47	var i: usize = 0;
63	while (i < count) : (i += 1)	48	while (i < count) : (i += 1)
...	@@ -65,7 +50,7 @@ pub const RangeDecoder = struct {	...	@@ -65,7 +50,7 @@ pub const RangeDecoder = struct {
65	return result;	50	return result;
66	}	51	}
67		52
68	pub inline fn decodeBit(self: RangeDecoder, reader: anytype, prob: u16, update: bool) !bool {	53	pub fn decodeBit(self: RangeDecoder, reader: Reader, prob: *u16, update: bool) !bool {
69	const bound = (self.range >> 11) * prob.*;	54	const bound = (self.range >> 11) * prob.*;
70		55
71	if (self.code < bound) {	56	if (self.code < bound) {
...	@@ -88,7 +73,7 @@ pub const RangeDecoder = struct {	...	@@ -88,7 +73,7 @@ pub const RangeDecoder = struct {
88		73
89	fn parseBitTree(	74	fn parseBitTree(
90	self: *RangeDecoder,	75	self: *RangeDecoder,
91	reader: anytype,	76	reader: *Reader,
92	num_bits: u5,	77	num_bits: u5,
93	probs: []u16,	78	probs: []u16,
94	update: bool,	79	update: bool,
...	@@ -104,7 +89,7 @@ pub const RangeDecoder = struct {	...	@@ -104,7 +89,7 @@ pub const RangeDecoder = struct {
104		89
105	pub fn parseReverseBitTree(	90	pub fn parseReverseBitTree(
106	self: *RangeDecoder,	91	self: *RangeDecoder,
107	reader: anytype,	92	reader: *Reader,
108	num_bits: u5,	93	num_bits: u5,
109	probs: []u16,	94	probs: []u16,
110	offset: usize,	95	offset: usize,
...	@@ -123,7 +108,7 @@ pub const RangeDecoder = struct {	...	@@ -123,7 +108,7 @@ pub const RangeDecoder = struct {
123	};	108	};
124		109
125	pub const Decode = struct {	110	pub const Decode = struct {
126	lzma_props: Properties,	111	properties: Properties,
127	unpacked_size: ?u64,	112	unpacked_size: ?u64,
128	literal_probs: Vec2d,	113	literal_probs: Vec2d,
129	pos_slot_decoder: [4]BitTree(6),	114	pos_slot_decoder: [4]BitTree(6),
...	@@ -141,14 +126,14 @@ pub const Decode = struct {	...	@@ -141,14 +126,14 @@ pub const Decode = struct {
141	rep_len_decoder: LenDecoder,	126	rep_len_decoder: LenDecoder,
142		127
143	pub fn init(	128	pub fn init(
144	allocator: Allocator,	129	gpa: Allocator,
145	lzma_props: Properties,	130	properties: Properties,
146	unpacked_size: ?u64,	131	unpacked_size: ?u64,
147	) !Decode {	132	) !Decode {
148	return .{	133	return .{
149	.lzma_props = lzma_props,	134	.properties = properties,
150	.unpacked_size = unpacked_size,	135	.unpacked_size = unpacked_size,
151	.literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),	136	.literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (properties.lc + properties.lp), 0x300 }),
152	.pos_slot_decoder = @splat(.{}),	137	.pos_slot_decoder = @splat(.{}),
153	.align_decoder = .{},	138	.align_decoder = .{},
154	.pos_decoders = @splat(0x400),	139	.pos_decoders = @splat(0x400),
...	@@ -165,21 +150,21 @@ pub const Decode = struct {	...	@@ -165,21 +150,21 @@ pub const Decode = struct {
165	};	150	};
166	}	151	}
167		152
168	pub fn deinit(self: *Decode, allocator: Allocator) void {	153	pub fn deinit(self: *Decode, gpa: Allocator) void {
169	self.literal_probs.deinit(allocator);	154	self.literal_probs.deinit(gpa);
170	self.* = undefined;	155	self.* = undefined;
171	}	156	}
172		157
173	pub fn resetState(self: *Decode, allocator: Allocator, new_props: Properties) !void {	158	pub fn resetState(self: *Decode, gpa: Allocator, new_props: Properties) !void {
174	new_props.validate();	159	new_props.validate();
175	if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {	160	if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
176	self.literal_probs.fill(0x400);	161	self.literal_probs.fill(0x400);
177	} else {	162	} else {
178	self.literal_probs.deinit(allocator);	163	self.literal_probs.deinit(gpa);
179	self.literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });	164	self.literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
180	}	165	}
181		166
182	self.lzma_props = new_props;	167	self.properties = new_props;
183	for (&self.pos_slot_decoder) \|*t\| t.reset();	168	for (&self.pos_slot_decoder) \|*t\| t.reset();
184	self.align_decoder.reset();	169	self.align_decoder.reset();
185	self.pos_decoders = @splat(0x400);	170	self.pos_decoders = @splat(0x400);
...	@@ -195,26 +180,23 @@ pub const Decode = struct {	...	@@ -195,26 +180,23 @@ pub const Decode = struct {
195	self.rep_len_decoder.reset();	180	self.rep_len_decoder.reset();
196	}	181	}
197		182
198	fn processNextInner(	183	fn processNext(
199	self: *Decode,	184	self: *Decode,
200	allocator: Allocator,	185	reader: *Reader,
201	reader: anytype,	186	allocating: *Writer.Allocating,
202	writer: anytype,	187	buffer: *CircularBuffer,
203	buffer: anytype,
204	decoder: *RangeDecoder,	188	decoder: *RangeDecoder,
205	update: bool,	189	update: bool,
206	) !ProcessingStatus {	190	) !ProcessingStatus {
207	const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);	191	const gpa = allocating.allocator;
		192	const writer = &allocating.writer;
		193	const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
208		194
209	if (!try decoder.decodeBit(	195	if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], update)) {
210	reader,
211	&self.is_match[(self.state << 4) + pos_state],
212	update,
213	)) {
214	const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);	196	const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
215		197
216	if (update) {	198	if (update) {
217	try buffer.appendLiteral(allocator, byte, writer);	199	try buffer.appendLiteral(gpa, byte, writer);
218		200
219	self.state = if (self.state < 4)	201	self.state = if (self.state < 4)
220	0	202	0
...	@@ -223,7 +205,7 @@ pub const Decode = struct {	...	@@ -223,7 +205,7 @@ pub const Decode = struct {
223	else	205	else
224	self.state - 6;	206	self.state - 6;
225	}	207	}
226	return .continue_;	208	return .more;
227	}	209	}
228		210
229	var len: usize = undefined;	211	var len: usize = undefined;
...	@@ -237,9 +219,9 @@ pub const Decode = struct {	...	@@ -237,9 +219,9 @@ pub const Decode = struct {
237	if (update) {	219	if (update) {
238	self.state = if (self.state < 7) 9 else 11;	220	self.state = if (self.state < 7) 9 else 11;
239	const dist = self.rep[0] + 1;	221	const dist = self.rep[0] + 1;
240	try buffer.appendLz(allocator, 1, dist, writer);	222	try buffer.appendLz(gpa, 1, dist, writer);
241	}	223	}
242	return .continue_;	224	return .more;
243	}	225	}
244	} else {	226	} else {
245	const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))	227	const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
...	@@ -293,31 +275,19 @@ pub const Decode = struct {	...	@@ -293,31 +275,19 @@ pub const Decode = struct {
293	len += 2;	275	len += 2;
294		276
295	const dist = self.rep[0] + 1;	277	const dist = self.rep[0] + 1;
296	try buffer.appendLz(allocator, len, dist, writer);	278	try buffer.appendLz(gpa, len, dist, writer);
297	}	279	}
298		280
299	return .continue_;	281	return .more;
300	}
301
302	fn processNext(
303	self: *Decode,
304	allocator: Allocator,
305	reader: anytype,
306	writer: anytype,
307	buffer: anytype,
308	decoder: *RangeDecoder,
309	) !ProcessingStatus {
310	return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
311	}	282	}
312		283
313	pub fn process(	284	pub fn process(
314	self: *Decode,	285	self: *Decode,
315	allocator: Allocator,	286	reader: *Reader,
316	reader: anytype,	287	allocating: *Writer.Allocating,
317	writer: anytype,	288	buffer: *CircularBuffer,
318	buffer: anytype,
319	decoder: *RangeDecoder,	289	decoder: *RangeDecoder,
320	) !ProcessingStatus {	290	) !void {
321	process_next: {	291	process_next: {
322	if (self.unpacked_size) \|unpacked_size\| {	292	if (self.unpacked_size) \|unpacked_size\| {
323	if (buffer.len >= unpacked_size) {	293	if (buffer.len >= unpacked_size) {
...	@@ -326,26 +296,24 @@ pub const Decode = struct {	...	@@ -326,26 +296,24 @@ pub const Decode = struct {
326	} else if (decoder.isFinished()) {	296	} else if (decoder.isFinished()) {
327	break :process_next;	297	break :process_next;
328	}	298	}
329		299	switch (try self.processNext(reader, allocating, buffer, decoder, true)) {
330	switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {	300	.more => return,
331	.continue_ => return .continue_,	301	.finished => {},
332	.finished => break :process_next,
333	}	302	}
334	}	303	}
335		304
336	if (self.unpacked_size) \|unpacked_size\| {	305	if (self.unpacked_size) \|unpacked_size\| {
337	if (buffer.len != unpacked_size) {	306	if (buffer.len != unpacked_size) return error.DecompressedSizeMismatch;
338	return error.CorruptInput;
339	}
340	}	307	}
341		308
342	return .finished;	309	try buffer.finish(&allocating.writer);
		310	self.state = math.maxInt(usize);
343	}	311	}
344		312
345	fn decodeLiteral(	313	fn decodeLiteral(
346	self: *Decode,	314	self: *Decode,
347	reader: anytype,	315	reader: *Reader,
348	buffer: anytype,	316	buffer: *CircularBuffer,
349	decoder: *RangeDecoder,	317	decoder: *RangeDecoder,
350	update: bool,	318	update: bool,
351	) !u8 {	319	) !u8 {
...	@@ -353,9 +321,9 @@ pub const Decode = struct {	...	@@ -353,9 +321,9 @@ pub const Decode = struct {
353	const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));	321	const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
354		322
355	var result: usize = 1;	323	var result: usize = 1;
356	const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +	324	const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
357	(prev_byte >> (8 - self.lzma_props.lc));	325	(prev_byte >> (8 - self.properties.lc));
358	const probs = try self.literal_probs.getMut(lit_state);	326	const probs = try self.literal_probs.get(lit_state);
359		327
360	if (self.state >= 7) {	328	if (self.state >= 7) {
361	var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));	329	var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
...	@@ -384,7 +352,7 @@ pub const Decode = struct {	...	@@ -384,7 +352,7 @@ pub const Decode = struct {
384		352
385	fn decodeDistance(	353	fn decodeDistance(
386	self: *Decode,	354	self: *Decode,
387	reader: anytype,	355	reader: *Reader,
388	decoder: *RangeDecoder,	356	decoder: *RangeDecoder,
389	length: usize,	357	length: usize,
390	update: bool,	358	update: bool,
...	@@ -415,46 +383,40 @@ pub const Decode = struct {	...	@@ -415,46 +383,40 @@ pub const Decode = struct {
415	}	383	}
416		384
417	/// A circular buffer for LZ sequences	385	/// A circular buffer for LZ sequences
418	pub const LzCircularBuffer = struct {	386	pub const CircularBuffer = struct {
419	/// Circular buffer	387	/// Circular buffer
420	buf: ArrayList(u8),	388	buf: ArrayList(u8),
421
422	/// Length of the buffer	389	/// Length of the buffer
423	dict_size: usize,	390	dict_size: usize,
424
425	/// Buffer memory limit	391	/// Buffer memory limit
426	memlimit: usize,	392	mem_limit: usize,
427
428	/// Current position	393	/// Current position
429	cursor: usize,	394	cursor: usize,
430
431	/// Total number of bytes sent through the buffer	395	/// Total number of bytes sent through the buffer
432	len: usize,	396	len: usize,
433		397
434	const Self = @This();	398	pub fn init(dict_size: usize, mem_limit: usize) CircularBuffer {
435		399	return .{
436	pub fn init(dict_size: usize, memlimit: usize) Self {
437	return Self{
438	.buf = .{},	400	.buf = .{},
439	.dict_size = dict_size,	401	.dict_size = dict_size,
440	.memlimit = memlimit,	402	.mem_limit = mem_limit,
441	.cursor = 0,	403	.cursor = 0,
442	.len = 0,	404	.len = 0,
443	};	405	};
444	}	406	}
445		407
446	pub fn get(self: Self, index: usize) u8 {	408	pub fn get(self: CircularBuffer, index: usize) u8 {
447	return if (0 <= index and index < self.buf.items.len)	409	return if (0 <= index and index < self.buf.items.len)
448	self.buf.items[index]	410	self.buf.items[index]
449	else	411	else
450	0;	412	0;
451	}	413	}
452		414
453	pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {	415	pub fn set(self: *CircularBuffer, gpa: Allocator, index: usize, value: u8) !void {
454	if (index >= self.memlimit) {	416	if (index >= self.mem_limit) {
455	return error.CorruptInput;	417	return error.CorruptInput;
456	}	418	}
457	try self.buf.ensureTotalCapacity(allocator, index + 1);	419	try self.buf.ensureTotalCapacity(gpa, index + 1);
458	while (self.buf.items.len < index) {	420	while (self.buf.items.len < index) {
459	self.buf.appendAssumeCapacity(0);	421	self.buf.appendAssumeCapacity(0);
460	}	422	}
...	@@ -462,7 +424,7 @@ pub const Decode = struct {	...	@@ -462,7 +424,7 @@ pub const Decode = struct {
462	}	424	}
463		425
464	/// Retrieve the last byte or return a default	426	/// Retrieve the last byte or return a default
465	pub fn lastOr(self: Self, lit: u8) u8 {	427	pub fn lastOr(self: CircularBuffer, lit: u8) u8 {
466	return if (self.len == 0)	428	return if (self.len == 0)
467	lit	429	lit
468	else	430	else
...	@@ -470,7 +432,7 @@ pub const Decode = struct {	...	@@ -470,7 +432,7 @@ pub const Decode = struct {
470	}	432	}
471		433
472	/// Retrieve the n-th last byte	434	/// Retrieve the n-th last byte
473	pub fn lastN(self: Self, dist: usize) !u8 {	435	pub fn lastN(self: CircularBuffer, dist: usize) !u8 {
474	if (dist > self.dict_size or dist > self.len) {	436	if (dist > self.dict_size or dist > self.len) {
475	return error.CorruptInput;	437	return error.CorruptInput;
476	}	438	}
...	@@ -481,12 +443,12 @@ pub const Decode = struct {	...	@@ -481,12 +443,12 @@ pub const Decode = struct {
481		443
482	/// Append a literal	444	/// Append a literal
483	pub fn appendLiteral(	445	pub fn appendLiteral(
484	self: *Self,	446	self: *CircularBuffer,
485	allocator: Allocator,	447	gpa: Allocator,
486	lit: u8,	448	lit: u8,
487	writer: anytype,	449	writer: *Writer,
488	) !void {	450	) !void {
489	try self.set(allocator, self.cursor, lit);	451	try self.set(gpa, self.cursor, lit);
490	self.cursor += 1;	452	self.cursor += 1;
491	self.len += 1;	453	self.len += 1;
492		454
...	@@ -499,11 +461,11 @@ pub const Decode = struct {	...	@@ -499,11 +461,11 @@ pub const Decode = struct {
499		461
500	/// Fetch an LZ sequence (length, distance) from inside the buffer	462	/// Fetch an LZ sequence (length, distance) from inside the buffer
501	pub fn appendLz(	463	pub fn appendLz(
502	self: *Self,	464	self: *CircularBuffer,
503	allocator: Allocator,	465	gpa: Allocator,
504	len: usize,	466	len: usize,
505	dist: usize,	467	dist: usize,
506	writer: anytype,	468	writer: *Writer,
507	) !void {	469	) !void {
508	if (dist > self.dict_size or dist > self.len) {	470	if (dist > self.dict_size or dist > self.len) {
509	return error.CorruptInput;	471	return error.CorruptInput;
...	@@ -513,7 +475,7 @@ pub const Decode = struct {	...	@@ -513,7 +475,7 @@ pub const Decode = struct {
513	var i: usize = 0;	475	var i: usize = 0;
514	while (i < len) : (i += 1) {	476	while (i < len) : (i += 1) {
515	const x = self.get(offset);	477	const x = self.get(offset);
516	try self.appendLiteral(allocator, x, writer);	478	try self.appendLiteral(gpa, x, writer);
517	offset += 1;	479	offset += 1;
518	if (offset == self.dict_size) {	480	if (offset == self.dict_size) {
519	offset = 0;	481	offset = 0;
...	@@ -521,15 +483,15 @@ pub const Decode = struct {	...	@@ -521,15 +483,15 @@ pub const Decode = struct {
521	}	483	}
522	}	484	}
523		485
524	pub fn finish(self: *Self, writer: anytype) !void {	486	pub fn finish(self: CircularBuffer, writer: Writer) !void {
525	if (self.cursor > 0) {	487	if (self.cursor > 0) {
526	try writer.writeAll(self.buf.items[0..self.cursor]);	488	try writer.writeAll(self.buf.items[0..self.cursor]);
527	self.cursor = 0;	489	self.cursor = 0;
528	}	490	}
529	}	491	}
530		492
531	pub fn deinit(self: *Self, allocator: Allocator) void {	493	pub fn deinit(self: *CircularBuffer, gpa: Allocator) void {
532	self.buf.deinit(allocator);	494	self.buf.deinit(gpa);
533	self.* = undefined;	495	self.* = undefined;
534	}	496	}
535	};	497	};
...	@@ -538,11 +500,9 @@ pub const Decode = struct {	...	@@ -538,11 +500,9 @@ pub const Decode = struct {
538	return struct {	500	return struct {
539	probs: [1 << num_bits]u16 = @splat(0x400),	501	probs: [1 << num_bits]u16 = @splat(0x400),
540		502
541	const Self = @This();
542
543	pub fn parse(	503	pub fn parse(
544	self: *Self,	504	self: *@This(),
545	reader: anytype,	505	reader: *Reader,
546	decoder: *RangeDecoder,	506	decoder: *RangeDecoder,
547	update: bool,	507	update: bool,
548	) !u32 {	508	) !u32 {
...	@@ -550,15 +510,15 @@ pub const Decode = struct {	...	@@ -550,15 +510,15 @@ pub const Decode = struct {
550	}	510	}
551		511
552	pub fn parseReverse(	512	pub fn parseReverse(
553	self: *Self,	513	self: *@This(),
554	reader: anytype,	514	reader: *Reader,
555	decoder: *RangeDecoder,	515	decoder: *RangeDecoder,
556	update: bool,	516	update: bool,
557	) !u32 {	517	) !u32 {
558	return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);	518	return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
559	}	519	}
560		520
561	pub fn reset(self: *Self) void {	521	pub fn reset(self: *@This()) void {
562	@memset(&self.probs, 0x400);	522	@memset(&self.probs, 0x400);
563	}	523	}
564	};	524	};
...	@@ -573,7 +533,7 @@ pub const Decode = struct {	...	@@ -573,7 +533,7 @@ pub const Decode = struct {
573		533
574	pub fn decode(	534	pub fn decode(
575	self: *LenDecoder,	535	self: *LenDecoder,
576	reader: anytype,	536	reader: *Reader,
577	decoder: *RangeDecoder,	537	decoder: *RangeDecoder,
578	pos_state: usize,	538	pos_state: usize,
579	update: bool,	539	update: bool,
...	@@ -600,45 +560,35 @@ pub const Decode = struct {	...	@@ -600,45 +560,35 @@ pub const Decode = struct {
600	data: []u16,	560	data: []u16,
601	cols: usize,	561	cols: usize,
602		562
603	const Self = @This();	563	pub fn init(gpa: Allocator, value: u16, size: struct { usize, usize }) !Vec2d {
604
605	pub fn init(allocator: Allocator, value: u16, size: struct { usize, usize }) !Self {
606	const len = try math.mul(usize, size[0], size[1]);	564	const len = try math.mul(usize, size[0], size[1]);
607	const data = try allocator.alloc(u16, len);	565	const data = try gpa.alloc(u16, len);
608	@memset(data, value);	566	@memset(data, value);
609	return Self{	567	return .{
610	.data = data,	568	.data = data,
611	.cols = size[1],	569	.cols = size[1],
612	};	570	};
613	}	571	}
614		572
615	pub fn deinit(self: *Self, allocator: Allocator) void {	573	pub fn deinit(self: *Vec2d, gpa: Allocator) void {
616	allocator.free(self.data);	574	gpa.free(self.data);
617	self.* = undefined;	575	self.* = undefined;
618	}	576	}
619		577
620	pub fn fill(self: *Self, value: u16) void {	578	pub fn fill(self: *Vec2d, value: u16) void {
621	@memset(self.data, value);	579	@memset(self.data, value);
622	}	580	}
623		581
624	inline fn _get(self: Self, row: usize) ![]u16 {	582	fn get(self: Vec2d, row: usize) ![]u16 {
625	const start_row = try math.mul(usize, row, self.cols);	583	const start_row = try math.mul(usize, row, self.cols);
626	const end_row = try math.add(usize, start_row, self.cols);	584	const end_row = try math.add(usize, start_row, self.cols);
627	return self.data[start_row..end_row];	585	return self.data[start_row..end_row];
628	}	586	}
629
630	pub fn get(self: Self, row: usize) ![]const u16 {
631	return self._get(row);
632	}
633
634	pub fn getMut(self: *Self, row: usize) ![]u16 {
635	return self._get(row);
636	}
637	};	587	};
638		588
639	pub const Options = struct {	589	pub const Options = struct {
640	unpacked_size: UnpackedSize = .read_from_header,	590	unpacked_size: UnpackedSize = .read_from_header,
641	memlimit: ?usize = null,	591	mem_limit: ?usize = null,
642	allow_incomplete: bool = false,	592	allow_incomplete: bool = false,
643	};	593	};
644		594
...	@@ -649,7 +599,7 @@ pub const Decode = struct {	...	@@ -649,7 +599,7 @@ pub const Decode = struct {
649	};	599	};
650		600
651	const ProcessingStatus = enum {	601	const ProcessingStatus = enum {
652	continue_,	602	more,
653	finished,	603	finished,
654	};	604	};
655		605
...	@@ -670,39 +620,34 @@ pub const Decode = struct {	...	@@ -670,39 +620,34 @@ pub const Decode = struct {
670	dict_size: u32,	620	dict_size: u32,
671	unpacked_size: ?u64,	621	unpacked_size: ?u64,
672		622
673	pub fn readHeader(reader: anytype, options: Options) !Params {	623	pub fn readHeader(reader: *Reader, options: Options) !Params {
674	var props = try reader.readByte();	624	var props = try reader.takeByte();
675	if (props >= 225) {	625	if (props >= 225) return error.CorruptInput;
676	return error.CorruptInput;
677	}
678		626
679	const lc = @as(u4, @intCast(props % 9));	627	const lc: u4 = @intCast(props % 9);
680	props /= 9;	628	props /= 9;
681	const lp = @as(u3, @intCast(props % 5));	629	const lp: u3 = @intCast(props % 5);
682	props /= 5;	630	props /= 5;
683	const pb = @as(u3, @intCast(props));	631	const pb: u3 = @intCast(props);
684		632
685	const dict_size_provided = try reader.readInt(u32, .little);	633	const dict_size_provided = try reader.takeInt(u32, .little);
686	const dict_size = @max(0x1000, dict_size_provided);	634	const dict_size = @max(0x1000, dict_size_provided);
687		635
688	const unpacked_size = switch (options.unpacked_size) {	636	const unpacked_size = switch (options.unpacked_size) {
689	.read_from_header => blk: {	637	.read_from_header => blk: {
690	const unpacked_size_provided = try reader.readInt(u64, .little);	638	const unpacked_size_provided = try reader.takeInt(u64, .little);
691	const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;	639	const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
692	break :blk if (marker_mandatory)	640	break :blk if (marker_mandatory) null else unpacked_size_provided;
693	null
694	else
695	unpacked_size_provided;
696	},	641	},
697	.read_header_but_use_provided => \|x\| blk: {	642	.read_header_but_use_provided => \|x\| blk: {
698	_ = try reader.readInt(u64, .little);	643	_ = try reader.takeInt(u64, .little);
699	break :blk x;	644	break :blk x;
700	},	645	},
701	.use_provided => \|x\| x,	646	.use_provided => \|x\| x,
702	};	647	};
703		648
704	return Params{	649	return .{
705	.properties = Properties{ .lc = lc, .lp = lp, .pb = pb },	650	.properties = .{ .lc = lc, .lp = lp, .pb = pb },
706	.dict_size = dict_size,	651	.dict_size = dict_size,
707	.unpacked_size = unpacked_size,	652	.unpacked_size = unpacked_size,
708	};	653	};
...	@@ -710,84 +655,121 @@ pub const Decode = struct {	...	@@ -710,84 +655,121 @@ pub const Decode = struct {
710	};	655	};
711	};	656	};
712		657
713	pub fn decompress(	658	pub const Decompress = struct {
714	allocator: Allocator,	659	gpa: Allocator,
715	reader: anytype,	660	input: *Reader,
716	) !Decompress(@TypeOf(reader)) {	661	reader: Reader,
717	return decompressWithOptions(allocator, reader, .{});	662	buffer: Decode.CircularBuffer,
718	}	663	range_decoder: RangeDecoder,
719		664	decode: Decode,
720	pub fn decompressWithOptions(	665	err: ?Error,
721	allocator: Allocator,	666
722	reader: anytype,	667	pub const Error = error{
723	options: Decode.Options,	668	OutOfMemory,
724	) !Decompress(@TypeOf(reader)) {	669	ReadFailed,
725	const params = try Decode.Params.readHeader(reader, options);	670	CorruptInput,
726	return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);	671	DecompressedSizeMismatch,
727	}	672	EndOfStream,
728		673	Overflow,
729	pub fn Decompress(comptime ReaderType: type) type {	674	};
730	return struct {
731	const Self = @This();
732
733	pub const Error =
734	ReaderType.Error \|\|
735	Allocator.Error \|\|
736	error{ CorruptInput, EndOfStream, Overflow };
737
738	pub const Reader = std.io.GenericReader(*Self, Error, read);
739		675
740	allocator: Allocator,	676	/// Takes ownership of `buffer` which may be resized with `gpa`.
741	in_reader: ReaderType,	677	///
742	to_read: std.ArrayListUnmanaged(u8),	678	/// LZMA was explicitly designed to take advantage of large heap memory
		679	/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
		680	/// this API dynamically allocates the dictionary as-needed.
		681	pub fn initParams(
		682	input: *Reader,
		683	gpa: Allocator,
		684	buffer: []u8,
		685	params: Decode.Params,
		686	mem_limit: usize,
		687	) !Decompress {
		688	return .{
		689	.gpa = gpa,
		690	.input = input,
		691	.buffer = Decode.CircularBuffer.init(params.dict_size, mem_limit),
		692	.range_decoder = try RangeDecoder.init(input),
		693	.decode = try Decode.init(gpa, params.properties, params.unpacked_size),
		694	.reader = .{
		695	.buffer = buffer,
		696	.vtable = &.{
		697	.readVec = readVec,
		698	.stream = stream,
		699	},
		700	.seek = 0,
		701	.end = 0,
		702	},
		703	.err = null,
		704	};
		705	}
743		706
744	buffer: Decode.LzCircularBuffer,	707	/// Takes ownership of `buffer` which may be resized with `gpa`.
745	decoder: RangeDecoder,	708	///
746	state: Decode,	709	/// LZMA was explicitly designed to take advantage of large heap memory
		710	/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
		711	/// this API dynamically allocates the dictionary as-needed.
		712	pub fn initOptions(
		713	input: *Reader,
		714	gpa: Allocator,
		715	buffer: []u8,
		716	options: Decode.Options,
		717	mem_limit: usize,
		718	) !Decompress {
		719	const params = try Decode.Params.readHeader(input, options);
		720	return initParams(input, gpa, buffer, params, mem_limit);
		721	}
747		722
748	pub fn init(allocator: Allocator, source: ReaderType, params: Decode.Params, memlimit: ?usize) !Self {	723	/// Reclaim ownership of the buffer passed to `init`.
749	return Self{	724	pub fn takeBuffer(d: *Decompress) []u8 {
750	.allocator = allocator,	725	const buffer = d.reader.buffer;
751	.in_reader = source,	726	d.reader.buffer = &.{};
752	.to_read = .{},	727	return buffer;
		728	}
753		729
754	.buffer = Decode.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),	730	pub fn deinit(d: *Decompress) void {
755	.decoder = try RangeDecoder.init(source),	731	const gpa = d.gpa;
756	.state = try Decode.init(allocator, params.properties, params.unpacked_size),	732	gpa.free(d.reader.buffer);
757	};	733	d.buffer.deinit(gpa);
758	}	734	d.decode.deinit(gpa);
		735	d.* = undefined;
		736	}
759		737
760	pub fn reader(self: *Self) Reader {	738	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
761	return .{ .context = self };	739	_ = data;
762	}	740	return readIndirect(r);
		741	}
763		742
764	pub fn deinit(self: *Self) void {	743	fn stream(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
765	self.to_read.deinit(self.allocator);	744	_ = w;
766	self.buffer.deinit(self.allocator);	745	_ = limit;
767	self.state.deinit(self.allocator);	746	return readIndirect(r);
768	self.* = undefined;	747	}
769	}
770		748
771	pub fn read(self: *Self, output: []u8) Error!usize {	749	fn readIndirect(r: *Reader) Reader.Error!usize {
772	const writer = self.to_read.writer(self.allocator);	750	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
773	while (self.to_read.items.len < output.len) {	751	const gpa = d.gpa;
774	switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {	752	var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
775	.continue_ => {},	753	allocating.writer.end = r.end;
776	.finished => {	754	defer r.end = allocating.writer.end;
777	try self.buffer.finish(writer);	755	if (d.decode.state == math.maxInt(usize)) return error.EndOfStream;
778	break;	756	d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch \|err\| switch (err) {
779	},	757	error.WriteFailed => {
780	}	758	d.err = error.OutOfMemory;
781	}	759	return error.ReadFailed;
782	const input = self.to_read.items;	760	},
783	const n = @min(input.len, output.len);	761	error.EndOfStream => {
784	@memcpy(output[0..n], input[0..n]);	762	d.err = error.EndOfStream;
785	std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);	763	return error.ReadFailed;
786	self.to_read.shrinkRetainingCapacity(input.len - n);	764	},
787	return n;	765	else => \|e\| {
788	}	766	d.err = e;
789	};	767	return error.ReadFailed;
790	}	768	},
		769	};
		770	return 0;
		771	}
		772	};
791		773
792	test {	774	test {
793	_ = @import("lzma/test.zig");	775	_ = @import("lzma/test.zig");

lib/std/compress/lzma/test.zig+14-12

...	@@ -1,19 +1,19 @@	...	@@ -1,19 +1,19 @@
1	const std = @import("../../std.zig");	1	const std = @import("../../std.zig");
2	const lzma = @import("../lzma.zig");	2	const lzma = std.compress.lzma;
3		3
4	fn testDecompress(compressed: []const u8) ![]u8 {	4	fn testDecompress(compressed: []const u8) ![]u8 {
5	const allocator = std.testing.allocator;	5	const gpa = std.testing.allocator;
6	var stream = std.io.fixedBufferStream(compressed);	6	var stream: std.Io.Reader = .fixed(compressed);
7	var decompressor = try lzma.decompress(allocator, stream.reader());	7
		8	var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
8	defer decompressor.deinit();	9	defer decompressor.deinit();
9	const reader = decompressor.reader();	10	return decompressor.reader.allocRemaining(gpa, .unlimited);
10	return reader.readAllAlloc(allocator, std.math.maxInt(usize));
11	}	11	}
12		12
13	fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {	13	fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14	const allocator = std.testing.allocator;	14	const gpa = std.testing.allocator;
15	const decomp = try testDecompress(compressed);	15	const decomp = try testDecompress(compressed);
16	defer allocator.free(decomp);	16	defer gpa.free(decomp);
17	try std.testing.expectEqualSlices(u8, expected, decomp);	17	try std.testing.expectEqualSlices(u8, expected, decomp);
18	}	18	}
19		19
...	@@ -89,11 +89,13 @@ test "too small uncompressed size in header" {	...	@@ -89,11 +89,13 @@ test "too small uncompressed size in header" {
89	}	89	}
90		90
91	test "reading one byte" {	91	test "reading one byte" {
		92	const gpa = std.testing.allocator;
92	const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");	93	const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
93	var stream = std.io.fixedBufferStream(compressed);	94	var stream: std.Io.Reader = .fixed(compressed);
94	var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());	95	var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
95	defer decompressor.deinit();	96	defer decompressor.deinit();
96		97
97	var buffer = [1]u8{0};	98	var buffer: [1]u8 = undefined;
98	_ = try decompressor.read(buffer[0..]);	99	try decompressor.reader.readSliceAll(&buffer);
		100	try std.testing.expectEqual(72, buffer[0]);
99	}	101	}

lib/std/compress/lzma2.zig+39-44

...	@@ -2,6 +2,8 @@ const std = @import("../std.zig");	...	@@ -2,6 +2,8 @@ const std = @import("../std.zig");
2	const Allocator = std.mem.Allocator;	2	const Allocator = std.mem.Allocator;
3	const ArrayList = std.ArrayList;	3	const ArrayList = std.ArrayList;
4	const lzma = std.compress.lzma;	4	const lzma = std.compress.lzma;
		5	const Writer = std.Io.Writer;
		6	const Reader = std.Io.Reader;
5		7
6	/// An accumulating buffer for LZ sequences	8	/// An accumulating buffer for LZ sequences
7	pub const LzAccumBuffer = struct {	9	pub const LzAccumBuffer = struct {
...	@@ -14,30 +16,28 @@ pub const LzAccumBuffer = struct {	...	@@ -14,30 +16,28 @@ pub const LzAccumBuffer = struct {
14	/// Total number of bytes sent through the buffer	16	/// Total number of bytes sent through the buffer
15	len: usize,	17	len: usize,
16		18
17	const Self = @This();	19	pub fn init(memlimit: usize) LzAccumBuffer {
18		20	return .{
19	pub fn init(memlimit: usize) Self {
20	return Self{
21	.buf = .{},	21	.buf = .{},
22	.memlimit = memlimit,	22	.memlimit = memlimit,
23	.len = 0,	23	.len = 0,
24	};	24	};
25	}	25	}
26		26
27	pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {	27	pub fn appendByte(self: *LzAccumBuffer, allocator: Allocator, byte: u8) !void {
28	try self.buf.append(allocator, byte);	28	try self.buf.append(allocator, byte);
29	self.len += 1;	29	self.len += 1;
30	}	30	}
31		31
32	/// Reset the internal dictionary	32	/// Reset the internal dictionary
33	pub fn reset(self: *Self, writer: anytype) !void {	33	pub fn reset(self: LzAccumBuffer, writer: Writer) !void {
34	try writer.writeAll(self.buf.items);	34	try writer.writeAll(self.buf.items);
35	self.buf.clearRetainingCapacity();	35	self.buf.clearRetainingCapacity();
36	self.len = 0;	36	self.len = 0;
37	}	37	}
38		38
39	/// Retrieve the last byte or return a default	39	/// Retrieve the last byte or return a default
40	pub fn lastOr(self: Self, lit: u8) u8 {	40	pub fn lastOr(self: LzAccumBuffer, lit: u8) u8 {
41	const buf_len = self.buf.items.len;	41	const buf_len = self.buf.items.len;
42	return if (buf_len == 0)	42	return if (buf_len == 0)
43	lit	43	lit
...	@@ -46,7 +46,7 @@ pub const LzAccumBuffer = struct {	...	@@ -46,7 +46,7 @@ pub const LzAccumBuffer = struct {
46	}	46	}
47		47
48	/// Retrieve the n-th last byte	48	/// Retrieve the n-th last byte
49	pub fn lastN(self: Self, dist: usize) !u8 {	49	pub fn lastN(self: LzAccumBuffer, dist: usize) !u8 {
50	const buf_len = self.buf.items.len;	50	const buf_len = self.buf.items.len;
51	if (dist > buf_len) {	51	if (dist > buf_len) {
52	return error.CorruptInput;	52	return error.CorruptInput;
...	@@ -57,10 +57,10 @@ pub const LzAccumBuffer = struct {	...	@@ -57,10 +57,10 @@ pub const LzAccumBuffer = struct {
57		57
58	/// Append a literal	58	/// Append a literal
59	pub fn appendLiteral(	59	pub fn appendLiteral(
60	self: *Self,	60	self: *LzAccumBuffer,
61	allocator: Allocator,	61	allocator: Allocator,
62	lit: u8,	62	lit: u8,
63	writer: anytype,	63	writer: *Writer,
64	) !void {	64	) !void {
65	_ = writer;	65	_ = writer;
66	if (self.len >= self.memlimit) {	66	if (self.len >= self.memlimit) {
...	@@ -72,11 +72,11 @@ pub const LzAccumBuffer = struct {	...	@@ -72,11 +72,11 @@ pub const LzAccumBuffer = struct {
72		72
73	/// Fetch an LZ sequence (length, distance) from inside the buffer	73	/// Fetch an LZ sequence (length, distance) from inside the buffer
74	pub fn appendLz(	74	pub fn appendLz(
75	self: *Self,	75	self: *LzAccumBuffer,
76	allocator: Allocator,	76	allocator: Allocator,
77	len: usize,	77	len: usize,
78	dist: usize,	78	dist: usize,
79	writer: anytype,	79	writer: *Writer,
80	) !void {	80	) !void {
81	_ = writer;	81	_ = writer;
82		82
...	@@ -95,23 +95,23 @@ pub const LzAccumBuffer = struct {	...	@@ -95,23 +95,23 @@ pub const LzAccumBuffer = struct {
95	self.len += len;	95	self.len += len;
96	}	96	}
97		97
98	pub fn finish(self: *Self, writer: anytype) !void {	98	pub fn finish(self: LzAccumBuffer, writer: Writer) !void {
99	try writer.writeAll(self.buf.items);	99	try writer.writeAll(self.buf.items);
100	self.buf.clearRetainingCapacity();	100	self.buf.clearRetainingCapacity();
101	}	101	}
102		102
103	pub fn deinit(self: *Self, allocator: Allocator) void {	103	pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
104	self.buf.deinit(allocator);	104	self.buf.deinit(allocator);
105	self.* = undefined;	105	self.* = undefined;
106	}	106	}
107	};	107	};
108		108
109	pub const Decode = struct {	109	pub const Decode = struct {
110	lzma_state: lzma.Decode,	110	lzma_decode: lzma.Decode,
111		111
112	pub fn init(allocator: Allocator) !Decode {	112	pub fn init(allocator: Allocator) !Decode {
113	return Decode{	113	return Decode{
114	.lzma_state = try lzma.Decode.init(	114	.lzma_decode = try lzma.Decode.init(
115	allocator,	115	allocator,
116	.{	116	.{
117	.lc = 0,	117	.lc = 0,
...	@@ -124,15 +124,15 @@ pub const Decode = struct {	...	@@ -124,15 +124,15 @@ pub const Decode = struct {
124	}	124	}
125		125
126	pub fn deinit(self: *Decode, allocator: Allocator) void {	126	pub fn deinit(self: *Decode, allocator: Allocator) void {
127	self.lzma_state.deinit(allocator);	127	self.lzma_decode.deinit(allocator);
128	self.* = undefined;	128	self.* = undefined;
129	}	129	}
130		130
131	pub fn decompress(	131	pub fn decompress(
132	self: *Decode,	132	self: *Decode,
133	allocator: Allocator,	133	allocator: Allocator,
134	reader: anytype,	134	reader: *Reader,
135	writer: anytype,	135	writer: *Writer,
136	) !void {	136	) !void {
137	var accum = LzAccumBuffer.init(std.math.maxInt(usize));	137	var accum = LzAccumBuffer.init(std.math.maxInt(usize));
138	defer accum.deinit(allocator);	138	defer accum.deinit(allocator);
...	@@ -154,8 +154,8 @@ pub const Decode = struct {	...	@@ -154,8 +154,8 @@ pub const Decode = struct {
154	fn parseLzma(	154	fn parseLzma(
155	self: *Decode,	155	self: *Decode,
156	allocator: Allocator,	156	allocator: Allocator,
157	reader: anytype,	157	reader: *Reader,
158	writer: anytype,	158	writer: *Writer,
159	accum: *LzAccumBuffer,	159	accum: *LzAccumBuffer,
160	status: u8,	160	status: u8,
161	) !void {	161	) !void {
...	@@ -210,7 +210,7 @@ pub const Decode = struct {	...	@@ -210,7 +210,7 @@ pub const Decode = struct {
210	}	210	}
211		211
212	if (reset.state) {	212	if (reset.state) {
213	var new_props = self.lzma_state.lzma_props;	213	var new_props = self.lzma_decode.properties;
214		214
215	if (reset.props) {	215	if (reset.props) {
216	var props = try reader.readByte();	216	var props = try reader.readByte();
...	@@ -231,16 +231,16 @@ pub const Decode = struct {	...	@@ -231,16 +231,16 @@ pub const Decode = struct {
231	new_props = .{ .lc = lc, .lp = lp, .pb = pb };	231	new_props = .{ .lc = lc, .lp = lp, .pb = pb };
232	}	232	}
233		233
234	try self.lzma_state.resetState(allocator, new_props);	234	try self.lzma_decode.resetState(allocator, new_props);
235	}	235	}
236		236
237	self.lzma_state.unpacked_size = unpacked_size + accum.len;	237	self.lzma_decode.unpacked_size = unpacked_size + accum.len;
238		238
239	var counter = std.io.countingReader(reader);	239	var counter = std.io.countingReader(reader);
240	const counter_reader = counter.reader();	240	const counter_reader = counter.reader();
241		241
242	var rangecoder = try lzma.RangeDecoder.init(counter_reader);	242	var rangecoder = try lzma.RangeDecoder.init(counter_reader);
243	while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}	243	while (try self.lzma_decode.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
244		244
245	if (counter.bytes_read != packed_size) {	245	if (counter.bytes_read != packed_size) {
246	return error.CorruptInput;	246	return error.CorruptInput;
...	@@ -249,8 +249,8 @@ pub const Decode = struct {	...	@@ -249,8 +249,8 @@ pub const Decode = struct {
249		249
250	fn parseUncompressed(	250	fn parseUncompressed(
251	allocator: Allocator,	251	allocator: Allocator,
252	reader: anytype,	252	reader: *Reader,
253	writer: anytype,	253	writer: *Writer,
254	accum: *LzAccumBuffer,	254	accum: *LzAccumBuffer,
255	reset_dict: bool,	255	reset_dict: bool,
256	) !void {	256	) !void {
...	@@ -267,24 +267,19 @@ pub const Decode = struct {	...	@@ -267,24 +267,19 @@ pub const Decode = struct {
267	}	267	}
268	};	268	};
269		269
270	pub fn decompress(	270	test "decompress hello world stream" {
271	allocator: Allocator,
272	reader: anytype,
273	writer: anytype,
274	) !void {
275	var decoder = try Decode.init(allocator);
276	defer decoder.deinit(allocator);
277	return decoder.decompress(allocator, reader, writer);
278	}
279
280	test {
281	const expected = "Hello\nWorld!\n";	271	const expected = "Hello\nWorld!\n";
282	const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };	272	const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
283		273
284	const allocator = std.testing.allocator;	274	const gpa = std.testing.allocator;
285	var decomp = std.array_list.Managed(u8).init(allocator);	275
286	defer decomp.deinit();	276	var stream: std.Io.Reader = .fixed(compressed);
287	var stream = std.io.fixedBufferStream(compressed);	277
288	try decompress(allocator, stream.reader(), decomp.writer());	278	var decode = try Decode.init(gpa, &stream);
289	try std.testing.expectEqualSlices(u8, expected, decomp.items);	279	defer decode.deinit(gpa);
		280
		281	const result = try decode.reader.allocRemaining(gpa, .unlimited);
		282	defer gpa.free(result);
		283
		284	try std.testing.expectEqualStrings(expected, result);
290	}	285	}

lib/std/compress/xz.zig+1-365

...	@@ -1,368 +1,4 @@	...	@@ -1,368 +1,4 @@
1	const std = @import("std");	1	pub const Decompress = @import("xz/Decompress.zig");
2	const Allocator = std.mem.Allocator;
3	const ArrayList = std.ArrayList;
4	const Crc32 = std.hash.Crc32;
5	const Crc64 = std.hash.crc.Crc64Xz;
6	const Sha256 = std.crypto.hash.sha2.Sha256;
7	const lzma2 = std.compress.lzma2;
8
9	pub const Check = enum(u4) {
10	none = 0x00,
11	crc32 = 0x01,
12	crc64 = 0x04,
13	sha256 = 0x0A,
14	_,
15	};
16
17	fn readStreamFlags(reader: anytype, check: *Check) !void {
18	const reserved1 = try reader.readByte();
19	if (reserved1 != 0) return error.CorruptInput;
20	const byte = try reader.readByte();
21	if ((byte >> 4) != 0) return error.CorruptInput;
22	check.* = @enumFromInt(@as(u4, @truncate(byte)));
23	}
24
25	pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
26	return Decompress(@TypeOf(reader)).init(allocator, reader);
27	}
28
29	pub fn Decompress(comptime ReaderType: type) type {
30	return struct {
31	const Self = @This();
32
33	pub const Error = ReaderType.Error \|\| Decoder(ReaderType).Error;
34	pub const Reader = std.io.GenericReader(*Self, Error, read);
35
36	allocator: Allocator,
37	block_decoder: Decoder(ReaderType),
38	in_reader: ReaderType,
39
40	fn init(allocator: Allocator, source: ReaderType) !Self {
41	const magic = try source.readBytesNoEof(6);
42	if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
43	return error.BadHeader;
44
45	var check: Check = undefined;
46	const hash_a = blk: {
47	var hasher = hashedReader(source, Crc32.init());
48	try readStreamFlags(hasher.reader(), &check);
49	break :blk hasher.hasher.final();
50	};
51
52	const hash_b = try source.readInt(u32, .little);
53	if (hash_a != hash_b)
54	return error.WrongChecksum;
55
56	return Self{
57	.allocator = allocator,
58	.block_decoder = try decoder(allocator, source, check),
59	.in_reader = source,
60	};
61	}
62
63	pub fn deinit(self: *Self) void {
64	self.block_decoder.deinit();
65	}
66
67	pub fn reader(self: *Self) Reader {
68	return .{ .context = self };
69	}
70
71	pub fn read(self: *Self, buffer: []u8) Error!usize {
72	if (buffer.len == 0)
73	return 0;
74
75	const r = try self.block_decoder.read(buffer);
76	if (r != 0)
77	return r;
78
79	const index_size = blk: {
80	var hasher = hashedReader(self.in_reader, Crc32.init());
81	hasher.hasher.update(&[1]u8{0x00});
82
83	var counter = std.io.countingReader(hasher.reader());
84	counter.bytes_read += 1;
85
86	const counting_reader = counter.reader();
87
88	const record_count = try std.leb.readUleb128(u64, counting_reader);
89	if (record_count != self.block_decoder.block_count)
90	return error.CorruptInput;
91
92	var i: usize = 0;
93	while (i < record_count) : (i += 1) {
94	// TODO: validate records
95	_ = try std.leb.readUleb128(u64, counting_reader);
96	_ = try std.leb.readUleb128(u64, counting_reader);
97	}
98
99	while (counter.bytes_read % 4 != 0) {
100	if (try counting_reader.readByte() != 0)
101	return error.CorruptInput;
102	}
103
104	const hash_a = hasher.hasher.final();
105	const hash_b = try counting_reader.readInt(u32, .little);
106	if (hash_a != hash_b)
107	return error.WrongChecksum;
108
109	break :blk counter.bytes_read;
110	};
111
112	const hash_a = try self.in_reader.readInt(u32, .little);
113
114	const hash_b = blk: {
115	var hasher = hashedReader(self.in_reader, Crc32.init());
116	const hashed_reader = hasher.reader();
117
118	const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
119	if (backward_size != index_size)
120	return error.CorruptInput;
121
122	var check: Check = undefined;
123	try readStreamFlags(hashed_reader, &check);
124
125	break :blk hasher.hasher.final();
126	};
127
128	if (hash_a != hash_b)
129	return error.WrongChecksum;
130
131	const magic = try self.in_reader.readBytesNoEof(2);
132	if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
133	return error.CorruptInput;
134
135	return 0;
136	}
137	};
138	}
139
140	pub fn HashedReader(ReaderType: type, HasherType: type) type {
141	return struct {
142	child_reader: ReaderType,
143	hasher: HasherType,
144
145	pub const Error = ReaderType.Error;
146	pub const Reader = std.io.GenericReader(*@This(), Error, read);
147
148	pub fn read(self: *@This(), buf: []u8) Error!usize {
149	const amt = try self.child_reader.read(buf);
150	self.hasher.update(buf[0..amt]);
151	return amt;
152	}
153
154	pub fn reader(self: *@This()) Reader {
155	return .{ .context = self };
156	}
157	};
158	}
159
160	pub fn hashedReader(
161	reader: anytype,
162	hasher: anytype,
163	) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
164	return .{ .child_reader = reader, .hasher = hasher };
165	}
166
167	const DecodeError = error{
168	CorruptInput,
169	EndOfStream,
170	EndOfStreamWithNoError,
171	WrongChecksum,
172	Unsupported,
173	Overflow,
174	};
175
176	pub fn decoder(allocator: Allocator, reader: anytype, check: Check) !Decoder(@TypeOf(reader)) {
177	return Decoder(@TypeOf(reader)).init(allocator, reader, check);
178	}
179
180	pub fn Decoder(comptime ReaderType: type) type {
181	return struct {
182	const Self = @This();
183	pub const Error =
184	ReaderType.Error \|\|
185	DecodeError \|\|
186	Allocator.Error;
187	pub const Reader = std.io.GenericReader(*Self, Error, read);
188
189	allocator: Allocator,
190	inner_reader: ReaderType,
191	check: Check,
192	err: ?Error,
193	to_read: ArrayList(u8),
194	read_pos: usize,
195	block_count: usize,
196
197	fn init(allocator: Allocator, in_reader: ReaderType, check: Check) !Self {
198	return Self{
199	.allocator = allocator,
200	.inner_reader = in_reader,
201	.check = check,
202	.err = null,
203	.to_read = .{},
204	.read_pos = 0,
205	.block_count = 0,
206	};
207	}
208
209	pub fn deinit(self: *Self) void {
210	self.to_read.deinit(self.allocator);
211	}
212
213	pub fn reader(self: *Self) Reader {
214	return .{ .context = self };
215	}
216
217	pub fn read(self: *Self, output: []u8) Error!usize {
218	while (true) {
219	const unread_len = self.to_read.items.len - self.read_pos;
220	if (unread_len > 0) {
221	const n = @min(unread_len, output.len);
222	@memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
223	self.read_pos += n;
224	return n;
225	}
226	if (self.err) \|e\| {
227	if (e == DecodeError.EndOfStreamWithNoError) {
228	return 0;
229	}
230	return e;
231	}
232	if (self.read_pos > 0) {
233	self.to_read.shrinkRetainingCapacity(0);
234	self.read_pos = 0;
235	}
236	self.readBlock() catch \|e\| {
237	self.err = e;
238	};
239	}
240	}
241
242	fn readBlock(self: *Self) Error!void {
243	var block_counter = std.io.countingReader(self.inner_reader);
244	const block_reader = block_counter.reader();
245
246	var packed_size: ?u64 = null;
247	var unpacked_size: ?u64 = null;
248
249	// Block Header
250	{
251	var header_hasher = hashedReader(block_reader, Crc32.init());
252	const header_reader = header_hasher.reader();
253
254	const header_size = @as(u64, try header_reader.readByte()) * 4;
255	if (header_size == 0)
256	return error.EndOfStreamWithNoError;
257
258	const Flags = packed struct(u8) {
259	last_filter_index: u2,
260	reserved: u4,
261	has_packed_size: bool,
262	has_unpacked_size: bool,
263	};
264
265	const flags = @as(Flags, @bitCast(try header_reader.readByte()));
266	const filter_count = @as(u3, flags.last_filter_index) + 1;
267	if (filter_count > 1)
268	return error.Unsupported;
269
270	if (flags.has_packed_size)
271	packed_size = try std.leb.readUleb128(u64, header_reader);
272
273	if (flags.has_unpacked_size)
274	unpacked_size = try std.leb.readUleb128(u64, header_reader);
275
276	const FilterId = enum(u64) {
277	lzma2 = 0x21,
278	_,
279	};
280
281	const filter_id = @as(
282	FilterId,
283	@enumFromInt(try std.leb.readUleb128(u64, header_reader)),
284	);
285
286	if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
287	return error.CorruptInput;
288
289	if (filter_id != .lzma2)
290	return error.Unsupported;
291
292	const properties_size = try std.leb.readUleb128(u64, header_reader);
293	if (properties_size != 1)
294	return error.CorruptInput;
295
296	// TODO: use filter properties
297	_ = try header_reader.readByte();
298
299	while (block_counter.bytes_read != header_size) {
300	if (try header_reader.readByte() != 0)
301	return error.CorruptInput;
302	}
303
304	const hash_a = header_hasher.hasher.final();
305	const hash_b = try header_reader.readInt(u32, .little);
306	if (hash_a != hash_b)
307	return error.WrongChecksum;
308	}
309
310	// Compressed Data
311	var packed_counter = std.io.countingReader(block_reader);
312	try lzma2.decompress(
313	self.allocator,
314	packed_counter.reader(),
315	self.to_read.writer(self.allocator),
316	);
317
318	if (packed_size) \|s\| {
319	if (s != packed_counter.bytes_read)
320	return error.CorruptInput;
321	}
322
323	const unpacked_bytes = self.to_read.items;
324	if (unpacked_size) \|s\| {
325	if (s != unpacked_bytes.len)
326	return error.CorruptInput;
327	}
328
329	// Block Padding
330	while (block_counter.bytes_read % 4 != 0) {
331	if (try block_reader.readByte() != 0)
332	return error.CorruptInput;
333	}
334
335	switch (self.check) {
336	.none => {},
337	.crc32 => {
338	const hash_a = Crc32.hash(unpacked_bytes);
339	const hash_b = try self.inner_reader.readInt(u32, .little);
340	if (hash_a != hash_b)
341	return error.WrongChecksum;
342	},
343	.crc64 => {
344	const hash_a = Crc64.hash(unpacked_bytes);
345	const hash_b = try self.inner_reader.readInt(u64, .little);
346	if (hash_a != hash_b)
347	return error.WrongChecksum;
348	},
349	.sha256 => {
350	var hash_a: [Sha256.digest_length]u8 = undefined;
351	Sha256.hash(unpacked_bytes, &hash_a, .{});
352
353	var hash_b: [Sha256.digest_length]u8 = undefined;
354	try self.inner_reader.readNoEof(&hash_b);
355
356	if (!std.mem.eql(u8, &hash_a, &hash_b))
357	return error.WrongChecksum;
358	},
359	else => return error.Unsupported,
360	}
361
362	self.block_count += 1;
363	}
364	};
365	}
366		2
367	test {	3	test {
368	_ = @import("xz/test.zig");	4	_ = @import("xz/test.zig");

lib/std/compress/xz/Decompress.zig created+288

...	@@ -0,0 +1,288 @@
1	const Decompress = @This();
2	const std = @import("../../std.zig");
3	const Allocator = std.mem.Allocator;
4	const ArrayList = std.ArrayList;
5	const Crc32 = std.hash.Crc32;
6	const Crc64 = std.hash.crc.Crc64Xz;
7	const Sha256 = std.crypto.hash.sha2.Sha256;
8	const lzma2 = std.compress.lzma2;
9	const Writer = std.Io.Writer;
10	const Reader = std.Io.Reader;
11
12	/// Underlying compressed data stream to pull bytes from.
13	input: *Reader,
14	/// Uncompressed bytes output by this stream implementation.
15	reader: Reader,
16	gpa: Allocator,
17	check: Check,
18	block_count: usize,
19	err: ?Error,
20
21	pub const Error = error{
22	ReadFailed,
23	OutOfMemory,
24	CorruptInput,
25	EndOfStream,
26	WrongChecksum,
27	Unsupported,
28	Overflow,
29	};
30
31	pub const Check = enum(u4) {
32	none = 0x00,
33	crc32 = 0x01,
34	crc64 = 0x04,
35	sha256 = 0x0A,
36	_,
37	};
38
39	pub const StreamFlags = packed struct(u16) {
40	null: u8 = 0,
41	check: Check,
42	reserved: u4 = 0,
43	};
44
45	pub const InitError = error{
46	NotXzStream,
47	WrongChecksum,
48	};
49
50	/// XZ uses a series of LZMA2 blocks which each specify a dictionary size
51	/// anywhere from 4K to 4G. Thus, this API dynamically allocates the dictionary
52	/// as-needed.
53	pub fn init(
54	input: *Reader,
55	gpa: Allocator,
56	/// Decompress takes ownership of this buffer and resizes it with `gpa`.
57	buffer: []u8,
58	) Decompress {
59	const magic = try input.takeBytes(6);
60	if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
61	return error.NotXzStream;
62
63	const actual_hash = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
64	const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
65	const stored_hash = try input.readInt(u32, .little);
66	if (actual_hash != stored_hash) return error.WrongChecksum;
67
68	return .{
69	.input = input,
70	.reader = .{
71	.vtable = &.{
72	.stream = stream,
73	.readVec = readVec,
74	},
75	.buffer = buffer,
76	.seek = 0,
77	.end = 0,
78	},
79	.gpa = gpa,
80	.check = stream_flags.check,
81	.block_count = 0,
82	.err = null,
83	};
84	}
85
86	fn stream(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
87	_ = w;
88	_ = limit;
89	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
90	_ = d;
91	@panic("TODO");
92	}
93
94	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
95	_ = data;
96	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
97	_ = d;
98	@panic("TODO");
99	}
100
101	// if (buffer.len == 0)
102	// return 0;
103	//
104	// const r = try self.block_decode.read(buffer);
105	// if (r != 0)
106	// return r;
107	//
108	// const index_size = blk: {
109	// var hasher = hashedReader(self.in_reader, Crc32.init());
110	// hasher.hasher.update(&[1]u8{0x00});
111	//
112	// var counter = std.io.countingReader(hasher.reader());
113	// counter.bytes_read += 1;
114	//
115	// const counting_reader = counter.reader();
116	//
117	// const record_count = try std.leb.readUleb128(u64, counting_reader);
118	// if (record_count != self.block_decode.block_count)
119	// return error.CorruptInput;
120	//
121	// var i: usize = 0;
122	// while (i < record_count) : (i += 1) {
123	// // TODO: validate records
124	// _ = try std.leb.readUleb128(u64, counting_reader);
125	// _ = try std.leb.readUleb128(u64, counting_reader);
126	// }
127	//
128	// while (counter.bytes_read % 4 != 0) {
129	// if (try counting_reader.readByte() != 0)
130	// return error.CorruptInput;
131	// }
132	//
133	// const hash_a = hasher.hasher.final();
134	// const hash_b = try counting_reader.readInt(u32, .little);
135	// if (hash_a != hash_b)
136	// return error.WrongChecksum;
137	//
138	// break :blk counter.bytes_read;
139	// };
140	//
141	// const hash_a = try self.in_reader.readInt(u32, .little);
142	//
143	// const hash_b = blk: {
144	// var hasher = hashedReader(self.in_reader, Crc32.init());
145	// const hashed_reader = hasher.reader();
146	//
147	// const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
148	// if (backward_size != index_size)
149	// return error.CorruptInput;
150	//
151	// var check: Check = undefined;
152	// try readStreamFlags(hashed_reader, &check);
153	//
154	// break :blk hasher.hasher.final();
155	// };
156	//
157	// if (hash_a != hash_b)
158	// return error.WrongChecksum;
159	//
160	// const magic = try self.in_reader.readBytesNoEof(2);
161	// if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
162	// return error.CorruptInput;
163	//
164	// return 0;
165	//}
166
167	//fn readBlock(self: *BlockDecode) Error!void {
168	// var block_counter = std.io.countingReader(self.inner_reader);
169	// const block_reader = block_counter.reader();
170	//
171	// var packed_size: ?u64 = null;
172	// var unpacked_size: ?u64 = null;
173	//
174	// // Block Header
175	// {
176	// var header_hasher = hashedReader(block_reader, Crc32.init());
177	// const header_reader = header_hasher.reader();
178	//
179	// const header_size = @as(u64, try header_reader.readByte()) * 4;
180	// if (header_size == 0)
181	// return error.EndOfStreamWithNoError;
182	//
183	// const Flags = packed struct(u8) {
184	// last_filter_index: u2,
185	// reserved: u4,
186	// has_packed_size: bool,
187	// has_unpacked_size: bool,
188	// };
189	//
190	// const flags = @as(Flags, @bitCast(try header_reader.readByte()));
191	// const filter_count = @as(u3, flags.last_filter_index) + 1;
192	// if (filter_count > 1)
193	// return error.Unsupported;
194	//
195	// if (flags.has_packed_size)
196	// packed_size = try std.leb.readUleb128(u64, header_reader);
197	//
198	// if (flags.has_unpacked_size)
199	// unpacked_size = try std.leb.readUleb128(u64, header_reader);
200	//
201	// const FilterId = enum(u64) {
202	// lzma2 = 0x21,
203	// _,
204	// };
205	//
206	// const filter_id = @as(
207	// FilterId,
208	// @enumFromInt(try std.leb.readUleb128(u64, header_reader)),
209	// );
210	//
211	// if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
212	// return error.CorruptInput;
213	//
214	// if (filter_id != .lzma2)
215	// return error.Unsupported;
216	//
217	// const properties_size = try std.leb.readUleb128(u64, header_reader);
218	// if (properties_size != 1)
219	// return error.CorruptInput;
220	//
221	// // TODO: use filter properties
222	// _ = try header_reader.readByte();
223	//
224	// while (block_counter.bytes_read != header_size) {
225	// if (try header_reader.readByte() != 0)
226	// return error.CorruptInput;
227	// }
228	//
229	// const hash_a = header_hasher.hasher.final();
230	// const hash_b = try header_reader.readInt(u32, .little);
231	// if (hash_a != hash_b)
232	// return error.WrongChecksum;
233	// }
234	//
235	// // Compressed Data
236	// var packed_counter = std.io.countingReader(block_reader);
237	// try lzma2.decompress(
238	// self.allocator,
239	// packed_counter.reader(),
240	// self.to_read.writer(self.allocator),
241	// );
242	//
243	// if (packed_size) \|s\| {
244	// if (s != packed_counter.bytes_read)
245	// return error.CorruptInput;
246	// }
247	//
248	// const unpacked_bytes = self.to_read.items;
249	// if (unpacked_size) \|s\| {
250	// if (s != unpacked_bytes.len)
251	// return error.CorruptInput;
252	// }
253	//
254	// // Block Padding
255	// while (block_counter.bytes_read % 4 != 0) {
256	// if (try block_reader.readByte() != 0)
257	// return error.CorruptInput;
258	// }
259	//
260	// switch (self.check) {
261	// .none => {},
262	// .crc32 => {
263	// const hash_a = Crc32.hash(unpacked_bytes);
264	// const hash_b = try self.inner_reader.readInt(u32, .little);
265	// if (hash_a != hash_b)
266	// return error.WrongChecksum;
267	// },
268	// .crc64 => {
269	// const hash_a = Crc64.hash(unpacked_bytes);
270	// const hash_b = try self.inner_reader.readInt(u64, .little);
271	// if (hash_a != hash_b)
272	// return error.WrongChecksum;
273	// },
274	// .sha256 => {
275	// var hash_a: [Sha256.digest_length]u8 = undefined;
276	// Sha256.hash(unpacked_bytes, &hash_a, .{});
277	//
278	// var hash_b: [Sha256.digest_length]u8 = undefined;
279	// try self.inner_reader.readNoEof(&hash_b);
280	//
281	// if (!std.mem.eql(u8, &hash_a, &hash_b))
282	// return error.WrongChecksum;
283	// },
284	// else => return error.Unsupported,
285	// }
286	//
287	// self.block_count += 1;
288	//}