authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-25 20:24:19-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
loga8ae6c2f4265a66c7a63d788a13549c48a1dd8c0
treed1e1dd03fef4ee98529511926e7617676b2f8505
parent3cb9baaf65abe6c658a5b8b4bd5c0a424ec923c1

std.compress.lzma2: tests passing

2 files changed, 131 insertions(+), 151 deletions(-)

lib/std/compress/lzma.zig+59-67
......@@ -105,7 +105,6 @@ pub const RangeDecoder = struct {
105105
106106pub const Decode = struct {
107107 properties: Properties,
108 unpacked_size: ?u64,
109108 literal_probs: Vec2d,
110109 pos_slot_decoder: [4]BitTree(6),
111110 align_decoder: BitTree(4),
......@@ -121,15 +120,10 @@ pub const Decode = struct {
121120 len_decoder: LenDecoder,
122121 rep_len_decoder: LenDecoder,
123122
124 pub fn init(
125 gpa: Allocator,
126 properties: Properties,
127 unpacked_size: ?u64,
128 ) !Decode {
123 pub fn init(gpa: Allocator, properties: Properties) !Decode {
129124 return .{
130125 .properties = properties,
131 .unpacked_size = unpacked_size,
132 .literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (properties.lc + properties.lp), 0x300 }),
126 .literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (properties.lc + properties.lp), 0x300),
133127 .pos_slot_decoder = @splat(.{}),
134128 .align_decoder = .{},
135129 .pos_decoders = @splat(0x400),
......@@ -157,7 +151,7 @@ pub const Decode = struct {
157151 self.literal_probs.fill(0x400);
158152 } else {
159153 self.literal_probs.deinit(gpa);
160 self.literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
154 self.literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
161155 }
162156
163157 self.properties = new_props;
......@@ -176,11 +170,12 @@ pub const Decode = struct {
176170 self.rep_len_decoder.reset();
177171 }
178172
179 fn processNext(
173 pub fn process(
180174 self: *Decode,
181175 reader: *Reader,
182176 allocating: *Writer.Allocating,
183 buffer: *CircularBuffer,
177 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
178 buffer: anytype,
184179 decoder: *RangeDecoder,
185180 ) !ProcessingStatus {
186181 const gpa = allocating.allocator;
......@@ -256,39 +251,11 @@ pub const Decode = struct {
256251 return .more;
257252 }
258253
259 pub fn process(
260 self: *Decode,
261 reader: *Reader,
262 allocating: *Writer.Allocating,
263 buffer: *CircularBuffer,
264 decoder: *RangeDecoder,
265 ) !void {
266 process_next: {
267 if (self.unpacked_size) |unpacked_size| {
268 if (buffer.len >= unpacked_size) {
269 break :process_next;
270 }
271 } else if (decoder.isFinished()) {
272 break :process_next;
273 }
274 switch (try self.processNext(reader, allocating, buffer, decoder)) {
275 .more => return,
276 .finished => {},
277 }
278 }
279
280 if (self.unpacked_size) |unpacked_size| {
281 if (buffer.len != unpacked_size) return error.DecompressedSizeMismatch;
282 }
283
284 try buffer.finish(&allocating.writer);
285 self.state = math.maxInt(usize);
286 }
287
288254 fn decodeLiteral(
289255 self: *Decode,
290256 reader: *Reader,
291 buffer: *CircularBuffer,
257 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
258 buffer: anytype,
292259 decoder: *RangeDecoder,
293260 ) !u8 {
294261 const def_prev_byte = 0;
......@@ -377,10 +344,7 @@ pub const Decode = struct {
377344 }
378345
379346 pub fn get(self: CircularBuffer, index: usize) u8 {
380 return if (0 <= index and index < self.buf.items.len)
381 self.buf.items[index]
382 else
383 0;
347 return if (0 <= index and index < self.buf.items.len) self.buf.items[index] else 0;
384348 }
385349
386350 pub fn set(self: *CircularBuffer, gpa: Allocator, index: usize, value: u8) !void {
......@@ -524,29 +488,29 @@ pub const Decode = struct {
524488 data: []u16,
525489 cols: usize,
526490
527 pub fn init(gpa: Allocator, value: u16, size: struct { usize, usize }) !Vec2d {
528 const len = try math.mul(usize, size[0], size[1]);
491 pub fn init(gpa: Allocator, value: u16, w: usize, h: usize) !Vec2d {
492 const len = try math.mul(usize, w, h);
529493 const data = try gpa.alloc(u16, len);
530494 @memset(data, value);
531495 return .{
532496 .data = data,
533 .cols = size[1],
497 .cols = h,
534498 };
535499 }
536500
537 pub fn deinit(self: *Vec2d, gpa: Allocator) void {
538 gpa.free(self.data);
539 self.* = undefined;
501 pub fn deinit(v: *Vec2d, gpa: Allocator) void {
502 gpa.free(v.data);
503 v.* = undefined;
540504 }
541505
542 pub fn fill(self: *Vec2d, value: u16) void {
543 @memset(self.data, value);
506 pub fn fill(v: *Vec2d, value: u16) void {
507 @memset(v.data, value);
544508 }
545509
546 fn get(self: Vec2d, row: usize) ![]u16 {
547 const start_row = try math.mul(usize, row, self.cols);
548 const end_row = try math.add(usize, start_row, self.cols);
549 return self.data[start_row..end_row];
510 fn get(v: Vec2d, row: usize) ![]u16 {
511 const start_row = try math.mul(usize, row, v.cols);
512 const end_row = try math.add(usize, start_row, v.cols);
513 return v.data[start_row..end_row];
550514 }
551515 };
552516
......@@ -627,6 +591,7 @@ pub const Decompress = struct {
627591 range_decoder: RangeDecoder,
628592 decode: Decode,
629593 err: ?Error,
594 unpacked_size: ?u64,
630595
631596 pub const Error = error{
632597 OutOfMemory,
......@@ -654,7 +619,7 @@ pub const Decompress = struct {
654619 .input = input,
655620 .buffer = Decode.CircularBuffer.init(params.dict_size, mem_limit),
656621 .range_decoder = try RangeDecoder.init(input),
657 .decode = try Decode.init(gpa, params.properties, params.unpacked_size),
622 .decode = try Decode.init(gpa, params.properties),
658623 .reader = .{
659624 .buffer = buffer,
660625 .vtable = &.{
......@@ -666,6 +631,7 @@ pub const Decompress = struct {
666631 .end = 0,
667632 },
668633 .err = null,
634 .unpacked_size = params.unpacked_size,
669635 };
670636 }
671637
......@@ -728,20 +694,46 @@ pub const Decompress = struct {
728694 r.end = allocating.writer.end;
729695 }
730696 if (d.decode.state == math.maxInt(usize)) return error.EndOfStream;
731 d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch |err| switch (err) {
697
698 process_next: {
699 if (d.unpacked_size) |unpacked_size| {
700 if (d.buffer.len >= unpacked_size) break :process_next;
701 } else if (d.range_decoder.isFinished()) {
702 break :process_next;
703 }
704 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch |err| switch (err) {
705 error.WriteFailed => {
706 d.err = error.OutOfMemory;
707 return error.ReadFailed;
708 },
709 error.EndOfStream => {
710 d.err = error.EndOfStream;
711 return error.ReadFailed;
712 },
713 else => |e| {
714 d.err = e;
715 return error.ReadFailed;
716 },
717 }) {
718 .more => return 0,
719 .finished => break :process_next,
720 }
721 }
722
723 if (d.unpacked_size) |unpacked_size| {
724 if (d.buffer.len != unpacked_size) {
725 d.err = error.DecompressedSizeMismatch;
726 return error.ReadFailed;
727 }
728 }
729
730 d.buffer.finish(&allocating.writer) catch |err| switch (err) {
732731 error.WriteFailed => {
733732 d.err = error.OutOfMemory;
734733 return error.ReadFailed;
735734 },
736 error.EndOfStream => {
737 d.err = error.EndOfStream;
738 return error.ReadFailed;
739 },
740 else => |e| {
741 d.err = e;
742 return error.ReadFailed;
743 },
744735 };
736 d.decode.state = math.maxInt(usize);
745737 return 0;
746738 }
747739};
lib/std/compress/lzma2.zig+72-84
......@@ -6,17 +6,15 @@ const Writer = std.Io.Writer;
66const Reader = std.Io.Reader;
77
88/// An accumulating buffer for LZ sequences
9pub const LzAccumBuffer = struct {
9pub const AccumBuffer = struct {
1010 /// Buffer
1111 buf: ArrayList(u8),
12
1312 /// Buffer memory limit
1413 memlimit: usize,
15
1614 /// Total number of bytes sent through the buffer
1715 len: usize,
1816
19 pub fn init(memlimit: usize) LzAccumBuffer {
17 pub fn init(memlimit: usize) AccumBuffer {
2018 return .{
2119 .buf = .{},
2220 .memlimit = memlimit,
......@@ -24,20 +22,20 @@ pub const LzAccumBuffer = struct {
2422 };
2523 }
2624
27 pub fn appendByte(self: *LzAccumBuffer, allocator: Allocator, byte: u8) !void {
25 pub fn appendByte(self: *AccumBuffer, allocator: Allocator, byte: u8) !void {
2826 try self.buf.append(allocator, byte);
2927 self.len += 1;
3028 }
3129
3230 /// Reset the internal dictionary
33 pub fn reset(self: *LzAccumBuffer, writer: *Writer) !void {
31 pub fn reset(self: *AccumBuffer, writer: *Writer) !void {
3432 try writer.writeAll(self.buf.items);
3533 self.buf.clearRetainingCapacity();
3634 self.len = 0;
3735 }
3836
3937 /// Retrieve the last byte or return a default
40 pub fn lastOr(self: LzAccumBuffer, lit: u8) u8 {
38 pub fn lastOr(self: AccumBuffer, lit: u8) u8 {
4139 const buf_len = self.buf.items.len;
4240 return if (buf_len == 0)
4341 lit
......@@ -46,7 +44,7 @@ pub const LzAccumBuffer = struct {
4644 }
4745
4846 /// Retrieve the n-th last byte
49 pub fn lastN(self: LzAccumBuffer, dist: usize) !u8 {
47 pub fn lastN(self: AccumBuffer, dist: usize) !u8 {
5048 const buf_len = self.buf.items.len;
5149 if (dist > buf_len) {
5250 return error.CorruptInput;
......@@ -57,7 +55,7 @@ pub const LzAccumBuffer = struct {
5755
5856 /// Append a literal
5957 pub fn appendLiteral(
60 self: *LzAccumBuffer,
58 self: *AccumBuffer,
6159 allocator: Allocator,
6260 lit: u8,
6361 writer: *Writer,
......@@ -72,7 +70,7 @@ pub const LzAccumBuffer = struct {
7270
7371 /// Fetch an LZ sequence (length, distance) from inside the buffer
7472 pub fn appendLz(
75 self: *LzAccumBuffer,
73 self: *AccumBuffer,
7674 allocator: Allocator,
7775 len: usize,
7876 dist: usize,
......@@ -95,12 +93,12 @@ pub const LzAccumBuffer = struct {
9593 self.len += len;
9694 }
9795
98 pub fn finish(self: *LzAccumBuffer, writer: *Writer) !void {
96 pub fn finish(self: *AccumBuffer, writer: *Writer) !void {
9997 try writer.writeAll(self.buf.items);
10098 self.buf.clearRetainingCapacity();
10199 }
102100
103 pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
101 pub fn deinit(self: *AccumBuffer, allocator: Allocator) void {
104102 self.buf.deinit(allocator);
105103 self.* = undefined;
106104 }
......@@ -109,59 +107,43 @@ pub const LzAccumBuffer = struct {
109107pub const Decode = struct {
110108 lzma_decode: lzma.Decode,
111109
112 pub fn init(allocator: Allocator) !Decode {
113 return Decode{
114 .lzma_decode = try lzma.Decode.init(
115 allocator,
116 .{
117 .lc = 0,
118 .lp = 0,
119 .pb = 0,
120 },
121 null,
122 ),
123 };
110 pub fn init(gpa: Allocator) !Decode {
111 return .{ .lzma_decode = try lzma.Decode.init(gpa, .{ .lc = 0, .lp = 0, .pb = 0 }) };
124112 }
125113
126 pub fn deinit(self: *Decode, allocator: Allocator) void {
127 self.lzma_decode.deinit(allocator);
114 pub fn deinit(self: *Decode, gpa: Allocator) void {
115 self.lzma_decode.deinit(gpa);
128116 self.* = undefined;
129117 }
130118
131 pub fn decompress(
132 self: *Decode,
133 allocator: Allocator,
134 reader: *Reader,
135 writer: *Writer,
136 ) !void {
137 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
138 defer accum.deinit(allocator);
119 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !void {
120 const gpa = allocating.allocator;
121
122 var accum = AccumBuffer.init(std.math.maxInt(usize));
123 defer accum.deinit(gpa);
139124
140125 while (true) {
141 const status = try reader.readByte();
126 const status = try reader.takeByte();
142127
143128 switch (status) {
144129 0 => break,
145 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
146 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
147 else => try self.parseLzma(allocator, reader, writer, &accum, status),
130 1 => try parseUncompressed(reader, allocating, &accum, true),
131 2 => try parseUncompressed(reader, allocating, &accum, false),
132 else => try d.parseLzma(reader, allocating, &accum, status),
148133 }
149134 }
150135
151 try accum.finish(writer);
136 try accum.finish(&allocating.writer);
152137 }
153138
154139 fn parseLzma(
155 self: *Decode,
156 allocator: Allocator,
140 d: *Decode,
157141 reader: *Reader,
158 writer: *Writer,
159 accum: *LzAccumBuffer,
142 allocating: *Writer.Allocating,
143 accum: *AccumBuffer,
160144 status: u8,
161145 ) !void {
162 if (status & 0x80 == 0) {
163 return error.CorruptInput;
164 }
146 if (status & 0x80 == 0) return error.CorruptInput;
165147
166148 const Reset = struct {
167149 dict: bool,
......@@ -169,23 +151,23 @@ pub const Decode = struct {
169151 props: bool,
170152 };
171153
172 const reset = switch ((status >> 5) & 0x3) {
173 0 => Reset{
154 const reset: Reset = switch ((status >> 5) & 0x3) {
155 0 => .{
174156 .dict = false,
175157 .state = false,
176158 .props = false,
177159 },
178 1 => Reset{
160 1 => .{
179161 .dict = false,
180162 .state = true,
181163 .props = false,
182164 },
183 2 => Reset{
165 2 => .{
184166 .dict = false,
185167 .state = true,
186168 .props = true,
187169 },
188 3 => Reset{
170 3 => .{
189171 .dict = true,
190172 .state = true,
191173 .props = true,
......@@ -196,24 +178,24 @@ pub const Decode = struct {
196178 const unpacked_size = blk: {
197179 var tmp: u64 = status & 0x1F;
198180 tmp <<= 16;
199 tmp |= try reader.readInt(u16, .big);
181 tmp |= try reader.takeInt(u16, .big);
200182 break :blk tmp + 1;
201183 };
202184
203185 const packed_size = blk: {
204 const tmp: u17 = try reader.readInt(u16, .big);
186 const tmp: u17 = try reader.takeInt(u16, .big);
205187 break :blk tmp + 1;
206188 };
207189
208 if (reset.dict) {
209 try accum.reset(writer);
210 }
190 if (reset.dict) try accum.reset(&allocating.writer);
191
192 const ld = &d.lzma_decode;
211193
212194 if (reset.state) {
213 var new_props = self.lzma_decode.properties;
195 var new_props = ld.properties;
214196
215197 if (reset.props) {
216 var props = try reader.readByte();
198 var props = try reader.takeByte();
217199 if (props >= 225) {
218200 return error.CorruptInput;
219201 }
......@@ -231,38 +213,44 @@ pub const Decode = struct {
231213 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
232214 }
233215
234 try self.lzma_decode.resetState(allocator, new_props);
216 try ld.resetState(allocating.allocator, new_props);
235217 }
236218
237 self.lzma_decode.unpacked_size = unpacked_size + accum.len;
219 var range_decoder = try lzma.RangeDecoder.init(reader);
238220
239 var counter = std.io.countingReader(reader);
240 const counter_reader = counter.reader();
241
242 var rangecoder = try lzma.RangeDecoder.init(counter_reader);
243 while (try self.lzma_decode.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
244
245 if (counter.bytes_read != packed_size) {
246 return error.CorruptInput;
221 while (true) {
222 if (accum.len >= unpacked_size) break;
223 if (range_decoder.isFinished()) break;
224 switch (try ld.process(reader, allocating, accum, &range_decoder)) {
225 .more => continue,
226 .finished => break,
227 }
247228 }
229 if (accum.len != unpacked_size) return error.DecompressedSizeMismatch;
230
231 // TODO restore this error
232 //if (counter.bytes_read != packed_size) {
233 // return error.CorruptInput;
234 //}
235 _ = packed_size;
248236 }
249237
250238 fn parseUncompressed(
251 allocator: Allocator,
252239 reader: *Reader,
253 writer: *Writer,
254 accum: *LzAccumBuffer,
240 allocating: *Writer.Allocating,
241 accum: *AccumBuffer,
255242 reset_dict: bool,
256243 ) !void {
257 const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
244 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
258245
259 if (reset_dict) {
260 try accum.reset(writer);
261 }
246 if (reset_dict) try accum.reset(&allocating.writer);
247
248 const gpa = allocating.allocator;
262249
263 var i: @TypeOf(unpacked_size) = 0;
264 while (i < unpacked_size) : (i += 1) {
265 try accum.appendByte(allocator, try reader.readByte());
250 var i = unpacked_size;
251 while (i != 0) {
252 try accum.appendByte(gpa, try reader.takeByte());
253 i -= 1;
266254 }
267255 }
268256};
......@@ -273,13 +261,13 @@ test "decompress hello world stream" {
273261
274262 const gpa = std.testing.allocator;
275263
276 var stream: std.Io.Reader = .fixed(compressed);
277
278 var decode = try Decode.init(gpa, &stream);
264 var decode = try Decode.init(gpa);
279265 defer decode.deinit(gpa);
280266
281 const result = try decode.reader.allocRemaining(gpa, .unlimited);
282 defer gpa.free(result);
267 var stream: std.Io.Reader = .fixed(compressed);
268 var result: std.Io.Writer.Allocating = .init(gpa);
269 defer result.deinit();
283270
284 try std.testing.expectEqualStrings(expected, result);
271 try decode.decompress(&stream, &result);
272 try std.testing.expectEqualStrings(expected, result.written());
285273}