authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-27 06:49:45-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-08-27 06:49:45-07:00
log50edad37ba745502174e49af922b179b1efdd99c
tree9b6fc34503d4aeb9d2d9ecab1be17ebb03df72fa
parent12a58087a423e33dd5fc8daa5c9fb556fe93f7a9
parent68f590d430bee6bc9b3bb4940f739d8b04435c08
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25034 from ziglang/lzma

std.compress: update lzma, lzma2, and xz to new I/O API

13 files changed, 1471 insertions(+), 1613 deletions(-)

lib/std/compress/lzma.zig+731-61
...@@ -2,89 +2,759 @@ const std = @import("../std.zig");...@@ -2,89 +2,759 @@ const std = @import("../std.zig");
2const math = std.math;2const math = std.math;
3const mem = std.mem;3const mem = std.mem;
4const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
5const assert = std.debug.assert;
6const ArrayList = std.ArrayList;
7const Writer = std.Io.Writer;
8const Reader = std.Io.Reader;
59
6pub const decode = @import("lzma/decode.zig");10pub const RangeDecoder = struct {
11 range: u32,
12 code: u32,
713
8pub fn decompress(14 pub fn init(reader: *Reader) !RangeDecoder {
9 allocator: Allocator,15 var counter: u64 = 0;
10 reader: anytype,16 return initCounting(reader, &counter);
11) !Decompress(@TypeOf(reader)) {17 }
12 return decompressWithOptions(allocator, reader, .{});
13}
1418
15pub fn decompressWithOptions(19 pub fn initCounting(reader: *Reader, n_read: *u64) !RangeDecoder {
16 allocator: Allocator,20 const reserved = try reader.takeByte();
17 reader: anytype,21 n_read.* += 1;
18 options: decode.Options,22 if (reserved != 0) return error.InvalidRangeCode;
19) !Decompress(@TypeOf(reader)) {23 const code = try reader.takeInt(u32, .big);
20 const params = try decode.Params.readHeader(reader, options);24 n_read.* += 4;
21 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);25 return .{
22}26 .range = 0xFFFF_FFFF,
27 .code = code,
28 };
29 }
30
31 pub fn isFinished(self: RangeDecoder) bool {
32 return self.code == 0;
33 }
34
35 fn normalize(self: *RangeDecoder, reader: *Reader, n_read: *u64) !void {
36 if (self.range < 0x0100_0000) {
37 self.range <<= 8;
38 self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
39 n_read.* += 1;
40 }
41 }
42
43 fn getBit(self: *RangeDecoder, reader: *Reader, n_read: *u64) !bool {
44 self.range >>= 1;
45
46 const bit = self.code >= self.range;
47 if (bit) self.code -= self.range;
48
49 try self.normalize(reader, n_read);
50 return bit;
51 }
52
53 pub fn get(self: *RangeDecoder, reader: *Reader, count: usize, n_read: *u64) !u32 {
54 var result: u32 = 0;
55 for (0..count) |_| {
56 result = (result << 1) ^ @intFromBool(try self.getBit(reader, n_read));
57 }
58 return result;
59 }
60
61 pub fn decodeBit(self: *RangeDecoder, reader: *Reader, prob: *u16, n_read: *u64) !bool {
62 const bound = (self.range >> 11) * prob.*;
63
64 if (self.code < bound) {
65 prob.* += (0x800 - prob.*) >> 5;
66 self.range = bound;
67
68 try self.normalize(reader, n_read);
69 return false;
70 } else {
71 prob.* -= prob.* >> 5;
72 self.code -= bound;
73 self.range -= bound;
74
75 try self.normalize(reader, n_read);
76 return true;
77 }
78 }
79
80 fn parseBitTree(
81 self: *RangeDecoder,
82 reader: *Reader,
83 num_bits: u5,
84 probs: []u16,
85 n_read: *u64,
86 ) !u32 {
87 var tmp: u32 = 1;
88 var i: @TypeOf(num_bits) = 0;
89 while (i < num_bits) : (i += 1) {
90 const bit = try self.decodeBit(reader, &probs[tmp], n_read);
91 tmp = (tmp << 1) ^ @intFromBool(bit);
92 }
93 return tmp - (@as(u32, 1) << num_bits);
94 }
95
96 pub fn parseReverseBitTree(
97 self: *RangeDecoder,
98 reader: *Reader,
99 num_bits: u5,
100 probs: []u16,
101 offset: usize,
102 n_read: *u64,
103 ) !u32 {
104 var result: u32 = 0;
105 var tmp: usize = 1;
106 var i: @TypeOf(num_bits) = 0;
107 while (i < num_bits) : (i += 1) {
108 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], n_read));
109 tmp = (tmp << 1) ^ bit;
110 result ^= @as(u32, bit) << i;
111 }
112 return result;
113 }
114};
115
116pub const Decode = struct {
117 properties: Properties,
118 literal_probs: Vec2d,
119 pos_slot_decoder: [4]BitTree(6),
120 align_decoder: BitTree(4),
121 pos_decoders: [115]u16,
122 is_match: [192]u16,
123 is_rep: [12]u16,
124 is_rep_g0: [12]u16,
125 is_rep_g1: [12]u16,
126 is_rep_g2: [12]u16,
127 is_rep_0long: [192]u16,
128 state: usize,
129 rep: [4]usize,
130 len_decoder: LenDecoder,
131 rep_len_decoder: LenDecoder,
132
133 pub fn init(gpa: Allocator, properties: Properties) !Decode {
134 return .{
135 .properties = properties,
136 .literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (properties.lc + properties.lp), 0x300),
137 .pos_slot_decoder = @splat(.{}),
138 .align_decoder = .{},
139 .pos_decoders = @splat(0x400),
140 .is_match = @splat(0x400),
141 .is_rep = @splat(0x400),
142 .is_rep_g0 = @splat(0x400),
143 .is_rep_g1 = @splat(0x400),
144 .is_rep_g2 = @splat(0x400),
145 .is_rep_0long = @splat(0x400),
146 .state = 0,
147 .rep = @splat(0),
148 .len_decoder = .{},
149 .rep_len_decoder = .{},
150 };
151 }
152
153 pub fn deinit(self: *Decode, gpa: Allocator) void {
154 self.literal_probs.deinit(gpa);
155 self.* = undefined;
156 }
157
158 pub fn resetState(self: *Decode, gpa: Allocator, new_props: Properties) !void {
159 new_props.validate();
160 if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
161 self.literal_probs.fill(0x400);
162 } else {
163 self.literal_probs.deinit(gpa);
164 self.literal_probs = try Vec2d.init(gpa, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
165 }
166
167 self.properties = new_props;
168 for (&self.pos_slot_decoder) |*t| t.reset();
169 self.align_decoder.reset();
170 self.pos_decoders = @splat(0x400);
171 self.is_match = @splat(0x400);
172 self.is_rep = @splat(0x400);
173 self.is_rep_g0 = @splat(0x400);
174 self.is_rep_g1 = @splat(0x400);
175 self.is_rep_g2 = @splat(0x400);
176 self.is_rep_0long = @splat(0x400);
177 self.state = 0;
178 self.rep = @splat(0);
179 self.len_decoder.reset();
180 self.rep_len_decoder.reset();
181 }
182
183 pub fn process(
184 self: *Decode,
185 reader: *Reader,
186 allocating: *Writer.Allocating,
187 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
188 buffer: anytype,
189 decoder: *RangeDecoder,
190 n_read: *u64,
191 ) !ProcessingStatus {
192 const gpa = allocating.allocator;
193 const writer = &allocating.writer;
194 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
195
196 if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], n_read)) {
197 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, n_read);
198
199 try buffer.appendLiteral(gpa, byte, writer);
200
201 self.state = if (self.state < 4)
202 0
203 else if (self.state < 10)
204 self.state - 3
205 else
206 self.state - 6;
207 return .more;
208 }
209
210 var len: usize = undefined;
211 if (try decoder.decodeBit(reader, &self.is_rep[self.state], n_read)) {
212 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], n_read)) {
213 if (!try decoder.decodeBit(reader, &self.is_rep_0long[(self.state << 4) + pos_state], n_read)) {
214 self.state = if (self.state < 7) 9 else 11;
215 const dist = self.rep[0] + 1;
216 try buffer.appendLz(gpa, 1, dist, writer);
217 return .more;
218 }
219 } else {
220 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], n_read))
221 1
222 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], n_read))
223 2
224 else
225 3;
226 const dist = self.rep[idx];
227 var i = idx;
228 while (i > 0) : (i -= 1) {
229 self.rep[i] = self.rep[i - 1];
230 }
231 self.rep[0] = dist;
232 }
233
234 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, n_read);
235
236 self.state = if (self.state < 7) 8 else 11;
237 } else {
238 self.rep[3] = self.rep[2];
239 self.rep[2] = self.rep[1];
240 self.rep[1] = self.rep[0];
23241
24pub fn Decompress(comptime ReaderType: type) type {242 len = try self.len_decoder.decode(reader, decoder, pos_state, n_read);
25 return struct {
26 const Self = @This();
27243
28 pub const Error =244 self.state = if (self.state < 7) 7 else 10;
29 ReaderType.Error ||
30 Allocator.Error ||
31 error{ CorruptInput, EndOfStream, Overflow };
32245
33 pub const Reader = std.io.GenericReader(*Self, Error, read);246 const rep_0 = try self.decodeDistance(reader, decoder, len, n_read);
34247
35 allocator: Allocator,248 self.rep[0] = rep_0;
36 in_reader: ReaderType,249 if (self.rep[0] == 0xFFFF_FFFF) {
37 to_read: std.ArrayListUnmanaged(u8),250 if (decoder.isFinished()) {
251 return .finished;
252 }
253 return error.CorruptInput;
254 }
255 }
256
257 len += 2;
258
259 const dist = self.rep[0] + 1;
260 try buffer.appendLz(gpa, len, dist, writer);
261
262 return .more;
263 }
264
265 fn decodeLiteral(
266 self: *Decode,
267 reader: *Reader,
268 /// `CircularBuffer` or `std.compress.lzma2.AccumBuffer`.
269 buffer: anytype,
270 decoder: *RangeDecoder,
271 n_read: *u64,
272 ) !u8 {
273 const def_prev_byte = 0;
274 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
275
276 var result: usize = 1;
277 const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
278 (prev_byte >> (8 - self.properties.lc));
279 const probs = try self.literal_probs.get(lit_state);
280
281 if (self.state >= 7) {
282 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
283
284 while (result < 0x100) {
285 const match_bit = (match_byte >> 7) & 1;
286 match_byte <<= 1;
287 const bit = @intFromBool(try decoder.decodeBit(
288 reader,
289 &probs[((@as(usize, 1) + match_bit) << 8) + result],
290 n_read,
291 ));
292 result = (result << 1) ^ bit;
293 if (match_bit != bit) {
294 break;
295 }
296 }
297 }
298
299 while (result < 0x100) {
300 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], n_read));
301 }
302
303 return @truncate(result - 0x100);
304 }
305
306 fn decodeDistance(
307 self: *Decode,
308 reader: *Reader,
309 decoder: *RangeDecoder,
310 length: usize,
311 n_read: *u64,
312 ) !usize {
313 const len_state = if (length > 3) 3 else length;
314
315 const pos_slot: usize = try self.pos_slot_decoder[len_state].parse(reader, decoder, n_read);
316 if (pos_slot < 4) return pos_slot;
38317
39 buffer: decode.lzbuffer.LzCircularBuffer,318 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
40 decoder: decode.rangecoder.RangeDecoder,319 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
41 state: decode.DecoderState,
42320
43 pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {321 if (pos_slot < 14) {
44 return Self{322 result += try decoder.parseReverseBitTree(
45 .allocator = allocator,323 reader,
46 .in_reader = source,324 num_direct_bits,
47 .to_read = .{},325 &self.pos_decoders,
326 result - pos_slot,
327 n_read,
328 );
329 } else {
330 result += @as(usize, try decoder.get(reader, num_direct_bits - 4, n_read)) << 4;
331 result += try self.align_decoder.parseReverse(reader, decoder, n_read);
332 }
333
334 return result;
335 }
336
337 /// A circular buffer for LZ sequences
338 pub const CircularBuffer = struct {
339 /// Circular buffer
340 buf: ArrayList(u8),
341 /// Length of the buffer
342 dict_size: usize,
343 /// Buffer memory limit
344 mem_limit: usize,
345 /// Current position
346 cursor: usize,
347 /// Total number of bytes sent through the buffer
348 len: usize,
48349
49 .buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),350 pub fn init(dict_size: usize, mem_limit: usize) CircularBuffer {
50 .decoder = try decode.rangecoder.RangeDecoder.init(source),351 return .{
51 .state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),352 .buf = .{},
353 .dict_size = dict_size,
354 .mem_limit = mem_limit,
355 .cursor = 0,
356 .len = 0,
52 };357 };
53 }358 }
54359
55 pub fn reader(self: *Self) Reader {360 pub fn get(self: CircularBuffer, index: usize) u8 {
56 return .{ .context = self };361 return if (0 <= index and index < self.buf.items.len) self.buf.items[index] else 0;
57 }362 }
58363
59 pub fn deinit(self: *Self) void {364 pub fn set(self: *CircularBuffer, gpa: Allocator, index: usize, value: u8) !void {
60 self.to_read.deinit(self.allocator);365 if (index >= self.mem_limit) {
61 self.buffer.deinit(self.allocator);366 return error.CorruptInput;
62 self.state.deinit(self.allocator);367 }
63 self.* = undefined;368 try self.buf.ensureTotalCapacity(gpa, index + 1);
369 while (self.buf.items.len < index) {
370 self.buf.appendAssumeCapacity(0);
371 }
372 self.buf.appendAssumeCapacity(value);
373 }
374
375 /// Retrieve the last byte or return a default
376 pub fn lastOr(self: CircularBuffer, lit: u8) u8 {
377 return if (self.len == 0)
378 lit
379 else
380 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
381 }
382
383 /// Retrieve the n-th last byte
384 pub fn lastN(self: CircularBuffer, dist: usize) !u8 {
385 if (dist > self.dict_size or dist > self.len) {
386 return error.CorruptInput;
387 }
388
389 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
390 return self.get(offset);
64 }391 }
65392
66 pub fn read(self: *Self, output: []u8) Error!usize {393 /// Append a literal
67 const writer = self.to_read.writer(self.allocator);394 pub fn appendLiteral(
68 while (self.to_read.items.len < output.len) {395 self: *CircularBuffer,
69 switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {396 gpa: Allocator,
70 .continue_ => {},397 lit: u8,
71 .finished => {398 writer: *Writer,
72 try self.buffer.finish(writer);399 ) !void {
73 break;400 try self.set(gpa, self.cursor, lit);
74 },401 self.cursor += 1;
402 self.len += 1;
403
404 // Flush the circular buffer to the output
405 if (self.cursor == self.dict_size) {
406 try writer.writeAll(self.buf.items);
407 self.cursor = 0;
408 }
409 }
410
411 /// Fetch an LZ sequence (length, distance) from inside the buffer
412 pub fn appendLz(
413 self: *CircularBuffer,
414 gpa: Allocator,
415 len: usize,
416 dist: usize,
417 writer: *Writer,
418 ) !void {
419 if (dist > self.dict_size or dist > self.len) {
420 return error.CorruptInput;
421 }
422
423 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
424 var i: usize = 0;
425 while (i < len) : (i += 1) {
426 const x = self.get(offset);
427 try self.appendLiteral(gpa, x, writer);
428 offset += 1;
429 if (offset == self.dict_size) {
430 offset = 0;
75 }431 }
76 }432 }
77 const input = self.to_read.items;433 }
78 const n = @min(input.len, output.len);434
79 @memcpy(output[0..n], input[0..n]);435 pub fn finish(self: *CircularBuffer, writer: *Writer) !void {
80 std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);436 if (self.cursor > 0) {
81 self.to_read.shrinkRetainingCapacity(input.len - n);437 try writer.writeAll(self.buf.items[0..self.cursor]);
82 return n;438 self.cursor = 0;
439 }
440 }
441
442 pub fn deinit(self: *CircularBuffer, gpa: Allocator) void {
443 self.buf.deinit(gpa);
444 self.* = undefined;
83 }445 }
84 };446 };
85}447
448 pub fn BitTree(comptime num_bits: usize) type {
449 return struct {
450 probs: [1 << num_bits]u16 = @splat(0x400),
451
452 pub fn parse(self: *@This(), reader: *Reader, decoder: *RangeDecoder, n_read: *u64) !u32 {
453 return decoder.parseBitTree(reader, num_bits, &self.probs, n_read);
454 }
455
456 pub fn parseReverse(
457 self: *@This(),
458 reader: *Reader,
459 decoder: *RangeDecoder,
460 n_read: *u64,
461 ) !u32 {
462 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, n_read);
463 }
464
465 pub fn reset(self: *@This()) void {
466 @memset(&self.probs, 0x400);
467 }
468 };
469 }
470
471 pub const LenDecoder = struct {
472 choice: u16 = 0x400,
473 choice2: u16 = 0x400,
474 low_coder: [16]BitTree(3) = @splat(.{}),
475 mid_coder: [16]BitTree(3) = @splat(.{}),
476 high_coder: BitTree(8) = .{},
477
478 pub fn decode(
479 self: *LenDecoder,
480 reader: *Reader,
481 decoder: *RangeDecoder,
482 pos_state: usize,
483 n_read: *u64,
484 ) !usize {
485 if (!try decoder.decodeBit(reader, &self.choice, n_read)) {
486 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, n_read));
487 } else if (!try decoder.decodeBit(reader, &self.choice2, n_read)) {
488 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, n_read)) + 8;
489 } else {
490 return @as(usize, try self.high_coder.parse(reader, decoder, n_read)) + 16;
491 }
492 }
493
494 pub fn reset(self: *LenDecoder) void {
495 self.choice = 0x400;
496 self.choice2 = 0x400;
497 for (&self.low_coder) |*t| t.reset();
498 for (&self.mid_coder) |*t| t.reset();
499 self.high_coder.reset();
500 }
501 };
502
503 pub const Vec2d = struct {
504 data: []u16,
505 cols: usize,
506
507 pub fn init(gpa: Allocator, value: u16, w: usize, h: usize) !Vec2d {
508 const len = try math.mul(usize, w, h);
509 const data = try gpa.alloc(u16, len);
510 @memset(data, value);
511 return .{
512 .data = data,
513 .cols = h,
514 };
515 }
516
517 pub fn deinit(v: *Vec2d, gpa: Allocator) void {
518 gpa.free(v.data);
519 v.* = undefined;
520 }
521
522 pub fn fill(v: *Vec2d, value: u16) void {
523 @memset(v.data, value);
524 }
525
526 fn get(v: Vec2d, row: usize) ![]u16 {
527 const start_row = try math.mul(usize, row, v.cols);
528 const end_row = try math.add(usize, start_row, v.cols);
529 return v.data[start_row..end_row];
530 }
531 };
532
533 pub const Options = struct {
534 unpacked_size: UnpackedSize = .read_from_header,
535 mem_limit: ?usize = null,
536 allow_incomplete: bool = false,
537 };
538
539 pub const UnpackedSize = union(enum) {
540 read_from_header,
541 read_header_but_use_provided: ?u64,
542 use_provided: ?u64,
543 };
544
545 const ProcessingStatus = enum {
546 more,
547 finished,
548 };
549
550 pub const Properties = struct {
551 lc: u4,
552 lp: u3,
553 pb: u3,
554
555 fn validate(self: Properties) void {
556 assert(self.lc <= 8);
557 assert(self.lp <= 4);
558 assert(self.pb <= 4);
559 }
560 };
561
562 pub const Params = struct {
563 properties: Properties,
564 dict_size: u32,
565 unpacked_size: ?u64,
566
567 pub fn readHeader(reader: *Reader, options: Options) !Params {
568 var props = try reader.takeByte();
569 if (props >= 225) return error.CorruptInput;
570
571 const lc: u4 = @intCast(props % 9);
572 props /= 9;
573 const lp: u3 = @intCast(props % 5);
574 props /= 5;
575 const pb: u3 = @intCast(props);
576
577 const dict_size_provided = try reader.takeInt(u32, .little);
578 const dict_size = @max(0x1000, dict_size_provided);
579
580 const unpacked_size = switch (options.unpacked_size) {
581 .read_from_header => blk: {
582 const unpacked_size_provided = try reader.takeInt(u64, .little);
583 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
584 break :blk if (marker_mandatory) null else unpacked_size_provided;
585 },
586 .read_header_but_use_provided => |x| blk: {
587 _ = try reader.takeInt(u64, .little);
588 break :blk x;
589 },
590 .use_provided => |x| x,
591 };
592
593 return .{
594 .properties = .{ .lc = lc, .lp = lp, .pb = pb },
595 .dict_size = dict_size,
596 .unpacked_size = unpacked_size,
597 };
598 }
599 };
600};
601
602pub const Decompress = struct {
603 gpa: Allocator,
604 input: *Reader,
605 reader: Reader,
606 buffer: Decode.CircularBuffer,
607 range_decoder: RangeDecoder,
608 decode: Decode,
609 err: ?Error,
610 unpacked_size: ?u64,
611
612 pub const Error = error{
613 OutOfMemory,
614 ReadFailed,
615 CorruptInput,
616 DecompressedSizeMismatch,
617 EndOfStream,
618 Overflow,
619 };
620
621 /// Takes ownership of `buffer` which may be resized with `gpa`.
622 ///
623 /// LZMA was explicitly designed to take advantage of large heap memory
624 /// being available, with a dictionary size anywhere from 4K to 4G. Thus,
625 /// this API dynamically allocates the dictionary as-needed.
626 pub fn initParams(
627 input: *Reader,
628 gpa: Allocator,
629 buffer: []u8,
630 params: Decode.Params,
631 mem_limit: usize,
632 ) !Decompress {
633 return .{
634 .gpa = gpa,
635 .input = input,
636 .buffer = Decode.CircularBuffer.init(params.dict_size, mem_limit),
637 .range_decoder = try RangeDecoder.init(input),
638 .decode = try Decode.init(gpa, params.properties),
639 .reader = .{
640 .buffer = buffer,
641 .vtable = &.{
642 .readVec = readVec,
643 .stream = stream,
644 .discard = discard,
645 },
646 .seek = 0,
647 .end = 0,
648 },
649 .err = null,
650 .unpacked_size = params.unpacked_size,
651 };
652 }
653
654 /// Takes ownership of `buffer` which may be resized with `gpa`.
655 ///
656 /// LZMA was explicitly designed to take advantage of large heap memory
657 /// being available, with a dictionary size anywhere from 4K to 4G. Thus,
658 /// this API dynamically allocates the dictionary as-needed.
659 pub fn initOptions(
660 input: *Reader,
661 gpa: Allocator,
662 buffer: []u8,
663 options: Decode.Options,
664 mem_limit: usize,
665 ) !Decompress {
666 const params = try Decode.Params.readHeader(input, options);
667 return initParams(input, gpa, buffer, params, mem_limit);
668 }
669
670 /// Reclaim ownership of the buffer passed to `init`.
671 pub fn takeBuffer(d: *Decompress) []u8 {
672 const buffer = d.reader.buffer;
673 d.reader.buffer = &.{};
674 return buffer;
675 }
676
677 pub fn deinit(d: *Decompress) void {
678 const gpa = d.gpa;
679 gpa.free(d.reader.buffer);
680 d.buffer.deinit(gpa);
681 d.decode.deinit(gpa);
682 d.* = undefined;
683 }
684
685 fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
686 _ = data;
687 return readIndirect(r);
688 }
689
690 fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
691 _ = w;
692 _ = limit;
693 return readIndirect(r);
694 }
695
696 fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
697 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
698 _ = d;
699 _ = limit;
700 @panic("TODO");
701 }
702
703 fn readIndirect(r: *Reader) Reader.Error!usize {
704 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
705 const gpa = d.gpa;
706 var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
707 allocating.writer.end = r.end;
708 defer {
709 r.buffer = allocating.writer.buffer;
710 r.end = allocating.writer.end;
711 }
712 if (d.decode.state == math.maxInt(usize)) return error.EndOfStream;
713
714 process_next: {
715 if (d.unpacked_size) |unpacked_size| {
716 if (d.buffer.len >= unpacked_size) break :process_next;
717 } else if (d.range_decoder.isFinished()) {
718 break :process_next;
719 }
720 var n_read: u64 = 0;
721 switch (d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder, &n_read) catch |err| switch (err) {
722 error.WriteFailed => {
723 d.err = error.OutOfMemory;
724 return error.ReadFailed;
725 },
726 error.EndOfStream => {
727 d.err = error.EndOfStream;
728 return error.ReadFailed;
729 },
730 else => |e| {
731 d.err = e;
732 return error.ReadFailed;
733 },
734 }) {
735 .more => return 0,
736 .finished => break :process_next,
737 }
738 }
739
740 if (d.unpacked_size) |unpacked_size| {
741 if (d.buffer.len != unpacked_size) {
742 d.err = error.DecompressedSizeMismatch;
743 return error.ReadFailed;
744 }
745 }
746
747 d.buffer.finish(&allocating.writer) catch |err| switch (err) {
748 error.WriteFailed => {
749 d.err = error.OutOfMemory;
750 return error.ReadFailed;
751 },
752 };
753 d.decode.state = math.maxInt(usize);
754 return 0;
755 }
756};
86757
87test {758test {
88 _ = @import("lzma/test.zig");759 _ = @import("lzma/test.zig");
89 _ = @import("lzma/vec2d.zig");
90}760}
lib/std/compress/lzma/decode.zig deleted-379
...@@ -1,379 +0,0 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
6pub const lzbuffer = @import("decode/lzbuffer.zig");
7pub const rangecoder = @import("decode/rangecoder.zig");
8
9const LzCircularBuffer = lzbuffer.LzCircularBuffer;
10const BitTree = rangecoder.BitTree;
11const LenDecoder = rangecoder.LenDecoder;
12const RangeDecoder = rangecoder.RangeDecoder;
13const Vec2D = @import("vec2d.zig").Vec2D;
14
15pub const Options = struct {
16 unpacked_size: UnpackedSize = .read_from_header,
17 memlimit: ?usize = null,
18 allow_incomplete: bool = false,
19};
20
21pub const UnpackedSize = union(enum) {
22 read_from_header,
23 read_header_but_use_provided: ?u64,
24 use_provided: ?u64,
25};
26
27const ProcessingStatus = enum {
28 continue_,
29 finished,
30};
31
32pub const Properties = struct {
33 lc: u4,
34 lp: u3,
35 pb: u3,
36
37 fn validate(self: Properties) void {
38 assert(self.lc <= 8);
39 assert(self.lp <= 4);
40 assert(self.pb <= 4);
41 }
42};
43
44pub const Params = struct {
45 properties: Properties,
46 dict_size: u32,
47 unpacked_size: ?u64,
48
49 pub fn readHeader(reader: anytype, options: Options) !Params {
50 var props = try reader.readByte();
51 if (props >= 225) {
52 return error.CorruptInput;
53 }
54
55 const lc = @as(u4, @intCast(props % 9));
56 props /= 9;
57 const lp = @as(u3, @intCast(props % 5));
58 props /= 5;
59 const pb = @as(u3, @intCast(props));
60
61 const dict_size_provided = try reader.readInt(u32, .little);
62 const dict_size = @max(0x1000, dict_size_provided);
63
64 const unpacked_size = switch (options.unpacked_size) {
65 .read_from_header => blk: {
66 const unpacked_size_provided = try reader.readInt(u64, .little);
67 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
68 break :blk if (marker_mandatory)
69 null
70 else
71 unpacked_size_provided;
72 },
73 .read_header_but_use_provided => |x| blk: {
74 _ = try reader.readInt(u64, .little);
75 break :blk x;
76 },
77 .use_provided => |x| x,
78 };
79
80 return Params{
81 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
82 .dict_size = dict_size,
83 .unpacked_size = unpacked_size,
84 };
85 }
86};
87
88pub const DecoderState = struct {
89 lzma_props: Properties,
90 unpacked_size: ?u64,
91 literal_probs: Vec2D(u16),
92 pos_slot_decoder: [4]BitTree(6),
93 align_decoder: BitTree(4),
94 pos_decoders: [115]u16,
95 is_match: [192]u16,
96 is_rep: [12]u16,
97 is_rep_g0: [12]u16,
98 is_rep_g1: [12]u16,
99 is_rep_g2: [12]u16,
100 is_rep_0long: [192]u16,
101 state: usize,
102 rep: [4]usize,
103 len_decoder: LenDecoder,
104 rep_len_decoder: LenDecoder,
105
106 pub fn init(
107 allocator: Allocator,
108 lzma_props: Properties,
109 unpacked_size: ?u64,
110 ) !DecoderState {
111 return .{
112 .lzma_props = lzma_props,
113 .unpacked_size = unpacked_size,
114 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
115 .pos_slot_decoder = @splat(.{}),
116 .align_decoder = .{},
117 .pos_decoders = @splat(0x400),
118 .is_match = @splat(0x400),
119 .is_rep = @splat(0x400),
120 .is_rep_g0 = @splat(0x400),
121 .is_rep_g1 = @splat(0x400),
122 .is_rep_g2 = @splat(0x400),
123 .is_rep_0long = @splat(0x400),
124 .state = 0,
125 .rep = @splat(0),
126 .len_decoder = .{},
127 .rep_len_decoder = .{},
128 };
129 }
130
131 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
132 self.literal_probs.deinit(allocator);
133 self.* = undefined;
134 }
135
136 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
137 new_props.validate();
138 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
139 self.literal_probs.fill(0x400);
140 } else {
141 self.literal_probs.deinit(allocator);
142 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
143 }
144
145 self.lzma_props = new_props;
146 for (&self.pos_slot_decoder) |*t| t.reset();
147 self.align_decoder.reset();
148 self.pos_decoders = @splat(0x400);
149 self.is_match = @splat(0x400);
150 self.is_rep = @splat(0x400);
151 self.is_rep_g0 = @splat(0x400);
152 self.is_rep_g1 = @splat(0x400);
153 self.is_rep_g2 = @splat(0x400);
154 self.is_rep_0long = @splat(0x400);
155 self.state = 0;
156 self.rep = @splat(0);
157 self.len_decoder.reset();
158 self.rep_len_decoder.reset();
159 }
160
161 fn processNextInner(
162 self: *DecoderState,
163 allocator: Allocator,
164 reader: anytype,
165 writer: anytype,
166 buffer: anytype,
167 decoder: *RangeDecoder,
168 update: bool,
169 ) !ProcessingStatus {
170 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
171
172 if (!try decoder.decodeBit(
173 reader,
174 &self.is_match[(self.state << 4) + pos_state],
175 update,
176 )) {
177 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
178
179 if (update) {
180 try buffer.appendLiteral(allocator, byte, writer);
181
182 self.state = if (self.state < 4)
183 0
184 else if (self.state < 10)
185 self.state - 3
186 else
187 self.state - 6;
188 }
189 return .continue_;
190 }
191
192 var len: usize = undefined;
193 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
194 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
195 if (!try decoder.decodeBit(
196 reader,
197 &self.is_rep_0long[(self.state << 4) + pos_state],
198 update,
199 )) {
200 if (update) {
201 self.state = if (self.state < 7) 9 else 11;
202 const dist = self.rep[0] + 1;
203 try buffer.appendLz(allocator, 1, dist, writer);
204 }
205 return .continue_;
206 }
207 } else {
208 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
209 1
210 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
211 2
212 else
213 3;
214 if (update) {
215 const dist = self.rep[idx];
216 var i = idx;
217 while (i > 0) : (i -= 1) {
218 self.rep[i] = self.rep[i - 1];
219 }
220 self.rep[0] = dist;
221 }
222 }
223
224 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
225
226 if (update) {
227 self.state = if (self.state < 7) 8 else 11;
228 }
229 } else {
230 if (update) {
231 self.rep[3] = self.rep[2];
232 self.rep[2] = self.rep[1];
233 self.rep[1] = self.rep[0];
234 }
235
236 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
237
238 if (update) {
239 self.state = if (self.state < 7) 7 else 10;
240 }
241
242 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
243
244 if (update) {
245 self.rep[0] = rep_0;
246 if (self.rep[0] == 0xFFFF_FFFF) {
247 if (decoder.isFinished()) {
248 return .finished;
249 }
250 return error.CorruptInput;
251 }
252 }
253 }
254
255 if (update) {
256 len += 2;
257
258 const dist = self.rep[0] + 1;
259 try buffer.appendLz(allocator, len, dist, writer);
260 }
261
262 return .continue_;
263 }
264
265 fn processNext(
266 self: *DecoderState,
267 allocator: Allocator,
268 reader: anytype,
269 writer: anytype,
270 buffer: anytype,
271 decoder: *RangeDecoder,
272 ) !ProcessingStatus {
273 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
274 }
275
276 pub fn process(
277 self: *DecoderState,
278 allocator: Allocator,
279 reader: anytype,
280 writer: anytype,
281 buffer: anytype,
282 decoder: *RangeDecoder,
283 ) !ProcessingStatus {
284 process_next: {
285 if (self.unpacked_size) |unpacked_size| {
286 if (buffer.len >= unpacked_size) {
287 break :process_next;
288 }
289 } else if (decoder.isFinished()) {
290 break :process_next;
291 }
292
293 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
294 .continue_ => return .continue_,
295 .finished => break :process_next,
296 }
297 }
298
299 if (self.unpacked_size) |unpacked_size| {
300 if (buffer.len != unpacked_size) {
301 return error.CorruptInput;
302 }
303 }
304
305 return .finished;
306 }
307
308 fn decodeLiteral(
309 self: *DecoderState,
310 reader: anytype,
311 buffer: anytype,
312 decoder: *RangeDecoder,
313 update: bool,
314 ) !u8 {
315 const def_prev_byte = 0;
316 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
317
318 var result: usize = 1;
319 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
320 (prev_byte >> (8 - self.lzma_props.lc));
321 const probs = try self.literal_probs.getMut(lit_state);
322
323 if (self.state >= 7) {
324 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
325
326 while (result < 0x100) {
327 const match_bit = (match_byte >> 7) & 1;
328 match_byte <<= 1;
329 const bit = @intFromBool(try decoder.decodeBit(
330 reader,
331 &probs[((@as(usize, 1) + match_bit) << 8) + result],
332 update,
333 ));
334 result = (result << 1) ^ bit;
335 if (match_bit != bit) {
336 break;
337 }
338 }
339 }
340
341 while (result < 0x100) {
342 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
343 }
344
345 return @as(u8, @truncate(result - 0x100));
346 }
347
348 fn decodeDistance(
349 self: *DecoderState,
350 reader: anytype,
351 decoder: *RangeDecoder,
352 length: usize,
353 update: bool,
354 ) !usize {
355 const len_state = if (length > 3) 3 else length;
356
357 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
358 if (pos_slot < 4)
359 return pos_slot;
360
361 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
362 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
363
364 if (pos_slot < 14) {
365 result += try decoder.parseReverseBitTree(
366 reader,
367 num_direct_bits,
368 &self.pos_decoders,
369 result - pos_slot,
370 update,
371 );
372 } else {
373 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
374 result += try self.align_decoder.parseReverse(reader, decoder, update);
375 }
376
377 return result;
378 }
379};
lib/std/compress/lzma/decode/lzbuffer.zig deleted-228
...@@ -1,228 +0,0 @@
1const std = @import("../../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6
7/// An accumulating buffer for LZ sequences
8pub const LzAccumBuffer = struct {
9 /// Buffer
10 buf: ArrayListUnmanaged(u8),
11
12 /// Buffer memory limit
13 memlimit: usize,
14
15 /// Total number of bytes sent through the buffer
16 len: usize,
17
18 const Self = @This();
19
20 pub fn init(memlimit: usize) Self {
21 return Self{
22 .buf = .{},
23 .memlimit = memlimit,
24 .len = 0,
25 };
26 }
27
28 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
29 try self.buf.append(allocator, byte);
30 self.len += 1;
31 }
32
33 /// Reset the internal dictionary
34 pub fn reset(self: *Self, writer: anytype) !void {
35 try writer.writeAll(self.buf.items);
36 self.buf.clearRetainingCapacity();
37 self.len = 0;
38 }
39
40 /// Retrieve the last byte or return a default
41 pub fn lastOr(self: Self, lit: u8) u8 {
42 const buf_len = self.buf.items.len;
43 return if (buf_len == 0)
44 lit
45 else
46 self.buf.items[buf_len - 1];
47 }
48
49 /// Retrieve the n-th last byte
50 pub fn lastN(self: Self, dist: usize) !u8 {
51 const buf_len = self.buf.items.len;
52 if (dist > buf_len) {
53 return error.CorruptInput;
54 }
55
56 return self.buf.items[buf_len - dist];
57 }
58
59 /// Append a literal
60 pub fn appendLiteral(
61 self: *Self,
62 allocator: Allocator,
63 lit: u8,
64 writer: anytype,
65 ) !void {
66 _ = writer;
67 if (self.len >= self.memlimit) {
68 return error.CorruptInput;
69 }
70 try self.buf.append(allocator, lit);
71 self.len += 1;
72 }
73
74 /// Fetch an LZ sequence (length, distance) from inside the buffer
75 pub fn appendLz(
76 self: *Self,
77 allocator: Allocator,
78 len: usize,
79 dist: usize,
80 writer: anytype,
81 ) !void {
82 _ = writer;
83
84 const buf_len = self.buf.items.len;
85 if (dist > buf_len) {
86 return error.CorruptInput;
87 }
88
89 var offset = buf_len - dist;
90 var i: usize = 0;
91 while (i < len) : (i += 1) {
92 const x = self.buf.items[offset];
93 try self.buf.append(allocator, x);
94 offset += 1;
95 }
96 self.len += len;
97 }
98
99 pub fn finish(self: *Self, writer: anytype) !void {
100 try writer.writeAll(self.buf.items);
101 self.buf.clearRetainingCapacity();
102 }
103
104 pub fn deinit(self: *Self, allocator: Allocator) void {
105 self.buf.deinit(allocator);
106 self.* = undefined;
107 }
108};
109
110/// A circular buffer for LZ sequences
111pub const LzCircularBuffer = struct {
112 /// Circular buffer
113 buf: ArrayListUnmanaged(u8),
114
115 /// Length of the buffer
116 dict_size: usize,
117
118 /// Buffer memory limit
119 memlimit: usize,
120
121 /// Current position
122 cursor: usize,
123
124 /// Total number of bytes sent through the buffer
125 len: usize,
126
127 const Self = @This();
128
129 pub fn init(dict_size: usize, memlimit: usize) Self {
130 return Self{
131 .buf = .{},
132 .dict_size = dict_size,
133 .memlimit = memlimit,
134 .cursor = 0,
135 .len = 0,
136 };
137 }
138
139 pub fn get(self: Self, index: usize) u8 {
140 return if (0 <= index and index < self.buf.items.len)
141 self.buf.items[index]
142 else
143 0;
144 }
145
146 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
147 if (index >= self.memlimit) {
148 return error.CorruptInput;
149 }
150 try self.buf.ensureTotalCapacity(allocator, index + 1);
151 while (self.buf.items.len < index) {
152 self.buf.appendAssumeCapacity(0);
153 }
154 self.buf.appendAssumeCapacity(value);
155 }
156
157 /// Retrieve the last byte or return a default
158 pub fn lastOr(self: Self, lit: u8) u8 {
159 return if (self.len == 0)
160 lit
161 else
162 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
163 }
164
165 /// Retrieve the n-th last byte
166 pub fn lastN(self: Self, dist: usize) !u8 {
167 if (dist > self.dict_size or dist > self.len) {
168 return error.CorruptInput;
169 }
170
171 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
172 return self.get(offset);
173 }
174
175 /// Append a literal
176 pub fn appendLiteral(
177 self: *Self,
178 allocator: Allocator,
179 lit: u8,
180 writer: anytype,
181 ) !void {
182 try self.set(allocator, self.cursor, lit);
183 self.cursor += 1;
184 self.len += 1;
185
186 // Flush the circular buffer to the output
187 if (self.cursor == self.dict_size) {
188 try writer.writeAll(self.buf.items);
189 self.cursor = 0;
190 }
191 }
192
193 /// Fetch an LZ sequence (length, distance) from inside the buffer
194 pub fn appendLz(
195 self: *Self,
196 allocator: Allocator,
197 len: usize,
198 dist: usize,
199 writer: anytype,
200 ) !void {
201 if (dist > self.dict_size or dist > self.len) {
202 return error.CorruptInput;
203 }
204
205 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
206 var i: usize = 0;
207 while (i < len) : (i += 1) {
208 const x = self.get(offset);
209 try self.appendLiteral(allocator, x, writer);
210 offset += 1;
211 if (offset == self.dict_size) {
212 offset = 0;
213 }
214 }
215 }
216
217 pub fn finish(self: *Self, writer: anytype) !void {
218 if (self.cursor > 0) {
219 try writer.writeAll(self.buf.items[0..self.cursor]);
220 self.cursor = 0;
221 }
222 }
223
224 pub fn deinit(self: *Self, allocator: Allocator) void {
225 self.buf.deinit(allocator);
226 self.* = undefined;
227 }
228};
lib/std/compress/lzma/decode/rangecoder.zig deleted-181
...@@ -1,181 +0,0 @@
1const std = @import("../../../std.zig");
2const mem = std.mem;
3
4pub const RangeDecoder = struct {
5 range: u32,
6 code: u32,
7
8 pub fn init(reader: anytype) !RangeDecoder {
9 const reserved = try reader.readByte();
10 if (reserved != 0) {
11 return error.CorruptInput;
12 }
13 return RangeDecoder{
14 .range = 0xFFFF_FFFF,
15 .code = try reader.readInt(u32, .big),
16 };
17 }
18
19 pub fn fromParts(
20 range: u32,
21 code: u32,
22 ) RangeDecoder {
23 return .{
24 .range = range,
25 .code = code,
26 };
27 }
28
29 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
30 self.range = range;
31 self.code = code;
32 }
33
34 pub inline fn isFinished(self: RangeDecoder) bool {
35 return self.code == 0;
36 }
37
38 inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
39 if (self.range < 0x0100_0000) {
40 self.range <<= 8;
41 self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
42 }
43 }
44
45 inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
46 self.range >>= 1;
47
48 const bit = self.code >= self.range;
49 if (bit)
50 self.code -= self.range;
51
52 try self.normalize(reader);
53 return bit;
54 }
55
56 pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
57 var result: u32 = 0;
58 var i: usize = 0;
59 while (i < count) : (i += 1)
60 result = (result << 1) ^ @intFromBool(try self.getBit(reader));
61 return result;
62 }
63
64 pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
65 const bound = (self.range >> 11) * prob.*;
66
67 if (self.code < bound) {
68 if (update)
69 prob.* += (0x800 - prob.*) >> 5;
70 self.range = bound;
71
72 try self.normalize(reader);
73 return false;
74 } else {
75 if (update)
76 prob.* -= prob.* >> 5;
77 self.code -= bound;
78 self.range -= bound;
79
80 try self.normalize(reader);
81 return true;
82 }
83 }
84
85 fn parseBitTree(
86 self: *RangeDecoder,
87 reader: anytype,
88 num_bits: u5,
89 probs: []u16,
90 update: bool,
91 ) !u32 {
92 var tmp: u32 = 1;
93 var i: @TypeOf(num_bits) = 0;
94 while (i < num_bits) : (i += 1) {
95 const bit = try self.decodeBit(reader, &probs[tmp], update);
96 tmp = (tmp << 1) ^ @intFromBool(bit);
97 }
98 return tmp - (@as(u32, 1) << num_bits);
99 }
100
101 pub fn parseReverseBitTree(
102 self: *RangeDecoder,
103 reader: anytype,
104 num_bits: u5,
105 probs: []u16,
106 offset: usize,
107 update: bool,
108 ) !u32 {
109 var result: u32 = 0;
110 var tmp: usize = 1;
111 var i: @TypeOf(num_bits) = 0;
112 while (i < num_bits) : (i += 1) {
113 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], update));
114 tmp = (tmp << 1) ^ bit;
115 result ^= @as(u32, bit) << i;
116 }
117 return result;
118 }
119};
120
121pub fn BitTree(comptime num_bits: usize) type {
122 return struct {
123 probs: [1 << num_bits]u16 = @splat(0x400),
124
125 const Self = @This();
126
127 pub fn parse(
128 self: *Self,
129 reader: anytype,
130 decoder: *RangeDecoder,
131 update: bool,
132 ) !u32 {
133 return decoder.parseBitTree(reader, num_bits, &self.probs, update);
134 }
135
136 pub fn parseReverse(
137 self: *Self,
138 reader: anytype,
139 decoder: *RangeDecoder,
140 update: bool,
141 ) !u32 {
142 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
143 }
144
145 pub fn reset(self: *Self) void {
146 @memset(&self.probs, 0x400);
147 }
148 };
149}
150
151pub const LenDecoder = struct {
152 choice: u16 = 0x400,
153 choice2: u16 = 0x400,
154 low_coder: [16]BitTree(3) = @splat(.{}),
155 mid_coder: [16]BitTree(3) = @splat(.{}),
156 high_coder: BitTree(8) = .{},
157
158 pub fn decode(
159 self: *LenDecoder,
160 reader: anytype,
161 decoder: *RangeDecoder,
162 pos_state: usize,
163 update: bool,
164 ) !usize {
165 if (!try decoder.decodeBit(reader, &self.choice, update)) {
166 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
167 } else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
168 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
169 } else {
170 return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
171 }
172 }
173
174 pub fn reset(self: *LenDecoder) void {
175 self.choice = 0x400;
176 self.choice2 = 0x400;
177 for (&self.low_coder) |*t| t.reset();
178 for (&self.mid_coder) |*t| t.reset();
179 self.high_coder.reset();
180 }
181};
lib/std/compress/lzma/test.zig+24-15
...@@ -1,24 +1,31 @@...@@ -1,24 +1,31 @@
1const std = @import("../../std.zig");1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");2const lzma = std.compress.lzma;
33
4fn testDecompress(compressed: []const u8) ![]u8 {4fn testDecompress(compressed: []const u8) ![]u8 {
5 const allocator = std.testing.allocator;5 const gpa = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);6 var stream: std.Io.Reader = .fixed(compressed);
7 var decompressor = try lzma.decompress(allocator, stream.reader());7
8 var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
8 defer decompressor.deinit();9 defer decompressor.deinit();
9 const reader = decompressor.reader();10 return decompressor.reader.allocRemaining(gpa, .unlimited);
10 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
11}11}
1212
13fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {13fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14 const allocator = std.testing.allocator;14 const gpa = std.testing.allocator;
15 const decomp = try testDecompress(compressed);15 const decomp = try testDecompress(compressed);
16 defer allocator.free(decomp);16 defer gpa.free(decomp);
17 try std.testing.expectEqualSlices(u8, expected, decomp);17 try std.testing.expectEqualSlices(u8, expected, decomp);
18}18}
1919
20fn testDecompressError(expected: anyerror, compressed: []const u8) !void {20fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
21 return std.testing.expectError(expected, testDecompress(compressed));21 const gpa = std.testing.allocator;
22 var stream: std.Io.Reader = .fixed(compressed);
23
24 var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
25 defer decompressor.deinit();
26
27 try std.testing.expectError(error.ReadFailed, decompressor.reader.allocRemaining(gpa, .unlimited));
28 try std.testing.expectEqual(expected, decompressor.err orelse return error.TestFailed);
22}29}
2330
24test "decompress empty world" {31test "decompress empty world" {
...@@ -76,24 +83,26 @@ test "known size with end of payload marker" {...@@ -76,24 +83,26 @@ test "known size with end of payload marker" {
7683
77test "too big uncompressed size in header" {84test "too big uncompressed size in header" {
78 try testDecompressError(85 try testDecompressError(
79 error.CorruptInput,86 error.DecompressedSizeMismatch,
80 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),87 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
81 );88 );
82}89}
8390
84test "too small uncompressed size in header" {91test "too small uncompressed size in header" {
85 try testDecompressError(92 try testDecompressError(
86 error.CorruptInput,93 error.DecompressedSizeMismatch,
87 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),94 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
88 );95 );
89}96}
9097
91test "reading one byte" {98test "reading one byte" {
99 const gpa = std.testing.allocator;
92 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");100 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
93 var stream = std.io.fixedBufferStream(compressed);101 var stream: std.Io.Reader = .fixed(compressed);
94 var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());102 var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
95 defer decompressor.deinit();103 defer decompressor.deinit();
96104
97 var buffer = [1]u8{0};105 var buffer: [1]u8 = undefined;
98 _ = try decompressor.read(buffer[0..]);106 try decompressor.reader.readSliceAll(&buffer);
107 try std.testing.expectEqual(72, buffer[0]);
99}108}
lib/std/compress/lzma/vec2d.zig deleted-128
...@@ -1,128 +0,0 @@
1const std = @import("../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub fn Vec2D(comptime T: type) type {
7 return struct {
8 data: []T,
9 cols: usize,
10
11 const Self = @This();
12
13 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
14 const len = try math.mul(usize, size[0], size[1]);
15 const data = try allocator.alloc(T, len);
16 @memset(data, value);
17 return Self{
18 .data = data,
19 .cols = size[1],
20 };
21 }
22
23 pub fn deinit(self: *Self, allocator: Allocator) void {
24 allocator.free(self.data);
25 self.* = undefined;
26 }
27
28 pub fn fill(self: *Self, value: T) void {
29 @memset(self.data, value);
30 }
31
32 inline fn _get(self: Self, row: usize) ![]T {
33 const start_row = try math.mul(usize, row, self.cols);
34 const end_row = try math.add(usize, start_row, self.cols);
35 return self.data[start_row..end_row];
36 }
37
38 pub fn get(self: Self, row: usize) ![]const T {
39 return self._get(row);
40 }
41
42 pub fn getMut(self: *Self, row: usize) ![]T {
43 return self._get(row);
44 }
45 };
46}
47
48const testing = std.testing;
49const expectEqualSlices = std.testing.expectEqualSlices;
50const expectError = std.testing.expectError;
51
52test "init" {
53 const allocator = testing.allocator;
54 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
55 defer vec2d.deinit(allocator);
56
57 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
58 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
59}
60
61test "init overflow" {
62 const allocator = testing.allocator;
63 try expectError(
64 error.Overflow,
65 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
66 );
67}
68
69test "fill" {
70 const allocator = testing.allocator;
71 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
72 defer vec2d.deinit(allocator);
73
74 vec2d.fill(7);
75
76 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
77 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
78}
79
80test "get" {
81 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
82 const vec2d = Vec2D(i32){
83 .data = &data,
84 .cols = 2,
85 };
86
87 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
88 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
89 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
90 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
91}
92
93test "getMut" {
94 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
95 var vec2d = Vec2D(i32){
96 .data = &data,
97 .cols = 2,
98 };
99
100 const row = try vec2d.getMut(1);
101 row[1] = 9;
102
103 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
104 // (1, 1) should be 9.
105 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
106 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
107 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
108}
109
110test "get multiplication overflow" {
111 const allocator = testing.allocator;
112 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
113 defer matrix.deinit(allocator);
114
115 const row = (math.maxInt(usize) / 4) + 1;
116 try expectError(error.Overflow, matrix.get(row));
117 try expectError(error.Overflow, matrix.getMut(row));
118}
119
120test "get addition overflow" {
121 const allocator = testing.allocator;
122 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
123 defer matrix.deinit(allocator);
124
125 const row = math.maxInt(usize) / 5;
126 try expectError(error.Overflow, matrix.get(row));
127 try expectError(error.Overflow, matrix.getMut(row));
128}
lib/std/compress/lzma2.zig+273-17
...@@ -1,26 +1,282 @@...@@ -1,26 +1,282 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;2const Allocator = std.mem.Allocator;
3const ArrayList = std.ArrayList;
4const lzma = std.compress.lzma;
5const Writer = std.Io.Writer;
6const Reader = std.Io.Reader;
37
4pub const decode = @import("lzma2/decode.zig");8/// An accumulating buffer for LZ sequences
9pub const AccumBuffer = struct {
10 /// Buffer
11 buf: ArrayList(u8),
12 /// Buffer memory limit
13 memlimit: usize,
14 /// Total number of bytes sent through the buffer
15 len: usize,
516
6pub fn decompress(17 pub fn init(memlimit: usize) AccumBuffer {
7 allocator: Allocator,18 return .{
8 reader: anytype,19 .buf = .{},
9 writer: anytype,20 .memlimit = memlimit,
10) !void {21 .len = 0,
11 var decoder = try decode.Decoder.init(allocator);22 };
12 defer decoder.deinit(allocator);23 }
13 return decoder.decompress(allocator, reader, writer);24
14}25 pub fn appendByte(self: *AccumBuffer, allocator: Allocator, byte: u8) !void {
26 try self.buf.append(allocator, byte);
27 self.len += 1;
28 }
29
30 /// Reset the internal dictionary
31 pub fn reset(self: *AccumBuffer, writer: *Writer) !void {
32 try writer.writeAll(self.buf.items);
33 self.buf.clearRetainingCapacity();
34 self.len = 0;
35 }
36
37 /// Retrieve the last byte or return a default
38 pub fn lastOr(self: AccumBuffer, lit: u8) u8 {
39 const buf_len = self.buf.items.len;
40 return if (buf_len == 0)
41 lit
42 else
43 self.buf.items[buf_len - 1];
44 }
45
46 /// Retrieve the n-th last byte
47 pub fn lastN(self: AccumBuffer, dist: usize) !u8 {
48 const buf_len = self.buf.items.len;
49 if (dist > buf_len) {
50 return error.CorruptInput;
51 }
52
53 return self.buf.items[buf_len - dist];
54 }
55
56 /// Append a literal
57 pub fn appendLiteral(
58 self: *AccumBuffer,
59 allocator: Allocator,
60 lit: u8,
61 writer: *Writer,
62 ) !void {
63 _ = writer;
64 if (self.len >= self.memlimit) {
65 return error.CorruptInput;
66 }
67 try self.buf.append(allocator, lit);
68 self.len += 1;
69 }
70
71 /// Fetch an LZ sequence (length, distance) from inside the buffer
72 pub fn appendLz(
73 self: *AccumBuffer,
74 allocator: Allocator,
75 len: usize,
76 dist: usize,
77 writer: *Writer,
78 ) !void {
79 _ = writer;
80
81 const buf_len = self.buf.items.len;
82 if (dist > buf_len) {
83 return error.CorruptInput;
84 }
85
86 var offset = buf_len - dist;
87 var i: usize = 0;
88 while (i < len) : (i += 1) {
89 const x = self.buf.items[offset];
90 try self.buf.append(allocator, x);
91 offset += 1;
92 }
93 self.len += len;
94 }
95
96 pub fn finish(self: *AccumBuffer, writer: *Writer) !void {
97 try writer.writeAll(self.buf.items);
98 self.buf.clearRetainingCapacity();
99 }
100
101 pub fn deinit(self: *AccumBuffer, allocator: Allocator) void {
102 self.buf.deinit(allocator);
103 self.* = undefined;
104 }
105};
106
107pub const Decode = struct {
108 lzma_decode: lzma.Decode,
109
110 pub fn init(gpa: Allocator) !Decode {
111 return .{ .lzma_decode = try lzma.Decode.init(gpa, .{ .lc = 0, .lp = 0, .pb = 0 }) };
112 }
113
114 pub fn deinit(self: *Decode, gpa: Allocator) void {
115 self.lzma_decode.deinit(gpa);
116 self.* = undefined;
117 }
118
119 /// Returns how many compressed bytes were consumed.
120 pub fn decompress(d: *Decode, reader: *Reader, allocating: *Writer.Allocating) !u64 {
121 const gpa = allocating.allocator;
122
123 var accum = AccumBuffer.init(std.math.maxInt(usize));
124 defer accum.deinit(gpa);
125
126 var n_read: u64 = 0;
127
128 while (true) {
129 const status = try reader.takeByte();
130 n_read += 1;
131
132 switch (status) {
133 0 => break,
134 1 => n_read += try parseUncompressed(reader, allocating, &accum, true),
135 2 => n_read += try parseUncompressed(reader, allocating, &accum, false),
136 else => n_read += try d.parseLzma(reader, allocating, &accum, status),
137 }
138 }
139
140 try accum.finish(&allocating.writer);
141 return n_read;
142 }
143
144 fn parseLzma(
145 d: *Decode,
146 reader: *Reader,
147 allocating: *Writer.Allocating,
148 accum: *AccumBuffer,
149 status: u8,
150 ) !u64 {
151 if (status & 0x80 == 0) return error.CorruptInput;
152
153 const Reset = struct {
154 dict: bool,
155 state: bool,
156 props: bool,
157 };
15158
16test {159 const reset: Reset = switch ((status >> 5) & 0x3) {
160 0 => .{
161 .dict = false,
162 .state = false,
163 .props = false,
164 },
165 1 => .{
166 .dict = false,
167 .state = true,
168 .props = false,
169 },
170 2 => .{
171 .dict = false,
172 .state = true,
173 .props = true,
174 },
175 3 => .{
176 .dict = true,
177 .state = true,
178 .props = true,
179 },
180 else => unreachable,
181 };
182
183 var n_read: u64 = 0;
184
185 const unpacked_size = blk: {
186 var tmp: u64 = status & 0x1F;
187 tmp <<= 16;
188 tmp |= try reader.takeInt(u16, .big);
189 n_read += 2;
190 break :blk tmp + 1;
191 };
192
193 const packed_size = blk: {
194 const tmp: u17 = try reader.takeInt(u16, .big);
195 n_read += 2;
196 break :blk tmp + 1;
197 };
198
199 if (reset.dict) try accum.reset(&allocating.writer);
200
201 const ld = &d.lzma_decode;
202
203 if (reset.state) {
204 var new_props = ld.properties;
205
206 if (reset.props) {
207 var props = try reader.takeByte();
208 n_read += 1;
209 if (props >= 225) {
210 return error.CorruptInput;
211 }
212
213 const lc = @as(u4, @intCast(props % 9));
214 props /= 9;
215 const lp = @as(u3, @intCast(props % 5));
216 props /= 5;
217 const pb = @as(u3, @intCast(props));
218
219 if (lc + lp > 4) {
220 return error.CorruptInput;
221 }
222
223 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
224 }
225
226 try ld.resetState(allocating.allocator, new_props);
227 }
228
229 const expected_unpacked_size = accum.len + unpacked_size;
230 const start_count = n_read;
231 var range_decoder = try lzma.RangeDecoder.initCounting(reader, &n_read);
232
233 while (true) {
234 if (accum.len >= expected_unpacked_size) break;
235 if (range_decoder.isFinished()) break;
236 switch (try ld.process(reader, allocating, accum, &range_decoder, &n_read)) {
237 .more => continue,
238 .finished => break,
239 }
240 }
241 if (accum.len != expected_unpacked_size) return error.DecompressedSizeMismatch;
242 if (n_read - start_count != packed_size) return error.CompressedSizeMismatch;
243
244 return n_read;
245 }
246
247 fn parseUncompressed(
248 reader: *Reader,
249 allocating: *Writer.Allocating,
250 accum: *AccumBuffer,
251 reset_dict: bool,
252 ) !usize {
253 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
254
255 if (reset_dict) try accum.reset(&allocating.writer);
256
257 const gpa = allocating.allocator;
258
259 for (0..unpacked_size) |_| {
260 try accum.appendByte(gpa, try reader.takeByte());
261 }
262 return 2 + unpacked_size;
263 }
264};
265
266test "decompress hello world stream" {
17 const expected = "Hello\nWorld!\n";267 const expected = "Hello\nWorld!\n";
18 const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };268 const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
19269
20 const allocator = std.testing.allocator;270 const gpa = std.testing.allocator;
21 var decomp = std.array_list.Managed(u8).init(allocator);271
22 defer decomp.deinit();272 var decode = try Decode.init(gpa);
23 var stream = std.io.fixedBufferStream(compressed);273 defer decode.deinit(gpa);
24 try decompress(allocator, stream.reader(), decomp.writer());274
25 try std.testing.expectEqualSlices(u8, expected, decomp.items);275 var stream: std.Io.Reader = .fixed(compressed);
276 var result: std.Io.Writer.Allocating = .init(gpa);
277 defer result.deinit();
278
279 const n_read = try decode.decompress(&stream, &result);
280 try std.testing.expectEqual(compressed.len, n_read);
281 try std.testing.expectEqualStrings(expected, result.written());
26}282}
lib/std/compress/lzma2/decode.zig deleted-169
...@@ -1,169 +0,0 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.readByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try reader.readInt(u16, .big);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try reader.readInt(u16, .big);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try reader.readByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @as(u4, @intCast(props % 9));
123 props /= 9;
124 const lp = @as(u3, @intCast(props % 5));
125 props /= 5;
126 const pb = @as(u3, @intCast(props));
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
142
143 var rangecoder = try RangeDecoder.init(counter_reader);
144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
167 }
168 }
169};
lib/std/compress/xz.zig+1-162
...@@ -1,165 +1,4 @@...@@ -1,165 +1,4 @@
1const std = @import("std");1pub const Decompress = @import("xz/Decompress.zig");
2const block = @import("xz/block.zig");
3const Allocator = std.mem.Allocator;
4const Crc32 = std.hash.Crc32;
5
6pub const Check = enum(u4) {
7 none = 0x00,
8 crc32 = 0x01,
9 crc64 = 0x04,
10 sha256 = 0x0A,
11 _,
12};
13
14fn readStreamFlags(reader: anytype, check: *Check) !void {
15 const reserved1 = try reader.readByte();
16 if (reserved1 != 0) return error.CorruptInput;
17 const byte = try reader.readByte();
18 if ((byte >> 4) != 0) return error.CorruptInput;
19 check.* = @enumFromInt(@as(u4, @truncate(byte)));
20}
21
22pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
23 return Decompress(@TypeOf(reader)).init(allocator, reader);
24}
25
26pub fn Decompress(comptime ReaderType: type) type {
27 return struct {
28 const Self = @This();
29
30 pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
31 pub const Reader = std.io.GenericReader(*Self, Error, read);
32
33 allocator: Allocator,
34 block_decoder: block.Decoder(ReaderType),
35 in_reader: ReaderType,
36
37 fn init(allocator: Allocator, source: ReaderType) !Self {
38 const magic = try source.readBytesNoEof(6);
39 if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
40 return error.BadHeader;
41
42 var check: Check = undefined;
43 const hash_a = blk: {
44 var hasher = hashedReader(source, Crc32.init());
45 try readStreamFlags(hasher.reader(), &check);
46 break :blk hasher.hasher.final();
47 };
48
49 const hash_b = try source.readInt(u32, .little);
50 if (hash_a != hash_b)
51 return error.WrongChecksum;
52
53 return Self{
54 .allocator = allocator,
55 .block_decoder = try block.decoder(allocator, source, check),
56 .in_reader = source,
57 };
58 }
59
60 pub fn deinit(self: *Self) void {
61 self.block_decoder.deinit();
62 }
63
64 pub fn reader(self: *Self) Reader {
65 return .{ .context = self };
66 }
67
68 pub fn read(self: *Self, buffer: []u8) Error!usize {
69 if (buffer.len == 0)
70 return 0;
71
72 const r = try self.block_decoder.read(buffer);
73 if (r != 0)
74 return r;
75
76 const index_size = blk: {
77 var hasher = hashedReader(self.in_reader, Crc32.init());
78 hasher.hasher.update(&[1]u8{0x00});
79
80 var counter = std.io.countingReader(hasher.reader());
81 counter.bytes_read += 1;
82
83 const counting_reader = counter.reader();
84
85 const record_count = try std.leb.readUleb128(u64, counting_reader);
86 if (record_count != self.block_decoder.block_count)
87 return error.CorruptInput;
88
89 var i: usize = 0;
90 while (i < record_count) : (i += 1) {
91 // TODO: validate records
92 _ = try std.leb.readUleb128(u64, counting_reader);
93 _ = try std.leb.readUleb128(u64, counting_reader);
94 }
95
96 while (counter.bytes_read % 4 != 0) {
97 if (try counting_reader.readByte() != 0)
98 return error.CorruptInput;
99 }
100
101 const hash_a = hasher.hasher.final();
102 const hash_b = try counting_reader.readInt(u32, .little);
103 if (hash_a != hash_b)
104 return error.WrongChecksum;
105
106 break :blk counter.bytes_read;
107 };
108
109 const hash_a = try self.in_reader.readInt(u32, .little);
110
111 const hash_b = blk: {
112 var hasher = hashedReader(self.in_reader, Crc32.init());
113 const hashed_reader = hasher.reader();
114
115 const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
116 if (backward_size != index_size)
117 return error.CorruptInput;
118
119 var check: Check = undefined;
120 try readStreamFlags(hashed_reader, &check);
121
122 break :blk hasher.hasher.final();
123 };
124
125 if (hash_a != hash_b)
126 return error.WrongChecksum;
127
128 const magic = try self.in_reader.readBytesNoEof(2);
129 if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
130 return error.CorruptInput;
131
132 return 0;
133 }
134 };
135}
136
137pub fn HashedReader(ReaderType: type, HasherType: type) type {
138 return struct {
139 child_reader: ReaderType,
140 hasher: HasherType,
141
142 pub const Error = ReaderType.Error;
143 pub const Reader = std.io.GenericReader(*@This(), Error, read);
144
145 pub fn read(self: *@This(), buf: []u8) Error!usize {
146 const amt = try self.child_reader.read(buf);
147 self.hasher.update(buf[0..amt]);
148 return amt;
149 }
150
151 pub fn reader(self: *@This()) Reader {
152 return .{ .context = self };
153 }
154 };
155}
156
157pub fn hashedReader(
158 reader: anytype,
159 hasher: anytype,
160) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
161 return .{ .child_reader = reader, .hasher = hasher };
162}
1632
164test {3test {
165 _ = @import("xz/test.zig");4 _ = @import("xz/test.zig");
lib/std/compress/xz/Decompress.zig created+319
...@@ -0,0 +1,319 @@
1const Decompress = @This();
2const std = @import("../../std.zig");
3const Allocator = std.mem.Allocator;
4const ArrayList = std.ArrayList;
5const Crc32 = std.hash.Crc32;
6const Crc64 = std.hash.crc.Crc64Xz;
7const Sha256 = std.crypto.hash.sha2.Sha256;
8const lzma2 = std.compress.lzma2;
9const Writer = std.Io.Writer;
10const Reader = std.Io.Reader;
11const assert = std.debug.assert;
12
13/// Underlying compressed data stream to pull bytes from.
14input: *Reader,
15/// Uncompressed bytes output by this stream implementation.
16reader: Reader,
17gpa: Allocator,
18check: Check,
19block_count: usize,
20err: ?Error,
21
22pub const Error = error{
23 ReadFailed,
24 OutOfMemory,
25 CorruptInput,
26 EndOfStream,
27 WrongChecksum,
28 Unsupported,
29 Overflow,
30 InvalidRangeCode,
31 DecompressedSizeMismatch,
32 CompressedSizeMismatch,
33};
34
35pub const Check = enum(u4) {
36 none = 0x00,
37 crc32 = 0x01,
38 crc64 = 0x04,
39 sha256 = 0x0A,
40 _,
41};
42
43pub const StreamFlags = packed struct(u16) {
44 null: u8 = 0,
45 check: Check,
46 reserved: u4 = 0,
47};
48
49pub const InitError = error{
50 NotXzStream,
51 WrongChecksum,
52};
53
54/// XZ uses a series of LZMA2 blocks which each specify a dictionary size
55/// anywhere from 4K to 4G. Thus, this API dynamically allocates the dictionary
56/// as-needed.
57pub fn init(
58 input: *Reader,
59 gpa: Allocator,
60 /// Decompress takes ownership of this buffer and resizes it with `gpa`.
61 buffer: []u8,
62) !Decompress {
63 const magic = try input.takeArray(6);
64 if (!std.mem.eql(u8, magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
65 return error.NotXzStream;
66
67 const computed_checksum = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
68 const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
69 const stored_hash = try input.takeInt(u32, .little);
70 if (computed_checksum != stored_hash) return error.WrongChecksum;
71
72 return .{
73 .input = input,
74 .reader = .{
75 .vtable = &.{
76 .stream = stream,
77 .readVec = readVec,
78 .discard = discard,
79 },
80 .buffer = buffer,
81 .seek = 0,
82 .end = 0,
83 },
84 .gpa = gpa,
85 .check = stream_flags.check,
86 .block_count = 0,
87 .err = null,
88 };
89}
90
91/// Reclaim ownership of the buffer passed to `init`.
92pub fn takeBuffer(d: *Decompress) []u8 {
93 const buffer = d.reader.buffer;
94 d.reader.buffer = &.{};
95 return buffer;
96}
97
98pub fn deinit(d: *Decompress) void {
99 const gpa = d.gpa;
100 gpa.free(d.reader.buffer);
101 d.* = undefined;
102}
103
104fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
105 _ = data;
106 return readIndirect(r);
107}
108
109fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
110 _ = w;
111 _ = limit;
112 return readIndirect(r);
113}
114
115fn discard(r: *Reader, limit: std.Io.Limit) Reader.Error!usize {
116 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
117 _ = d;
118 _ = limit;
119 @panic("TODO");
120}
121
122fn readIndirect(r: *Reader) Reader.Error!usize {
123 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
124 const gpa = d.gpa;
125 const input = d.input;
126
127 var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
128 allocating.writer.end = r.end;
129 defer {
130 r.buffer = allocating.writer.buffer;
131 r.end = allocating.writer.end;
132 }
133
134 if (d.err != null) return error.ReadFailed;
135 if (d.block_count == std.math.maxInt(usize)) return error.EndOfStream;
136
137 readBlock(input, &allocating) catch |err| switch (err) {
138 error.WriteFailed => {
139 d.err = error.OutOfMemory;
140 return error.ReadFailed;
141 },
142 error.SuccessfulEndOfStream => {
143 finish(d) catch |finish_err| {
144 d.err = finish_err;
145 return error.ReadFailed;
146 };
147 d.block_count = std.math.maxInt(usize);
148 return error.EndOfStream;
149 },
150 else => |e| {
151 d.err = e;
152 return error.ReadFailed;
153 },
154 };
155 switch (d.check) {
156 .none => {},
157 .crc32 => {
158 const declared_checksum = try input.takeInt(u32, .little);
159 // TODO
160 //const hash_a = Crc32.hash(unpacked_bytes);
161 //if (hash_a != hash_b) return error.WrongChecksum;
162 _ = declared_checksum;
163 },
164 .crc64 => {
165 const declared_checksum = try input.takeInt(u64, .little);
166 // TODO
167 //const hash_a = Crc64.hash(unpacked_bytes);
168 //if (hash_a != hash_b) return error.WrongChecksum;
169 _ = declared_checksum;
170 },
171 .sha256 => {
172 const declared_hash = try input.take(Sha256.digest_length);
173 // TODO
174 //var hash_a: [Sha256.digest_length]u8 = undefined;
175 //Sha256.hash(unpacked_bytes, &hash_a, .{});
176 //if (!std.mem.eql(u8, &hash_a, &hash_b))
177 // return error.WrongChecksum;
178 _ = declared_hash;
179 },
180 else => {
181 d.err = error.Unsupported;
182 return error.ReadFailed;
183 },
184 }
185 d.block_count += 1;
186 return 0;
187}
188
189fn readBlock(input: *Reader, allocating: *Writer.Allocating) !void {
190 var packed_size: ?u64 = null;
191 var unpacked_size: ?u64 = null;
192
193 const header_size = h: {
194 // Read the block header via peeking so that we can hash the whole thing too.
195 const first_byte: usize = try input.peekByte();
196 if (first_byte == 0) return error.SuccessfulEndOfStream;
197
198 const declared_header_size = first_byte * 4;
199 try input.fill(declared_header_size);
200 const header_seek_start = input.seek;
201 input.toss(1);
202
203 const Flags = packed struct(u8) {
204 last_filter_index: u2,
205 reserved: u4,
206 has_packed_size: bool,
207 has_unpacked_size: bool,
208 };
209 const flags = try input.takeStruct(Flags, .little);
210
211 const filter_count = @as(u3, flags.last_filter_index) + 1;
212 if (filter_count > 1) return error.Unsupported;
213
214 if (flags.has_packed_size) packed_size = try input.takeLeb128(u64);
215 if (flags.has_unpacked_size) unpacked_size = try input.takeLeb128(u64);
216
217 const FilterId = enum(u64) {
218 lzma2 = 0x21,
219 _,
220 };
221
222 const filter_id: FilterId = @enumFromInt(try input.takeLeb128(u64));
223 if (filter_id != .lzma2) return error.Unsupported;
224
225 const properties_size = try input.takeLeb128(u64);
226 if (properties_size != 1) return error.CorruptInput;
227 // TODO: use filter properties
228 _ = try input.takeByte();
229
230 const actual_header_size = input.seek - header_seek_start;
231 if (actual_header_size > declared_header_size) return error.CorruptInput;
232 const remaining_bytes = declared_header_size - actual_header_size;
233 for (0..remaining_bytes) |_| {
234 if (try input.takeByte() != 0) return error.CorruptInput;
235 }
236
237 const header_slice = input.buffer[header_seek_start..][0..declared_header_size];
238 const computed_checksum = Crc32.hash(header_slice);
239 const declared_checksum = try input.takeInt(u32, .little);
240 if (computed_checksum != declared_checksum) return error.WrongChecksum;
241 break :h declared_header_size;
242 };
243
244 // Compressed Data
245
246 var lzma2_decode = try lzma2.Decode.init(allocating.allocator);
247 defer lzma2_decode.deinit(allocating.allocator);
248 const before_size = allocating.writer.end;
249 const packed_bytes_read = try lzma2_decode.decompress(input, allocating);
250 const unpacked_bytes = allocating.writer.end - before_size;
251
252 if (packed_size) |s| {
253 if (s != packed_bytes_read) return error.CorruptInput;
254 }
255
256 if (unpacked_size) |s| {
257 if (s != unpacked_bytes) return error.CorruptInput;
258 }
259
260 // Block Padding
261 const block_counter = header_size + packed_bytes_read;
262 const padding = try input.take(@intCast((4 - (block_counter % 4)) % 4));
263 for (padding) |byte| {
264 if (byte != 0) return error.CorruptInput;
265 }
266}
267
268fn finish(d: *Decompress) !void {
269 const input = d.input;
270 const index_size = blk: {
271 // Assume that we already peeked a zero in readBlock().
272 assert(input.buffered()[0] == 0);
273 var input_counter: u64 = 1;
274 var checksum: Crc32 = .init();
275 checksum.update(&.{0});
276 input.toss(1);
277
278 const record_count = try countLeb128(input, u64, &input_counter, &checksum);
279 if (record_count != d.block_count)
280 return error.CorruptInput;
281
282 for (0..@intCast(record_count)) |_| {
283 // TODO: validate records
284 _ = try countLeb128(input, u64, &input_counter, &checksum);
285 _ = try countLeb128(input, u64, &input_counter, &checksum);
286 }
287
288 const padding = try input.take(@intCast((4 - (input_counter % 4)) % 4));
289 for (padding) |byte| {
290 if (byte != 0) return error.CorruptInput;
291 }
292 checksum.update(padding);
293
294 const declared_checksum = try input.takeInt(u32, .little);
295 const computed_checksum = checksum.final();
296 if (computed_checksum != declared_checksum) return error.WrongChecksum;
297
298 break :blk input_counter + padding.len + 4;
299 };
300
301 const declared_checksum = try input.takeInt(u32, .little);
302 const computed_checksum = Crc32.hash(try input.peek(4 + @sizeOf(StreamFlags)));
303 if (declared_checksum != computed_checksum) return error.WrongChecksum;
304 const backward_size = (@as(u64, try input.takeInt(u32, .little)) + 1) * 4;
305 if (backward_size != index_size) return error.CorruptInput;
306 input.toss(@sizeOf(StreamFlags));
307 if (!std.mem.eql(u8, try input.takeArray(2), &.{ 'Y', 'Z' }))
308 return error.CorruptInput;
309}
310
311fn countLeb128(reader: *Reader, comptime T: type, counter: *u64, hasher: *Crc32) !T {
312 try reader.fill(8);
313 const start = reader.seek;
314 const result = try reader.takeLeb128(T);
315 const read_slice = reader.buffer[start..reader.seek];
316 hasher.update(read_slice);
317 counter.* += read_slice.len;
318 return result;
319}
lib/std/compress/xz/block.zig deleted-208
...@@ -1,208 +0,0 @@
1const std = @import("../../std.zig");
2const lzma2 = std.compress.lzma2;
3const Allocator = std.mem.Allocator;
4const ArrayListUnmanaged = std.ArrayListUnmanaged;
5const Crc32 = std.hash.Crc32;
6const Crc64 = std.hash.crc.Crc64Xz;
7const Sha256 = std.crypto.hash.sha2.Sha256;
8const xz = std.compress.xz;
9
10const DecodeError = error{
11 CorruptInput,
12 EndOfStream,
13 EndOfStreamWithNoError,
14 WrongChecksum,
15 Unsupported,
16 Overflow,
17};
18
19pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
20 return Decoder(@TypeOf(reader)).init(allocator, reader, check);
21}
22
23pub fn Decoder(comptime ReaderType: type) type {
24 return struct {
25 const Self = @This();
26 pub const Error =
27 ReaderType.Error ||
28 DecodeError ||
29 Allocator.Error;
30 pub const Reader = std.io.GenericReader(*Self, Error, read);
31
32 allocator: Allocator,
33 inner_reader: ReaderType,
34 check: xz.Check,
35 err: ?Error,
36 to_read: ArrayListUnmanaged(u8),
37 read_pos: usize,
38 block_count: usize,
39
40 fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
41 return Self{
42 .allocator = allocator,
43 .inner_reader = in_reader,
44 .check = check,
45 .err = null,
46 .to_read = .{},
47 .read_pos = 0,
48 .block_count = 0,
49 };
50 }
51
52 pub fn deinit(self: *Self) void {
53 self.to_read.deinit(self.allocator);
54 }
55
56 pub fn reader(self: *Self) Reader {
57 return .{ .context = self };
58 }
59
60 pub fn read(self: *Self, output: []u8) Error!usize {
61 while (true) {
62 const unread_len = self.to_read.items.len - self.read_pos;
63 if (unread_len > 0) {
64 const n = @min(unread_len, output.len);
65 @memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
66 self.read_pos += n;
67 return n;
68 }
69 if (self.err) |e| {
70 if (e == DecodeError.EndOfStreamWithNoError) {
71 return 0;
72 }
73 return e;
74 }
75 if (self.read_pos > 0) {
76 self.to_read.shrinkRetainingCapacity(0);
77 self.read_pos = 0;
78 }
79 self.readBlock() catch |e| {
80 self.err = e;
81 };
82 }
83 }
84
85 fn readBlock(self: *Self) Error!void {
86 var block_counter = std.io.countingReader(self.inner_reader);
87 const block_reader = block_counter.reader();
88
89 var packed_size: ?u64 = null;
90 var unpacked_size: ?u64 = null;
91
92 // Block Header
93 {
94 var header_hasher = xz.hashedReader(block_reader, Crc32.init());
95 const header_reader = header_hasher.reader();
96
97 const header_size = @as(u64, try header_reader.readByte()) * 4;
98 if (header_size == 0)
99 return error.EndOfStreamWithNoError;
100
101 const Flags = packed struct(u8) {
102 last_filter_index: u2,
103 reserved: u4,
104 has_packed_size: bool,
105 has_unpacked_size: bool,
106 };
107
108 const flags = @as(Flags, @bitCast(try header_reader.readByte()));
109 const filter_count = @as(u3, flags.last_filter_index) + 1;
110 if (filter_count > 1)
111 return error.Unsupported;
112
113 if (flags.has_packed_size)
114 packed_size = try std.leb.readUleb128(u64, header_reader);
115
116 if (flags.has_unpacked_size)
117 unpacked_size = try std.leb.readUleb128(u64, header_reader);
118
119 const FilterId = enum(u64) {
120 lzma2 = 0x21,
121 _,
122 };
123
124 const filter_id = @as(
125 FilterId,
126 @enumFromInt(try std.leb.readUleb128(u64, header_reader)),
127 );
128
129 if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
130 return error.CorruptInput;
131
132 if (filter_id != .lzma2)
133 return error.Unsupported;
134
135 const properties_size = try std.leb.readUleb128(u64, header_reader);
136 if (properties_size != 1)
137 return error.CorruptInput;
138
139 // TODO: use filter properties
140 _ = try header_reader.readByte();
141
142 while (block_counter.bytes_read != header_size) {
143 if (try header_reader.readByte() != 0)
144 return error.CorruptInput;
145 }
146
147 const hash_a = header_hasher.hasher.final();
148 const hash_b = try header_reader.readInt(u32, .little);
149 if (hash_a != hash_b)
150 return error.WrongChecksum;
151 }
152
153 // Compressed Data
154 var packed_counter = std.io.countingReader(block_reader);
155 try lzma2.decompress(
156 self.allocator,
157 packed_counter.reader(),
158 self.to_read.writer(self.allocator),
159 );
160
161 if (packed_size) |s| {
162 if (s != packed_counter.bytes_read)
163 return error.CorruptInput;
164 }
165
166 const unpacked_bytes = self.to_read.items;
167 if (unpacked_size) |s| {
168 if (s != unpacked_bytes.len)
169 return error.CorruptInput;
170 }
171
172 // Block Padding
173 while (block_counter.bytes_read % 4 != 0) {
174 if (try block_reader.readByte() != 0)
175 return error.CorruptInput;
176 }
177
178 switch (self.check) {
179 .none => {},
180 .crc32 => {
181 const hash_a = Crc32.hash(unpacked_bytes);
182 const hash_b = try self.inner_reader.readInt(u32, .little);
183 if (hash_a != hash_b)
184 return error.WrongChecksum;
185 },
186 .crc64 => {
187 const hash_a = Crc64.hash(unpacked_bytes);
188 const hash_b = try self.inner_reader.readInt(u64, .little);
189 if (hash_a != hash_b)
190 return error.WrongChecksum;
191 },
192 .sha256 => {
193 var hash_a: [Sha256.digest_length]u8 = undefined;
194 Sha256.hash(unpacked_bytes, &hash_a, .{});
195
196 var hash_b: [Sha256.digest_length]u8 = undefined;
197 try self.inner_reader.readNoEof(&hash_b);
198
199 if (!std.mem.eql(u8, &hash_a, &hash_b))
200 return error.WrongChecksum;
201 },
202 else => return error.Unsupported,
203 }
204
205 self.block_count += 1;
206 }
207 };
208}
lib/std/compress/xz/test.zig+120-60
...@@ -3,80 +3,138 @@ const testing = std.testing;...@@ -3,80 +3,138 @@ const testing = std.testing;
3const xz = std.compress.xz;3const xz = std.compress.xz;
44
5fn decompress(data: []const u8) ![]u8 {5fn decompress(data: []const u8) ![]u8 {
6 var in_stream = std.io.fixedBufferStream(data);6 const gpa = testing.allocator;
77
8 var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());8 var in_stream: std.Io.Reader = .fixed(data);
9
10 var xz_stream = try xz.Decompress.init(&in_stream, gpa, &.{});
9 defer xz_stream.deinit();11 defer xz_stream.deinit();
1012
11 return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));13 return xz_stream.reader.allocRemaining(gpa, .unlimited);
12}14}
1315
14fn testReader(data: []const u8, comptime expected: []const u8) !void {16fn testReader(data: []const u8, expected: []const u8) !void {
15 const buf = try decompress(data);17 const gpa = testing.allocator;
16 defer testing.allocator.free(buf);18
19 const result = try decompress(data);
20 defer gpa.free(result);
21
22 try testing.expectEqualSlices(u8, expected, result);
23}
24
25fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
26 const gpa = std.testing.allocator;
27 var stream: std.Io.Reader = .fixed(compressed);
1728
18 try testing.expectEqualSlices(u8, expected, buf);29 var decompressor = try xz.Decompress.init(&stream, gpa, &.{});
30 defer decompressor.deinit();
31
32 try std.testing.expectError(error.ReadFailed, decompressor.reader.allocRemaining(gpa, .unlimited));
33 try std.testing.expectEqual(expected, decompressor.err orelse return error.TestFailed);
19}34}
2035
21test "compressed data" {36test "fixture good-0-empty.xz" {
22 try testReader(@embedFile("testdata/good-0-empty.xz"), "");37 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
38}
39
40const hello_world_text =
41 \\Hello
42 \\World!
43 \\
44;
2345
24 inline for ([_][]const u8{46test "fixture good-1-check-none.xz" {
25 "good-1-check-none.xz",47 try testReader(@embedFile("testdata/good-1-check-none.xz"), hello_world_text);
26 "good-1-check-crc32.xz",48}
27 "good-1-check-crc64.xz",49
28 "good-1-check-sha256.xz",50test "fixture good-1-check-crc32.xz" {
29 "good-2-lzma2.xz",51 try testReader(@embedFile("testdata/good-1-check-crc32.xz"), hello_world_text);
30 "good-1-block_header-1.xz",52}
31 "good-1-block_header-2.xz",
32 "good-1-block_header-3.xz",
33 }) |filename| {
34 try testReader(@embedFile("testdata/" ++ filename),
35 \\Hello
36 \\World!
37 \\
38 );
39 }
40
41 inline for ([_][]const u8{
42 "good-1-lzma2-1.xz",
43 "good-1-lzma2-2.xz",
44 "good-1-lzma2-3.xz",
45 "good-1-lzma2-4.xz",
46 }) |filename| {
47 try testReader(@embedFile("testdata/" ++ filename),
48 \\Lorem ipsum dolor sit amet, consectetur adipisicing
49 \\elit, sed do eiusmod tempor incididunt ut
50 \\labore et dolore magna aliqua. Ut enim
51 \\ad minim veniam, quis nostrud exercitation ullamco
52 \\laboris nisi ut aliquip ex ea commodo
53 \\consequat. Duis aute irure dolor in reprehenderit
54 \\in voluptate velit esse cillum dolore eu
55 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
56 \\non proident, sunt in culpa qui officia
57 \\deserunt mollit anim id est laborum.
58 \\
59 );
60 }
6153
54test "fixture good-1-check-crc64.xz" {
55 try testReader(@embedFile("testdata/good-1-check-crc64.xz"), hello_world_text);
56}
57
58test "fixture good-1-check-sha256.xz" {
59 try testReader(@embedFile("testdata/good-1-check-sha256.xz"), hello_world_text);
60}
61
62test "fixture good-2-lzma2.xz" {
63 try testReader(@embedFile("testdata/good-2-lzma2.xz"), hello_world_text);
64}
65
66test "fixture good-1-block_header-1.xz" {
67 try testReader(@embedFile("testdata/good-1-block_header-1.xz"), hello_world_text);
68}
69
70test "fixture good-1-block_header-2.xz" {
71 try testReader(@embedFile("testdata/good-1-block_header-2.xz"), hello_world_text);
72}
73
74test "fixture good-1-block_header-3.xz" {
75 try testReader(@embedFile("testdata/good-1-block_header-3.xz"), hello_world_text);
76}
77
78const lorem_ipsum_text =
79 \\Lorem ipsum dolor sit amet, consectetur adipisicing
80 \\elit, sed do eiusmod tempor incididunt ut
81 \\labore et dolore magna aliqua. Ut enim
82 \\ad minim veniam, quis nostrud exercitation ullamco
83 \\laboris nisi ut aliquip ex ea commodo
84 \\consequat. Duis aute irure dolor in reprehenderit
85 \\in voluptate velit esse cillum dolore eu
86 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
87 \\non proident, sunt in culpa qui officia
88 \\deserunt mollit anim id est laborum.
89 \\
90;
91
92test "fixture good-1-lzma2-1.xz" {
93 try testReader(@embedFile("testdata/good-1-lzma2-1.xz"), lorem_ipsum_text);
94}
95
96test "fixture good-1-lzma2-2.xz" {
97 try testReader(@embedFile("testdata/good-1-lzma2-2.xz"), lorem_ipsum_text);
98}
99
100test "fixture good-1-lzma2-3.xz" {
101 try testReader(@embedFile("testdata/good-1-lzma2-3.xz"), lorem_ipsum_text);
102}
103
104test "fixture good-1-lzma2-4.xz" {
105 try testReader(@embedFile("testdata/good-1-lzma2-4.xz"), lorem_ipsum_text);
106}
107
108test "fixture good-1-lzma2-5.xz" {
62 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");109 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
63}110}
64111
65test "unsupported" {112test "fixture good-1-delta-lzma2.tiff.xz" {
66 inline for ([_][]const u8{113 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-delta-lzma2.tiff.xz"));
67 "good-1-delta-lzma2.tiff.xz",114}
68 "good-1-x86-lzma2.xz",115
69 "good-1-sparc-lzma2.xz",116test "fixture good-1-x86-lzma2.xz" {
70 "good-1-arm64-lzma2-1.xz",117 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-x86-lzma2.xz"));
71 "good-1-arm64-lzma2-2.xz",118}
72 "good-1-3delta-lzma2.xz",119
73 "good-1-empty-bcj-lzma2.xz",120test "fixture good-1-sparc-lzma2.xz" {
74 }) |filename| {121 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-sparc-lzma2.xz"));
75 try testing.expectError(122}
76 error.Unsupported,123
77 decompress(@embedFile("testdata/" ++ filename)),124test "fixture good-1-arm64-lzma2-1.xz" {
78 );125 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-arm64-lzma2-1.xz"));
79 }126}
127
128test "fixture good-1-arm64-lzma2-2.xz" {
129 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-arm64-lzma2-2.xz"));
130}
131
132test "fixture good-1-3delta-lzma2.xz" {
133 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-3delta-lzma2.xz"));
134}
135
136test "fixture good-1-empty-bcj-lzma2.xz" {
137 try testDecompressError(error.Unsupported, @embedFile("testdata/good-1-empty-bcj-lzma2.xz"));
80}138}
81139
82fn testDontPanic(data: []const u8) !void {140fn testDontPanic(data: []const u8) !void {
...@@ -91,6 +149,8 @@ test "size fields: integer overflow avoidance" {...@@ -91,6 +149,8 @@ test "size fields: integer overflow avoidance" {
91 // These cases were found via fuzz testing and each previously caused149 // These cases were found via fuzz testing and each previously caused
92 // an integer overflow when decoding. We just want to ensure they no longer150 // an integer overflow when decoding. We just want to ensure they no longer
93 // cause a panic151 // cause a panic
152 // TODO this not a sufficient way to test. tests should always check the result,
153 // not merely ensure that the code does not crash.
94 const header_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6z";154 const header_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6z";
95 try testDontPanic(header_size_overflow);155 try testDontPanic(header_size_overflow);
96 const lzma2_chunk_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6\x02\x00!\x01\x08\x00\x00\x00\xd8\x0f#\x13\x01\xff\xff";156 const lzma2_chunk_size_overflow = "\xfd7zXZ\x00\x00\x01i\"\xde6\x02\x00!\x01\x08\x00\x00\x00\xd8\x0f#\x13\x01\xff\xff";
src/Package/Fetch.zig+3-5
...@@ -1204,12 +1204,10 @@ fn unpackResource(...@@ -1204,12 +1204,10 @@ fn unpackResource(
1204 },1204 },
1205 .@"tar.xz" => {1205 .@"tar.xz" => {
1206 const gpa = f.arena.child_allocator;1206 const gpa = f.arena.child_allocator;
1207 var dcp = std.compress.xz.decompress(gpa, resource.reader().adaptToOldInterface()) catch |err|1207 var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err|
1208 return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));1208 return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
1209 defer dcp.deinit();1209 defer decompress.deinit();
1210 var adapter_buffer: [1024]u8 = undefined;1210 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1211 var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);
1212 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1213 },1211 },
1214 .@"tar.zst" => {1212 .@"tar.zst" => {
1215 const window_len = std.compress.zstd.default_window_len;1213 const window_len = std.compress.zstd.default_window_len;