authorgravatar for fncontroloption@noreply.codeberg.orgFnControlOption <fncontroloption@noreply.codeberg.org> 2023-02-05 06:52:28-08:00
committergravatar for fncontroloption@noreply.codeberg.orgFnControlOption <fncontroloption@noreply.codeberg.org> 2023-02-05 06:52:28-08:00
loge03d6c42ea8b65a3b283c4da8c9593b82762874c
treeb82f444aa38ea462c26d5c162ef898200ed45498
parentd57813e3e94e77f54e989fa0814216389cf04a2b

Delete redundant `lzma`/`lzma2` prefix in function/struct names

11 files changed, 683 insertions(+), 705 deletions(-)

lib/std/compress.zig+2
......@@ -3,6 +3,7 @@ const std = @import("std.zig");
33pub const deflate = @import("compress/deflate.zig");
44pub const gzip = @import("compress/gzip.zig");
55pub const lzma = @import("compress/lzma.zig");
6pub const lzma2 = @import("compress/lzma2.zig");
67pub const xz = @import("compress/xz.zig");
78pub const zlib = @import("compress/zlib.zig");
89
......@@ -40,6 +41,7 @@ test {
4041 _ = deflate;
4142 _ = gzip;
4243 _ = lzma;
44 _ = lzma2;
4345 _ = xz;
4446 _ = zlib;
4547}
lib/std/compress/lzma.zig+4-19
......@@ -1,36 +1,21 @@
11const std = @import("../std.zig");
22const Allocator = std.mem.Allocator;
3const FixedBufferStream = std.io.FixedBufferStream;
43
54pub const decode = @import("lzma/decode.zig");
6pub const LzmaParams = decode.lzma.LzmaParams;
7pub const LzmaDecoder = decode.lzma.LzmaDecoder;
8pub const Lzma2Decoder = decode.lzma2.Lzma2Decoder;
95
10pub fn lzmaDecompress(
6pub fn decompress(
117 allocator: Allocator,
128 reader: anytype,
139 writer: anytype,
1410 options: decode.Options,
1511) !void {
16 const params = try LzmaParams.readHeader(reader, options);
17 var decoder = try LzmaDecoder.init(allocator, params, options.memlimit);
18 defer decoder.deinit(allocator);
19 return decoder.decompress(allocator, reader, writer);
20}
21
22pub fn lzma2Decompress(
23 allocator: Allocator,
24 reader: anytype,
25 writer: anytype,
26) !void {
27 var decoder = try Lzma2Decoder.init(allocator);
12 const params = try decode.Params.readHeader(reader, options);
13 var decoder = try decode.Decoder.init(allocator, params, options.memlimit);
2814 defer decoder.deinit(allocator);
2915 return decoder.decompress(allocator, reader, writer);
3016}
3117
3218test {
33 _ = @import("lzma/lzma_test.zig");
34 _ = @import("lzma/lzma2_test.zig");
19 _ = @import("lzma/test.zig");
3520 _ = @import("lzma/vec2d.zig");
3621}
lib/std/compress/lzma/decode.zig+395-2
......@@ -1,8 +1,17 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
16pub const lzbuffer = @import("decode/lzbuffer.zig");
2pub const lzma = @import("decode/lzma.zig");
3pub const lzma2 = @import("decode/lzma2.zig");
47pub const rangecoder = @import("decode/rangecoder.zig");
58
9const LzCircularBuffer = lzbuffer.LzCircularBuffer;
10const BitTree = rangecoder.BitTree;
11const LenDecoder = rangecoder.LenDecoder;
12const RangeDecoder = rangecoder.RangeDecoder;
13const Vec2D = @import("vec2d.zig").Vec2D;
14
615pub const Options = struct {
716 unpacked_size: UnpackedSize = .read_from_header,
817 memlimit: ?usize = null,
......@@ -14,3 +23,387 @@ pub const UnpackedSize = union(enum) {
1423 read_header_but_use_provided: ?u64,
1524 use_provided: ?u64,
1625};
26
27const ProcessingStatus = enum {
28 continue_,
29 finished,
30};
31
32pub const Properties = struct {
33 lc: u4,
34 lp: u3,
35 pb: u3,
36
37 fn validate(self: Properties) void {
38 assert(self.lc <= 8);
39 assert(self.lp <= 4);
40 assert(self.pb <= 4);
41 }
42};
43
44pub const Params = struct {
45 properties: Properties,
46 dict_size: u32,
47 unpacked_size: ?u64,
48
49 pub fn readHeader(reader: anytype, options: Options) !Params {
50 var props = try reader.readByte();
51 if (props >= 225) {
52 return error.CorruptInput;
53 }
54
55 const lc = @intCast(u4, props % 9);
56 props /= 9;
57 const lp = @intCast(u3, props % 5);
58 props /= 5;
59 const pb = @intCast(u3, props);
60
61 const dict_size_provided = try reader.readIntLittle(u32);
62 const dict_size = math.max(0x1000, dict_size_provided);
63
64 const unpacked_size = switch (options.unpacked_size) {
65 .read_from_header => blk: {
66 const unpacked_size_provided = try reader.readIntLittle(u64);
67 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
68 break :blk if (marker_mandatory)
69 null
70 else
71 unpacked_size_provided;
72 },
73 .read_header_but_use_provided => |x| blk: {
74 _ = try reader.readIntLittle(u64);
75 break :blk x;
76 },
77 .use_provided => |x| x,
78 };
79
80 return Params{
81 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
82 .dict_size = dict_size,
83 .unpacked_size = unpacked_size,
84 };
85 }
86};
87
88pub const DecoderState = struct {
89 lzma_props: Properties,
90 unpacked_size: ?u64,
91 literal_probs: Vec2D(u16),
92 pos_slot_decoder: [4]BitTree(6),
93 align_decoder: BitTree(4),
94 pos_decoders: [115]u16,
95 is_match: [192]u16,
96 is_rep: [12]u16,
97 is_rep_g0: [12]u16,
98 is_rep_g1: [12]u16,
99 is_rep_g2: [12]u16,
100 is_rep_0long: [192]u16,
101 state: usize,
102 rep: [4]usize,
103 len_decoder: LenDecoder,
104 rep_len_decoder: LenDecoder,
105
106 pub fn init(
107 allocator: Allocator,
108 lzma_props: Properties,
109 unpacked_size: ?u64,
110 ) !DecoderState {
111 return .{
112 .lzma_props = lzma_props,
113 .unpacked_size = unpacked_size,
114 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
115 .pos_slot_decoder = .{.{}} ** 4,
116 .align_decoder = .{},
117 .pos_decoders = .{0x400} ** 115,
118 .is_match = .{0x400} ** 192,
119 .is_rep = .{0x400} ** 12,
120 .is_rep_g0 = .{0x400} ** 12,
121 .is_rep_g1 = .{0x400} ** 12,
122 .is_rep_g2 = .{0x400} ** 12,
123 .is_rep_0long = .{0x400} ** 192,
124 .state = 0,
125 .rep = .{0} ** 4,
126 .len_decoder = .{},
127 .rep_len_decoder = .{},
128 };
129 }
130
131 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
132 self.literal_probs.deinit(allocator);
133 self.* = undefined;
134 }
135
136 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
137 new_props.validate();
138 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
139 self.literal_probs.fill(0x400);
140 } else {
141 self.literal_probs.deinit(allocator);
142 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
143 }
144
145 self.lzma_props = new_props;
146 for (self.pos_slot_decoder) |*t| t.reset();
147 self.align_decoder.reset();
148 self.pos_decoders = .{0x400} ** 115;
149 self.is_match = .{0x400} ** 192;
150 self.is_rep = .{0x400} ** 12;
151 self.is_rep_g0 = .{0x400} ** 12;
152 self.is_rep_g1 = .{0x400} ** 12;
153 self.is_rep_g2 = .{0x400} ** 12;
154 self.is_rep_0long = .{0x400} ** 192;
155 self.state = 0;
156 self.rep = .{0} ** 4;
157 self.len_decoder.reset();
158 self.rep_len_decoder.reset();
159 }
160
161 fn processNextInner(
162 self: *DecoderState,
163 allocator: Allocator,
164 reader: anytype,
165 writer: anytype,
166 buffer: anytype,
167 decoder: *RangeDecoder,
168 update: bool,
169 ) !ProcessingStatus {
170 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
171
172 if (!try decoder.decodeBit(
173 reader,
174 &self.is_match[(self.state << 4) + pos_state],
175 update,
176 )) {
177 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
178
179 if (update) {
180 try buffer.appendLiteral(allocator, byte, writer);
181
182 self.state = if (self.state < 4)
183 0
184 else if (self.state < 10)
185 self.state - 3
186 else
187 self.state - 6;
188 }
189 return .continue_;
190 }
191
192 var len: usize = undefined;
193 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
194 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
195 if (!try decoder.decodeBit(
196 reader,
197 &self.is_rep_0long[(self.state << 4) + pos_state],
198 update,
199 )) {
200 if (update) {
201 self.state = if (self.state < 7) 9 else 11;
202 const dist = self.rep[0] + 1;
203 try buffer.appendLz(allocator, 1, dist, writer);
204 }
205 return .continue_;
206 }
207 } else {
208 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
209 1
210 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
211 2
212 else
213 3;
214 if (update) {
215 const dist = self.rep[idx];
216 var i = idx;
217 while (i > 0) : (i -= 1) {
218 self.rep[i] = self.rep[i - 1];
219 }
220 self.rep[0] = dist;
221 }
222 }
223
224 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
225
226 if (update) {
227 self.state = if (self.state < 7) 8 else 11;
228 }
229 } else {
230 if (update) {
231 self.rep[3] = self.rep[2];
232 self.rep[2] = self.rep[1];
233 self.rep[1] = self.rep[0];
234 }
235
236 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
237
238 if (update) {
239 self.state = if (self.state < 7) 7 else 10;
240 }
241
242 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
243
244 if (update) {
245 self.rep[0] = rep_0;
246 if (self.rep[0] == 0xFFFF_FFFF) {
247 if (decoder.isFinished()) {
248 return .finished;
249 }
250 return error.CorruptInput;
251 }
252 }
253 }
254
255 if (update) {
256 len += 2;
257
258 const dist = self.rep[0] + 1;
259 try buffer.appendLz(allocator, len, dist, writer);
260 }
261
262 return .continue_;
263 }
264
265 fn processNext(
266 self: *DecoderState,
267 allocator: Allocator,
268 reader: anytype,
269 writer: anytype,
270 buffer: anytype,
271 decoder: *RangeDecoder,
272 ) !ProcessingStatus {
273 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
274 }
275
276 pub fn process(
277 self: *DecoderState,
278 allocator: Allocator,
279 reader: anytype,
280 writer: anytype,
281 buffer: anytype,
282 decoder: *RangeDecoder,
283 ) !void {
284 while (true) {
285 if (self.unpacked_size) |unpacked_size| {
286 if (buffer.len >= unpacked_size) {
287 break;
288 }
289 } else if (decoder.isFinished()) {
290 break;
291 }
292
293 if (try self.processNext(allocator, reader, writer, buffer, decoder) == .finished) {
294 break;
295 }
296 }
297
298 if (self.unpacked_size) |len| {
299 if (len != buffer.len) {
300 return error.CorruptInput;
301 }
302 }
303 }
304
305 fn decodeLiteral(
306 self: *DecoderState,
307 reader: anytype,
308 buffer: anytype,
309 decoder: *RangeDecoder,
310 update: bool,
311 ) !u8 {
312 const def_prev_byte = 0;
313 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
314
315 var result: usize = 1;
316 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
317 (prev_byte >> (8 - self.lzma_props.lc));
318 const probs = try self.literal_probs.getMut(lit_state);
319
320 if (self.state >= 7) {
321 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
322
323 while (result < 0x100) {
324 const match_bit = (match_byte >> 7) & 1;
325 match_byte <<= 1;
326 const bit = @boolToInt(try decoder.decodeBit(
327 reader,
328 &probs[((@as(usize, 1) + match_bit) << 8) + result],
329 update,
330 ));
331 result = (result << 1) ^ bit;
332 if (match_bit != bit) {
333 break;
334 }
335 }
336 }
337
338 while (result < 0x100) {
339 result = (result << 1) ^ @boolToInt(try decoder.decodeBit(reader, &probs[result], update));
340 }
341
342 return @truncate(u8, result - 0x100);
343 }
344
345 fn decodeDistance(
346 self: *DecoderState,
347 reader: anytype,
348 decoder: *RangeDecoder,
349 length: usize,
350 update: bool,
351 ) !usize {
352 const len_state = if (length > 3) 3 else length;
353
354 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
355 if (pos_slot < 4)
356 return pos_slot;
357
358 const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
359 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
360
361 if (pos_slot < 14) {
362 result += try decoder.parseReverseBitTree(
363 reader,
364 num_direct_bits,
365 &self.pos_decoders,
366 result - pos_slot,
367 update,
368 );
369 } else {
370 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
371 result += try self.align_decoder.parseReverse(reader, decoder, update);
372 }
373
374 return result;
375 }
376};
377
378pub const Decoder = struct {
379 params: Params,
380 memlimit: usize,
381 state: DecoderState,
382
383 pub fn init(allocator: Allocator, params: Params, memlimit: ?usize) !Decoder {
384 return Decoder{
385 .params = params,
386 .memlimit = memlimit orelse math.maxInt(usize),
387 .state = try DecoderState.init(allocator, params.properties, params.unpacked_size),
388 };
389 }
390
391 pub fn deinit(self: *Decoder, allocator: Allocator) void {
392 self.state.deinit(allocator);
393 self.* = undefined;
394 }
395
396 pub fn decompress(
397 self: *Decoder,
398 allocator: Allocator,
399 reader: anytype,
400 writer: anytype,
401 ) !void {
402 var buffer = LzCircularBuffer.init(self.params.dict_size, self.memlimit);
403 defer buffer.deinit(allocator);
404
405 var decoder = try RangeDecoder.init(reader);
406 try self.state.process(allocator, reader, writer, &buffer, &decoder);
407 try buffer.finish(writer);
408 }
409};
lib/std/compress/lzma/decode/lzma.zig deleted-398
......@@ -1,398 +0,0 @@
1const std = @import("../../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6const FixedBufferStream = std.io.FixedBufferStream;
7
8const LzCircularBuffer = @import("lzbuffer.zig").LzCircularBuffer;
9const Options = @import("../decode.zig").Options;
10const Vec2D = @import("../vec2d.zig").Vec2D;
11const rangecoder = @import("rangecoder.zig");
12const BitTree = rangecoder.BitTree;
13const LenDecoder = rangecoder.LenDecoder;
14const RangeDecoder = rangecoder.RangeDecoder;
15
16const ProcessingStatus = enum {
17 continue_,
18 finished,
19};
20
21pub const LzmaProperties = struct {
22 lc: u4,
23 lp: u3,
24 pb: u3,
25
26 fn validate(self: LzmaProperties) void {
27 assert(self.lc <= 8);
28 assert(self.lp <= 4);
29 assert(self.pb <= 4);
30 }
31};
32
33pub const LzmaParams = struct {
34 properties: LzmaProperties,
35 dict_size: u32,
36 unpacked_size: ?u64,
37
38 pub fn readHeader(reader: anytype, options: Options) !LzmaParams {
39 var props = try reader.readByte();
40 if (props >= 225) {
41 return error.CorruptInput;
42 }
43
44 const lc = @intCast(u4, props % 9);
45 props /= 9;
46 const lp = @intCast(u3, props % 5);
47 props /= 5;
48 const pb = @intCast(u3, props);
49
50 const dict_size_provided = try reader.readIntLittle(u32);
51 const dict_size = math.max(0x1000, dict_size_provided);
52
53 const unpacked_size = switch (options.unpacked_size) {
54 .read_from_header => blk: {
55 const unpacked_size_provided = try reader.readIntLittle(u64);
56 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
57 break :blk if (marker_mandatory)
58 null
59 else
60 unpacked_size_provided;
61 },
62 .read_header_but_use_provided => |x| blk: {
63 _ = try reader.readIntLittle(u64);
64 break :blk x;
65 },
66 .use_provided => |x| x,
67 };
68
69 return LzmaParams{
70 .properties = LzmaProperties{ .lc = lc, .lp = lp, .pb = pb },
71 .dict_size = dict_size,
72 .unpacked_size = unpacked_size,
73 };
74 }
75};
76
77pub const DecoderState = struct {
78 lzma_props: LzmaProperties,
79 unpacked_size: ?u64,
80 literal_probs: Vec2D(u16),
81 pos_slot_decoder: [4]BitTree(6),
82 align_decoder: BitTree(4),
83 pos_decoders: [115]u16,
84 is_match: [192]u16,
85 is_rep: [12]u16,
86 is_rep_g0: [12]u16,
87 is_rep_g1: [12]u16,
88 is_rep_g2: [12]u16,
89 is_rep_0long: [192]u16,
90 state: usize,
91 rep: [4]usize,
92 len_decoder: LenDecoder,
93 rep_len_decoder: LenDecoder,
94
95 pub fn init(
96 allocator: Allocator,
97 lzma_props: LzmaProperties,
98 unpacked_size: ?u64,
99 ) !DecoderState {
100 return .{
101 .lzma_props = lzma_props,
102 .unpacked_size = unpacked_size,
103 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
104 .pos_slot_decoder = .{.{}} ** 4,
105 .align_decoder = .{},
106 .pos_decoders = .{0x400} ** 115,
107 .is_match = .{0x400} ** 192,
108 .is_rep = .{0x400} ** 12,
109 .is_rep_g0 = .{0x400} ** 12,
110 .is_rep_g1 = .{0x400} ** 12,
111 .is_rep_g2 = .{0x400} ** 12,
112 .is_rep_0long = .{0x400} ** 192,
113 .state = 0,
114 .rep = .{0} ** 4,
115 .len_decoder = .{},
116 .rep_len_decoder = .{},
117 };
118 }
119
120 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
121 self.literal_probs.deinit(allocator);
122 self.* = undefined;
123 }
124
125 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: LzmaProperties) !void {
126 new_props.validate();
127 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
128 self.literal_probs.fill(0x400);
129 } else {
130 self.literal_probs.deinit(allocator);
131 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
132 }
133
134 self.lzma_props = new_props;
135 for (self.pos_slot_decoder) |*t| t.reset();
136 self.align_decoder.reset();
137 self.pos_decoders = .{0x400} ** 115;
138 self.is_match = .{0x400} ** 192;
139 self.is_rep = .{0x400} ** 12;
140 self.is_rep_g0 = .{0x400} ** 12;
141 self.is_rep_g1 = .{0x400} ** 12;
142 self.is_rep_g2 = .{0x400} ** 12;
143 self.is_rep_0long = .{0x400} ** 192;
144 self.state = 0;
145 self.rep = .{0} ** 4;
146 self.len_decoder.reset();
147 self.rep_len_decoder.reset();
148 }
149
150 fn processNextInner(
151 self: *DecoderState,
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 buffer: anytype,
156 decoder: *RangeDecoder,
157 update: bool,
158 ) !ProcessingStatus {
159 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
160
161 if (!try decoder.decodeBit(
162 reader,
163 &self.is_match[(self.state << 4) + pos_state],
164 update,
165 )) {
166 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
167
168 if (update) {
169 try buffer.appendLiteral(allocator, byte, writer);
170
171 self.state = if (self.state < 4)
172 0
173 else if (self.state < 10)
174 self.state - 3
175 else
176 self.state - 6;
177 }
178 return .continue_;
179 }
180
181 var len: usize = undefined;
182 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
183 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
184 if (!try decoder.decodeBit(
185 reader,
186 &self.is_rep_0long[(self.state << 4) + pos_state],
187 update,
188 )) {
189 if (update) {
190 self.state = if (self.state < 7) 9 else 11;
191 const dist = self.rep[0] + 1;
192 try buffer.appendLz(allocator, 1, dist, writer);
193 }
194 return .continue_;
195 }
196 } else {
197 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
198 1
199 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
200 2
201 else
202 3;
203 if (update) {
204 const dist = self.rep[idx];
205 var i = idx;
206 while (i > 0) : (i -= 1) {
207 self.rep[i] = self.rep[i - 1];
208 }
209 self.rep[0] = dist;
210 }
211 }
212
213 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
214
215 if (update) {
216 self.state = if (self.state < 7) 8 else 11;
217 }
218 } else {
219 if (update) {
220 self.rep[3] = self.rep[2];
221 self.rep[2] = self.rep[1];
222 self.rep[1] = self.rep[0];
223 }
224
225 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
226
227 if (update) {
228 self.state = if (self.state < 7) 7 else 10;
229 }
230
231 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
232
233 if (update) {
234 self.rep[0] = rep_0;
235 if (self.rep[0] == 0xFFFF_FFFF) {
236 if (decoder.isFinished()) {
237 return .finished;
238 }
239 return error.CorruptInput;
240 }
241 }
242 }
243
244 if (update) {
245 len += 2;
246
247 const dist = self.rep[0] + 1;
248 try buffer.appendLz(allocator, len, dist, writer);
249 }
250
251 return .continue_;
252 }
253
254 fn processNext(
255 self: *DecoderState,
256 allocator: Allocator,
257 reader: anytype,
258 writer: anytype,
259 buffer: anytype,
260 decoder: *RangeDecoder,
261 ) !ProcessingStatus {
262 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
263 }
264
265 pub fn process(
266 self: *DecoderState,
267 allocator: Allocator,
268 reader: anytype,
269 writer: anytype,
270 buffer: anytype,
271 decoder: *RangeDecoder,
272 ) !void {
273 while (true) {
274 if (self.unpacked_size) |unpacked_size| {
275 if (buffer.len >= unpacked_size) {
276 break;
277 }
278 } else if (decoder.isFinished()) {
279 break;
280 }
281
282 if (try self.processNext(allocator, reader, writer, buffer, decoder) == .finished) {
283 break;
284 }
285 }
286
287 if (self.unpacked_size) |len| {
288 if (len != buffer.len) {
289 return error.CorruptInput;
290 }
291 }
292 }
293
294 fn decodeLiteral(
295 self: *DecoderState,
296 reader: anytype,
297 buffer: anytype,
298 decoder: *RangeDecoder,
299 update: bool,
300 ) !u8 {
301 const def_prev_byte = 0;
302 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
303
304 var result: usize = 1;
305 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
306 (prev_byte >> (8 - self.lzma_props.lc));
307 const probs = try self.literal_probs.getMut(lit_state);
308
309 if (self.state >= 7) {
310 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
311
312 while (result < 0x100) {
313 const match_bit = (match_byte >> 7) & 1;
314 match_byte <<= 1;
315 const bit = @boolToInt(try decoder.decodeBit(
316 reader,
317 &probs[((@as(usize, 1) + match_bit) << 8) + result],
318 update,
319 ));
320 result = (result << 1) ^ bit;
321 if (match_bit != bit) {
322 break;
323 }
324 }
325 }
326
327 while (result < 0x100) {
328 result = (result << 1) ^ @boolToInt(try decoder.decodeBit(reader, &probs[result], update));
329 }
330
331 return @truncate(u8, result - 0x100);
332 }
333
334 fn decodeDistance(
335 self: *DecoderState,
336 reader: anytype,
337 decoder: *RangeDecoder,
338 length: usize,
339 update: bool,
340 ) !usize {
341 const len_state = if (length > 3) 3 else length;
342
343 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
344 if (pos_slot < 4)
345 return pos_slot;
346
347 const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
348 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
349
350 if (pos_slot < 14) {
351 result += try decoder.parseReverseBitTree(
352 reader,
353 num_direct_bits,
354 &self.pos_decoders,
355 result - pos_slot,
356 update,
357 );
358 } else {
359 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
360 result += try self.align_decoder.parseReverse(reader, decoder, update);
361 }
362
363 return result;
364 }
365};
366
367pub const LzmaDecoder = struct {
368 params: LzmaParams,
369 memlimit: usize,
370 state: DecoderState,
371
372 pub fn init(allocator: Allocator, params: LzmaParams, memlimit: ?usize) !LzmaDecoder {
373 return LzmaDecoder{
374 .params = params,
375 .memlimit = memlimit orelse math.maxInt(usize),
376 .state = try DecoderState.init(allocator, params.properties, params.unpacked_size),
377 };
378 }
379
380 pub fn deinit(self: *LzmaDecoder, allocator: Allocator) void {
381 self.state.deinit(allocator);
382 self.* = undefined;
383 }
384
385 pub fn decompress(
386 self: *LzmaDecoder,
387 allocator: Allocator,
388 reader: anytype,
389 writer: anytype,
390 ) !void {
391 var buffer = LzCircularBuffer.init(self.params.dict_size, self.memlimit);
392 defer buffer.deinit(allocator);
393
394 var decoder = try RangeDecoder.init(reader);
395 try self.state.process(allocator, reader, writer, &buffer, &decoder);
396 try buffer.finish(writer);
397 }
398};
lib/std/compress/lzma/decode/lzma2.zig deleted-169
......@@ -1,169 +0,0 @@
1const std = @import("../../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("lzma.zig");
5const DecoderState = lzma.DecoderState;
6const LzmaProperties = lzma.LzmaProperties;
7const LzAccumBuffer = @import("lzbuffer.zig").LzAccumBuffer;
8const RangeDecoder = @import("rangecoder.zig").RangeDecoder;
9
10pub const Lzma2Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Lzma2Decoder {
14 return Lzma2Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 LzmaProperties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Lzma2Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Lzma2Decoder,
34 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.readByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Lzma2Decoder,
57 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try reader.readIntBig(u16);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try reader.readIntBig(u16);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try reader.readByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @intCast(u4, props % 9);
123 props /= 9;
124 const lp = @intCast(u3, props % 5);
125 props /= 5;
126 const pb = @intCast(u3, props);
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = LzmaProperties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
142
143 var rangecoder = try RangeDecoder.init(counter_reader);
144 try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder);
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.readIntBig(u16)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
167 }
168 }
169};
lib/std/compress/lzma/decode/rangecoder.zig-3
......@@ -1,8 +1,5 @@
11const std = @import("../../../std.zig");
22const mem = std.mem;
3const Allocator = std.mem.Allocator;
4const ArrayListUnmanaged = std.ArrayListUnmanaged;
5const FixedBufferStream = std.io.FixedBufferStream;
63
74pub const RangeDecoder = struct {
85 range: u32,
lib/std/compress/lzma/lzma2_test.zig deleted-27
......@@ -1,27 +0,0 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8, writer: anytype) !void {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 try lzma.lzma2Decompress(allocator, stream.reader(), writer);
8}
9
10fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
11 const allocator = std.testing.allocator;
12 var decomp = std.ArrayList(u8).init(allocator);
13 defer decomp.deinit();
14 try testDecompress(compressed, decomp.writer());
15 try std.testing.expectEqualSlices(u8, expected, decomp.items);
16}
17
18fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
19 return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer));
20}
21
22test {
23 try testDecompressEqual(
24 "Hello\nWorld!\n",
25 &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 },
26 );
27}
lib/std/compress/lzma/lzma_test.zig deleted-87
......@@ -1,87 +0,0 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8, writer: anytype) !void {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 try lzma.lzmaDecompress(allocator, stream.reader(), writer, .{});
8}
9
10fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
11 const allocator = std.testing.allocator;
12 var decomp = std.ArrayList(u8).init(allocator);
13 defer decomp.deinit();
14 try testDecompress(compressed, decomp.writer());
15 try std.testing.expectEqualSlices(u8, expected, decomp.items);
16}
17
18fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
19 return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer));
20}
21
22test "decompress empty world" {
23 try testDecompressEqual(
24 "",
25 &[_]u8{
26 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
27 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
28 },
29 );
30}
31
32test "decompress hello world" {
33 try testDecompressEqual(
34 "Hello world\n",
35 &[_]u8{
36 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
37 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
38 0xa5, 0xb0, 0x00,
39 },
40 );
41}
42
43test "decompress huge dict" {
44 try testDecompressEqual(
45 "Hello world\n",
46 &[_]u8{
47 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
48 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
49 0xa5, 0xb0, 0x00,
50 },
51 );
52}
53
54test "unknown size with end of payload marker" {
55 try testDecompressEqual(
56 "Hello\nWorld!\n",
57 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
58 );
59}
60
61test "known size without end of payload marker" {
62 try testDecompressEqual(
63 "Hello\nWorld!\n",
64 @embedFile("testdata/good-known_size-without_eopm.lzma"),
65 );
66}
67
68test "known size with end of payload marker" {
69 try testDecompressEqual(
70 "Hello\nWorld!\n",
71 @embedFile("testdata/good-known_size-with_eopm.lzma"),
72 );
73}
74
75test "too big uncompressed size in header" {
76 try testDecompressError(
77 error.CorruptInput,
78 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
79 );
80}
81
82test "too small uncompressed size in header" {
83 try testDecompressError(
84 error.CorruptInput,
85 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
86 );
87}
lib/std/compress/lzma/test.zig created+87
......@@ -0,0 +1,87 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8, writer: anytype) !void {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 try lzma.decompress(allocator, stream.reader(), writer, .{});
8}
9
10fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
11 const allocator = std.testing.allocator;
12 var decomp = std.ArrayList(u8).init(allocator);
13 defer decomp.deinit();
14 try testDecompress(compressed, decomp.writer());
15 try std.testing.expectEqualSlices(u8, expected, decomp.items);
16}
17
18fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
19 return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer));
20}
21
22test "LZMA: decompress empty world" {
23 try testDecompressEqual(
24 "",
25 &[_]u8{
26 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
27 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
28 },
29 );
30}
31
32test "LZMA: decompress hello world" {
33 try testDecompressEqual(
34 "Hello world\n",
35 &[_]u8{
36 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
37 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
38 0xa5, 0xb0, 0x00,
39 },
40 );
41}
42
43test "LZMA: decompress huge dict" {
44 try testDecompressEqual(
45 "Hello world\n",
46 &[_]u8{
47 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
48 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
49 0xa5, 0xb0, 0x00,
50 },
51 );
52}
53
54test "LZMA: unknown size with end of payload marker" {
55 try testDecompressEqual(
56 "Hello\nWorld!\n",
57 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
58 );
59}
60
61test "LZMA: known size without end of payload marker" {
62 try testDecompressEqual(
63 "Hello\nWorld!\n",
64 @embedFile("testdata/good-known_size-without_eopm.lzma"),
65 );
66}
67
68test "LZMA: known size with end of payload marker" {
69 try testDecompressEqual(
70 "Hello\nWorld!\n",
71 @embedFile("testdata/good-known_size-with_eopm.lzma"),
72 );
73}
74
75test "LZMA: too big uncompressed size in header" {
76 try testDecompressError(
77 error.CorruptInput,
78 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
79 );
80}
81
82test "LZMA: too small uncompressed size in header" {
83 try testDecompressError(
84 error.CorruptInput,
85 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
86 );
87}
lib/std/compress/lzma2.zig created+26
......@@ -0,0 +1,26 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3
4pub const decode = @import("lzma2/decode.zig");
5
6pub fn decompress(
7 allocator: Allocator,
8 reader: anytype,
9 writer: anytype,
10) !void {
11 var decoder = try decode.Decoder.init(allocator);
12 defer decoder.deinit(allocator);
13 return decoder.decompress(allocator, reader, writer);
14}
15
16test {
17 const expected = "Hello\nWorld!\n";
18 const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
19
20 const allocator = std.testing.allocator;
21 var decomp = std.ArrayList(u8).init(allocator);
22 defer decomp.deinit();
23 var stream = std.io.fixedBufferStream(compressed);
24 try decompress(allocator, stream.reader(), decomp.writer());
25 try std.testing.expectEqualSlices(u8, expected, decomp.items);
26}
lib/std/compress/lzma2/decode.zig created+169
......@@ -0,0 +1,169 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.readByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try reader.readIntBig(u16);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try reader.readIntBig(u16);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try reader.readByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @intCast(u4, props % 9);
123 props /= 9;
124 const lp = @intCast(u3, props % 5);
125 props /= 5;
126 const pb = @intCast(u3, props);
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
142
143 var rangecoder = try RangeDecoder.init(counter_reader);
144 try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder);
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.readIntBig(u16)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
167 }
168 }
169};