| author | |
| committer | |
| log | e03d6c42ea8b65a3b283c4da8c9593b82762874c |
| tree | b82f444aa38ea462c26d5c162ef898200ed45498 |
| parent | d57813e3e94e77f54e989fa0814216389cf04a2b |
11 files changed, 683 insertions(+), 705 deletions(-)
lib/std/compress.zig+2| ... | @@ -3,6 +3,7 @@ const std = @import("std.zig"); | ... | @@ -3,6 +3,7 @@ const std = @import("std.zig"); |
| 3 | pub const deflate = @import("compress/deflate.zig"); | 3 | pub const deflate = @import("compress/deflate.zig"); |
| 4 | pub const gzip = @import("compress/gzip.zig"); | 4 | pub const gzip = @import("compress/gzip.zig"); |
| 5 | pub const lzma = @import("compress/lzma.zig"); | 5 | pub const lzma = @import("compress/lzma.zig"); |
| 6 | pub const lzma2 = @import("compress/lzma2.zig"); | ||
| 6 | pub const xz = @import("compress/xz.zig"); | 7 | pub const xz = @import("compress/xz.zig"); |
| 7 | pub const zlib = @import("compress/zlib.zig"); | 8 | pub const zlib = @import("compress/zlib.zig"); |
| 8 | 9 | ||
| ... | @@ -40,6 +41,7 @@ test { | ... | @@ -40,6 +41,7 @@ test { |
| 40 | _ = deflate; | 41 | _ = deflate; |
| 41 | _ = gzip; | 42 | _ = gzip; |
| 42 | _ = lzma; | 43 | _ = lzma; |
| 44 | _ = lzma2; | ||
| 43 | _ = xz; | 45 | _ = xz; |
| 44 | _ = zlib; | 46 | _ = zlib; |
| 45 | } | 47 | } |
lib/std/compress/lzma.zig+4-19| ... | @@ -1,36 +1,21 @@ | ... | @@ -1,36 +1,21 @@ |
| 1 | const std = @import("../std.zig"); | 1 | const std = @import("../std.zig"); |
| 2 | const Allocator = std.mem.Allocator; | 2 | const Allocator = std.mem.Allocator; |
| 3 | const FixedBufferStream = std.io.FixedBufferStream; | ||
| 4 | 3 | ||
| 5 | pub const decode = @import("lzma/decode.zig"); | 4 | pub const decode = @import("lzma/decode.zig"); |
| 6 | pub const LzmaParams = decode.lzma.LzmaParams; | ||
| 7 | pub const LzmaDecoder = decode.lzma.LzmaDecoder; | ||
| 8 | pub const Lzma2Decoder = decode.lzma2.Lzma2Decoder; | ||
| 9 | 5 | ||
| 10 | pub fn lzmaDecompress( | 6 | pub fn decompress( |
| 11 | allocator: Allocator, | 7 | allocator: Allocator, |
| 12 | reader: anytype, | 8 | reader: anytype, |
| 13 | writer: anytype, | 9 | writer: anytype, |
| 14 | options: decode.Options, | 10 | options: decode.Options, |
| 15 | ) !void { | 11 | ) !void { |
| 16 | const params = try LzmaParams.readHeader(reader, options); | 12 | const params = try decode.Params.readHeader(reader, options); |
| 17 | var decoder = try LzmaDecoder.init(allocator, params, options.memlimit); | 13 | var decoder = try decode.Decoder.init(allocator, params, options.memlimit); |
| 18 | defer decoder.deinit(allocator); | ||
| 19 | return decoder.decompress(allocator, reader, writer); | ||
| 20 | } | ||
| 21 | |||
| 22 | pub fn lzma2Decompress( | ||
| 23 | allocator: Allocator, | ||
| 24 | reader: anytype, | ||
| 25 | writer: anytype, | ||
| 26 | ) !void { | ||
| 27 | var decoder = try Lzma2Decoder.init(allocator); | ||
| 28 | defer decoder.deinit(allocator); | 14 | defer decoder.deinit(allocator); |
| 29 | return decoder.decompress(allocator, reader, writer); | 15 | return decoder.decompress(allocator, reader, writer); |
| 30 | } | 16 | } |
| 31 | 17 | ||
| 32 | test { | 18 | test { |
| 33 | _ = @import("lzma/lzma_test.zig"); | 19 | _ = @import("lzma/test.zig"); |
| 34 | _ = @import("lzma/lzma2_test.zig"); | ||
| 35 | _ = @import("lzma/vec2d.zig"); | 20 | _ = @import("lzma/vec2d.zig"); |
| 36 | } | 21 | } |
lib/std/compress/lzma/decode.zig+395-2| ... | @@ -1,8 +1,17 @@ | ... | @@ -1,8 +1,17 @@ |
| 1 | const std = @import("../../std.zig"); | ||
| 2 | const assert = std.debug.assert; | ||
| 3 | const math = std.math; | ||
| 4 | const Allocator = std.mem.Allocator; | ||
| 5 | |||
| 1 | pub const lzbuffer = @import("decode/lzbuffer.zig"); | 6 | pub const lzbuffer = @import("decode/lzbuffer.zig"); |
| 2 | pub const lzma = @import("decode/lzma.zig"); | ||
| 3 | pub const lzma2 = @import("decode/lzma2.zig"); | ||
| 4 | pub const rangecoder = @import("decode/rangecoder.zig"); | 7 | pub const rangecoder = @import("decode/rangecoder.zig"); |
| 5 | 8 | ||
| 9 | const LzCircularBuffer = lzbuffer.LzCircularBuffer; | ||
| 10 | const BitTree = rangecoder.BitTree; | ||
| 11 | const LenDecoder = rangecoder.LenDecoder; | ||
| 12 | const RangeDecoder = rangecoder.RangeDecoder; | ||
| 13 | const Vec2D = @import("vec2d.zig").Vec2D; | ||
| 14 | |||
| 6 | pub const Options = struct { | 15 | pub const Options = struct { |
| 7 | unpacked_size: UnpackedSize = .read_from_header, | 16 | unpacked_size: UnpackedSize = .read_from_header, |
| 8 | memlimit: ?usize = null, | 17 | memlimit: ?usize = null, |
| ... | @@ -14,3 +23,387 @@ pub const UnpackedSize = union(enum) { | ... | @@ -14,3 +23,387 @@ pub const UnpackedSize = union(enum) { |
| 14 | read_header_but_use_provided: ?u64, | 23 | read_header_but_use_provided: ?u64, |
| 15 | use_provided: ?u64, | 24 | use_provided: ?u64, |
| 16 | }; | 25 | }; |
| 26 | |||
| 27 | const ProcessingStatus = enum { | ||
| 28 | continue_, | ||
| 29 | finished, | ||
| 30 | }; | ||
| 31 | |||
| 32 | pub const Properties = struct { | ||
| 33 | lc: u4, | ||
| 34 | lp: u3, | ||
| 35 | pb: u3, | ||
| 36 | |||
| 37 | fn validate(self: Properties) void { | ||
| 38 | assert(self.lc <= 8); | ||
| 39 | assert(self.lp <= 4); | ||
| 40 | assert(self.pb <= 4); | ||
| 41 | } | ||
| 42 | }; | ||
| 43 | |||
| 44 | pub const Params = struct { | ||
| 45 | properties: Properties, | ||
| 46 | dict_size: u32, | ||
| 47 | unpacked_size: ?u64, | ||
| 48 | |||
| 49 | pub fn readHeader(reader: anytype, options: Options) !Params { | ||
| 50 | var props = try reader.readByte(); | ||
| 51 | if (props >= 225) { | ||
| 52 | return error.CorruptInput; | ||
| 53 | } | ||
| 54 | |||
| 55 | const lc = @intCast(u4, props % 9); | ||
| 56 | props /= 9; | ||
| 57 | const lp = @intCast(u3, props % 5); | ||
| 58 | props /= 5; | ||
| 59 | const pb = @intCast(u3, props); | ||
| 60 | |||
| 61 | const dict_size_provided = try reader.readIntLittle(u32); | ||
| 62 | const dict_size = math.max(0x1000, dict_size_provided); | ||
| 63 | |||
| 64 | const unpacked_size = switch (options.unpacked_size) { | ||
| 65 | .read_from_header => blk: { | ||
| 66 | const unpacked_size_provided = try reader.readIntLittle(u64); | ||
| 67 | const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF; | ||
| 68 | break :blk if (marker_mandatory) | ||
| 69 | null | ||
| 70 | else | ||
| 71 | unpacked_size_provided; | ||
| 72 | }, | ||
| 73 | .read_header_but_use_provided => |x| blk: { | ||
| 74 | _ = try reader.readIntLittle(u64); | ||
| 75 | break :blk x; | ||
| 76 | }, | ||
| 77 | .use_provided => |x| x, | ||
| 78 | }; | ||
| 79 | |||
| 80 | return Params{ | ||
| 81 | .properties = Properties{ .lc = lc, .lp = lp, .pb = pb }, | ||
| 82 | .dict_size = dict_size, | ||
| 83 | .unpacked_size = unpacked_size, | ||
| 84 | }; | ||
| 85 | } | ||
| 86 | }; | ||
| 87 | |||
| 88 | pub const DecoderState = struct { | ||
| 89 | lzma_props: Properties, | ||
| 90 | unpacked_size: ?u64, | ||
| 91 | literal_probs: Vec2D(u16), | ||
| 92 | pos_slot_decoder: [4]BitTree(6), | ||
| 93 | align_decoder: BitTree(4), | ||
| 94 | pos_decoders: [115]u16, | ||
| 95 | is_match: [192]u16, | ||
| 96 | is_rep: [12]u16, | ||
| 97 | is_rep_g0: [12]u16, | ||
| 98 | is_rep_g1: [12]u16, | ||
| 99 | is_rep_g2: [12]u16, | ||
| 100 | is_rep_0long: [192]u16, | ||
| 101 | state: usize, | ||
| 102 | rep: [4]usize, | ||
| 103 | len_decoder: LenDecoder, | ||
| 104 | rep_len_decoder: LenDecoder, | ||
| 105 | |||
| 106 | pub fn init( | ||
| 107 | allocator: Allocator, | ||
| 108 | lzma_props: Properties, | ||
| 109 | unpacked_size: ?u64, | ||
| 110 | ) !DecoderState { | ||
| 111 | return .{ | ||
| 112 | .lzma_props = lzma_props, | ||
| 113 | .unpacked_size = unpacked_size, | ||
| 114 | .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }), | ||
| 115 | .pos_slot_decoder = .{.{}} ** 4, | ||
| 116 | .align_decoder = .{}, | ||
| 117 | .pos_decoders = .{0x400} ** 115, | ||
| 118 | .is_match = .{0x400} ** 192, | ||
| 119 | .is_rep = .{0x400} ** 12, | ||
| 120 | .is_rep_g0 = .{0x400} ** 12, | ||
| 121 | .is_rep_g1 = .{0x400} ** 12, | ||
| 122 | .is_rep_g2 = .{0x400} ** 12, | ||
| 123 | .is_rep_0long = .{0x400} ** 192, | ||
| 124 | .state = 0, | ||
| 125 | .rep = .{0} ** 4, | ||
| 126 | .len_decoder = .{}, | ||
| 127 | .rep_len_decoder = .{}, | ||
| 128 | }; | ||
| 129 | } | ||
| 130 | |||
| 131 | pub fn deinit(self: *DecoderState, allocator: Allocator) void { | ||
| 132 | self.literal_probs.deinit(allocator); | ||
| 133 | self.* = undefined; | ||
| 134 | } | ||
| 135 | |||
| 136 | pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void { | ||
| 137 | new_props.validate(); | ||
| 138 | if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) { | ||
| 139 | self.literal_probs.fill(0x400); | ||
| 140 | } else { | ||
| 141 | self.literal_probs.deinit(allocator); | ||
| 142 | self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 }); | ||
| 143 | } | ||
| 144 | |||
| 145 | self.lzma_props = new_props; | ||
| 146 | for (self.pos_slot_decoder) |*t| t.reset(); | ||
| 147 | self.align_decoder.reset(); | ||
| 148 | self.pos_decoders = .{0x400} ** 115; | ||
| 149 | self.is_match = .{0x400} ** 192; | ||
| 150 | self.is_rep = .{0x400} ** 12; | ||
| 151 | self.is_rep_g0 = .{0x400} ** 12; | ||
| 152 | self.is_rep_g1 = .{0x400} ** 12; | ||
| 153 | self.is_rep_g2 = .{0x400} ** 12; | ||
| 154 | self.is_rep_0long = .{0x400} ** 192; | ||
| 155 | self.state = 0; | ||
| 156 | self.rep = .{0} ** 4; | ||
| 157 | self.len_decoder.reset(); | ||
| 158 | self.rep_len_decoder.reset(); | ||
| 159 | } | ||
| 160 | |||
| 161 | fn processNextInner( | ||
| 162 | self: *DecoderState, | ||
| 163 | allocator: Allocator, | ||
| 164 | reader: anytype, | ||
| 165 | writer: anytype, | ||
| 166 | buffer: anytype, | ||
| 167 | decoder: *RangeDecoder, | ||
| 168 | update: bool, | ||
| 169 | ) !ProcessingStatus { | ||
| 170 | const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1); | ||
| 171 | |||
| 172 | if (!try decoder.decodeBit( | ||
| 173 | reader, | ||
| 174 | &self.is_match[(self.state << 4) + pos_state], | ||
| 175 | update, | ||
| 176 | )) { | ||
| 177 | const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update); | ||
| 178 | |||
| 179 | if (update) { | ||
| 180 | try buffer.appendLiteral(allocator, byte, writer); | ||
| 181 | |||
| 182 | self.state = if (self.state < 4) | ||
| 183 | 0 | ||
| 184 | else if (self.state < 10) | ||
| 185 | self.state - 3 | ||
| 186 | else | ||
| 187 | self.state - 6; | ||
| 188 | } | ||
| 189 | return .continue_; | ||
| 190 | } | ||
| 191 | |||
| 192 | var len: usize = undefined; | ||
| 193 | if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) { | ||
| 194 | if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) { | ||
| 195 | if (!try decoder.decodeBit( | ||
| 196 | reader, | ||
| 197 | &self.is_rep_0long[(self.state << 4) + pos_state], | ||
| 198 | update, | ||
| 199 | )) { | ||
| 200 | if (update) { | ||
| 201 | self.state = if (self.state < 7) 9 else 11; | ||
| 202 | const dist = self.rep[0] + 1; | ||
| 203 | try buffer.appendLz(allocator, 1, dist, writer); | ||
| 204 | } | ||
| 205 | return .continue_; | ||
| 206 | } | ||
| 207 | } else { | ||
| 208 | const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update)) | ||
| 209 | 1 | ||
| 210 | else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update)) | ||
| 211 | 2 | ||
| 212 | else | ||
| 213 | 3; | ||
| 214 | if (update) { | ||
| 215 | const dist = self.rep[idx]; | ||
| 216 | var i = idx; | ||
| 217 | while (i > 0) : (i -= 1) { | ||
| 218 | self.rep[i] = self.rep[i - 1]; | ||
| 219 | } | ||
| 220 | self.rep[0] = dist; | ||
| 221 | } | ||
| 222 | } | ||
| 223 | |||
| 224 | len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update); | ||
| 225 | |||
| 226 | if (update) { | ||
| 227 | self.state = if (self.state < 7) 8 else 11; | ||
| 228 | } | ||
| 229 | } else { | ||
| 230 | if (update) { | ||
| 231 | self.rep[3] = self.rep[2]; | ||
| 232 | self.rep[2] = self.rep[1]; | ||
| 233 | self.rep[1] = self.rep[0]; | ||
| 234 | } | ||
| 235 | |||
| 236 | len = try self.len_decoder.decode(reader, decoder, pos_state, update); | ||
| 237 | |||
| 238 | if (update) { | ||
| 239 | self.state = if (self.state < 7) 7 else 10; | ||
| 240 | } | ||
| 241 | |||
| 242 | const rep_0 = try self.decodeDistance(reader, decoder, len, update); | ||
| 243 | |||
| 244 | if (update) { | ||
| 245 | self.rep[0] = rep_0; | ||
| 246 | if (self.rep[0] == 0xFFFF_FFFF) { | ||
| 247 | if (decoder.isFinished()) { | ||
| 248 | return .finished; | ||
| 249 | } | ||
| 250 | return error.CorruptInput; | ||
| 251 | } | ||
| 252 | } | ||
| 253 | } | ||
| 254 | |||
| 255 | if (update) { | ||
| 256 | len += 2; | ||
| 257 | |||
| 258 | const dist = self.rep[0] + 1; | ||
| 259 | try buffer.appendLz(allocator, len, dist, writer); | ||
| 260 | } | ||
| 261 | |||
| 262 | return .continue_; | ||
| 263 | } | ||
| 264 | |||
| 265 | fn processNext( | ||
| 266 | self: *DecoderState, | ||
| 267 | allocator: Allocator, | ||
| 268 | reader: anytype, | ||
| 269 | writer: anytype, | ||
| 270 | buffer: anytype, | ||
| 271 | decoder: *RangeDecoder, | ||
| 272 | ) !ProcessingStatus { | ||
| 273 | return self.processNextInner(allocator, reader, writer, buffer, decoder, true); | ||
| 274 | } | ||
| 275 | |||
| 276 | pub fn process( | ||
| 277 | self: *DecoderState, | ||
| 278 | allocator: Allocator, | ||
| 279 | reader: anytype, | ||
| 280 | writer: anytype, | ||
| 281 | buffer: anytype, | ||
| 282 | decoder: *RangeDecoder, | ||
| 283 | ) !void { | ||
| 284 | while (true) { | ||
| 285 | if (self.unpacked_size) |unpacked_size| { | ||
| 286 | if (buffer.len >= unpacked_size) { | ||
| 287 | break; | ||
| 288 | } | ||
| 289 | } else if (decoder.isFinished()) { | ||
| 290 | break; | ||
| 291 | } | ||
| 292 | |||
| 293 | if (try self.processNext(allocator, reader, writer, buffer, decoder) == .finished) { | ||
| 294 | break; | ||
| 295 | } | ||
| 296 | } | ||
| 297 | |||
| 298 | if (self.unpacked_size) |len| { | ||
| 299 | if (len != buffer.len) { | ||
| 300 | return error.CorruptInput; | ||
| 301 | } | ||
| 302 | } | ||
| 303 | } | ||
| 304 | |||
| 305 | fn decodeLiteral( | ||
| 306 | self: *DecoderState, | ||
| 307 | reader: anytype, | ||
| 308 | buffer: anytype, | ||
| 309 | decoder: *RangeDecoder, | ||
| 310 | update: bool, | ||
| 311 | ) !u8 { | ||
| 312 | const def_prev_byte = 0; | ||
| 313 | const prev_byte = @as(usize, buffer.lastOr(def_prev_byte)); | ||
| 314 | |||
| 315 | var result: usize = 1; | ||
| 316 | const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) + | ||
| 317 | (prev_byte >> (8 - self.lzma_props.lc)); | ||
| 318 | const probs = try self.literal_probs.getMut(lit_state); | ||
| 319 | |||
| 320 | if (self.state >= 7) { | ||
| 321 | var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1)); | ||
| 322 | |||
| 323 | while (result < 0x100) { | ||
| 324 | const match_bit = (match_byte >> 7) & 1; | ||
| 325 | match_byte <<= 1; | ||
| 326 | const bit = @boolToInt(try decoder.decodeBit( | ||
| 327 | reader, | ||
| 328 | &probs[((@as(usize, 1) + match_bit) << 8) + result], | ||
| 329 | update, | ||
| 330 | )); | ||
| 331 | result = (result << 1) ^ bit; | ||
| 332 | if (match_bit != bit) { | ||
| 333 | break; | ||
| 334 | } | ||
| 335 | } | ||
| 336 | } | ||
| 337 | |||
| 338 | while (result < 0x100) { | ||
| 339 | result = (result << 1) ^ @boolToInt(try decoder.decodeBit(reader, &probs[result], update)); | ||
| 340 | } | ||
| 341 | |||
| 342 | return @truncate(u8, result - 0x100); | ||
| 343 | } | ||
| 344 | |||
| 345 | fn decodeDistance( | ||
| 346 | self: *DecoderState, | ||
| 347 | reader: anytype, | ||
| 348 | decoder: *RangeDecoder, | ||
| 349 | length: usize, | ||
| 350 | update: bool, | ||
| 351 | ) !usize { | ||
| 352 | const len_state = if (length > 3) 3 else length; | ||
| 353 | |||
| 354 | const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update)); | ||
| 355 | if (pos_slot < 4) | ||
| 356 | return pos_slot; | ||
| 357 | |||
| 358 | const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1); | ||
| 359 | var result = (2 ^ (pos_slot & 1)) << num_direct_bits; | ||
| 360 | |||
| 361 | if (pos_slot < 14) { | ||
| 362 | result += try decoder.parseReverseBitTree( | ||
| 363 | reader, | ||
| 364 | num_direct_bits, | ||
| 365 | &self.pos_decoders, | ||
| 366 | result - pos_slot, | ||
| 367 | update, | ||
| 368 | ); | ||
| 369 | } else { | ||
| 370 | result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4; | ||
| 371 | result += try self.align_decoder.parseReverse(reader, decoder, update); | ||
| 372 | } | ||
| 373 | |||
| 374 | return result; | ||
| 375 | } | ||
| 376 | }; | ||
| 377 | |||
| 378 | pub const Decoder = struct { | ||
| 379 | params: Params, | ||
| 380 | memlimit: usize, | ||
| 381 | state: DecoderState, | ||
| 382 | |||
| 383 | pub fn init(allocator: Allocator, params: Params, memlimit: ?usize) !Decoder { | ||
| 384 | return Decoder{ | ||
| 385 | .params = params, | ||
| 386 | .memlimit = memlimit orelse math.maxInt(usize), | ||
| 387 | .state = try DecoderState.init(allocator, params.properties, params.unpacked_size), | ||
| 388 | }; | ||
| 389 | } | ||
| 390 | |||
| 391 | pub fn deinit(self: *Decoder, allocator: Allocator) void { | ||
| 392 | self.state.deinit(allocator); | ||
| 393 | self.* = undefined; | ||
| 394 | } | ||
| 395 | |||
| 396 | pub fn decompress( | ||
| 397 | self: *Decoder, | ||
| 398 | allocator: Allocator, | ||
| 399 | reader: anytype, | ||
| 400 | writer: anytype, | ||
| 401 | ) !void { | ||
| 402 | var buffer = LzCircularBuffer.init(self.params.dict_size, self.memlimit); | ||
| 403 | defer buffer.deinit(allocator); | ||
| 404 | |||
| 405 | var decoder = try RangeDecoder.init(reader); | ||
| 406 | try self.state.process(allocator, reader, writer, &buffer, &decoder); | ||
| 407 | try buffer.finish(writer); | ||
| 408 | } | ||
| 409 | }; |
lib/std/compress/lzma/decode/lzma.zig deleted-398| ... | @@ -1,398 +0,0 @@ | ||
| 1 | const std = @import("../../../std.zig"); | ||
| 2 | const assert = std.debug.assert; | ||
| 3 | const math = std.math; | ||
| 4 | const Allocator = std.mem.Allocator; | ||
| 5 | const ArrayListUnmanaged = std.ArrayListUnmanaged; | ||
| 6 | const FixedBufferStream = std.io.FixedBufferStream; | ||
| 7 | |||
| 8 | const LzCircularBuffer = @import("lzbuffer.zig").LzCircularBuffer; | ||
| 9 | const Options = @import("../decode.zig").Options; | ||
| 10 | const Vec2D = @import("../vec2d.zig").Vec2D; | ||
| 11 | const rangecoder = @import("rangecoder.zig"); | ||
| 12 | const BitTree = rangecoder.BitTree; | ||
| 13 | const LenDecoder = rangecoder.LenDecoder; | ||
| 14 | const RangeDecoder = rangecoder.RangeDecoder; | ||
| 15 | |||
| 16 | const ProcessingStatus = enum { | ||
| 17 | continue_, | ||
| 18 | finished, | ||
| 19 | }; | ||
| 20 | |||
| 21 | pub const LzmaProperties = struct { | ||
| 22 | lc: u4, | ||
| 23 | lp: u3, | ||
| 24 | pb: u3, | ||
| 25 | |||
| 26 | fn validate(self: LzmaProperties) void { | ||
| 27 | assert(self.lc <= 8); | ||
| 28 | assert(self.lp <= 4); | ||
| 29 | assert(self.pb <= 4); | ||
| 30 | } | ||
| 31 | }; | ||
| 32 | |||
| 33 | pub const LzmaParams = struct { | ||
| 34 | properties: LzmaProperties, | ||
| 35 | dict_size: u32, | ||
| 36 | unpacked_size: ?u64, | ||
| 37 | |||
| 38 | pub fn readHeader(reader: anytype, options: Options) !LzmaParams { | ||
| 39 | var props = try reader.readByte(); | ||
| 40 | if (props >= 225) { | ||
| 41 | return error.CorruptInput; | ||
| 42 | } | ||
| 43 | |||
| 44 | const lc = @intCast(u4, props % 9); | ||
| 45 | props /= 9; | ||
| 46 | const lp = @intCast(u3, props % 5); | ||
| 47 | props /= 5; | ||
| 48 | const pb = @intCast(u3, props); | ||
| 49 | |||
| 50 | const dict_size_provided = try reader.readIntLittle(u32); | ||
| 51 | const dict_size = math.max(0x1000, dict_size_provided); | ||
| 52 | |||
| 53 | const unpacked_size = switch (options.unpacked_size) { | ||
| 54 | .read_from_header => blk: { | ||
| 55 | const unpacked_size_provided = try reader.readIntLittle(u64); | ||
| 56 | const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF; | ||
| 57 | break :blk if (marker_mandatory) | ||
| 58 | null | ||
| 59 | else | ||
| 60 | unpacked_size_provided; | ||
| 61 | }, | ||
| 62 | .read_header_but_use_provided => |x| blk: { | ||
| 63 | _ = try reader.readIntLittle(u64); | ||
| 64 | break :blk x; | ||
| 65 | }, | ||
| 66 | .use_provided => |x| x, | ||
| 67 | }; | ||
| 68 | |||
| 69 | return LzmaParams{ | ||
| 70 | .properties = LzmaProperties{ .lc = lc, .lp = lp, .pb = pb }, | ||
| 71 | .dict_size = dict_size, | ||
| 72 | .unpacked_size = unpacked_size, | ||
| 73 | }; | ||
| 74 | } | ||
| 75 | }; | ||
| 76 | |||
| 77 | pub const DecoderState = struct { | ||
| 78 | lzma_props: LzmaProperties, | ||
| 79 | unpacked_size: ?u64, | ||
| 80 | literal_probs: Vec2D(u16), | ||
| 81 | pos_slot_decoder: [4]BitTree(6), | ||
| 82 | align_decoder: BitTree(4), | ||
| 83 | pos_decoders: [115]u16, | ||
| 84 | is_match: [192]u16, | ||
| 85 | is_rep: [12]u16, | ||
| 86 | is_rep_g0: [12]u16, | ||
| 87 | is_rep_g1: [12]u16, | ||
| 88 | is_rep_g2: [12]u16, | ||
| 89 | is_rep_0long: [192]u16, | ||
| 90 | state: usize, | ||
| 91 | rep: [4]usize, | ||
| 92 | len_decoder: LenDecoder, | ||
| 93 | rep_len_decoder: LenDecoder, | ||
| 94 | |||
| 95 | pub fn init( | ||
| 96 | allocator: Allocator, | ||
| 97 | lzma_props: LzmaProperties, | ||
| 98 | unpacked_size: ?u64, | ||
| 99 | ) !DecoderState { | ||
| 100 | return .{ | ||
| 101 | .lzma_props = lzma_props, | ||
| 102 | .unpacked_size = unpacked_size, | ||
| 103 | .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }), | ||
| 104 | .pos_slot_decoder = .{.{}} ** 4, | ||
| 105 | .align_decoder = .{}, | ||
| 106 | .pos_decoders = .{0x400} ** 115, | ||
| 107 | .is_match = .{0x400} ** 192, | ||
| 108 | .is_rep = .{0x400} ** 12, | ||
| 109 | .is_rep_g0 = .{0x400} ** 12, | ||
| 110 | .is_rep_g1 = .{0x400} ** 12, | ||
| 111 | .is_rep_g2 = .{0x400} ** 12, | ||
| 112 | .is_rep_0long = .{0x400} ** 192, | ||
| 113 | .state = 0, | ||
| 114 | .rep = .{0} ** 4, | ||
| 115 | .len_decoder = .{}, | ||
| 116 | .rep_len_decoder = .{}, | ||
| 117 | }; | ||
| 118 | } | ||
| 119 | |||
| 120 | pub fn deinit(self: *DecoderState, allocator: Allocator) void { | ||
| 121 | self.literal_probs.deinit(allocator); | ||
| 122 | self.* = undefined; | ||
| 123 | } | ||
| 124 | |||
| 125 | pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: LzmaProperties) !void { | ||
| 126 | new_props.validate(); | ||
| 127 | if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) { | ||
| 128 | self.literal_probs.fill(0x400); | ||
| 129 | } else { | ||
| 130 | self.literal_probs.deinit(allocator); | ||
| 131 | self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 }); | ||
| 132 | } | ||
| 133 | |||
| 134 | self.lzma_props = new_props; | ||
| 135 | for (self.pos_slot_decoder) |*t| t.reset(); | ||
| 136 | self.align_decoder.reset(); | ||
| 137 | self.pos_decoders = .{0x400} ** 115; | ||
| 138 | self.is_match = .{0x400} ** 192; | ||
| 139 | self.is_rep = .{0x400} ** 12; | ||
| 140 | self.is_rep_g0 = .{0x400} ** 12; | ||
| 141 | self.is_rep_g1 = .{0x400} ** 12; | ||
| 142 | self.is_rep_g2 = .{0x400} ** 12; | ||
| 143 | self.is_rep_0long = .{0x400} ** 192; | ||
| 144 | self.state = 0; | ||
| 145 | self.rep = .{0} ** 4; | ||
| 146 | self.len_decoder.reset(); | ||
| 147 | self.rep_len_decoder.reset(); | ||
| 148 | } | ||
| 149 | |||
| 150 | fn processNextInner( | ||
| 151 | self: *DecoderState, | ||
| 152 | allocator: Allocator, | ||
| 153 | reader: anytype, | ||
| 154 | writer: anytype, | ||
| 155 | buffer: anytype, | ||
| 156 | decoder: *RangeDecoder, | ||
| 157 | update: bool, | ||
| 158 | ) !ProcessingStatus { | ||
| 159 | const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1); | ||
| 160 | |||
| 161 | if (!try decoder.decodeBit( | ||
| 162 | reader, | ||
| 163 | &self.is_match[(self.state << 4) + pos_state], | ||
| 164 | update, | ||
| 165 | )) { | ||
| 166 | const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update); | ||
| 167 | |||
| 168 | if (update) { | ||
| 169 | try buffer.appendLiteral(allocator, byte, writer); | ||
| 170 | |||
| 171 | self.state = if (self.state < 4) | ||
| 172 | 0 | ||
| 173 | else if (self.state < 10) | ||
| 174 | self.state - 3 | ||
| 175 | else | ||
| 176 | self.state - 6; | ||
| 177 | } | ||
| 178 | return .continue_; | ||
| 179 | } | ||
| 180 | |||
| 181 | var len: usize = undefined; | ||
| 182 | if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) { | ||
| 183 | if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) { | ||
| 184 | if (!try decoder.decodeBit( | ||
| 185 | reader, | ||
| 186 | &self.is_rep_0long[(self.state << 4) + pos_state], | ||
| 187 | update, | ||
| 188 | )) { | ||
| 189 | if (update) { | ||
| 190 | self.state = if (self.state < 7) 9 else 11; | ||
| 191 | const dist = self.rep[0] + 1; | ||
| 192 | try buffer.appendLz(allocator, 1, dist, writer); | ||
| 193 | } | ||
| 194 | return .continue_; | ||
| 195 | } | ||
| 196 | } else { | ||
| 197 | const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update)) | ||
| 198 | 1 | ||
| 199 | else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update)) | ||
| 200 | 2 | ||
| 201 | else | ||
| 202 | 3; | ||
| 203 | if (update) { | ||
| 204 | const dist = self.rep[idx]; | ||
| 205 | var i = idx; | ||
| 206 | while (i > 0) : (i -= 1) { | ||
| 207 | self.rep[i] = self.rep[i - 1]; | ||
| 208 | } | ||
| 209 | self.rep[0] = dist; | ||
| 210 | } | ||
| 211 | } | ||
| 212 | |||
| 213 | len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update); | ||
| 214 | |||
| 215 | if (update) { | ||
| 216 | self.state = if (self.state < 7) 8 else 11; | ||
| 217 | } | ||
| 218 | } else { | ||
| 219 | if (update) { | ||
| 220 | self.rep[3] = self.rep[2]; | ||
| 221 | self.rep[2] = self.rep[1]; | ||
| 222 | self.rep[1] = self.rep[0]; | ||
| 223 | } | ||
| 224 | |||
| 225 | len = try self.len_decoder.decode(reader, decoder, pos_state, update); | ||
| 226 | |||
| 227 | if (update) { | ||
| 228 | self.state = if (self.state < 7) 7 else 10; | ||
| 229 | } | ||
| 230 | |||
| 231 | const rep_0 = try self.decodeDistance(reader, decoder, len, update); | ||
| 232 | |||
| 233 | if (update) { | ||
| 234 | self.rep[0] = rep_0; | ||
| 235 | if (self.rep[0] == 0xFFFF_FFFF) { | ||
| 236 | if (decoder.isFinished()) { | ||
| 237 | return .finished; | ||
| 238 | } | ||
| 239 | return error.CorruptInput; | ||
| 240 | } | ||
| 241 | } | ||
| 242 | } | ||
| 243 | |||
| 244 | if (update) { | ||
| 245 | len += 2; | ||
| 246 | |||
| 247 | const dist = self.rep[0] + 1; | ||
| 248 | try buffer.appendLz(allocator, len, dist, writer); | ||
| 249 | } | ||
| 250 | |||
| 251 | return .continue_; | ||
| 252 | } | ||
| 253 | |||
| 254 | fn processNext( | ||
| 255 | self: *DecoderState, | ||
| 256 | allocator: Allocator, | ||
| 257 | reader: anytype, | ||
| 258 | writer: anytype, | ||
| 259 | buffer: anytype, | ||
| 260 | decoder: *RangeDecoder, | ||
| 261 | ) !ProcessingStatus { | ||
| 262 | return self.processNextInner(allocator, reader, writer, buffer, decoder, true); | ||
| 263 | } | ||
| 264 | |||
| 265 | pub fn process( | ||
| 266 | self: *DecoderState, | ||
| 267 | allocator: Allocator, | ||
| 268 | reader: anytype, | ||
| 269 | writer: anytype, | ||
| 270 | buffer: anytype, | ||
| 271 | decoder: *RangeDecoder, | ||
| 272 | ) !void { | ||
| 273 | while (true) { | ||
| 274 | if (self.unpacked_size) |unpacked_size| { | ||
| 275 | if (buffer.len >= unpacked_size) { | ||
| 276 | break; | ||
| 277 | } | ||
| 278 | } else if (decoder.isFinished()) { | ||
| 279 | break; | ||
| 280 | } | ||
| 281 | |||
| 282 | if (try self.processNext(allocator, reader, writer, buffer, decoder) == .finished) { | ||
| 283 | break; | ||
| 284 | } | ||
| 285 | } | ||
| 286 | |||
| 287 | if (self.unpacked_size) |len| { | ||
| 288 | if (len != buffer.len) { | ||
| 289 | return error.CorruptInput; | ||
| 290 | } | ||
| 291 | } | ||
| 292 | } | ||
| 293 | |||
| 294 | fn decodeLiteral( | ||
| 295 | self: *DecoderState, | ||
| 296 | reader: anytype, | ||
| 297 | buffer: anytype, | ||
| 298 | decoder: *RangeDecoder, | ||
| 299 | update: bool, | ||
| 300 | ) !u8 { | ||
| 301 | const def_prev_byte = 0; | ||
| 302 | const prev_byte = @as(usize, buffer.lastOr(def_prev_byte)); | ||
| 303 | |||
| 304 | var result: usize = 1; | ||
| 305 | const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) + | ||
| 306 | (prev_byte >> (8 - self.lzma_props.lc)); | ||
| 307 | const probs = try self.literal_probs.getMut(lit_state); | ||
| 308 | |||
| 309 | if (self.state >= 7) { | ||
| 310 | var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1)); | ||
| 311 | |||
| 312 | while (result < 0x100) { | ||
| 313 | const match_bit = (match_byte >> 7) & 1; | ||
| 314 | match_byte <<= 1; | ||
| 315 | const bit = @boolToInt(try decoder.decodeBit( | ||
| 316 | reader, | ||
| 317 | &probs[((@as(usize, 1) + match_bit) << 8) + result], | ||
| 318 | update, | ||
| 319 | )); | ||
| 320 | result = (result << 1) ^ bit; | ||
| 321 | if (match_bit != bit) { | ||
| 322 | break; | ||
| 323 | } | ||
| 324 | } | ||
| 325 | } | ||
| 326 | |||
| 327 | while (result < 0x100) { | ||
| 328 | result = (result << 1) ^ @boolToInt(try decoder.decodeBit(reader, &probs[result], update)); | ||
| 329 | } | ||
| 330 | |||
| 331 | return @truncate(u8, result - 0x100); | ||
| 332 | } | ||
| 333 | |||
| 334 | fn decodeDistance( | ||
| 335 | self: *DecoderState, | ||
| 336 | reader: anytype, | ||
| 337 | decoder: *RangeDecoder, | ||
| 338 | length: usize, | ||
| 339 | update: bool, | ||
| 340 | ) !usize { | ||
| 341 | const len_state = if (length > 3) 3 else length; | ||
| 342 | |||
| 343 | const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update)); | ||
| 344 | if (pos_slot < 4) | ||
| 345 | return pos_slot; | ||
| 346 | |||
| 347 | const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1); | ||
| 348 | var result = (2 ^ (pos_slot & 1)) << num_direct_bits; | ||
| 349 | |||
| 350 | if (pos_slot < 14) { | ||
| 351 | result += try decoder.parseReverseBitTree( | ||
| 352 | reader, | ||
| 353 | num_direct_bits, | ||
| 354 | &self.pos_decoders, | ||
| 355 | result - pos_slot, | ||
| 356 | update, | ||
| 357 | ); | ||
| 358 | } else { | ||
| 359 | result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4; | ||
| 360 | result += try self.align_decoder.parseReverse(reader, decoder, update); | ||
| 361 | } | ||
| 362 | |||
| 363 | return result; | ||
| 364 | } | ||
| 365 | }; | ||
| 366 | |||
| 367 | pub const LzmaDecoder = struct { | ||
| 368 | params: LzmaParams, | ||
| 369 | memlimit: usize, | ||
| 370 | state: DecoderState, | ||
| 371 | |||
| 372 | pub fn init(allocator: Allocator, params: LzmaParams, memlimit: ?usize) !LzmaDecoder { | ||
| 373 | return LzmaDecoder{ | ||
| 374 | .params = params, | ||
| 375 | .memlimit = memlimit orelse math.maxInt(usize), | ||
| 376 | .state = try DecoderState.init(allocator, params.properties, params.unpacked_size), | ||
| 377 | }; | ||
| 378 | } | ||
| 379 | |||
| 380 | pub fn deinit(self: *LzmaDecoder, allocator: Allocator) void { | ||
| 381 | self.state.deinit(allocator); | ||
| 382 | self.* = undefined; | ||
| 383 | } | ||
| 384 | |||
| 385 | pub fn decompress( | ||
| 386 | self: *LzmaDecoder, | ||
| 387 | allocator: Allocator, | ||
| 388 | reader: anytype, | ||
| 389 | writer: anytype, | ||
| 390 | ) !void { | ||
| 391 | var buffer = LzCircularBuffer.init(self.params.dict_size, self.memlimit); | ||
| 392 | defer buffer.deinit(allocator); | ||
| 393 | |||
| 394 | var decoder = try RangeDecoder.init(reader); | ||
| 395 | try self.state.process(allocator, reader, writer, &buffer, &decoder); | ||
| 396 | try buffer.finish(writer); | ||
| 397 | } | ||
| 398 | }; | ||
lib/std/compress/lzma/decode/lzma2.zig deleted-169| ... | @@ -1,169 +0,0 @@ | ||
| 1 | const std = @import("../../../std.zig"); | ||
| 2 | const Allocator = std.mem.Allocator; | ||
| 3 | |||
| 4 | const lzma = @import("lzma.zig"); | ||
| 5 | const DecoderState = lzma.DecoderState; | ||
| 6 | const LzmaProperties = lzma.LzmaProperties; | ||
| 7 | const LzAccumBuffer = @import("lzbuffer.zig").LzAccumBuffer; | ||
| 8 | const RangeDecoder = @import("rangecoder.zig").RangeDecoder; | ||
| 9 | |||
| 10 | pub const Lzma2Decoder = struct { | ||
| 11 | lzma_state: DecoderState, | ||
| 12 | |||
| 13 | pub fn init(allocator: Allocator) !Lzma2Decoder { | ||
| 14 | return Lzma2Decoder{ | ||
| 15 | .lzma_state = try DecoderState.init( | ||
| 16 | allocator, | ||
| 17 | LzmaProperties{ | ||
| 18 | .lc = 0, | ||
| 19 | .lp = 0, | ||
| 20 | .pb = 0, | ||
| 21 | }, | ||
| 22 | null, | ||
| 23 | ), | ||
| 24 | }; | ||
| 25 | } | ||
| 26 | |||
| 27 | pub fn deinit(self: *Lzma2Decoder, allocator: Allocator) void { | ||
| 28 | self.lzma_state.deinit(allocator); | ||
| 29 | self.* = undefined; | ||
| 30 | } | ||
| 31 | |||
| 32 | pub fn decompress( | ||
| 33 | self: *Lzma2Decoder, | ||
| 34 | allocator: Allocator, | ||
| 35 | reader: anytype, | ||
| 36 | writer: anytype, | ||
| 37 | ) !void { | ||
| 38 | var accum = LzAccumBuffer.init(std.math.maxInt(usize)); | ||
| 39 | defer accum.deinit(allocator); | ||
| 40 | |||
| 41 | while (true) { | ||
| 42 | const status = try reader.readByte(); | ||
| 43 | |||
| 44 | switch (status) { | ||
| 45 | 0 => break, | ||
| 46 | 1 => try parseUncompressed(allocator, reader, writer, &accum, true), | ||
| 47 | 2 => try parseUncompressed(allocator, reader, writer, &accum, false), | ||
| 48 | else => try self.parseLzma(allocator, reader, writer, &accum, status), | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | try accum.finish(writer); | ||
| 53 | } | ||
| 54 | |||
| 55 | fn parseLzma( | ||
| 56 | self: *Lzma2Decoder, | ||
| 57 | allocator: Allocator, | ||
| 58 | reader: anytype, | ||
| 59 | writer: anytype, | ||
| 60 | accum: *LzAccumBuffer, | ||
| 61 | status: u8, | ||
| 62 | ) !void { | ||
| 63 | if (status & 0x80 == 0) { | ||
| 64 | return error.CorruptInput; | ||
| 65 | } | ||
| 66 | |||
| 67 | const Reset = struct { | ||
| 68 | dict: bool, | ||
| 69 | state: bool, | ||
| 70 | props: bool, | ||
| 71 | }; | ||
| 72 | |||
| 73 | const reset = switch ((status >> 5) & 0x3) { | ||
| 74 | 0 => Reset{ | ||
| 75 | .dict = false, | ||
| 76 | .state = false, | ||
| 77 | .props = false, | ||
| 78 | }, | ||
| 79 | 1 => Reset{ | ||
| 80 | .dict = false, | ||
| 81 | .state = true, | ||
| 82 | .props = false, | ||
| 83 | }, | ||
| 84 | 2 => Reset{ | ||
| 85 | .dict = false, | ||
| 86 | .state = true, | ||
| 87 | .props = true, | ||
| 88 | }, | ||
| 89 | 3 => Reset{ | ||
| 90 | .dict = true, | ||
| 91 | .state = true, | ||
| 92 | .props = true, | ||
| 93 | }, | ||
| 94 | else => unreachable, | ||
| 95 | }; | ||
| 96 | |||
| 97 | const unpacked_size = blk: { | ||
| 98 | var tmp: u64 = status & 0x1F; | ||
| 99 | tmp <<= 16; | ||
| 100 | tmp |= try reader.readIntBig(u16); | ||
| 101 | break :blk tmp + 1; | ||
| 102 | }; | ||
| 103 | |||
| 104 | const packed_size = blk: { | ||
| 105 | const tmp: u17 = try reader.readIntBig(u16); | ||
| 106 | break :blk tmp + 1; | ||
| 107 | }; | ||
| 108 | |||
| 109 | if (reset.dict) { | ||
| 110 | try accum.reset(writer); | ||
| 111 | } | ||
| 112 | |||
| 113 | if (reset.state) { | ||
| 114 | var new_props = self.lzma_state.lzma_props; | ||
| 115 | |||
| 116 | if (reset.props) { | ||
| 117 | var props = try reader.readByte(); | ||
| 118 | if (props >= 225) { | ||
| 119 | return error.CorruptInput; | ||
| 120 | } | ||
| 121 | |||
| 122 | const lc = @intCast(u4, props % 9); | ||
| 123 | props /= 9; | ||
| 124 | const lp = @intCast(u3, props % 5); | ||
| 125 | props /= 5; | ||
| 126 | const pb = @intCast(u3, props); | ||
| 127 | |||
| 128 | if (lc + lp > 4) { | ||
| 129 | return error.CorruptInput; | ||
| 130 | } | ||
| 131 | |||
| 132 | new_props = LzmaProperties{ .lc = lc, .lp = lp, .pb = pb }; | ||
| 133 | } | ||
| 134 | |||
| 135 | try self.lzma_state.resetState(allocator, new_props); | ||
| 136 | } | ||
| 137 | |||
| 138 | self.lzma_state.unpacked_size = unpacked_size + accum.len; | ||
| 139 | |||
| 140 | var counter = std.io.countingReader(reader); | ||
| 141 | const counter_reader = counter.reader(); | ||
| 142 | |||
| 143 | var rangecoder = try RangeDecoder.init(counter_reader); | ||
| 144 | try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder); | ||
| 145 | |||
| 146 | if (counter.bytes_read != packed_size) { | ||
| 147 | return error.CorruptInput; | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | fn parseUncompressed( | ||
| 152 | allocator: Allocator, | ||
| 153 | reader: anytype, | ||
| 154 | writer: anytype, | ||
| 155 | accum: *LzAccumBuffer, | ||
| 156 | reset_dict: bool, | ||
| 157 | ) !void { | ||
| 158 | const unpacked_size = @as(u17, try reader.readIntBig(u16)) + 1; | ||
| 159 | |||
| 160 | if (reset_dict) { | ||
| 161 | try accum.reset(writer); | ||
| 162 | } | ||
| 163 | |||
| 164 | var i: @TypeOf(unpacked_size) = 0; | ||
| 165 | while (i < unpacked_size) : (i += 1) { | ||
| 166 | try accum.appendByte(allocator, try reader.readByte()); | ||
| 167 | } | ||
| 168 | } | ||
| 169 | }; | ||
lib/std/compress/lzma/decode/rangecoder.zig-3| ... | @@ -1,8 +1,5 @@ | ... | @@ -1,8 +1,5 @@ |
| 1 | const std = @import("../../../std.zig"); | 1 | const std = @import("../../../std.zig"); |
| 2 | const mem = std.mem; | 2 | const mem = std.mem; |
| 3 | const Allocator = std.mem.Allocator; | ||
| 4 | const ArrayListUnmanaged = std.ArrayListUnmanaged; | ||
| 5 | const FixedBufferStream = std.io.FixedBufferStream; | ||
| 6 | 3 | ||
| 7 | pub const RangeDecoder = struct { | 4 | pub const RangeDecoder = struct { |
| 8 | range: u32, | 5 | range: u32, |
lib/std/compress/lzma/lzma2_test.zig deleted-27| ... | @@ -1,27 +0,0 @@ | ||
| 1 | const std = @import("../../std.zig"); | ||
| 2 | const lzma = @import("../lzma.zig"); | ||
| 3 | |||
| 4 | fn testDecompress(compressed: []const u8, writer: anytype) !void { | ||
| 5 | const allocator = std.testing.allocator; | ||
| 6 | var stream = std.io.fixedBufferStream(compressed); | ||
| 7 | try lzma.lzma2Decompress(allocator, stream.reader(), writer); | ||
| 8 | } | ||
| 9 | |||
| 10 | fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void { | ||
| 11 | const allocator = std.testing.allocator; | ||
| 12 | var decomp = std.ArrayList(u8).init(allocator); | ||
| 13 | defer decomp.deinit(); | ||
| 14 | try testDecompress(compressed, decomp.writer()); | ||
| 15 | try std.testing.expectEqualSlices(u8, expected, decomp.items); | ||
| 16 | } | ||
| 17 | |||
| 18 | fn testDecompressError(expected: anyerror, compressed: []const u8) !void { | ||
| 19 | return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer)); | ||
| 20 | } | ||
| 21 | |||
| 22 | test { | ||
| 23 | try testDecompressEqual( | ||
| 24 | "Hello\nWorld!\n", | ||
| 25 | &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 }, | ||
| 26 | ); | ||
| 27 | } | ||
lib/std/compress/lzma/lzma_test.zig deleted-87| ... | @@ -1,87 +0,0 @@ | ||
| 1 | const std = @import("../../std.zig"); | ||
| 2 | const lzma = @import("../lzma.zig"); | ||
| 3 | |||
| 4 | fn testDecompress(compressed: []const u8, writer: anytype) !void { | ||
| 5 | const allocator = std.testing.allocator; | ||
| 6 | var stream = std.io.fixedBufferStream(compressed); | ||
| 7 | try lzma.lzmaDecompress(allocator, stream.reader(), writer, .{}); | ||
| 8 | } | ||
| 9 | |||
| 10 | fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void { | ||
| 11 | const allocator = std.testing.allocator; | ||
| 12 | var decomp = std.ArrayList(u8).init(allocator); | ||
| 13 | defer decomp.deinit(); | ||
| 14 | try testDecompress(compressed, decomp.writer()); | ||
| 15 | try std.testing.expectEqualSlices(u8, expected, decomp.items); | ||
| 16 | } | ||
| 17 | |||
| 18 | fn testDecompressError(expected: anyerror, compressed: []const u8) !void { | ||
| 19 | return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer)); | ||
| 20 | } | ||
| 21 | |||
| 22 | test "decompress empty world" { | ||
| 23 | try testDecompressEqual( | ||
| 24 | "", | ||
| 25 | &[_]u8{ | ||
| 26 | 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff, | ||
| 27 | 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, | ||
| 28 | }, | ||
| 29 | ); | ||
| 30 | } | ||
| 31 | |||
| 32 | test "decompress hello world" { | ||
| 33 | try testDecompressEqual( | ||
| 34 | "Hello world\n", | ||
| 35 | &[_]u8{ | ||
| 36 | 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19, | ||
| 37 | 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe, | ||
| 38 | 0xa5, 0xb0, 0x00, | ||
| 39 | }, | ||
| 40 | ); | ||
| 41 | } | ||
| 42 | |||
| 43 | test "decompress huge dict" { | ||
| 44 | try testDecompressEqual( | ||
| 45 | "Hello world\n", | ||
| 46 | &[_]u8{ | ||
| 47 | 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19, | ||
| 48 | 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe, | ||
| 49 | 0xa5, 0xb0, 0x00, | ||
| 50 | }, | ||
| 51 | ); | ||
| 52 | } | ||
| 53 | |||
| 54 | test "unknown size with end of payload marker" { | ||
| 55 | try testDecompressEqual( | ||
| 56 | "Hello\nWorld!\n", | ||
| 57 | @embedFile("testdata/good-unknown_size-with_eopm.lzma"), | ||
| 58 | ); | ||
| 59 | } | ||
| 60 | |||
| 61 | test "known size without end of payload marker" { | ||
| 62 | try testDecompressEqual( | ||
| 63 | "Hello\nWorld!\n", | ||
| 64 | @embedFile("testdata/good-known_size-without_eopm.lzma"), | ||
| 65 | ); | ||
| 66 | } | ||
| 67 | |||
| 68 | test "known size with end of payload marker" { | ||
| 69 | try testDecompressEqual( | ||
| 70 | "Hello\nWorld!\n", | ||
| 71 | @embedFile("testdata/good-known_size-with_eopm.lzma"), | ||
| 72 | ); | ||
| 73 | } | ||
| 74 | |||
| 75 | test "too big uncompressed size in header" { | ||
| 76 | try testDecompressError( | ||
| 77 | error.CorruptInput, | ||
| 78 | @embedFile("testdata/bad-too_big_size-with_eopm.lzma"), | ||
| 79 | ); | ||
| 80 | } | ||
| 81 | |||
| 82 | test "too small uncompressed size in header" { | ||
| 83 | try testDecompressError( | ||
| 84 | error.CorruptInput, | ||
| 85 | @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"), | ||
| 86 | ); | ||
| 87 | } | ||
lib/std/compress/lzma/test.zig created+87| ... | @@ -0,0 +1,87 @@ | ||
| 1 | const std = @import("../../std.zig"); | ||
| 2 | const lzma = @import("../lzma.zig"); | ||
| 3 | |||
| 4 | fn testDecompress(compressed: []const u8, writer: anytype) !void { | ||
| 5 | const allocator = std.testing.allocator; | ||
| 6 | var stream = std.io.fixedBufferStream(compressed); | ||
| 7 | try lzma.decompress(allocator, stream.reader(), writer, .{}); | ||
| 8 | } | ||
| 9 | |||
| 10 | fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void { | ||
| 11 | const allocator = std.testing.allocator; | ||
| 12 | var decomp = std.ArrayList(u8).init(allocator); | ||
| 13 | defer decomp.deinit(); | ||
| 14 | try testDecompress(compressed, decomp.writer()); | ||
| 15 | try std.testing.expectEqualSlices(u8, expected, decomp.items); | ||
| 16 | } | ||
| 17 | |||
| 18 | fn testDecompressError(expected: anyerror, compressed: []const u8) !void { | ||
| 19 | return std.testing.expectError(expected, testDecompress(compressed, std.io.null_writer)); | ||
| 20 | } | ||
| 21 | |||
| 22 | test "LZMA: decompress empty world" { | ||
| 23 | try testDecompressEqual( | ||
| 24 | "", | ||
| 25 | &[_]u8{ | ||
| 26 | 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff, | ||
| 27 | 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00, | ||
| 28 | }, | ||
| 29 | ); | ||
| 30 | } | ||
| 31 | |||
| 32 | test "LZMA: decompress hello world" { | ||
| 33 | try testDecompressEqual( | ||
| 34 | "Hello world\n", | ||
| 35 | &[_]u8{ | ||
| 36 | 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19, | ||
| 37 | 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe, | ||
| 38 | 0xa5, 0xb0, 0x00, | ||
| 39 | }, | ||
| 40 | ); | ||
| 41 | } | ||
| 42 | |||
| 43 | test "LZMA: decompress huge dict" { | ||
| 44 | try testDecompressEqual( | ||
| 45 | "Hello world\n", | ||
| 46 | &[_]u8{ | ||
| 47 | 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19, | ||
| 48 | 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe, | ||
| 49 | 0xa5, 0xb0, 0x00, | ||
| 50 | }, | ||
| 51 | ); | ||
| 52 | } | ||
| 53 | |||
| 54 | test "LZMA: unknown size with end of payload marker" { | ||
| 55 | try testDecompressEqual( | ||
| 56 | "Hello\nWorld!\n", | ||
| 57 | @embedFile("testdata/good-unknown_size-with_eopm.lzma"), | ||
| 58 | ); | ||
| 59 | } | ||
| 60 | |||
| 61 | test "LZMA: known size without end of payload marker" { | ||
| 62 | try testDecompressEqual( | ||
| 63 | "Hello\nWorld!\n", | ||
| 64 | @embedFile("testdata/good-known_size-without_eopm.lzma"), | ||
| 65 | ); | ||
| 66 | } | ||
| 67 | |||
| 68 | test "LZMA: known size with end of payload marker" { | ||
| 69 | try testDecompressEqual( | ||
| 70 | "Hello\nWorld!\n", | ||
| 71 | @embedFile("testdata/good-known_size-with_eopm.lzma"), | ||
| 72 | ); | ||
| 73 | } | ||
| 74 | |||
| 75 | test "LZMA: too big uncompressed size in header" { | ||
| 76 | try testDecompressError( | ||
| 77 | error.CorruptInput, | ||
| 78 | @embedFile("testdata/bad-too_big_size-with_eopm.lzma"), | ||
| 79 | ); | ||
| 80 | } | ||
| 81 | |||
| 82 | test "LZMA: too small uncompressed size in header" { | ||
| 83 | try testDecompressError( | ||
| 84 | error.CorruptInput, | ||
| 85 | @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"), | ||
| 86 | ); | ||
| 87 | } | ||
lib/std/compress/lzma2.zig created+26| ... | @@ -0,0 +1,26 @@ | ||
| 1 | const std = @import("../std.zig"); | ||
| 2 | const Allocator = std.mem.Allocator; | ||
| 3 | |||
| 4 | pub const decode = @import("lzma2/decode.zig"); | ||
| 5 | |||
| 6 | pub fn decompress( | ||
| 7 | allocator: Allocator, | ||
| 8 | reader: anytype, | ||
| 9 | writer: anytype, | ||
| 10 | ) !void { | ||
| 11 | var decoder = try decode.Decoder.init(allocator); | ||
| 12 | defer decoder.deinit(allocator); | ||
| 13 | return decoder.decompress(allocator, reader, writer); | ||
| 14 | } | ||
| 15 | |||
| 16 | test { | ||
| 17 | const expected = "Hello\nWorld!\n"; | ||
| 18 | const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 }; | ||
| 19 | |||
| 20 | const allocator = std.testing.allocator; | ||
| 21 | var decomp = std.ArrayList(u8).init(allocator); | ||
| 22 | defer decomp.deinit(); | ||
| 23 | var stream = std.io.fixedBufferStream(compressed); | ||
| 24 | try decompress(allocator, stream.reader(), decomp.writer()); | ||
| 25 | try std.testing.expectEqualSlices(u8, expected, decomp.items); | ||
| 26 | } | ||
lib/std/compress/lzma2/decode.zig created+169| ... | @@ -0,0 +1,169 @@ | ||
| 1 | const std = @import("../../std.zig"); | ||
| 2 | const Allocator = std.mem.Allocator; | ||
| 3 | |||
| 4 | const lzma = @import("../lzma.zig"); | ||
| 5 | const DecoderState = lzma.decode.DecoderState; | ||
| 6 | const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer; | ||
| 7 | const Properties = lzma.decode.Properties; | ||
| 8 | const RangeDecoder = lzma.decode.rangecoder.RangeDecoder; | ||
| 9 | |||
| 10 | pub const Decoder = struct { | ||
| 11 | lzma_state: DecoderState, | ||
| 12 | |||
| 13 | pub fn init(allocator: Allocator) !Decoder { | ||
| 14 | return Decoder{ | ||
| 15 | .lzma_state = try DecoderState.init( | ||
| 16 | allocator, | ||
| 17 | Properties{ | ||
| 18 | .lc = 0, | ||
| 19 | .lp = 0, | ||
| 20 | .pb = 0, | ||
| 21 | }, | ||
| 22 | null, | ||
| 23 | ), | ||
| 24 | }; | ||
| 25 | } | ||
| 26 | |||
| 27 | pub fn deinit(self: *Decoder, allocator: Allocator) void { | ||
| 28 | self.lzma_state.deinit(allocator); | ||
| 29 | self.* = undefined; | ||
| 30 | } | ||
| 31 | |||
| 32 | pub fn decompress( | ||
| 33 | self: *Decoder, | ||
| 34 | allocator: Allocator, | ||
| 35 | reader: anytype, | ||
| 36 | writer: anytype, | ||
| 37 | ) !void { | ||
| 38 | var accum = LzAccumBuffer.init(std.math.maxInt(usize)); | ||
| 39 | defer accum.deinit(allocator); | ||
| 40 | |||
| 41 | while (true) { | ||
| 42 | const status = try reader.readByte(); | ||
| 43 | |||
| 44 | switch (status) { | ||
| 45 | 0 => break, | ||
| 46 | 1 => try parseUncompressed(allocator, reader, writer, &accum, true), | ||
| 47 | 2 => try parseUncompressed(allocator, reader, writer, &accum, false), | ||
| 48 | else => try self.parseLzma(allocator, reader, writer, &accum, status), | ||
| 49 | } | ||
| 50 | } | ||
| 51 | |||
| 52 | try accum.finish(writer); | ||
| 53 | } | ||
| 54 | |||
| 55 | fn parseLzma( | ||
| 56 | self: *Decoder, | ||
| 57 | allocator: Allocator, | ||
| 58 | reader: anytype, | ||
| 59 | writer: anytype, | ||
| 60 | accum: *LzAccumBuffer, | ||
| 61 | status: u8, | ||
| 62 | ) !void { | ||
| 63 | if (status & 0x80 == 0) { | ||
| 64 | return error.CorruptInput; | ||
| 65 | } | ||
| 66 | |||
| 67 | const Reset = struct { | ||
| 68 | dict: bool, | ||
| 69 | state: bool, | ||
| 70 | props: bool, | ||
| 71 | }; | ||
| 72 | |||
| 73 | const reset = switch ((status >> 5) & 0x3) { | ||
| 74 | 0 => Reset{ | ||
| 75 | .dict = false, | ||
| 76 | .state = false, | ||
| 77 | .props = false, | ||
| 78 | }, | ||
| 79 | 1 => Reset{ | ||
| 80 | .dict = false, | ||
| 81 | .state = true, | ||
| 82 | .props = false, | ||
| 83 | }, | ||
| 84 | 2 => Reset{ | ||
| 85 | .dict = false, | ||
| 86 | .state = true, | ||
| 87 | .props = true, | ||
| 88 | }, | ||
| 89 | 3 => Reset{ | ||
| 90 | .dict = true, | ||
| 91 | .state = true, | ||
| 92 | .props = true, | ||
| 93 | }, | ||
| 94 | else => unreachable, | ||
| 95 | }; | ||
| 96 | |||
| 97 | const unpacked_size = blk: { | ||
| 98 | var tmp: u64 = status & 0x1F; | ||
| 99 | tmp <<= 16; | ||
| 100 | tmp |= try reader.readIntBig(u16); | ||
| 101 | break :blk tmp + 1; | ||
| 102 | }; | ||
| 103 | |||
| 104 | const packed_size = blk: { | ||
| 105 | const tmp: u17 = try reader.readIntBig(u16); | ||
| 106 | break :blk tmp + 1; | ||
| 107 | }; | ||
| 108 | |||
| 109 | if (reset.dict) { | ||
| 110 | try accum.reset(writer); | ||
| 111 | } | ||
| 112 | |||
| 113 | if (reset.state) { | ||
| 114 | var new_props = self.lzma_state.lzma_props; | ||
| 115 | |||
| 116 | if (reset.props) { | ||
| 117 | var props = try reader.readByte(); | ||
| 118 | if (props >= 225) { | ||
| 119 | return error.CorruptInput; | ||
| 120 | } | ||
| 121 | |||
| 122 | const lc = @intCast(u4, props % 9); | ||
| 123 | props /= 9; | ||
| 124 | const lp = @intCast(u3, props % 5); | ||
| 125 | props /= 5; | ||
| 126 | const pb = @intCast(u3, props); | ||
| 127 | |||
| 128 | if (lc + lp > 4) { | ||
| 129 | return error.CorruptInput; | ||
| 130 | } | ||
| 131 | |||
| 132 | new_props = Properties{ .lc = lc, .lp = lp, .pb = pb }; | ||
| 133 | } | ||
| 134 | |||
| 135 | try self.lzma_state.resetState(allocator, new_props); | ||
| 136 | } | ||
| 137 | |||
| 138 | self.lzma_state.unpacked_size = unpacked_size + accum.len; | ||
| 139 | |||
| 140 | var counter = std.io.countingReader(reader); | ||
| 141 | const counter_reader = counter.reader(); | ||
| 142 | |||
| 143 | var rangecoder = try RangeDecoder.init(counter_reader); | ||
| 144 | try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder); | ||
| 145 | |||
| 146 | if (counter.bytes_read != packed_size) { | ||
| 147 | return error.CorruptInput; | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | fn parseUncompressed( | ||
| 152 | allocator: Allocator, | ||
| 153 | reader: anytype, | ||
| 154 | writer: anytype, | ||
| 155 | accum: *LzAccumBuffer, | ||
| 156 | reset_dict: bool, | ||
| 157 | ) !void { | ||
| 158 | const unpacked_size = @as(u17, try reader.readIntBig(u16)) + 1; | ||
| 159 | |||
| 160 | if (reset_dict) { | ||
| 161 | try accum.reset(writer); | ||
| 162 | } | ||
| 163 | |||
| 164 | var i: @TypeOf(unpacked_size) = 0; | ||
| 165 | while (i < unpacked_size) : (i += 1) { | ||
| 166 | try accum.appendByte(allocator, try reader.readByte()); | ||
| 167 | } | ||
| 168 | } | ||
| 169 | }; | ||