authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-04-04 18:33:42-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:26-07:00
log6c48aad991f64f7e5bb92af498cc4cbddca9895e
tree0e43f0121bbdd66efe7be915807bbec84d3dd1f1
parent31e0b5c3c76022daababd9065a9359343cf2b34f

update some more std lib API to new Reader/Writer

std.compress needs an audit, I see some problems

15 files changed, 1012 insertions(+), 1081 deletions(-)

lib/std/compress/flate/BitWriter.zig created+93
......@@ -0,0 +1,93 @@
1//! Bit writer for use in deflate (compression).
2//!
3//! Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
4//! When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
5//! accumulate 240 bytes they are flushed to the underlying inner_writer.
6
7const std = @import("std");
8const assert = std.debug.assert;
9const BitWriter = @This();
10
11// buffer_flush_size indicates the buffer size
12// after which bytes are flushed to the writer.
13// Should preferably be a multiple of 6, since
14// we accumulate 6 bytes between writes to the buffer.
15const buffer_flush_size = 240;
16
17// buffer_size is the actual output byte buffer size.
18// It must have additional headroom for a flush
19// which can contain up to 8 bytes.
20const buffer_size = buffer_flush_size + 8;
21
22inner_writer: *std.io.BufferedWriter,
23
24// Data waiting to be written is bytes[0 .. nbytes]
25// and then the low nbits of bits. Data is always written
26// sequentially into the bytes array.
27bits: u64 = 0,
28nbits: u32 = 0, // number of bits
29bytes: [buffer_size]u8 = undefined,
30nbytes: u32 = 0, // number of bytes
31
32const Self = @This();
33
34pub fn init(bw: *std.io.BufferedWriter) Self {
35 return .{ .inner_writer = bw };
36}
37
38pub fn setWriter(self: *Self, new_writer: *std.io.BufferedWriter) void {
39 self.inner_writer = new_writer;
40}
41
42pub fn flush(self: *Self) anyerror!void {
43 var n = self.nbytes;
44 while (self.nbits != 0) {
45 self.bytes[n] = @as(u8, @truncate(self.bits));
46 self.bits >>= 8;
47 if (self.nbits > 8) { // Avoid underflow
48 self.nbits -= 8;
49 } else {
50 self.nbits = 0;
51 }
52 n += 1;
53 }
54 self.bits = 0;
55 _ = try self.inner_writer.write(self.bytes[0..n]);
56 self.nbytes = 0;
57}
58
59pub fn writeBits(self: *Self, b: u32, nb: u32) anyerror!void {
60 self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
61 self.nbits += nb;
62 if (self.nbits < 48)
63 return;
64
65 var n = self.nbytes;
66 std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
67 n += 6;
68 if (n >= buffer_flush_size) {
69 _ = try self.inner_writer.write(self.bytes[0..n]);
70 n = 0;
71 }
72 self.nbytes = n;
73 self.bits >>= 48;
74 self.nbits -= 48;
75}
76
77pub fn writeBytes(self: *Self, bytes: []const u8) anyerror!void {
78 var n = self.nbytes;
79 if (self.nbits & 7 != 0) {
80 return error.UnfinishedBits;
81 }
82 while (self.nbits != 0) {
83 self.bytes[n] = @as(u8, @truncate(self.bits));
84 self.bits >>= 8;
85 self.nbits -= 8;
86 n += 1;
87 }
88 if (n != 0) {
89 _ = try self.inner_writer.write(self.bytes[0..n]);
90 }
91 self.nbytes = 0;
92 _ = try self.inner_writer.write(bytes);
93}
lib/std/compress/flate/BlockWriter.zig created+696
......@@ -0,0 +1,696 @@
1//! Accepts list of tokens, decides what is best block type to write. What block
2//! type will provide best compression. Writes header and body of the block.
3const std = @import("std");
4const io = std.io;
5const assert = std.debug.assert;
6
7const hc = @import("huffman_encoder.zig");
8const consts = @import("consts.zig").huffman;
9const Token = @import("Token.zig");
10const BitWriter = @import("BitWriter.zig");
11const BlockWriter = @This();
12
13const codegen_order = consts.codegen_order;
14const end_code_mark = 255;
15const Self = @This();
16
17bit_writer: BitWriter,
18
19codegen_freq: [consts.codegen_code_count]u16 = undefined,
20literal_freq: [consts.max_num_lit]u16 = undefined,
21distance_freq: [consts.distance_code_count]u16 = undefined,
22codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
23literal_encoding: hc.LiteralEncoder = .{},
24distance_encoding: hc.DistanceEncoder = .{},
25codegen_encoding: hc.CodegenEncoder = .{},
26fixed_literal_encoding: hc.LiteralEncoder,
27fixed_distance_encoding: hc.DistanceEncoder,
28huff_distance: hc.DistanceEncoder,
29
30pub fn init(writer: *std.io.BufferedWriter) Self {
31 return .{
32 .bit_writer = BitWriter.init(writer),
33 .fixed_literal_encoding = hc.fixedLiteralEncoder(),
34 .fixed_distance_encoding = hc.fixedDistanceEncoder(),
35 .huff_distance = hc.huffmanDistanceEncoder(),
36 };
37}
38
39/// Flush intrenal bit buffer to the writer.
40/// Should be called only when bit stream is at byte boundary.
41///
42/// That is after final block; when last byte could be incomplete or
43/// after stored block; which is aligned to the byte boundary (it has x
44/// padding bits after first 3 bits).
45pub fn flush(self: *Self) anyerror!void {
46 try self.bit_writer.flush();
47}
48
49pub fn setWriter(self: *Self, new_writer: *std.io.BufferedWriter) void {
50 self.bit_writer.setWriter(new_writer);
51}
52
53fn writeCode(self: *Self, c: hc.HuffCode) anyerror!void {
54 try self.bit_writer.writeBits(c.code, c.len);
55}
56
57// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
58// the literal and distance lengths arrays (which are concatenated into a single
59// array). This method generates that run-length encoding.
60//
61// The result is written into the codegen array, and the frequencies
62// of each code is written into the codegen_freq array.
63// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
64// information. Code bad_code is an end marker
65//
66// num_literals: The number of literals in literal_encoding
67// num_distances: The number of distances in distance_encoding
68// lit_enc: The literal encoder to use
69// dist_enc: The distance encoder to use
70fn generateCodegen(
71 self: *Self,
72 num_literals: u32,
73 num_distances: u32,
74 lit_enc: *hc.LiteralEncoder,
75 dist_enc: *hc.DistanceEncoder,
76) void {
77 for (self.codegen_freq, 0..) |_, i| {
78 self.codegen_freq[i] = 0;
79 }
80
81 // Note that we are using codegen both as a temporary variable for holding
82 // a copy of the frequencies, and as the place where we put the result.
83 // This is fine because the output is always shorter than the input used
84 // so far.
85 var codegen = &self.codegen; // cache
86 // Copy the concatenated code sizes to codegen. Put a marker at the end.
87 var cgnl = codegen[0..num_literals];
88 for (cgnl, 0..) |_, i| {
89 cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
90 }
91
92 cgnl = codegen[num_literals .. num_literals + num_distances];
93 for (cgnl, 0..) |_, i| {
94 cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
95 }
96 codegen[num_literals + num_distances] = end_code_mark;
97
98 var size = codegen[0];
99 var count: i32 = 1;
100 var out_index: u32 = 0;
101 var in_index: u32 = 1;
102 while (size != end_code_mark) : (in_index += 1) {
103 // INVARIANT: We have seen "count" copies of size that have not yet
104 // had output generated for them.
105 const next_size = codegen[in_index];
106 if (next_size == size) {
107 count += 1;
108 continue;
109 }
110 // We need to generate codegen indicating "count" of size.
111 if (size != 0) {
112 codegen[out_index] = size;
113 out_index += 1;
114 self.codegen_freq[size] += 1;
115 count -= 1;
116 while (count >= 3) {
117 var n: i32 = 6;
118 if (n > count) {
119 n = count;
120 }
121 codegen[out_index] = 16;
122 out_index += 1;
123 codegen[out_index] = @as(u8, @intCast(n - 3));
124 out_index += 1;
125 self.codegen_freq[16] += 1;
126 count -= n;
127 }
128 } else {
129 while (count >= 11) {
130 var n: i32 = 138;
131 if (n > count) {
132 n = count;
133 }
134 codegen[out_index] = 18;
135 out_index += 1;
136 codegen[out_index] = @as(u8, @intCast(n - 11));
137 out_index += 1;
138 self.codegen_freq[18] += 1;
139 count -= n;
140 }
141 if (count >= 3) {
142 // 3 <= count <= 10
143 codegen[out_index] = 17;
144 out_index += 1;
145 codegen[out_index] = @as(u8, @intCast(count - 3));
146 out_index += 1;
147 self.codegen_freq[17] += 1;
148 count = 0;
149 }
150 }
151 count -= 1;
152 while (count >= 0) : (count -= 1) {
153 codegen[out_index] = size;
154 out_index += 1;
155 self.codegen_freq[size] += 1;
156 }
157 // Set up invariant for next time through the loop.
158 size = next_size;
159 count = 1;
160 }
161 // Marker indicating the end of the codegen.
162 codegen[out_index] = end_code_mark;
163}
164
165const DynamicSize = struct {
166 size: u32,
167 num_codegens: u32,
168};
169
170// dynamicSize returns the size of dynamically encoded data in bits.
171fn dynamicSize(
172 self: *Self,
173 lit_enc: *hc.LiteralEncoder, // literal encoder
174 dist_enc: *hc.DistanceEncoder, // distance encoder
175 extra_bits: u32,
176) DynamicSize {
177 var num_codegens = self.codegen_freq.len;
178 while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
179 num_codegens -= 1;
180 }
181 const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
182 self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
183 self.codegen_freq[16] * 2 +
184 self.codegen_freq[17] * 3 +
185 self.codegen_freq[18] * 7;
186 const size = header +
187 lit_enc.bitLength(&self.literal_freq) +
188 dist_enc.bitLength(&self.distance_freq) +
189 extra_bits;
190
191 return DynamicSize{
192 .size = @as(u32, @intCast(size)),
193 .num_codegens = @as(u32, @intCast(num_codegens)),
194 };
195}
196
197// fixedSize returns the size of dynamically encoded data in bits.
198fn fixedSize(self: *Self, extra_bits: u32) u32 {
199 return 3 +
200 self.fixed_literal_encoding.bitLength(&self.literal_freq) +
201 self.fixed_distance_encoding.bitLength(&self.distance_freq) +
202 extra_bits;
203}
204
205const StoredSize = struct {
206 size: u32,
207 storable: bool,
208};
209
210// storedSizeFits calculates the stored size, including header.
211// The function returns the size in bits and whether the block
212// fits inside a single block.
213fn storedSizeFits(in: ?[]const u8) StoredSize {
214 if (in == null) {
215 return .{ .size = 0, .storable = false };
216 }
217 if (in.?.len <= consts.max_store_block_size) {
218 return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
219 }
220 return .{ .size = 0, .storable = false };
221}
222
223// Write the header of a dynamic Huffman block to the output stream.
224//
225// num_literals: The number of literals specified in codegen
226// num_distances: The number of distances specified in codegen
227// num_codegens: The number of codegens used in codegen
228// eof: Is it the end-of-file? (end of stream)
229fn dynamicHeader(
230 self: *Self,
231 num_literals: u32,
232 num_distances: u32,
233 num_codegens: u32,
234 eof: bool,
235) anyerror!void {
236 const first_bits: u32 = if (eof) 5 else 4;
237 try self.bit_writer.writeBits(first_bits, 3);
238 try self.bit_writer.writeBits(num_literals - 257, 5);
239 try self.bit_writer.writeBits(num_distances - 1, 5);
240 try self.bit_writer.writeBits(num_codegens - 4, 4);
241
242 var i: u32 = 0;
243 while (i < num_codegens) : (i += 1) {
244 const value = self.codegen_encoding.codes[codegen_order[i]].len;
245 try self.bit_writer.writeBits(value, 3);
246 }
247
248 i = 0;
249 while (true) {
250 const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
251 i += 1;
252 if (code_word == end_code_mark) {
253 break;
254 }
255 try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
256
257 switch (code_word) {
258 16 => {
259 try self.bit_writer.writeBits(self.codegen[i], 2);
260 i += 1;
261 },
262 17 => {
263 try self.bit_writer.writeBits(self.codegen[i], 3);
264 i += 1;
265 },
266 18 => {
267 try self.bit_writer.writeBits(self.codegen[i], 7);
268 i += 1;
269 },
270 else => {},
271 }
272 }
273}
274
275fn storedHeader(self: *Self, length: usize, eof: bool) anyerror!void {
276 assert(length <= 65535);
277 const flag: u32 = if (eof) 1 else 0;
278 try self.bit_writer.writeBits(flag, 3);
279 try self.flush();
280 const l: u16 = @intCast(length);
281 try self.bit_writer.writeBits(l, 16);
282 try self.bit_writer.writeBits(~l, 16);
283}
284
285fn fixedHeader(self: *Self, eof: bool) anyerror!void {
286 // Indicate that we are a fixed Huffman block
287 var value: u32 = 2;
288 if (eof) {
289 value = 3;
290 }
291 try self.bit_writer.writeBits(value, 3);
292}
293
294// Write a block of tokens with the smallest encoding. Will choose block type.
295// The original input can be supplied, and if the huffman encoded data
296// is larger than the original bytes, the data will be written as a
297// stored block.
298// If the input is null, the tokens will always be Huffman encoded.
299pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) anyerror!void {
300 const lit_and_dist = self.indexTokens(tokens);
301 const num_literals = lit_and_dist.num_literals;
302 const num_distances = lit_and_dist.num_distances;
303
304 var extra_bits: u32 = 0;
305 const ret = storedSizeFits(input);
306 const stored_size = ret.size;
307 const storable = ret.storable;
308
309 if (storable) {
310 // We only bother calculating the costs of the extra bits required by
311 // the length of distance fields (which will be the same for both fixed
312 // and dynamic encoding), if we need to compare those two encodings
313 // against stored encoding.
314 var length_code: u16 = Token.length_codes_start + 8;
315 while (length_code < num_literals) : (length_code += 1) {
316 // First eight length codes have extra size = 0.
317 extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
318 @as(u32, @intCast(Token.lengthExtraBits(length_code)));
319 }
320 var distance_code: u16 = 4;
321 while (distance_code < num_distances) : (distance_code += 1) {
322 // First four distance codes have extra size = 0.
323 extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
324 @as(u32, @intCast(Token.distanceExtraBits(distance_code)));
325 }
326 }
327
328 // Figure out smallest code.
329 // Fixed Huffman baseline.
330 var literal_encoding = &self.fixed_literal_encoding;
331 var distance_encoding = &self.fixed_distance_encoding;
332 var size = self.fixedSize(extra_bits);
333
334 // Dynamic Huffman?
335 var num_codegens: u32 = 0;
336
337 // Generate codegen and codegenFrequencies, which indicates how to encode
338 // the literal_encoding and the distance_encoding.
339 self.generateCodegen(
340 num_literals,
341 num_distances,
342 &self.literal_encoding,
343 &self.distance_encoding,
344 );
345 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
346 const dynamic_size = self.dynamicSize(
347 &self.literal_encoding,
348 &self.distance_encoding,
349 extra_bits,
350 );
351 const dyn_size = dynamic_size.size;
352 num_codegens = dynamic_size.num_codegens;
353
354 if (dyn_size < size) {
355 size = dyn_size;
356 literal_encoding = &self.literal_encoding;
357 distance_encoding = &self.distance_encoding;
358 }
359
360 // Stored bytes?
361 if (storable and stored_size < size) {
362 try self.storedBlock(input.?, eof);
363 return;
364 }
365
366 // Huffman.
367 if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
368 try self.fixedHeader(eof);
369 } else {
370 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
371 }
372
373 // Write the tokens.
374 try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
375}
376
377pub fn storedBlock(self: *Self, input: []const u8, eof: bool) anyerror!void {
378 try self.storedHeader(input.len, eof);
379 try self.bit_writer.writeBytes(input);
380}
381
382// writeBlockDynamic encodes a block using a dynamic Huffman table.
383// This should be used if the symbols used have a disproportionate
384// histogram distribution.
385// If input is supplied and the compression savings are below 1/16th of the
386// input size the block is stored.
387fn dynamicBlock(
388 self: *Self,
389 tokens: []const Token,
390 eof: bool,
391 input: ?[]const u8,
392) anyerror!void {
393 const total_tokens = self.indexTokens(tokens);
394 const num_literals = total_tokens.num_literals;
395 const num_distances = total_tokens.num_distances;
396
397 // Generate codegen and codegenFrequencies, which indicates how to encode
398 // the literal_encoding and the distance_encoding.
399 self.generateCodegen(
400 num_literals,
401 num_distances,
402 &self.literal_encoding,
403 &self.distance_encoding,
404 );
405 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
406 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
407 const size = dynamic_size.size;
408 const num_codegens = dynamic_size.num_codegens;
409
410 // Store bytes, if we don't get a reasonable improvement.
411
412 const stored_size = storedSizeFits(input);
413 const ssize = stored_size.size;
414 const storable = stored_size.storable;
415 if (storable and ssize < (size + (size >> 4))) {
416 try self.storedBlock(input.?, eof);
417 return;
418 }
419
420 // Write Huffman table.
421 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
422
423 // Write the tokens.
424 try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
425}
426
427const TotalIndexedTokens = struct {
428 num_literals: u32,
429 num_distances: u32,
430};
431
432// Indexes a slice of tokens followed by an end_block_marker, and updates
433// literal_freq and distance_freq, and generates literal_encoding
434// and distance_encoding.
435// The number of literal and distance tokens is returned.
436fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
437 var num_literals: u32 = 0;
438 var num_distances: u32 = 0;
439
440 for (self.literal_freq, 0..) |_, i| {
441 self.literal_freq[i] = 0;
442 }
443 for (self.distance_freq, 0..) |_, i| {
444 self.distance_freq[i] = 0;
445 }
446
447 for (tokens) |t| {
448 if (t.kind == Token.Kind.literal) {
449 self.literal_freq[t.literal()] += 1;
450 continue;
451 }
452 self.literal_freq[t.lengthCode()] += 1;
453 self.distance_freq[t.distanceCode()] += 1;
454 }
455 // add end_block_marker token at the end
456 self.literal_freq[consts.end_block_marker] += 1;
457
458 // get the number of literals
459 num_literals = @as(u32, @intCast(self.literal_freq.len));
460 while (self.literal_freq[num_literals - 1] == 0) {
461 num_literals -= 1;
462 }
463 // get the number of distances
464 num_distances = @as(u32, @intCast(self.distance_freq.len));
465 while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
466 num_distances -= 1;
467 }
468 if (num_distances == 0) {
469 // We haven't found a single match. If we want to go with the dynamic encoding,
470 // we should count at least one distance to be sure that the distance huffman tree could be encoded.
471 self.distance_freq[0] = 1;
472 num_distances = 1;
473 }
474 self.literal_encoding.generate(&self.literal_freq, 15);
475 self.distance_encoding.generate(&self.distance_freq, 15);
476 return TotalIndexedTokens{
477 .num_literals = num_literals,
478 .num_distances = num_distances,
479 };
480}
481
482// Writes a slice of tokens to the output followed by and end_block_marker.
483// codes for literal and distance encoding must be supplied.
484fn writeTokens(
485 self: *Self,
486 tokens: []const Token,
487 le_codes: []hc.HuffCode,
488 oe_codes: []hc.HuffCode,
489) anyerror!void {
490 for (tokens) |t| {
491 if (t.kind == Token.Kind.literal) {
492 try self.writeCode(le_codes[t.literal()]);
493 continue;
494 }
495
496 // Write the length
497 const le = t.lengthEncoding();
498 try self.writeCode(le_codes[le.code]);
499 if (le.extra_bits > 0) {
500 try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
501 }
502
503 // Write the distance
504 const oe = t.distanceEncoding();
505 try self.writeCode(oe_codes[oe.code]);
506 if (oe.extra_bits > 0) {
507 try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
508 }
509 }
510 // add end_block_marker at the end
511 try self.writeCode(le_codes[consts.end_block_marker]);
512}
513
514// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
515// if the results only gains very little from compression.
516pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) anyerror!void {
517 // Add everything as literals
518 histogram(input, &self.literal_freq);
519
520 self.literal_freq[consts.end_block_marker] = 1;
521
522 const num_literals = consts.end_block_marker + 1;
523 self.distance_freq[0] = 1;
524 const num_distances = 1;
525
526 self.literal_encoding.generate(&self.literal_freq, 15);
527
528 // Figure out smallest code.
529 // Always use dynamic Huffman or Store
530 var num_codegens: u32 = 0;
531
532 // Generate codegen and codegenFrequencies, which indicates how to encode
533 // the literal_encoding and the distance_encoding.
534 self.generateCodegen(
535 num_literals,
536 num_distances,
537 &self.literal_encoding,
538 &self.huff_distance,
539 );
540 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
541 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
542 const size = dynamic_size.size;
543 num_codegens = dynamic_size.num_codegens;
544
545 // Store bytes, if we don't get a reasonable improvement.
546 const stored_size_ret = storedSizeFits(input);
547 const ssize = stored_size_ret.size;
548 const storable = stored_size_ret.storable;
549
550 if (storable and ssize < (size + (size >> 4))) {
551 try self.storedBlock(input, eof);
552 return;
553 }
554
555 // Huffman.
556 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
557 const encoding = self.literal_encoding.codes[0..257];
558
559 for (input) |t| {
560 const c = encoding[t];
561 try self.bit_writer.writeBits(c.code, c.len);
562 }
563 try self.writeCode(encoding[consts.end_block_marker]);
564}
565
566// histogram accumulates a histogram of b in h.
567fn histogram(b: []const u8, h: *[286]u16) void {
568 // Clear histogram
569 for (h, 0..) |_, i| {
570 h[i] = 0;
571 }
572
573 var lh = h.*[0..256];
574 for (b) |t| {
575 lh[t] += 1;
576 }
577}
578
579// tests
580const expect = std.testing.expect;
581const fmt = std.fmt;
582const testing = std.testing;
583const ArrayList = std.ArrayList;
584
585const TestCase = @import("testdata/block_writer.zig").TestCase;
586const testCases = @import("testdata/block_writer.zig").testCases;
587
588// tests if the writeBlock encoding has changed.
589test "write" {
590 inline for (0..testCases.len) |i| {
591 try testBlock(testCases[i], .write_block);
592 }
593}
594
595// tests if the writeBlockDynamic encoding has changed.
596test "dynamicBlock" {
597 inline for (0..testCases.len) |i| {
598 try testBlock(testCases[i], .write_dyn_block);
599 }
600}
601
602test "huffmanBlock" {
603 inline for (0..testCases.len) |i| {
604 try testBlock(testCases[i], .write_huffman_block);
605 }
606 try testBlock(.{
607 .tokens = &[_]Token{},
608 .input = "huffman-rand-max.input",
609 .want = "huffman-rand-max.{s}.expect",
610 }, .write_huffman_block);
611}
612
613const TestFn = enum {
614 write_block,
615 write_dyn_block, // write dynamic block
616 write_huffman_block,
617
618 fn to_s(self: TestFn) []const u8 {
619 return switch (self) {
620 .write_block => "wb",
621 .write_dyn_block => "dyn",
622 .write_huffman_block => "huff",
623 };
624 }
625
626 fn write(
627 comptime self: TestFn,
628 bw: anytype,
629 tok: []const Token,
630 input: ?[]const u8,
631 final: bool,
632 ) !void {
633 switch (self) {
634 .write_block => try bw.write(tok, final, input),
635 .write_dyn_block => try bw.dynamicBlock(tok, final, input),
636 .write_huffman_block => try bw.huffmanBlock(input.?, final),
637 }
638 try bw.flush();
639 }
640};
641
642// testBlock tests a block against its references
643//
644// size
645// 64K [file-name].input - input non compressed file
646// 8.1K [file-name].golden -
647// 78 [file-name].dyn.expect - output with writeBlockDynamic
648// 78 [file-name].wb.expect - output with writeBlock
649// 8.1K [file-name].huff.expect - output with writeBlockHuff
650// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
651// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
652//
653// wb - writeBlock
654// dyn - writeBlockDynamic
655// huff - writeBlockHuff
656//
657fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
658 if (tc.input.len != 0 and tc.want.len != 0) {
659 const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
660 const input = @embedFile("testdata/block_writer/" ++ tc.input);
661 const want = @embedFile("testdata/block_writer/" ++ want_name);
662 try testWriteBlock(tfn, input, want, tc.tokens);
663 }
664
665 if (tfn == .write_huffman_block) {
666 return;
667 }
668
669 const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
670 const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
671 try testWriteBlock(tfn, null, want, tc.tokens);
672}
673
674// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
675fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
676 var buf = ArrayList(u8).init(testing.allocator);
677 var bw: BlockWriter = .init(buf.writer());
678 try tfn.write(&bw, tokens, input, false);
679 var got = buf.items;
680 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
681 try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
682 //
683 // Test if the writer produces the same output after reset.
684 buf.deinit();
685 buf = ArrayList(u8).init(testing.allocator);
686 defer buf.deinit();
687 bw.setWriter(buf.writer());
688
689 try tfn.write(&bw, tokens, input, true);
690 try bw.flush();
691 got = buf.items;
692
693 try expect(got[0] & 1 == 1); // bfinal is set
694 buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
695 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
696}
lib/std/compress/flate/bit_writer.zig deleted-99
......@@ -1,99 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
4/// Bit writer for use in deflate (compression).
5///
6/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9///
10pub fn BitWriter(comptime WriterType: type) type {
11 // buffer_flush_size indicates the buffer size
12 // after which bytes are flushed to the writer.
13 // Should preferably be a multiple of 6, since
14 // we accumulate 6 bytes between writes to the buffer.
15 const buffer_flush_size = 240;
16
17 // buffer_size is the actual output byte buffer size.
18 // It must have additional headroom for a flush
19 // which can contain up to 8 bytes.
20 const buffer_size = buffer_flush_size + 8;
21
22 return struct {
23 inner_writer: WriterType,
24
25 // Data waiting to be written is bytes[0 .. nbytes]
26 // and then the low nbits of bits. Data is always written
27 // sequentially into the bytes array.
28 bits: u64 = 0,
29 nbits: u32 = 0, // number of bits
30 bytes: [buffer_size]u8 = undefined,
31 nbytes: u32 = 0, // number of bytes
32
33 const Self = @This();
34
35 pub const Error = WriterType.Error || error{UnfinishedBits};
36
37 pub fn init(writer: WriterType) Self {
38 return .{ .inner_writer = writer };
39 }
40
41 pub fn setWriter(self: *Self, new_writer: WriterType) void {
42 //assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43 self.inner_writer = new_writer;
44 }
45
46 pub fn flush(self: *Self) Error!void {
47 var n = self.nbytes;
48 while (self.nbits != 0) {
49 self.bytes[n] = @as(u8, @truncate(self.bits));
50 self.bits >>= 8;
51 if (self.nbits > 8) { // Avoid underflow
52 self.nbits -= 8;
53 } else {
54 self.nbits = 0;
55 }
56 n += 1;
57 }
58 self.bits = 0;
59 _ = try self.inner_writer.write(self.bytes[0..n]);
60 self.nbytes = 0;
61 }
62
63 pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64 self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65 self.nbits += nb;
66 if (self.nbits < 48)
67 return;
68
69 var n = self.nbytes;
70 std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71 n += 6;
72 if (n >= buffer_flush_size) {
73 _ = try self.inner_writer.write(self.bytes[0..n]);
74 n = 0;
75 }
76 self.nbytes = n;
77 self.bits >>= 48;
78 self.nbits -= 48;
79 }
80
81 pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82 var n = self.nbytes;
83 if (self.nbits & 7 != 0) {
84 return error.UnfinishedBits;
85 }
86 while (self.nbits != 0) {
87 self.bytes[n] = @as(u8, @truncate(self.bits));
88 self.bits >>= 8;
89 self.nbits -= 8;
90 n += 1;
91 }
92 if (n != 0) {
93 _ = try self.inner_writer.write(self.bytes[0..n]);
94 }
95 self.nbytes = 0;
96 _ = try self.inner_writer.write(bytes);
97 }
98 };
99}
lib/std/compress/flate/block_writer.zig deleted-706
......@@ -1,706 +0,0 @@
1const std = @import("std");
2const io = std.io;
3const assert = std.debug.assert;
4
5const hc = @import("huffman_encoder.zig");
6const consts = @import("consts.zig").huffman;
7const Token = @import("Token.zig");
8const BitWriter = @import("bit_writer.zig").BitWriter;
9
10pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11 return BlockWriter(@TypeOf(writer)).init(writer);
12}
13
14/// Accepts list of tokens, decides what is best block type to write. What block
15/// type will provide best compression. Writes header and body of the block.
16///
17pub fn BlockWriter(comptime WriterType: type) type {
18 const BitWriterType = BitWriter(WriterType);
19 return struct {
20 const codegen_order = consts.codegen_order;
21 const end_code_mark = 255;
22 const Self = @This();
23
24 pub const Error = BitWriterType.Error;
25 bit_writer: BitWriterType,
26
27 codegen_freq: [consts.codegen_code_count]u16 = undefined,
28 literal_freq: [consts.max_num_lit]u16 = undefined,
29 distance_freq: [consts.distance_code_count]u16 = undefined,
30 codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31 literal_encoding: hc.LiteralEncoder = .{},
32 distance_encoding: hc.DistanceEncoder = .{},
33 codegen_encoding: hc.CodegenEncoder = .{},
34 fixed_literal_encoding: hc.LiteralEncoder,
35 fixed_distance_encoding: hc.DistanceEncoder,
36 huff_distance: hc.DistanceEncoder,
37
38 pub fn init(writer: WriterType) Self {
39 return .{
40 .bit_writer = BitWriterType.init(writer),
41 .fixed_literal_encoding = hc.fixedLiteralEncoder(),
42 .fixed_distance_encoding = hc.fixedDistanceEncoder(),
43 .huff_distance = hc.huffmanDistanceEncoder(),
44 };
45 }
46
47 /// Flush intrenal bit buffer to the writer.
48 /// Should be called only when bit stream is at byte boundary.
49 ///
50 /// That is after final block; when last byte could be incomplete or
51 /// after stored block; which is aligned to the byte boundary (it has x
52 /// padding bits after first 3 bits).
53 pub fn flush(self: *Self) Error!void {
54 try self.bit_writer.flush();
55 }
56
57 pub fn setWriter(self: *Self, new_writer: WriterType) void {
58 self.bit_writer.setWriter(new_writer);
59 }
60
61 fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62 try self.bit_writer.writeBits(c.code, c.len);
63 }
64
65 // RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66 // the literal and distance lengths arrays (which are concatenated into a single
67 // array). This method generates that run-length encoding.
68 //
69 // The result is written into the codegen array, and the frequencies
70 // of each code is written into the codegen_freq array.
71 // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72 // information. Code bad_code is an end marker
73 //
74 // num_literals: The number of literals in literal_encoding
75 // num_distances: The number of distances in distance_encoding
76 // lit_enc: The literal encoder to use
77 // dist_enc: The distance encoder to use
78 fn generateCodegen(
79 self: *Self,
80 num_literals: u32,
81 num_distances: u32,
82 lit_enc: *hc.LiteralEncoder,
83 dist_enc: *hc.DistanceEncoder,
84 ) void {
85 for (self.codegen_freq, 0..) |_, i| {
86 self.codegen_freq[i] = 0;
87 }
88
89 // Note that we are using codegen both as a temporary variable for holding
90 // a copy of the frequencies, and as the place where we put the result.
91 // This is fine because the output is always shorter than the input used
92 // so far.
93 var codegen = &self.codegen; // cache
94 // Copy the concatenated code sizes to codegen. Put a marker at the end.
95 var cgnl = codegen[0..num_literals];
96 for (cgnl, 0..) |_, i| {
97 cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98 }
99
100 cgnl = codegen[num_literals .. num_literals + num_distances];
101 for (cgnl, 0..) |_, i| {
102 cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103 }
104 codegen[num_literals + num_distances] = end_code_mark;
105
106 var size = codegen[0];
107 var count: i32 = 1;
108 var out_index: u32 = 0;
109 var in_index: u32 = 1;
110 while (size != end_code_mark) : (in_index += 1) {
111 // INVARIANT: We have seen "count" copies of size that have not yet
112 // had output generated for them.
113 const next_size = codegen[in_index];
114 if (next_size == size) {
115 count += 1;
116 continue;
117 }
118 // We need to generate codegen indicating "count" of size.
119 if (size != 0) {
120 codegen[out_index] = size;
121 out_index += 1;
122 self.codegen_freq[size] += 1;
123 count -= 1;
124 while (count >= 3) {
125 var n: i32 = 6;
126 if (n > count) {
127 n = count;
128 }
129 codegen[out_index] = 16;
130 out_index += 1;
131 codegen[out_index] = @as(u8, @intCast(n - 3));
132 out_index += 1;
133 self.codegen_freq[16] += 1;
134 count -= n;
135 }
136 } else {
137 while (count >= 11) {
138 var n: i32 = 138;
139 if (n > count) {
140 n = count;
141 }
142 codegen[out_index] = 18;
143 out_index += 1;
144 codegen[out_index] = @as(u8, @intCast(n - 11));
145 out_index += 1;
146 self.codegen_freq[18] += 1;
147 count -= n;
148 }
149 if (count >= 3) {
150 // 3 <= count <= 10
151 codegen[out_index] = 17;
152 out_index += 1;
153 codegen[out_index] = @as(u8, @intCast(count - 3));
154 out_index += 1;
155 self.codegen_freq[17] += 1;
156 count = 0;
157 }
158 }
159 count -= 1;
160 while (count >= 0) : (count -= 1) {
161 codegen[out_index] = size;
162 out_index += 1;
163 self.codegen_freq[size] += 1;
164 }
165 // Set up invariant for next time through the loop.
166 size = next_size;
167 count = 1;
168 }
169 // Marker indicating the end of the codegen.
170 codegen[out_index] = end_code_mark;
171 }
172
173 const DynamicSize = struct {
174 size: u32,
175 num_codegens: u32,
176 };
177
178 // dynamicSize returns the size of dynamically encoded data in bits.
179 fn dynamicSize(
180 self: *Self,
181 lit_enc: *hc.LiteralEncoder, // literal encoder
182 dist_enc: *hc.DistanceEncoder, // distance encoder
183 extra_bits: u32,
184 ) DynamicSize {
185 var num_codegens = self.codegen_freq.len;
186 while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187 num_codegens -= 1;
188 }
189 const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190 self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191 self.codegen_freq[16] * 2 +
192 self.codegen_freq[17] * 3 +
193 self.codegen_freq[18] * 7;
194 const size = header +
195 lit_enc.bitLength(&self.literal_freq) +
196 dist_enc.bitLength(&self.distance_freq) +
197 extra_bits;
198
199 return DynamicSize{
200 .size = @as(u32, @intCast(size)),
201 .num_codegens = @as(u32, @intCast(num_codegens)),
202 };
203 }
204
205 // fixedSize returns the size of dynamically encoded data in bits.
206 fn fixedSize(self: *Self, extra_bits: u32) u32 {
207 return 3 +
208 self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209 self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210 extra_bits;
211 }
212
213 const StoredSize = struct {
214 size: u32,
215 storable: bool,
216 };
217
218 // storedSizeFits calculates the stored size, including header.
219 // The function returns the size in bits and whether the block
220 // fits inside a single block.
221 fn storedSizeFits(in: ?[]const u8) StoredSize {
222 if (in == null) {
223 return .{ .size = 0, .storable = false };
224 }
225 if (in.?.len <= consts.max_store_block_size) {
226 return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227 }
228 return .{ .size = 0, .storable = false };
229 }
230
231 // Write the header of a dynamic Huffman block to the output stream.
232 //
233 // num_literals: The number of literals specified in codegen
234 // num_distances: The number of distances specified in codegen
235 // num_codegens: The number of codegens used in codegen
236 // eof: Is it the end-of-file? (end of stream)
237 fn dynamicHeader(
238 self: *Self,
239 num_literals: u32,
240 num_distances: u32,
241 num_codegens: u32,
242 eof: bool,
243 ) Error!void {
244 const first_bits: u32 = if (eof) 5 else 4;
245 try self.bit_writer.writeBits(first_bits, 3);
246 try self.bit_writer.writeBits(num_literals - 257, 5);
247 try self.bit_writer.writeBits(num_distances - 1, 5);
248 try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250 var i: u32 = 0;
251 while (i < num_codegens) : (i += 1) {
252 const value = self.codegen_encoding.codes[codegen_order[i]].len;
253 try self.bit_writer.writeBits(value, 3);
254 }
255
256 i = 0;
257 while (true) {
258 const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259 i += 1;
260 if (code_word == end_code_mark) {
261 break;
262 }
263 try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265 switch (code_word) {
266 16 => {
267 try self.bit_writer.writeBits(self.codegen[i], 2);
268 i += 1;
269 },
270 17 => {
271 try self.bit_writer.writeBits(self.codegen[i], 3);
272 i += 1;
273 },
274 18 => {
275 try self.bit_writer.writeBits(self.codegen[i], 7);
276 i += 1;
277 },
278 else => {},
279 }
280 }
281 }
282
283 fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284 assert(length <= 65535);
285 const flag: u32 = if (eof) 1 else 0;
286 try self.bit_writer.writeBits(flag, 3);
287 try self.flush();
288 const l: u16 = @intCast(length);
289 try self.bit_writer.writeBits(l, 16);
290 try self.bit_writer.writeBits(~l, 16);
291 }
292
293 fn fixedHeader(self: *Self, eof: bool) Error!void {
294 // Indicate that we are a fixed Huffman block
295 var value: u32 = 2;
296 if (eof) {
297 value = 3;
298 }
299 try self.bit_writer.writeBits(value, 3);
300 }
301
302 // Write a block of tokens with the smallest encoding. Will choose block type.
303 // The original input can be supplied, and if the huffman encoded data
304 // is larger than the original bytes, the data will be written as a
305 // stored block.
306 // If the input is null, the tokens will always be Huffman encoded.
307 pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308 const lit_and_dist = self.indexTokens(tokens);
309 const num_literals = lit_and_dist.num_literals;
310 const num_distances = lit_and_dist.num_distances;
311
312 var extra_bits: u32 = 0;
313 const ret = storedSizeFits(input);
314 const stored_size = ret.size;
315 const storable = ret.storable;
316
317 if (storable) {
318 // We only bother calculating the costs of the extra bits required by
319 // the length of distance fields (which will be the same for both fixed
320 // and dynamic encoding), if we need to compare those two encodings
321 // against stored encoding.
322 var length_code: u16 = Token.length_codes_start + 8;
323 while (length_code < num_literals) : (length_code += 1) {
324 // First eight length codes have extra size = 0.
325 extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326 @as(u32, @intCast(Token.lengthExtraBits(length_code)));
327 }
328 var distance_code: u16 = 4;
329 while (distance_code < num_distances) : (distance_code += 1) {
330 // First four distance codes have extra size = 0.
331 extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332 @as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333 }
334 }
335
336 // Figure out smallest code.
337 // Fixed Huffman baseline.
338 var literal_encoding = &self.fixed_literal_encoding;
339 var distance_encoding = &self.fixed_distance_encoding;
340 var size = self.fixedSize(extra_bits);
341
342 // Dynamic Huffman?
343 var num_codegens: u32 = 0;
344
345 // Generate codegen and codegenFrequencies, which indicates how to encode
346 // the literal_encoding and the distance_encoding.
347 self.generateCodegen(
348 num_literals,
349 num_distances,
350 &self.literal_encoding,
351 &self.distance_encoding,
352 );
353 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354 const dynamic_size = self.dynamicSize(
355 &self.literal_encoding,
356 &self.distance_encoding,
357 extra_bits,
358 );
359 const dyn_size = dynamic_size.size;
360 num_codegens = dynamic_size.num_codegens;
361
362 if (dyn_size < size) {
363 size = dyn_size;
364 literal_encoding = &self.literal_encoding;
365 distance_encoding = &self.distance_encoding;
366 }
367
368 // Stored bytes?
369 if (storable and stored_size < size) {
370 try self.storedBlock(input.?, eof);
371 return;
372 }
373
374 // Huffman.
375 if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376 try self.fixedHeader(eof);
377 } else {
378 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379 }
380
381 // Write the tokens.
382 try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383 }
384
385 pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386 try self.storedHeader(input.len, eof);
387 try self.bit_writer.writeBytes(input);
388 }
389
390 // writeBlockDynamic encodes a block using a dynamic Huffman table.
391 // This should be used if the symbols used have a disproportionate
392 // histogram distribution.
393 // If input is supplied and the compression savings are below 1/16th of the
394 // input size the block is stored.
395 fn dynamicBlock(
396 self: *Self,
397 tokens: []const Token,
398 eof: bool,
399 input: ?[]const u8,
400 ) Error!void {
401 const total_tokens = self.indexTokens(tokens);
402 const num_literals = total_tokens.num_literals;
403 const num_distances = total_tokens.num_distances;
404
405 // Generate codegen and codegenFrequencies, which indicates how to encode
406 // the literal_encoding and the distance_encoding.
407 self.generateCodegen(
408 num_literals,
409 num_distances,
410 &self.literal_encoding,
411 &self.distance_encoding,
412 );
413 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415 const size = dynamic_size.size;
416 const num_codegens = dynamic_size.num_codegens;
417
418 // Store bytes, if we don't get a reasonable improvement.
419
420 const stored_size = storedSizeFits(input);
421 const ssize = stored_size.size;
422 const storable = stored_size.storable;
423 if (storable and ssize < (size + (size >> 4))) {
424 try self.storedBlock(input.?, eof);
425 return;
426 }
427
428 // Write Huffman table.
429 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431 // Write the tokens.
432 try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433 }
434
435 const TotalIndexedTokens = struct {
436 num_literals: u32,
437 num_distances: u32,
438 };
439
440 // Indexes a slice of tokens followed by an end_block_marker, and updates
441 // literal_freq and distance_freq, and generates literal_encoding
442 // and distance_encoding.
443 // The number of literal and distance tokens is returned.
444 fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445 var num_literals: u32 = 0;
446 var num_distances: u32 = 0;
447
448 for (self.literal_freq, 0..) |_, i| {
449 self.literal_freq[i] = 0;
450 }
451 for (self.distance_freq, 0..) |_, i| {
452 self.distance_freq[i] = 0;
453 }
454
455 for (tokens) |t| {
456 if (t.kind == Token.Kind.literal) {
457 self.literal_freq[t.literal()] += 1;
458 continue;
459 }
460 self.literal_freq[t.lengthCode()] += 1;
461 self.distance_freq[t.distanceCode()] += 1;
462 }
463 // add end_block_marker token at the end
464 self.literal_freq[consts.end_block_marker] += 1;
465
466 // get the number of literals
467 num_literals = @as(u32, @intCast(self.literal_freq.len));
468 while (self.literal_freq[num_literals - 1] == 0) {
469 num_literals -= 1;
470 }
471 // get the number of distances
472 num_distances = @as(u32, @intCast(self.distance_freq.len));
473 while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474 num_distances -= 1;
475 }
476 if (num_distances == 0) {
477 // We haven't found a single match. If we want to go with the dynamic encoding,
478 // we should count at least one distance to be sure that the distance huffman tree could be encoded.
479 self.distance_freq[0] = 1;
480 num_distances = 1;
481 }
482 self.literal_encoding.generate(&self.literal_freq, 15);
483 self.distance_encoding.generate(&self.distance_freq, 15);
484 return TotalIndexedTokens{
485 .num_literals = num_literals,
486 .num_distances = num_distances,
487 };
488 }
489
490 // Writes a slice of tokens to the output followed by and end_block_marker.
491 // codes for literal and distance encoding must be supplied.
492 fn writeTokens(
493 self: *Self,
494 tokens: []const Token,
495 le_codes: []hc.HuffCode,
496 oe_codes: []hc.HuffCode,
497 ) Error!void {
498 for (tokens) |t| {
499 if (t.kind == Token.Kind.literal) {
500 try self.writeCode(le_codes[t.literal()]);
501 continue;
502 }
503
504 // Write the length
505 const le = t.lengthEncoding();
506 try self.writeCode(le_codes[le.code]);
507 if (le.extra_bits > 0) {
508 try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509 }
510
511 // Write the distance
512 const oe = t.distanceEncoding();
513 try self.writeCode(oe_codes[oe.code]);
514 if (oe.extra_bits > 0) {
515 try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516 }
517 }
518 // add end_block_marker at the end
519 try self.writeCode(le_codes[consts.end_block_marker]);
520 }
521
522 // Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523 // if the results only gains very little from compression.
524 pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525 // Add everything as literals
526 histogram(input, &self.literal_freq);
527
528 self.literal_freq[consts.end_block_marker] = 1;
529
530 const num_literals = consts.end_block_marker + 1;
531 self.distance_freq[0] = 1;
532 const num_distances = 1;
533
534 self.literal_encoding.generate(&self.literal_freq, 15);
535
536 // Figure out smallest code.
537 // Always use dynamic Huffman or Store
538 var num_codegens: u32 = 0;
539
540 // Generate codegen and codegenFrequencies, which indicates how to encode
541 // the literal_encoding and the distance_encoding.
542 self.generateCodegen(
543 num_literals,
544 num_distances,
545 &self.literal_encoding,
546 &self.huff_distance,
547 );
548 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550 const size = dynamic_size.size;
551 num_codegens = dynamic_size.num_codegens;
552
553 // Store bytes, if we don't get a reasonable improvement.
554 const stored_size_ret = storedSizeFits(input);
555 const ssize = stored_size_ret.size;
556 const storable = stored_size_ret.storable;
557
558 if (storable and ssize < (size + (size >> 4))) {
559 try self.storedBlock(input, eof);
560 return;
561 }
562
563 // Huffman.
564 try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565 const encoding = self.literal_encoding.codes[0..257];
566
567 for (input) |t| {
568 const c = encoding[t];
569 try self.bit_writer.writeBits(c.code, c.len);
570 }
571 try self.writeCode(encoding[consts.end_block_marker]);
572 }
573
574 // histogram accumulates a histogram of b in h.
575 fn histogram(b: []const u8, h: *[286]u16) void {
576 // Clear histogram
577 for (h, 0..) |_, i| {
578 h[i] = 0;
579 }
580
581 var lh = h.*[0..256];
582 for (b) |t| {
583 lh[t] += 1;
584 }
585 }
586 };
587}
588
589// tests
590const expect = std.testing.expect;
591const fmt = std.fmt;
592const testing = std.testing;
593const ArrayList = std.ArrayList;
594
595const TestCase = @import("testdata/block_writer.zig").TestCase;
596const testCases = @import("testdata/block_writer.zig").testCases;
597
598// tests if the writeBlock encoding has changed.
599test "write" {
600 inline for (0..testCases.len) |i| {
601 try testBlock(testCases[i], .write_block);
602 }
603}
604
605// tests if the writeBlockDynamic encoding has changed.
606test "dynamicBlock" {
607 inline for (0..testCases.len) |i| {
608 try testBlock(testCases[i], .write_dyn_block);
609 }
610}
611
612test "huffmanBlock" {
613 inline for (0..testCases.len) |i| {
614 try testBlock(testCases[i], .write_huffman_block);
615 }
616 try testBlock(.{
617 .tokens = &[_]Token{},
618 .input = "huffman-rand-max.input",
619 .want = "huffman-rand-max.{s}.expect",
620 }, .write_huffman_block);
621}
622
623const TestFn = enum {
624 write_block,
625 write_dyn_block, // write dynamic block
626 write_huffman_block,
627
628 fn to_s(self: TestFn) []const u8 {
629 return switch (self) {
630 .write_block => "wb",
631 .write_dyn_block => "dyn",
632 .write_huffman_block => "huff",
633 };
634 }
635
636 fn write(
637 comptime self: TestFn,
638 bw: anytype,
639 tok: []const Token,
640 input: ?[]const u8,
641 final: bool,
642 ) !void {
643 switch (self) {
644 .write_block => try bw.write(tok, final, input),
645 .write_dyn_block => try bw.dynamicBlock(tok, final, input),
646 .write_huffman_block => try bw.huffmanBlock(input.?, final),
647 }
648 try bw.flush();
649 }
650};
651
652// testBlock tests a block against its references
653//
654// size
655// 64K [file-name].input - input non compressed file
656// 8.1K [file-name].golden -
657// 78 [file-name].dyn.expect - output with writeBlockDynamic
658// 78 [file-name].wb.expect - output with writeBlock
659// 8.1K [file-name].huff.expect - output with writeBlockHuff
660// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662//
663// wb - writeBlock
664// dyn - writeBlockDynamic
665// huff - writeBlockHuff
666//
667fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668 if (tc.input.len != 0 and tc.want.len != 0) {
669 const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670 const input = @embedFile("testdata/block_writer/" ++ tc.input);
671 const want = @embedFile("testdata/block_writer/" ++ want_name);
672 try testWriteBlock(tfn, input, want, tc.tokens);
673 }
674
675 if (tfn == .write_huffman_block) {
676 return;
677 }
678
679 const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680 const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681 try testWriteBlock(tfn, null, want, tc.tokens);
682}
683
684// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686 var buf = ArrayList(u8).init(testing.allocator);
687 var bw = blockWriter(buf.writer());
688 try tfn.write(&bw, tokens, input, false);
689 var got = buf.items;
690 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691 try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692 //
693 // Test if the writer produces the same output after reset.
694 buf.deinit();
695 buf = ArrayList(u8).init(testing.allocator);
696 defer buf.deinit();
697 bw.setWriter(buf.writer());
698
699 try tfn.write(&bw, tokens, input, true);
700 try bw.flush();
701 got = buf.items;
702
703 try expect(got[0] & 1 == 1); // bfinal is set
704 buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706}
lib/std/compress/flate/deflate.zig+10-14
......@@ -7,7 +7,7 @@ const print = std.debug.print;
77
88const Token = @import("Token.zig");
99const consts = @import("consts.zig");
10const BlockWriter = @import("block_writer.zig").BlockWriter;
10const BlockWriter = @import("BlockWriter.zig");
1111const Container = @import("container.zig").Container;
1212const SlidingWindow = @import("SlidingWindow.zig");
1313const Lookup = @import("Lookup.zig");
......@@ -53,24 +53,20 @@ const LevelArgs = struct {
5353};
5454
5555/// Compress plain data from reader into compressed stream written to writer.
56pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57 var c = try compressor(container, writer, options);
56pub fn compress(
57 comptime container: Container,
58 reader: *std.io.BufferedReader,
59 writer: *std.io.BufferedWriter,
60 options: Options,
61) !void {
62 var c = try Compressor.init(container, writer, options);
5863 try c.compress(reader);
5964 try c.finish();
6065}
6166
62/// Create compressor for writer type.
63pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64 container,
65 @TypeOf(writer),
66) {
67 return try Compressor(container, @TypeOf(writer)).init(writer, options);
68}
69
7067/// Compressor type.
71pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72 const TokenWriterType = BlockWriter(WriterType);
73 return Deflate(container, WriterType, TokenWriterType);
68pub fn Compressor(comptime container: Container) type {
69 return Deflate(container, BlockWriter);
7470}
7571
7672/// Default compression algorithm. Has two steps: tokenization and token
lib/std/compress/flate/inflate.zig+12-13
......@@ -11,22 +11,22 @@ const codegen_order = @import("consts.zig").huffman.codegen_order;
1111
1212/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
1313/// `writer` stream.
14pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
14pub fn decompress(comptime container: Container, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
1515 var d = decompressor(container, reader);
1616 try d.decompress(writer);
1717}
1818
1919/// Inflate decompressor for the reader type.
20pub fn decompressor(comptime container: Container, reader: anytype) Decompressor(container, @TypeOf(reader)) {
21 return Decompressor(container, @TypeOf(reader)).init(reader);
20pub fn decompressor(comptime container: Container, reader: *std.io.BufferedReader) Decompressor(container) {
21 return Decompressor(container).init(reader);
2222}
2323
24pub fn Decompressor(comptime container: Container, comptime ReaderType: type) type {
24pub fn Decompressor(comptime container: Container) type {
2525 // zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
2626 // gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
2727 // For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
2828 const lookahead: type = if (container == .zlib) u32 else u64;
29 return Inflate(container, lookahead, ReaderType);
29 return Inflate(container, lookahead);
3030}
3131
3232/// Inflate decompresses deflate bit stream. Reads compressed data from reader
......@@ -48,15 +48,14 @@ pub fn Decompressor(comptime container: Container, comptime ReaderType: type) ty
4848/// * 64K for history (CircularBuffer)
4949/// * ~10K huffman decoders (Literal and DistanceDecoder)
5050///
51pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comptime ReaderType: type) type {
51pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type {
5252 assert(LookaheadType == u32 or LookaheadType == u64);
53 const BitReaderType = BitReader(LookaheadType, ReaderType);
53 const BitReaderType = BitReader(LookaheadType);
5454
5555 return struct {
56 //const BitReaderType = BitReader(ReaderType);
5756 const F = BitReaderType.flag;
5857
59 bits: BitReaderType = .{},
58 bits: BitReaderType,
6059 hist: CircularBuffer = .{},
6160 // Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
6261 hasher: container.Hasher() = .{},
......@@ -88,8 +87,8 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comp
8887 InvalidDynamicBlockHeader,
8988 };
9089
91 pub fn init(rt: ReaderType) Self {
92 return .{ .bits = BitReaderType.init(rt) };
90 pub fn init(bw: *std.io.BufferedReader) Self {
91 return .{ .bits = BitReaderType.init(bw) };
9392 }
9493
9594 fn blockHeader(self: *Self) !void {
......@@ -289,7 +288,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comp
289288 }
290289
291290 /// Replaces the inner reader with new reader.
292 pub fn setReader(self: *Self, new_reader: ReaderType) void {
291 pub fn setReader(self: *Self, new_reader: *std.io.BufferedReader) void {
293292 self.bits.forward_reader = new_reader;
294293 if (self.state == .end or self.state == .protocol_footer) {
295294 self.state = .protocol_header;
......@@ -298,7 +297,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type, comp
298297
299298 // Reads all compressed data from the internal reader and outputs plain
300299 // (uncompressed) data to the provided writer.
301 pub fn decompress(self: *Self, writer: anytype) !void {
300 pub fn decompress(self: *Self, writer: *std.io.BufferedWriter) !void {
302301 while (try self.next()) |buf| {
303302 try writer.writeAll(buf);
304303 }
lib/std/compress/lzma/decode.zig+164-4
......@@ -4,14 +4,174 @@ const math = std.math;
44const Allocator = std.mem.Allocator;
55
66pub const lzbuffer = @import("decode/lzbuffer.zig");
7pub const rangecoder = @import("decode/rangecoder.zig");
87
98const LzCircularBuffer = lzbuffer.LzCircularBuffer;
10const BitTree = rangecoder.BitTree;
11const LenDecoder = rangecoder.LenDecoder;
12const RangeDecoder = rangecoder.RangeDecoder;
139const Vec2D = @import("vec2d.zig").Vec2D;
1410
11pub const RangeDecoder = struct {
12 range: u32,
13 code: u32,
14
15 pub fn init(br: *std.io.BufferedReader) !RangeDecoder {
16 const reserved = try br.takeByte();
17 if (reserved != 0) {
18 return error.CorruptInput;
19 }
20 return .{
21 .range = 0xFFFF_FFFF,
22 .code = try br.readInt(u32, .big),
23 };
24 }
25
26 pub inline fn isFinished(self: RangeDecoder) bool {
27 return self.code == 0;
28 }
29
30 inline fn normalize(self: *RangeDecoder, br: *std.io.BufferedReader) !void {
31 if (self.range < 0x0100_0000) {
32 self.range <<= 8;
33 self.code = (self.code << 8) ^ @as(u32, try br.takeByte());
34 }
35 }
36
37 inline fn getBit(self: *RangeDecoder, br: *std.io.BufferedReader) !bool {
38 self.range >>= 1;
39
40 const bit = self.code >= self.range;
41 if (bit)
42 self.code -= self.range;
43
44 try self.normalize(br);
45 return bit;
46 }
47
48 pub fn get(self: *RangeDecoder, br: *std.io.BufferedReader, count: usize) !u32 {
49 var result: u32 = 0;
50 var i: usize = 0;
51 while (i < count) : (i += 1)
52 result = (result << 1) ^ @intFromBool(try self.getBit(br));
53 return result;
54 }
55
56 pub inline fn decodeBit(self: *RangeDecoder, br: *std.io.BufferedReader, prob: *u16, update: bool) !bool {
57 const bound = (self.range >> 11) * prob.*;
58
59 if (self.code < bound) {
60 if (update)
61 prob.* += (0x800 - prob.*) >> 5;
62 self.range = bound;
63
64 try self.normalize(br);
65 return false;
66 } else {
67 if (update)
68 prob.* -= prob.* >> 5;
69 self.code -= bound;
70 self.range -= bound;
71
72 try self.normalize(br);
73 return true;
74 }
75 }
76
77 fn parseBitTree(
78 self: *RangeDecoder,
79 br: *std.io.BufferedReader,
80 num_bits: u5,
81 probs: []u16,
82 update: bool,
83 ) !u32 {
84 var tmp: u32 = 1;
85 var i: @TypeOf(num_bits) = 0;
86 while (i < num_bits) : (i += 1) {
87 const bit = try self.decodeBit(br, &probs[tmp], update);
88 tmp = (tmp << 1) ^ @intFromBool(bit);
89 }
90 return tmp - (@as(u32, 1) << num_bits);
91 }
92
93 pub fn parseReverseBitTree(
94 self: *RangeDecoder,
95 br: *std.io.BufferedReader,
96 num_bits: u5,
97 probs: []u16,
98 offset: usize,
99 update: bool,
100 ) !u32 {
101 var result: u32 = 0;
102 var tmp: usize = 1;
103 var i: @TypeOf(num_bits) = 0;
104 while (i < num_bits) : (i += 1) {
105 const bit = @intFromBool(try self.decodeBit(br, &probs[offset + tmp], update));
106 tmp = (tmp << 1) ^ bit;
107 result ^= @as(u32, bit) << i;
108 }
109 return result;
110 }
111};
112
113pub fn BitTree(comptime num_bits: usize) type {
114 return struct {
115 probs: [1 << num_bits]u16 = @splat(0x400),
116
117 const Self = @This();
118
119 pub fn parse(
120 self: *Self,
121 br: *std.io.BufferedReader,
122 decoder: *RangeDecoder,
123 update: bool,
124 ) !u32 {
125 return decoder.parseBitTree(br, num_bits, &self.probs, update);
126 }
127
128 pub fn parseReverse(
129 self: *Self,
130 br: *std.io.BufferedReader,
131 decoder: *RangeDecoder,
132 update: bool,
133 ) !u32 {
134 return decoder.parseReverseBitTree(br, num_bits, &self.probs, 0, update);
135 }
136
137 pub fn reset(self: *Self) void {
138 @memset(&self.probs, 0x400);
139 }
140 };
141}
142
143pub const LenDecoder = struct {
144 choice: u16 = 0x400,
145 choice2: u16 = 0x400,
146 low_coder: [16]BitTree(3) = @splat(.{}),
147 mid_coder: [16]BitTree(3) = @splat(.{}),
148 high_coder: BitTree(8) = .{},
149
150 pub fn decode(
151 self: *LenDecoder,
152 br: *std.io.BufferedReader,
153 decoder: *RangeDecoder,
154 pos_state: usize,
155 update: bool,
156 ) !usize {
157 if (!try decoder.decodeBit(br, &self.choice, update)) {
158 return @as(usize, try self.low_coder[pos_state].parse(br, decoder, update));
159 } else if (!try decoder.decodeBit(br, &self.choice2, update)) {
160 return @as(usize, try self.mid_coder[pos_state].parse(br, decoder, update)) + 8;
161 } else {
162 return @as(usize, try self.high_coder.parse(br, decoder, update)) + 16;
163 }
164 }
165
166 pub fn reset(self: *LenDecoder) void {
167 self.choice = 0x400;
168 self.choice2 = 0x400;
169 for (&self.low_coder) |*t| t.reset();
170 for (&self.mid_coder) |*t| t.reset();
171 self.high_coder.reset();
172 }
173};
174
15175pub const Options = struct {
16176 unpacked_size: UnpackedSize = .read_from_header,
17177 memlimit: ?usize = null,
lib/std/compress/lzma/decode/rangecoder.zig deleted-181
......@@ -1,181 +0,0 @@
1const std = @import("../../../std.zig");
2const mem = std.mem;
3
4pub const RangeDecoder = struct {
5 range: u32,
6 code: u32,
7
8 pub fn init(reader: anytype) !RangeDecoder {
9 const reserved = try reader.readByte();
10 if (reserved != 0) {
11 return error.CorruptInput;
12 }
13 return RangeDecoder{
14 .range = 0xFFFF_FFFF,
15 .code = try reader.readInt(u32, .big),
16 };
17 }
18
19 pub fn fromParts(
20 range: u32,
21 code: u32,
22 ) RangeDecoder {
23 return .{
24 .range = range,
25 .code = code,
26 };
27 }
28
29 pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
30 self.range = range;
31 self.code = code;
32 }
33
34 pub inline fn isFinished(self: RangeDecoder) bool {
35 return self.code == 0;
36 }
37
38 inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
39 if (self.range < 0x0100_0000) {
40 self.range <<= 8;
41 self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
42 }
43 }
44
45 inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
46 self.range >>= 1;
47
48 const bit = self.code >= self.range;
49 if (bit)
50 self.code -= self.range;
51
52 try self.normalize(reader);
53 return bit;
54 }
55
56 pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
57 var result: u32 = 0;
58 var i: usize = 0;
59 while (i < count) : (i += 1)
60 result = (result << 1) ^ @intFromBool(try self.getBit(reader));
61 return result;
62 }
63
64 pub inline fn decodeBit(self: *RangeDecoder, reader: anytype, prob: *u16, update: bool) !bool {
65 const bound = (self.range >> 11) * prob.*;
66
67 if (self.code < bound) {
68 if (update)
69 prob.* += (0x800 - prob.*) >> 5;
70 self.range = bound;
71
72 try self.normalize(reader);
73 return false;
74 } else {
75 if (update)
76 prob.* -= prob.* >> 5;
77 self.code -= bound;
78 self.range -= bound;
79
80 try self.normalize(reader);
81 return true;
82 }
83 }
84
85 fn parseBitTree(
86 self: *RangeDecoder,
87 reader: anytype,
88 num_bits: u5,
89 probs: []u16,
90 update: bool,
91 ) !u32 {
92 var tmp: u32 = 1;
93 var i: @TypeOf(num_bits) = 0;
94 while (i < num_bits) : (i += 1) {
95 const bit = try self.decodeBit(reader, &probs[tmp], update);
96 tmp = (tmp << 1) ^ @intFromBool(bit);
97 }
98 return tmp - (@as(u32, 1) << num_bits);
99 }
100
101 pub fn parseReverseBitTree(
102 self: *RangeDecoder,
103 reader: anytype,
104 num_bits: u5,
105 probs: []u16,
106 offset: usize,
107 update: bool,
108 ) !u32 {
109 var result: u32 = 0;
110 var tmp: usize = 1;
111 var i: @TypeOf(num_bits) = 0;
112 while (i < num_bits) : (i += 1) {
113 const bit = @intFromBool(try self.decodeBit(reader, &probs[offset + tmp], update));
114 tmp = (tmp << 1) ^ bit;
115 result ^= @as(u32, bit) << i;
116 }
117 return result;
118 }
119};
120
121pub fn BitTree(comptime num_bits: usize) type {
122 return struct {
123 probs: [1 << num_bits]u16 = @splat(0x400),
124
125 const Self = @This();
126
127 pub fn parse(
128 self: *Self,
129 reader: anytype,
130 decoder: *RangeDecoder,
131 update: bool,
132 ) !u32 {
133 return decoder.parseBitTree(reader, num_bits, &self.probs, update);
134 }
135
136 pub fn parseReverse(
137 self: *Self,
138 reader: anytype,
139 decoder: *RangeDecoder,
140 update: bool,
141 ) !u32 {
142 return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
143 }
144
145 pub fn reset(self: *Self) void {
146 @memset(&self.probs, 0x400);
147 }
148 };
149}
150
151pub const LenDecoder = struct {
152 choice: u16 = 0x400,
153 choice2: u16 = 0x400,
154 low_coder: [16]BitTree(3) = @splat(.{}),
155 mid_coder: [16]BitTree(3) = @splat(.{}),
156 high_coder: BitTree(8) = .{},
157
158 pub fn decode(
159 self: *LenDecoder,
160 reader: anytype,
161 decoder: *RangeDecoder,
162 pos_state: usize,
163 update: bool,
164 ) !usize {
165 if (!try decoder.decodeBit(reader, &self.choice, update)) {
166 return @as(usize, try self.low_coder[pos_state].parse(reader, decoder, update));
167 } else if (!try decoder.decodeBit(reader, &self.choice2, update)) {
168 return @as(usize, try self.mid_coder[pos_state].parse(reader, decoder, update)) + 8;
169 } else {
170 return @as(usize, try self.high_coder.parse(reader, decoder, update)) + 16;
171 }
172 }
173
174 pub fn reset(self: *LenDecoder) void {
175 self.choice = 0x400;
176 self.choice2 = 0x400;
177 for (&self.low_coder) |*t| t.reset();
178 for (&self.mid_coder) |*t| t.reset();
179 self.high_coder.reset();
180 }
181};
lib/std/compress/lzma2.zig+1-1
......@@ -11,7 +11,7 @@ pub fn decompress(allocator: Allocator, reader: *std.io.BufferedReader, writer:
1111
1212test {
1313 const expected = "Hello\nWorld!\n";
14 const compressed = &[_]u8{
14 const compressed = [_]u8{
1515 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02,
1616 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00,
1717 };
lib/std/compress/lzma2/decode.zig+16-16
......@@ -5,7 +5,7 @@ const lzma = @import("../lzma.zig");
55const DecoderState = lzma.decode.DecoderState;
66const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
77const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.rangecoder.RangeDecoder;
8const RangeDecoder = lzma.decode.RangeDecoder;
99
1010pub const Decoder = struct {
1111 lzma_state: DecoderState,
......@@ -32,14 +32,14 @@ pub const Decoder = struct {
3232 pub fn decompress(
3333 self: *Decoder,
3434 allocator: Allocator,
35 reader: anytype,
36 writer: anytype,
35 reader: *std.io.BufferedReader,
36 writer: *std.io.BufferedWriter,
3737 ) !void {
3838 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
3939 defer accum.deinit(allocator);
4040
4141 while (true) {
42 const status = try reader.readByte();
42 const status = try reader.takeByte();
4343
4444 switch (status) {
4545 0 => break,
......@@ -55,8 +55,8 @@ pub const Decoder = struct {
5555 fn parseLzma(
5656 self: *Decoder,
5757 allocator: Allocator,
58 reader: anytype,
59 writer: anytype,
58 br: *std.io.BufferedReader,
59 writer: *std.io.BufferedWriter,
6060 accum: *LzAccumBuffer,
6161 status: u8,
6262 ) !void {
......@@ -97,12 +97,12 @@ pub const Decoder = struct {
9797 const unpacked_size = blk: {
9898 var tmp: u64 = status & 0x1F;
9999 tmp <<= 16;
100 tmp |= try reader.readInt(u16, .big);
100 tmp |= try br.takeInt(u16, .big);
101101 break :blk tmp + 1;
102102 };
103103
104104 const packed_size = blk: {
105 const tmp: u17 = try reader.readInt(u16, .big);
105 const tmp: u17 = try br.takeInt(u16, .big);
106106 break :blk tmp + 1;
107107 };
108108
......@@ -114,7 +114,7 @@ pub const Decoder = struct {
114114 var new_props = self.lzma_state.lzma_props;
115115
116116 if (reset.props) {
117 var props = try reader.readByte();
117 var props = try br.takeByte();
118118 if (props >= 225) {
119119 return error.CorruptInput;
120120 }
......@@ -137,10 +137,10 @@ pub const Decoder = struct {
137137
138138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139139
140 var counter = std.io.countingReader(reader);
141 const counter_reader = counter.reader();
140 var counter: std.io.CountingReader = .{ .child_reader = br.reader() };
141 var counter_reader = counter.reader().unbuffered();
142142
143 var rangecoder = try RangeDecoder.init(counter_reader);
143 var rangecoder = try RangeDecoder.init(&counter_reader);
144144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145145
146146 if (counter.bytes_read != packed_size) {
......@@ -150,12 +150,12 @@ pub const Decoder = struct {
150150
151151 fn parseUncompressed(
152152 allocator: Allocator,
153 reader: anytype,
154 writer: anytype,
153 reader: *std.io.BufferedReader,
154 writer: *std.io.BufferedWriter,
155155 accum: *LzAccumBuffer,
156156 reset_dict: bool,
157157 ) !void {
158 const unpacked_size = @as(u17, try reader.readInt(u16, .big)) + 1;
158 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
159159
160160 if (reset_dict) {
161161 try accum.reset(writer);
......@@ -163,7 +163,7 @@ pub const Decoder = struct {
163163
164164 var i: @TypeOf(unpacked_size) = 0;
165165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.readByte());
166 try accum.appendByte(allocator, try reader.takeByte());
167167 }
168168 }
169169};
lib/std/compress/zlib.zig+10-39
......@@ -1,73 +1,44 @@
1const std = @import("../std.zig");
12const deflate = @import("flate/deflate.zig");
23const inflate = @import("flate/inflate.zig");
34
45/// Decompress compressed data from reader and write plain data to the writer.
5pub fn decompress(reader: anytype, writer: anytype) !void {
6pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
67 try inflate.decompress(.zlib, reader, writer);
78}
89
9/// Decompressor type
10pub fn Decompressor(comptime ReaderType: type) type {
11 return inflate.Decompressor(.zlib, ReaderType);
12}
13
14/// Create Decompressor which will read compressed data from reader.
15pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16 return inflate.decompressor(.zlib, reader);
17}
10pub const Decompressor = inflate.Decompressor(.zlib);
1811
1912/// Compression level, trades between speed and compression size.
2013pub const Options = deflate.Options;
2114
2215/// Compress plain data from reader and write compressed data to the writer.
23pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
16pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) !void {
2417 try deflate.compress(.zlib, reader, writer, options);
2518}
2619
27/// Compressor type
28pub fn Compressor(comptime WriterType: type) type {
29 return deflate.Compressor(.zlib, WriterType);
30}
31
32/// Create Compressor which outputs compressed data to the writer.
33pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34 return try deflate.compressor(.zlib, writer, options);
35}
20pub const Compressor = deflate.Compressor(.zlib);
3621
3722/// Huffman only compression. Without Lempel-Ziv match searching. Faster
3823/// compression, less memory requirements but bigger compressed sizes.
3924pub const huffman = struct {
40 pub fn compress(reader: anytype, writer: anytype) !void {
25 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
4126 try deflate.huffman.compress(.zlib, reader, writer);
4227 }
4328
44 pub fn Compressor(comptime WriterType: type) type {
45 return deflate.huffman.Compressor(.zlib, WriterType);
46 }
47
48 pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49 return deflate.huffman.compressor(.zlib, writer);
50 }
29 pub const Compressor = deflate.huffman.Compressor(.zlib);
5130};
5231
5332// No compression store only. Compressed size is slightly bigger than plain.
5433pub const store = struct {
55 pub fn compress(reader: anytype, writer: anytype) !void {
34 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
5635 try deflate.store.compress(.zlib, reader, writer);
5736 }
5837
59 pub fn Compressor(comptime WriterType: type) type {
60 return deflate.store.Compressor(.zlib, WriterType);
61 }
62
63 pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64 return deflate.store.compressor(.zlib, writer);
65 }
38 pub const Compressor = deflate.store.Compressor(.zlib);
6639};
6740
6841test "should not overshoot" {
69 const std = @import("std");
70
7142 // Compressed zlib data with extra 4 bytes at the end.
7243 const data = [_]u8{
7344 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
......@@ -79,7 +50,7 @@ test "should not overshoot" {
7950 var stream = std.io.fixedBufferStream(data[0..]);
8051 const reader = stream.reader();
8152
82 var dcp = decompressor(reader);
53 var dcp = Decompressor.init(reader);
8354 var out: [128]u8 = undefined;
8455
8556 // Decompress
lib/std/debug/Dwarf.zig+1-1
......@@ -2241,7 +2241,7 @@ pub const ElfModule = struct {
22412241 if (chdr.ch_type != .ZLIB) continue;
22422242 const ch_size = chdr.ch_size;
22432243
2244 var zlib_stream = std.compress.zlib.decompressor(&section_reader);
2244 var zlib_stream: std.compress.zlib.Decompressor = .init(&section_reader);
22452245
22462246 const decompressed_section = try gpa.alloc(u8, ch_size);
22472247 errdefer gpa.free(decompressed_section);
lib/std/debug/SelfInfo.zig+4-2
......@@ -2027,8 +2027,10 @@ pub const VirtualMachine = struct {
20272027
20282028 var prev_row: Row = self.current_row;
20292029
2030 var cie_stream: std.io.FixedBufferStream = .{ .buffer = cie.initial_instructions };
2031 var fde_stream: std.io.FixedBufferStream = .{ .buffer = fde.instructions };
2030 var cie_stream: std.io.BufferedReader = undefined;
2031 cie_stream.initFixed(&cie.initial_instructions);
2032 var fde_stream: std.io.BufferedReader = undefined;
2033 fde_stream.initFixed(&fde.instructions);
20322034 const streams: [2]*std.io.FixedBufferStream = .{ &cie_stream, &fde_stream };
20332035
20342036 for (&streams, 0..) |stream, i| {
lib/std/fs/File.zig+2-2
......@@ -1613,11 +1613,11 @@ pub fn writer(file: File) std.io.Writer {
16131613const max_buffers_len = 16;
16141614
16151615pub fn reader_posRead(
1616 context: *anyopaque,
1616 context: ?*anyopaque,
16171617 bw: *std.io.BufferedWriter,
16181618 limit: std.io.Reader.Limit,
16191619 offset: u64,
1620) anyerror!usize {
1620) std.io.Reader.Result {
16211621 const file = opaqueToHandle(context);
16221622 const len: std.io.Writer.Len = if (limit.unwrap()) |l| .init(l) else .entire_file;
16231623 return writer.writeFile(bw, file, .init(offset), len, &.{}, 0);
lib/std/io/bit_reader.zig+3-3
......@@ -13,9 +13,9 @@ const std = @import("../std.zig");
1313// of the byte.
1414
1515/// Creates a bit reader which allows for reading bits from an underlying standard reader
16pub fn BitReader(comptime endian: std.builtin.Endian, comptime Reader: type) type {
16pub fn BitReader(comptime endian: std.builtin.Endian) type {
1717 return struct {
18 reader: Reader,
18 reader: *std.io.BufferedReader,
1919 bits: u8 = 0,
2020 count: u4 = 0,
2121
......@@ -157,7 +157,7 @@ pub fn BitReader(comptime endian: std.builtin.Endian, comptime Reader: type) typ
157157 };
158158}
159159
160pub fn bitReader(comptime endian: std.builtin.Endian, reader: anytype) BitReader(endian, @TypeOf(reader)) {
160pub fn bitReader(comptime endian: std.builtin.Endian, reader: *std.io.BufferedReader) BitReader(endian) {
161161 return .{ .reader = reader };
162162}
163163