authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-03-06 19:10:47-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-03-11 02:28:19+01:00
log6e5a95bd7c1f6b6bb659908fc7ba70cb5fd1b42c
treebda8d876893b28f29c4d886dd2d5ce4bc581046f
parentc2587582c8af957f032bee1951d923d472c89e2f

implement proper deflate flush semantics

To end a flate stream, `finish` must now be called. `flush` now follows regular semantics and byte-aligns the stream. Byte-aligning the stream is done with empty fixed or store blocks. To implement flush, a variable history length was added and it is tracked if the final bytes of history have been hashed yet.

2 files changed, 209 insertions(+), 110 deletions(-)

lib/std/compress/flate/Compress.zig+208-109
...@@ -2,7 +2,8 @@...@@ -2,7 +2,8 @@
2//!2//!
3//! The source of an `error.WriteFailed` is always the backing writer. After an3//! The source of an `error.WriteFailed` is always the backing writer. After an
4//! `error.WriteFailed`, the `.writer` becomes `.failing` and is unrecoverable.4//! `error.WriteFailed`, the `.writer` becomes `.failing` and is unrecoverable.
5//! After a `flush`, the writer also becomes `.failing` since the stream has5//!
6//! After `finish`, the writer also becomes `.failing` since the stream has
6//! been finished. This behavior also applies to `Raw` and `Huffman`.7//! been finished. This behavior also applies to `Raw` and `Huffman`.
78
8// Implementation details:9// Implementation details:
...@@ -43,9 +44,10 @@ const PackedOptionalU15 = packed struct(u16) {...@@ -43,9 +44,10 @@ const PackedOptionalU15 = packed struct(u16) {
43 pub const null_bit: PackedOptionalU15 = .{ .value = 0, .is_null = true };44 pub const null_bit: PackedOptionalU15 = .{ .value = 0, .is_null = true };
44};45};
4546
46/// After `flush` is called, all vtable calls with result in `error.WriteFailed.`47/// After `finish` is called, all vtable calls with result in `error.WriteFailed`.
47writer: Writer,48writer: Writer,
48has_history: bool,49history_len: u16,
50history_end_unhashed: bool,
49bit_writer: BitWriter,51bit_writer: BitWriter,
50buffered_tokens: struct {52buffered_tokens: struct {
51 /// List of `TokenBufferEntryHeader`s and their trailing data.53 /// List of `TokenBufferEntryHeader`s and their trailing data.
...@@ -108,7 +110,7 @@ const BitWriter = struct {...@@ -108,7 +110,7 @@ const BitWriter = struct {
108 b.buffered = @intCast(combined >> (combined_bits - b.buffered_n));110 b.buffered = @intCast(combined >> (combined_bits - b.buffered_n));
109 }111 }
110112
111 /// Assserts one byte can be written to `b.otuput` without rebasing.113 /// Asserts one byte can be written to `b.output` without rebasing.
112 pub fn byteAlign(b: *BitWriter) void {114 pub fn byteAlign(b: *BitWriter) void {
113 b.output.unusedCapacitySlice()[0] = b.buffered;115 b.output.unusedCapacitySlice()[0] = b.buffered;
114 b.output.advance(@intFromBool(b.buffered_n != 0));116 b.output.advance(@intFromBool(b.buffered_n != 0));
...@@ -116,6 +118,35 @@ const BitWriter = struct {...@@ -116,6 +118,35 @@ const BitWriter = struct {
116 b.buffered_n = 0;118 b.buffered_n = 0;
117 }119 }
118120
121 /// Byte align using only empty flate blocks
122 pub fn byteAlignBlocks(b: *BitWriter) Writer.Error!void {
123 if (b.buffered_n == 0) return;
124
125 // There are two methods to do this:
126 // 1. A store block (5 or 6 bytes)
127 // 2. Outputting empty 10-bit fixed blocks until aligned
128 //
129 // Fixed blocks advance the bit alignment by two, and so can only used for even numbers
130 // requiring a maximum of four bytes (three blocks = 30 bits) to which is always more
131 // efficient than store blocks.
132 if (b.buffered_n & 1 == 0) {
133 const splat = (8 - @as(u5, b.buffered_n)) >> 1;
134 const bits = splat * 10;
135 // fixed eos code is 0, so the only bits are for the block header
136 const pattern: u32 = BlockHeader.int(.{ .kind = .fixed, .final = false });
137 const splatted = ((pattern << 20) | (pattern << 10) | pattern) >> (30 - bits);
138 try b.write(splatted, bits);
139 } else {
140 try b.write(BlockHeader.int(.{ .kind = .stored, .final = false }), 3);
141 try b.output.rebase(0, 5);
142 b.byteAlign();
143 b.output.writeInt(u16, 0x0000, .little) catch unreachable;
144 b.output.writeInt(u16, 0xffff, .little) catch unreachable;
145 }
146
147 assert(b.buffered_n == 0);
148 }
149
119 pub fn writeClen(150 pub fn writeClen(
120 b: *BitWriter,151 b: *BitWriter,
121 hclen: u4,152 hclen: u4,
...@@ -159,9 +190,9 @@ const BitWriter = struct {...@@ -159,9 +190,9 @@ const BitWriter = struct {
159/// The maximum value is `math.maxInt(u16) - 1` since one token is reserved for end-of-block.190/// The maximum value is `math.maxInt(u16) - 1` since one token is reserved for end-of-block.
160const block_tokens: u16 = 1 << 15;191const block_tokens: u16 = 1 << 15;
161const lookup_hash_bits = 15;192const lookup_hash_bits = 15;
162const Hash = u16; // `u[lookup_hash_bits]` is not used due to worse optimization (with LLVM 21)193const Hash = u16; // `@Int(.unsigned, lookup_hash_bits)` is not used due to worse optimization (with LLVM 21)
163const seq_bytes = 3; // not intended to be changed194const seq_bytes = 3; // not intended to be changed
164const Seq = std.meta.Int(.unsigned, seq_bytes * 8);195const Seq = @Int(.unsigned, seq_bytes * 8);
165196
166const TokenBufferEntryHeader = packed struct(u16) {197const TokenBufferEntryHeader = packed struct(u16) {
167 kind: enum(u1) {198 kind: enum(u1) {
...@@ -295,7 +326,8 @@ pub fn init(...@@ -295,7 +326,8 @@ pub fn init(
295 .rebase = rebase,326 .rebase = rebase,
296 },327 },
297 },328 },
298 .has_history = false,329 .history_len = 0,
330 .history_end_unhashed = false,
299 .bit_writer = .init(output),331 .bit_writer = .init(output),
300 .buffered_tokens = .empty,332 .buffered_tokens = .empty,
301 .lookup = .{333 .lookup = .{
...@@ -314,68 +346,110 @@ fn drain(w: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize...@@ -314,68 +346,110 @@ fn drain(w: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize
314 errdefer w.* = .failing;346 errdefer w.* = .failing;
315 // There may have not been enough space in the buffer and the write was sent directly here.347 // There may have not been enough space in the buffer and the write was sent directly here.
316 // However, it is required that all data goes through the buffer to keep a history.348 // However, it is required that all data goes through the buffer to keep a history.
317 //
318 // Additionally, ensuring the buffer is always full ensures there is always a full history
319 // after.
320 const data_n = w.buffer.len - w.end;349 const data_n = w.buffer.len - w.end;
321 _ = w.fixedDrain(data, splat) catch {};350 _ = w.fixedDrain(data, splat) catch {};
322 assert(w.end == w.buffer.len);351 assert(w.end == w.buffer.len);
323 try rebaseInner(w, 0, 1, false);352 try rebaseInner(w, 0, 1, false, false);
324 return data_n;353 return data_n;
325}354}
326355
327fn flush(w: *Writer) Writer.Error!void {356fn flush(w: *Writer) Writer.Error!void {
328 defer w.* = .failing;357 errdefer w.* = .failing;
358 try rebaseInner(w, 0, w.buffer.len - flate.history_len, true, false);
329 const c: *Compress = @fieldParentPtr("writer", w);359 const c: *Compress = @fieldParentPtr("writer", w);
330 try rebaseInner(w, 0, w.buffer.len - flate.history_len, true);360 try c.bit_writer.byteAlignBlocks();
361}
362
363pub fn finish(c: *Compress) Writer.Error!void {
364 defer c.writer = .failing;
365 try rebaseInner(&c.writer, 0, c.writer.buffer.len - flate.history_len, true, true);
331 try c.bit_writer.output.rebase(0, 1);366 try c.bit_writer.output.rebase(0, 1);
332 c.bit_writer.byteAlign();367 c.bit_writer.byteAlign();
333 try c.hasher.writeFooter(c.bit_writer.output);368 try c.hasher.writeFooter(c.bit_writer.output);
334}369}
335370
336fn rebase(w: *Writer, preserve: usize, capacity: usize) Writer.Error!void {371fn rebase(w: *Writer, preserve: usize, capacity: usize) Writer.Error!void {
337 return rebaseInner(w, preserve, capacity, false);372 errdefer w.* = .failing;
373 return rebaseInner(w, preserve, capacity, false, false);
338}374}
339375
340pub const rebase_min_preserve = flate.history_len;376pub const rebase_min_preserve = flate.history_len;
341pub const rebase_reserved_capacity = (token.max_length + 1) + seq_bytes;377pub const rebase_reserved_capacity = (token.max_length + 1) + seq_bytes;
342378
343fn rebaseInner(w: *Writer, preserve: usize, capacity: usize, eos: bool) Writer.Error!void {379fn rebaseInner(
344 if (!eos) {380 w: *Writer,
381 preserve: usize,
382 capacity: usize,
383 is_flush: bool,
384 is_finish: bool,
385) Writer.Error!void {
386 if (!is_flush) {
345 assert(@max(preserve, rebase_min_preserve) + (capacity + rebase_reserved_capacity) <= w.buffer.len);387 assert(@max(preserve, rebase_min_preserve) + (capacity + rebase_reserved_capacity) <= w.buffer.len);
346 assert(w.end >= flate.history_len + rebase_reserved_capacity); // Above assert should
347 // fail since rebase is only called when `capacity` is not present. This assertion is
348 // important because a full history is required at the end.
349 } else {388 } else {
389 // Preverse is not considered for `matching_end`
350 assert(preserve == 0 and capacity == w.buffer.len - flate.history_len);390 assert(preserve == 0 and capacity == w.buffer.len - flate.history_len);
351 }391 }
392 if (is_finish) assert(is_flush);
352393
353 const c: *Compress = @fieldParentPtr("writer", w);394 const c: *Compress = @fieldParentPtr("writer", w);
354 const buffered = w.buffered();395 const buffered = w.buffered();
355396
356 const start = @as(usize, flate.history_len) * @intFromBool(c.has_history);397 const start: usize = c.history_len;
357 const lit_end: usize = if (!eos)398 const hashable_len = buffered.len -| (seq_bytes - 1);
399 const matching_end: usize = if (!is_flush)
358 buffered.len - rebase_reserved_capacity - (preserve -| flate.history_len)400 buffered.len - rebase_reserved_capacity - (preserve -| flate.history_len)
359 else401 else
360 buffered.len -| (seq_bytes - 1);402 hashable_len;
361403
362 var i = start;404 var i = start;
363 var last_unmatched = i;405 var last_unmatched = i;
364 // Read from `w.buffer` instead of `buffered` since the latter may not406 var seq: Seq = start_seq: {
365 // have enough bytes. If this is the case, this variable is not used.407 if (c.history_end_unhashed) {
366 var seq: Seq = mem.readInt(408 @branchHint(.unlikely);
367 std.meta.Int(.unsigned, (seq_bytes - 1) * 8),409
368 w.buffer[i..][0 .. seq_bytes - 1],410 assert(i != 0);
369 .big,411 i -|= seq_bytes - 1;
370 );412 var seq: Seq = mem.readInt(
371 if (buffered[i..].len < seq_bytes - 1) {413 @Int(.unsigned, (seq_bytes - 1) * 8),
372 @branchHint(.unlikely);414 w.buffer[i..][0 .. seq_bytes - 1],
373 assert(eos);415 .big,
374 seq = undefined;416 );
375 assert(i >= lit_end);417
376 }418 while (i < @min(start, hashable_len)) {
419 seq <<= 8;
420 seq |= buffered[i + (seq_bytes - 1)];
421 c.addHash(i, hash(seq));
422 i += 1;
423 }
424
425 if (i < start) {
426 @branchHint(.unlikely);
427 i = start;
428 assert(i >= hashable_len);
429 assert(i >= matching_end);
430 assert(is_flush);
431 break :start_seq undefined; // Unused
432 }
433
434 c.history_end_unhashed = false;
435 break :start_seq seq;
436 }
437
438 if (i >= hashable_len) {
439 @branchHint(.unlikely);
440 assert(i >= matching_end);
441 assert(is_flush);
442 break :start_seq undefined; // Unused
443 }
377444
378 while (i < lit_end) {445 break :start_seq mem.readInt(
446 @Int(.unsigned, (seq_bytes - 1) * 8),
447 buffered[i..][0 .. seq_bytes - 1],
448 .big,
449 );
450 };
451
452 while (i < matching_end) {
379 var match_start = i;453 var match_start = i;
380 seq <<= 8;454 seq <<= 8;
381 seq |= buffered[i + (seq_bytes - 1)];455 seq |= buffered[i + (seq_bytes - 1)];
...@@ -420,43 +494,50 @@ fn rebaseInner(w: *Writer, preserve: usize, capacity: usize, eos: bool) Writer.E...@@ -420,43 +494,50 @@ fn rebaseInner(w: *Writer, preserve: usize, capacity: usize, eos: bool) Writer.E
420494
421 try c.outputBytes(buffered[last_unmatched..match_start]);495 try c.outputBytes(buffered[last_unmatched..match_start]);
422 try c.outputMatch(@intCast(match.dist), @intCast(match.len - 3));496 try c.outputMatch(@intCast(match.dist), @intCast(match.len - 3));
423
424 last_unmatched = match_start + match.len;497 last_unmatched = match_start + match.len;
425 if (last_unmatched + seq_bytes >= w.end) {
426 @branchHint(.unlikely);
427 assert(eos);
428 i = undefined;
429 break;
430 }
431498
432 while (true) {499 while (i < hashable_len) {
433 seq <<= 8;500 seq <<= 8;
434 seq |= buffered[i + (seq_bytes - 1)];501 seq |= buffered[i + (seq_bytes - 1)];
435 _ = c.addHash(i, hash(seq));502 c.addHash(i, hash(seq));
436 i += 1;503 i += 1;
437504
438 match_unadded -= 1;505 match_unadded -= 1;
439 if (match_unadded == 0) break;506 if (match_unadded == 0) break;
507 } else {
508 @branchHint(.unlikely);
509 assert(is_flush);
510 // `c.history_end_unhashed` is set down below
511 break;
440 }512 }
441 assert(i == match_start + match.len);513 assert(i == match_start + match.len);
442 }514 }
443515
444 if (eos) {516 if (is_flush) {
445 i = undefined; // (from match hashing logic)
446 try c.outputBytes(buffered[last_unmatched..]);517 try c.outputBytes(buffered[last_unmatched..]);
447 c.hasher.update(buffered[start..]);518 c.hasher.update(buffered[start..]);
448 try c.writeBlock(true);
449 return;
450 }
451519
452 try c.outputBytes(buffered[last_unmatched..i]);520 if (is_finish) {
453 c.hasher.update(buffered[start..i]);521 try c.writeBlock(true);
522 return; // Other state does not need updated since the writer transitions to `.failing`
523 }
524
525 i = buffered.len;
526 c.history_end_unhashed = i != 0;
527
528 if (c.buffered_tokens.n != 0) {
529 try c.writeBlock(false);
530 }
531 } else {
532 try c.outputBytes(buffered[last_unmatched..i]);
533 c.hasher.update(buffered[start..i]);
534 }
454535
455 const preserved = buffered[i - flate.history_len ..];536 c.history_len = @min(i, flate.history_len);
456 assert(preserved.len > @max(rebase_min_preserve, preserve));537 const preserved = buffered[i - c.history_len ..];
538 if (!is_flush) assert(preserved.len >= @max(rebase_min_preserve, preserve));
457 @memmove(w.buffer[0..preserved.len], preserved);539 @memmove(w.buffer[0..preserved.len], preserved);
458 w.end = preserved.len;540 w.end = preserved.len;
459 c.has_history = true;
460}541}
461542
462fn addHash(c: *Compress, i: usize, h: Hash) void {543fn addHash(c: *Compress, i: usize, h: Hash) void {
...@@ -499,7 +580,7 @@ fn betterMatchLen(old: u16, prev: []const u8, bytes: []const u8) u16 {...@@ -499,7 +580,7 @@ fn betterMatchLen(old: u16, prev: []const u8, bytes: []const u8) u16 {
499 assert(bytes.len >= token.min_length);580 assert(bytes.len >= token.min_length);
500581
501 var i: u16 = 0;582 var i: u16 = 0;
502 const Block = std.meta.Int(.unsigned, @min(math.divCeil(583 const Block = @Int(.unsigned, @min(math.divCeil(
503 comptime_int,584 comptime_int,
504 math.ceilPowerOfTwoAssert(usize, @bitSizeOf(usize)),585 math.ceilPowerOfTwoAssert(usize, @bitSizeOf(usize)),
505 8,586 8,
...@@ -798,7 +879,6 @@ test buildClen {...@@ -798,7 +879,6 @@ test buildClen {
798879
799fn writeBlock(c: *Compress, eos: bool) Writer.Error!void {880fn writeBlock(c: *Compress, eos: bool) Writer.Error!void {
800 const toks = &c.buffered_tokens;881 const toks = &c.buffered_tokens;
801 if (!eos) assert(toks.n == block_tokens);
802 assert(toks.lit_freqs[256] == 0);882 assert(toks.lit_freqs[256] == 0);
803 toks.lit_freqs[256] = 1;883 toks.lit_freqs[256] = 1;
804884
...@@ -1438,20 +1518,8 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith...@@ -1438,20 +1518,8 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith
1438 .chain = chain,1518 .chain = chain,
1439 });1519 });
14401520
1441 // It is ensured that more bytes are not written then this to ensure this run1521 var max_output: usize = 32; // Headers / footer
1442 // does not take too long and that `flate_buf` does not run out of space.
1443 const flate_buf_blocks = flate_buf.len / block_tokens;
1444 // Allow a max overhead of 64 bytes per block since the implementation does not gaurauntee it
1445 // writes store blocks when optimal. This comes from taking less than 32 bytes to write an
1446 // optimal dynamic block header of mostly bitlen 8 codes and the end of block literal plus
1447 // `(65536 / 256) / 8`, which is is the maximum number of extra bytes from bitlen 9 codes. An
1448 // extra 32 bytes is reserved on top of that for container headers and footers.
1449 const max_size = flate_buf.len - (flate_buf_blocks * 64 + 32);
1450
1451 while (!smith.eosWeightedSimple(7, 1)) {1522 while (!smith.eosWeightedSimple(7, 1)) {
1452 const max_bytes = max_size -| expected_size;
1453 if (max_bytes == 0) break;
1454
1455 const buffered = deflate_w.writer.buffered();1523 const buffered = deflate_w.writer.buffered();
1456 // Required for repeating patterns and since writing from `buffered` is illegal1524 // Required for repeating patterns and since writing from `buffered` is illegal
1457 var copy_buf: [512]u8 = undefined;1525 var copy_buf: [512]u8 = undefined;
...@@ -1459,13 +1527,13 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith...@@ -1459,13 +1527,13 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith
1459 const bytes = bytes: switch (smith.valueRangeAtMost(1527 const bytes = bytes: switch (smith.valueRangeAtMost(
1460 u2,1528 u2,
1461 @intFromBool(buffered.len == 0),1529 @intFromBool(buffered.len == 0),
1462 2,1530 3,
1463 )) {1531 )) {
1464 0 => { // Copy1532 0 => { // Copy
1465 const start = smith.valueRangeLessThan(u32, 0, @intCast(buffered.len));1533 const start = smith.valueRangeLessThan(u32, 0, @intCast(buffered.len));
1466 // Reuse the implementation's history; otherwise, our own would need maintained.1534 // Reuse the implementation's history; otherwise, our own would need maintained.
1467 const from = buffered[start..];1535 const from = buffered[start..];
1468 const len = smith.valueRangeAtMost(u16, 1, @min(copy_buf.len, max_bytes));1536 const len = smith.valueRangeAtMost(u16, 1, copy_buf.len);
14691537
1470 const history_bytes = from[0..@min(from.len, len)];1538 const history_bytes = from[0..@min(from.len, len)];
1471 @memcpy(copy_buf[0..history_bytes.len], history_bytes);1539 @memcpy(copy_buf[0..history_bytes.len], history_bytes);
...@@ -1485,7 +1553,7 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith...@@ -1485,7 +1553,7 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith
1485 .value(FreqBufIndex, .random, 1),1553 .value(FreqBufIndex, .random, 1),
1486 })1554 })
1487 ];1555 ];
1488 const len = smith.valueRangeAtMost(u32, 1, @min(fbuf.len, max_bytes));1556 const len = smith.valueRangeAtMost(u32, 1, fbuf.len);
1489 const off = smith.valueRangeAtMost(u32, 0, @intCast(fbuf.len - len));1557 const off = smith.valueRangeAtMost(u32, 0, @intCast(fbuf.len - len));
1490 break :bytes fbuf[off..][0..len];1558 break :bytes fbuf[off..][0..len];
1491 },1559 },
...@@ -1493,25 +1561,42 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith...@@ -1493,25 +1561,42 @@ fn testFuzzedCompressInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith
1493 const rebaseable = bufsize - rebase_reserved_capacity;1561 const rebaseable = bufsize - rebase_reserved_capacity;
1494 const capacity = smith.valueRangeAtMost(u32, 1, rebaseable - rebase_min_preserve);1562 const capacity = smith.valueRangeAtMost(u32, 1, rebaseable - rebase_min_preserve);
1495 const preserve = smith.valueRangeAtMost(u32, 0, rebaseable - capacity);1563 const preserve = smith.valueRangeAtMost(u32, 0, rebaseable - capacity);
1496 try deflate_w.writer.rebase(preserve, capacity);1564 const failed = deflate_w.writer.rebase(preserve, capacity);
1565 if (flate_w.buffered().len > max_output) return error.OverheadTooLarge;
1566 failed catch return; // Wrote too much data and ran out of space
1567 continue;
1568 },
1569 3 => { // Flush
1570 max_output += 8; // Alignment data
1571 const failed = deflate_w.writer.flush();
1572 if (flate_w.buffered().len > max_output) return error.OverheadTooLarge;
1573 failed catch return; // Wrote too much data and ran out of space
1497 continue;1574 continue;
1498 },1575 },
1499 else => unreachable,
1500 };1576 };
15011577
1502 assert(bytes.len <= max_bytes);1578 // An overhead of 64 bytes is given for each block since the implementation does not
1503 try deflate_w.writer.writeAll(bytes);1579 // gaurauntee it writes store blocks when optimal. This comes from taking less than 32
1580 // bytes to write an optimal dynamic block header of mostly bitlen 8 codes and the end
1581 // of block literal plus `(65536 / 256) / 8`, which is is the maximum number of extra
1582 // bytes from bitlen 9 codes.
1583 max_output += bytes.len + ((bytes.len + flate_buf.len - 1) / block_tokens) * 64;
1584 const failed = deflate_w.writer.writeAll(bytes);
1585 if (flate_w.buffered().len > max_output) return error.OverheadTooLarge;
1586 failed catch return; // Wrote too much data and ran out of space
1504 expected_hash.update(bytes);1587 expected_hash.update(bytes);
1505 expected_size += @intCast(bytes.len);1588 expected_size += @intCast(bytes.len);
1506 }1589 }
15071590
1508 try deflate_w.writer.flush();1591 const failed = deflate_w.finish();
1592 if (flate_w.buffered().len > max_output) return error.OverheadTooLarge;
1593 failed catch return; // Wrote too much data and ran out of space
1509 try testingCheckDecompressedMatches(flate_w.buffered(), expected_size, expected_hash);1594 try testingCheckDecompressedMatches(flate_w.buffered(), expected_size, expected_hash);
1510}1595}
15111596
1512/// Does not compress data1597/// Does not compress data
1513pub const Raw = struct {1598pub const Raw = struct {
1514 /// After `flush` is called, all vtable calls with result in `error.WriteFailed.`1599 /// After `finish` is called, all vtable calls with result in `error.WriteFailed`.
1515 writer: Writer,1600 writer: Writer,
1516 output: *Writer,1601 output: *Writer,
1517 hasher: flate.Container.Hasher,1602 hasher: flate.Container.Hasher,
...@@ -1654,8 +1739,12 @@ pub const Raw = struct {...@@ -1654,8 +1739,12 @@ pub const Raw = struct {
1654 }1739 }
16551740
1656 fn flush(w: *Writer) Writer.Error!void {1741 fn flush(w: *Writer) Writer.Error!void {
1657 defer w.* = .failing;1742 errdefer w.* = .failing;
1658 try Raw.rebaseInner(w, 0, w.buffer.len, true);1743 try Raw.rebaseInner(w, 0, w.buffer.len, false);
1744 }
1745
1746 fn finish(r: *Raw) Writer.Error!void {
1747 try Raw.rebaseInner(&r.writer, 0, r.writer.buffer.len, true);
1659 }1748 }
16601749
1661 fn rebase(w: *Writer, preserve: usize, capacity: usize) Writer.Error!void {1750 fn rebase(w: *Writer, preserve: usize, capacity: usize) Writer.Error!void {
...@@ -1899,19 +1988,21 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)...@@ -1899,19 +1988,21 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)
1899 const Op = packed struct {1988 const Op = packed struct {
1900 drain: bool = false,1989 drain: bool = false,
1901 add_vec: bool = false,1990 add_vec: bool = false,
1902 rebase: bool = false,1991 rebase: enum(u2) { none, rebase, flush } = .none,
19031992
1904 pub const drain_only: @This() = .{ .drain = true };1993 pub const drain_only: @This() = .{ .drain = true };
1905 pub const add_vec_only: @This() = .{ .add_vec = true };1994 pub const add_vec_only: @This() = .{ .add_vec = true };
1906 pub const add_vec_and_drain: @This() = .{ .add_vec = true, .drain = true };1995 pub const add_vec_and_drain: @This() = .{ .add_vec = true, .drain = true };
1907 pub const drain_and_rebase: @This() = .{ .drain = true, .rebase = true };1996 pub const drain_and_rebase: @This() = .{ .drain = true, .rebase = .rebase };
1997 pub const drain_and_flush: @This() = .{ .drain = true, .rebase = .flush };
1908 };1998 };
19091999
1910 const is_eos = expected_size == max_size or smith.eosWeightedSimple(7, 1);2000 const is_eos = expected_size == max_size or smith.eosWeightedSimple(7, 1);
1911 var op: Op = if (!is_eos) smith.valueWeighted(Op, &.{2001 var op: Op = if (!is_eos) smith.valueWeighted(Op, &.{
1912 .value(Op, .add_vec_only, 6),2002 .value(Op, .add_vec_only, 5),
1913 .value(Op, .add_vec_and_drain, 1),2003 .value(Op, .add_vec_and_drain, 1),
1914 .value(Op, .drain_and_rebase, 1),2004 .value(Op, .drain_and_rebase, 1),
2005 .value(Op, .drain_and_flush, 1),
1915 }) else .drain_only;2006 }) else .drain_only;
19162007
1917 if (op.add_vec) {2008 if (op.add_vec) {
...@@ -1965,16 +2056,20 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)...@@ -1965,16 +2056,20 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)
1965 vecs_n = 0;2056 vecs_n = 0;
1966 }2057 }
19672058
1968 if (op.rebase) {2059 switch (op.rebase) {
1969 const capacity = smith.valueRangeAtMost(u32, 0, raw_buf_len);2060 .none => {},
1970 const preserve = smith.valueRangeAtMost(u32, 0, raw_buf_len - capacity);2061 .rebase => {
1971 try raw.writer.rebase(preserve, capacity);2062 const capacity = smith.valueRangeAtMost(u32, 0, raw_buf_len);
2063 const preserve = smith.valueRangeAtMost(u32, 0, raw_buf_len - capacity);
2064 try raw.writer.rebase(preserve, capacity);
2065 },
2066 .flush => try raw.writer.flush(),
1972 }2067 }
19732068
1974 if (is_eos) break;2069 if (is_eos) break;
1975 }2070 }
19762071
1977 try raw.writer.flush();2072 try raw.finish();
1978 try output.writer.flush();2073 try output.writer.flush();
19792074
1980 try std.testing.expectEqual(.end, output.state);2075 try std.testing.expectEqual(.end, output.state);
...@@ -1997,6 +2092,7 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)...@@ -1997,6 +2092,7 @@ fn testFuzzedRawInput(data_buf: *const [4 * 65536]u8, smith: *std.testing.Smith)
19972092
1998/// Only performs huffman compression on data, does no matching.2093/// Only performs huffman compression on data, does no matching.
1999pub const Huffman = struct {2094pub const Huffman = struct {
2095 /// After `finish` is called, all vtable calls with result in `error.WriteFailed`.
2000 writer: Writer,2096 writer: Writer,
2001 bit_writer: BitWriter,2097 bit_writer: BitWriter,
2002 hasher: flate.Container.Hasher,2098 hasher: flate.Container.Hasher,
...@@ -2026,12 +2122,6 @@ pub const Huffman = struct {...@@ -2026,12 +2122,6 @@ pub const Huffman = struct {
2026 }2122 }
20272123
2028 fn drain(w: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {2124 fn drain(w: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {
2029 {
2030 //std.debug.print("drain {} (buffered)", .{w.buffered().len});
2031 //for (data) |d| std.debug.print("\n\t+ {}", .{d.len});
2032 //std.debug.print(" x {}\n\n", .{splat});
2033 }
2034
2035 const h: *Huffman = @fieldParentPtr("writer", w);2125 const h: *Huffman = @fieldParentPtr("writer", w);
2036 const min_block = @min(w.buffer.len, max_tokens);2126 const min_block = @min(w.buffer.len, max_tokens);
2037 const pattern = data[data.len - 1];2127 const pattern = data[data.len - 1];
...@@ -2238,9 +2328,15 @@ pub const Huffman = struct {...@@ -2238,9 +2328,15 @@ pub const Huffman = struct {
2238 }2328 }
22392329
2240 fn flush(w: *Writer) Writer.Error!void {2330 fn flush(w: *Writer) Writer.Error!void {
2241 defer w.* = .failing;2331 errdefer w.* = .failing;
2242 const h: *Huffman = @fieldParentPtr("writer", w);2332 const h: *Huffman = @fieldParentPtr("writer", w);
2243 try Huffman.rebaseInner(w, 0, w.buffer.len, true);2333 try Huffman.rebaseInner(w, 0, w.buffer.len, false);
2334 try h.bit_writer.byteAlignBlocks();
2335 }
2336
2337 fn finish(h: *Huffman) Writer.Error!void {
2338 defer h.writer = .failing;
2339 try Huffman.rebaseInner(&h.writer, 0, h.writer.buffer.len, true);
2244 try h.bit_writer.output.rebase(0, 1);2340 try h.bit_writer.output.rebase(0, 1);
2245 h.bit_writer.byteAlign();2341 h.bit_writer.byteAlign();
2246 try h.hasher.writeFooter(h.bit_writer.output);2342 try h.hasher.writeFooter(h.bit_writer.output);
...@@ -2359,9 +2455,6 @@ pub const Huffman = struct {...@@ -2359,9 +2455,6 @@ pub const Huffman = struct {
2359 break :n stored_align_bits + @as(u32, 32) + @as(u32, bytes) * 8;2455 break :n stored_align_bits + @as(u32, 32) + @as(u32, bytes) * 8;
2360 };2456 };
23612457
2362 //std.debug.print("@ {}{{{}}} ", .{ h.bit_writer.output.end, h.bit_writer.buffered_n });
2363 //std.debug.print("#{} -> s {} f {} d {}\n", .{ bytes, stored_bitsize, fixed_bitsize, dynamic_bitsize });
2364
2365 if (stored_bitsize <= @min(dynamic_bitsize, fixed_bitsize)) {2458 if (stored_bitsize <= @min(dynamic_bitsize, fixed_bitsize)) {
2366 try h.bit_writer.write(BlockHeader.int(.{ .kind = .stored, .final = eos }), 3);2459 try h.bit_writer.write(BlockHeader.int(.{ .kind = .stored, .final = eos }), 3);
2367 try h.bit_writer.output.rebase(0, 5);2460 try h.bit_writer.output.rebase(0, 5);
...@@ -2434,19 +2527,21 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)...@@ -2434,19 +2527,21 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)
2434 const Op = packed struct {2527 const Op = packed struct {
2435 drain: bool = false,2528 drain: bool = false,
2436 add_vec: bool = false,2529 add_vec: bool = false,
2437 rebase: bool = false,2530 rebase: enum(u2) { none, rebase, flush } = .none,
24382531
2439 pub const drain_only: @This() = .{ .drain = true };2532 pub const drain_only: @This() = .{ .drain = true };
2440 pub const add_vec_only: @This() = .{ .add_vec = true };2533 pub const add_vec_only: @This() = .{ .add_vec = true };
2441 pub const add_vec_and_drain: @This() = .{ .add_vec = true, .drain = true };2534 pub const add_vec_and_drain: @This() = .{ .add_vec = true, .drain = true };
2442 pub const drain_and_rebase: @This() = .{ .drain = true, .rebase = true };2535 pub const drain_and_rebase: @This() = .{ .drain = true, .rebase = .rebase };
2536 pub const drain_and_flush: @This() = .{ .drain = true, .rebase = .flush };
2443 };2537 };
24442538
2445 const is_eos = expected_size == max_size or smith.eosWeightedSimple(7, 1);2539 const is_eos = expected_size == max_size or smith.eosWeightedSimple(7, 1);
2446 var op: Op = if (!is_eos) smith.valueWeighted(Op, &.{2540 var op: Op = if (!is_eos) smith.valueWeighted(Op, &.{
2447 .value(Op, .add_vec_only, 6),2541 .value(Op, .add_vec_only, 5),
2448 .value(Op, .add_vec_and_drain, 1),2542 .value(Op, .add_vec_and_drain, 1),
2449 .value(Op, .drain_and_rebase, 1),2543 .value(Op, .drain_and_rebase, 1),
2544 .value(Op, .drain_and_flush, 1),
2450 }) else .drain_only;2545 }) else .drain_only;
24512546
2452 if (op.add_vec) {2547 if (op.add_vec) {
...@@ -2517,7 +2612,7 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)...@@ -2517,7 +2612,7 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)
2517 vecs_n = 0;2612 vecs_n = 0;
2518 }2613 }
25192614
2520 if (op.rebase) {2615 if (op.rebase != .none) {
2521 const capacity = smith.valueRangeAtMost(u32, 0, h_buf_len);2616 const capacity = smith.valueRangeAtMost(u32, 0, h_buf_len);
2522 const preserve = smith.valueRangeAtMost(u32, 0, h_buf_len - capacity);2617 const preserve = smith.valueRangeAtMost(u32, 0, h_buf_len - capacity);
25232618
...@@ -2525,9 +2620,14 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)...@@ -2525,9 +2620,14 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)
2525 h.writer.buffered().len,2620 h.writer.buffered().len,
2526 flate_w.buffered().len,2621 flate_w.buffered().len,
2527 false,2622 false,
2528 );2623 ) + @as(usize, 8) * @intFromBool(op.rebase == .flush); // Overhead from byte alignment
2529 h.writer.rebase(preserve, capacity) catch2624 switch (op.rebase) {
2530 return if (max_space <= flate_w.buffer.len) error.OverheadTooLarge else {};2625 .none => unreachable,
2626 .rebase => h.writer.rebase(preserve, capacity) catch
2627 return if (max_space <= flate_w.buffer.len) error.OverheadTooLarge else {},
2628 .flush => h.writer.flush() catch
2629 return if (max_space <= flate_w.buffer.len) error.OverheadTooLarge else {},
2630 }
2531 if (flate_w.buffered().len > max_space) return error.OverheadTooLarge;2631 if (flate_w.buffered().len > max_space) return error.OverheadTooLarge;
2532 }2632 }
25332633
...@@ -2539,8 +2639,7 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)...@@ -2539,8 +2639,7 @@ fn testFuzzedHuffmanInput(fbufs: *const [2][65536]u8, smith: *std.testing.Smith)
2539 flate_w.buffered().len,2639 flate_w.buffered().len,
2540 true,2640 true,
2541 );2641 );
2542 h.writer.flush() catch2642 h.finish() catch return if (max_space <= flate_w.buffer.len) error.OverheadTooLarge else {};
2543 return if (max_space <= flate_w.buffer.len) error.OverheadTooLarge else {};
2544 if (flate_w.buffered().len > max_space) return error.OverheadTooLarge;2643 if (flate_w.buffered().len > max_space) return error.OverheadTooLarge;
25452644
2546 try testingCheckDecompressedMatches(flate_w.buffered(), expected_size, expected_hash);2645 try testingCheckDecompressedMatches(flate_w.buffered(), expected_size, expected_hash);
src/Package/Fetch.zig+1-1
...@@ -447,7 +447,7 @@ pub const JobQueue = struct {...@@ -447,7 +447,7 @@ pub const JobQueue = struct {
447447
448 // intentionally omitting the pointless trailer448 // intentionally omitting the pointless trailer
449 //try archiver.finish();449 //try archiver.finish();
450 compress.writer.flush() catch |err| switch (err) {450 compress.finish() catch |err| switch (err) {
451 error.WriteFailed => return file_writer.err.?,451 error.WriteFailed => return file_writer.err.?,
452 };452 };
453 try file_writer.flush();453 try file_writer.flush();