authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-02-18 18:17:15-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-25 20:05:48+01:00
logbb304796f466b4fd15a4adac7ffbb81ace95d2a4
treee4825a8af25fcb0063b5a29675e15adf41c2db0d
parent333055ced72ae73c1ec74a66aa9436291d8b7d1d

optimize flate decompression

Matches now use memcpy and memset when possible. Block loops have been rewritten to be more optimizer friendly. Reworks Symbol and HuffmanDecoder * Symbol now only includes the value and number of code bits. decodeSymbol returns only the value. * HuffmanDecoder now takes the regular bits instead of the reversed. * Code table construction now uses buckets instead of sorting. * For linked codes, the value field of Symbol is now used as the next index. The actual value is the element index. * InvalidCode is now detected only once with a special linked index. Performance is 39.7% faster than before and 1.1% faster than gzip using a sample created from compressing a tar of the src directory.

1 files changed, 170 insertions(+), 189 deletions(-)

lib/std/compress/flate/Decompress.zig+170-189
......@@ -229,11 +229,11 @@ fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usiz
229229 }
230230}
231231
232fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
232fn decodeSymbol(self: *Decompress, decoder: anytype) !u16 {
233233 // Maximum code len is 15 bits.
234 const sym = try decoder.find(@bitReverse(try self.peekIntBitsShort(u15)));
234 const sym = try decoder.find(try self.peekIntBitsShort(u15));
235235 try self.tossBitsShort(sym.code_bits);
236 return sym;
236 return sym.value;
237237}
238238
239239fn streamDirect(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
......@@ -348,10 +348,10 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader
348348 var dec_lens: [286 + 30]u4 = @splat(0);
349349 var pos: usize = 0;
350350 while (pos < hlit + hdist) {
351 const peeked = @bitReverse(try d.peekIntBitsShort(u7));
351 const peeked = try d.peekIntBitsShort(u7);
352352 const sym = try cl_dec.find(peeked);
353353 try d.tossBitsShort(sym.code_bits);
354 pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos);
354 pos += try d.dynamicCodeLength(sym.value, &dec_lens, pos);
355355 }
356356 if (pos > hlit + hdist) {
357357 return error.InvalidDynamicBlockHeader;
......@@ -383,35 +383,34 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader
383383 w.advance(n);
384384 return @intFromEnum(limit) - remaining + n;
385385 },
386 .fixed_block => {
387 while (remaining > 0) {
388 const code = try d.readFixedCode();
389 switch (code) {
390 0...255 => {
391 if (remaining != 0) {
392 @branchHint(.likely);
393 try w.writeBytePreserve(flate.history_len, @intCast(code));
394 remaining -= 1;
395 } else {
396 d.state = .{ .fixed_block_literal = @intCast(code) };
397 return @intFromEnum(limit) - remaining;
398 }
399 },
400 256 => {
401 d.state = if (d.final_block) .protocol_footer else .block_header;
402 return @intFromEnum(limit) - remaining;
403 },
404 257...285 => {
405 // Handles fixed block non literal (length) code.
406 // Length code is followed by 5 bits of distance code.
407 const length = try d.decodeLength(@intCast(code - 257));
408 continue :sw .{ .fixed_block_match = length };
409 },
410 else => return error.InvalidCode,
386 .fixed_block => while (true) {
387 // Consume bytes
388 const sym = try d.readFixedCode();
389
390 if (sym >= 256) {
391 @branchHint(.unlikely);
392
393 if (sym == 256) {
394 @branchHint(.unlikely);
395 // End
396 d.state = if (d.final_block) .protocol_footer else .block_header;
397 continue :sw d.state;
411398 }
399
400 // Match
401 const length = try d.decodeLength(@intCast(sym - 257));
402 continue :sw .{ .fixed_block_match = length };
403 }
404
405 const byte: u8 = @intCast(sym);
406 if (remaining != 0) {
407 @branchHint(.likely);
408 remaining -= 1;
409 try w.writeBytePreserve(flate.history_len, byte);
410 } else {
411 d.state = .{ .fixed_block_literal = byte };
412 return @intFromEnum(limit) - remaining;
412413 }
413 d.state = .fixed_block;
414 return @intFromEnum(limit) - remaining;
415414 },
416415 .fixed_block_literal => |symbol| {
417416 assert(remaining != 0);
......@@ -431,32 +430,35 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader
431430 return @intFromEnum(limit) - remaining;
432431 }
433432 },
434 .dynamic_block => {
435 // In larger archives most blocks are usually dynamic, so
436 // decompression performance depends on this logic.
437 var sym = try d.decodeSymbol(&d.lit_dec);
438 sym: switch (sym.kind) {
439 .literal => {
440 if (remaining != 0) {
441 @branchHint(.likely);
442 remaining -= 1;
443 try w.writeBytePreserve(flate.history_len, sym.symbol);
444 sym = try d.decodeSymbol(&d.lit_dec);
445 continue :sym sym.kind;
446 } else {
447 d.state = .{ .dynamic_block_literal = sym.symbol };
448 return @intFromEnum(limit) - remaining;
449 }
450 },
451 .match => {
452 // Decode match backreference <length, distance>
453 const length = try d.decodeLength(@intCast(sym.symbol));
454 continue :sw .{ .dynamic_block_match = length };
455 },
456 .end_of_block => {
433 // In larger archives most blocks are usually dynamic, so
434 // decompression performance depends on this logic.
435 .dynamic_block => while (true) {
436 // Consume bytes
437 const sym = try d.decodeSymbol(&d.lit_dec);
438
439 if (sym >= 256) {
440 @branchHint(.unlikely);
441
442 if (sym == 256) {
443 @branchHint(.unlikely);
444 // End
457445 d.state = if (d.final_block) .protocol_footer else .block_header;
458446 continue :sw d.state;
459 },
447 }
448
449 // Match
450 const length = try d.decodeLength(@intCast(sym - 257));
451 continue :sw .{ .dynamic_block_match = length };
452 }
453
454 const byte: u8 = @intCast(sym);
455 if (remaining != 0) {
456 @branchHint(.likely);
457 remaining -= 1;
458 try w.writeBytePreserve(flate.history_len, byte);
459 } else {
460 d.state = .{ .dynamic_block_literal = byte };
461 return @intFromEnum(limit) - remaining;
460462 }
461463 },
462464 .dynamic_block_literal => |symbol| {
......@@ -470,7 +472,7 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader
470472 @branchHint(.likely);
471473 remaining -= length;
472474 const dsm = try d.decodeSymbol(&d.dst_dec);
473 const distance = try d.decodeDistance(@intCast(dsm.symbol));
475 const distance = try d.decodeDistance(@intCast(dsm));
474476 try writeMatch(w, length, distance);
475477 continue :sw .dynamic_block;
476478 } else {
......@@ -501,17 +503,25 @@ fn streamInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader
501503/// back from current write position, and `length` of bytes.
502504fn writeMatch(w: *Writer, length: u16, distance: u16) !void {
503505 if (w.end < distance) return error.InvalidMatch;
504 if (length < token.min_length) return error.InvalidMatch;
505 if (length > token.max_length) return error.InvalidMatch;
506 if (distance < token.min_distance) return error.InvalidMatch;
507 if (distance > token.max_distance) return error.InvalidMatch;
506 assert(length >= token.min_length);
507 assert(length <= token.max_length);
508 assert(distance >= token.min_distance);
509 assert(distance <= token.max_distance);
508510
509511 // This is not a @memmove; it intentionally repeats patterns caused by
510512 // iterating one byte at a time.
511513 const dest = try w.writableSlicePreserve(flate.history_len, length);
512514 const end = dest.ptr - w.buffer.ptr;
513515 const src = w.buffer[end - distance ..][0..length];
514 for (dest, src) |*d, s| d.* = s;
516 if (distance >= length) {
517 @memcpy(dest, src);
518 } else if (distance == 1) {
519 // Repeating copy of single byte
520 @memset(dest, src[0]);
521 } else {
522 // Repeating copy of multiple bytes
523 for (dest, src) |*d, s| d.* = s;
524 }
515525}
516526
517527fn peekBits(d: *Decompress, n: u4) !u16 {
......@@ -603,31 +613,9 @@ fn readFixedCode(d: *Decompress) !u16 {
603613 };
604614}
605615
606pub const Symbol = packed struct {
607 pub const Kind = enum(u2) {
608 literal,
609 end_of_block,
610 match,
611 };
612
613 symbol: u8 = 0, // symbol from alphabet
616pub const Symbol = packed struct(u16) {
617 value: u12 = 0,
614618 code_bits: u4 = 0, // number of bits in code 0-15
615 kind: Kind = .literal,
616
617 code: u16 = 0, // huffman code of the symbol
618 next: u16 = 0, // pointer to the next symbol in linked list
619 // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
620
621 // Sorting less than function.
622 pub fn asc(_: void, a: Symbol, b: Symbol) bool {
623 if (a.code_bits == b.code_bits) {
624 if (a.kind == b.kind) {
625 return a.symbol < b.symbol;
626 }
627 return @intFromEnum(a.kind) < @intFromEnum(b.kind);
628 }
629 return a.code_bits < b.code_bits;
630 }
631619};
632620
633621pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
......@@ -646,69 +634,85 @@ pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
646634/// Small lookup table is optimization for faster search.
647635/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
648636/// with difference that we here use statically allocated arrays.
649///
650637fn HuffmanDecoder(
651638 comptime alphabet_size: u16,
652639 comptime max_code_bits: u4,
653640 comptime lookup_bits: u4,
654641) type {
655642 const lookup_shift = max_code_bits - lookup_bits;
643 const lookup_mask = (1 << lookup_bits) - 1;
656644
657645 return struct {
658 // all symbols in alaphabet, sorted by code_len, symbol
659 symbols: [alphabet_size]Symbol = undefined,
660646 // lookup table code -> symbol
647 // for values with code_bits == 0, symbol is the index of the first node in linked
648 // if the index of the first node is 0xfff, it is an invalid code
661649 lookup: [1 << lookup_bits]Symbol = undefined,
650 linked: if (lookup_bits == max_code_bits) void else [alphabet_size]struct {
651 // sym.value is the next index in linked where the current index ends the chain
652 // the actual symbol is this nodes's index
653 sym: Symbol,
654 code: u16,
655 } = undefined,
662656
663657 const Self = @This();
664658
659 fn reverseIdx(idx: usize) u16 {
660 return @bitReverse(@as(@Int(.unsigned, lookup_bits), @intCast(idx)));
661 }
662
665663 /// Generates symbols and lookup tables from list of code lens for each symbol.
666664 pub fn generate(self: *Self, lens: []const u4) !void {
667665 try checkCompleteness(lens);
668666
669 // init alphabet with code_bits
670 for (self.symbols, 0..) |_, i| {
671 const cb: u4 = if (i < lens.len) lens[i] else 0;
672 self.symbols[i] = if (i < 256)
673 .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
674 else if (i == 256)
675 .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
676 else
677 .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
678 }
679 std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
680
681 // reset lookup table
682 for (0..self.lookup.len) |i| {
683 self.lookup[i] = .{};
667 var buckets: [1 + @as(usize, max_code_bits)][alphabet_size]Symbol = undefined;
668 var bucket_len: [buckets.len]u16 = @splat(0);
669 for (0.., lens) |symbol, bits| {
670 buckets[bits][bucket_len[bits]] = .{
671 .value = @intCast(symbol),
672 .code_bits = bits,
673 };
674 bucket_len[bits] += 1;
684675 }
685676
686 // assign code to symbols
687 // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
688677 var code: u16 = 0;
689678 var idx: u16 = 0;
690 for (&self.symbols, 0..) |*sym, pos| {
691 if (sym.code_bits == 0) continue; // skip unused
692 sym.code = code;
693
694 const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
695 const next_idx = next_code >> lookup_shift;
696
697 if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
698 if (sym.code_bits <= lookup_bits) {
699 // fill small lookup table
700 for (idx..next_idx) |j|
701 self.lookup[j] = sym.*;
702 } else {
703 // insert into linked table starting at root
704 const root = &self.lookup[idx];
705 const root_next = root.next;
706 root.next = @intCast(pos);
707 sym.next = root_next;
679 for (1..lookup_bits + 1) |bits| {
680 const inc = @as(u16, 1) << @intCast(max_code_bits - bits);
681 for (buckets[bits][0..bucket_len[bits]]) |lookup_sym| {
682 const next_code = code + inc;
683 const next_idx = next_code >> lookup_shift;
684 for (idx..next_idx) |i| {
685 self.lookup[reverseIdx(i)] = lookup_sym;
686 }
687 code = next_code;
688 idx = next_idx;
689 }
690 }
691 for (lookup_bits + 1..buckets.len) |bits| {
692 const inc = @as(u16, 1) << @intCast(max_code_bits - bits);
693 for (buckets[bits][0..bucket_len[bits]]) |linked_sym| {
694 const next_code = code + inc;
695 const next_idx = next_code >> lookup_shift;
696
697 const ri = reverseIdx(idx);
698 const next: Symbol = .{
699 .value = self.lookup[ri].value,
700 .code_bits = linked_sym.code_bits,
701 };
702 self.linked[linked_sym.value] = .{
703 .sym = next,
704 .code = @bitReverse(@as(@Int(.unsigned, max_code_bits), @intCast(code))),
705 };
706 self.lookup[ri] = .{ .value = linked_sym.value, .code_bits = 0 };
707
708 code = next_code;
709 idx = next_idx;
708710 }
711 }
709712
710 idx = next_idx;
711 code = next_code;
713 // Invalid codes
714 for (idx..self.lookup.len) |i| {
715 self.lookup[reverseIdx(i)] = .{ .value = 0xfff, .code_bits = 0 };
712716 }
713717 }
714718
......@@ -748,23 +752,25 @@ fn HuffmanDecoder(
748752 /// Finds symbol for lookup table code.
749753 pub fn find(self: *Self, code: u16) !Symbol {
750754 // try to find in lookup table
751 const idx = code >> lookup_shift;
755 const idx = code & lookup_mask;
752756 const sym = self.lookup[idx];
753757 if (sym.code_bits != 0) return sym;
754758 // if not use linked list of symbols with same prefix
755 return self.findLinked(code, sym.next);
759 return self.findLinked(code, sym.value);
756760 }
757761
758762 fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
763 if (start == 0xfff) return error.InvalidCode;
764 if (lookup_bits == max_code_bits) unreachable;
759765 var pos = start;
760 while (pos > 0) {
761 const sym = self.symbols[pos];
762 const shift = max_code_bits - sym.code_bits;
766 while (true) {
767 const node = self.linked[pos];
768 const shift = -%node.sym.code_bits;
763769 // compare code_bits number of upper bits
764 if ((code ^ sym.code) >> shift == 0) return sym;
765 pos = sym.next;
770 if ((code ^ node.code) << shift == 0)
771 return .{ .value = @intCast(pos), .code_bits = node.sym.code_bits };
772 pos = node.sym.value;
766773 }
767 return error.InvalidCode;
768774 }
769775 };
770776}
......@@ -775,74 +781,49 @@ test "init/find" {
775781 var h: CodegenDecoder = .{};
776782 try h.generate(&code_lens);
777783
778 const expected = [_]struct {
779 sym: Symbol,
780 code: u16,
781 }{
782 .{
783 .code = 0b00_00000,
784 .sym = .{ .symbol = 3, .code_bits = 2 },
785 },
786 .{
787 .code = 0b01_00000,
788 .sym = .{ .symbol = 18, .code_bits = 2 },
789 },
790 .{
791 .code = 0b100_0000,
792 .sym = .{ .symbol = 1, .code_bits = 3 },
793 },
794 .{
795 .code = 0b101_0000,
796 .sym = .{ .symbol = 4, .code_bits = 3 },
797 },
798 .{
799 .code = 0b110_0000,
800 .sym = .{ .symbol = 17, .code_bits = 3 },
801 },
802 .{
803 .code = 0b1110_000,
804 .sym = .{ .symbol = 0, .code_bits = 4 },
805 },
806 .{
807 .code = 0b1111_000,
808 .sym = .{ .symbol = 16, .code_bits = 4 },
809 },
810 };
811
812 // unused symbols
813 for (0..12) |i| {
814 try testing.expectEqual(0, h.symbols[i].code_bits);
815 }
816 // used, from index 12
817 for (expected, 12..) |e, i| {
818 try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
819 try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
820 const sym_from_code = try h.find(e.code);
821 try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
822 }
823
824784 // All possible codes for each symbol.
825785 // Lookup table has 126 elements, to cover all possible 7 bit codes.
826786 for (0b0000_000..0b0100_000) |c| // 0..32 (32)
827 try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
787 try testing.expectEqual(
788 Symbol{ .value = 3, .code_bits = 2 },
789 try h.find(@bitReverse(@as(u7, @intCast(c)))),
790 );
828791
829792 for (0b0100_000..0b1000_000) |c| // 32..64 (32)
830 try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
793 try testing.expectEqual(
794 Symbol{ .value = 18, .code_bits = 2 },
795 try h.find(@bitReverse(@as(u7, @intCast(c)))),
796 );
831797
832798 for (0b1000_000..0b1010_000) |c| // 64..80 (16)
833 try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
799 try testing.expectEqual(
800 Symbol{ .value = 1, .code_bits = 3 },
801 try h.find(@bitReverse(@as(u7, @intCast(c)))),
802 );
834803
835804 for (0b1010_000..0b1100_000) |c| // 80..96 (16)
836 try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
805 try testing.expectEqual(
806 Symbol{ .value = 4, .code_bits = 3 },
807 try h.find(@bitReverse(@as(u7, @intCast(c)))),
808 );
837809
838810 for (0b1100_000..0b1110_000) |c| // 96..112 (16)
839 try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
811 try testing.expectEqual(
812 Symbol{ .value = 17, .code_bits = 3 },
813 try h.find(@bitReverse(@as(u7, @intCast(c)))),
814 );
840815
841816 for (0b1110_000..0b1111_000) |c| // 112..120 (8)
842 try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
817 try testing.expectEqual(
818 Symbol{ .value = 0, .code_bits = 4 },
819 try h.find(@bitReverse(@as(u7, @intCast(c)))),
820 );
843821
844822 for (0b1111_000..0b1_0000_000) |c| // 120...128 (8)
845 try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
823 try testing.expectEqual(
824 Symbol{ .value = 16, .code_bits = 4 },
825 try h.find(@bitReverse(@as(u7, @intCast(c)))),
826 );
846827}
847828
848829test "encode/decode literals" {
......@@ -867,8 +848,8 @@ test "encode/decode literals" {
867848 if (bits == 0) continue;
868849 for (0..1 << (max_bits - bits)) |extra| {
869850 const full = (@as(u16, code) << (max_bits - bits)) | @as(u16, @intCast(extra));
870 const symbol = try decoder.find(full);
871 try testing.expectEqual(i, symbol.symbol);
851 const symbol = try decoder.find(@bitReverse(@as(u5, @intCast(full))));
852 try testing.expectEqual(i, symbol.value);
872853 try testing.expectEqual(bits, symbol.code_bits);
873854 }
874855 }