authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-02-17 18:49:49+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-17 15:31:13-08:00
log3e8cb153ea44ad1f403fbf6d23a67c839df57925
tree0a196079d351a3a64b3d19818064801a8323a732
parent79e99c401ca32031d29aff77e43758ca0b67d5d9

fix flate regression

Until now literal and distance code lengths where treated as two different arrays. But according to rfc they can overlap: The code length repeat codes can cross from HLIT + 257 to the HDIST + 1 code lengths. In other words, all code lengths form a single sequence of HLIT + HDIST + 258 values. Now code lengths are decoded in single array which is then split to literal and distance part.

3 files changed, 42 insertions(+), 20 deletions(-)

lib/std/compress/flate/inflate.zig+25-20
......@@ -157,30 +157,23 @@ pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
157157 var cl_dec: hfd.CodegenDecoder = .{};
158158 try cl_dec.generate(&cl_lens);
159159
160 // literal code lengths
161 var lit_lens = [_]u4{0} ** (286);
160 // decoded code lengths
161 var dec_lens = [_]u4{0} ** (286 + 30);
162162 var pos: usize = 0;
163 while (pos < hlit) {
163 while (pos < hlit + hdist) {
164164 const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
165165 try self.bits.shift(sym.code_bits);
166 pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos);
166 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
167167 }
168 if (pos > hlit)
168 if (pos > hlit + hdist) {
169169 return error.InvalidDynamicBlockHeader;
170
171 // distance code lenths
172 var dst_lens = [_]u4{0} ** (30);
173 pos = 0;
174 while (pos < hdist) {
175 const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176 try self.bits.shift(sym.code_bits);
177 pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos);
178170 }
179 if (pos > hdist)
180 return error.InvalidDynamicBlockHeader;
181171
182 try self.lit_dec.generate(&lit_lens);
183 try self.dst_dec.generate(&dst_lens);
172 // literal code lengts to literal decoder
173 try self.lit_dec.generate(dec_lens[0..hlit]);
174
175 // distance code lengths to distance decoder
176 try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
184177 }
185178
186179 // Decode code length symbol to code length. Writes decoded length into
......@@ -496,7 +489,7 @@ test "flate.Inflate fuzzing tests" {
496489 .{ .input = "puff14", .err = error.EndOfStream },
497490 .{ .input = "puff15", .err = error.IncompleteHuffmanTree },
498491 .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
499 .{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25
492 .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
500493 .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
501494 .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
502495 .{ .input = "fuzz3", .err = error.InvalidMatch },
......@@ -506,8 +499,8 @@ test "flate.Inflate fuzzing tests" {
506499 .{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
507500 .{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
508501 .{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
509 .{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35
510 .{ .input = "puff24", .err = error.InvalidDynamicBlockHeader },
502 .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
503 .{ .input = "puff24", .err = error.IncompleteHuffmanTree },
511504 .{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
512505 .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
513506 .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
......@@ -527,3 +520,15 @@ test "flate.Inflate fuzzing tests" {
527520 }
528521 }
529522}
523
524test "flate bug 18966" {
525 const input = @embedFile("testdata/fuzz/bug_18966.input");
526 const expect = @embedFile("testdata/fuzz/bug_18966.expect");
527
528 var in = std.io.fixedBufferStream(input);
529 var out = std.ArrayList(u8).init(testing.allocator);
530 defer out.deinit();
531
532 try decompress(.gzip, in.reader(), out.writer());
533 try testing.expectEqualStrings(expect, out.items);
534}
lib/std/compress/flate/testdata/fuzz/bug_18966.expect created+17
......@@ -0,0 +1,17 @@
1conditions,correction,exchange,id,participant_timestamp,price,sequence_number,sip_timestamp,size,tape,trf_id,trf_timestamp
2,,1,,0,9.92,1929969624145677,1063224024145000000,200,1,,0
3,,1,,0,9.86,1929967211195252,1063221611195000000,100,1,,0
4,,1,,0,9.86,1929967094672373,1063221494672000000,200,1,,0
5,,1,,0,9.85,1929967050174259,1063221450174000000,100,1,,0
6,,1,,0,9.85,1929967013672246,1063221413672000000,200,1,,0
7,,1,,0,9.85,1929966984671861,1063221384671000000,200,1,,0
8,,1,,0,9.9,1929966976177115,1063221376177000000,200,1,,0
9,,1,,0,9.85,1929966961186410,1063221361186000000,1000,1,,0
10,,1,,0,9.85,1929957130426170,1063211530426000000,100,1,,0
11,,1,,0,9.85,1929956938455804,1063211338455000000,100,1,,0
12,,1,,0,9.85,1929956786452149,1063211186452000000,300,1,,0
13,,1,,0,9.85,1929956658456522,1063211058456000000,500,1,,0
14,,1,,0,9.85,1929956650952287,1063211050952000000,300,1,,0
15,,1,,0,9.85,1929956528452578,1063210928452000000,1500,1,,0
16,,1,,0,9.85,1929950990948195,1063205390948000000,100,1,,0
17,,1,,0,9.85,1929947489432428,1063201889432000000,100,1,,0
lib/std/compress/flate/testdata/fuzz/bug_18966.input created
Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/bug_18966.input differ