authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-12 22:04:07+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:06+11:00
loga53cf299a6a22422d734d54b6abed4ff8b6473c5
tree5b432d404112e31fe32aca2f830811498c6ef4a3
parent5a31fc2014ed6c1d806d08f1393e10b597ec427d

std.compress.zstandard: add error condition to ring buffer decoding

Previously `executeSequenceRingBuffer()` would not verify the offset against the number of bytes already decoded, so it would happily copy garbage bytes rather than return an error before the window was filled. To fix this a new `written_count` is added to the decode state that tracks the total number of bytes decoded.

1 files changed, 19 insertions(+), 1 deletions(-)

lib/std/compress/zstandard/decode/block.zig+19-1
...@@ -45,6 +45,7 @@ pub const DecodeState = struct {...@@ -45,6 +45,7 @@ pub const DecodeState = struct {
45 huffman_tree: ?LiteralsSection.HuffmanTree,45 huffman_tree: ?LiteralsSection.HuffmanTree,
4646
47 literal_written_count: usize,47 literal_written_count: usize,
48 written_count: usize = 0,
4849
49 fn StateData(comptime max_accuracy_log: comptime_int) type {50 fn StateData(comptime max_accuracy_log: comptime_int) type {
50 return struct {51 return struct {
...@@ -84,6 +85,8 @@ pub const DecodeState = struct {...@@ -84,6 +85,8 @@ pub const DecodeState = struct {
84 .literal_stream_reader = undefined,85 .literal_stream_reader = undefined,
85 .literal_stream_index = undefined,86 .literal_stream_index = undefined,
86 .huffman_tree = null,87 .huffman_tree = null,
88
89 .written_count = 0,
87 };90 };
88 }91 }
8992
...@@ -296,6 +299,7 @@ pub const DecodeState = struct {...@@ -296,6 +299,7 @@ pub const DecodeState = struct {
296 // NOTE: we ignore the usage message for std.mem.copy and copy with dest.ptr >= src.ptr299 // NOTE: we ignore the usage message for std.mem.copy and copy with dest.ptr >= src.ptr
297 // to allow repeats300 // to allow repeats
298 std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);301 std.mem.copy(u8, dest[write_pos + sequence.literal_length ..], dest[copy_start..copy_end]);
302 self.written_count += sequence.match_length;
299 }303 }
300304
301 fn executeSequenceRingBuffer(305 fn executeSequenceRingBuffer(
...@@ -303,7 +307,8 @@ pub const DecodeState = struct {...@@ -303,7 +307,8 @@ pub const DecodeState = struct {
303 dest: *RingBuffer,307 dest: *RingBuffer,
304 sequence: Sequence,308 sequence: Sequence,
305 ) (error{MalformedSequence} || DecodeLiteralsError)!void {309 ) (error{MalformedSequence} || DecodeLiteralsError)!void {
306 if (sequence.offset > dest.data.len) return error.MalformedSequence;310 if (sequence.offset > @min(dest.data.len, self.written_count + sequence.literal_length))
311 return error.MalformedSequence;
307312
308 try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);313 try self.decodeLiteralsRingBuffer(dest, sequence.literal_length);
309 const copy_start = dest.write_index + dest.data.len - sequence.offset;314 const copy_start = dest.write_index + dest.data.len - sequence.offset;
...@@ -311,6 +316,7 @@ pub const DecodeState = struct {...@@ -311,6 +316,7 @@ pub const DecodeState = struct {
311 // TODO: would std.mem.copy and figuring out dest slice be better/faster?316 // TODO: would std.mem.copy and figuring out dest slice be better/faster?
312 for (copy_slice.first) |b| dest.writeAssumeCapacity(b);317 for (copy_slice.first) |b| dest.writeAssumeCapacity(b);
313 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);318 for (copy_slice.second) |b| dest.writeAssumeCapacity(b);
319 self.written_count += sequence.match_length;
314 }320 }
315321
316 const DecodeSequenceError = error{322 const DecodeSequenceError = error{
...@@ -444,6 +450,7 @@ pub const DecodeState = struct {...@@ -444,6 +450,7 @@ pub const DecodeState = struct {
444 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];450 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
445 std.mem.copy(u8, dest, literal_data);451 std.mem.copy(u8, dest, literal_data);
446 self.literal_written_count += len;452 self.literal_written_count += len;
453 self.written_count += len;
447 },454 },
448 .rle => {455 .rle => {
449 var i: usize = 0;456 var i: usize = 0;
...@@ -451,6 +458,7 @@ pub const DecodeState = struct {...@@ -451,6 +458,7 @@ pub const DecodeState = struct {
451 dest[i] = self.literal_streams.one[0];458 dest[i] = self.literal_streams.one[0];
452 }459 }
453 self.literal_written_count += len;460 self.literal_written_count += len;
461 self.written_count += len;
454 },462 },
455 .compressed, .treeless => {463 .compressed, .treeless => {
456 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;464 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
...@@ -497,6 +505,7 @@ pub const DecodeState = struct {...@@ -497,6 +505,7 @@ pub const DecodeState = struct {
497 }505 }
498 }506 }
499 self.literal_written_count += len;507 self.literal_written_count += len;
508 self.written_count += len;
500 },509 },
501 }510 }
502 }511 }
...@@ -516,6 +525,7 @@ pub const DecodeState = struct {...@@ -516,6 +525,7 @@ pub const DecodeState = struct {
516 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];525 const literal_data = self.literal_streams.one[self.literal_written_count..literals_end];
517 dest.writeSliceAssumeCapacity(literal_data);526 dest.writeSliceAssumeCapacity(literal_data);
518 self.literal_written_count += len;527 self.literal_written_count += len;
528 self.written_count += len;
519 },529 },
520 .rle => {530 .rle => {
521 var i: usize = 0;531 var i: usize = 0;
...@@ -523,6 +533,7 @@ pub const DecodeState = struct {...@@ -523,6 +533,7 @@ pub const DecodeState = struct {
523 dest.writeAssumeCapacity(self.literal_streams.one[0]);533 dest.writeAssumeCapacity(self.literal_streams.one[0]);
524 }534 }
525 self.literal_written_count += len;535 self.literal_written_count += len;
536 self.written_count += len;
526 },537 },
527 .compressed, .treeless => {538 .compressed, .treeless => {
528 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;539 // const written_bytes_per_stream = (literals.header.regenerated_size + 3) / 4;
...@@ -565,6 +576,7 @@ pub const DecodeState = struct {...@@ -565,6 +576,7 @@ pub const DecodeState = struct {
565 }576 }
566 }577 }
567 self.literal_written_count += len;578 self.literal_written_count += len;
579 self.written_count += len;
568 },580 },
569 }581 }
570 }582 }
...@@ -612,6 +624,7 @@ pub fn decodeBlock(...@@ -612,6 +624,7 @@ pub fn decodeBlock(
612 const data = src[0..block_size];624 const data = src[0..block_size];
613 std.mem.copy(u8, dest[written_count..], data);625 std.mem.copy(u8, dest[written_count..], data);
614 consumed_count.* += block_size;626 consumed_count.* += block_size;
627 decode_state.written_count += block_size;
615 return block_size;628 return block_size;
616 },629 },
617 .rle => {630 .rle => {
...@@ -622,6 +635,7 @@ pub fn decodeBlock(...@@ -622,6 +635,7 @@ pub fn decodeBlock(
622 dest[write_pos] = src[0];635 dest[write_pos] = src[0];
623 }636 }
624 consumed_count.* += 1;637 consumed_count.* += 1;
638 decode_state.written_count += block_size;
625 return block_size;639 return block_size;
626 },640 },
627 .compressed => {641 .compressed => {
...@@ -712,6 +726,7 @@ pub fn decodeBlockRingBuffer(...@@ -712,6 +726,7 @@ pub fn decodeBlockRingBuffer(
712 const data = src[0..block_size];726 const data = src[0..block_size];
713 dest.writeSliceAssumeCapacity(data);727 dest.writeSliceAssumeCapacity(data);
714 consumed_count.* += block_size;728 consumed_count.* += block_size;
729 decode_state.written_count += block_size;
715 return block_size;730 return block_size;
716 },731 },
717 .rle => {732 .rle => {
...@@ -721,6 +736,7 @@ pub fn decodeBlockRingBuffer(...@@ -721,6 +736,7 @@ pub fn decodeBlockRingBuffer(
721 dest.writeAssumeCapacity(src[0]);736 dest.writeAssumeCapacity(src[0]);
722 }737 }
723 consumed_count.* += 1;738 consumed_count.* += 1;
739 decode_state.written_count += block_size;
724 return block_size;740 return block_size;
725 },741 },
726 .compressed => {742 .compressed => {
...@@ -814,6 +830,7 @@ pub fn decodeBlockReader(...@@ -814,6 +830,7 @@ pub fn decodeBlockReader(
814 try source.readNoEof(slice.first);830 try source.readNoEof(slice.first);
815 try source.readNoEof(slice.second);831 try source.readNoEof(slice.second);
816 dest.write_index = dest.mask2(dest.write_index + block_size);832 dest.write_index = dest.mask2(dest.write_index + block_size);
833 decode_state.written_count += block_size;
817 },834 },
818 .rle => {835 .rle => {
819 const byte = try source.readByte();836 const byte = try source.readByte();
...@@ -821,6 +838,7 @@ pub fn decodeBlockReader(...@@ -821,6 +838,7 @@ pub fn decodeBlockReader(
821 while (i < block_size) : (i += 1) {838 while (i < block_size) : (i += 1) {
822 dest.writeAssumeCapacity(byte);839 dest.writeAssumeCapacity(byte);
823 }840 }
841 decode_state.written_count += block_size;
824 },842 },
825 .compressed => {843 .compressed => {
826 const literals = try decodeLiteralsSection(block_reader, literals_buffer);844 const literals = try decodeLiteralsSection(block_reader, literals_buffer);