authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-01 14:52:41-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:28-07:00
loge05af2da131b5d9353711ffb5979c67f4bd8b5af
tree11d98d1777956110f2611635181cce9a48fe5ecd
parent685d55c1a49615b6601c02c174d1097489b0fe95

std.compress.zstd: it's compiling

5 files changed, 378 insertions(+), 434 deletions(-)

lib/std/compress/zstd.zig+1-1
......@@ -109,7 +109,7 @@ fn testExpectDecompressError(err: anyerror, compressed: []const u8) !void {
109109 in.initFixed(@constCast(compressed));
110110 var zstd_stream: Decompress = .init(&in, .{});
111111 try std.testing.expectError(error.ReadFailed, zstd_stream.reader().readRemainingArrayList(gpa, null, &out, .unlimited));
112 try std.testing.expectError(err, zstd_stream.err.?);
112 try std.testing.expectError(err, zstd_stream.err orelse {});
113113
114114 return error.TestFailed;
115115}
lib/std/compress/zstd/Decompress.zig+374-133
......@@ -11,12 +11,10 @@ state: State,
1111verify_checksum: bool,
1212err: ?Error = null,
1313
14const table_size_max = zstd.compressed_block.table_size_max;
15
1614const State = union(enum) {
1715 new_frame,
1816 in_frame: InFrame,
19 skipping_frame: u32,
17 skipping_frame: usize,
2018 end,
2119
2220 const InFrame = struct {
......@@ -31,11 +29,38 @@ pub const Options = struct {
3129};
3230
3331pub const Error = error{
32 BadMagic,
33 BlockOversize,
3434 ChecksumFailure,
35 ContentOversize,
3536 DictionaryIdFlagUnsupported,
37 EndOfStream,
38 HuffmanTreeIncomplete,
39 InvalidBitStream,
40 LiteralsBufferUndersize,
41 MalformedAccuracyLog,
3642 MalformedBlock,
43 MalformedCompressedBlock,
3744 MalformedFrame,
38 EndOfStream,
45 MalformedFseBits,
46 MalformedFseTable,
47 MalformedHuffmanTree,
48 MalformedLiteralsHeader,
49 MalformedLiteralsLength,
50 MalformedLiteralsSection,
51 MalformedSequence,
52 MissingStartBit,
53 OutputBufferUndersize,
54 InputBufferUndersize,
55 ReadFailed,
56 RepeatModeFirst,
57 ReservedBitSet,
58 ReservedBlock,
59 SequenceBufferUndersize,
60 TreelessLiteralsFirst,
61 UnexpectedEndOfLiteralStream,
62 WindowOversize,
63 WindowSizeUnknown,
3964};
4065
4166pub fn init(input: *BufferedReader, options: Options) Decompress {
......@@ -69,25 +94,32 @@ fn read(context: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit) Reader.R
6994 d.err = err;
7095 return error.ReadFailed;
7196 };
72 return readInFrame(d, bw, limit, &d.state.in_frame) catch |err| {
73 d.err = err;
74 return error.ReadFailed;
97 return readInFrame(d, bw, limit, &d.state.in_frame) catch |err| switch (err) {
98 error.ReadFailed => return error.ReadFailed,
99 error.WriteFailed => return error.WriteFailed,
100 else => |e| {
101 d.err = e;
102 return error.ReadFailed;
103 },
75104 };
76105 },
77106 .in_frame => |*in_frame| {
78 return readInFrame(d, bw, limit, in_frame) catch |err| {
79 d.err = err;
80 return error.ReadFailed;
107 return readInFrame(d, bw, limit, in_frame) catch |err| switch (err) {
108 error.ReadFailed => return error.ReadFailed,
109 error.WriteFailed => return error.WriteFailed,
110 else => |e| {
111 d.err = e;
112 return error.ReadFailed;
113 },
81114 };
82115 },
83116 .skipping_frame => |*remaining| {
84 const requested = remaining.*;
85 const n = in.discard(.limited(requested)) catch |err| {
117 const n = in.discard(.limited(remaining.*)) catch |err| {
86118 d.err = err;
87119 return error.ReadFailed;
88120 };
89 if (requested == n) d.state = .new_frame;
90 remaining.* = requested - n;
121 remaining.* -= n;
122 if (remaining.* == 0) d.state = .new_frame;
91123 return 0;
92124 },
93125 .end => return error.EndOfStream,
......@@ -115,9 +147,9 @@ fn initFrame(d: *Decompress, window_size_max: usize, magic: Frame.Magic) !void {
115147fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state: *State.InFrame) !usize {
116148 const in = d.input;
117149
118 var literal_fse_buffer: [table_size_max.literal]Table.Fse = undefined;
119 var match_fse_buffer: [table_size_max.match]Table.Fse = undefined;
120 var offset_fse_buffer: [table_size_max.offset]Table.Fse = undefined;
150 var literal_fse_buffer: [zstd.table_size_max.literal]Table.Fse = undefined;
151 var match_fse_buffer: [zstd.table_size_max.match]Table.Fse = undefined;
152 var offset_fse_buffer: [zstd.table_size_max.offset]Table.Fse = undefined;
121153 var literals_buffer: [zstd.block_size_max]u8 = undefined;
122154 var sequence_buffer: [zstd.block_size_max]u8 = undefined;
123155
......@@ -125,10 +157,10 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
125157
126158 const header_bytes = try in.takeArray(3);
127159 const block_header: Frame.Zstandard.Block.Header = @bitCast(header_bytes.*);
128 const block_size = block_header.block_size;
160 const block_size = block_header.size;
129161 if (state.frame.block_size_max < block_size) return error.BlockOversize;
130162 if (@intFromEnum(limit) < block_size) return error.OutputBufferUndersize;
131 switch (block_header.block_type) {
163 switch (block_header.type) {
132164 .raw => {
133165 try in.readAll(bw, .limited(block_size));
134166 return block_size;
......@@ -151,9 +183,10 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
151183 var bytes_written: usize = 0;
152184 {
153185 if (sequence_buffer.len < @intFromEnum(remaining))
154 return error.SequenceBufferTooSmall;
155 const seq_len = try in.readSlice(remaining.slice(&sequence_buffer));
156 var bit_stream = try ReverseBitReader.init(sequence_buffer[0..seq_len]);
186 return error.SequenceBufferUndersize;
187 const seq_slice = remaining.slice(&sequence_buffer);
188 try in.readSlice(seq_slice);
189 var bit_stream = try ReverseBitReader.init(seq_slice);
157190
158191 if (sequences_header.sequence_count > 0) {
159192 try decode.readInitialFseState(&bit_stream);
......@@ -205,16 +238,16 @@ fn readInFrame(d: *Decompress, bw: *BufferedWriter, limit: Reader.Limit, state:
205238 }
206239 }
207240
208 if (block_header.last_block) {
241 if (block_header.last) {
209242 if (state.frame.has_checksum) {
210 const expected_checksum = try in.readInt(u32, .little);
243 const expected_checksum = try in.takeInt(u32, .little);
211244 if (state.frame.hasher_opt) |*hasher| {
212245 const actual_checksum: u32 = @truncate(hasher.final());
213246 if (expected_checksum != actual_checksum) return error.ChecksumFailure;
214247 }
215248 }
216 if (d.frame.content_size) |content_size| {
217 if (content_size != d.current_frame_decompressed_size) {
249 if (state.frame.content_size) |content_size| {
250 if (content_size != state.decompressed_size) {
218251 return error.MalformedFrame;
219252 }
220253 }
......@@ -249,16 +282,16 @@ pub const Frame = struct {
249282 _,
250283
251284 pub fn kind(m: Magic) ?Kind {
252 return switch (m) {
253 .zstandard => .zstandard,
254 Skippable.magic_min...Skippable.magic_max => .skippable,
285 return switch (@intFromEnum(m)) {
286 @intFromEnum(Magic.zstandard) => .zstandard,
287 @intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => .skippable,
255288 else => null,
256289 };
257290 }
258291
259292 pub fn isSkippable(m: Magic) bool {
260 return switch (m) {
261 Skippable.magic_min...Skippable.magic_max => true,
293 return switch (@intFromEnum(m)) {
294 @intFromEnum(Skippable.magic_min)...@intFromEnum(Skippable.magic_max) => true,
262295 else => false,
263296 };
264297 }
......@@ -384,9 +417,9 @@ pub const Frame = struct {
384417 ) Decode {
385418 return .{
386419 .repeat_offsets = .{
387 zstd.compressed_block.start_repeated_offset_1,
388 zstd.compressed_block.start_repeated_offset_2,
389 zstd.compressed_block.start_repeated_offset_3,
420 zstd.start_repeated_offset_1,
421 zstd.start_repeated_offset_2,
422 zstd.start_repeated_offset_3,
390423 },
391424
392425 .offset = undefined,
......@@ -410,7 +443,7 @@ pub const Frame = struct {
410443
411444 pub const PrepareError = error{
412445 /// the (reversed) literal bitstream's first byte does not have any bits set
413 BitStreamHasNoStartBit,
446 MissingStartBit,
414447 /// `literals` is a treeless literals section and the decode state does not
415448 /// have a Huffman tree from a previous block
416449 TreelessLiteralsFirst,
......@@ -422,6 +455,8 @@ pub const Frame = struct {
422455 MalformedFseTable,
423456 /// input stream ends before all FSE tables are read
424457 EndOfStream,
458 ReadFailed,
459 InputBufferUndersize,
425460 };
426461
427462 /// Prepare the decoder to decode a compressed block. Loads the literals
......@@ -430,6 +465,7 @@ pub const Frame = struct {
430465 pub fn prepare(
431466 self: *Decode,
432467 in: *BufferedReader,
468 remaining: *Reader.Limit,
433469 literals: LiteralsSection,
434470 sequences_header: SequencesSection.Header,
435471 ) PrepareError!void {
......@@ -455,17 +491,14 @@ pub const Frame = struct {
455491 }
456492
457493 if (sequences_header.sequence_count > 0) {
458 try self.updateFseTable(in, .literal, sequences_header.literal_lengths);
459 try self.updateFseTable(in, .offset, sequences_header.offsets);
460 try self.updateFseTable(in, .match, sequences_header.match_lengths);
494 try self.updateFseTable(in, remaining, .literal, sequences_header.literal_lengths);
495 try self.updateFseTable(in, remaining, .offset, sequences_header.offsets);
496 try self.updateFseTable(in, remaining, .match, sequences_header.match_lengths);
461497 self.fse_tables_undefined = false;
462498 }
463499 }
464500
465501 /// Read initial FSE states for sequence decoding.
466 ///
467 /// Errors returned:
468 /// - `error.EndOfStream` if `bit_reader` does not contain enough bits.
469502 pub fn readInitialFseState(self: *Decode, bit_reader: *ReverseBitReader) error{EndOfStream}!void {
470503 self.literal.state = try bit_reader.readBitsNoEof(u9, self.literal.accuracy_log);
471504 self.offset.state = try bit_reader.readBitsNoEof(u8, self.offset.accuracy_log);
......@@ -490,6 +523,7 @@ pub const Frame = struct {
490523
491524 const DataType = enum { offset, match, literal };
492525
526 /// TODO: don't use `@field`
493527 fn updateState(
494528 self: *Decode,
495529 comptime choice: DataType,
......@@ -517,9 +551,11 @@ pub const Frame = struct {
517551 EndOfStream,
518552 };
519553
554 /// TODO: don't use `@field`
520555 fn updateFseTable(
521556 self: *Decode,
522 source: *BufferedReader,
557 in: *BufferedReader,
558 remaining: *Reader.Limit,
523559 comptime choice: DataType,
524560 mode: SequencesSection.Header.Mode,
525561 ) !void {
......@@ -527,28 +563,32 @@ pub const Frame = struct {
527563 switch (mode) {
528564 .predefined => {
529565 @field(self, field_name).accuracy_log =
530 @field(zstd.compressed_block.default_accuracy_log, field_name);
566 @field(zstd.default_accuracy_log, field_name);
531567
532568 @field(self, field_name).table =
533569 @field(Table, "predefined_" ++ field_name);
534570 },
535571 .rle => {
536572 @field(self, field_name).accuracy_log = 0;
537 @field(self, field_name).table = .{ .rle = try source.readByte() };
573 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
574 @field(self, field_name).table = .{ .rle = try in.takeByte() };
538575 },
539576 .fse => {
540 var bit_reader: std.io.BitReader(.little) = .init(source);
541
577 if (in.buffer.len < @intFromEnum(remaining.*)) return error.InputBufferUndersize;
578 const limited_buffer = try in.peek(@intFromEnum(remaining.*));
579 var bit_reader: BitReader = .{ .bytes = limited_buffer };
542580 const table_size = try Table.decode(
543581 &bit_reader,
544 @field(zstd.compressed_block.table_symbol_count_max, field_name),
545 @field(zstd.compressed_block.table_accuracy_log_max, field_name),
582 @field(zstd.table_symbol_count_max, field_name),
583 @field(zstd.table_accuracy_log_max, field_name),
546584 @field(self, field_name ++ "_fse_buffer"),
547585 );
548586 @field(self, field_name).table = .{
549587 .fse = @field(self, field_name ++ "_fse_buffer")[0..table_size],
550588 };
551589 @field(self, field_name).accuracy_log = std.math.log2_int_ceil(usize, table_size);
590 in.toss(bit_reader.index);
591 remaining.* = remaining.subtract(bit_reader.index).?;
552592 },
553593 .repeat => if (self.fse_tables_undefined) return error.RepeatModeFirst,
554594 }
......@@ -571,15 +611,15 @@ pub const Frame = struct {
571611 const offset_value = (@as(u32, 1) << offset_code) + try bit_reader.readBitsNoEof(u32, offset_code);
572612
573613 const match_code = self.getCode(.match);
574 if (match_code >= zstd.compressed_block.match_length_code_table.len)
614 if (match_code >= zstd.match_length_code_table.len)
575615 return error.InvalidBitStream;
576 const match = zstd.compressed_block.match_length_code_table[match_code];
616 const match = zstd.match_length_code_table[match_code];
577617 const match_length = match[0] + try bit_reader.readBitsNoEof(u32, match[1]);
578618
579619 const literal_code = self.getCode(.literal);
580 if (literal_code >= zstd.compressed_block.literals_length_code_table.len)
620 if (literal_code >= zstd.literals_length_code_table.len)
581621 return error.InvalidBitStream;
582 const literal = zstd.compressed_block.literals_length_code_table[literal_code];
622 const literal = zstd.literals_length_code_table[literal_code];
583623 const literal_length = literal[0] + try bit_reader.readBitsNoEof(u32, literal[1]);
584624
585625 const offset = if (offset_value > 3) offset: {
......@@ -622,12 +662,17 @@ pub const Frame = struct {
622662 /// The `BufferedWriter` storage capacity is not large enough to
623663 /// accept this stream.
624664 OutputBufferUndersize,
665 WriteFailed,
666 MalformedLiteralsLength,
667 MalformedFseBits,
668 MissingStartBit,
669 HuffmanTreeIncomplete,
625670 };
626671
627672 /// Decode one sequence from `bit_reader` into `dest`. Updates FSE states
628673 /// if `last_sequence` is `false`. Assumes `prepare` called for the block
629674 /// before attempting to decode sequences.
630 pub fn decodeSequence(
675 fn decodeSequence(
631676 self: *Decode,
632677 dest: *BufferedWriter,
633678 bit_reader: *ReverseBitReader,
......@@ -662,13 +707,13 @@ pub const Frame = struct {
662707 return sequence_length;
663708 }
664709
665 fn nextLiteralMultiStream(self: *Decode) error{BitStreamHasNoStartBit}!void {
710 fn nextLiteralMultiStream(self: *Decode) error{MissingStartBit}!void {
666711 self.literal_stream_index += 1;
667712 try self.initLiteralStream(self.literal_streams.four[self.literal_stream_index]);
668713 }
669714
670 fn initLiteralStream(self: *Decode, bytes: []const u8) error{BitStreamHasNoStartBit}!void {
671 try self.literal_stream_reader.init(bytes);
715 fn initLiteralStream(self: *Decode, bytes: []const u8) error{MissingStartBit}!void {
716 self.literal_stream_reader = try ReverseBitReader.init(bytes);
672717 }
673718
674719 fn isLiteralStreamEmpty(self: *Decode) bool {
......@@ -679,7 +724,7 @@ pub const Frame = struct {
679724 }
680725
681726 const LiteralBitsError = error{
682 BitStreamHasNoStartBit,
727 MissingStartBit,
683728 UnexpectedEndOfLiteralStream,
684729 };
685730 fn readLiteralsBits(
......@@ -704,6 +749,9 @@ pub const Frame = struct {
704749 /// Problems decoding Huffman compressed literals
705750 UnexpectedEndOfLiteralStream,
706751 OutputBufferUndersize,
752 WriteFailed,
753 MissingStartBit,
754 HuffmanTreeIncomplete,
707755 };
708756
709757 /// Decode `len` bytes of literals into `dest`.
......@@ -765,6 +813,7 @@ pub const Frame = struct {
765813 }
766814 }
767815
816 /// TODO: don't use `@field`
768817 fn getCode(self: *Decode, comptime choice: DataType) u32 {
769818 return switch (@field(self, @tagName(choice)).table) {
770819 .rle => |value| value,
......@@ -785,21 +834,17 @@ pub const Frame = struct {
785834 };
786835
787836 const InitError = error{
837 /// Frame uses a dictionary.
788838 DictionaryIdFlagUnsupported,
839 /// Frame does not have a valid window size.
789840 WindowSizeUnknown,
790 WindowTooLarge,
791 ContentSizeTooLarge,
841 /// Window size exceeds `window_size_max` or max `usize` value.
842 WindowOversize,
843 /// Frame header indicates a content size exceeding max `usize` value.
844 ContentOversize,
792845 };
846
793847 /// Validates `frame_header` and returns the associated `Frame`.
794 ///
795 /// Errors returned:
796 /// - `error.DictionaryIdFlagUnsupported` if the frame uses a dictionary
797 /// - `error.WindowSizeUnknown` if the frame does not have a valid window
798 /// size
799 /// - `error.WindowTooLarge` if the window size is larger than
800 /// `window_size_max` or `std.math.intMax(usize)`
801 /// - `error.ContentSizeTooLarge` if the frame header indicates a content
802 /// size larger than `std.math.maxInt(usize)`
803848 pub fn init(
804849 frame_header: Frame.Zstandard.Header,
805850 window_size_max: usize,
......@@ -810,15 +855,15 @@ pub const Frame = struct {
810855
811856 const window_size_raw = frame_header.windowSize() orelse return error.WindowSizeUnknown;
812857 const window_size = if (window_size_raw > window_size_max)
813 return error.WindowTooLarge
858 return error.WindowOversize
814859 else
815 std.math.cast(usize, window_size_raw) orelse return error.WindowTooLarge;
860 std.math.cast(usize, window_size_raw) orelse return error.WindowOversize;
816861
817862 const should_compute_checksum =
818863 frame_header.descriptor.content_checksum_flag and verify_checksum;
819864
820865 const content_size = if (frame_header.content_size) |size|
821 std.math.cast(usize, size) orelse return error.ContentSizeTooLarge
866 std.math.cast(usize, size) orelse return error.ContentOversize
822867 else
823868 null;
824869
......@@ -875,13 +920,11 @@ pub const LiteralsSection = struct {
875920 compressed_size: ?u18,
876921
877922 /// Decode a literals section header.
878 ///
879 /// Errors returned:
880 /// - `error.EndOfStream` if there are not enough bytes in `source`
881 pub fn decode(source: *BufferedReader) !Header {
882 const byte0 = try source.readByte();
883 const block_type = @as(BlockType, @enumFromInt(byte0 & 0b11));
884 const size_format = @as(u2, @intCast((byte0 & 0b1100) >> 2));
923 pub fn decode(in: *BufferedReader, remaining: *Reader.Limit) !Header {
924 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
925 const byte0 = try in.takeByte();
926 const block_type: BlockType = @enumFromInt(byte0 & 0b11);
927 const size_format: u2 = @intCast((byte0 & 0b1100) >> 2);
885928 var regenerated_size: u20 = undefined;
886929 var compressed_size: ?u18 = null;
887930 switch (block_type) {
......@@ -890,28 +933,37 @@ pub const LiteralsSection = struct {
890933 0, 2 => {
891934 regenerated_size = byte0 >> 3;
892935 },
893 1 => regenerated_size = (byte0 >> 4) + (@as(u20, try source.readByte()) << 4),
894 3 => regenerated_size = (byte0 >> 4) +
895 (@as(u20, try source.readByte()) << 4) +
896 (@as(u20, try source.readByte()) << 12),
936 1 => {
937 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
938 regenerated_size = (byte0 >> 4) + (@as(u20, try in.takeByte()) << 4);
939 },
940 3 => {
941 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
942 regenerated_size = (byte0 >> 4) +
943 (@as(u20, try in.takeByte()) << 4) +
944 (@as(u20, try in.takeByte()) << 12);
945 },
897946 }
898947 },
899948 .compressed, .treeless => {
900 const byte1 = try source.readByte();
901 const byte2 = try source.readByte();
949 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
950 const byte1 = try in.takeByte();
951 const byte2 = try in.takeByte();
902952 switch (size_format) {
903953 0, 1 => {
904954 regenerated_size = (byte0 >> 4) + ((@as(u20, byte1) & 0b00111111) << 4);
905955 compressed_size = ((byte1 & 0b11000000) >> 6) + (@as(u18, byte2) << 2);
906956 },
907957 2 => {
908 const byte3 = try source.readByte();
958 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
959 const byte3 = try in.takeByte();
909960 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00000011) << 12);
910961 compressed_size = ((byte2 & 0b11111100) >> 2) + (@as(u18, byte3) << 6);
911962 },
912963 3 => {
913 const byte3 = try source.readByte();
914 const byte4 = try source.readByte();
964 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
965 const byte3 = try in.takeByte();
966 const byte4 = try in.takeByte();
915967 regenerated_size = (byte0 >> 4) + (@as(u20, byte1) << 4) + ((@as(u20, byte2) & 0b00111111) << 12);
916968 compressed_size = ((byte2 & 0b11000000) >> 6) + (@as(u18, byte3) << 2) + (@as(u18, byte4) << 10);
917969 },
......@@ -950,17 +1002,17 @@ pub const LiteralsSection = struct {
9501002 index: usize,
9511003 };
9521004
953 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{NotFound}!Result {
1005 pub fn query(self: HuffmanTree, index: usize, prefix: u16) error{HuffmanTreeIncomplete}!Result {
9541006 var node = self.nodes[index];
9551007 const weight = node.weight;
9561008 var i: usize = index;
9571009 while (node.weight == weight) {
958 if (node.prefix == prefix) return Result{ .symbol = node.symbol };
959 if (i == 0) return error.NotFound;
1010 if (node.prefix == prefix) return .{ .symbol = node.symbol };
1011 if (i == 0) return error.HuffmanTreeIncomplete;
9601012 i -= 1;
9611013 node = self.nodes[i];
9621014 }
963 return Result{ .index = i };
1015 return .{ .index = i };
9641016 }
9651017
9661018 pub fn weightToBitCount(weight: u4, max_bit_count: u4) u4 {
......@@ -975,20 +1027,26 @@ pub const LiteralsSection = struct {
9751027 MissingStartBit,
9761028 };
9771029
978 pub fn decode(in: *BufferedReader) HuffmanTree.DecodeError!HuffmanTree {
1030 pub fn decode(in: *BufferedReader, remaining: *Reader.Limit) HuffmanTree.DecodeError!HuffmanTree {
1031 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
9791032 const header = try in.takeByte();
9801033 if (header < 128) {
981 return decodeFse(in, header);
1034 return decodeFse(in, remaining, header);
9821035 } else {
983 return decodeDirect(in, header - 127);
1036 return decodeDirect(in, remaining, header - 127);
9841037 }
9851038 }
9861039
987 fn decodeDirect(source: *BufferedReader, encoded_symbol_count: usize) HuffmanTree.DecodeError!HuffmanTree {
1040 fn decodeDirect(
1041 in: *BufferedReader,
1042 remaining: *Reader.Limit,
1043 encoded_symbol_count: usize,
1044 ) HuffmanTree.DecodeError!HuffmanTree {
9881045 var weights: [256]u4 = undefined;
9891046 const weights_byte_count = (encoded_symbol_count + 1) / 2;
1047 remaining.* = remaining.subtract(weights_byte_count) orelse return error.EndOfStream;
9901048 for (0..weights_byte_count) |i| {
991 const byte = try source.takeByte();
1049 const byte = try in.takeByte();
9921050 weights[2 * i] = @as(u4, @intCast(byte >> 4));
9931051 weights[2 * i + 1] = @as(u4, @intCast(byte & 0xF));
9941052 }
......@@ -996,22 +1054,25 @@ pub const LiteralsSection = struct {
9961054 return build(&weights, symbol_count);
9971055 }
9981056
999 fn decodeFse(in: *BufferedReader, compressed_size: usize) HuffmanTree.DecodeError!HuffmanTree {
1057 fn decodeFse(
1058 in: *BufferedReader,
1059 remaining: *Reader.Limit,
1060 compressed_size: usize,
1061 ) HuffmanTree.DecodeError!HuffmanTree {
10001062 var weights: [256]u4 = undefined;
1063 remaining.* = remaining.subtract(compressed_size) orelse return error.EndOfStream;
10011064 const compressed_buffer = try in.take(compressed_size);
1002 var limited_stream: BufferedReader = undefined;
1003 limited_stream.initFixed(compressed_buffer);
1004 var bit_reader: std.io.BitReader(.little) = .init(&limited_stream);
1065 var bit_reader: BitReader = .{ .bytes = compressed_buffer };
10051066 var entries: [1 << 6]Table.Fse = undefined;
10061067 const table_size = try Table.decode(&bit_reader, 256, 6, &entries);
10071068 const accuracy_log = std.math.log2_int_ceil(usize, table_size);
1008 const remaining = limited_stream.bufferContents();
1009 const symbol_count = try assignWeights(remaining, accuracy_log, &entries, weights);
1069 const remaining_buffer = bit_reader.bytes[bit_reader.index..];
1070 const symbol_count = try assignWeights(remaining_buffer, accuracy_log, &entries, &weights);
10101071 return build(&weights, symbol_count);
10111072 }
10121073
10131074 fn assignWeights(
1014 huff_bits_buffer: []u8,
1075 huff_bits_buffer: []const u8,
10151076 accuracy_log: u16,
10161077 entries: *[1 << 6]Table.Fse,
10171078 weights: *[256]u4,
......@@ -1159,14 +1220,18 @@ pub const LiteralsSection = struct {
11591220 MalformedHuffmanTree,
11601221 /// Not enough bytes to complete the section.
11611222 EndOfStream,
1223 ReadFailed,
1224 LiteralsBufferUndersize,
1225 MissingStartBit,
11621226 };
11631227
1164 pub fn decode(source: *BufferedReader, buffer: []u8) DecodeError!LiteralsSection {
1165 const header = try Header.decode(source);
1228 pub fn decode(in: *BufferedReader, remaining: *Reader.Limit, buffer: []u8) DecodeError!LiteralsSection {
1229 const header = try Header.decode(in, remaining);
11661230 switch (header.block_type) {
11671231 .raw => {
1168 if (buffer.len < header.regenerated_size) return error.LiteralsBufferTooSmall;
1169 try source.readNoEof(buffer[0..header.regenerated_size]);
1232 if (buffer.len < header.regenerated_size) return error.LiteralsBufferUndersize;
1233 remaining.* = remaining.subtract(header.regenerated_size) orelse return error.EndOfStream;
1234 try in.readSlice(buffer[0..header.regenerated_size]);
11701235 return .{
11711236 .header = header,
11721237 .huffman_tree = null,
......@@ -1174,7 +1239,8 @@ pub const LiteralsSection = struct {
11741239 };
11751240 },
11761241 .rle => {
1177 buffer[0] = try source.readByte();
1242 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1243 buffer[0] = try in.takeByte();
11781244 return .{
11791245 .header = header,
11801246 .huffman_tree = null,
......@@ -1182,19 +1248,18 @@ pub const LiteralsSection = struct {
11821248 };
11831249 },
11841250 .compressed, .treeless => {
1185 var counting_reader = std.io.countingReader(source);
1251 const before_remaining = remaining.*;
11861252 const huffman_tree = if (header.block_type == .compressed)
1187 try HuffmanTree.decode(counting_reader.reader(), buffer)
1253 try HuffmanTree.decode(in, remaining)
11881254 else
11891255 null;
1190 const huffman_tree_size = @as(usize, @intCast(counting_reader.bytes_read));
1256 const huffman_tree_size = @intFromEnum(before_remaining) - @intFromEnum(remaining.*);
11911257 const total_streams_size = std.math.sub(usize, header.compressed_size.?, huffman_tree_size) catch
11921258 return error.MalformedLiteralsSection;
1193
1194 if (total_streams_size > buffer.len) return error.LiteralsBufferTooSmall;
1195 try source.readNoEof(buffer[0..total_streams_size]);
1259 if (total_streams_size > buffer.len) return error.LiteralsBufferUndersize;
1260 remaining.* = remaining.subtract(total_streams_size) orelse return error.EndOfStream;
1261 try in.readSlice(buffer[0..total_streams_size]);
11961262 const stream_data = buffer[0..total_streams_size];
1197
11981263 const streams = try Streams.decode(header.size_format, stream_data);
11991264 return .{
12001265 .header = header,
......@@ -1207,7 +1272,7 @@ pub const LiteralsSection = struct {
12071272};
12081273
12091274pub const SequencesSection = struct {
1210 header: SequencesSection.Header,
1275 header: Header,
12111276 literals_length_table: Table,
12121277 offset_table: Table,
12131278 match_length_table: Table,
......@@ -1228,32 +1293,37 @@ pub const SequencesSection = struct {
12281293 pub const DecodeError = error{
12291294 ReservedBitSet,
12301295 EndOfStream,
1296 ReadFailed,
12311297 };
12321298
1233 pub fn decode(source: *BufferedReader) DecodeError!Header {
1299 pub fn decode(in: *BufferedReader, remaining: *Reader.Limit) DecodeError!Header {
12341300 var sequence_count: u24 = undefined;
12351301
1236 const byte0 = try source.readByte();
1302 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
1303 const byte0 = try in.takeByte();
12371304 if (byte0 == 0) {
1238 return SequencesSection.Header{
1305 return .{
12391306 .sequence_count = 0,
12401307 .offsets = undefined,
12411308 .match_lengths = undefined,
12421309 .literal_lengths = undefined,
12431310 };
12441311 } else if (byte0 < 128) {
1312 remaining.* = remaining.subtract(1) orelse return error.EndOfStream;
12451313 sequence_count = byte0;
12461314 } else if (byte0 < 255) {
1247 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try source.readByte();
1315 remaining.* = remaining.subtract(2) orelse return error.EndOfStream;
1316 sequence_count = (@as(u24, (byte0 - 128)) << 8) + try in.takeByte();
12481317 } else {
1249 sequence_count = (try source.readByte()) + (@as(u24, try source.readByte()) << 8) + 0x7F00;
1318 remaining.* = remaining.subtract(3) orelse return error.EndOfStream;
1319 sequence_count = (try in.takeByte()) + (@as(u24, try in.takeByte()) << 8) + 0x7F00;
12501320 }
12511321
1252 const compression_modes = try source.readByte();
1322 const compression_modes = try in.takeByte();
12531323
1254 const matches_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00001100) >> 2));
1255 const offsets_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b00110000) >> 4));
1256 const literal_mode = @as(SequencesSection.Header.Mode, @enumFromInt((compression_modes & 0b11000000) >> 6));
1324 const matches_mode: Header.Mode = @enumFromInt((compression_modes & 0b00001100) >> 2);
1325 const offsets_mode: Header.Mode = @enumFromInt((compression_modes & 0b00110000) >> 4);
1326 const literal_mode: Header.Mode = @enumFromInt((compression_modes & 0b11000000) >> 6);
12571327 if (compression_modes & 0b11 != 0) return error.ReservedBitSet;
12581328
12591329 return .{
......@@ -1277,7 +1347,7 @@ pub const Table = union(enum) {
12771347 };
12781348
12791349 pub fn decode(
1280 bit_reader: *std.io.BitReader(.little),
1350 bit_reader: *BitReader,
12811351 expected_symbol_count: usize,
12821352 max_accuracy_log: u4,
12831353 entries: []Table.Fse,
......@@ -1600,6 +1670,22 @@ pub const Table = union(enum) {
16001670 };
16011671};
16021672
1673const low_bit_mask = [9]u8{
1674 0b00000000,
1675 0b00000001,
1676 0b00000011,
1677 0b00000111,
1678 0b00001111,
1679 0b00011111,
1680 0b00111111,
1681 0b01111111,
1682 0b11111111,
1683};
1684
1685fn Bits(comptime T: type) type {
1686 return struct { T, u16 };
1687}
1688
16031689/// For reading the reversed bit streams used to encode FSE compressed data.
16041690const ReverseBitReader = struct {
16051691 bytes: []const u8,
......@@ -1619,20 +1705,175 @@ const ReverseBitReader = struct {
16191705 return error.MissingStartBit;
16201706 }
16211707
1622 fn readBitsNoEof(self: *ReverseBitReader, comptime U: type, num_bits: u16) error{EndOfStream}!U {
1623 return self.bit_reader.readBitsNoEof(U, num_bits);
1708 fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1709 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1710 return .{
1711 @bitCast(@as(UT, @intCast(out))),
1712 num,
1713 };
1714 }
1715
1716 fn readBitsNoEof(self: *ReverseBitReader, comptime T: type, num: u16) error{EndOfStream}!T {
1717 const b, const c = try self.readBitsTuple(T, num);
1718 if (c < num) return error.EndOfStream;
1719 return b;
16241720 }
16251721
1626 fn readBits(self: *ReverseBitReader, comptime U: type, num_bits: u16, out_bits: *u16) error{}!U {
1627 return try self.bit_reader.readBits(U, num_bits, out_bits);
1722 fn readBits(self: *ReverseBitReader, comptime T: type, num: u16, out_bits: *u16) !T {
1723 const b, const c = try self.readBitsTuple(T, num);
1724 out_bits.* = c;
1725 return b;
16281726 }
16291727
1630 fn alignToByte(self: *ReverseBitReader) void {
1631 self.bit_reader.alignToByte();
1728 fn readBitsTuple(self: *ReverseBitReader, comptime T: type, num: u16) !Bits(T) {
1729 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1730 const U = if (@bitSizeOf(T) < 8) u8 else UT;
1731
1732 if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1733
1734 var out_count: u16 = self.count;
1735 var out: U = self.removeBits(self.count);
1736
1737 const full_bytes_left = (num - out_count) / 8;
1738
1739 for (0..full_bytes_left) |_| {
1740 const byte = takeByte(self) catch |err| switch (err) {
1741 error.EndOfStream => return initBits(T, out, out_count),
1742 };
1743 if (U == u8) out = 0 else out <<= 8;
1744 out |= byte;
1745 out_count += 8;
1746 }
1747
1748 const bits_left = num - out_count;
1749 const keep = 8 - bits_left;
1750
1751 if (bits_left == 0) return initBits(T, out, out_count);
1752
1753 const final_byte = takeByte(self) catch |err| switch (err) {
1754 error.EndOfStream => return initBits(T, out, out_count),
1755 };
1756
1757 out <<= @intCast(bits_left);
1758 out |= final_byte >> @intCast(keep);
1759 self.bits = final_byte & low_bit_mask[keep];
1760
1761 self.count = @intCast(keep);
1762 return initBits(T, out, num);
1763 }
1764
1765 fn takeByte(rbr: *ReverseBitReader) error{EndOfStream}!u8 {
1766 if (rbr.remaining == 0) return error.EndOfStream;
1767 rbr.remaining -= 1;
1768 return rbr.bytes[rbr.remaining];
16321769 }
16331770
16341771 fn isEmpty(self: *const ReverseBitReader) bool {
1635 return self.byte_reader.remaining_bytes == 0 and self.bit_reader.count == 0;
1772 return self.remaining == 0 and self.count == 0;
1773 }
1774
1775 fn removeBits(self: *ReverseBitReader, num: u4) u8 {
1776 if (num == 8) {
1777 self.count = 0;
1778 return self.bits;
1779 }
1780
1781 const keep = self.count - num;
1782 const bits = self.bits >> @intCast(keep);
1783 self.bits &= low_bit_mask[keep];
1784
1785 self.count = keep;
1786 return bits;
1787 }
1788};
1789
1790const BitReader = struct {
1791 bytes: []const u8,
1792 index: usize = 0,
1793 bits: u8 = 0,
1794 count: u4 = 0,
1795
1796 fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
1797 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1798 return .{
1799 @bitCast(@as(UT, @intCast(out))),
1800 num,
1801 };
1802 }
1803
1804 fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
1805 const b, const c = try self.readBitsTuple(T, num);
1806 if (c < num) return error.EndOfStream;
1807 return b;
1808 }
1809
1810 fn readBits(self: *@This(), comptime T: type, num: u16, out_bits: *u16) !T {
1811 const b, const c = try self.readBitsTuple(T, num);
1812 out_bits.* = c;
1813 return b;
1814 }
1815
1816 fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
1817 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
1818 const U = if (@bitSizeOf(T) < 8) u8 else UT;
1819
1820 if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
1821
1822 var out_count: u16 = self.count;
1823 var out: U = self.removeBits(self.count);
1824
1825 const full_bytes_left = (num - out_count) / 8;
1826
1827 for (0..full_bytes_left) |_| {
1828 const byte = takeByte(self) catch |err| switch (err) {
1829 error.EndOfStream => return initBits(T, out, out_count),
1830 };
1831
1832 const pos = @as(U, byte) << @intCast(out_count);
1833 out |= pos;
1834 out_count += 8;
1835 }
1836
1837 const bits_left = num - out_count;
1838 const keep = 8 - bits_left;
1839
1840 if (bits_left == 0) return initBits(T, out, out_count);
1841
1842 const final_byte = takeByte(self) catch |err| switch (err) {
1843 error.EndOfStream => return initBits(T, out, out_count),
1844 };
1845
1846 const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
1847 out |= pos;
1848 self.bits = final_byte >> @intCast(bits_left);
1849
1850 self.count = @intCast(keep);
1851 return initBits(T, out, num);
1852 }
1853
1854 fn takeByte(br: *BitReader) error{EndOfStream}!u8 {
1855 if (br.bytes.len - br.index == 0) return error.EndOfStream;
1856 const result = br.bytes[br.index];
1857 br.index += 1;
1858 return result;
1859 }
1860
1861 fn removeBits(self: *@This(), num: u4) u8 {
1862 if (num == 8) {
1863 self.count = 0;
1864 return self.bits;
1865 }
1866
1867 const keep = self.count - num;
1868 const bits = self.bits & low_bit_mask[num];
1869 self.bits >>= @intCast(num);
1870 self.count = keep;
1871 return bits;
1872 }
1873
1874 fn alignToByte(self: *@This()) void {
1875 self.bits = 0;
1876 self.count = 0;
16361877 }
16371878};
16381879
lib/std/io.zig+3-9
......@@ -19,16 +19,12 @@ pub const AllocatingWriter = @import("io/AllocatingWriter.zig");
1919pub const MultiWriter = @import("io/multi_writer.zig").MultiWriter;
2020pub const multiWriter = @import("io/multi_writer.zig").multiWriter;
2121
22pub const BitReader = @import("io/bit_reader.zig").Type;
23
2422pub const BitWriter = @import("io/bit_writer.zig").BitWriter;
2523pub const bitWriter = @import("io/bit_writer.zig").bitWriter;
2624
2725pub const ChangeDetectionStream = @import("io/change_detection_stream.zig").ChangeDetectionStream;
2826pub const changeDetectionStream = @import("io/change_detection_stream.zig").changeDetectionStream;
2927
30pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile;
31
3228pub const tty = @import("io/tty.zig");
3329
3430pub fn poll(
......@@ -437,13 +433,11 @@ pub fn PollFiles(comptime StreamEnum: type) type {
437433}
438434
439435test {
440 _ = BufferedWriter;
436 _ = AllocatingWriter;
437 _ = BitWriter;
441438 _ = BufferedReader;
439 _ = BufferedWriter;
442440 _ = Reader;
443441 _ = Writer;
444 _ = AllocatingWriter;
445 _ = @import("io/bit_reader.zig");
446 _ = @import("io/bit_writer.zig");
447 _ = @import("io/buffered_atomic_file.zig");
448442 _ = @import("io/test.zig");
449443}
lib/std/io/bit_reader.zig deleted-236
......@@ -1,236 +0,0 @@
1const std = @import("../std.zig");
2const bit_reader = @This();
3
4//General note on endianess:
5//Big endian is packed starting in the most significant part of the byte and subsequent
6// bytes contain less significant bits. Thus we always take bits from the high
7// end and place them below existing bits in our output.
8//Little endian is packed starting in the least significant part of the byte and
9// subsequent bytes contain more significant bits. Thus we always take bits from
10// the low end and place them above existing bits in our output.
11//Regardless of endianess, within any given byte the bits are always in most
12// to least significant order.
13//Also regardless of endianess, the buffer always aligns bits to the low end
14// of the byte.
15
16/// Creates a bit reader which allows for reading bits from an underlying standard reader
17pub fn Type(comptime endian: std.builtin.Endian) type {
18 return struct {
19 reader: *std.io.BufferedReader,
20 bits: u8,
21 count: u4,
22
23 const low_bit_mask = [9]u8{
24 0b00000000,
25 0b00000001,
26 0b00000011,
27 0b00000111,
28 0b00001111,
29 0b00011111,
30 0b00111111,
31 0b01111111,
32 0b11111111,
33 };
34
35 pub fn init(reader: *std.io.BufferedReader) @This() {
36 return .{ .reader = reader, .bits = 0, .count = 0 };
37 }
38
39 fn Bits(comptime T: type) type {
40 return struct { T, u16 };
41 }
42
43 fn initBits(comptime T: type, out: anytype, num: u16) Bits(T) {
44 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
45 return .{
46 @bitCast(@as(UT, @intCast(out))),
47 num,
48 };
49 }
50
51 /// Reads `bits` bits from the reader and returns a specified type
52 /// containing them in the least significant end, returning an error if the
53 /// specified number of bits could not be read.
54 pub fn readBitsNoEof(self: *@This(), comptime T: type, num: u16) !T {
55 const b, const c = try self.readBitsTuple(T, num);
56 if (c < num) return error.EndOfStream;
57 return b;
58 }
59
60 /// Reads `bits` bits from the reader and returns a specified type
61 /// containing them in the least significant end. The number of bits successfully
62 /// read is placed in `out_bits`, as reaching the end of the stream is not an error.
63 pub fn readBits(self: *@This(), comptime T: type, num: u16, out_bits: *u16) !T {
64 const b, const c = try self.readBitsTuple(T, num);
65 out_bits.* = c;
66 return b;
67 }
68
69 /// Reads `bits` bits from the reader and returns a tuple of the specified type
70 /// containing them in the least significant end, and the number of bits successfully
71 /// read. Reaching the end of the stream is not an error.
72 pub fn readBitsTuple(self: *@This(), comptime T: type, num: u16) !Bits(T) {
73 const UT = std.meta.Int(.unsigned, @bitSizeOf(T));
74 const U = if (@bitSizeOf(T) < 8) u8 else UT; //it is a pain to work with <u8
75
76 //dump any bits in our buffer first
77 if (num <= self.count) return initBits(T, self.removeBits(@intCast(num)), num);
78
79 var out_count: u16 = self.count;
80 var out: U = self.removeBits(self.count);
81
82 //grab all the full bytes we need and put their
83 //bits where they belong
84 const full_bytes_left = (num - out_count) / 8;
85
86 for (0..full_bytes_left) |_| {
87 const byte = self.reader.takeByte() catch |err| switch (err) {
88 error.EndOfStream => return initBits(T, out, out_count),
89 else => |e| return e,
90 };
91
92 switch (endian) {
93 .big => {
94 if (U == u8) out = 0 else out <<= 8; //shifting u8 by 8 is illegal in Zig
95 out |= byte;
96 },
97 .little => {
98 const pos = @as(U, byte) << @intCast(out_count);
99 out |= pos;
100 },
101 }
102 out_count += 8;
103 }
104
105 const bits_left = num - out_count;
106 const keep = 8 - bits_left;
107
108 if (bits_left == 0) return initBits(T, out, out_count);
109
110 const final_byte = self.reader.takeByte() catch |err| switch (err) {
111 error.EndOfStream => return initBits(T, out, out_count),
112 else => |e| return e,
113 };
114
115 switch (endian) {
116 .big => {
117 out <<= @intCast(bits_left);
118 out |= final_byte >> @intCast(keep);
119 self.bits = final_byte & low_bit_mask[keep];
120 },
121 .little => {
122 const pos = @as(U, final_byte & low_bit_mask[bits_left]) << @intCast(out_count);
123 out |= pos;
124 self.bits = final_byte >> @intCast(bits_left);
125 },
126 }
127
128 self.count = @intCast(keep);
129 return initBits(T, out, num);
130 }
131
132 //convenience function for removing bits from
133 //the appropriate part of the buffer based on
134 //endianess.
135 fn removeBits(self: *@This(), num: u4) u8 {
136 if (num == 8) {
137 self.count = 0;
138 return self.bits;
139 }
140
141 const keep = self.count - num;
142 const bits = switch (endian) {
143 .big => self.bits >> @intCast(keep),
144 .little => self.bits & low_bit_mask[num],
145 };
146 switch (endian) {
147 .big => self.bits &= low_bit_mask[keep],
148 .little => self.bits >>= @intCast(num),
149 }
150
151 self.count = keep;
152 return bits;
153 }
154
155 pub fn alignToByte(self: *@This()) void {
156 self.bits = 0;
157 self.count = 0;
158 }
159 };
160}
161
162///////////////////////////////
163
164test "api coverage" {
165 const mem_be = [_]u8{ 0b11001101, 0b00001011 };
166 const mem_le = [_]u8{ 0b00011101, 0b10010101 };
167
168 var mem_in_be = std.io.fixedBufferStream(&mem_be);
169 var bit_stream_be: bit_reader.Type(.big) = .init(mem_in_be.reader());
170
171 var out_bits: u16 = undefined;
172
173 const expect = std.testing.expect;
174 const expectError = std.testing.expectError;
175
176 try expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits));
177 try expect(out_bits == 1);
178 try expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits));
179 try expect(out_bits == 2);
180 try expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits));
181 try expect(out_bits == 3);
182 try expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits));
183 try expect(out_bits == 4);
184 try expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits));
185 try expect(out_bits == 5);
186 try expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits));
187 try expect(out_bits == 1);
188
189 mem_in_be.pos = 0;
190 bit_stream_be.count = 0;
191 try expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits));
192 try expect(out_bits == 15);
193
194 mem_in_be.pos = 0;
195 bit_stream_be.count = 0;
196 try expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits));
197 try expect(out_bits == 16);
198
199 _ = try bit_stream_be.readBits(u0, 0, &out_bits);
200
201 try expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits));
202 try expect(out_bits == 0);
203 try expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1));
204
205 var mem_in_le = std.io.fixedBufferStream(&mem_le);
206 var bit_stream_le: bit_reader.Type(.little) = .init(mem_in_le.reader());
207
208 try expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits));
209 try expect(out_bits == 1);
210 try expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits));
211 try expect(out_bits == 2);
212 try expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits));
213 try expect(out_bits == 3);
214 try expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits));
215 try expect(out_bits == 4);
216 try expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits));
217 try expect(out_bits == 5);
218 try expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits));
219 try expect(out_bits == 1);
220
221 mem_in_le.pos = 0;
222 bit_stream_le.count = 0;
223 try expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits));
224 try expect(out_bits == 15);
225
226 mem_in_le.pos = 0;
227 bit_stream_le.count = 0;
228 try expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits));
229 try expect(out_bits == 16);
230
231 _ = try bit_stream_le.readBits(u0, 0, &out_bits);
232
233 try expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits));
234 try expect(out_bits == 0);
235 try expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1));
236}
lib/std/io/buffered_atomic_file.zig deleted-55
......@@ -1,55 +0,0 @@
1const std = @import("../std.zig");
2const mem = std.mem;
3const fs = std.fs;
4const File = std.fs.File;
5
6pub const BufferedAtomicFile = struct {
7 atomic_file: fs.AtomicFile,
8 file_writer: File.Writer,
9 buffered_writer: BufferedWriter,
10 allocator: mem.Allocator,
11
12 pub const buffer_size = 4096;
13 pub const BufferedWriter = std.io.BufferedWriter(buffer_size, File.Writer);
14 pub const Writer = std.io.Writer(*BufferedWriter, BufferedWriter.Error, BufferedWriter.write);
15
16 /// TODO when https://github.com/ziglang/zig/issues/2761 is solved
17 /// this API will not need an allocator
18 pub fn create(
19 allocator: mem.Allocator,
20 dir: fs.Dir,
21 dest_path: []const u8,
22 atomic_file_options: fs.Dir.AtomicFileOptions,
23 ) !*BufferedAtomicFile {
24 var self = try allocator.create(BufferedAtomicFile);
25 self.* = BufferedAtomicFile{
26 .atomic_file = undefined,
27 .file_writer = undefined,
28 .buffered_writer = undefined,
29 .allocator = allocator,
30 };
31 errdefer allocator.destroy(self);
32
33 self.atomic_file = try dir.atomicFile(dest_path, atomic_file_options);
34 errdefer self.atomic_file.deinit();
35
36 self.file_writer = self.atomic_file.file.writer();
37 self.buffered_writer = .{ .unbuffered_writer = self.file_writer };
38 return self;
39 }
40
41 /// always call destroy, even after successful finish()
42 pub fn destroy(self: *BufferedAtomicFile) void {
43 self.atomic_file.deinit();
44 self.allocator.destroy(self);
45 }
46
47 pub fn finish(self: *BufferedAtomicFile) !void {
48 try self.buffered_writer.flush();
49 try self.atomic_file.finish();
50 }
51
52 pub fn writer(self: *BufferedAtomicFile) Writer {
53 return .{ .context = &self.buffered_writer };
54 }
55};