authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-02 21:10:36+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-03-04 09:53:01+01:00
log8a963fd66e22820119bac1b0a568f0cf3af4fa8e
tree3bce31a144e7f100438d8945c53747ea48e01bf1
parent90c1a2c41aafb1e35e075fb7d0bdcf04c00db913

flate: 32 bit BitReader

Extend BitReader to accept size of internal buffer. It can be u64 (only option until now) or u32.

2 files changed, 84 insertions(+), 52 deletions(-)

lib/std/compress/flate/bit_reader.zig+83-51
......@@ -2,8 +2,16 @@ const std = @import("std");
22const assert = std.debug.assert;
33const testing = std.testing;
44
5pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
6 return BitReader(@TypeOf(reader)).init(reader);
5pub fn bitReader(comptime T: type, reader: anytype) BitReader(T, @TypeOf(reader)) {
6 return BitReader(T, @TypeOf(reader)).init(reader);
7}
8
9pub fn BitReader64(comptime ReaderType: type) type {
10 return BitReader(u64, ReaderType);
11}
12
13pub fn BitReader32(comptime ReaderType: type) type {
14 return BitReader(u32, ReaderType);
715}
816
917/// Bit reader used during inflate (decompression). Has internal buffer of 64
......@@ -15,12 +23,16 @@ pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
1523/// fill buffer from forward_reader by calling fill in advance and readF with
1624/// buffered flag set.
1725///
18pub fn BitReader(comptime ReaderType: type) type {
26pub fn BitReader(T: type, comptime ReaderType: type) type {
27 assert(T == u32 or T == u64);
28 const t_bytes: usize = @sizeOf(T);
29 const Tshift = if (T == u64) u6 else u5;
30
1931 return struct {
2032 // Underlying reader used for filling internal bits buffer
2133 forward_reader: ReaderType = undefined,
2234 // Internal buffer of 64 bits
23 bits: u64 = 0,
35 bits: T = 0,
2436 // Number of bits in the buffer
2537 nbits: u32 = 0,
2638
......@@ -51,14 +63,14 @@ pub fn BitReader(comptime ReaderType: type) type {
5163
5264 // Number of empty bytes in bits, round nbits to whole bytes.
5365 const empty_bytes =
54 @as(u8, if (self.nbits & 0x7 == 0) 8 else 7) - // 8 for 8, 16, 24..., 7 otherwise
66 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
5567 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
5668
57 var buf: [8]u8 = [_]u8{0} ** 8;
69 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
5870 const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
5971 if (bytes_read > 0) {
60 const u: u64 = std.mem.readInt(u64, buf[0..8], .little);
61 self.bits |= u << @as(u6, @intCast(self.nbits));
72 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
73 self.bits |= u << @as(Tshift, @intCast(self.nbits));
6274 self.nbits += 8 * @as(u8, @intCast(bytes_read));
6375 return;
6476 }
......@@ -99,7 +111,17 @@ pub fn BitReader(comptime ReaderType: type) type {
99111
100112 /// Read with flags provided.
101113 pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
102 const n: u6 = @bitSizeOf(U);
114 if (U == T) {
115 assert(how == 0);
116 assert(self.alignBits() == 0);
117 try self.fill(@bitSizeOf(T));
118 assert(self.nbits == @bitSizeOf(T));
119 const v = self.bits;
120 self.nbits = 0;
121 self.bits = 0;
122 return v;
123 }
124 const n: Tshift = @bitSizeOf(U);
103125 switch (how) {
104126 0 => { // `normal` read
105127 try self.fill(n); // ensure that there are n bits in the buffer
......@@ -157,7 +179,7 @@ pub fn BitReader(comptime ReaderType: type) type {
157179 }
158180
159181 /// Advance buffer for n bits.
160 pub fn shift(self: *Self, n: u6) !void {
182 pub fn shift(self: *Self, n: Tshift) !void {
161183 if (n > self.nbits) return error.EndOfStream;
162184 self.bits >>= n;
163185 self.nbits -= n;
......@@ -218,10 +240,10 @@ pub fn BitReader(comptime ReaderType: type) type {
218240 };
219241}
220242
221test "BitReader" {
243test "readF" {
222244 var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
223 var br = bitReader(fbs.reader());
224 const F = BitReader(@TypeOf(fbs.reader())).flag;
245 var br = bitReader(u64, fbs.reader());
246 const F = BitReader64(@TypeOf(fbs.reader())).flag;
225247
226248 try testing.expectEqual(@as(u8, 48), br.nbits);
227249 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
......@@ -254,36 +276,38 @@ test "BitReader" {
254276}
255277
256278test "read block type 1 data" {
257 const data = [_]u8{
258 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
259 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
260 0x0c, 0x01, 0x02, 0x03, //
261 0xaa, 0xbb, 0xcc, 0xdd,
262 };
263 var fbs = std.io.fixedBufferStream(&data);
264 var br = bitReader(fbs.reader());
265 const F = BitReader(@TypeOf(fbs.reader())).flag;
279 inline for ([_]type{ u64, u32 }) |T| {
280 const data = [_]u8{
281 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
282 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
283 0x0c, 0x01, 0x02, 0x03, //
284 0xaa, 0xbb, 0xcc, 0xdd,
285 };
286 var fbs = std.io.fixedBufferStream(&data);
287 var br = bitReader(T, fbs.reader());
288 const F = BitReader(T, @TypeOf(fbs.reader())).flag;
266289
267 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
268 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
290 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
291 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
269292
270 for ("Hello world\n") |c| {
271 try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
293 for ("Hello world\n") |c| {
294 try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
295 }
296 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
297 br.alignToByte();
298 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
299 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
300 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
272301 }
273 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
274 br.alignToByte();
275 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
276 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
277 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
278302}
279303
280test "init" {
304test "shift/fill" {
281305 const data = [_]u8{
282306 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
283307 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
284308 };
285309 var fbs = std.io.fixedBufferStream(&data);
286 var br = bitReader(fbs.reader());
310 var br = bitReader(u64, fbs.reader());
287311
288312 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
289313 try br.shift(8);
......@@ -303,31 +327,39 @@ test "init" {
303327}
304328
305329test "readAll" {
306 const data = [_]u8{
307 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
309 };
310 var fbs = std.io.fixedBufferStream(&data);
311 var br = bitReader(fbs.reader());
330 inline for ([_]type{ u64, u32 }) |T| {
331 const data = [_]u8{
332 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
333 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
334 };
335 var fbs = std.io.fixedBufferStream(&data);
336 var br = bitReader(T, fbs.reader());
312337
313 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
338 switch (T) {
339 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
340 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
341 else => unreachable,
342 }
314343
315 var out: [16]u8 = undefined;
316 try br.readAll(out[0..]);
317 try testing.expect(br.nbits == 0);
318 try testing.expect(br.bits == 0);
344 var out: [16]u8 = undefined;
345 try br.readAll(out[0..]);
346 try testing.expect(br.nbits == 0);
347 try testing.expect(br.bits == 0);
319348
320 try testing.expectEqualSlices(u8, data[0..16], &out);
349 try testing.expectEqualSlices(u8, data[0..16], &out);
350 }
321351}
322352
323353test "readFixedCode" {
324 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
354 inline for ([_]type{ u64, u32 }) |T| {
355 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
325356
326 var fbs = std.io.fixedBufferStream(&fixed_codes);
327 var rdr = bitReader(fbs.reader());
357 var fbs = std.io.fixedBufferStream(&fixed_codes);
358 var rdr = bitReader(T, fbs.reader());
328359
329 for (0..286) |c| {
330 try testing.expectEqual(c, try rdr.readFixedCode());
360 for (0..286) |c| {
361 try testing.expectEqual(c, try rdr.readFixedCode());
362 }
363 try testing.expect(rdr.nbits == 0);
331364 }
332 try testing.expect(rdr.nbits == 0);
333365}
lib/std/compress/flate/inflate.zig+1-1
......@@ -3,7 +3,7 @@ const assert = std.debug.assert;
33const testing = std.testing;
44
55const hfd = @import("huffman_decoder.zig");
6const BitReader = @import("bit_reader.zig").BitReader;
6const BitReader = @import("bit_reader.zig").BitReader64;
77const CircularBuffer = @import("CircularBuffer.zig");
88const Container = @import("container.zig").Container;
99const Token = @import("Token.zig");