authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-04-09 19:58:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:26-07:00
log6ac7931bec29f1cd4c889d6913a21f28e17c13a8
treea91d44087acb1c8886d9e92e6df555539db8b59f
parent890a02c3456dce7242aa65e5093b31f9d8a417bc

std: hacking around with buffered reader / writer semantics

I think I'm going to back out these vtable changes in the next commit

29 files changed, 2941 insertions(+), 2793 deletions(-)

lib/std/compress/flate.zig+14-41
......@@ -1,3 +1,5 @@
1const std = @import("../std.zig");
2
13/// Deflate is a lossless data compression file format that uses a combination
24/// of LZ77 and Huffman coding.
35pub const deflate = @import("flate/deflate.zig");
......@@ -7,77 +9,48 @@ pub const deflate = @import("flate/deflate.zig");
79pub const inflate = @import("flate/inflate.zig");
810
911/// Decompress compressed data from reader and write plain data to the writer.
10pub fn decompress(reader: anytype, writer: anytype) !void {
12pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
1113 try inflate.decompress(.raw, reader, writer);
1214}
1315
14/// Decompressor type
15pub fn Decompressor(comptime ReaderType: type) type {
16 return inflate.Decompressor(.raw, ReaderType);
17}
18
19/// Create Decompressor which will read compressed data from reader.
20pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21 return inflate.decompressor(.raw, reader);
22}
16pub const Decompressor = inflate.Decompressor(.raw);
2317
2418/// Compression level, trades between speed and compression size.
2519pub const Options = deflate.Options;
2620
2721/// Compress plain data from reader and write compressed data to the writer.
28pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
22pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) anyerror!void {
2923 try deflate.compress(.raw, reader, writer, options);
3024}
3125
32/// Compressor type
33pub fn Compressor(comptime WriterType: type) type {
34 return deflate.Compressor(.raw, WriterType);
35}
36
37/// Create Compressor which outputs compressed data to the writer.
38pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
39 return try deflate.compressor(.raw, writer, options);
40}
26pub const Compressor = deflate.Compressor(.raw);
4127
4228/// Huffman only compression. Without Lempel-Ziv match searching. Faster
4329/// compression, less memory requirements but bigger compressed sizes.
4430pub const huffman = struct {
45 pub fn compress(reader: anytype, writer: anytype) !void {
31 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
4632 try deflate.huffman.compress(.raw, reader, writer);
4733 }
4834
49 pub fn Compressor(comptime WriterType: type) type {
50 return deflate.huffman.Compressor(.raw, WriterType);
51 }
52
53 pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
54 return deflate.huffman.compressor(.raw, writer);
55 }
35 pub const Compressor = deflate.huffman.Compressor(.raw);
5636};
5737
5838// No compression store only. Compressed size is slightly bigger than plain.
5939pub const store = struct {
60 pub fn compress(reader: anytype, writer: anytype) !void {
40 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
6141 try deflate.store.compress(.raw, reader, writer);
6242 }
6343
64 pub fn Compressor(comptime WriterType: type) type {
65 return deflate.store.Compressor(.raw, WriterType);
66 }
67
68 pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
69 return deflate.store.compressor(.raw, writer);
70 }
44 pub const Compressor = deflate.store.Compressor(.raw);
7145};
7246
73/// Container defines header/footer around deflate bit stream. Gzip and zlib
74/// compression algorithms are containers around deflate bit stream body.
75const Container = @import("flate/container.zig").Container;
76const std = @import("std");
47const builtin = @import("builtin");
7748const testing = std.testing;
7849const fixedBufferStream = std.io.fixedBufferStream;
7950const print = std.debug.print;
80const builtin = @import("builtin");
51/// Container defines header/footer around deflate bit stream. Gzip and zlib
52/// compression algorithms are containers around deflate bit stream body.
53const Container = @import("flate/container.zig").Container;
8154
8255test {
8356 _ = deflate;
lib/std/compress/flate/bit_reader.zig deleted-421
......@@ -1,421 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4
5pub const Flags = packed struct(u3) {
6 /// dont advance internal buffer, just get bits, leave them in buffer
7 peek: bool = false,
8 /// assume that there is no need to fill, fill should be called before
9 buffered: bool = false,
10 /// bit reverse read bits
11 reverse: bool = false,
12};
13
14/// Bit reader used during inflate (decompression). Has internal buffer of 64
15/// bits which shifts right after bits are consumed. Uses forward_reader to fill
16/// that internal buffer when needed.
17///
18/// readF is the core function. Supports few different ways of getting bits
19/// controlled by flags. In hot path we try to avoid checking whether we need to
20/// fill buffer from forward_reader by calling fill in advance and readF with
21/// buffered flag set.
22///
23pub fn BitReader(comptime T: type) type {
24 assert(T == u32 or T == u64);
25 const t_bytes: usize = @sizeOf(T);
26 const Tshift = if (T == u64) u6 else u5;
27
28 return struct {
29 // Underlying reader used for filling internal bits buffer
30 forward_reader: *std.io.BufferedReader,
31 // Internal buffer of 64 bits
32 bits: T = 0,
33 // Number of bits in the buffer
34 nbits: u32 = 0,
35
36 const Self = @This();
37
38 pub fn init(forward_reader: *std.io.BufferedReader) Self {
39 var self = Self{ .forward_reader = forward_reader };
40 self.fill(1) catch {};
41 return self;
42 }
43
44 /// Try to have `nice` bits are available in buffer. Reads from
45 /// forward reader if there is no `nice` bits in buffer. Returns error
46 /// if end of forward stream is reached and internal buffer is empty.
47 /// It will not error if less than `nice` bits are in buffer, only when
48 /// all bits are exhausted. During inflate we usually know what is the
49 /// maximum bits for the next step but usually that step will need less
50 /// bits to decode. So `nice` is not hard limit, it will just try to have
51 /// that number of bits available. If end of forward stream is reached
52 /// it may be some extra zero bits in buffer.
53 pub fn fill(self: *Self, nice: u6) !void {
54 if (self.nbits >= nice and nice != 0) {
55 return; // We have enough bits
56 }
57 // Read more bits from forward reader
58
59 // Number of empty bytes in bits, round nbits to whole bytes.
60 const empty_bytes =
61 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
62 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
63
64 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
65 const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
66 if (bytes_read > 0) {
67 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
68 self.bits |= u << @as(Tshift, @intCast(self.nbits));
69 self.nbits += 8 * @as(u8, @intCast(bytes_read));
70 return;
71 }
72
73 if (self.nbits == 0)
74 return error.EndOfStream;
75 }
76
77 /// Read exactly buf.len bytes into buf.
78 pub fn readAll(self: *Self, buf: []u8) !void {
79 assert(self.alignBits() == 0); // internal bits must be at byte boundary
80
81 // First read from internal bits buffer.
82 var n: usize = 0;
83 while (self.nbits > 0 and n < buf.len) {
84 buf[n] = try self.readF(u8, .{ .buffered = true });
85 n += 1;
86 }
87 // Then use forward reader for all other bytes.
88 try self.forward_reader.readNoEof(buf[n..]);
89 }
90
91 /// Alias for readF(U, 0).
92 pub fn read(self: *Self, comptime U: type) !U {
93 return self.readF(U, 0);
94 }
95
96 /// Alias for readF with flag.peak set.
97 pub inline fn peekF(self: *Self, comptime U: type, comptime how: Flags) !U {
98 return self.readF(U, .{
99 .peek = true,
100 .buffered = how.buffered,
101 .reverse = how.reverse,
102 });
103 }
104
105 /// Read with flags provided.
106 pub fn readF(self: *Self, comptime U: type, comptime how: Flags) !U {
107 if (U == T) {
108 assert(how == 0);
109 assert(self.alignBits() == 0);
110 try self.fill(@bitSizeOf(T));
111 if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
112 const v = self.bits;
113 self.nbits = 0;
114 self.bits = 0;
115 return v;
116 }
117 const n: Tshift = @bitSizeOf(U);
118 switch (how) {
119 0 => { // `normal` read
120 try self.fill(n); // ensure that there are n bits in the buffer
121 const u: U = @truncate(self.bits); // get n bits
122 try self.shift(n); // advance buffer for n
123 return u;
124 },
125 .{ .peek = true } => { // no shift, leave bits in the buffer
126 try self.fill(n);
127 return @truncate(self.bits);
128 },
129 .{ .buffered = true } => { // no fill, assume that buffer has enough bits
130 const u: U = @truncate(self.bits);
131 try self.shift(n);
132 return u;
133 },
134 .{ .reverse = true } => { // same as 0 with bit reverse
135 try self.fill(n);
136 const u: U = @truncate(self.bits);
137 try self.shift(n);
138 return @bitReverse(u);
139 },
140 .{ .peek = true, .reverse = true } => {
141 try self.fill(n);
142 return @bitReverse(@as(U, @truncate(self.bits)));
143 },
144 .{ .buffered = true, .reverse = true } => {
145 const u: U = @truncate(self.bits);
146 try self.shift(n);
147 return @bitReverse(u);
148 },
149 .{ .peek = true, .buffered = true },
150 => {
151 return @truncate(self.bits);
152 },
153 .{ .peek = true, .buffered = true, .reverse = true } => {
154 return @bitReverse(@as(U, @truncate(self.bits)));
155 },
156 }
157 }
158
159 /// Read n number of bits.
160 /// Only buffered flag can be used in how.
161 pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
162 switch (how) {
163 0 => {
164 try self.fill(n);
165 },
166 .{ .buffered = true } => {},
167 else => unreachable,
168 }
169 const mask: u16 = (@as(u16, 1) << n) - 1;
170 const u: u16 = @as(u16, @truncate(self.bits)) & mask;
171 try self.shift(n);
172 return u;
173 }
174
175 /// Advance buffer for n bits.
176 pub fn shift(self: *Self, n: Tshift) !void {
177 if (n > self.nbits) return error.EndOfStream;
178 self.bits >>= n;
179 self.nbits -= n;
180 }
181
182 /// Skip n bytes.
183 pub fn skipBytes(self: *Self, n: u16) !void {
184 for (0..n) |_| {
185 try self.fill(8);
186 try self.shift(8);
187 }
188 }
189
190 // Number of bits to align stream to the byte boundary.
191 fn alignBits(self: *Self) u3 {
192 return @intCast(self.nbits & 0x7);
193 }
194
195 /// Align stream to the byte boundary.
196 pub fn alignToByte(self: *Self) void {
197 const ab = self.alignBits();
198 if (ab > 0) self.shift(ab) catch unreachable;
199 }
200
201 /// Skip zero terminated string.
202 pub fn skipStringZ(self: *Self) !void {
203 while (true) {
204 if (try self.readF(u8, 0) == 0) break;
205 }
206 }
207
208 /// Read deflate fixed fixed code.
209 /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
210 /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
211 /// Lit Value Bits Codes
212 /// --------- ---- -----
213 /// 0 - 143 8 00110000 through
214 /// 10111111
215 /// 144 - 255 9 110010000 through
216 /// 111111111
217 /// 256 - 279 7 0000000 through
218 /// 0010111
219 /// 280 - 287 8 11000000 through
220 /// 11000111
221 pub fn readFixedCode(self: *Self) !u16 {
222 try self.fill(7 + 2);
223 const code7 = try self.readF(u7, .{ .buffered = true, .reverse = true });
224 if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
225 return @as(u16, code7) + 256;
226 } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
227 return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, .{ .buffered = true })) - 0b0011_0000;
228 } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
229 return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, .{ .buffered = true }) + 280;
230 } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
231 return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, .{ .buffered = true, .reverse = true })) + 144;
232 }
233 }
234 };
235}
236
237test "readF" {
238 var input: std.io.BufferedReader = undefined;
239 input.initFixed(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
240 var br: BitReader(u64) = .init(&input);
241
242 try testing.expectEqual(@as(u8, 48), br.nbits);
243 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
244
245 try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
246 try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
247 try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
248 try testing.expectEqual(@as(u3, 5), br.alignBits());
249
250 try testing.expect(try br.readF(u8, .{ .peek = true }) == 0b0001_1110);
251 try testing.expect(try br.readF(u9, .{ .peek = true }) == 0b1_0001_1110);
252 try br.shift(9);
253 try testing.expectEqual(@as(u8, 36), br.nbits);
254 try testing.expectEqual(@as(u3, 4), br.alignBits());
255
256 try testing.expect(try br.readF(u4, 0) == 0b0100);
257 try testing.expectEqual(@as(u8, 32), br.nbits);
258 try testing.expectEqual(@as(u3, 0), br.alignBits());
259
260 try br.shift(1);
261 try testing.expectEqual(@as(u3, 7), br.alignBits());
262 try br.shift(1);
263 try testing.expectEqual(@as(u3, 6), br.alignBits());
264 br.alignToByte();
265 try testing.expectEqual(@as(u3, 0), br.alignBits());
266
267 try testing.expectEqual(@as(u64, 0xc9), br.bits);
268 try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
269 try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
270}
271
272test "read block type 1 data" {
273 inline for ([_]type{ u64, u32 }) |T| {
274 const data = [_]u8{
275 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
276 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
277 0x0c, 0x01, 0x02, 0x03, //
278 0xaa, 0xbb, 0xcc, 0xdd,
279 };
280 var fbs: std.io.BufferedReader = undefined;
281 fbs.initFixed(&data);
282 var br: BitReader(T) = .init(&fbs);
283
284 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
285 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
286
287 for ("Hello world\n") |c| {
288 try testing.expectEqual(@as(u8, c), try br.readF(u8, .{ .reverse = true }) - 0x30);
289 }
290 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
291 br.alignToByte();
292 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
293 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
294 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
295 }
296}
297
298test "shift/fill" {
299 const data = [_]u8{
300 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
301 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
302 };
303 var fbs: std.io.BufferedReader = undefined;
304 fbs.initFixed(&data);
305 var br: BitReader(u64) = .init(&fbs);
306
307 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
308 try br.shift(8);
309 try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
310 try br.fill(60); // fill with 1 byte
311 try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
312 try br.shift(8 * 4 + 4);
313 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
314
315 try br.fill(60); // fill with 4 bytes (shift by 4)
316 try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
317 try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
318
319 try br.shift(@intCast(br.nbits)); // clear buffer
320 try br.fill(8); // refill with the rest of the bytes
321 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
322}
323
324test "readAll" {
325 inline for ([_]type{ u64, u32 }) |T| {
326 const data = [_]u8{
327 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
328 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
329 };
330 var fbs: std.io.BufferedReader = undefined;
331 fbs.initFixed(&data);
332 var br: BitReader(T) = .init(&fbs);
333
334 switch (T) {
335 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
336 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
337 else => unreachable,
338 }
339
340 var out: [16]u8 = undefined;
341 try br.readAll(out[0..]);
342 try testing.expect(br.nbits == 0);
343 try testing.expect(br.bits == 0);
344
345 try testing.expectEqualSlices(u8, data[0..16], &out);
346 }
347}
348
349test "readFixedCode" {
350 inline for ([_]type{ u64, u32 }) |T| {
351 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
352
353 var fbs: std.io.BufferedReader = undefined;
354 fbs.initFixed(&fixed_codes);
355 var rdr: BitReader(T) = .init(&fbs);
356
357 for (0..286) |c| {
358 try testing.expectEqual(c, try rdr.readFixedCode());
359 }
360 try testing.expect(rdr.nbits == 0);
361 }
362}
363
364test "u32 leaves no bits on u32 reads" {
365 const data = [_]u8{
366 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
367 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
368 };
369 var fbs: std.io.BufferedReader = undefined;
370 fbs.initFixed(&data);
371 var br: BitReader(u32) = .init(&fbs);
372
373 _ = try br.read(u3);
374 try testing.expectEqual(29, br.nbits);
375 br.alignToByte();
376 try testing.expectEqual(24, br.nbits);
377 try testing.expectEqual(0x04_03_02_01, try br.read(u32));
378 try testing.expectEqual(0, br.nbits);
379 try testing.expectEqual(0x08_07_06_05, try br.read(u32));
380 try testing.expectEqual(0, br.nbits);
381
382 _ = try br.read(u9);
383 try testing.expectEqual(23, br.nbits);
384 br.alignToByte();
385 try testing.expectEqual(16, br.nbits);
386 try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
387 try testing.expectEqual(0, br.nbits);
388}
389
390test "u64 need fill after alignToByte" {
391 const data = [_]u8{
392 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
393 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
394 };
395
396 // without fill
397 var fbs: std.io.BufferedReader = undefined;
398 fbs.initFixed(&data);
399 var br: BitReader(u64) = .init(&fbs);
400 _ = try br.read(u23);
401 try testing.expectEqual(41, br.nbits);
402 br.alignToByte();
403 try testing.expectEqual(40, br.nbits);
404 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
405 try testing.expectEqual(8, br.nbits);
406 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
407 try testing.expectEqual(32, br.nbits);
408
409 // fill after align ensures all bits filled
410 fbs.reset();
411 br = .init(&fbs);
412 _ = try br.read(u23);
413 try testing.expectEqual(41, br.nbits);
414 br.alignToByte();
415 try br.fill(0);
416 try testing.expectEqual(64, br.nbits);
417 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
418 try testing.expectEqual(32, br.nbits);
419 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
420 try testing.expectEqual(0, br.nbits);
421}
lib/std/compress/flate/inflate.zig+480-30
......@@ -3,7 +3,6 @@ const assert = std.debug.assert;
33const testing = std.testing;
44
55const hfd = @import("huffman_decoder.zig");
6const BitReader = @import("bit_reader.zig").BitReader;
76const CircularBuffer = @import("CircularBuffer.zig");
87const Container = @import("container.zig").Container;
98const Token = @import("Token.zig");
......@@ -48,16 +47,14 @@ pub fn Decompressor(comptime container: Container) type {
4847/// * 64K for history (CircularBuffer)
4948/// * ~10K huffman decoders (Literal and DistanceDecoder)
5049///
51pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type {
52 assert(LookaheadType == u32 or LookaheadType == u64);
53 const BitReaderType = BitReader(LookaheadType);
50pub fn Inflate(comptime container: Container, comptime Lookahead: type) type {
51 assert(Lookahead == u32 or Lookahead == u64);
52 const LookaheadBitReader = BitReader(Lookahead);
5453
5554 return struct {
56 const F = BitReaderType.flag;
57
58 bits: BitReaderType,
55 bits: LookaheadBitReader,
5956 hist: CircularBuffer = .{},
60 // Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
57 // Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
6158 hasher: container.Hasher() = .{},
6259
6360 // dynamic block huffman code decoders
......@@ -79,7 +76,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
7976
8077 const Self = @This();
8178
82 pub const Error = BitReaderType.Error || Container.Error || hfd.Error || error{
79 pub const Error = anyerror || Container.Error || hfd.Error || error{
8380 InvalidCode,
8481 InvalidMatch,
8582 InvalidBlockType,
......@@ -88,10 +85,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
8885 };
8986
9087 pub fn init(bw: *std.io.BufferedReader) Self {
91 return .{ .bits = BitReaderType.init(bw) };
88 return .{ .bits = LookaheadBitReader.init(bw) };
9289 }
9390
94 fn blockHeader(self: *Self) !void {
91 fn blockHeader(self: *Self) anyerror!void {
9592 self.bfinal = try self.bits.read(u1);
9693 self.block_type = try self.bits.read(u2);
9794 }
......@@ -129,7 +126,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
129126 fn fixedDistanceCode(self: *Self, code: u8) !void {
130127 try self.bits.fill(5 + 5 + 13);
131128 const length = try self.decodeLength(code);
132 const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered | F.reverse));
129 const distance = try self.decodeDistance(try self.bits.readF(u5, .{
130 .buffered = true,
131 .reverse = true,
132 }));
133133 try self.hist.writeMatch(length, distance);
134134 }
135135
......@@ -139,7 +139,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
139139 return if (ml.extra_bits == 0) // 0 - 5 extra bits
140140 ml.base
141141 else
142 ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
142 ml.base + try self.bits.readN(ml.extra_bits, .{ .buffered = true });
143143 }
144144
145145 fn decodeDistance(self: *Self, code: u8) !u16 {
......@@ -148,7 +148,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
148148 return if (md.extra_bits == 0) // 0 - 13 extra bits
149149 md.base
150150 else
151 md.base + try self.bits.readN(md.extra_bits, F.buffered);
151 md.base + try self.bits.readN(md.extra_bits, .{ .buffered = true });
152152 }
153153
154154 fn dynamicBlockHeader(self: *Self) !void {
......@@ -171,7 +171,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
171171 var dec_lens = [_]u4{0} ** (286 + 30);
172172 var pos: usize = 0;
173173 while (pos < hlit + hdist) {
174 const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
174 const sym = try cl_dec.find(try self.bits.peekF(u7, .{ .reverse = true }));
175175 try self.bits.shift(sym.code_bits);
176176 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
177177 }
......@@ -230,13 +230,13 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
230230 .literal => self.hist.write(sym.symbol),
231231 .match => { // Decode match backreference <length, distance>
232232 // fill so we can use buffered reads
233 if (LookaheadType == u32)
233 if (Lookahead == u32)
234234 try self.bits.fill(5 + 15)
235235 else
236236 try self.bits.fill(5 + 15 + 13);
237237 const length = try self.decodeLength(sym.symbol);
238238 const dsm = try self.decodeSymbol(&self.dst_dec);
239 if (LookaheadType == u32) try self.bits.fill(13);
239 if (Lookahead == u32) try self.bits.fill(13);
240240 const distance = try self.decodeDistance(dsm.symbol);
241241 try self.hist.writeMatch(length, distance);
242242 },
......@@ -251,7 +251,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
251251 // used. Shift bit reader for that much bits, those bits are used. And
252252 // return symbol.
253253 fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
254 const sym = try decoder.find(try self.bits.peekF(u15, F.buffered | F.reverse));
254 const sym = try decoder.find(try self.bits.peekF(u15, .{ .buffered = true, .reverse = true }));
255255 try self.bits.shift(sym.code_bits);
256256 return sym;
257257 }
......@@ -338,22 +338,48 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
338338 }
339339 }
340340
341 // Reader interface
341 fn reader_streamRead(
342 ctx: ?*anyopaque,
343 bw: *std.io.BufferedWriter,
344 limit: std.io.Reader.Limit,
345 ) std.io.Reader.RwResult {
346 const self: *Self = @alignCast(@ptrCast(ctx));
347 const out = bw.writableSlice(1) catch |err| return .{ .write_err = err };
348 const in = self.get(limit.min(out.len)) catch |err| return .{ .read_err = err };
349 if (in.len == 0) return .{ .read_end = true };
350 @memcpy(out[0..in.len], in);
351 return .{ .len = in.len };
352 }
342353
343 pub const Reader = std.io.Reader(*Self, Error, read);
354 fn reader_streamReadVec(ctx: ?*anyopaque, data: []const []u8) std.io.Reader.Result {
355 const self: *Self = @alignCast(@ptrCast(ctx));
356 var total: usize = 0;
357 for (data) |buffer| {
358 if (buffer.len == 0) break;
359 const out = self.get(buffer.len) catch |err| {
360 return .{ .len = total, .err = err };
361 };
362 if (out.len == 0) break;
363 @memcpy(buffer[0..out.len], out);
364 total += out.len;
365 }
366 return .{ .len = total, .end = total == 0 };
367 }
344368
345 /// Returns the number of bytes read. It may be less than buffer.len.
346 /// If the number of bytes read is 0, it means end of stream.
347 /// End of stream is not an error condition.
348 pub fn read(self: *Self, buffer: []u8) Error!usize {
349 if (buffer.len == 0) return 0;
350 const out = try self.get(buffer.len);
351 @memcpy(buffer[0..out.len], out);
352 return out.len;
369 pub fn streamReadVec(self: *Self, data: []const []u8) std.io.Reader.Result {
370 return reader_streamReadVec(self, data);
353371 }
354372
355 pub fn reader(self: *Self) Reader {
356 return .{ .context = self };
373 pub fn reader(self: *Self) std.io.Reader {
374 return .{
375 .context = self,
376 .vtable = &.{
377 .posRead = null,
378 .posReadVec = null,
379 .streamRead = reader_streamRead,
380 .streamReadVec = reader_streamReadVec,
381 },
382 };
357383 }
358384 };
359385}
......@@ -567,3 +593,427 @@ test "bug 19895" {
567593 var buf: [0]u8 = undefined;
568594 try testing.expectEqual(0, try decomp.read(&buf));
569595}
596
597/// Bit reader used during inflate (decompression). Has internal buffer of 64
598/// bits which shifts right after bits are consumed. Uses forward_reader to fill
599/// that internal buffer when needed.
600///
601/// readF is the core function. Supports few different ways of getting bits
602/// controlled by flags. In hot path we try to avoid checking whether we need to
603/// fill buffer from forward_reader by calling fill in advance and readF with
604/// buffered flag set.
605///
606pub fn BitReader(comptime T: type) type {
607 assert(T == u32 or T == u64);
608 const t_bytes: usize = @sizeOf(T);
609 const Tshift = if (T == u64) u6 else u5;
610
611 return struct {
612 // Underlying reader used for filling internal bits buffer
613 forward_reader: *std.io.BufferedReader,
614 // Internal buffer of 64 bits
615 bits: T = 0,
616 // Number of bits in the buffer
617 nbits: u32 = 0,
618
619 const Self = @This();
620
621 pub const Flags = packed struct(u3) {
622 /// dont advance internal buffer, just get bits, leave them in buffer
623 peek: bool = false,
624 /// assume that there is no need to fill, fill should be called before
625 buffered: bool = false,
626 /// bit reverse read bits
627 reverse: bool = false,
628
629 /// work around https://github.com/ziglang/zig/issues/18882
630 pub inline fn toInt(f: Flags) u3 {
631 return @bitCast(f);
632 }
633 };
634
635 pub fn init(forward_reader: *std.io.BufferedReader) Self {
636 var self = Self{ .forward_reader = forward_reader };
637 self.fill(1) catch {};
638 return self;
639 }
640
641 /// Try to have `nice` bits are available in buffer. Reads from
642 /// forward reader if there is no `nice` bits in buffer. Returns error
643 /// if end of forward stream is reached and internal buffer is empty.
644 /// It will not error if less than `nice` bits are in buffer, only when
645 /// all bits are exhausted. During inflate we usually know what is the
646 /// maximum bits for the next step but usually that step will need less
647 /// bits to decode. So `nice` is not hard limit, it will just try to have
648 /// that number of bits available. If end of forward stream is reached
649 /// it may be some extra zero bits in buffer.
650 pub fn fill(self: *Self, nice: u6) !void {
651 if (self.nbits >= nice and nice != 0) {
652 return; // We have enough bits
653 }
654 // Read more bits from forward reader
655
656 // Number of empty bytes in bits, round nbits to whole bytes.
657 const empty_bytes =
658 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
659 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
660
661 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
662 const bytes_read = self.forward_reader.partialRead(buf[0..empty_bytes]) catch 0;
663 if (bytes_read > 0) {
664 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
665 self.bits |= u << @as(Tshift, @intCast(self.nbits));
666 self.nbits += 8 * @as(u8, @intCast(bytes_read));
667 return;
668 }
669
670 if (self.nbits == 0)
671 return error.EndOfStream;
672 }
673
674 /// Read exactly buf.len bytes into buf.
675 pub fn readAll(self: *Self, buf: []u8) anyerror!void {
676 assert(self.alignBits() == 0); // internal bits must be at byte boundary
677
678 // First read from internal bits buffer.
679 var n: usize = 0;
680 while (self.nbits > 0 and n < buf.len) {
681 buf[n] = try self.readF(u8, .{ .buffered = true });
682 n += 1;
683 }
684 // Then use forward reader for all other bytes.
685 try self.forward_reader.read(buf[n..]);
686 }
687
688 /// Alias for readF(U, 0).
689 pub fn read(self: *Self, comptime U: type) !U {
690 return self.readF(U, .{});
691 }
692
693 /// Alias for readF with flag.peak set.
694 pub inline fn peekF(self: *Self, comptime U: type, comptime how: Flags) !U {
695 return self.readF(U, .{
696 .peek = true,
697 .buffered = how.buffered,
698 .reverse = how.reverse,
699 });
700 }
701
702 /// Read with flags provided.
703 pub fn readF(self: *Self, comptime U: type, comptime how: Flags) !U {
704 if (U == T) {
705 assert(how.toInt() == 0);
706 assert(self.alignBits() == 0);
707 try self.fill(@bitSizeOf(T));
708 if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
709 const v = self.bits;
710 self.nbits = 0;
711 self.bits = 0;
712 return v;
713 }
714 const n: Tshift = @bitSizeOf(U);
715 // work around https://github.com/ziglang/zig/issues/18882
716 switch (how.toInt()) {
717 @as(Flags, .{}).toInt() => { // `normal` read
718 try self.fill(n); // ensure that there are n bits in the buffer
719 const u: U = @truncate(self.bits); // get n bits
720 try self.shift(n); // advance buffer for n
721 return u;
722 },
723 @as(Flags, .{ .peek = true }).toInt() => { // no shift, leave bits in the buffer
724 try self.fill(n);
725 return @truncate(self.bits);
726 },
727 @as(Flags, .{ .buffered = true }).toInt() => { // no fill, assume that buffer has enough bits
728 const u: U = @truncate(self.bits);
729 try self.shift(n);
730 return u;
731 },
732 @as(Flags, .{ .reverse = true }).toInt() => { // same as 0 with bit reverse
733 try self.fill(n);
734 const u: U = @truncate(self.bits);
735 try self.shift(n);
736 return @bitReverse(u);
737 },
738 @as(Flags, .{ .peek = true, .reverse = true }).toInt() => {
739 try self.fill(n);
740 return @bitReverse(@as(U, @truncate(self.bits)));
741 },
742 @as(Flags, .{ .buffered = true, .reverse = true }).toInt() => {
743 const u: U = @truncate(self.bits);
744 try self.shift(n);
745 return @bitReverse(u);
746 },
747 @as(Flags, .{ .peek = true, .buffered = true }).toInt() => {
748 return @truncate(self.bits);
749 },
750 @as(Flags, .{ .peek = true, .buffered = true, .reverse = true }).toInt() => {
751 return @bitReverse(@as(U, @truncate(self.bits)));
752 },
753 }
754 }
755
756 /// Read n number of bits.
757 /// Only buffered flag can be used in how.
758 pub fn readN(self: *Self, n: u4, comptime how: Flags) !u16 {
759 // work around https://github.com/ziglang/zig/issues/18882
760 switch (how.toInt()) {
761 @as(Flags, .{}).toInt() => {
762 try self.fill(n);
763 },
764 @as(Flags, .{ .buffered = true }).toInt() => {},
765 else => unreachable,
766 }
767 const mask: u16 = (@as(u16, 1) << n) - 1;
768 const u: u16 = @as(u16, @truncate(self.bits)) & mask;
769 try self.shift(n);
770 return u;
771 }
772
773 /// Advance buffer for n bits.
774 pub fn shift(self: *Self, n: Tshift) !void {
775 if (n > self.nbits) return error.EndOfStream;
776 self.bits >>= n;
777 self.nbits -= n;
778 }
779
780 /// Skip n bytes.
781 pub fn skipBytes(self: *Self, n: u16) !void {
782 for (0..n) |_| {
783 try self.fill(8);
784 try self.shift(8);
785 }
786 }
787
788 // Number of bits to align stream to the byte boundary.
789 fn alignBits(self: *Self) u3 {
790 return @intCast(self.nbits & 0x7);
791 }
792
793 /// Align stream to the byte boundary.
794 pub fn alignToByte(self: *Self) void {
795 const ab = self.alignBits();
796 if (ab > 0) self.shift(ab) catch unreachable;
797 }
798
799 /// Skip zero terminated string.
800 pub fn skipStringZ(self: *Self) !void {
801 while (true) {
802 if (try self.readF(u8, 0) == 0) break;
803 }
804 }
805
806 /// Read deflate fixed fixed code.
807 /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
808 /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
809 /// Lit Value Bits Codes
810 /// --------- ---- -----
811 /// 0 - 143 8 00110000 through
812 /// 10111111
813 /// 144 - 255 9 110010000 through
814 /// 111111111
815 /// 256 - 279 7 0000000 through
816 /// 0010111
817 /// 280 - 287 8 11000000 through
818 /// 11000111
819 pub fn readFixedCode(self: *Self) !u16 {
820 try self.fill(7 + 2);
821 const code7 = try self.readF(u7, .{ .buffered = true, .reverse = true });
822 if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
823 return @as(u16, code7) + 256;
824 } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
825 return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, .{ .buffered = true })) - 0b0011_0000;
826 } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
827 return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, .{ .buffered = true }) + 280;
828 } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
829 return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, .{ .buffered = true, .reverse = true })) + 144;
830 }
831 }
832 };
833}
834
835test "readF" {
836 var input: std.io.BufferedReader = undefined;
837 input.initFixed(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
838 var br: BitReader(u64) = .init(&input);
839
840 try testing.expectEqual(@as(u8, 48), br.nbits);
841 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
842
843 try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
844 try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
845 try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
846 try testing.expectEqual(@as(u3, 5), br.alignBits());
847
848 try testing.expect(try br.readF(u8, .{ .peek = true }) == 0b0001_1110);
849 try testing.expect(try br.readF(u9, .{ .peek = true }) == 0b1_0001_1110);
850 try br.shift(9);
851 try testing.expectEqual(@as(u8, 36), br.nbits);
852 try testing.expectEqual(@as(u3, 4), br.alignBits());
853
854 try testing.expect(try br.readF(u4, 0) == 0b0100);
855 try testing.expectEqual(@as(u8, 32), br.nbits);
856 try testing.expectEqual(@as(u3, 0), br.alignBits());
857
858 try br.shift(1);
859 try testing.expectEqual(@as(u3, 7), br.alignBits());
860 try br.shift(1);
861 try testing.expectEqual(@as(u3, 6), br.alignBits());
862 br.alignToByte();
863 try testing.expectEqual(@as(u3, 0), br.alignBits());
864
865 try testing.expectEqual(@as(u64, 0xc9), br.bits);
866 try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
867 try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
868}
869
870test "read block type 1 data" {
871 inline for ([_]type{ u64, u32 }) |T| {
872 const data = [_]u8{
873 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
874 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
875 0x0c, 0x01, 0x02, 0x03, //
876 0xaa, 0xbb, 0xcc, 0xdd,
877 };
878 var fbs: std.io.BufferedReader = undefined;
879 fbs.initFixed(&data);
880 var br: BitReader(T) = .init(&fbs);
881
882 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
883 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
884
885 for ("Hello world\n") |c| {
886 try testing.expectEqual(@as(u8, c), try br.readF(u8, .{ .reverse = true }) - 0x30);
887 }
888 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
889 br.alignToByte();
890 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
891 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
892 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
893 }
894}
895
896test "shift/fill" {
897 const data = [_]u8{
898 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
899 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
900 };
901 var fbs: std.io.BufferedReader = undefined;
902 fbs.initFixed(&data);
903 var br: BitReader(u64) = .init(&fbs);
904
905 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
906 try br.shift(8);
907 try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
908 try br.fill(60); // fill with 1 byte
909 try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
910 try br.shift(8 * 4 + 4);
911 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
912
913 try br.fill(60); // fill with 4 bytes (shift by 4)
914 try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
915 try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
916
917 try br.shift(@intCast(br.nbits)); // clear buffer
918 try br.fill(8); // refill with the rest of the bytes
919 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
920}
921
922test "readAll" {
923 inline for ([_]type{ u64, u32 }) |T| {
924 const data = [_]u8{
925 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
926 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
927 };
928 var fbs: std.io.BufferedReader = undefined;
929 fbs.initFixed(&data);
930 var br: BitReader(T) = .init(&fbs);
931
932 switch (T) {
933 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
934 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
935 else => unreachable,
936 }
937
938 var out: [16]u8 = undefined;
939 try br.readAll(out[0..]);
940 try testing.expect(br.nbits == 0);
941 try testing.expect(br.bits == 0);
942
943 try testing.expectEqualSlices(u8, data[0..16], &out);
944 }
945}
946
947test "readFixedCode" {
948 inline for ([_]type{ u64, u32 }) |T| {
949 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
950
951 var fbs: std.io.BufferedReader = undefined;
952 fbs.initFixed(&fixed_codes);
953 var rdr: BitReader(T) = .init(&fbs);
954
955 for (0..286) |c| {
956 try testing.expectEqual(c, try rdr.readFixedCode());
957 }
958 try testing.expect(rdr.nbits == 0);
959 }
960}
961
962test "u32 leaves no bits on u32 reads" {
963 const data = [_]u8{
964 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
965 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
966 };
967 var fbs: std.io.BufferedReader = undefined;
968 fbs.initFixed(&data);
969 var br: BitReader(u32) = .init(&fbs);
970
971 _ = try br.read(u3);
972 try testing.expectEqual(29, br.nbits);
973 br.alignToByte();
974 try testing.expectEqual(24, br.nbits);
975 try testing.expectEqual(0x04_03_02_01, try br.read(u32));
976 try testing.expectEqual(0, br.nbits);
977 try testing.expectEqual(0x08_07_06_05, try br.read(u32));
978 try testing.expectEqual(0, br.nbits);
979
980 _ = try br.read(u9);
981 try testing.expectEqual(23, br.nbits);
982 br.alignToByte();
983 try testing.expectEqual(16, br.nbits);
984 try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
985 try testing.expectEqual(0, br.nbits);
986}
987
988test "u64 need fill after alignToByte" {
989 const data = [_]u8{
990 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
991 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
992 };
993
994 // without fill
995 var fbs: std.io.BufferedReader = undefined;
996 fbs.initFixed(&data);
997 var br: BitReader(u64) = .init(&fbs);
998 _ = try br.read(u23);
999 try testing.expectEqual(41, br.nbits);
1000 br.alignToByte();
1001 try testing.expectEqual(40, br.nbits);
1002 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1003 try testing.expectEqual(8, br.nbits);
1004 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1005 try testing.expectEqual(32, br.nbits);
1006
1007 // fill after align ensures all bits filled
1008 fbs.reset();
1009 br = .init(&fbs);
1010 _ = try br.read(u23);
1011 try testing.expectEqual(41, br.nbits);
1012 br.alignToByte();
1013 try br.fill(0);
1014 try testing.expectEqual(64, br.nbits);
1015 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1016 try testing.expectEqual(32, br.nbits);
1017 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1018 try testing.expectEqual(0, br.nbits);
1019}
lib/std/compress/lzma.zig+906-60
......@@ -1,90 +1,936 @@
11const std = @import("../std.zig");
2const assert = std.debug.assert;
23const math = std.math;
34const mem = std.mem;
45const Allocator = std.mem.Allocator;
6const testing = std.testing;
7const expectEqualSlices = std.testing.expectEqualSlices;
8const expectError = std.testing.expectError;
59
6pub const decode = @import("lzma/decode.zig");
10pub const RangeDecoder = struct {
11 range: u32,
12 code: u32,
713
8pub fn decompress(
9 allocator: Allocator,
10 reader: anytype,
11) !Decompress(@TypeOf(reader)) {
12 return decompressWithOptions(allocator, reader, .{});
13}
14 pub fn init(rd: *RangeDecoder, br: *std.io.BufferedReader) anyerror!usize {
15 const reserved = try br.takeByte();
16 if (reserved != 0) return error.CorruptInput;
17 rd.* = .{
18 .range = 0xFFFF_FFFF,
19 .code = try br.takeInt(u32, .big),
20 };
21 return 5;
22 }
1423
15pub fn decompressWithOptions(
16 allocator: Allocator,
17 reader: anytype,
18 options: decode.Options,
19) !Decompress(@TypeOf(reader)) {
20 const params = try decode.Params.readHeader(reader, options);
21 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);
22}
24 pub inline fn isFinished(self: RangeDecoder) bool {
25 return self.code == 0;
26 }
27
28 inline fn normalize(self: *RangeDecoder, br: *std.io.BufferedReader) !void {
29 if (self.range < 0x0100_0000) {
30 self.range <<= 8;
31 self.code = (self.code << 8) ^ @as(u32, try br.takeByte());
32 }
33 }
34
35 inline fn getBit(self: *RangeDecoder, br: *std.io.BufferedReader) !bool {
36 self.range >>= 1;
37
38 const bit = self.code >= self.range;
39 if (bit)
40 self.code -= self.range;
41
42 try self.normalize(br);
43 return bit;
44 }
45
46 pub fn get(self: *RangeDecoder, br: *std.io.BufferedReader, count: usize) !u32 {
47 var result: u32 = 0;
48 var i: usize = 0;
49 while (i < count) : (i += 1)
50 result = (result << 1) ^ @intFromBool(try self.getBit(br));
51 return result;
52 }
53
54 pub inline fn decodeBit(self: *RangeDecoder, br: *std.io.BufferedReader, prob: *u16, update: bool) !bool {
55 const bound = (self.range >> 11) * prob.*;
56
57 if (self.code < bound) {
58 if (update)
59 prob.* += (0x800 - prob.*) >> 5;
60 self.range = bound;
61
62 try self.normalize(br);
63 return false;
64 } else {
65 if (update)
66 prob.* -= prob.* >> 5;
67 self.code -= bound;
68 self.range -= bound;
2369
24pub fn Decompress(comptime ReaderType: type) type {
70 try self.normalize(br);
71 return true;
72 }
73 }
74
75 fn parseBitTree(
76 self: *RangeDecoder,
77 br: *std.io.BufferedReader,
78 num_bits: u5,
79 probs: []u16,
80 update: bool,
81 ) !u32 {
82 var tmp: u32 = 1;
83 var i: @TypeOf(num_bits) = 0;
84 while (i < num_bits) : (i += 1) {
85 const bit = try self.decodeBit(br, &probs[tmp], update);
86 tmp = (tmp << 1) ^ @intFromBool(bit);
87 }
88 return tmp - (@as(u32, 1) << num_bits);
89 }
90
91 pub fn parseReverseBitTree(
92 self: *RangeDecoder,
93 br: *std.io.BufferedReader,
94 num_bits: u5,
95 probs: []u16,
96 offset: usize,
97 update: bool,
98 ) !u32 {
99 var result: u32 = 0;
100 var tmp: usize = 1;
101 var i: @TypeOf(num_bits) = 0;
102 while (i < num_bits) : (i += 1) {
103 const bit = @intFromBool(try self.decodeBit(br, &probs[offset + tmp], update));
104 tmp = (tmp << 1) ^ bit;
105 result ^= @as(u32, bit) << i;
106 }
107 return result;
108 }
109};
110
111pub const LenDecoder = struct {
112 choice: u16 = 0x400,
113 choice2: u16 = 0x400,
114 low_coder: [16]BitTree(3) = @splat(.{}),
115 mid_coder: [16]BitTree(3) = @splat(.{}),
116 high_coder: BitTree(8) = .{},
117
118 pub fn decode(
119 self: *LenDecoder,
120 br: *std.io.BufferedReader,
121 decoder: *RangeDecoder,
122 pos_state: usize,
123 update: bool,
124 ) !usize {
125 if (!try decoder.decodeBit(br, &self.choice, update)) {
126 return @as(usize, try self.low_coder[pos_state].parse(br, decoder, update));
127 } else if (!try decoder.decodeBit(br, &self.choice2, update)) {
128 return @as(usize, try self.mid_coder[pos_state].parse(br, decoder, update)) + 8;
129 } else {
130 return @as(usize, try self.high_coder.parse(br, decoder, update)) + 16;
131 }
132 }
133
134 pub fn reset(self: *LenDecoder) void {
135 self.choice = 0x400;
136 self.choice2 = 0x400;
137 for (&self.low_coder) |*t| t.reset();
138 for (&self.mid_coder) |*t| t.reset();
139 self.high_coder.reset();
140 }
141};
142
143pub fn BitTree(comptime num_bits: usize) type {
25144 return struct {
145 probs: [1 << num_bits]u16 = @splat(0x400),
146
26147 const Self = @This();
27148
28 pub const Error =
29 ReaderType.Error ||
30 Allocator.Error ||
31 error{ CorruptInput, EndOfStream, Overflow };
149 pub fn parse(
150 self: *Self,
151 br: *std.io.BufferedReader,
152 decoder: *RangeDecoder,
153 update: bool,
154 ) !u32 {
155 return decoder.parseBitTree(br, num_bits, &self.probs, update);
156 }
157
158 pub fn parseReverse(
159 self: *Self,
160 br: *std.io.BufferedReader,
161 decoder: *RangeDecoder,
162 update: bool,
163 ) !u32 {
164 return decoder.parseReverseBitTree(br, num_bits, &self.probs, 0, update);
165 }
32166
33 pub const Reader = std.io.Reader(*Self, Error, read);
167 pub fn reset(self: *Self) void {
168 @memset(&self.probs, 0x400);
169 }
170 };
171}
34172
35 allocator: Allocator,
36 in_reader: ReaderType,
37 to_read: std.ArrayListUnmanaged(u8),
173pub const Decode = struct {
174 properties: Properties,
175 unpacked_size: ?u64,
176 literal_probs: Vec2D(u16),
177 pos_slot_decoder: [4]BitTree(6),
178 align_decoder: BitTree(4),
179 pos_decoders: [115]u16,
180 is_match: [192]u16,
181 is_rep: [12]u16,
182 is_rep_g0: [12]u16,
183 is_rep_g1: [12]u16,
184 is_rep_g2: [12]u16,
185 is_rep_0long: [192]u16,
186 state: usize,
187 rep: [4]usize,
188 len_decoder: LenDecoder,
189 rep_len_decoder: LenDecoder,
38190
39 buffer: decode.lzbuffer.LzCircularBuffer,
40 decoder: decode.rangecoder.RangeDecoder,
41 state: decode.DecoderState,
191 pub const Options = struct {
192 unpacked_size: UnpackedSize = .read_from_header,
193 memlimit: ?usize = null,
194 allow_incomplete: bool = false,
195 };
42196
43 pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {
44 return Self{
45 .allocator = allocator,
46 .in_reader = source,
47 .to_read = .{},
197 pub const UnpackedSize = union(enum) {
198 read_from_header,
199 read_header_but_use_provided: ?u64,
200 use_provided: ?u64,
201 };
202
203 const ProcessingStatus = enum {
204 cont,
205 finished,
206 };
207
208 pub const Properties = struct {
209 lc: u4,
210 lp: u3,
211 pb: u3,
212
213 fn validate(self: Properties) void {
214 assert(self.lc <= 8);
215 assert(self.lp <= 4);
216 assert(self.pb <= 4);
217 }
218 };
219
220 pub const Params = struct {
221 properties: Properties,
222 dict_size: u32,
223 unpacked_size: ?u64,
224
225 pub fn readHeader(br: *std.io.BufferedReader, options: Options) anyerror!Params {
226 var props = try br.readByte();
227 if (props >= 225) {
228 return error.CorruptInput;
229 }
48230
49 .buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
50 .decoder = try decode.rangecoder.RangeDecoder.init(source),
51 .state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),
231 const lc = @as(u4, @intCast(props % 9));
232 props /= 9;
233 const lp = @as(u3, @intCast(props % 5));
234 props /= 5;
235 const pb = @as(u3, @intCast(props));
236
237 const dict_size_provided = try br.readInt(u32, .little);
238 const dict_size = @max(0x1000, dict_size_provided);
239
240 const unpacked_size = switch (options.unpacked_size) {
241 .read_from_header => blk: {
242 const unpacked_size_provided = try br.readInt(u64, .little);
243 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
244 break :blk if (marker_mandatory)
245 null
246 else
247 unpacked_size_provided;
248 },
249 .read_header_but_use_provided => |x| blk: {
250 _ = try br.readInt(u64, .little);
251 break :blk x;
252 },
253 .use_provided => |x| x,
254 };
255
256 return Params{
257 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
258 .dict_size = dict_size,
259 .unpacked_size = unpacked_size,
52260 };
53261 }
262 };
263
264 pub fn init(
265 allocator: Allocator,
266 properties: Properties,
267 unpacked_size: ?u64,
268 ) !Decode {
269 return .{
270 .properties = properties,
271 .unpacked_size = unpacked_size,
272 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (properties.lc + properties.lp), 0x300 }),
273 .pos_slot_decoder = @splat(.{}),
274 .align_decoder = .{},
275 .pos_decoders = @splat(0x400),
276 .is_match = @splat(0x400),
277 .is_rep = @splat(0x400),
278 .is_rep_g0 = @splat(0x400),
279 .is_rep_g1 = @splat(0x400),
280 .is_rep_g2 = @splat(0x400),
281 .is_rep_0long = @splat(0x400),
282 .state = 0,
283 .rep = @splat(0),
284 .len_decoder = .{},
285 .rep_len_decoder = .{},
286 };
287 }
288
289 pub fn deinit(self: *Decode, allocator: Allocator) void {
290 self.literal_probs.deinit(allocator);
291 self.* = undefined;
292 }
54293
55 pub fn reader(self: *Self) Reader {
56 return .{ .context = self };
294 pub fn resetState(self: *Decode, allocator: Allocator, new_props: Properties) !void {
295 new_props.validate();
296 if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
297 self.literal_probs.fill(0x400);
298 } else {
299 self.literal_probs.deinit(allocator);
300 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
57301 }
58302
59 pub fn deinit(self: *Self) void {
60 self.to_read.deinit(self.allocator);
61 self.buffer.deinit(self.allocator);
62 self.state.deinit(self.allocator);
63 self.* = undefined;
303 self.properties = new_props;
304 for (&self.pos_slot_decoder) |*t| t.reset();
305 self.align_decoder.reset();
306 self.pos_decoders = @splat(0x400);
307 self.is_match = @splat(0x400);
308 self.is_rep = @splat(0x400);
309 self.is_rep_g0 = @splat(0x400);
310 self.is_rep_g1 = @splat(0x400);
311 self.is_rep_g2 = @splat(0x400);
312 self.is_rep_0long = @splat(0x400);
313 self.state = 0;
314 self.rep = @splat(0);
315 self.len_decoder.reset();
316 self.rep_len_decoder.reset();
317 }
318
319 fn processNextInner(
320 self: *Decode,
321 allocator: Allocator,
322 br: *std.io.BufferedReader,
323 bw: *std.io.BufferedWriter,
324 buffer: anytype,
325 decoder: *RangeDecoder,
326 bytes_read: *usize,
327 update: bool,
328 ) !ProcessingStatus {
329 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
330
331 if (!try decoder.decodeBit(br, &self.is_match[(self.state << 4) + pos_state], update, bytes_read)) {
332 const byte: u8 = try self.decodeLiteral(br, buffer, decoder, update, bytes_read);
333
334 if (update) {
335 try buffer.appendLiteral(allocator, byte, bw);
336
337 self.state = if (self.state < 4)
338 0
339 else if (self.state < 10)
340 self.state - 3
341 else
342 self.state - 6;
343 }
344 return .cont;
345 }
346
347 var len: usize = undefined;
348 if (try decoder.decodeBit(br, &self.is_rep[self.state], update, bytes_read)) {
349 if (!try decoder.decodeBit(br, &self.is_rep_g0[self.state], update, bytes_read)) {
350 if (!try decoder.decodeBit(br, &self.is_rep_0long[(self.state << 4) + pos_state], update, bytes_read)) {
351 if (update) {
352 self.state = if (self.state < 7) 9 else 11;
353 const dist = self.rep[0] + 1;
354 try buffer.appendLz(allocator, 1, dist, bw);
355 }
356 return .cont;
357 }
358 } else {
359 const idx: usize = if (!try decoder.decodeBit(br, &self.is_rep_g1[self.state], update, bytes_read))
360 1
361 else if (!try decoder.decodeBit(br, &self.is_rep_g2[self.state], update, bytes_read))
362 2
363 else
364 3;
365 if (update) {
366 const dist = self.rep[idx];
367 var i = idx;
368 while (i > 0) : (i -= 1) {
369 self.rep[i] = self.rep[i - 1];
370 }
371 self.rep[0] = dist;
372 }
373 }
374
375 len = try self.rep_len_decoder.decode(br, decoder, pos_state, update, bytes_read);
376
377 if (update) {
378 self.state = if (self.state < 7) 8 else 11;
379 }
380 } else {
381 if (update) {
382 self.rep[3] = self.rep[2];
383 self.rep[2] = self.rep[1];
384 self.rep[1] = self.rep[0];
385 }
386
387 len = try self.len_decoder.decode(br, decoder, pos_state, update, bytes_read);
388
389 if (update) {
390 self.state = if (self.state < 7) 7 else 10;
391 }
392
393 const rep_0 = try self.decodeDistance(br, decoder, len, update, bytes_read);
394
395 if (update) {
396 self.rep[0] = rep_0;
397 if (self.rep[0] == 0xFFFF_FFFF) {
398 if (decoder.isFinished()) {
399 return .finished;
400 }
401 return error.CorruptInput;
402 }
403 }
404 }
405
406 if (update) {
407 len += 2;
408
409 const dist = self.rep[0] + 1;
410 try buffer.appendLz(allocator, len, dist, bw);
411 }
412
413 return .cont;
414 }
415
416 fn processNext(
417 self: *Decode,
418 allocator: Allocator,
419 br: *std.io.BufferedReader,
420 bw: *std.io.BufferedWriter,
421 buffer: anytype,
422 decoder: *RangeDecoder,
423 bytes_read: *usize,
424 ) !ProcessingStatus {
425 return self.processNextInner(allocator, br, bw, buffer, decoder, bytes_read, true);
426 }
427
428 pub fn process(
429 self: *Decode,
430 allocator: Allocator,
431 br: *std.io.BufferedReader,
432 bw: *std.io.BufferedWriter,
433 buffer: anytype,
434 decoder: *RangeDecoder,
435 bytes_read: *usize,
436 ) !ProcessingStatus {
437 process_next: {
438 if (self.unpacked_size) |unpacked_size| {
439 if (buffer.len >= unpacked_size) {
440 break :process_next;
441 }
442 } else if (decoder.isFinished()) {
443 break :process_next;
444 }
445
446 switch (try self.processNext(allocator, br, bw, buffer, decoder, bytes_read)) {
447 .cont => return .cont,
448 .finished => break :process_next,
449 }
450 }
451
452 if (self.unpacked_size) |unpacked_size| {
453 if (buffer.len != unpacked_size) {
454 return error.CorruptInput;
455 }
64456 }
65457
66 pub fn read(self: *Self, output: []u8) Error!usize {
67 const writer = self.to_read.writer(self.allocator);
68 while (self.to_read.items.len < output.len) {
69 switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
70 .continue_ => {},
71 .finished => {
72 try self.buffer.finish(writer);
73 break;
74 },
458 return .finished;
459 }
460
461 fn decodeLiteral(
462 self: *Decode,
463 br: *std.io.BufferedReader,
464 buffer: anytype,
465 decoder: *RangeDecoder,
466 update: bool,
467 bytes_read: *usize,
468 ) !u8 {
469 const def_prev_byte = 0;
470 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
471
472 var result: usize = 1;
473 const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
474 (prev_byte >> (8 - self.properties.lc));
475 const probs = try self.literal_probs.getMut(lit_state);
476
477 if (self.state >= 7) {
478 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
479
480 while (result < 0x100) {
481 const match_bit = (match_byte >> 7) & 1;
482 match_byte <<= 1;
483 const bit = @intFromBool(try decoder.decodeBit(
484 br,
485 &probs[((@as(usize, 1) + match_bit) << 8) + result],
486 update,
487 bytes_read,
488 ));
489 result = (result << 1) ^ bit;
490 if (match_bit != bit) {
491 break;
75492 }
76493 }
77 const input = self.to_read.items;
78 const n = @min(input.len, output.len);
79 @memcpy(output[0..n], input[0..n]);
80 std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
81 self.to_read.shrinkRetainingCapacity(input.len - n);
82 return n;
83494 }
495
496 while (result < 0x100) {
497 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(br, &probs[result], update, bytes_read));
498 }
499
500 return @as(u8, @truncate(result - 0x100));
501 }
502
503 fn decodeDistance(
504 self: *Decode,
505 br: *std.io.BufferedReader,
506 decoder: *RangeDecoder,
507 length: usize,
508 update: bool,
509 bytes_read: *usize,
510 ) !usize {
511 const len_state = if (length > 3) 3 else length;
512
513 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(br, decoder, update, bytes_read));
514 if (pos_slot < 4)
515 return pos_slot;
516
517 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
518 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
519
520 if (pos_slot < 14) {
521 result += try decoder.parseReverseBitTree(
522 br,
523 num_direct_bits,
524 &self.pos_decoders,
525 result - pos_slot,
526 update,
527 bytes_read,
528 );
529 } else {
530 result += @as(usize, try decoder.get(br, num_direct_bits - 4, bytes_read)) << 4;
531 result += try self.align_decoder.parseReverse(br, decoder, update, bytes_read);
532 }
533
534 return result;
535 }
536};
537
538pub const Decompress = struct {
539 pub const Error =
540 anyerror ||
541 Allocator.Error ||
542 error{ CorruptInput, EndOfStream, Overflow };
543
544 allocator: Allocator,
545 in_reader: *std.io.BufferedReader,
546 to_read: std.ArrayListUnmanaged(u8),
547
548 buffer: LzCircularBuffer,
549 decoder: RangeDecoder,
550 state: Decode,
551
552 pub fn initOptions(allocator: Allocator, br: *std.io.BufferedReader, options: Decode.Options) !Decompress {
553 const params = try Decode.Params.readHeader(br, options);
554 return init(allocator, br, params, options.memlimit);
555 }
556
557 pub fn init(allocator: Allocator, source: *std.io.BufferedReader, params: Decode.Params, memlimit: ?usize) !Decompress {
558 return .{
559 .allocator = allocator,
560 .in_reader = source,
561 .to_read = .{},
562
563 .buffer = LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
564 .decoder = try RangeDecoder.init(source),
565 .state = try Decode.init(allocator, params.properties, params.unpacked_size),
566 };
567 }
568
569 pub fn reader(self: *Decompress) std.io.Reader {
570 return .{ .context = self };
571 }
572
573 pub fn deinit(self: *Decompress) void {
574 self.to_read.deinit(self.allocator);
575 self.buffer.deinit(self.allocator);
576 self.state.deinit(self.allocator);
577 self.* = undefined;
578 }
579
580 pub fn read(self: *Decompress, output: []u8) Error!usize {
581 const bw = self.to_read.writer(self.allocator);
582 while (self.to_read.items.len < output.len) {
583 switch (try self.state.process(self.allocator, self.in_reader, bw, &self.buffer, &self.decoder)) {
584 .cont => {},
585 .finished => {
586 try self.buffer.finish(bw);
587 break;
588 },
589 }
590 }
591 const input = self.to_read.items;
592 const n = @min(input.len, output.len);
593 @memcpy(output[0..n], input[0..n]);
594 std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
595 self.to_read.shrinkRetainingCapacity(input.len - n);
596 return n;
597 }
598};
599
600/// A circular buffer for LZ sequences
601const LzCircularBuffer = struct {
602 /// Circular buffer
603 buf: std.ArrayListUnmanaged(u8),
604
605 /// Length of the buffer
606 dict_size: usize,
607
608 /// Buffer memory limit
609 memlimit: usize,
610
611 /// Current position
612 cursor: usize,
613
614 /// Total number of bytes sent through the buffer
615 len: usize,
616
617 const Self = @This();
618
619 pub fn init(dict_size: usize, memlimit: usize) Self {
620 return Self{
621 .buf = .{},
622 .dict_size = dict_size,
623 .memlimit = memlimit,
624 .cursor = 0,
625 .len = 0,
626 };
627 }
628
629 pub fn get(self: Self, index: usize) u8 {
630 return if (0 <= index and index < self.buf.items.len)
631 self.buf.items[index]
632 else
633 0;
634 }
635
636 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
637 if (index >= self.memlimit) {
638 return error.CorruptInput;
639 }
640 try self.buf.ensureTotalCapacity(allocator, index + 1);
641 while (self.buf.items.len < index) {
642 self.buf.appendAssumeCapacity(0);
643 }
644 self.buf.appendAssumeCapacity(value);
645 }
646
647 /// Retrieve the last byte or return a default
648 pub fn lastOr(self: Self, lit: u8) u8 {
649 return if (self.len == 0)
650 lit
651 else
652 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
653 }
654
655 /// Retrieve the n-th last byte
656 pub fn lastN(self: Self, dist: usize) !u8 {
657 if (dist > self.dict_size or dist > self.len) {
658 return error.CorruptInput;
659 }
660
661 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
662 return self.get(offset);
663 }
664
665 /// Append a literal
666 pub fn appendLiteral(
667 self: *Self,
668 allocator: Allocator,
669 lit: u8,
670 bw: *std.io.BufferedWriter,
671 ) anyerror!void {
672 try self.set(allocator, self.cursor, lit);
673 self.cursor += 1;
674 self.len += 1;
675
676 // Flush the circular buffer to the output
677 if (self.cursor == self.dict_size) {
678 try bw.writeAll(self.buf.items);
679 self.cursor = 0;
680 }
681 }
682
683 /// Fetch an LZ sequence (length, distance) from inside the buffer
684 pub fn appendLz(
685 self: *Self,
686 allocator: Allocator,
687 len: usize,
688 dist: usize,
689 bw: *std.io.BufferedWriter,
690 ) anyerror!void {
691 if (dist > self.dict_size or dist > self.len) {
692 return error.CorruptInput;
693 }
694
695 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
696 var i: usize = 0;
697 while (i < len) : (i += 1) {
698 const x = self.get(offset);
699 try self.appendLiteral(allocator, x, bw);
700 offset += 1;
701 if (offset == self.dict_size) {
702 offset = 0;
703 }
704 }
705 }
706
707 pub fn finish(self: *Self, bw: *std.io.BufferedWriter) anyerror!void {
708 if (self.cursor > 0) {
709 try bw.writeAll(self.buf.items[0..self.cursor]);
710 self.cursor = 0;
711 }
712 }
713
714 pub fn deinit(self: *Self, allocator: Allocator) void {
715 self.buf.deinit(allocator);
716 self.* = undefined;
717 }
718};
719
720pub fn Vec2D(comptime T: type) type {
721 return struct {
722 data: []T,
723 cols: usize,
724
725 const Self = @This();
726
727 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
728 const len = try math.mul(usize, size[0], size[1]);
729 const data = try allocator.alloc(T, len);
730 @memset(data, value);
731 return Self{
732 .data = data,
733 .cols = size[1],
734 };
735 }
736
737 pub fn deinit(self: *Self, allocator: Allocator) void {
738 allocator.free(self.data);
739 self.* = undefined;
740 }
741
742 pub fn fill(self: *Self, value: T) void {
743 @memset(self.data, value);
744 }
745
746 inline fn _get(self: Self, row: usize) ![]T {
747 const start_row = try math.mul(usize, row, self.cols);
748 const end_row = try math.add(usize, start_row, self.cols);
749 return self.data[start_row..end_row];
750 }
751
752 pub fn get(self: Self, row: usize) ![]const T {
753 return self._get(row);
754 }
755
756 pub fn getMut(self: *Self, row: usize) ![]T {
757 return self._get(row);
758 }
759 };
760}
761
762test "Vec2D init" {
763 const allocator = testing.allocator;
764 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
765 defer vec2d.deinit(allocator);
766
767 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
768 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
769}
770
771test "Vec2D init overflow" {
772 const allocator = testing.allocator;
773 try expectError(
774 error.Overflow,
775 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
776 );
777}
778
779test "Vec2D fill" {
780 const allocator = testing.allocator;
781 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
782 defer vec2d.deinit(allocator);
783
784 vec2d.fill(7);
785
786 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
787 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
788}
789
790test "Vec2D get" {
791 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
792 const vec2d = Vec2D(i32){
793 .data = &data,
794 .cols = 2,
84795 };
796
797 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
798 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
799 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
800 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
801}
802
803test "Vec2D getMut" {
804 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
805 var vec2d = Vec2D(i32){
806 .data = &data,
807 .cols = 2,
808 };
809
810 const row = try vec2d.getMut(1);
811 row[1] = 9;
812
813 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
814 // (1, 1) should be 9.
815 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
816 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
817 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
818}
819
820test "Vec2D get multiplication overflow" {
821 const allocator = testing.allocator;
822 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
823 defer matrix.deinit(allocator);
824
825 const row = (math.maxInt(usize) / 4) + 1;
826 try expectError(error.Overflow, matrix.get(row));
827 try expectError(error.Overflow, matrix.getMut(row));
828}
829
830test "Vec2D get addition overflow" {
831 const allocator = testing.allocator;
832 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
833 defer matrix.deinit(allocator);
834
835 const row = math.maxInt(usize) / 5;
836 try expectError(error.Overflow, matrix.get(row));
837 try expectError(error.Overflow, matrix.getMut(row));
838}
839
840fn testDecompress(compressed: []const u8) ![]u8 {
841 const allocator = std.testing.allocator;
842 var br: std.io.BufferedReader = undefined;
843 br.initFixed(compressed);
844 var decompressor = try Decompress.initOptions(allocator, &br, .{});
845 defer decompressor.deinit();
846 const reader = decompressor.reader();
847 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
848}
849
850fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
851 const allocator = std.testing.allocator;
852 const decomp = try testDecompress(compressed);
853 defer allocator.free(decomp);
854 try std.testing.expectEqualSlices(u8, expected, decomp);
855}
856
857fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
858 return std.testing.expectError(expected, testDecompress(compressed));
859}
860
861test "decompress empty world" {
862 try testDecompressEqual(
863 "",
864 &[_]u8{
865 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
866 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
867 },
868 );
869}
870
871test "decompress hello world" {
872 try testDecompressEqual(
873 "Hello world\n",
874 &[_]u8{
875 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
876 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
877 0xa5, 0xb0, 0x00,
878 },
879 );
880}
881
882test "decompress huge dict" {
883 try testDecompressEqual(
884 "Hello world\n",
885 &[_]u8{
886 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
887 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
888 0xa5, 0xb0, 0x00,
889 },
890 );
891}
892
893test "unknown size with end of payload marker" {
894 try testDecompressEqual(
895 "Hello\nWorld!\n",
896 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
897 );
898}
899
900test "known size without end of payload marker" {
901 try testDecompressEqual(
902 "Hello\nWorld!\n",
903 @embedFile("testdata/good-known_size-without_eopm.lzma"),
904 );
905}
906
907test "known size with end of payload marker" {
908 try testDecompressEqual(
909 "Hello\nWorld!\n",
910 @embedFile("testdata/good-known_size-with_eopm.lzma"),
911 );
912}
913
914test "too big uncompressed size in header" {
915 try testDecompressError(
916 error.CorruptInput,
917 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
918 );
919}
920
921test "too small uncompressed size in header" {
922 try testDecompressError(
923 error.CorruptInput,
924 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
925 );
85926}
86927
87test {
88 _ = @import("lzma/test.zig");
89 _ = @import("lzma/vec2d.zig");
928test "reading one byte" {
929 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
930 var br: std.io.BufferedReader = undefined;
931 br.initFixed(compressed);
932 var decompressor = try Decompress.initOptions(std.testing.allocator, &br, .{});
933 defer decompressor.deinit();
934 var buffer = [1]u8{0};
935 _ = try decompressor.read(buffer[0..]);
90936}
lib/std/compress/lzma/decode.zig deleted-539
......@@ -1,539 +0,0 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
6pub const lzbuffer = @import("decode/lzbuffer.zig");
7
8const LzCircularBuffer = lzbuffer.LzCircularBuffer;
9const Vec2D = @import("vec2d.zig").Vec2D;
10
11pub const RangeDecoder = struct {
12 range: u32,
13 code: u32,
14
15 pub fn init(br: *std.io.BufferedReader) !RangeDecoder {
16 const reserved = try br.takeByte();
17 if (reserved != 0) {
18 return error.CorruptInput;
19 }
20 return .{
21 .range = 0xFFFF_FFFF,
22 .code = try br.readInt(u32, .big),
23 };
24 }
25
26 pub inline fn isFinished(self: RangeDecoder) bool {
27 return self.code == 0;
28 }
29
30 inline fn normalize(self: *RangeDecoder, br: *std.io.BufferedReader) !void {
31 if (self.range < 0x0100_0000) {
32 self.range <<= 8;
33 self.code = (self.code << 8) ^ @as(u32, try br.takeByte());
34 }
35 }
36
37 inline fn getBit(self: *RangeDecoder, br: *std.io.BufferedReader) !bool {
38 self.range >>= 1;
39
40 const bit = self.code >= self.range;
41 if (bit)
42 self.code -= self.range;
43
44 try self.normalize(br);
45 return bit;
46 }
47
48 pub fn get(self: *RangeDecoder, br: *std.io.BufferedReader, count: usize) !u32 {
49 var result: u32 = 0;
50 var i: usize = 0;
51 while (i < count) : (i += 1)
52 result = (result << 1) ^ @intFromBool(try self.getBit(br));
53 return result;
54 }
55
56 pub inline fn decodeBit(self: *RangeDecoder, br: *std.io.BufferedReader, prob: *u16, update: bool) !bool {
57 const bound = (self.range >> 11) * prob.*;
58
59 if (self.code < bound) {
60 if (update)
61 prob.* += (0x800 - prob.*) >> 5;
62 self.range = bound;
63
64 try self.normalize(br);
65 return false;
66 } else {
67 if (update)
68 prob.* -= prob.* >> 5;
69 self.code -= bound;
70 self.range -= bound;
71
72 try self.normalize(br);
73 return true;
74 }
75 }
76
77 fn parseBitTree(
78 self: *RangeDecoder,
79 br: *std.io.BufferedReader,
80 num_bits: u5,
81 probs: []u16,
82 update: bool,
83 ) !u32 {
84 var tmp: u32 = 1;
85 var i: @TypeOf(num_bits) = 0;
86 while (i < num_bits) : (i += 1) {
87 const bit = try self.decodeBit(br, &probs[tmp], update);
88 tmp = (tmp << 1) ^ @intFromBool(bit);
89 }
90 return tmp - (@as(u32, 1) << num_bits);
91 }
92
93 pub fn parseReverseBitTree(
94 self: *RangeDecoder,
95 br: *std.io.BufferedReader,
96 num_bits: u5,
97 probs: []u16,
98 offset: usize,
99 update: bool,
100 ) !u32 {
101 var result: u32 = 0;
102 var tmp: usize = 1;
103 var i: @TypeOf(num_bits) = 0;
104 while (i < num_bits) : (i += 1) {
105 const bit = @intFromBool(try self.decodeBit(br, &probs[offset + tmp], update));
106 tmp = (tmp << 1) ^ bit;
107 result ^= @as(u32, bit) << i;
108 }
109 return result;
110 }
111};
112
113pub fn BitTree(comptime num_bits: usize) type {
114 return struct {
115 probs: [1 << num_bits]u16 = @splat(0x400),
116
117 const Self = @This();
118
119 pub fn parse(
120 self: *Self,
121 br: *std.io.BufferedReader,
122 decoder: *RangeDecoder,
123 update: bool,
124 ) !u32 {
125 return decoder.parseBitTree(br, num_bits, &self.probs, update);
126 }
127
128 pub fn parseReverse(
129 self: *Self,
130 br: *std.io.BufferedReader,
131 decoder: *RangeDecoder,
132 update: bool,
133 ) !u32 {
134 return decoder.parseReverseBitTree(br, num_bits, &self.probs, 0, update);
135 }
136
137 pub fn reset(self: *Self) void {
138 @memset(&self.probs, 0x400);
139 }
140 };
141}
142
143pub const LenDecoder = struct {
144 choice: u16 = 0x400,
145 choice2: u16 = 0x400,
146 low_coder: [16]BitTree(3) = @splat(.{}),
147 mid_coder: [16]BitTree(3) = @splat(.{}),
148 high_coder: BitTree(8) = .{},
149
150 pub fn decode(
151 self: *LenDecoder,
152 br: *std.io.BufferedReader,
153 decoder: *RangeDecoder,
154 pos_state: usize,
155 update: bool,
156 ) !usize {
157 if (!try decoder.decodeBit(br, &self.choice, update)) {
158 return @as(usize, try self.low_coder[pos_state].parse(br, decoder, update));
159 } else if (!try decoder.decodeBit(br, &self.choice2, update)) {
160 return @as(usize, try self.mid_coder[pos_state].parse(br, decoder, update)) + 8;
161 } else {
162 return @as(usize, try self.high_coder.parse(br, decoder, update)) + 16;
163 }
164 }
165
166 pub fn reset(self: *LenDecoder) void {
167 self.choice = 0x400;
168 self.choice2 = 0x400;
169 for (&self.low_coder) |*t| t.reset();
170 for (&self.mid_coder) |*t| t.reset();
171 self.high_coder.reset();
172 }
173};
174
175pub const Options = struct {
176 unpacked_size: UnpackedSize = .read_from_header,
177 memlimit: ?usize = null,
178 allow_incomplete: bool = false,
179};
180
181pub const UnpackedSize = union(enum) {
182 read_from_header,
183 read_header_but_use_provided: ?u64,
184 use_provided: ?u64,
185};
186
187const ProcessingStatus = enum {
188 continue_,
189 finished,
190};
191
192pub const Properties = struct {
193 lc: u4,
194 lp: u3,
195 pb: u3,
196
197 fn validate(self: Properties) void {
198 assert(self.lc <= 8);
199 assert(self.lp <= 4);
200 assert(self.pb <= 4);
201 }
202};
203
204pub const Params = struct {
205 properties: Properties,
206 dict_size: u32,
207 unpacked_size: ?u64,
208
209 pub fn readHeader(reader: anytype, options: Options) !Params {
210 var props = try reader.readByte();
211 if (props >= 225) {
212 return error.CorruptInput;
213 }
214
215 const lc = @as(u4, @intCast(props % 9));
216 props /= 9;
217 const lp = @as(u3, @intCast(props % 5));
218 props /= 5;
219 const pb = @as(u3, @intCast(props));
220
221 const dict_size_provided = try reader.readInt(u32, .little);
222 const dict_size = @max(0x1000, dict_size_provided);
223
224 const unpacked_size = switch (options.unpacked_size) {
225 .read_from_header => blk: {
226 const unpacked_size_provided = try reader.readInt(u64, .little);
227 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
228 break :blk if (marker_mandatory)
229 null
230 else
231 unpacked_size_provided;
232 },
233 .read_header_but_use_provided => |x| blk: {
234 _ = try reader.readInt(u64, .little);
235 break :blk x;
236 },
237 .use_provided => |x| x,
238 };
239
240 return Params{
241 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
242 .dict_size = dict_size,
243 .unpacked_size = unpacked_size,
244 };
245 }
246};
247
248pub const DecoderState = struct {
249 lzma_props: Properties,
250 unpacked_size: ?u64,
251 literal_probs: Vec2D(u16),
252 pos_slot_decoder: [4]BitTree(6),
253 align_decoder: BitTree(4),
254 pos_decoders: [115]u16,
255 is_match: [192]u16,
256 is_rep: [12]u16,
257 is_rep_g0: [12]u16,
258 is_rep_g1: [12]u16,
259 is_rep_g2: [12]u16,
260 is_rep_0long: [192]u16,
261 state: usize,
262 rep: [4]usize,
263 len_decoder: LenDecoder,
264 rep_len_decoder: LenDecoder,
265
266 pub fn init(
267 allocator: Allocator,
268 lzma_props: Properties,
269 unpacked_size: ?u64,
270 ) !DecoderState {
271 return .{
272 .lzma_props = lzma_props,
273 .unpacked_size = unpacked_size,
274 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
275 .pos_slot_decoder = @splat(.{}),
276 .align_decoder = .{},
277 .pos_decoders = @splat(0x400),
278 .is_match = @splat(0x400),
279 .is_rep = @splat(0x400),
280 .is_rep_g0 = @splat(0x400),
281 .is_rep_g1 = @splat(0x400),
282 .is_rep_g2 = @splat(0x400),
283 .is_rep_0long = @splat(0x400),
284 .state = 0,
285 .rep = @splat(0),
286 .len_decoder = .{},
287 .rep_len_decoder = .{},
288 };
289 }
290
291 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
292 self.literal_probs.deinit(allocator);
293 self.* = undefined;
294 }
295
296 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
297 new_props.validate();
298 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
299 self.literal_probs.fill(0x400);
300 } else {
301 self.literal_probs.deinit(allocator);
302 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
303 }
304
305 self.lzma_props = new_props;
306 for (&self.pos_slot_decoder) |*t| t.reset();
307 self.align_decoder.reset();
308 self.pos_decoders = @splat(0x400);
309 self.is_match = @splat(0x400);
310 self.is_rep = @splat(0x400);
311 self.is_rep_g0 = @splat(0x400);
312 self.is_rep_g1 = @splat(0x400);
313 self.is_rep_g2 = @splat(0x400);
314 self.is_rep_0long = @splat(0x400);
315 self.state = 0;
316 self.rep = @splat(0);
317 self.len_decoder.reset();
318 self.rep_len_decoder.reset();
319 }
320
321 fn processNextInner(
322 self: *DecoderState,
323 allocator: Allocator,
324 reader: anytype,
325 writer: anytype,
326 buffer: anytype,
327 decoder: *RangeDecoder,
328 update: bool,
329 ) !ProcessingStatus {
330 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
331
332 if (!try decoder.decodeBit(
333 reader,
334 &self.is_match[(self.state << 4) + pos_state],
335 update,
336 )) {
337 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
338
339 if (update) {
340 try buffer.appendLiteral(allocator, byte, writer);
341
342 self.state = if (self.state < 4)
343 0
344 else if (self.state < 10)
345 self.state - 3
346 else
347 self.state - 6;
348 }
349 return .continue_;
350 }
351
352 var len: usize = undefined;
353 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
354 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
355 if (!try decoder.decodeBit(
356 reader,
357 &self.is_rep_0long[(self.state << 4) + pos_state],
358 update,
359 )) {
360 if (update) {
361 self.state = if (self.state < 7) 9 else 11;
362 const dist = self.rep[0] + 1;
363 try buffer.appendLz(allocator, 1, dist, writer);
364 }
365 return .continue_;
366 }
367 } else {
368 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
369 1
370 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
371 2
372 else
373 3;
374 if (update) {
375 const dist = self.rep[idx];
376 var i = idx;
377 while (i > 0) : (i -= 1) {
378 self.rep[i] = self.rep[i - 1];
379 }
380 self.rep[0] = dist;
381 }
382 }
383
384 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
385
386 if (update) {
387 self.state = if (self.state < 7) 8 else 11;
388 }
389 } else {
390 if (update) {
391 self.rep[3] = self.rep[2];
392 self.rep[2] = self.rep[1];
393 self.rep[1] = self.rep[0];
394 }
395
396 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
397
398 if (update) {
399 self.state = if (self.state < 7) 7 else 10;
400 }
401
402 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
403
404 if (update) {
405 self.rep[0] = rep_0;
406 if (self.rep[0] == 0xFFFF_FFFF) {
407 if (decoder.isFinished()) {
408 return .finished;
409 }
410 return error.CorruptInput;
411 }
412 }
413 }
414
415 if (update) {
416 len += 2;
417
418 const dist = self.rep[0] + 1;
419 try buffer.appendLz(allocator, len, dist, writer);
420 }
421
422 return .continue_;
423 }
424
425 fn processNext(
426 self: *DecoderState,
427 allocator: Allocator,
428 reader: anytype,
429 writer: anytype,
430 buffer: anytype,
431 decoder: *RangeDecoder,
432 ) !ProcessingStatus {
433 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
434 }
435
436 pub fn process(
437 self: *DecoderState,
438 allocator: Allocator,
439 reader: anytype,
440 writer: anytype,
441 buffer: anytype,
442 decoder: *RangeDecoder,
443 ) !ProcessingStatus {
444 process_next: {
445 if (self.unpacked_size) |unpacked_size| {
446 if (buffer.len >= unpacked_size) {
447 break :process_next;
448 }
449 } else if (decoder.isFinished()) {
450 break :process_next;
451 }
452
453 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
454 .continue_ => return .continue_,
455 .finished => break :process_next,
456 }
457 }
458
459 if (self.unpacked_size) |unpacked_size| {
460 if (buffer.len != unpacked_size) {
461 return error.CorruptInput;
462 }
463 }
464
465 return .finished;
466 }
467
468 fn decodeLiteral(
469 self: *DecoderState,
470 reader: anytype,
471 buffer: anytype,
472 decoder: *RangeDecoder,
473 update: bool,
474 ) !u8 {
475 const def_prev_byte = 0;
476 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
477
478 var result: usize = 1;
479 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
480 (prev_byte >> (8 - self.lzma_props.lc));
481 const probs = try self.literal_probs.getMut(lit_state);
482
483 if (self.state >= 7) {
484 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
485
486 while (result < 0x100) {
487 const match_bit = (match_byte >> 7) & 1;
488 match_byte <<= 1;
489 const bit = @intFromBool(try decoder.decodeBit(
490 reader,
491 &probs[((@as(usize, 1) + match_bit) << 8) + result],
492 update,
493 ));
494 result = (result << 1) ^ bit;
495 if (match_bit != bit) {
496 break;
497 }
498 }
499 }
500
501 while (result < 0x100) {
502 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
503 }
504
505 return @as(u8, @truncate(result - 0x100));
506 }
507
508 fn decodeDistance(
509 self: *DecoderState,
510 reader: anytype,
511 decoder: *RangeDecoder,
512 length: usize,
513 update: bool,
514 ) !usize {
515 const len_state = if (length > 3) 3 else length;
516
517 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
518 if (pos_slot < 4)
519 return pos_slot;
520
521 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
522 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
523
524 if (pos_slot < 14) {
525 result += try decoder.parseReverseBitTree(
526 reader,
527 num_direct_bits,
528 &self.pos_decoders,
529 result - pos_slot,
530 update,
531 );
532 } else {
533 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
534 result += try self.align_decoder.parseReverse(reader, decoder, update);
535 }
536
537 return result;
538 }
539};
lib/std/compress/lzma/decode/lzbuffer.zig deleted-228
......@@ -1,228 +0,0 @@
1const std = @import("../../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6
7/// An accumulating buffer for LZ sequences
8pub const LzAccumBuffer = struct {
9 /// Buffer
10 buf: ArrayListUnmanaged(u8),
11
12 /// Buffer memory limit
13 memlimit: usize,
14
15 /// Total number of bytes sent through the buffer
16 len: usize,
17
18 const Self = @This();
19
20 pub fn init(memlimit: usize) Self {
21 return Self{
22 .buf = .{},
23 .memlimit = memlimit,
24 .len = 0,
25 };
26 }
27
28 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
29 try self.buf.append(allocator, byte);
30 self.len += 1;
31 }
32
33 /// Reset the internal dictionary
34 pub fn reset(self: *Self, writer: anytype) !void {
35 try writer.writeAll(self.buf.items);
36 self.buf.clearRetainingCapacity();
37 self.len = 0;
38 }
39
40 /// Retrieve the last byte or return a default
41 pub fn lastOr(self: Self, lit: u8) u8 {
42 const buf_len = self.buf.items.len;
43 return if (buf_len == 0)
44 lit
45 else
46 self.buf.items[buf_len - 1];
47 }
48
49 /// Retrieve the n-th last byte
50 pub fn lastN(self: Self, dist: usize) !u8 {
51 const buf_len = self.buf.items.len;
52 if (dist > buf_len) {
53 return error.CorruptInput;
54 }
55
56 return self.buf.items[buf_len - dist];
57 }
58
59 /// Append a literal
60 pub fn appendLiteral(
61 self: *Self,
62 allocator: Allocator,
63 lit: u8,
64 writer: anytype,
65 ) !void {
66 _ = writer;
67 if (self.len >= self.memlimit) {
68 return error.CorruptInput;
69 }
70 try self.buf.append(allocator, lit);
71 self.len += 1;
72 }
73
74 /// Fetch an LZ sequence (length, distance) from inside the buffer
75 pub fn appendLz(
76 self: *Self,
77 allocator: Allocator,
78 len: usize,
79 dist: usize,
80 writer: anytype,
81 ) !void {
82 _ = writer;
83
84 const buf_len = self.buf.items.len;
85 if (dist > buf_len) {
86 return error.CorruptInput;
87 }
88
89 var offset = buf_len - dist;
90 var i: usize = 0;
91 while (i < len) : (i += 1) {
92 const x = self.buf.items[offset];
93 try self.buf.append(allocator, x);
94 offset += 1;
95 }
96 self.len += len;
97 }
98
99 pub fn finish(self: *Self, writer: anytype) !void {
100 try writer.writeAll(self.buf.items);
101 self.buf.clearRetainingCapacity();
102 }
103
104 pub fn deinit(self: *Self, allocator: Allocator) void {
105 self.buf.deinit(allocator);
106 self.* = undefined;
107 }
108};
109
110/// A circular buffer for LZ sequences
111pub const LzCircularBuffer = struct {
112 /// Circular buffer
113 buf: ArrayListUnmanaged(u8),
114
115 /// Length of the buffer
116 dict_size: usize,
117
118 /// Buffer memory limit
119 memlimit: usize,
120
121 /// Current position
122 cursor: usize,
123
124 /// Total number of bytes sent through the buffer
125 len: usize,
126
127 const Self = @This();
128
129 pub fn init(dict_size: usize, memlimit: usize) Self {
130 return Self{
131 .buf = .{},
132 .dict_size = dict_size,
133 .memlimit = memlimit,
134 .cursor = 0,
135 .len = 0,
136 };
137 }
138
139 pub fn get(self: Self, index: usize) u8 {
140 return if (0 <= index and index < self.buf.items.len)
141 self.buf.items[index]
142 else
143 0;
144 }
145
146 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
147 if (index >= self.memlimit) {
148 return error.CorruptInput;
149 }
150 try self.buf.ensureTotalCapacity(allocator, index + 1);
151 while (self.buf.items.len < index) {
152 self.buf.appendAssumeCapacity(0);
153 }
154 self.buf.appendAssumeCapacity(value);
155 }
156
157 /// Retrieve the last byte or return a default
158 pub fn lastOr(self: Self, lit: u8) u8 {
159 return if (self.len == 0)
160 lit
161 else
162 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
163 }
164
165 /// Retrieve the n-th last byte
166 pub fn lastN(self: Self, dist: usize) !u8 {
167 if (dist > self.dict_size or dist > self.len) {
168 return error.CorruptInput;
169 }
170
171 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
172 return self.get(offset);
173 }
174
175 /// Append a literal
176 pub fn appendLiteral(
177 self: *Self,
178 allocator: Allocator,
179 lit: u8,
180 writer: anytype,
181 ) !void {
182 try self.set(allocator, self.cursor, lit);
183 self.cursor += 1;
184 self.len += 1;
185
186 // Flush the circular buffer to the output
187 if (self.cursor == self.dict_size) {
188 try writer.writeAll(self.buf.items);
189 self.cursor = 0;
190 }
191 }
192
193 /// Fetch an LZ sequence (length, distance) from inside the buffer
194 pub fn appendLz(
195 self: *Self,
196 allocator: Allocator,
197 len: usize,
198 dist: usize,
199 writer: anytype,
200 ) !void {
201 if (dist > self.dict_size or dist > self.len) {
202 return error.CorruptInput;
203 }
204
205 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
206 var i: usize = 0;
207 while (i < len) : (i += 1) {
208 const x = self.get(offset);
209 try self.appendLiteral(allocator, x, writer);
210 offset += 1;
211 if (offset == self.dict_size) {
212 offset = 0;
213 }
214 }
215 }
216
217 pub fn finish(self: *Self, writer: anytype) !void {
218 if (self.cursor > 0) {
219 try writer.writeAll(self.buf.items[0..self.cursor]);
220 self.cursor = 0;
221 }
222 }
223
224 pub fn deinit(self: *Self, allocator: Allocator) void {
225 self.buf.deinit(allocator);
226 self.* = undefined;
227 }
228};
lib/std/compress/lzma/test.zig deleted-99
......@@ -1,99 +0,0 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8) ![]u8 {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 var decompressor = try lzma.decompress(allocator, stream.reader());
8 defer decompressor.deinit();
9 const reader = decompressor.reader();
10 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
11}
12
13fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14 const allocator = std.testing.allocator;
15 const decomp = try testDecompress(compressed);
16 defer allocator.free(decomp);
17 try std.testing.expectEqualSlices(u8, expected, decomp);
18}
19
20fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
21 return std.testing.expectError(expected, testDecompress(compressed));
22}
23
24test "decompress empty world" {
25 try testDecompressEqual(
26 "",
27 &[_]u8{
28 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
29 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
30 },
31 );
32}
33
34test "decompress hello world" {
35 try testDecompressEqual(
36 "Hello world\n",
37 &[_]u8{
38 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
39 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
40 0xa5, 0xb0, 0x00,
41 },
42 );
43}
44
45test "decompress huge dict" {
46 try testDecompressEqual(
47 "Hello world\n",
48 &[_]u8{
49 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
50 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
51 0xa5, 0xb0, 0x00,
52 },
53 );
54}
55
56test "unknown size with end of payload marker" {
57 try testDecompressEqual(
58 "Hello\nWorld!\n",
59 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
60 );
61}
62
63test "known size without end of payload marker" {
64 try testDecompressEqual(
65 "Hello\nWorld!\n",
66 @embedFile("testdata/good-known_size-without_eopm.lzma"),
67 );
68}
69
70test "known size with end of payload marker" {
71 try testDecompressEqual(
72 "Hello\nWorld!\n",
73 @embedFile("testdata/good-known_size-with_eopm.lzma"),
74 );
75}
76
77test "too big uncompressed size in header" {
78 try testDecompressError(
79 error.CorruptInput,
80 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
81 );
82}
83
84test "too small uncompressed size in header" {
85 try testDecompressError(
86 error.CorruptInput,
87 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
88 );
89}
90
91test "reading one byte" {
92 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
93 var stream = std.io.fixedBufferStream(compressed);
94 var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());
95 defer decompressor.deinit();
96
97 var buffer = [1]u8{0};
98 _ = try decompressor.read(buffer[0..]);
99}
lib/std/compress/lzma/vec2d.zig deleted-128
......@@ -1,128 +0,0 @@
1const std = @import("../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub fn Vec2D(comptime T: type) type {
7 return struct {
8 data: []T,
9 cols: usize,
10
11 const Self = @This();
12
13 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
14 const len = try math.mul(usize, size[0], size[1]);
15 const data = try allocator.alloc(T, len);
16 @memset(data, value);
17 return Self{
18 .data = data,
19 .cols = size[1],
20 };
21 }
22
23 pub fn deinit(self: *Self, allocator: Allocator) void {
24 allocator.free(self.data);
25 self.* = undefined;
26 }
27
28 pub fn fill(self: *Self, value: T) void {
29 @memset(self.data, value);
30 }
31
32 inline fn _get(self: Self, row: usize) ![]T {
33 const start_row = try math.mul(usize, row, self.cols);
34 const end_row = try math.add(usize, start_row, self.cols);
35 return self.data[start_row..end_row];
36 }
37
38 pub fn get(self: Self, row: usize) ![]const T {
39 return self._get(row);
40 }
41
42 pub fn getMut(self: *Self, row: usize) ![]T {
43 return self._get(row);
44 }
45 };
46}
47
48const testing = std.testing;
49const expectEqualSlices = std.testing.expectEqualSlices;
50const expectError = std.testing.expectError;
51
52test "init" {
53 const allocator = testing.allocator;
54 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
55 defer vec2d.deinit(allocator);
56
57 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
58 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
59}
60
61test "init overflow" {
62 const allocator = testing.allocator;
63 try expectError(
64 error.Overflow,
65 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
66 );
67}
68
69test "fill" {
70 const allocator = testing.allocator;
71 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
72 defer vec2d.deinit(allocator);
73
74 vec2d.fill(7);
75
76 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
77 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
78}
79
80test "get" {
81 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
82 const vec2d = Vec2D(i32){
83 .data = &data,
84 .cols = 2,
85 };
86
87 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
88 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
89 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
90 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
91}
92
93test "getMut" {
94 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
95 var vec2d = Vec2D(i32){
96 .data = &data,
97 .cols = 2,
98 };
99
100 const row = try vec2d.getMut(1);
101 row[1] = 9;
102
103 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
104 // (1, 1) should be 9.
105 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
106 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
107 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
108}
109
110test "get multiplication overflow" {
111 const allocator = testing.allocator;
112 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
113 defer matrix.deinit(allocator);
114
115 const row = (math.maxInt(usize) / 4) + 1;
116 try expectError(error.Overflow, matrix.get(row));
117 try expectError(error.Overflow, matrix.getMut(row));
118}
119
120test "get addition overflow" {
121 const allocator = testing.allocator;
122 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
123 defer matrix.deinit(allocator);
124
125 const row = math.maxInt(usize) / 5;
126 try expectError(error.Overflow, matrix.get(row));
127 try expectError(error.Overflow, matrix.getMut(row));
128}
lib/std/compress/lzma2.zig+268-7
......@@ -1,15 +1,276 @@
11const std = @import("../std.zig");
22const Allocator = std.mem.Allocator;
3const lzma = std.compress.lzma;
34
4pub const decode = @import("lzma2/decode.zig");
5
6pub fn decompress(allocator: Allocator, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
7 var decoder = try decode.Decoder.init(allocator);
8 defer decoder.deinit(allocator);
9 return decoder.decompress(allocator, reader, writer);
5pub fn decompress(gpa: Allocator, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
6 var decoder = try Decode.init(gpa);
7 defer decoder.deinit(gpa);
8 return decoder.decompress(gpa, reader, writer);
109}
1110
12test {
11pub const Decode = struct {
12 lzma1: lzma.Decode,
13
14 pub fn init(allocator: Allocator) !Decode {
15 return .{
16 .lzma1 = try lzma.Decode.init(
17 allocator,
18 .{
19 .lc = 0,
20 .lp = 0,
21 .pb = 0,
22 },
23 null,
24 ),
25 };
26 }
27
28 pub fn deinit(self: *Decode, allocator: Allocator) void {
29 self.lzma1.deinit(allocator);
30 self.* = undefined;
31 }
32
33 pub fn decompress(
34 self: *Decode,
35 allocator: Allocator,
36 reader: *std.io.BufferedReader,
37 writer: *std.io.BufferedWriter,
38 ) !void {
39 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
40 defer accum.deinit(allocator);
41
42 while (true) {
43 const status = try reader.takeByte();
44
45 switch (status) {
46 0 => break,
47 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
48 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
49 else => try self.parseLzma(allocator, reader, writer, &accum, status),
50 }
51 }
52
53 try accum.finish(writer);
54 }
55
56 fn parseLzma(
57 self: *Decode,
58 allocator: Allocator,
59 br: *std.io.BufferedReader,
60 writer: *std.io.BufferedWriter,
61 accum: *LzAccumBuffer,
62 status: u8,
63 ) !void {
64 if (status & 0x80 == 0) {
65 return error.CorruptInput;
66 }
67
68 const Reset = struct {
69 dict: bool,
70 state: bool,
71 props: bool,
72 };
73
74 const reset = switch ((status >> 5) & 0x3) {
75 0 => Reset{
76 .dict = false,
77 .state = false,
78 .props = false,
79 },
80 1 => Reset{
81 .dict = false,
82 .state = true,
83 .props = false,
84 },
85 2 => Reset{
86 .dict = false,
87 .state = true,
88 .props = true,
89 },
90 3 => Reset{
91 .dict = true,
92 .state = true,
93 .props = true,
94 },
95 else => unreachable,
96 };
97
98 const unpacked_size = blk: {
99 var tmp: u64 = status & 0x1F;
100 tmp <<= 16;
101 tmp |= try br.takeInt(u16, .big);
102 break :blk tmp + 1;
103 };
104
105 const packed_size = blk: {
106 const tmp: u17 = try br.takeInt(u16, .big);
107 break :blk tmp + 1;
108 };
109
110 if (reset.dict) {
111 try accum.reset(writer);
112 }
113
114 if (reset.state) {
115 var new_props = self.lzma1.properties;
116
117 if (reset.props) {
118 var props = try br.takeByte();
119 if (props >= 225) {
120 return error.CorruptInput;
121 }
122
123 const lc = @as(u4, @intCast(props % 9));
124 props /= 9;
125 const lp = @as(u3, @intCast(props % 5));
126 props /= 5;
127 const pb = @as(u3, @intCast(props));
128
129 if (lc + lp > 4) {
130 return error.CorruptInput;
131 }
132
133 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
134 }
135
136 try self.lzma1.resetState(allocator, new_props);
137 }
138
139 self.lzma1.unpacked_size = unpacked_size + accum.len;
140
141 var range_decoder: lzma.RangeDecoder = undefined;
142 var bytes_read = try lzma.RangeDecoder.init(br);
143 while (try self.lzma1.process(allocator, br, writer, accum, &range_decoder, &bytes_read) == .cont) {}
144
145 if (bytes_read != packed_size) {
146 return error.CorruptInput;
147 }
148 }
149
150 fn parseUncompressed(
151 allocator: Allocator,
152 reader: *std.io.BufferedReader,
153 writer: *std.io.BufferedWriter,
154 accum: *LzAccumBuffer,
155 reset_dict: bool,
156 ) !void {
157 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
158
159 if (reset_dict) {
160 try accum.reset(writer);
161 }
162
163 var i: @TypeOf(unpacked_size) = 0;
164 while (i < unpacked_size) : (i += 1) {
165 try accum.appendByte(allocator, try reader.takeByte());
166 }
167 }
168};
169
170/// An accumulating buffer for LZ sequences
171const LzAccumBuffer = struct {
172 /// Buffer
173 buf: std.ArrayListUnmanaged(u8),
174
175 /// Buffer memory limit
176 memlimit: usize,
177
178 /// Total number of bytes sent through the buffer
179 len: usize,
180
181 const Self = @This();
182
183 pub fn init(memlimit: usize) Self {
184 return Self{
185 .buf = .{},
186 .memlimit = memlimit,
187 .len = 0,
188 };
189 }
190
191 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
192 try self.buf.append(allocator, byte);
193 self.len += 1;
194 }
195
196 /// Reset the internal dictionary
197 pub fn reset(self: *Self, writer: anytype) !void {
198 try writer.writeAll(self.buf.items);
199 self.buf.clearRetainingCapacity();
200 self.len = 0;
201 }
202
203 /// Retrieve the last byte or return a default
204 pub fn lastOr(self: Self, lit: u8) u8 {
205 const buf_len = self.buf.items.len;
206 return if (buf_len == 0)
207 lit
208 else
209 self.buf.items[buf_len - 1];
210 }
211
212 /// Retrieve the n-th last byte
213 pub fn lastN(self: Self, dist: usize) !u8 {
214 const buf_len = self.buf.items.len;
215 if (dist > buf_len) {
216 return error.CorruptInput;
217 }
218
219 return self.buf.items[buf_len - dist];
220 }
221
222 /// Append a literal
223 pub fn appendLiteral(
224 self: *Self,
225 allocator: Allocator,
226 lit: u8,
227 writer: anytype,
228 ) !void {
229 _ = writer;
230 if (self.len >= self.memlimit) {
231 return error.CorruptInput;
232 }
233 try self.buf.append(allocator, lit);
234 self.len += 1;
235 }
236
237 /// Fetch an LZ sequence (length, distance) from inside the buffer
238 pub fn appendLz(
239 self: *Self,
240 allocator: Allocator,
241 len: usize,
242 dist: usize,
243 writer: anytype,
244 ) !void {
245 _ = writer;
246
247 const buf_len = self.buf.items.len;
248 if (dist > buf_len) {
249 return error.CorruptInput;
250 }
251
252 var offset = buf_len - dist;
253 var i: usize = 0;
254 while (i < len) : (i += 1) {
255 const x = self.buf.items[offset];
256 try self.buf.append(allocator, x);
257 offset += 1;
258 }
259 self.len += len;
260 }
261
262 pub fn finish(self: *Self, writer: anytype) !void {
263 try writer.writeAll(self.buf.items);
264 self.buf.clearRetainingCapacity();
265 }
266
267 pub fn deinit(self: *Self, allocator: Allocator) void {
268 self.buf.deinit(allocator);
269 self.* = undefined;
270 }
271};
272
273test decompress {
13274 const expected = "Hello\nWorld!\n";
14275 const compressed = [_]u8{
15276 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02,
lib/std/compress/lzma2/decode.zig deleted-169
......@@ -1,169 +0,0 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: *std.io.BufferedReader,
36 writer: *std.io.BufferedWriter,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.takeByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 br: *std.io.BufferedReader,
59 writer: *std.io.BufferedWriter,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try br.takeInt(u16, .big);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try br.takeInt(u16, .big);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try br.takeByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @as(u4, @intCast(props % 9));
123 props /= 9;
124 const lp = @as(u3, @intCast(props % 5));
125 props /= 5;
126 const pb = @as(u3, @intCast(props));
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter: std.io.CountingReader = .{ .child_reader = br.reader() };
141 var counter_reader = counter.reader().unbuffered();
142
143 var rangecoder = try RangeDecoder.init(&counter_reader);
144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: *std.io.BufferedReader,
154 writer: *std.io.BufferedWriter,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.takeByte());
167 }
168 }
169};
lib/std/compress/zstandard.zig+159-163
......@@ -16,191 +16,187 @@ pub const DecompressorOptions = struct {
1616 pub const default_window_buffer_len = 8 * 1024 * 1024;
1717};
1818
19pub fn Decompressor(comptime ReaderType: type) type {
20 return struct {
21 const Self = @This();
22
23 const table_size_max = types.compressed_block.table_size_max;
24
25 source: std.io.CountingReader(ReaderType),
26 state: enum { NewFrame, InFrame, LastBlock },
27 decode_state: decompress.block.DecodeState,
28 frame_context: decompress.FrameContext,
29 buffer: WindowBuffer,
30 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
31 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
32 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
33 literals_buffer: [types.block_size_max]u8,
34 sequence_buffer: [types.block_size_max]u8,
35 verify_checksum: bool,
36 checksum: ?u32,
37 current_frame_decompressed_size: usize,
38
39 const WindowBuffer = struct {
40 data: []u8 = undefined,
41 read_index: usize = 0,
42 write_index: usize = 0,
43 };
19pub const Decompressor = struct {
20 const Self = @This();
21
22 const table_size_max = types.compressed_block.table_size_max;
23
24 source: std.io.CountingReader,
25 state: enum { NewFrame, InFrame, LastBlock },
26 decode_state: decompress.block.DecodeState,
27 frame_context: decompress.FrameContext,
28 buffer: WindowBuffer,
29 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
30 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
31 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
32 literals_buffer: [types.block_size_max]u8,
33 sequence_buffer: [types.block_size_max]u8,
34 verify_checksum: bool,
35 checksum: ?u32,
36 current_frame_decompressed_size: usize,
37
38 const WindowBuffer = struct {
39 data: []u8 = undefined,
40 read_index: usize = 0,
41 write_index: usize = 0,
42 };
43
44 pub const Error = anyerror || error{
45 ChecksumFailure,
46 DictionaryIdFlagUnsupported,
47 MalformedBlock,
48 MalformedFrame,
49 OutOfMemory,
50 };
4451
45 pub const Error = ReaderType.Error || error{
46 ChecksumFailure,
47 DictionaryIdFlagUnsupported,
48 MalformedBlock,
49 MalformedFrame,
50 OutOfMemory,
52 pub fn init(source: *std.io.BufferedReader, options: DecompressorOptions) Self {
53 return .{
54 .source = std.io.countingReader(source),
55 .state = .NewFrame,
56 .decode_state = undefined,
57 .frame_context = undefined,
58 .buffer = .{ .data = options.window_buffer },
59 .literal_fse_buffer = undefined,
60 .match_fse_buffer = undefined,
61 .offset_fse_buffer = undefined,
62 .literals_buffer = undefined,
63 .sequence_buffer = undefined,
64 .verify_checksum = options.verify_checksum,
65 .checksum = undefined,
66 .current_frame_decompressed_size = undefined,
5167 };
68 }
5269
53 pub const Reader = std.io.Reader(*Self, Error, read);
54
55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {
56 return .{
57 .source = std.io.countingReader(source),
58 .state = .NewFrame,
59 .decode_state = undefined,
60 .frame_context = undefined,
61 .buffer = .{ .data = options.window_buffer },
62 .literal_fse_buffer = undefined,
63 .match_fse_buffer = undefined,
64 .offset_fse_buffer = undefined,
65 .literals_buffer = undefined,
66 .sequence_buffer = undefined,
67 .verify_checksum = options.verify_checksum,
68 .checksum = undefined,
69 .current_frame_decompressed_size = undefined,
70 };
70 fn frameInit(self: *Self) !void {
71 const source_reader = self.source;
72 switch (try decompress.decodeFrameHeader(source_reader)) {
73 .skippable => |header| {
74 try source_reader.skipBytes(header.frame_size, .{});
75 self.state = .NewFrame;
76 },
77 .zstandard => |header| {
78 const frame_context = try decompress.FrameContext.init(
79 header,
80 self.buffer.data.len,
81 self.verify_checksum,
82 );
83
84 const decode_state = decompress.block.DecodeState.init(
85 &self.literal_fse_buffer,
86 &self.match_fse_buffer,
87 &self.offset_fse_buffer,
88 );
89
90 self.decode_state = decode_state;
91 self.frame_context = frame_context;
92
93 self.checksum = null;
94 self.current_frame_decompressed_size = 0;
95
96 self.state = .InFrame;
97 },
7198 }
99 }
72100
73 fn frameInit(self: *Self) !void {
74 const source_reader = self.source.reader();
75 switch (try decompress.decodeFrameHeader(source_reader)) {
76 .skippable => |header| {
77 try source_reader.skipBytes(header.frame_size, .{});
78 self.state = .NewFrame;
79 },
80 .zstandard => |header| {
81 const frame_context = try decompress.FrameContext.init(
82 header,
83 self.buffer.data.len,
84 self.verify_checksum,
85 );
86
87 const decode_state = decompress.block.DecodeState.init(
88 &self.literal_fse_buffer,
89 &self.match_fse_buffer,
90 &self.offset_fse_buffer,
91 );
92
93 self.decode_state = decode_state;
94 self.frame_context = frame_context;
95
96 self.checksum = null;
97 self.current_frame_decompressed_size = 0;
98
99 self.state = .InFrame;
100 },
101 pub fn reader(self: *Self) std.io.Reader {
102 return .{ .context = self };
103 }
104
105 pub fn read(self: *Self, buffer: []u8) Error!usize {
106 if (buffer.len == 0) return 0;
107
108 var size: usize = 0;
109 while (size == 0) {
110 while (self.state == .NewFrame) {
111 const initial_count = self.source.bytes_read;
112 self.frameInit() catch |err| switch (err) {
113 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
114 error.EndOfStream => return if (self.source.bytes_read == initial_count)
115 0
116 else
117 error.MalformedFrame,
118 else => return error.MalformedFrame,
119 };
101120 }
121 size = try self.readInner(buffer);
102122 }
123 return size;
124 }
103125
104 pub fn reader(self: *Self) Reader {
105 return .{ .context = self };
106 }
126 fn readInner(self: *Self, buffer: []u8) Error!usize {
127 std.debug.assert(self.state != .NewFrame);
107128
108 pub fn read(self: *Self, buffer: []u8) Error!usize {
109 if (buffer.len == 0) return 0;
110
111 var size: usize = 0;
112 while (size == 0) {
113 while (self.state == .NewFrame) {
114 const initial_count = self.source.bytes_read;
115 self.frameInit() catch |err| switch (err) {
116 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
117 error.EndOfStream => return if (self.source.bytes_read == initial_count)
118 0
119 else
120 error.MalformedFrame,
121 else => return error.MalformedFrame,
122 };
123 }
124 size = try self.readInner(buffer);
125 }
126 return size;
129 var ring_buffer = RingBuffer{
130 .data = self.buffer.data,
131 .read_index = self.buffer.read_index,
132 .write_index = self.buffer.write_index,
133 };
134 defer {
135 self.buffer.read_index = ring_buffer.read_index;
136 self.buffer.write_index = ring_buffer.write_index;
127137 }
128138
129 fn readInner(self: *Self, buffer: []u8) Error!usize {
130 std.debug.assert(self.state != .NewFrame);
131
132 var ring_buffer = RingBuffer{
133 .data = self.buffer.data,
134 .read_index = self.buffer.read_index,
135 .write_index = self.buffer.write_index,
136 };
137 defer {
138 self.buffer.read_index = ring_buffer.read_index;
139 self.buffer.write_index = ring_buffer.write_index;
139 const source_reader = self.source;
140 while (ring_buffer.isEmpty() and self.state != .LastBlock) {
141 const header_bytes = source_reader.readBytesNoEof(3) catch
142 return error.MalformedFrame;
143 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
144
145 decompress.block.decodeBlockReader(
146 &ring_buffer,
147 source_reader,
148 block_header,
149 &self.decode_state,
150 self.frame_context.block_size_max,
151 &self.literals_buffer,
152 &self.sequence_buffer,
153 ) catch
154 return error.MalformedBlock;
155
156 if (self.frame_context.content_size) |size| {
157 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
140158 }
141159
142 const source_reader = self.source.reader();
143 while (ring_buffer.isEmpty() and self.state != .LastBlock) {
144 const header_bytes = source_reader.readBytesNoEof(3) catch
145 return error.MalformedFrame;
146 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
147
148 decompress.block.decodeBlockReader(
149 &ring_buffer,
150 source_reader,
151 block_header,
152 &self.decode_state,
153 self.frame_context.block_size_max,
154 &self.literals_buffer,
155 &self.sequence_buffer,
156 ) catch
157 return error.MalformedBlock;
158
159 if (self.frame_context.content_size) |size| {
160 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
161 }
160 const size = ring_buffer.len();
161 self.current_frame_decompressed_size += size;
162162
163 const size = ring_buffer.len();
164 self.current_frame_decompressed_size += size;
165
166 if (self.frame_context.hasher_opt) |*hasher| {
167 if (size > 0) {
168 const written_slice = ring_buffer.sliceLast(size);
169 hasher.update(written_slice.first);
170 hasher.update(written_slice.second);
171 }
163 if (self.frame_context.hasher_opt) |*hasher| {
164 if (size > 0) {
165 const written_slice = ring_buffer.sliceLast(size);
166 hasher.update(written_slice.first);
167 hasher.update(written_slice.second);
172168 }
173 if (block_header.last_block) {
174 self.state = .LastBlock;
175 if (self.frame_context.has_checksum) {
176 const checksum = source_reader.readInt(u32, .little) catch
177 return error.MalformedFrame;
178 if (self.verify_checksum) {
179 if (self.frame_context.hasher_opt) |*hasher| {
180 if (checksum != decompress.computeChecksum(hasher))
181 return error.ChecksumFailure;
182 }
169 }
170 if (block_header.last_block) {
171 self.state = .LastBlock;
172 if (self.frame_context.has_checksum) {
173 const checksum = source_reader.readInt(u32, .little) catch
174 return error.MalformedFrame;
175 if (self.verify_checksum) {
176 if (self.frame_context.hasher_opt) |*hasher| {
177 if (checksum != decompress.computeChecksum(hasher))
178 return error.ChecksumFailure;
183179 }
184180 }
185 if (self.frame_context.content_size) |content_size| {
186 if (content_size != self.current_frame_decompressed_size) {
187 return error.MalformedFrame;
188 }
181 }
182 if (self.frame_context.content_size) |content_size| {
183 if (content_size != self.current_frame_decompressed_size) {
184 return error.MalformedFrame;
189185 }
190186 }
191187 }
188 }
192189
193 const size = @min(ring_buffer.len(), buffer.len);
194 if (size > 0) {
195 ring_buffer.readFirstAssumeLength(buffer, size);
196 }
197 if (self.state == .LastBlock and ring_buffer.len() == 0) {
198 self.state = .NewFrame;
199 }
200 return size;
190 const size = @min(ring_buffer.len(), buffer.len);
191 if (size > 0) {
192 ring_buffer.readFirstAssumeLength(buffer, size);
201193 }
202 };
203}
194 if (self.state == .LastBlock and ring_buffer.len() == 0) {
195 self.state = .NewFrame;
196 }
197 return size;
198 }
199};
204200
205201pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206202 return Decompressor(@TypeOf(reader)).init(reader, options);
lib/std/debug/Dwarf.zig+17-3
......@@ -2212,7 +2212,7 @@ pub const ElfModule = struct {
22122212 var separate_debug_filename: ?[]const u8 = null;
22132213 var separate_debug_crc: ?u32 = null;
22142214
2215 for (shdrs) |*shdr| {
2215 shdrs: for (shdrs) |*shdr| {
22162216 if (shdr.sh_type == elf.SHT_NULL or shdr.sh_type == elf.SHT_NOBITS) continue;
22172217 const name = mem.sliceTo(header_strings[shdr.sh_name..], 0);
22182218
......@@ -2246,8 +2246,22 @@ pub const ElfModule = struct {
22462246 const decompressed_section = try gpa.alloc(u8, ch_size);
22472247 errdefer gpa.free(decompressed_section);
22482248
2249 const read = zlib_stream.reader().readAll(decompressed_section) catch continue;
2250 assert(read == decompressed_section.len);
2249 {
2250 var read_index: usize = 0;
2251 while (true) {
2252 const read_result = zlib_stream.streamReadVec(&.{decompressed_section[read_index..]});
2253 read_result.err catch {
2254 gpa.free(decompressed_section);
2255 continue :shdrs;
2256 };
2257 read_index += read_result.len;
2258 if (read_index == decompressed_section.len) break;
2259 if (read_result.end) {
2260 gpa.free(decompressed_section);
2261 continue :shdrs;
2262 }
2263 }
2264 }
22512265
22522266 break :blk .{
22532267 .data = decompressed_section,
lib/std/debug/FixedBufferReader.zig+21-9
......@@ -1,5 +1,7 @@
11//! Optimized for performance in debug builds.
22
3// TODO I'm pretty sure this can be deleted thanks to the new std.io.BufferedReader semantics
4
35const std = @import("../std.zig");
46const MemoryAccessor = std.debug.MemoryAccessor;
57
......@@ -9,20 +11,20 @@ buf: []const u8,
911pos: usize = 0,
1012endian: std.builtin.Endian,
1113
12pub const Error = error{ EndOfBuffer, Overflow, InvalidBuffer };
14pub const Error = error{ EndOfStream, Overflow, InvalidBuffer };
1315
1416pub fn seekTo(fbr: *FixedBufferReader, pos: u64) Error!void {
15 if (pos > fbr.buf.len) return error.EndOfBuffer;
17 if (pos > fbr.buf.len) return error.EndOfStream;
1618 fbr.pos = @intCast(pos);
1719}
1820
1921pub fn seekForward(fbr: *FixedBufferReader, amount: u64) Error!void {
20 if (fbr.buf.len - fbr.pos < amount) return error.EndOfBuffer;
22 if (fbr.buf.len - fbr.pos < amount) return error.EndOfStream;
2123 fbr.pos += @intCast(amount);
2224}
2325
2426pub inline fn readByte(fbr: *FixedBufferReader) Error!u8 {
25 if (fbr.pos >= fbr.buf.len) return error.EndOfBuffer;
27 if (fbr.pos >= fbr.buf.len) return error.EndOfStream;
2628 defer fbr.pos += 1;
2729 return fbr.buf[fbr.pos];
2830}
......@@ -33,7 +35,7 @@ pub fn readByteSigned(fbr: *FixedBufferReader) Error!i8 {
3335
3436pub fn readInt(fbr: *FixedBufferReader, comptime T: type) Error!T {
3537 const size = @divExact(@typeInfo(T).int.bits, 8);
36 if (fbr.buf.len - fbr.pos < size) return error.EndOfBuffer;
38 if (fbr.buf.len - fbr.pos < size) return error.EndOfStream;
3739 defer fbr.pos += size;
3840 return std.mem.readInt(T, fbr.buf[fbr.pos..][0..size], fbr.endian);
3941}
......@@ -50,11 +52,21 @@ pub fn readIntChecked(
5052}
5153
5254pub fn readUleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {
53 return std.leb.readUleb128(T, fbr);
55 var br: std.io.BufferedReader = undefined;
56 br.initFixed(fbr.buf);
57 br.seek = fbr.pos;
58 const result = br.takeUleb128(T);
59 fbr.pos = br.seek;
60 return @errorCast(result);
5461}
5562
5663pub fn readIleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {
57 return std.leb.readIleb128(T, fbr);
64 var br: std.io.BufferedReader = undefined;
65 br.initFixed(fbr.buf);
66 br.seek = fbr.pos;
67 const result = br.takeIleb128(T);
68 fbr.pos = br.seek;
69 return @errorCast(result);
5870}
5971
6072pub fn readAddress(fbr: *FixedBufferReader, format: std.dwarf.Format) Error!u64 {
......@@ -76,7 +88,7 @@ pub fn readAddressChecked(
7688}
7789
7890pub fn readBytes(fbr: *FixedBufferReader, len: usize) Error![]const u8 {
79 if (fbr.buf.len - fbr.pos < len) return error.EndOfBuffer;
91 if (fbr.buf.len - fbr.pos < len) return error.EndOfStream;
8092 defer fbr.pos += len;
8193 return fbr.buf[fbr.pos..][0..len];
8294}
......@@ -87,7 +99,7 @@ pub fn readBytesTo(fbr: *FixedBufferReader, comptime sentinel: u8) Error![:senti
8799 fbr.buf,
88100 fbr.pos,
89101 sentinel,
90 }) orelse return error.EndOfBuffer;
102 }) orelse return error.EndOfStream;
91103 defer fbr.pos = end + 1;
92104 return fbr.buf[fbr.pos..end :sentinel];
93105}
lib/std/debug/SelfInfo.zig+4-4
......@@ -2028,13 +2028,13 @@ pub const VirtualMachine = struct {
20282028 var prev_row: Row = self.current_row;
20292029
20302030 var cie_stream: std.io.BufferedReader = undefined;
2031 cie_stream.initFixed(&cie.initial_instructions);
2031 cie_stream.initFixed(cie.initial_instructions);
20322032 var fde_stream: std.io.BufferedReader = undefined;
2033 fde_stream.initFixed(&fde.instructions);
2034 const streams: [2]*std.io.FixedBufferStream = .{ &cie_stream, &fde_stream };
2033 fde_stream.initFixed(fde.instructions);
2034 const streams: [2]*std.io.BufferedReader = .{ &cie_stream, &fde_stream };
20352035
20362036 for (&streams, 0..) |stream, i| {
2037 while (stream.pos < stream.buffer.len) {
2037 while (stream.seek < stream.buffer.len) {
20382038 const instruction = try std.debug.Dwarf.call_frame.Instruction.read(stream, addr_size_bytes, endian);
20392039 prev_row = try self.step(allocator, cie, i == 0, instruction);
20402040 if (pc < fde.pc_begin + self.current_row.offset) return prev_row;
lib/std/fmt.zig+10-8
......@@ -91,7 +91,7 @@ pub const Options = struct {
9191/// A user type may be a `struct`, `vector`, `union` or `enum` type.
9292///
9393/// To print literal curly braces, escape them by writing them twice, e.g. `{{` or `}}`.
94pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytype) anyerror!void {
94pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytype) anyerror!usize {
9595 const ArgsType = @TypeOf(args);
9696 const args_type_info = @typeInfo(ArgsType);
9797 if (args_type_info != .@"struct") {
......@@ -107,6 +107,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
107107 comptime var arg_state: ArgState = .{ .args_len = fields_info.len };
108108 comptime var i = 0;
109109 comptime var literal: []const u8 = "";
110 var bytes_written: usize = 0;
110111 inline while (true) {
111112 const start_index = i;
112113
......@@ -136,7 +137,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
136137
137138 // Write out the literal
138139 if (literal.len != 0) {
139 try bw.writeAll(literal);
140 bytes_written += try bw.writeAllCount(literal);
140141 literal = "";
141142 }
142143
......@@ -196,7 +197,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
196197 const arg_to_print = comptime arg_state.nextArg(arg_pos) orelse
197198 @compileError("too few arguments");
198199
199 try bw.printValue(
200 bytes_written += try bw.printValue(
200201 placeholder.specifier_arg,
201202 .{
202203 .fill = placeholder.fill,
......@@ -217,6 +218,8 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
217218 else => @compileError(comptimePrint("{d}", .{missing_count}) ++ " unused arguments in '" ++ fmt ++ "'"),
218219 }
219220 }
221
222 return bytes_written;
220223}
221224
222225fn cacheString(str: anytype) []const u8 {
......@@ -852,11 +855,10 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr
852855}
853856
854857/// Count the characters needed for format.
855pub fn count(comptime fmt: []const u8, args: anytype) u64 {
856 var counting_writer: std.io.CountingWriter = .{ .child_writer = std.io.null_writer };
857 var bw = counting_writer.writer().unbuffered();
858 bw.print(fmt, args) catch unreachable;
859 return counting_writer.bytes_written;
858pub fn count(comptime fmt: []const u8, args: anytype) usize {
859 var buffer: [std.atomic.cache_line]u8 = undefined;
860 var bw = std.io.Writer.null.buffered(&buffer);
861 return bw.printCount(fmt, args) catch unreachable;
860862}
861863
862864pub const AllocPrintError = error{OutOfMemory};
lib/std/fs/File.zig+43-18
......@@ -1512,23 +1512,48 @@ pub fn writeFileAllUnseekable(self: File, in_file: File, args: WriteFileOptions)
15121512 return @errorCast(writeFileAllUnseekableInner(self, in_file, args));
15131513}
15141514
1515fn writeFileAllUnseekableInner(self: File, in_file: File, args: WriteFileOptions) anyerror!void {
1515fn writeFileAllUnseekableInner(out_file: File, in_file: File, args: WriteFileOptions) anyerror!void {
15161516 const headers = args.headers_and_trailers[0..args.header_count];
15171517 const trailers = args.headers_and_trailers[args.header_count..];
15181518
1519 try self.writevAll(headers);
1519 try out_file.writevAll(headers);
15201520
1521 try in_file.reader().skipBytes(args.in_offset, .{ .buf_size = 4096 });
1521 // Some possible optimizations here:
1522 // * Could writev buffer multiple times if the amount to discard is larger than 4096
1523 // * Could combine discard and read in one readv if amount to discard is small
15221524
1523 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1525 var buffer: [4096]u8 = undefined;
1526 var remaining = args.in_offset;
1527 while (remaining > 0) {
1528 const n = try in_file.read(buffer[0..@min(buffer.len, remaining)]);
1529 if (n == 0) return error.EndOfStream;
1530 remaining -= n;
1531 }
15241532 if (args.in_len) |len| {
1525 var stream = std.io.limitedReader(in_file.reader(), len);
1526 try fifo.pump(stream.reader(), self.writer());
1533 remaining = len;
1534 var buffer_index: usize = 0;
1535 while (remaining > 0) {
1536 const n = buffer_index + try in_file.read(buffer[buffer_index..@min(buffer.len, remaining)]);
1537 if (n == 0) return error.EndOfStream;
1538 const written = try out_file.write(buffer[0..n]);
1539 if (written == 0) return error.EndOfStream;
1540 remaining -= written;
1541 std.mem.copyForwards(u8, &buffer, buffer[written..n]);
1542 buffer_index = n - written;
1543 }
15271544 } else {
1528 try fifo.pump(in_file.reader(), self.writer());
1545 var buffer_index: usize = 0;
1546 while (true) {
1547 const n = buffer_index + try in_file.read(buffer[buffer_index..]);
1548 if (n == 0) break;
1549 const written = try out_file.write(buffer[0..n]);
1550 if (written == 0) return error.EndOfStream;
1551 std.mem.copyForwards(u8, &buffer, buffer[written..n]);
1552 buffer_index = n - written;
1553 }
15291554 }
15301555
1531 try self.writevAll(trailers);
1556 try out_file.writevAll(trailers);
15321557}
15331558
15341559/// Low level function which can fail for OS-specific reasons.
......@@ -1645,7 +1670,7 @@ pub fn reader_posReadVec(context: *anyopaque, data: []const []u8, offset: u64) a
16451670}
16461671
16471672pub fn reader_streamRead(
1648 context: *anyopaque,
1673 context: ?*anyopaque,
16491674 bw: *std.io.BufferedWriter,
16501675 limit: std.io.Reader.Limit,
16511676) anyerror!std.io.Reader.Status {
......@@ -1658,7 +1683,7 @@ pub fn reader_streamRead(
16581683 };
16591684}
16601685
1661pub fn reader_streamReadVec(context: *anyopaque, data: []const []u8) anyerror!std.io.Reader.Status {
1686pub fn reader_streamReadVec(context: ?*anyopaque, data: []const []u8) anyerror!std.io.Reader.Status {
16621687 const file = opaqueToHandle(context);
16631688 const n = try file.readv(data);
16641689 return .{
......@@ -1667,12 +1692,12 @@ pub fn reader_streamReadVec(context: *anyopaque, data: []const []u8) anyerror!st
16671692 };
16681693}
16691694
1670pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
1695pub fn writer_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Result {
16711696 const file = opaqueToHandle(context);
16721697 var splat_buffer: [256]u8 = undefined;
16731698 if (is_windows) {
16741699 if (data.len == 1 and splat == 0) return 0;
1675 return windows.WriteFile(file, data[0], null);
1700 return .{ .len = windows.WriteFile(file, data[0], null) catch |err| return .{ .err = err } };
16761701 }
16771702 var iovecs: [max_buffers_len]std.posix.iovec_const = undefined;
16781703 var len: usize = @min(iovecs.len, data.len);
......@@ -1681,8 +1706,8 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u
16811706 .len = d.len,
16821707 };
16831708 switch (splat) {
1684 0 => return std.posix.writev(file, iovecs[0 .. len - 1]),
1685 1 => return std.posix.writev(file, iovecs[0..len]),
1709 0 => return .{ .len = std.posix.writev(file, iovecs[0 .. len - 1]) catch |err| return .{ .err = err } },
1710 1 => return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } },
16861711 else => {
16871712 const pattern = data[data.len - 1];
16881713 if (pattern.len == 1) {
......@@ -1700,21 +1725,21 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u
17001725 iovecs[len] = .{ .base = &splat_buffer, .len = remaining_splat };
17011726 len += 1;
17021727 }
1703 return std.posix.writev(file, iovecs[0..len]);
1728 return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } };
17041729 }
17051730 },
17061731 }
1707 return std.posix.writev(file, iovecs[0..len]);
1732 return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } };
17081733}
17091734
17101735pub fn writer_writeFile(
1711 context: *anyopaque,
1736 context: ?*anyopaque,
17121737 in_file: std.fs.File,
17131738 in_offset: u64,
17141739 in_len: std.io.Writer.FileLen,
17151740 headers_and_trailers: []const []const u8,
17161741 headers_len: usize,
1717) anyerror!usize {
1742) std.io.Writer.Result {
17181743 const out_fd = opaqueToHandle(context);
17191744 const in_fd = in_file.handle;
17201745 const len_int = switch (in_len) {
lib/std/io.zig-44
......@@ -20,8 +20,6 @@ pub const Writer = @import("io/Writer.zig");
2020pub const BufferedReader = @import("io/BufferedReader.zig");
2121pub const BufferedWriter = @import("io/BufferedWriter.zig");
2222pub const AllocatingWriter = @import("io/AllocatingWriter.zig");
23pub const CountingWriter = @import("io/CountingWriter.zig");
24pub const CountingReader = @import("io/CountingReader.zig");
2523
2624pub const CWriter = @import("io/c_writer.zig").CWriter;
2725pub const cWriter = @import("io/c_writer.zig").cWriter;
......@@ -48,46 +46,6 @@ pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAt
4846
4947pub const tty = @import("io/tty.zig");
5048
51/// A `Writer` that discards all data.
52pub const null_writer: Writer = .{
53 .context = undefined,
54 .vtable = &.{
55 .writeSplat = null_writeSplat,
56 .writeFile = null_writeFile,
57 },
58};
59
60fn null_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
61 _ = context;
62 const headers = data[0 .. data.len - 1];
63 const pattern = data[headers.len..];
64 var written: usize = pattern.len * splat;
65 for (headers) |bytes| written += bytes.len;
66 return written;
67}
68
69fn null_writeFile(
70 context: *anyopaque,
71 file: std.fs.File,
72 offset: u64,
73 len: Writer.FileLen,
74 headers_and_trailers: []const []const u8,
75 headers_len: usize,
76) anyerror!usize {
77 _ = context;
78 _ = offset;
79 _ = headers_len;
80 _ = file;
81 if (len == .entire_file) return error.Unimplemented;
82 var n: usize = 0;
83 for (headers_and_trailers) |bytes| n += bytes.len;
84 return len.int() + n;
85}
86
87test null_writer {
88 try null_writer.writeAll("yay");
89}
90
9149pub fn poll(
9250 allocator: Allocator,
9351 comptime StreamEnum: type,
......@@ -494,8 +452,6 @@ test {
494452 _ = BufferedReader;
495453 _ = Reader;
496454 _ = Writer;
497 _ = CountingWriter;
498 _ = CountingReader;
499455 _ = AllocatingWriter;
500456 _ = @import("io/bit_reader.zig");
501457 _ = @import("io/bit_writer.zig");
lib/std/io/BufferedReader.zig+162-25
......@@ -14,26 +14,37 @@ seek: usize,
1414storage: BufferedWriter,
1515unbuffered_reader: Reader,
1616
17pub fn init(br: *BufferedReader, r: Reader, buffer: []u8) void {
18 br.* = .{
19 .seek = 0,
20 .storage = undefined,
21 .unbuffered_reader = r,
22 };
23 br.storage.initFixed(buffer);
24}
25
26/// Constructs `br` such that it will read from `buffer` and then end.
1727pub fn initFixed(br: *BufferedReader, buffer: []const u8) void {
1828 br.* = .{
1929 .seek = 0,
2030 .storage = .{
21 .buffer = buffer,
22 .mode = .fixed,
23 },
24 .reader = .{
25 .context = br,
26 .vtable = &.{
27 .streamRead = null,
28 .posRead = null,
31 .buffer = .initBuffer(@constCast(buffer)),
32 .unbuffered_writer = .{
33 .context = undefined,
34 .vtable = &std.io.Writer.VTable.eof,
2935 },
3036 },
37 .unbuffered_reader = &.{
38 .context = undefined,
39 .vtable = &std.io.Reader.VTable.eof,
40 },
3141 };
3242}
3343
34pub fn deinit(br: *BufferedReader) void {
35 br.storage.deinit();
36 br.* = undefined;
44pub fn storageBuffer(br: *BufferedReader) []u8 {
45 assert(br.storage.unbuffered_writer.vtable == &std.io.Writer.VTable.eof);
46 assert(br.unbuffered_reader.vtable == &std.io.Reader.VTable.eof);
47 return br.storage.buffer.allocatedSlice();
3748}
3849
3950/// Although `BufferedReader` can easily satisfy the `Reader` interface, it's
......@@ -51,30 +62,31 @@ pub fn reader(br: *BufferedReader) Reader {
5162 };
5263}
5364
54fn passthru_streamRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit) anyerror!Reader.Status {
65fn passthru_streamRead(ctx: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit) anyerror!Reader.RwResult {
5566 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
5667 const buffer = br.storage.buffer.items;
5768 const buffered = buffer[br.seek..];
5869 const limited = buffered[0..limit.min(buffered.len)];
5970 if (limited.len > 0) {
60 const n = try bw.writeSplat(limited, 1);
61 br.seek += n;
71 const result = bw.writeSplat(limited, 1);
72 br.seek += result.len;
6273 return .{
63 .end = false,
64 .len = @intCast(n),
74 .len = result.len,
75 .write_err = result.err,
76 .write_end = result.end,
6577 };
6678 }
6779 return br.unbuffered_reader.streamRead(bw, limit);
6880}
6981
70fn passthru_streamReadVec(ctx: *anyopaque, data: []const []u8) anyerror!Reader.Status {
82fn passthru_streamReadVec(ctx: ?*anyopaque, data: []const []u8) anyerror!Reader.Status {
7183 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
7284 _ = br;
7385 _ = data;
7486 @panic("TODO");
7587}
7688
77fn passthru_posRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit, off: u64) anyerror!Reader.Status {
89fn passthru_posRead(ctx: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit, off: u64) anyerror!Reader.Status {
7890 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
7991 const buffer = br.storage.buffer.items;
8092 if (off < buffer.len) {
......@@ -84,7 +96,7 @@ fn passthru_posRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit, o
8496 return br.unbuffered_reader.posRead(bw, limit, off - buffer.len);
8597}
8698
87fn passthru_posReadVec(ctx: *anyopaque, data: []const []u8, off: u64) anyerror!Reader.Status {
99fn passthru_posReadVec(ctx: ?*anyopaque, data: []const []u8, off: u64) anyerror!Reader.Status {
88100 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
89101 _ = br;
90102 _ = data;
......@@ -155,8 +167,24 @@ pub fn takeArray(br: *BufferedReader, comptime n: usize) anyerror!*[n]u8 {
155167///
156168/// See also:
157169/// * `toss`
158/// * `discardAll`
170/// * `discardUntilEnd`
171/// * `discardUpTo`
159172pub fn discard(br: *BufferedReader, n: usize) anyerror!void {
173 if ((try discardUpTo(br, n)) != n) return error.EndOfStream;
174}
175
176/// Skips the next `n` bytes from the stream, advancing the seek position.
177///
178/// Unlike `toss` which is infallible, in this function `n` can be any amount.
179///
180/// Returns the number of bytes discarded, which is less than `n` if and only
181/// if the stream reached the end.
182///
183/// See also:
184/// * `discard`
185/// * `toss`
186/// * `discardUntilEnd`
187pub fn discardUpTo(br: *BufferedReader, n: usize) anyerror!usize {
160188 const list = &br.storage.buffer;
161189 var remaining = n;
162190 while (remaining > 0) {
......@@ -168,19 +196,22 @@ pub fn discard(br: *BufferedReader, n: usize) anyerror!void {
168196 remaining -= (list.items.len - br.seek);
169197 list.items.len = 0;
170198 br.seek = 0;
171 const status = try br.unbuffered_reader.streamRead(&br.storage, .none);
199 const result = try br.unbuffered_reader.streamRead(&br.storage, .none);
200 result.write_err catch unreachable;
201 try result.read_err;
202 assert(result.len == list.items.len);
172203 if (remaining <= list.items.len) continue;
173 if (status.end) return error.EndOfStream;
204 if (result.end) return n - remaining;
174205 }
175206}
176207
177208/// Reads the stream until the end, ignoring all the data.
178209/// Returns the number of bytes discarded.
179pub fn discardAll(br: *BufferedReader) anyerror!usize {
210pub fn discardUntilEnd(br: *BufferedReader) anyerror!usize {
180211 const list = &br.storage.buffer;
181212 var total: usize = list.items.len;
182213 list.items.len = 0;
183 total += try br.unbuffered_reader.discardAll();
214 total += try br.unbuffered_reader.discardUntilEnd();
184215 return total;
185216}
186217
......@@ -224,6 +255,15 @@ pub fn read(br: *BufferedReader, buffer: []u8) anyerror!void {
224255 }
225256}
226257
258/// Returns the number of bytes read. If the number read is smaller than `buffer.len`, it
259/// means the stream reached the end. Reaching the end of a stream is not an error
260/// condition.
261pub fn partialRead(br: *BufferedReader, buffer: []u8) anyerror!usize {
262 _ = br;
263 _ = buffer;
264 @panic("TODO");
265}
266
227267/// Returns a slice of the next bytes of buffered data from the stream until
228268/// `delimiter` is found, advancing the seek position.
229269///
......@@ -463,6 +503,95 @@ pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.En
463503 return std.meta.intToEnum(Enum, int);
464504}
465505
506/// Read a single unsigned LEB128 value from the given reader as type T,
507/// or error.Overflow if the value cannot fit.
508pub fn takeUleb128(br: *std.io.BufferedReader, comptime T: type) anyerror!T {
509 const U = if (@typeInfo(T).int.bits < 8) u8 else T;
510 const ShiftT = std.math.Log2Int(U);
511
512 const max_group = (@typeInfo(U).int.bits + 6) / 7;
513
514 var value: U = 0;
515 var group: ShiftT = 0;
516
517 while (group < max_group) : (group += 1) {
518 const byte = try br.takeByte();
519
520 const ov = @shlWithOverflow(@as(U, byte & 0x7f), group * 7);
521 if (ov[1] != 0) return error.Overflow;
522
523 value |= ov[0];
524 if (byte & 0x80 == 0) break;
525 } else {
526 return error.Overflow;
527 }
528
529 // only applies in the case that we extended to u8
530 if (U != T) {
531 if (value > std.math.maxInt(T)) return error.Overflow;
532 }
533
534 return @truncate(value);
535}
536
537/// Read a single signed LEB128 value from the given reader as type T,
538/// or `error.Overflow` if the value cannot fit.
539pub fn takeIleb128(br: *std.io.BufferedReader, comptime T: type) anyerror!T {
540 const S = if (@typeInfo(T).int.bits < 8) i8 else T;
541 const U = std.meta.Int(.unsigned, @typeInfo(S).int.bits);
542 const ShiftU = std.math.Log2Int(U);
543
544 const max_group = (@typeInfo(U).int.bits + 6) / 7;
545
546 var value = @as(U, 0);
547 var group = @as(ShiftU, 0);
548
549 while (group < max_group) : (group += 1) {
550 const byte = try br.takeByte();
551
552 const shift = group * 7;
553 const ov = @shlWithOverflow(@as(U, byte & 0x7f), shift);
554 if (ov[1] != 0) {
555 // Overflow is ok so long as the sign bit is set and this is the last byte
556 if (byte & 0x80 != 0) return error.Overflow;
557 if (@as(S, @bitCast(ov[0])) >= 0) return error.Overflow;
558
559 // and all the overflowed bits are 1
560 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
561 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
562 if (remaining_bits != -1) return error.Overflow;
563 } else {
564 // If we don't overflow and this is the last byte and the number being decoded
565 // is negative, check that the remaining bits are 1
566 if ((byte & 0x80 == 0) and (@as(S, @bitCast(ov[0])) < 0)) {
567 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
568 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
569 if (remaining_bits != -1) return error.Overflow;
570 }
571 }
572
573 value |= ov[0];
574 if (byte & 0x80 == 0) {
575 const needs_sign_ext = group + 1 < max_group;
576 if (byte & 0x40 != 0 and needs_sign_ext) {
577 const ones = @as(S, -1);
578 value |= @as(U, @bitCast(ones)) << (shift + 7);
579 }
580 break;
581 }
582 } else {
583 return error.Overflow;
584 }
585
586 const result = @as(S, @bitCast(value));
587 // Only applies if we extended to i8
588 if (S != T) {
589 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
590 }
591
592 return @truncate(result);
593}
594
466595test initFixed {
467596 var br: BufferedReader = undefined;
468597 br.initFixed("a\x02");
......@@ -501,7 +630,7 @@ test discard {
501630 try testing.expectError(error.EndOfStream, br.discard(1));
502631}
503632
504test discardAll {
633test discardUntilEnd {
505634 return error.Unimplemented;
506635}
507636
......@@ -576,3 +705,11 @@ test takeStructEndian {
576705test takeEnum {
577706 return error.Unimplemented;
578707}
708
709test takeUleb128 {
710 return error.Unimplemented;
711}
712
713test takeIleb128 {
714 return error.Unimplemented;
715}
lib/std/io/BufferedWriter.zig+334-142
......@@ -43,7 +43,7 @@ const fixed_vtable: Writer.VTable = .{
4343};
4444
4545/// Replaces the `BufferedWriter` with a new one that writes to `buffer` and
46/// returns `error.NoSpaceLeft` when it is full.
46/// then ends when it is full.
4747pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void {
4848 bw.* = .{
4949 .unbuffered_writer = .{
......@@ -77,6 +77,36 @@ pub fn unusedCapacitySlice(bw: *const BufferedWriter) []u8 {
7777 return bw.buffer.unusedCapacitySlice();
7878}
7979
80pub fn writableSlice(bw: *BufferedWriter, minimum_length: usize) anyerror![]u8 {
81 const list = &bw.buffer;
82 assert(list.capacity >= minimum_length);
83 const cap_slice = list.unusedCapacitySlice();
84 if (cap_slice.len >= minimum_length) {
85 @branchHint(.likely);
86 return cap_slice;
87 }
88 const buffer = list.items;
89 const result = bw.unbuffered_writer.write(buffer);
90 if (result.len == buffer.len) {
91 @branchHint(.likely);
92 list.items.len = 0;
93 try result.err;
94 return list.unusedCapacitySlice();
95 }
96 if (result.len > 0) {
97 const remainder = buffer[result.len..];
98 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
99 list.items.len = remainder.len;
100 }
101 try result.err;
102 return list.unusedCapacitySlice();
103}
104
105/// After calling `writableSlice`, this function tracks how many bytes were written to it.
106pub fn advance(bw: *BufferedWriter, n: usize) void {
107 bw.items.len += n;
108}
109
80110/// The `data` parameter is mutable because this function needs to mutate the
81111/// fields in order to handle partial writes from `Writer.VTable.writev`.
82112pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {
......@@ -92,15 +122,15 @@ pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {
92122 }
93123}
94124
95pub fn writeSplat(bw: *BufferedWriter, data: []const []const u8, splat: usize) anyerror!usize {
125pub fn writeSplat(bw: *BufferedWriter, data: []const []const u8, splat: usize) Writer.Result {
96126 return passthru_writeSplat(bw, data, splat);
97127}
98128
99pub fn writev(bw: *BufferedWriter, data: []const []const u8) anyerror!usize {
129pub fn writev(bw: *BufferedWriter, data: []const []const u8) Writer.Result {
100130 return passthru_writeSplat(bw, data, 1);
101131}
102132
103fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
133fn passthru_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Result {
104134 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
105135 const list = &bw.buffer;
106136 const buffer = list.allocatedSlice();
......@@ -126,27 +156,45 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
126156 if (len >= remaining_data.len) {
127157 @branchHint(.likely);
128158 // Made it past the headers, so we can enable splatting.
129 const n = try bw.unbuffered_writer.writeSplat(send_buffers, splat);
159 const result = bw.unbuffered_writer.writeSplat(send_buffers, splat);
160 const n = result.len;
130161 if (n < end) {
131162 @branchHint(.unlikely);
132163 const remainder = buffer[n..end];
133164 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
134165 list.items.len = remainder.len;
135 return end - start_end;
166 return .{
167 .err = result.err,
168 .len = end - start_end,
169 .end = result.end,
170 };
136171 }
137172 list.items.len = 0;
138 return n - start_end;
173 return .{
174 .err = result.err,
175 .len = n - start_end,
176 .end = result.end,
177 };
139178 }
140 const n = try bw.unbuffered_writer.writeSplat(send_buffers, 1);
179 const result = try bw.unbuffered_writer.writeSplat(send_buffers, 1);
180 const n = result.len;
141181 if (n < end) {
142182 @branchHint(.unlikely);
143183 const remainder = buffer[n..end];
144184 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
145185 list.items.len = remainder.len;
146 return end - start_end;
186 return .{
187 .err = result.err,
188 .len = end - start_end,
189 .end = result.end,
190 };
147191 }
148192 list.items.len = 0;
149 return n - start_end;
193 return .{
194 .err = result.err,
195 .len = n - start_end,
196 .end = result.end,
197 };
150198 }
151199
152200 const pattern = data[data.len - 1];
......@@ -156,7 +204,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
156204 // It was added in the loop above; undo it here.
157205 end -= pattern.len;
158206 list.items.len = end;
159 return end - start_end;
207 return .{ .len = end - start_end };
160208 }
161209
162210 const remaining_splat = splat - 1;
......@@ -164,7 +212,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
164212 switch (pattern.len) {
165213 0 => {
166214 list.items.len = end;
167 return end - start_end;
215 return .{ .len = end - start_end };
168216 },
169217 1 => {
170218 const new_end = end + remaining_splat;
......@@ -172,20 +220,29 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
172220 @branchHint(.likely);
173221 @memset(buffer[end..new_end], pattern[0]);
174222 list.items.len = new_end;
175 return new_end - start_end;
223 return .{ .len = new_end - start_end };
176224 }
177225 buffers[0] = buffer[0..end];
178226 buffers[1] = pattern;
179 const n = try bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
227 const result = bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
228 const n = result.len;
180229 if (n < end) {
181230 @branchHint(.unlikely);
182231 const remainder = buffer[n..end];
183232 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
184233 list.items.len = remainder.len;
185 return end - start_end;
234 return .{
235 .err = result.err,
236 .len = end - start_end,
237 .end = result.end,
238 };
186239 }
187240 list.items.len = 0;
188 return n - start_end;
241 return .{
242 .err = result.err,
243 .len = n - start_end,
244 .end = result.end,
245 };
189246 },
190247 else => {
191248 const new_end = end + pattern.len * remaining_splat;
......@@ -195,46 +252,43 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
195252 @memcpy(buffer[end..][0..pattern.len], pattern);
196253 }
197254 list.items.len = new_end;
198 return new_end - start_end;
255 return .{ .len = new_end - start_end };
199256 }
200257 buffers[0] = buffer[0..end];
201258 buffers[1] = pattern;
202 const n = try bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
259 const result = bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
260 const n = result.len;
203261 if (n < end) {
204262 @branchHint(.unlikely);
205263 const remainder = buffer[n..end];
206264 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
207265 list.items.len = remainder.len;
208 return end - start_end;
266 return .{
267 .err = result.err,
268 .len = end - start_end,
269 .end = result.end,
270 };
209271 }
210272 list.items.len = 0;
211 return n - start_end;
273 return .{
274 .err = result.err,
275 .len = n - start_end,
276 .end = result.end,
277 };
212278 },
213279 }
214280}
215281
216fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
217 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
218 const list = &bw.buffer;
219 // When this function is called it means the buffer got full, so it's time
220 // to return an error. However, we still need to make sure all of the
221 // available buffer has been used.
222 const first = data[0];
223 const dest = list.unusedCapacitySlice();
224 @memcpy(dest, first[0..dest.len]);
225 list.items.len = list.capacity;
226 return error.NoSpaceLeft;
227}
228
229282/// When this function is called it means the buffer got full, so it's time
230283/// to return an error. However, we still need to make sure all of the
231284/// available buffer has been filled.
232fn fixed_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
285fn fixed_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Result {
233286 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
234287 const list = &bw.buffer;
288 const start_len = list.items.len;
235289 for (data) |bytes| {
236290 const dest = list.unusedCapacitySlice();
237 if (dest.len == 0) return error.NoSpaceLeft;
291 if (dest.len == 0) return .{ .len = list.items.len - start_len, .end = true };
238292 const len = @min(bytes.len, dest.len);
239293 @memcpy(dest[0..len], bytes[0..len]);
240294 list.items.len += len;
......@@ -247,90 +301,153 @@ fn fixed_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize)
247301 else => for (0..splat - 1) |i| @memcpy(dest[i * pattern.len ..][0..pattern.len], pattern),
248302 }
249303 list.items.len = list.capacity;
250 return error.NoSpaceLeft;
304 return .{ .len = list.items.len - start_len, .end = true };
251305}
252306
253pub fn write(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
307pub fn write(bw: *BufferedWriter, bytes: []const u8) Writer.Result {
254308 const list = &bw.buffer;
255309 const buffer = list.allocatedSlice();
256310 const end = list.items.len;
257311 const new_end = end + bytes.len;
258312 if (new_end > buffer.len) {
259313 var data: [2][]const u8 = .{ buffer[0..end], bytes };
260 const n = try bw.unbuffered_writer.writev(&data);
314 const result = bw.unbuffered_writer.writev(&data);
315 const n = result.len;
261316 if (n < end) {
262317 @branchHint(.unlikely);
263318 const remainder = buffer[n..end];
264319 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
265320 list.items.len = remainder.len;
266 return 0;
321 return .{
322 .err = result.err,
323 .len = 0,
324 .end = result.end,
325 };
267326 }
268327 list.items.len = 0;
269 return n - end;
328 return .{
329 .err = result.err,
330 .len = n - end,
331 .end = result.end,
332 };
270333 }
271334 @memcpy(buffer[end..new_end], bytes);
272335 list.items.len = new_end;
273336 return bytes.len;
274337}
275338
276/// This function is provided by the `Writer`, however it is
277/// duplicated here so that `bw` can be passed to `std.fmt.format` directly,
278/// avoiding one indirect function call.
279339pub fn writeAll(bw: *BufferedWriter, bytes: []const u8) anyerror!void {
340 if ((try writeUntilEnd(bw, bytes)) != bytes.len) return error.WriteStreamEnd;
341}
342
343pub fn writeAllCount(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
344 try writeAll(bw, bytes);
345 return bytes.len;
346}
347
348/// If the number returned is less than `bytes.len` it indicates end of stream.
349pub fn writeUntilEnd(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
280350 var index: usize = 0;
281 while (index < bytes.len) index += try write(bw, bytes[index..]);
351 while (true) {
352 const result = write(bw, bytes[index..]);
353 try result.err;
354 index += result.len;
355 assert(index <= bytes.len);
356 if (index == bytes.len or result.end) return index;
357 }
282358}
283359
284360pub fn print(bw: *BufferedWriter, comptime format: []const u8, args: anytype) anyerror!void {
361 _ = try std.fmt.format(bw, format, args);
362}
363
364pub fn printCount(bw: *BufferedWriter, comptime format: []const u8, args: anytype) anyerror!usize {
285365 return std.fmt.format(bw, format, args);
286366}
287367
288368pub fn writeByte(bw: *BufferedWriter, byte: u8) anyerror!void {
369 if ((try writeByteUntilEnd(bw, byte)) == 0) return error.WriteStreamEnd;
370}
371
372pub fn writeByteCount(bw: *BufferedWriter, byte: u8) anyerror!usize {
373 try writeByte(bw, byte);
374 return 1;
375}
376
377/// Returns 0 or 1 indicating how many bytes were written.
378/// `0` means end of stream encountered.
379pub fn writeByteUntilEnd(bw: *BufferedWriter, byte: u8) anyerror!usize {
289380 const list = &bw.buffer;
290381 const buffer = list.items;
291382 if (buffer.len < list.capacity) {
292383 @branchHint(.likely);
293384 buffer.ptr[buffer.len] = byte;
294385 list.items.len = buffer.len + 1;
295 return;
386 return 1;
296387 }
297388 var buffers: [2][]const u8 = .{ buffer, &.{byte} };
298389 while (true) {
299 const n = try bw.unbuffered_writer.writev(&buffers);
390 const result = bw.unbuffered_writer.writev(&buffers);
391 try result.err;
392 const n = result.len;
300393 if (n == 0) {
301394 @branchHint(.unlikely);
395 if (result.end) return 0;
302396 continue;
303397 } else if (n >= buffer.len) {
304398 @branchHint(.likely);
305399 if (n > buffer.len) {
306400 @branchHint(.likely);
307401 list.items.len = 0;
308 return;
402 return 1;
309403 } else {
310404 buffer[0] = byte;
311405 list.items.len = 1;
312 return;
406 return 1;
313407 }
314408 }
315409 const remainder = buffer[n..];
316410 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
317411 buffer[remainder.len] = byte;
318412 list.items.len = remainder.len + 1;
319 return;
413 return 1;
320414 }
321415}
322416
323417/// Writes the same byte many times, performing the underlying write call as
324/// many times as necessary.
418/// many times as necessary, returning `error.WriteStreamEnd` if the byte
419/// could not be repeated `n` times.
325420pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {
326 var remaining: usize = n;
327 while (remaining > 0) remaining -= try splatByte(bw, byte, remaining);
421 if ((try splatByteUntilEnd(bw, byte, n)) != n) return error.WriteStreamEnd;
422}
423
424/// Writes the same byte many times, performing the underlying write call as
425/// many times as necessary, returning `error.WriteStreamEnd` if the byte
426/// could not be repeated `n` times, or returning `n` on success.
427pub fn splatByteAllCount(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
428 try splatByteAll(bw, byte, n);
429 return n;
430}
431
432/// Writes the same byte many times, performing the underlying write call as
433/// many times as necessary.
434///
435/// If the number returned is less than `n` it indicates end of stream.
436pub fn splatByteUntilEnd(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
437 var index: usize = 0;
438 while (true) {
439 const result = splatByte(bw, byte, n - index);
440 try result.err;
441 index += result.len;
442 assert(index <= n);
443 if (index == n or result.end) return index;
444 }
328445}
329446
330447/// Writes the same byte many times, allowing short writes.
331448///
332/// Does maximum of one underlying `Writer.VTable.writev`.
333pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
449/// Does maximum of one underlying `Writer.VTable.writeSplat`.
450pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) Writer.Result {
334451 return passthru_writeSplat(bw, &.{&.{byte}}, n);
335452}
336453
......@@ -389,7 +506,7 @@ pub fn writeFile(
389506}
390507
391508fn passthru_writeFile(
392 context: *anyopaque,
509 context: ?*anyopaque,
393510 file: std.fs.File,
394511 offset: u64,
395512 len: Writer.FileLen,
......@@ -544,32 +661,34 @@ pub fn alignBuffer(
544661 width: usize,
545662 alignment: std.fmt.Alignment,
546663 fill: u8,
547) anyerror!void {
664) anyerror!usize {
548665 const padding = if (buffer.len < width) width - buffer.len else 0;
549666 if (padding == 0) {
550667 @branchHint(.likely);
551 return bw.writeAll(buffer);
668 return bw.writeAllCount(buffer);
552669 }
670 var n: usize = 0;
553671 switch (alignment) {
554672 .left => {
555 try bw.writeAll(buffer);
556 try bw.splatByteAll(fill, padding);
673 n += try bw.writeAllCount(buffer);
674 n += try bw.splatByteAllCount(fill, padding);
557675 },
558676 .center => {
559677 const left_padding = padding / 2;
560678 const right_padding = (padding + 1) / 2;
561 try bw.splatByteAll(fill, left_padding);
562 try bw.writeAll(buffer);
563 try bw.splatByteAll(fill, right_padding);
679 n += try bw.splatByteAllCount(fill, left_padding);
680 n += try bw.writeAllCount(buffer);
681 n += try bw.splatByteAllCount(fill, right_padding);
564682 },
565683 .right => {
566 try bw.splatByteAll(fill, padding);
567 try bw.writeAll(buffer);
684 n += try bw.splatByteAllCount(fill, padding);
685 n += try bw.writeAllCount(buffer);
568686 },
569687 }
688 return n;
570689}
571690
572pub fn alignBufferOptions(bw: *BufferedWriter, buffer: []const u8, options: std.fmt.Options) anyerror!void {
691pub fn alignBufferOptions(bw: *BufferedWriter, buffer: []const u8, options: std.fmt.Options) anyerror!usize {
573692 return alignBuffer(bw, buffer, options.width orelse buffer.len, options.alignment, options.fill);
574693}
575694
......@@ -604,7 +723,7 @@ pub fn printValue(
604723 options: std.fmt.Options,
605724 value: anytype,
606725 max_depth: usize,
607) anyerror!void {
726) anyerror!usize {
608727 const T = @TypeOf(value);
609728 const actual_fmt = comptime if (std.mem.eql(u8, fmt, ANY))
610729 defaultFormatString(T)
......@@ -619,13 +738,10 @@ pub fn printValue(
619738
620739 if (std.meta.hasMethod(T, "format")) {
621740 if (fmt.len > 0 and fmt[0] == 'f') {
622 return value.format(fmt[1..], options, bw);
623 } else {
624 //@deprecated();
625 // After 0.14.0 is tagged, uncomment this next line:
626 //@compileError("ambiguous format string; specify {f} to call format method, or {any} to skip it");
627 //and then delete the `hasMethod` condition
628 return value.format(fmt, options, bw);
741 return value.format(bw, fmt[1..]);
742 } else if (fmt.len == 0) {
743 // after 0.15.0 is tagged, delete the hasMethod condition and this compile error
744 @compileError("ambiguous format string; specify {f} to call format method, or {any} to skip it");
629745 }
630746 }
631747
......@@ -662,92 +778,104 @@ pub fn printValue(
662778 },
663779 .error_set => {
664780 if (actual_fmt.len > 0 and actual_fmt[0] == 's') {
665 return bw.writeAll(@errorName(value));
781 return bw.writeAllCount(@errorName(value));
666782 } else if (actual_fmt.len != 0) {
667783 invalidFmtError(fmt, value);
668784 } else {
669 try bw.writeAll("error.");
670 return bw.writeAll(@errorName(value));
785 var n: usize = 0;
786 n += try bw.writeAllCount("error.");
787 n += try bw.writeAllCount(@errorName(value));
788 return n;
671789 }
672790 },
673 .@"enum" => |enumInfo| {
674 try bw.writeAll(@typeName(T));
675 if (enumInfo.is_exhaustive) {
791 .@"enum" => |enum_info| {
792 var n: usize = 0;
793 n += try bw.writeAllCount(@typeName(T));
794 if (enum_info.is_exhaustive) {
676795 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
677 try bw.writeAll(".");
678 try bw.writeAll(@tagName(value));
679 return;
796 n += try bw.writeAllCount(".");
797 n += try bw.writeAllCount(@tagName(value));
798 return n;
680799 }
681800
682801 // Use @tagName only if value is one of known fields
683 @setEvalBranchQuota(3 * enumInfo.fields.len);
684 inline for (enumInfo.fields) |enumField| {
802 @setEvalBranchQuota(3 * enum_info.fields.len);
803 inline for (enum_info.fields) |enumField| {
685804 if (@intFromEnum(value) == enumField.value) {
686 try bw.writeAll(".");
687 try bw.writeAll(@tagName(value));
805 n += try bw.writeAllCount(".");
806 n += try bw.writeAllCount(@tagName(value));
688807 return;
689808 }
690809 }
691810
692 try bw.writeByte('(');
693 try printValue(bw, actual_fmt, options, @intFromEnum(value), max_depth);
694 try bw.writeByte(')');
811 n += try bw.writeByteCount('(');
812 n += try printValue(bw, actual_fmt, options, @intFromEnum(value), max_depth);
813 n += try bw.writeByteCount(')');
814 return n;
695815 },
696816 .@"union" => |info| {
697817 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
698 try bw.writeAll(@typeName(T));
818 var n: usize = 0;
819 n += try bw.writeAllCount(@typeName(T));
699820 if (max_depth == 0) {
700 return bw.writeAll("{ ... }");
821 n += bw.writeAllCount("{ ... }");
822 return n;
701823 }
702824 if (info.tag_type) |UnionTagType| {
703 try bw.writeAll("{ .");
704 try bw.writeAll(@tagName(@as(UnionTagType, value)));
705 try bw.writeAll(" = ");
825 n += try bw.writeAllCount("{ .");
826 n += try bw.writeAllCount(@tagName(@as(UnionTagType, value)));
827 n += try bw.writeAllCount(" = ");
706828 inline for (info.fields) |u_field| {
707829 if (value == @field(UnionTagType, u_field.name)) {
708 try printValue(bw, ANY, options, @field(value, u_field.name), max_depth - 1);
830 n += try printValue(bw, ANY, options, @field(value, u_field.name), max_depth - 1);
709831 }
710832 }
711 try bw.writeAll(" }");
833 n += try bw.writeAllCount(" }");
712834 } else {
713 try bw.writeByte('@');
714 try bw.printIntOptions(@intFromPtr(&value), 16, .lower);
835 n += try bw.writeByte('@');
836 n += try bw.printIntOptions(@intFromPtr(&value), 16, .lower);
715837 }
838 return n;
716839 },
717840 .@"struct" => |info| {
718841 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
842 var n: usize = 0;
719843 if (info.is_tuple) {
720844 // Skip the type and field names when formatting tuples.
721845 if (max_depth == 0) {
722 return bw.writeAll("{ ... }");
846 n += try bw.writeAllCount("{ ... }");
847 return n;
723848 }
724 try bw.writeAll("{");
849 n += try bw.writeAllCount("{");
725850 inline for (info.fields, 0..) |f, i| {
726851 if (i == 0) {
727 try bw.writeAll(" ");
852 n += try bw.writeAllCount(" ");
728853 } else {
729 try bw.writeAll(", ");
854 n += try bw.writeAllCount(", ");
730855 }
731 try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
856 n += try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
732857 }
733 return bw.writeAll(" }");
858 n += try bw.writeAllCount(" }");
859 return n;
734860 }
735 try bw.writeAll(@typeName(T));
861 n += try bw.writeAllCount(@typeName(T));
736862 if (max_depth == 0) {
737 return bw.writeAll("{ ... }");
863 n += try bw.writeAllCount("{ ... }");
864 return n;
738865 }
739 try bw.writeAll("{");
866 n += try bw.writeAllCount("{");
740867 inline for (info.fields, 0..) |f, i| {
741868 if (i == 0) {
742 try bw.writeAll(" .");
869 n += try bw.writeAllCount(" .");
743870 } else {
744 try bw.writeAll(", .");
871 n += try bw.writeAllCount(", .");
745872 }
746 try bw.writeAll(f.name);
747 try bw.writeAll(" = ");
748 try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
873 n += try bw.writeAllCount(f.name);
874 n += try bw.writeAllCount(" = ");
875 n += try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
749876 }
750 try bw.writeAll(" }");
877 n += try bw.writeAllCount(" }");
878 return n;
751879 },
752880 .pointer => |ptr_info| switch (ptr_info.size) {
753881 .one => switch (@typeInfo(ptr_info.child)) {
......@@ -756,8 +884,10 @@ pub fn printValue(
756884 },
757885 else => {
758886 var buffers: [2][]const u8 = .{ @typeName(ptr_info.child), "@" };
759 try writevAll(bw, &buffers);
760 try printIntOptions(bw, @intFromPtr(value), 16, .lower, options);
887 var n: usize = 0;
888 n += try writevAll(bw, &buffers);
889 n += try printIntOptions(bw, @intFromPtr(value), 16, .lower, options);
890 return n;
761891 },
762892 },
763893 .many, .c => {
......@@ -775,7 +905,7 @@ pub fn printValue(
775905 if (actual_fmt.len == 0)
776906 @compileError("cannot format slice without a specifier (i.e. {s}, {x}, {b64}, or {any})");
777907 if (max_depth == 0) {
778 return bw.writeAll("{ ... }");
908 return bw.writeAllCount("{ ... }");
779909 }
780910 if (ptr_info.child == u8) switch (actual_fmt.len) {
781911 1 => switch (actual_fmt[0]) {
......@@ -789,21 +919,23 @@ pub fn printValue(
789919 },
790920 else => {},
791921 };
792 try bw.writeAll("{ ");
922 var n: usize = 0;
923 n += try bw.writeAllCount("{ ");
793924 for (value, 0..) |elem, i| {
794 try printValue(bw, actual_fmt, options, elem, max_depth - 1);
925 n += try printValue(bw, actual_fmt, options, elem, max_depth - 1);
795926 if (i != value.len - 1) {
796 try bw.writeAll(", ");
927 n += try bw.writeAllCount(", ");
797928 }
798929 }
799 try bw.writeAll(" }");
930 n += try bw.writeAllCount(" }");
931 return n;
800932 },
801933 },
802934 .array => |info| {
803935 if (actual_fmt.len == 0)
804936 @compileError("cannot format array without a specifier (i.e. {s} or {any})");
805937 if (max_depth == 0) {
806 return bw.writeAll("{ ... }");
938 return bw.writeAllCount("{ ... }");
807939 }
808940 if (info.child == u8) {
809941 if (actual_fmt[0] == 's') {
......@@ -814,28 +946,32 @@ pub fn printValue(
814946 return printHex(bw, &value, .upper);
815947 }
816948 }
817 try bw.writeAll("{ ");
949 var n: usize = 0;
950 n += try bw.writeAllCount("{ ");
818951 for (value, 0..) |elem, i| {
819 try printValue(bw, actual_fmt, options, elem, max_depth - 1);
952 n += try printValue(bw, actual_fmt, options, elem, max_depth - 1);
820953 if (i < value.len - 1) {
821 try bw.writeAll(", ");
954 n += try bw.writeAllCount(", ");
822955 }
823956 }
824 try bw.writeAll(" }");
957 n += try bw.writeAllCount(" }");
958 return n;
825959 },
826960 .vector => |info| {
827961 if (max_depth == 0) {
828 return bw.writeAll("{ ... }");
962 return bw.writeAllCount("{ ... }");
829963 }
830 try bw.writeAll("{ ");
964 var n: usize = 0;
965 n += try bw.writeAllCount("{ ");
831966 var i: usize = 0;
832967 while (i < info.len) : (i += 1) {
833 try printValue(bw, actual_fmt, options, value[i], max_depth - 1);
968 n += try printValue(bw, actual_fmt, options, value[i], max_depth - 1);
834969 if (i < info.len - 1) {
835 try bw.writeAll(", ");
970 n += try bw.writeAllCount(", ");
836971 }
837972 }
838 try bw.writeAll(" }");
973 n += try bw.writeAllCount(" }");
974 return n;
839975 },
840976 .@"fn" => @compileError("unable to format function body type, use '*const " ++ @typeName(T) ++ "' for a function pointer type"),
841977 .type => {
......@@ -860,7 +996,7 @@ pub fn printInt(
860996 comptime fmt: []const u8,
861997 options: std.fmt.Options,
862998 value: anytype,
863) anyerror!void {
999) anyerror!usize {
8641000 const int_value = if (@TypeOf(value) == comptime_int) blk: {
8651001 const Int = std.math.IntFittingRange(value, value);
8661002 break :blk @as(Int, value);
......@@ -904,15 +1040,15 @@ pub fn printInt(
9041040 comptime unreachable;
9051041}
9061042
907pub fn printAsciiChar(bw: *BufferedWriter, c: u8, options: std.fmt.Options) anyerror!void {
1043pub fn printAsciiChar(bw: *BufferedWriter, c: u8, options: std.fmt.Options) anyerror!usize {
9081044 return alignBufferOptions(bw, @as(*const [1]u8, &c), options);
9091045}
9101046
911pub fn printAscii(bw: *BufferedWriter, bytes: []const u8, options: std.fmt.Options) anyerror!void {
1047pub fn printAscii(bw: *BufferedWriter, bytes: []const u8, options: std.fmt.Options) anyerror!usize {
9121048 return alignBufferOptions(bw, bytes, options);
9131049}
9141050
915pub fn printUnicodeCodepoint(bw: *BufferedWriter, c: u21, options: std.fmt.Options) anyerror!void {
1051pub fn printUnicodeCodepoint(bw: *BufferedWriter, c: u21, options: std.fmt.Options) anyerror!usize {
9161052 var buf: [4]u8 = undefined;
9171053 const len = try std.unicode.utf8Encode(c, &buf);
9181054 return alignBufferOptions(bw, buf[0..len], options);
......@@ -924,7 +1060,7 @@ pub fn printIntOptions(
9241060 base: u8,
9251061 case: std.fmt.Case,
9261062 options: std.fmt.Options,
927) anyerror!void {
1063) anyerror!usize {
9281064 assert(base >= 2);
9291065
9301066 const int_value = if (@TypeOf(value) == comptime_int) blk: {
......@@ -991,7 +1127,7 @@ pub fn printFloat(
9911127 comptime fmt: []const u8,
9921128 options: std.fmt.Options,
9931129 value: anytype,
994) anyerror!void {
1130) anyerror!usize {
9951131 var buf: [std.fmt.float.bufferSize(.decimal, f64)]u8 = undefined;
9961132
9971133 if (fmt.len > 1) invalidFmtError(fmt, value);
......@@ -1279,7 +1415,7 @@ pub fn printDuration(bw: *BufferedWriter, nanoseconds: anytype, options: std.fmt
12791415 return alignBufferOptions(bw, sub_bw.getWritten(), options);
12801416}
12811417
1282pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anyerror!void {
1418pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anyerror!usize {
12831419 const charset = switch (case) {
12841420 .upper => "0123456789ABCDEF",
12851421 .lower => "0123456789abcdef",
......@@ -1288,12 +1424,68 @@ pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anye
12881424 try writeByte(bw, charset[c >> 4]);
12891425 try writeByte(bw, charset[c & 15]);
12901426 }
1427 return bytes.len * 2;
12911428}
12921429
1293pub fn printBase64(bw: *BufferedWriter, bytes: []const u8) anyerror!void {
1430pub fn printBase64(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
12941431 var chunker = std.mem.window(u8, bytes, 3, 3);
12951432 var temp: [5]u8 = undefined;
1296 while (chunker.next()) |chunk| try bw.writeAll(std.base64.standard.Encoder.encode(&temp, chunk));
1433 var n: usize = 0;
1434 while (chunker.next()) |chunk| {
1435 n += try bw.writeAllCount(std.base64.standard.Encoder.encode(&temp, chunk));
1436 }
1437 return n;
1438}
1439
1440/// Write a single unsigned integer as unsigned LEB128 to the given writer.
1441pub fn writeUleb128(bw: *std.io.BufferedWriter, arg: anytype) anyerror!usize {
1442 const Arg = @TypeOf(arg);
1443 const Int = switch (Arg) {
1444 comptime_int => std.math.IntFittingRange(arg, arg),
1445 else => Arg,
1446 };
1447 const Value = if (@typeInfo(Int).int.bits < 8) u8 else Int;
1448 var value: Value = arg;
1449 var n: usize = 0;
1450
1451 while (true) {
1452 const byte: u8 = @truncate(value & 0x7f);
1453 value >>= 7;
1454 if (value == 0) {
1455 try bw.writeByte(byte);
1456 return n + 1;
1457 } else {
1458 try bw.writeByte(byte | 0x80);
1459 n += 1;
1460 }
1461 }
1462}
1463
1464/// Write a single signed integer as signed LEB128 to the given writer.
1465pub fn writeIleb128(bw: *std.io.BufferedWriter, arg: anytype) anyerror!usize {
1466 const Arg = @TypeOf(arg);
1467 const Int = switch (Arg) {
1468 comptime_int => std.math.IntFittingRange(-@abs(arg), @abs(arg)),
1469 else => Arg,
1470 };
1471 const Signed = if (@typeInfo(Int).int.bits < 8) i8 else Int;
1472 const Unsigned = std.meta.Int(.unsigned, @typeInfo(Signed).int.bits);
1473 var value: Signed = arg;
1474 var n: usize = 0;
1475
1476 while (true) {
1477 const unsigned: Unsigned = @bitCast(value);
1478 const byte: u8 = @truncate(unsigned);
1479 value >>= 6;
1480 if (value == -1 or value == 0) {
1481 try bw.writeByte(byte & 0x7F);
1482 return n + 1;
1483 } else {
1484 value >>= 1;
1485 try bw.writeByte(byte | 0x80);
1486 n += 1;
1487 }
1488 }
12971489}
12981490
12991491test "formatValue max_depth" {
......@@ -1590,15 +1782,15 @@ test "fixed output" {
15901782 try bw.writeAll("world");
15911783 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));
15921784
1593 try testing.expectError(error.NoSpaceLeft, bw.writeAll("!"));
1785 try testing.expectError(error.WriteStreamEnd, bw.writeAll("!"));
15941786 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));
15951787
15961788 bw.reset();
15971789 try testing.expect(bw.getWritten().len == 0);
15981790
1599 try testing.expectError(error.NoSpaceLeft, bw.writeAll("Hello world!"));
1791 try testing.expectError(error.WriteStreamEnd, bw.writeAll("Hello world!"));
16001792 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Hello worl"));
16011793
16021794 try bw.seekTo((try bw.getEndPos()) + 1);
1603 try testing.expectError(error.NoSpaceLeft, bw.writeAll("H"));
1795 try testing.expectError(error.WriteStreamEnd, bw.writeAll("H"));
16041796}
lib/std/io/CountingReader.zig deleted-29
......@@ -1,29 +0,0 @@
1//! A Reader that counts how many bytes has been read from it.
2
3const std = @import("../std.zig");
4const CountingReader = @This();
5
6child_reader: std.io.Reader,
7bytes_read: u64 = 0,
8
9pub fn read(self: *@This(), buf: []u8) anyerror!usize {
10 const amt = try self.child_reader.read(buf);
11 self.bytes_read += amt;
12 return amt;
13}
14
15pub fn reader(self: *@This()) std.io.Reader {
16 return .{ .context = self };
17}
18
19test CountingReader {
20 const bytes = "yay" ** 20;
21 var fbs: std.io.BufferedReader = undefined;
22 fbs.initFixed(bytes);
23 var counting_stream: CountingReader = .{ .child_reader = fbs.reader() };
24 var stream = counting_stream.reader().unbuffered();
25 while (stream.readByte()) |_| {} else |err| {
26 try std.testing.expectError(error.EndOfStream, err);
27 }
28 try std.testing.expect(counting_stream.bytes_read == bytes.len);
29}
lib/std/io/CountingWriter.zig deleted-52
......@@ -1,52 +0,0 @@
1//! TODO make this more like AllocatingWriter, managing the state of
2//! BufferedWriter both as the output and the input, but with only
3//! one buffer.
4const std = @import("../std.zig");
5const CountingWriter = @This();
6const assert = std.debug.assert;
7const native_endian = @import("builtin").target.cpu.arch.endian();
8const Writer = std.io.Writer;
9const testing = std.testing;
10
11/// Underlying stream to passthrough bytes to.
12child_writer: Writer,
13bytes_written: u64 = 0,
14
15pub fn writer(cw: *CountingWriter) Writer {
16 return .{
17 .context = cw,
18 .vtable = &.{
19 .writeSplat = passthru_writeSplat,
20 .writeFile = passthru_writeFile,
21 },
22 };
23}
24
25fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
26 const cw: *CountingWriter = @alignCast(@ptrCast(context));
27 const n = try cw.child_writer.writeSplat(data, splat);
28 cw.bytes_written += n;
29 return n;
30}
31
32fn passthru_writeFile(
33 context: *anyopaque,
34 file: std.fs.File,
35 offset: u64,
36 len: Writer.FileLen,
37 headers_and_trailers: []const []const u8,
38 headers_len: usize,
39) anyerror!usize {
40 const cw: *CountingWriter = @alignCast(@ptrCast(context));
41 const n = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);
42 cw.bytes_written += n;
43 return n;
44}
45
46test CountingWriter {
47 var cw: CountingWriter = .{ .child_writer = std.io.null_writer };
48 var bw = cw.writer().unbuffered();
49 const bytes = "yay";
50 try bw.writeAll(bytes);
51 try testing.expect(cw.bytes_written == bytes.len);
52}
lib/std/io/Reader.zig+51-41
......@@ -19,8 +19,8 @@ pub const VTable = struct {
1919 ///
2020 /// If this is `null` it is equivalent to always returning
2121 /// `error.Unseekable`.
22 posRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) Result,
23 posReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8, offset: u64) VecResult,
22 posRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) RwResult,
23 posReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8, offset: u64) Result,
2424
2525 /// Writes bytes from the internally tracked stream position to `bw`, or
2626 /// returns `error.Unstreamable`, indicating `posRead` should be used
......@@ -37,38 +37,34 @@ pub const VTable = struct {
3737 ///
3838 /// If this is `null` it is equivalent to always returning
3939 /// `error.Unstreamable`.
40 streamRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) Result,
41 streamReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8) VecResult,
42};
43
44pub const Len = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(usize) - 1 } });
45
46pub const VecResult = struct {
47 /// Even when a failure occurs, `Effect.written` may be nonzero, and
48 /// `Effect.end` may be true.
49 failure: anyerror!void,
50 effect: VecEffect,
40 streamRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) RwResult,
41 streamReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8) Result,
42
43 pub const eof: VTable = .{
44 .posRead = eof_posRead,
45 .posReadVec = eof_posReadVec,
46 .streamRead = eof_streamRead,
47 .streamReadVec = eof_streamReadVec,
48 };
5149};
5250
53pub const Result = struct {
54 /// Even when a failure occurs, `Effect.written` may be nonzero, and
55 /// `Effect.end` may be true.
56 failure: anyerror!void,
57 write_effect: Effect,
58 read_effect: Effect,
59};
51pub const Result = std.io.Writer.Result;
6052
61pub const Effect = packed struct(usize) {
62 /// Number of bytes that were read from the reader or written to the
63 /// writer.
64 len: Len,
65 /// Indicates end of stream.
66 end: bool,
53pub const RwResult = struct {
54 len: usize = 0,
55 read_err: anyerror!void = {},
56 write_err: anyerror!void = {},
57 read_end: bool = false,
58 write_end: bool = false,
6759};
6860
6961pub const Limit = enum(usize) {
7062 none = std.math.maxInt(usize),
7163 _,
64
65 pub fn min(l: Limit, int: usize) usize {
66 return @min(int, @intFromEnum(l));
67 }
7268};
7369
7470/// Returns total number of bytes written to `w`.
......@@ -133,25 +129,11 @@ pub fn streamReadAlloc(r: Reader, gpa: std.mem.Allocator, max_size: usize) anyer
133129
134130/// Reads the stream until the end, ignoring all the data.
135131/// Returns the number of bytes discarded.
136pub fn discardAll(r: Reader) anyerror!usize {
132pub fn discardUntilEnd(r: Reader) anyerror!usize {
137133 var bw = std.io.null_writer.unbuffered();
138134 return streamReadAll(r, &bw);
139135}
140136
141pub fn buffered(r: Reader, buffer: []u8) std.io.BufferedReader {
142 return .{
143 .reader = r,
144 .buffered_writer = .{
145 .buffer = buffer,
146 .mode = .fixed,
147 },
148 };
149}
150
151pub fn unbuffered(r: Reader) std.io.BufferedReader {
152 return buffered(r, &.{});
153}
154
155137pub fn allocating(r: Reader, gpa: std.mem.Allocator) std.io.BufferedReader {
156138 return .{
157139 .reader = r,
......@@ -189,3 +171,31 @@ test "when the backing reader provides one byte at a time" {
189171 defer std.testing.allocator.free(res);
190172 try std.testing.expectEqualStrings(str, res);
191173}
174
175fn eof_posRead(ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) RwResult {
176 _ = ctx;
177 _ = bw;
178 _ = limit;
179 _ = offset;
180 return .{ .end = true };
181}
182
183fn eof_posReadVec(ctx: ?*anyopaque, data: []const []u8, offset: u64) Result {
184 _ = ctx;
185 _ = data;
186 _ = offset;
187 return .{ .end = true };
188}
189
190fn eof_streamRead(ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) RwResult {
191 _ = ctx;
192 _ = bw;
193 _ = limit;
194 return .{ .end = true };
195}
196
197fn eof_streamReadVec(ctx: ?*anyopaque, data: []const []u8) Result {
198 _ = ctx;
199 _ = data;
200 return .{ .end = true };
201}
lib/std/io/Writer.zig+98-21
......@@ -2,7 +2,7 @@ const std = @import("../std.zig");
22const assert = std.debug.assert;
33const Writer = @This();
44
5context: *anyopaque,
5context: ?*anyopaque,
66vtable: *const VTable,
77
88pub const VTable = struct {
......@@ -17,7 +17,7 @@ pub const VTable = struct {
1717 /// Number of bytes returned may be zero, which does not mean
1818 /// end-of-stream. A subsequent call may return nonzero, or may signal end
1919 /// of stream via an error.
20 writeSplat: *const fn (ctx: *anyopaque, data: []const []const u8, splat: usize) Result,
20 writeSplat: *const fn (ctx: ?*anyopaque, data: []const []const u8, splat: usize) Result,
2121
2222 /// Writes contents from an open file. `headers` are written first, then `len`
2323 /// bytes of `file` starting from `offset`, then `trailers`.
......@@ -29,7 +29,7 @@ pub const VTable = struct {
2929 /// end-of-stream. A subsequent call may return nonzero, or may signal end
3030 /// of stream via an error.
3131 writeFile: *const fn (
32 ctx: *anyopaque,
32 ctx: ?*anyopaque,
3333 file: std.fs.File,
3434 offset: Offset,
3535 /// When zero, it means copy until the end of the file is reached.
......@@ -39,25 +39,26 @@ pub const VTable = struct {
3939 headers_and_trailers: []const []const u8,
4040 headers_len: usize,
4141 ) Result,
42};
43
44pub const Len = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(usize) - 1 } });
4542
46pub const Result = struct {
47 /// Even when a failure occurs, `Effect.written` may be nonzero, and
48 /// `Effect.end` may be true.
49 failure: anyerror!void,
50 effect: Effect,
43 pub const eof: VTable = .{
44 .writeSplat = eof_writeSplat,
45 .writeFile = eof_writeFile,
46 };
5147};
5248
53pub const Effect = packed struct(usize) {
54 /// Number of bytes that were written to `writer`.
55 len: Len,
49pub const Result = struct {
50 /// Even when a failure occurs, `len` may be nonzero, and `end` may be
51 /// true.
52 err: anyerror!void = {},
53 /// Number of bytes that were transferred. When an error occurs, ideally
54 /// this will be zero, but may not always be the case.
55 len: usize = 0,
5656 /// Indicates end of stream.
57 end: bool,
57 end: bool = false,
5858};
5959
6060pub const Offset = enum(u64) {
61 /// Indicates to read the file as a stream.
6162 none = std.math.maxInt(u64),
6263 _,
6364
......@@ -66,6 +67,11 @@ pub const Offset = enum(u64) {
6667 assert(result != .none);
6768 return result;
6869 }
70
71 pub fn toInt(o: Offset) ?u64 {
72 if (o == .none) return null;
73 return @intFromEnum(o);
74 }
6975};
7076
7177pub const FileLen = enum(u64) {
......@@ -84,11 +90,11 @@ pub const FileLen = enum(u64) {
8490 }
8591};
8692
87pub fn writev(w: Writer, data: []const []const u8) anyerror!usize {
93pub fn writev(w: Writer, data: []const []const u8) Result {
8894 return w.vtable.writeSplat(w.context, data, 1);
8995}
9096
91pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) anyerror!usize {
97pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) Result {
9298 return w.vtable.writeSplat(w.context, data, splat);
9399}
94100
......@@ -99,25 +105,25 @@ pub fn writeFile(
99105 len: FileLen,
100106 headers_and_trailers: []const []const u8,
101107 headers_len: usize,
102) anyerror!usize {
108) Result {
103109 return w.vtable.writeFile(w.context, file, offset, len, headers_and_trailers, headers_len);
104110}
105111
106112pub fn unimplemented_writeFile(
107 context: *anyopaque,
113 context: ?*anyopaque,
108114 file: std.fs.File,
109115 offset: u64,
110116 len: FileLen,
111117 headers_and_trailers: []const []const u8,
112118 headers_len: usize,
113) anyerror!usize {
119) Result {
114120 _ = context;
115121 _ = file;
116122 _ = offset;
117123 _ = len;
118124 _ = headers_and_trailers;
119125 _ = headers_len;
120 return error.Unimplemented;
126 return .{ .err = error.Unimplemented };
121127}
122128
123129pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
......@@ -130,3 +136,74 @@ pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
130136pub fn unbuffered(w: Writer) std.io.BufferedWriter {
131137 return buffered(w, &.{});
132138}
139
140/// A `Writer` that discards all data.
141pub const @"null": Writer = .{
142 .context = undefined,
143 .vtable = &.{
144 .writeSplat = null_writeSplat,
145 .writeFile = null_writeFile,
146 },
147};
148
149fn null_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Result {
150 _ = context;
151 const headers = data[0 .. data.len - 1];
152 const pattern = data[headers.len..];
153 var written: usize = pattern.len * splat;
154 for (headers) |bytes| written += bytes.len;
155 return .{ .len = written };
156}
157
158fn null_writeFile(
159 context: ?*anyopaque,
160 file: std.fs.File,
161 offset: Offset,
162 len: FileLen,
163 headers_and_trailers: []const []const u8,
164 headers_len: usize,
165) Result {
166 _ = context;
167 var n: usize = 0;
168 if (len == .entire_file) {
169 const headers = headers_and_trailers[0..headers_len];
170 for (headers) |bytes| n += bytes.len;
171 if (offset.toInt()) |off| {
172 const stat = file.stat() catch |err| return .{ .err = err, .len = n };
173 n += stat.size - off;
174 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
175 return .{ .len = n };
176 }
177 @panic("TODO stream from file until eof, counting");
178 }
179 for (headers_and_trailers) |bytes| n += bytes.len;
180 return .{ .len = len.int() + n };
181}
182
183test @"null" {
184 try @"null".writeAll("yay");
185}
186
187fn eof_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Result {
188 _ = context;
189 _ = data;
190 _ = splat;
191 return .{ .end = true };
192}
193
194fn eof_writeFile(
195 context: ?*anyopaque,
196 file: std.fs.File,
197 offset: u64,
198 len: FileLen,
199 headers_and_trailers: []const []const u8,
200 headers_len: usize,
201) Result {
202 _ = context;
203 _ = file;
204 _ = offset;
205 _ = len;
206 _ = headers_and_trailers;
207 _ = headers_len;
208 return .{ .end = true };
209}
lib/std/leb128.zig+31-175
......@@ -2,151 +2,6 @@ const builtin = @import("builtin");
22const std = @import("std");
33const testing = std.testing;
44
5/// Read a single unsigned LEB128 value from the given reader as type T,
6/// or error.Overflow if the value cannot fit.
7pub fn readUleb128(comptime T: type, reader: anytype) !T {
8 const U = if (@typeInfo(T).int.bits < 8) u8 else T;
9 const ShiftT = std.math.Log2Int(U);
10
11 const max_group = (@typeInfo(U).int.bits + 6) / 7;
12
13 var value: U = 0;
14 var group: ShiftT = 0;
15
16 while (group < max_group) : (group += 1) {
17 const byte = try reader.readByte();
18
19 const ov = @shlWithOverflow(@as(U, byte & 0x7f), group * 7);
20 if (ov[1] != 0) return error.Overflow;
21
22 value |= ov[0];
23 if (byte & 0x80 == 0) break;
24 } else {
25 return error.Overflow;
26 }
27
28 // only applies in the case that we extended to u8
29 if (U != T) {
30 if (value > std.math.maxInt(T)) return error.Overflow;
31 }
32
33 return @as(T, @truncate(value));
34}
35
36/// Deprecated: use `readUleb128`
37pub const readULEB128 = readUleb128;
38
39/// Write a single unsigned integer as unsigned LEB128 to the given writer.
40pub fn writeUleb128(writer: anytype, arg: anytype) !void {
41 const Arg = @TypeOf(arg);
42 const Int = switch (Arg) {
43 comptime_int => std.math.IntFittingRange(arg, arg),
44 else => Arg,
45 };
46 const Value = if (@typeInfo(Int).int.bits < 8) u8 else Int;
47 var value: Value = arg;
48
49 while (true) {
50 const byte: u8 = @truncate(value & 0x7f);
51 value >>= 7;
52 if (value == 0) {
53 try writer.writeByte(byte);
54 break;
55 } else {
56 try writer.writeByte(byte | 0x80);
57 }
58 }
59}
60
61/// Deprecated: use `writeUleb128`
62pub const writeULEB128 = writeUleb128;
63
64/// Read a single signed LEB128 value from the given reader as type T,
65/// or error.Overflow if the value cannot fit.
66pub fn readIleb128(comptime T: type, reader: anytype) !T {
67 const S = if (@typeInfo(T).int.bits < 8) i8 else T;
68 const U = std.meta.Int(.unsigned, @typeInfo(S).int.bits);
69 const ShiftU = std.math.Log2Int(U);
70
71 const max_group = (@typeInfo(U).int.bits + 6) / 7;
72
73 var value = @as(U, 0);
74 var group = @as(ShiftU, 0);
75
76 while (group < max_group) : (group += 1) {
77 const byte = try reader.readByte();
78
79 const shift = group * 7;
80 const ov = @shlWithOverflow(@as(U, byte & 0x7f), shift);
81 if (ov[1] != 0) {
82 // Overflow is ok so long as the sign bit is set and this is the last byte
83 if (byte & 0x80 != 0) return error.Overflow;
84 if (@as(S, @bitCast(ov[0])) >= 0) return error.Overflow;
85
86 // and all the overflowed bits are 1
87 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
88 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
89 if (remaining_bits != -1) return error.Overflow;
90 } else {
91 // If we don't overflow and this is the last byte and the number being decoded
92 // is negative, check that the remaining bits are 1
93 if ((byte & 0x80 == 0) and (@as(S, @bitCast(ov[0])) < 0)) {
94 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
95 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
96 if (remaining_bits != -1) return error.Overflow;
97 }
98 }
99
100 value |= ov[0];
101 if (byte & 0x80 == 0) {
102 const needs_sign_ext = group + 1 < max_group;
103 if (byte & 0x40 != 0 and needs_sign_ext) {
104 const ones = @as(S, -1);
105 value |= @as(U, @bitCast(ones)) << (shift + 7);
106 }
107 break;
108 }
109 } else {
110 return error.Overflow;
111 }
112
113 const result = @as(S, @bitCast(value));
114 // Only applies if we extended to i8
115 if (S != T) {
116 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
117 }
118
119 return @as(T, @truncate(result));
120}
121
122/// Deprecated: use `readIleb128`
123pub const readILEB128 = readIleb128;
124
125/// Write a single signed integer as signed LEB128 to the given writer.
126pub fn writeIleb128(writer: anytype, arg: anytype) !void {
127 const Arg = @TypeOf(arg);
128 const Int = switch (Arg) {
129 comptime_int => std.math.IntFittingRange(-@abs(arg), @abs(arg)),
130 else => Arg,
131 };
132 const Signed = if (@typeInfo(Int).int.bits < 8) i8 else Int;
133 const Unsigned = std.meta.Int(.unsigned, @typeInfo(Signed).int.bits);
134 var value: Signed = arg;
135
136 while (true) {
137 const unsigned: Unsigned = @bitCast(value);
138 const byte: u8 = @truncate(unsigned);
139 value >>= 6;
140 if (value == -1 or value == 0) {
141 try writer.writeByte(byte & 0x7F);
142 break;
143 } else {
144 value >>= 1;
145 try writer.writeByte(byte | 0x80);
146 }
147 }
148}
149
1505/// This is an "advanced" function. It allows one to use a fixed amount of memory to store a
1516/// ULEB128. This defeats the entire purpose of using this data encoding; it will no longer use
1527/// fewer bytes to store smaller numbers. The advantage of using a fixed width is that it makes
......@@ -176,9 +31,6 @@ pub fn writeUnsignedExtended(slice: []u8, arg: anytype) void {
17631 slice[slice.len - 1] = @as(u7, @intCast(value));
17732}
17833
179/// Deprecated: use `writeIleb128`
180pub const writeILEB128 = writeIleb128;
181
18234test writeUnsignedFixed {
18335 {
18436 var buf: [4]u8 = undefined;
......@@ -261,42 +113,45 @@ test writeSignedFixed {
261113 }
262114}
263115
264// tests
265116fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {
266 var reader = std.io.fixedBufferStream(encoded);
267 return try readIleb128(T, reader.reader());
117 var br: std.io.BufferedReader = undefined;
118 br.initFixed(encoded);
119 return br.takeIleb128(T);
268120}
269121
270122fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {
271 var reader = std.io.fixedBufferStream(encoded);
272 return try readUleb128(T, reader.reader());
123 var br: std.io.BufferedReader = undefined;
124 br.initFixed(encoded);
125 return br.takeUleb128(T);
273126}
274127
275128fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
276 var reader = std.io.fixedBufferStream(encoded);
277 const v1 = try readIleb128(T, reader.reader());
278 return v1;
129 var br: std.io.BufferedReader = undefined;
130 br.initFixed(encoded);
131 return br.readIleb128(T);
279132}
280133
281134fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
282 var reader = std.io.fixedBufferStream(encoded);
283 const v1 = try readUleb128(T, reader.reader());
284 return v1;
135 var br: std.io.BufferedReader = undefined;
136 br.initFixed(encoded);
137 return br.readUleb128(T);
285138}
286139
287140fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
288 var reader = std.io.fixedBufferStream(encoded);
141 var br: std.io.BufferedReader = undefined;
142 br.initFixed(encoded);
289143 var i: usize = 0;
290144 while (i < N) : (i += 1) {
291 _ = try readIleb128(T, reader.reader());
145 _ = try br.readIleb128(T);
292146 }
293147}
294148
295149fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
296 var reader = std.io.fixedBufferStream(encoded);
150 var br: std.io.BufferedReader = undefined;
151 br.initFixed(encoded);
297152 var i: usize = 0;
298153 while (i < N) : (i += 1) {
299 _ = try readUleb128(T, reader.reader());
154 _ = try br.readUleb128(T);
300155 }
301156}
302157
......@@ -392,8 +247,8 @@ fn test_write_leb128(value: anytype) !void {
392247 const signedness = @typeInfo(T).int.signedness;
393248 const t_signed = signedness == .signed;
394249
395 const writeStream = if (t_signed) writeIleb128 else writeUleb128;
396 const readStream = if (t_signed) readIleb128 else readUleb128;
250 const writeStream = if (t_signed) std.io.BufferedWriter.writeIleb128 else std.io.BufferedWriter.writeUleb128;
251 const readStream = if (t_signed) std.io.BufferedReader.readIleb128 else std.io.BufferedReader.readUleb128;
397252
398253 // decode to a larger bit size too, to ensure sign extension
399254 // is working as expected
......@@ -412,23 +267,24 @@ fn test_write_leb128(value: anytype) !void {
412267 const max_groups = if (@typeInfo(T).int.bits == 0) 1 else (@typeInfo(T).int.bits + 6) / 7;
413268
414269 var buf: [max_groups]u8 = undefined;
415 var fbs = std.io.fixedBufferStream(&buf);
270 var bw: std.io.BufferedWriter = undefined;
271 bw.initFixed(&buf);
416272
417273 // stream write
418 try writeStream(fbs.writer(), value);
419 const w1_pos = fbs.pos;
420 try testing.expect(w1_pos == bytes_needed);
274 try testing.expect((try writeStream(&bw, value)) == bytes_needed);
275 try testing.expect(bw.buffer.items.len == bytes_needed);
421276
422277 // stream read
423 fbs.pos = 0;
424 const sr = try readStream(T, fbs.reader());
425 try testing.expect(fbs.pos == w1_pos);
278 var br: std.io.BufferedReader = undefined;
279 br.initFixed(&buf);
280 const sr = try readStream(&br, T);
281 try testing.expect(br.seek == bytes_needed);
426282 try testing.expect(sr == value);
427283
428284 // bigger type stream read
429 fbs.pos = 0;
430 const bsr = try readStream(B, fbs.reader());
431 try testing.expect(fbs.pos == w1_pos);
285 bw.buffer.items.len = 0;
286 const bsr = try readStream(&bw, B);
287 try testing.expect(bw.buffer.items.len == bytes_needed);
432288 try testing.expect(bsr == value);
433289}
434290
lib/std/zig/ErrorBundle.zig+7-9
......@@ -189,24 +189,22 @@ fn renderErrorMessageToWriter(
189189 indent: usize,
190190) anyerror!void {
191191 const ttyconf = options.ttyconf;
192 var counting_writer: std.io.CountingWriter = .{ .child_writer = bw.writer() };
193 var counting_bw = counting_writer.writer().unbuffered();
194192 const err_msg = eb.getErrorMessage(err_msg_index);
193 // This is the length of the part before the error message:
194 // e.g. "file.zig:4:5: error: "
195 var prefix_len: usize = 0;
195196 if (err_msg.src_loc != .none) {
196197 const src = eb.extraData(SourceLocation, @intFromEnum(err_msg.src_loc));
197 try counting_bw.splatByteAll(' ', indent);
198 prefix_len += try bw.splatByteAllCount(' ', indent);
198199 try ttyconf.setColor(bw, .bold);
199 try counting_bw.print("{s}:{d}:{d}: ", .{
200 prefix_len += try bw.printCount("{s}:{d}:{d}: ", .{
200201 eb.nullTerminatedString(src.data.src_path),
201202 src.data.line + 1,
202203 src.data.column + 1,
203204 });
204205 try ttyconf.setColor(bw, color);
205 try counting_bw.writeAll(kind);
206 try counting_bw.writeAll(": ");
207 // This is the length of the part before the error message:
208 // e.g. "file.zig:4:5: error: "
209 const prefix_len: usize = @intCast(counting_writer.bytes_written);
206 prefix_len += try bw.writeAllCount(kind);
207 prefix_len += try bw.writeAllCount(": ");
210208 try ttyconf.setColor(bw, .reset);
211209 try ttyconf.setColor(bw, .bold);
212210 if (err_msg.count == 1) {
lib/std/zip/test.zig+151-152
......@@ -103,13 +103,13 @@ pub const Zip64Options = struct {
103103};
104104
105105pub fn writeZip(
106 writer: anytype,
106 writer: *std.io.BufferedWriter,
107107 files: []const File,
108108 store: []FileStore,
109109 options: WriteZipOptions,
110110) !void {
111111 if (store.len < files.len) return error.FileStoreTooSmall;
112 var zipper = initZipper(writer);
112 var zipper: Zipper = .init(writer);
113113 for (files, 0..) |file, i| {
114114 store[i] = try zipper.writeFile(.{
115115 .name = file.name,
......@@ -126,173 +126,172 @@ pub fn writeZip(
126126 try zipper.writeEndRecord(if (options.end) |e| e else .{});
127127}
128128
129pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
130 return .{ .counting_writer = std.io.countingWriter(writer) };
131}
132
133129/// Provides methods to format and write the contents of a zip archive
134130/// to the underlying Writer.
135pub fn Zipper(comptime Writer: type) type {
136 return struct {
137 counting_writer: std.io.CountingWriter(Writer),
138 central_count: u64 = 0,
139 first_central_offset: ?u64 = null,
140 last_central_limit: ?u64 = null,
131pub const Zipper = struct {
132 writer: *std.io.BufferedWriter,
133 bytes_written: u64,
134 central_count: u64 = 0,
135 first_central_offset: ?u64 = null,
136 last_central_limit: ?u64 = null,
141137
142 const Self = @This();
138 const Self = @This();
143139
144 pub fn writeFile(
145 self: *Self,
146 opt: struct {
147 name: []const u8,
148 content: []const u8,
149 compression: zip.CompressionMethod,
150 write_options: WriteZipOptions,
151 },
152 ) !FileStore {
153 const writer = self.counting_writer.writer();
154
155 const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
156 const crc32 = std.hash.Crc32.hash(opt.content);
140 pub fn init(writer: *std.io.BufferedWriter) Zipper {
141 return .{ .writer = writer, .bytes_written = 0 };
142 }
157143
158 const header_options = opt.write_options.local_header;
159 {
160 var compressed_size: u32 = 0;
161 var uncompressed_size: u32 = 0;
162 var extra_len: u16 = 0;
163 if (header_options) |hdr_options| {
164 compressed_size = if (hdr_options.compressed_size) |size| size else 0;
165 uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);
166 extra_len = if (hdr_options.extra_len) |len| len else 0;
167 }
168 const hdr: zip.LocalFileHeader = .{
169 .signature = zip.local_file_header_sig,
170 .version_needed_to_extract = 10,
171 .flags = .{ .encrypted = false, ._ = 0 },
172 .compression_method = opt.compression,
173 .last_modification_time = 0,
174 .last_modification_date = 0,
175 .crc32 = crc32,
176 .compressed_size = compressed_size,
177 .uncompressed_size = uncompressed_size,
178 .filename_len = @intCast(opt.name.len),
179 .extra_len = extra_len,
180 };
181 try writer.writeStructEndian(hdr, .little);
182 }
183 try writer.writeAll(opt.name);
144 pub fn writeFile(
145 self: *Self,
146 opt: struct {
147 name: []const u8,
148 content: []const u8,
149 compression: zip.CompressionMethod,
150 write_options: WriteZipOptions,
151 },
152 ) !FileStore {
153 const writer = self.writer;
184154
185 if (header_options) |hdr| {
186 if (hdr.zip64) |options| {
187 try writer.writeInt(u16, 0x0001, .little);
188 const data_size = if (options.data_size) |size| size else 8;
189 try writer.writeInt(u16, data_size, .little);
190 try writer.writeInt(u64, 0, .little);
191 try writer.writeInt(u64, @intCast(opt.content.len), .little);
192 }
193 }
155 const file_offset: u64 = @intCast(self.bytes_written);
156 const crc32 = std.hash.Crc32.hash(opt.content);
194157
195 var compressed_size: u32 = undefined;
196 switch (opt.compression) {
197 .store => {
198 try writer.writeAll(opt.content);
199 compressed_size = @intCast(opt.content.len);
200 },
201 .deflate => {
202 const offset = self.counting_writer.bytes_written;
203 var fbs = std.io.fixedBufferStream(opt.content);
204 try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205 std.debug.assert(fbs.pos == opt.content.len);
206 compressed_size = @intCast(self.counting_writer.bytes_written - offset);
207 },
208 else => unreachable,
158 const header_options = opt.write_options.local_header;
159 {
160 var compressed_size: u32 = 0;
161 var uncompressed_size: u32 = 0;
162 var extra_len: u16 = 0;
163 if (header_options) |hdr_options| {
164 compressed_size = if (hdr_options.compressed_size) |size| size else 0;
165 uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);
166 extra_len = if (hdr_options.extra_len) |len| len else 0;
209167 }
210 return .{
211 .compression = opt.compression,
212 .file_offset = file_offset,
168 const hdr: zip.LocalFileHeader = .{
169 .signature = zip.local_file_header_sig,
170 .version_needed_to_extract = 10,
171 .flags = .{ .encrypted = false, ._ = 0 },
172 .compression_method = opt.compression,
173 .last_modification_time = 0,
174 .last_modification_date = 0,
213175 .crc32 = crc32,
214176 .compressed_size = compressed_size,
215 .uncompressed_size = opt.content.len,
177 .uncompressed_size = uncompressed_size,
178 .filename_len = @intCast(opt.name.len),
179 .extra_len = extra_len,
216180 };
181 self.bytes_written += try writer.writeStructEndian(hdr, .little);
217182 }
183 self.bytes_written += try writer.writeAll(opt.name);
218184
219 pub fn writeCentralRecord(
220 self: *Self,
221 store: FileStore,
222 opt: struct {
223 name: []const u8,
224 version_needed_to_extract: u16 = 10,
225 },
226 ) !void {
227 if (self.first_central_offset == null) {
228 self.first_central_offset = self.counting_writer.bytes_written;
185 if (header_options) |hdr| {
186 if (hdr.zip64) |options| {
187 self.bytes_written += try writer.writeInt(u16, 0x0001, .little);
188 const data_size = if (options.data_size) |size| size else 8;
189 self.bytes_written += try writer.writeInt(u16, data_size, .little);
190 self.bytes_written += try writer.writeInt(u64, 0, .little);
191 self.bytes_written += try writer.writeInt(u64, @intCast(opt.content.len), .little);
229192 }
230 self.central_count += 1;
193 }
231194
232 const hdr: zip.CentralDirectoryFileHeader = .{
233 .signature = zip.central_file_header_sig,
234 .version_made_by = 0,
235 .version_needed_to_extract = opt.version_needed_to_extract,
236 .flags = .{ .encrypted = false, ._ = 0 },
237 .compression_method = store.compression,
238 .last_modification_time = 0,
239 .last_modification_date = 0,
240 .crc32 = store.crc32,
241 .compressed_size = store.compressed_size,
242 .uncompressed_size = @intCast(store.uncompressed_size),
243 .filename_len = @intCast(opt.name.len),
244 .extra_len = 0,
245 .comment_len = 0,
246 .disk_number = 0,
247 .internal_file_attributes = 0,
248 .external_file_attributes = 0,
249 .local_file_header_offset = @intCast(store.file_offset),
250 };
251 try self.counting_writer.writer().writeStructEndian(hdr, .little);
252 try self.counting_writer.writer().writeAll(opt.name);
253 self.last_central_limit = self.counting_writer.bytes_written;
195 var compressed_size: u32 = undefined;
196 switch (opt.compression) {
197 .store => {
198 self.bytes_written += try writer.writeAll(opt.content);
199 compressed_size = @intCast(opt.content.len);
200 },
201 .deflate => {
202 const offset = self.bytes_written;
203 var fbs = std.io.fixedBufferStream(opt.content);
204 self.bytes_written += try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205 std.debug.assert(fbs.pos == opt.content.len);
206 compressed_size = @intCast(self.bytes_written - offset);
207 },
208 else => unreachable,
254209 }
210 return .{
211 .compression = opt.compression,
212 .file_offset = file_offset,
213 .crc32 = crc32,
214 .compressed_size = compressed_size,
215 .uncompressed_size = opt.content.len,
216 };
217 }
255218
256 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
257 const cd_offset = self.first_central_offset orelse 0;
258 const cd_end = self.last_central_limit orelse 0;
219 pub fn writeCentralRecord(
220 self: *Self,
221 store: FileStore,
222 opt: struct {
223 name: []const u8,
224 version_needed_to_extract: u16 = 10,
225 },
226 ) !void {
227 if (self.first_central_offset == null) {
228 self.first_central_offset = self.bytes_written;
229 }
230 self.central_count += 1;
259231
260 if (opt.zip64) |zip64| {
261 const end64_off = cd_end;
262 const fixed: zip.EndRecord64 = .{
263 .signature = zip.end_record64_sig,
264 .end_record_size = @sizeOf(zip.EndRecord64) - 12,
265 .version_made_by = 0,
266 .version_needed_to_extract = 45,
267 .disk_number = 0,
268 .central_directory_disk_number = 0,
269 .record_count_disk = @intCast(self.central_count),
270 .record_count_total = @intCast(self.central_count),
271 .central_directory_size = @intCast(cd_end - cd_offset),
272 .central_directory_offset = @intCast(cd_offset),
273 };
274 try self.counting_writer.writer().writeStructEndian(fixed, .little);
275 const locator: zip.EndLocator64 = .{
276 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
277 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
278 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
279 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
280 };
281 try self.counting_writer.writer().writeStructEndian(locator, .little);
282 }
283 const hdr: zip.EndRecord = .{
284 .signature = if (opt.sig) |s| s else zip.end_record_sig,
285 .disk_number = if (opt.disk_number) |n| n else 0,
286 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
287 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
288 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
289 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
290 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
291 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
232 const hdr: zip.CentralDirectoryFileHeader = .{
233 .signature = zip.central_file_header_sig,
234 .version_made_by = 0,
235 .version_needed_to_extract = opt.version_needed_to_extract,
236 .flags = .{ .encrypted = false, ._ = 0 },
237 .compression_method = store.compression,
238 .last_modification_time = 0,
239 .last_modification_date = 0,
240 .crc32 = store.crc32,
241 .compressed_size = store.compressed_size,
242 .uncompressed_size = @intCast(store.uncompressed_size),
243 .filename_len = @intCast(opt.name.len),
244 .extra_len = 0,
245 .comment_len = 0,
246 .disk_number = 0,
247 .internal_file_attributes = 0,
248 .external_file_attributes = 0,
249 .local_file_header_offset = @intCast(store.file_offset),
250 };
251 self.bytes_written += try self.writer.writeStructEndian(hdr, .little);
252 self.bytes_written += try self.writer.writeAll(opt.name);
253 self.last_central_limit = self.bytes_written;
254 }
255
256 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
257 const cd_offset = self.first_central_offset orelse 0;
258 const cd_end = self.last_central_limit orelse 0;
259
260 if (opt.zip64) |zip64| {
261 const end64_off = cd_end;
262 const fixed: zip.EndRecord64 = .{
263 .signature = zip.end_record64_sig,
264 .end_record_size = @sizeOf(zip.EndRecord64) - 12,
265 .version_made_by = 0,
266 .version_needed_to_extract = 45,
267 .disk_number = 0,
268 .central_directory_disk_number = 0,
269 .record_count_disk = @intCast(self.central_count),
270 .record_count_total = @intCast(self.central_count),
271 .central_directory_size = @intCast(cd_end - cd_offset),
272 .central_directory_offset = @intCast(cd_offset),
273 };
274 self.bytes_written += try self.writer.writeStructEndian(fixed, .little);
275 const locator: zip.EndLocator64 = .{
276 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
277 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
278 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
279 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
292280 };
293 try self.counting_writer.writer().writeStructEndian(hdr, .little);
294 if (opt.comment) |c|
295 try self.counting_writer.writer().writeAll(c);
281 self.bytes_written += try self.writer.writeStructEndian(locator, .little);
296282 }
297 };
298}
283 const hdr: zip.EndRecord = .{
284 .signature = if (opt.sig) |s| s else zip.end_record_sig,
285 .disk_number = if (opt.disk_number) |n| n else 0,
286 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
287 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
288 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
289 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
290 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
291 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
292 };
293 self.bytes_written += try self.writer.writeStructEndian(hdr, .little);
294 if (opt.comment) |c|
295 self.bytes_written += try self.writer.writeAll(c);
296 }
297};
src/Type.zig+92-85
......@@ -121,11 +121,10 @@ pub fn eql(a: Type, b: Type, zcu: *const Zcu) bool {
121121 return a.toIntern() == b.toIntern();
122122}
123123
124pub fn format(ty: Type, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void {
124pub fn format(ty: Type, bw: *std.io.BufferedWriter, comptime f: []const u8) anyerror!usize {
125125 _ = ty;
126 _ = unused_fmt_string;
127 _ = options;
128 _ = writer;
126 _ = f;
127 _ = bw;
129128 @compileError("do not format types directly; use either ty.fmtDebug() or ty.fmt()");
130129}
131130
......@@ -143,15 +142,9 @@ const FormatContext = struct {
143142 pt: Zcu.PerThread,
144143};
145144
146fn format2(
147 ctx: FormatContext,
148 comptime unused_format_string: []const u8,
149 options: std.fmt.FormatOptions,
150 writer: anytype,
151) !void {
152 comptime assert(unused_format_string.len == 0);
153 _ = options;
154 return print(ctx.ty, writer, ctx.pt);
145fn format2(ctx: FormatContext, bw: *std.io.BufferedWriter, comptime f: []const u8) anyerror!usize {
146 comptime assert(f.len == 0);
147 return print(ctx.ty, bw, ctx.pt);
155148}
156149
157150pub fn fmtDebug(ty: Type) std.fmt.Formatter(dump) {
......@@ -173,7 +166,7 @@ pub fn dump(
173166
174167/// Prints a name suitable for `@typeName`.
175168/// TODO: take an `opt_sema` to pass to `fmtValue` when printing sentinels.
176pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerror!void {
169pub fn print(ty: Type, bw: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerror!usize {
177170 const zcu = pt.zcu;
178171 const ip = &zcu.intern_pool;
179172 switch (ip.indexToKey(ty.toIntern())) {
......@@ -183,22 +176,23 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
183176 .signed => 'i',
184177 .unsigned => 'u',
185178 };
186 return writer.print("{c}{d}", .{ sign_char, int_type.bits });
179 return bw.print("{c}{d}", .{ sign_char, int_type.bits });
187180 },
188181 .ptr_type => {
182 var n: usize = 0;
189183 const info = ty.ptrInfo(zcu);
190184
191185 if (info.sentinel != .none) switch (info.flags.size) {
192186 .one, .c => unreachable,
193 .many => try writer.print("[*:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
194 .slice => try writer.print("[:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
187 .many => n += try bw.print("[*:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
188 .slice => n += try bw.print("[:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
195189 } else switch (info.flags.size) {
196 .one => try writer.writeAll("*"),
197 .many => try writer.writeAll("[*]"),
198 .c => try writer.writeAll("[*c]"),
199 .slice => try writer.writeAll("[]"),
190 .one => n += try bw.writeAll("*"),
191 .many => n += try bw.writeAll("[*]"),
192 .c => n += try bw.writeAll("[*c]"),
193 .slice => n += try bw.writeAll("[]"),
200194 }
201 if (info.flags.is_allowzero and info.flags.size != .c) try writer.writeAll("allowzero ");
195 if (info.flags.is_allowzero and info.flags.size != .c) n += try bw.writeAll("allowzero ");
202196 if (info.flags.alignment != .none or
203197 info.packed_offset.host_size != 0 or
204198 info.flags.vector_index != .none)
......@@ -207,76 +201,83 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
207201 info.flags.alignment
208202 else
209203 Type.fromInterned(info.child).abiAlignment(pt.zcu);
210 try writer.print("align({d}", .{alignment.toByteUnits() orelse 0});
204 n += try bw.print("align({d}", .{alignment.toByteUnits() orelse 0});
211205
212206 if (info.packed_offset.bit_offset != 0 or info.packed_offset.host_size != 0) {
213 try writer.print(":{d}:{d}", .{
207 n += try bw.print(":{d}:{d}", .{
214208 info.packed_offset.bit_offset, info.packed_offset.host_size,
215209 });
216210 }
217211 if (info.flags.vector_index == .runtime) {
218 try writer.writeAll(":?");
212 n += try bw.writeAll(":?");
219213 } else if (info.flags.vector_index != .none) {
220 try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)});
214 n += try bw.print(":{d}", .{@intFromEnum(info.flags.vector_index)});
221215 }
222 try writer.writeAll(") ");
216 n += try bw.writeAll(") ");
223217 }
224218 if (info.flags.address_space != .generic) {
225 try writer.print("addrspace(.{s}) ", .{@tagName(info.flags.address_space)});
219 n += try bw.print("addrspace(.{s}) ", .{@tagName(info.flags.address_space)});
226220 }
227 if (info.flags.is_const) try writer.writeAll("const ");
228 if (info.flags.is_volatile) try writer.writeAll("volatile ");
221 if (info.flags.is_const) n += try bw.writeAll("const ");
222 if (info.flags.is_volatile) n += try bw.writeAll("volatile ");
229223
230 try print(Type.fromInterned(info.child), writer, pt);
231 return;
224 n += try print(Type.fromInterned(info.child), bw, pt);
225 return n;
232226 },
233227 .array_type => |array_type| {
228 var n: usize = 0;
234229 if (array_type.sentinel == .none) {
235 try writer.print("[{d}]", .{array_type.len});
236 try print(Type.fromInterned(array_type.child), writer, pt);
230 n += try bw.print("[{d}]", .{array_type.len});
231 n += try print(Type.fromInterned(array_type.child), bw, pt);
237232 } else {
238 try writer.print("[{d}:{}]", .{
233 n += try bw.print("[{d}:{}]", .{
239234 array_type.len,
240235 Value.fromInterned(array_type.sentinel).fmtValue(pt),
241236 });
242 try print(Type.fromInterned(array_type.child), writer, pt);
237 n += try print(Type.fromInterned(array_type.child), bw, pt);
243238 }
244 return;
239 return n;
245240 },
246241 .vector_type => |vector_type| {
247 try writer.print("@Vector({d}, ", .{vector_type.len});
248 try print(Type.fromInterned(vector_type.child), writer, pt);
249 try writer.writeAll(")");
250 return;
242 var n: usize = 0;
243 n += try bw.print("@Vector({d}, ", .{vector_type.len});
244 n += try print(Type.fromInterned(vector_type.child), bw, pt);
245 n += try bw.writeAll(")");
246 return n;
251247 },
252248 .opt_type => |child| {
253 try writer.writeByte('?');
254 return print(Type.fromInterned(child), writer, pt);
249 var n: usize = 0;
250 n += try bw.writeByte('?');
251 n += try print(Type.fromInterned(child), bw, pt);
252 return n;
255253 },
256254 .error_union_type => |error_union_type| {
257 try print(Type.fromInterned(error_union_type.error_set_type), writer, pt);
258 try writer.writeByte('!');
255 var n: usize = 0;
256 n += try print(Type.fromInterned(error_union_type.error_set_type), bw, pt);
257 n += try bw.writeByte('!');
259258 if (error_union_type.payload_type == .generic_poison_type) {
260 try writer.writeAll("anytype");
259 n += try bw.writeAll("anytype");
261260 } else {
262 try print(Type.fromInterned(error_union_type.payload_type), writer, pt);
261 n += try print(Type.fromInterned(error_union_type.payload_type), bw, pt);
263262 }
264 return;
263 return n;
265264 },
266265 .inferred_error_set_type => |func_index| {
267266 const func_nav = ip.getNav(zcu.funcInfo(func_index).owner_nav);
268 try writer.print("@typeInfo(@typeInfo(@TypeOf({})).@\"fn\".return_type.?).error_union.error_set", .{
267 return bw.print("@typeInfo(@typeInfo(@TypeOf({})).@\"fn\".return_type.?).error_union.error_set", .{
269268 func_nav.fqn.fmt(ip),
270269 });
271270 },
272271 .error_set_type => |error_set_type| {
272 var n: usize = 0;
273273 const names = error_set_type.names;
274 try writer.writeAll("error{");
274 n += try bw.writeAll("error{");
275275 for (names.get(ip), 0..) |name, i| {
276 if (i != 0) try writer.writeByte(',');
277 try writer.print("{}", .{name.fmt(ip)});
276 if (i != 0) n += try bw.writeByte(',');
277 n += try bw.print("{}", .{name.fmt(ip)});
278278 }
279 try writer.writeAll("}");
279 n += try bw.writeAll("}");
280 return n;
280281 },
281282 .simple_type => |s| switch (s) {
282283 .f16,
......@@ -305,97 +306,103 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
305306 .comptime_float,
306307 .noreturn,
307308 .adhoc_inferred_error_set,
308 => return writer.writeAll(@tagName(s)),
309 => return bw.writeAll(@tagName(s)),
309310
310311 .null,
311312 .undefined,
312 => try writer.print("@TypeOf({s})", .{@tagName(s)}),
313 => return bw.print("@TypeOf({s})", .{@tagName(s)}),
313314
314 .enum_literal => try writer.writeAll("@Type(.enum_literal)"),
315 .enum_literal => return bw.writeAll("@Type(.enum_literal)"),
315316
316317 .generic_poison => unreachable,
317318 },
318319 .struct_type => {
319320 const name = ip.loadStructType(ty.toIntern()).name;
320 try writer.print("{}", .{name.fmt(ip)});
321 return bw.print("{}", .{name.fmt(ip)});
321322 },
322323 .tuple_type => |tuple| {
323324 if (tuple.types.len == 0) {
324 return writer.writeAll("@TypeOf(.{})");
325 return bw.writeAll("@TypeOf(.{})");
325326 }
326 try writer.writeAll("struct {");
327 var n: usize = 0;
328 n += try bw.writeAll("struct {");
327329 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, val, i| {
328 try writer.writeAll(if (i == 0) " " else ", ");
329 if (val != .none) try writer.writeAll("comptime ");
330 try print(Type.fromInterned(field_ty), writer, pt);
331 if (val != .none) try writer.print(" = {}", .{Value.fromInterned(val).fmtValue(pt)});
330 n += try bw.writeAll(if (i == 0) " " else ", ");
331 if (val != .none) n += try bw.writeAll("comptime ");
332 n += try print(Type.fromInterned(field_ty), bw, pt);
333 if (val != .none) n += try bw.print(" = {}", .{Value.fromInterned(val).fmtValue(pt)});
332334 }
333 try writer.writeAll(" }");
335 n += try bw.writeAll(" }");
336 return n;
334337 },
335338
336339 .union_type => {
337340 const name = ip.loadUnionType(ty.toIntern()).name;
338 try writer.print("{}", .{name.fmt(ip)});
341 return bw.print("{}", .{name.fmt(ip)});
339342 },
340343 .opaque_type => {
341344 const name = ip.loadOpaqueType(ty.toIntern()).name;
342 try writer.print("{}", .{name.fmt(ip)});
345 return bw.print("{}", .{name.fmt(ip)});
343346 },
344347 .enum_type => {
345348 const name = ip.loadEnumType(ty.toIntern()).name;
346 try writer.print("{}", .{name.fmt(ip)});
349 return bw.print("{}", .{name.fmt(ip)});
347350 },
348351 .func_type => |fn_info| {
352 var n: usize = 0;
349353 if (fn_info.is_noinline) {
350 try writer.writeAll("noinline ");
354 n += try bw.writeAll("noinline ");
351355 }
352 try writer.writeAll("fn (");
356 n += try bw.writeAll("fn (");
353357 const param_types = fn_info.param_types.get(&zcu.intern_pool);
354358 for (param_types, 0..) |param_ty, i| {
355 if (i != 0) try writer.writeAll(", ");
359 if (i != 0) n += try bw.writeAll(", ");
356360 if (std.math.cast(u5, i)) |index| {
357361 if (fn_info.paramIsComptime(index)) {
358 try writer.writeAll("comptime ");
362 n += try bw.writeAll("comptime ");
359363 }
360364 if (fn_info.paramIsNoalias(index)) {
361 try writer.writeAll("noalias ");
365 n += try bw.writeAll("noalias ");
362366 }
363367 }
364368 if (param_ty == .generic_poison_type) {
365 try writer.writeAll("anytype");
369 n += try bw.writeAll("anytype");
366370 } else {
367 try print(Type.fromInterned(param_ty), writer, pt);
371 n += try print(Type.fromInterned(param_ty), bw, pt);
368372 }
369373 }
370374 if (fn_info.is_var_args) {
371375 if (param_types.len != 0) {
372 try writer.writeAll(", ");
376 n += try bw.writeAll(", ");
373377 }
374 try writer.writeAll("...");
378 n += try bw.writeAll("...");
375379 }
376 try writer.writeAll(") ");
380 n += try bw.writeAll(") ");
377381 if (fn_info.cc != .auto) print_cc: {
378382 if (zcu.getTarget().cCallingConvention()) |ccc| {
379383 if (fn_info.cc.eql(ccc)) {
380 try writer.writeAll("callconv(.c) ");
384 n += try bw.writeAll("callconv(.c) ");
381385 break :print_cc;
382386 }
383387 }
384388 switch (fn_info.cc) {
385 .auto, .@"async", .naked, .@"inline" => try writer.print("callconv(.{}) ", .{std.zig.fmtId(@tagName(fn_info.cc))}),
386 else => try writer.print("callconv({any}) ", .{fn_info.cc}),
389 .auto, .@"async", .naked, .@"inline" => n += try bw.print("callconv(.{}) ", .{std.zig.fmtId(@tagName(fn_info.cc))}),
390 else => n += try bw.print("callconv({any}) ", .{fn_info.cc}),
387391 }
388392 }
389393 if (fn_info.return_type == .generic_poison_type) {
390 try writer.writeAll("anytype");
394 n += try bw.writeAll("anytype");
391395 } else {
392 try print(Type.fromInterned(fn_info.return_type), writer, pt);
396 n += try print(Type.fromInterned(fn_info.return_type), bw, pt);
393397 }
398 return n;
394399 },
395400 .anyframe_type => |child| {
396 if (child == .none) return writer.writeAll("anyframe");
397 try writer.writeAll("anyframe->");
398 return print(Type.fromInterned(child), writer, pt);
401 if (child == .none) return bw.writeAll("anyframe");
402 var n: usize = 0;
403 n += try bw.writeAll("anyframe->");
404 n += print(Type.fromInterned(child), bw, pt);
405 return n;
399406 },
400407
401408 // values, not types
src/codegen/c.zig+61-69
......@@ -1270,7 +1270,7 @@ pub const DeclGen = struct {
12701270 }
12711271 const ai = ty.arrayInfo(zcu);
12721272 if (ai.elem_type.eql(.u8, zcu)) {
1273 var literal = stringLiteral(writer, ty.arrayLenIncludingSentinel(zcu));
1273 var literal: StringLiteral = .init(writer, ty.arrayLenIncludingSentinel(zcu));
12741274 try literal.start();
12751275 var index: usize = 0;
12761276 while (index < ai.len) : (index += 1) {
......@@ -1829,7 +1829,7 @@ pub const DeclGen = struct {
18291829 const ai = ty.arrayInfo(zcu);
18301830 if (ai.elem_type.eql(.u8, zcu)) {
18311831 const c_len = ty.arrayLenIncludingSentinel(zcu);
1832 var literal = stringLiteral(writer, c_len);
1832 var literal: StringLiteral = .init(writer, c_len);
18331833 try literal.start();
18341834 var index: u64 = 0;
18351835 while (index < c_len) : (index += 1)
......@@ -8111,7 +8111,12 @@ fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {
81118111 };
81128112}
81138113
8114fn StringLiteral(comptime WriterType: type) type {
8114const StringLiteral = struct {
8115 len: usize,
8116 cur_len: usize,
8117 bytes_written: usize,
8118 writer: *std.io.BufferedWriter,
8119
81158120 // MSVC throws C2078 if an array of size 65536 or greater is initialized with a string literal,
81168121 // regardless of the length of the string literal initializing it. Array initializer syntax is
81178122 // used instead.
......@@ -8123,81 +8128,68 @@ fn StringLiteral(comptime WriterType: type) type {
81238128 const max_char_len = 4;
81248129 const max_literal_len = @min(16380 - max_char_len, 4095);
81258130
8126 return struct {
8127 len: u64,
8128 cur_len: u64 = 0,
8129 counting_writer: std.io.CountingWriter(WriterType),
8130
8131 pub const Error = WriterType.Error;
8132
8133 const Self = @This();
8131 fn init(writer: *std.io.BufferedWriter, len: usize) StringLiteral {
8132 return .{
8133 .cur_len = 0,
8134 .len = len,
8135 .writer = writer,
8136 .bytes_written = 0,
8137 };
8138 }
81348139
8135 pub fn start(self: *Self) Error!void {
8136 const writer = self.counting_writer.writer();
8137 if (self.len <= max_string_initializer_len) {
8138 try writer.writeByte('\"');
8139 } else {
8140 try writer.writeByte('{');
8141 }
8140 pub fn start(self: *StringLiteral) anyerror!void {
8141 const writer = self.writer;
8142 if (self.len <= max_string_initializer_len) {
8143 self.bytes_written += try writer.writeByteCount('\"');
8144 } else {
8145 self.bytes_written += try writer.writeByteCount('{');
81428146 }
8147 }
81438148
8144 pub fn end(self: *Self) Error!void {
8145 const writer = self.counting_writer.writer();
8146 if (self.len <= max_string_initializer_len) {
8147 try writer.writeByte('\"');
8148 } else {
8149 try writer.writeByte('}');
8150 }
8149 pub fn end(self: *StringLiteral) anyerror!void {
8150 const writer = self.writer;
8151 if (self.len <= max_string_initializer_len) {
8152 self.bytes_written += try writer.writeByteCount('\"');
8153 } else {
8154 self.bytes_written += try writer.writeByteCount('}');
81518155 }
8156 }
81528157
8153 fn writeStringLiteralChar(writer: anytype, c: u8) !void {
8154 switch (c) {
8155 7 => try writer.writeAll("\\a"),
8156 8 => try writer.writeAll("\\b"),
8157 '\t' => try writer.writeAll("\\t"),
8158 '\n' => try writer.writeAll("\\n"),
8159 11 => try writer.writeAll("\\v"),
8160 12 => try writer.writeAll("\\f"),
8161 '\r' => try writer.writeAll("\\r"),
8162 '"', '\'', '?', '\\' => try writer.print("\\{c}", .{c}),
8163 else => switch (c) {
8164 ' '...'~' => try writer.writeByte(c),
8165 else => try writer.print("\\{o:0>3}", .{c}),
8166 },
8167 }
8158 fn writeStringLiteralChar(writer: *std.io.BufferedWriter, c: u8) anyerror!usize {
8159 switch (c) {
8160 7 => return writer.writeAllCount("\\a"),
8161 8 => return writer.writeAllCount("\\b"),
8162 '\t' => return writer.writeAllCount("\\t"),
8163 '\n' => return writer.writeAllCount("\\n"),
8164 11 => return writer.writeAllCount("\\v"),
8165 12 => return writer.writeAllCount("\\f"),
8166 '\r' => return writer.writeAllCount("\\r"),
8167 '"', '\'', '?', '\\' => return writer.printCount("\\{c}", .{c}),
8168 else => switch (c) {
8169 ' '...'~' => return writer.writeByteCount(c),
8170 else => return writer.printCount("\\{o:0>3}", .{c}),
8171 },
81688172 }
8173 }
81698174
8170 pub fn writeChar(self: *Self, c: u8) Error!void {
8171 const writer = self.counting_writer.writer();
8172 if (self.len <= max_string_initializer_len) {
8173 if (self.cur_len == 0 and self.counting_writer.bytes_written > 1)
8174 try writer.writeAll("\"\"");
8175
8176 const len = self.counting_writer.bytes_written;
8177 try writeStringLiteralChar(writer, c);
8175 pub fn writeChar(self: *StringLiteral, c: u8) anyerror!void {
8176 const writer = self.writer;
8177 if (self.len <= max_string_initializer_len) {
8178 if (self.cur_len == 0 and self.bytes_written > 1)
8179 self.bytes_written += try writer.writeAllCount("\"\"");
81788180
8179 const char_length = self.counting_writer.bytes_written - len;
8180 assert(char_length <= max_char_len);
8181 self.cur_len += char_length;
8181 const char_length = try writeStringLiteralChar(writer, c);
8182 self.bytes_written += char_length;
8183 assert(char_length <= max_char_len);
8184 self.cur_len += char_length;
81828185
8183 if (self.cur_len >= max_literal_len) self.cur_len = 0;
8184 } else {
8185 if (self.counting_writer.bytes_written > 1) try writer.writeByte(',');
8186 try writer.print("'\\x{x}'", .{c});
8187 }
8186 if (self.cur_len >= max_literal_len) self.cur_len = 0;
8187 } else {
8188 if (self.bytes_written > 1) self.bytes_written += try writer.writeByteCount(',');
8189 self.bytes_written += try writer.printCount("'\\x{x}'", .{c});
81888190 }
8189 };
8190}
8191
8192fn stringLiteral(
8193 child_stream: anytype,
8194 len: u64,
8195) StringLiteral(@TypeOf(child_stream)) {
8196 return .{
8197 .len = len,
8198 .counting_writer = std.io.countingWriter(child_stream),
8199 };
8200}
8191 }
8192};
82018193
82028194const FormatStringContext = struct { str: []const u8, sentinel: ?u8 };
82038195fn formatStringLiteral(
......@@ -8208,7 +8200,7 @@ fn formatStringLiteral(
82088200) @TypeOf(writer).Error!void {
82098201 if (fmt.len != 1 or fmt[0] != 's') @compileError("Invalid fmt: " ++ fmt);
82108202
8211 var literal = stringLiteral(writer, data.str.len + @intFromBool(data.sentinel != null));
8203 var literal: StringLiteral = .init(writer, data.str.len + @intFromBool(data.sentinel != null));
82128204 try literal.start();
82138205 for (data.str) |c| try literal.writeChar(c);
82148206 if (data.sentinel) |sentinel| if (sentinel != 0) try literal.writeChar(sentinel);
src/link/Dwarf.zig+32-22
......@@ -1768,34 +1768,36 @@ pub const WipNav = struct {
17681768 }
17691769
17701770 const ExprLocCounter = struct {
1771 const Stream = std.io.CountingWriter(std.io.NullWriter);
1772 stream: Stream,
1771 stream: *std.io.BufferedWriter,
17731772 section_offset_bytes: u32,
17741773 address_size: AddressSize,
1775 fn init(dwarf: *Dwarf) ExprLocCounter {
1774 counter: usize,
1775 fn init(dwarf: *Dwarf, stream: *std.io.BufferedWriter) ExprLocCounter {
17761776 return .{
1777 .stream = std.io.countingWriter(std.io.null_writer),
1777 .stream = stream,
17781778 .section_offset_bytes = dwarf.sectionOffsetBytes(),
17791779 .address_size = dwarf.address_size,
17801780 };
17811781 }
1782 fn writer(counter: *ExprLocCounter) Stream.Writer {
1783 return counter.stream.writer();
1782 fn writer(counter: *ExprLocCounter) *std.io.BufferedWriter {
1783 return counter.stream;
17841784 }
17851785 fn endian(_: ExprLocCounter) std.builtin.Endian {
17861786 return @import("builtin").cpu.arch.endian();
17871787 }
17881788 fn addrSym(counter: *ExprLocCounter, _: u32) error{}!void {
1789 counter.stream.bytes_written += @intFromEnum(counter.address_size);
1789 counter.count += @intFromEnum(counter.address_size);
17901790 }
17911791 fn infoEntry(counter: *ExprLocCounter, _: Unit.Index, _: Entry.Index) error{}!void {
1792 counter.stream.bytes_written += counter.section_offset_bytes;
1792 counter.count += counter.section_offset_bytes;
17931793 }
17941794 };
17951795
17961796 fn infoExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {
1797 var counter: ExprLocCounter = .init(wip_nav.dwarf);
1798 try loc.write(&counter);
1797 var buffer: [std.atomic.cache_line]u8 = undefined;
1798 var counter_bw = std.io.Writer.null.buffered(&buffer);
1799 var counter: ExprLocCounter = .init(wip_nav.dwarf, &counter_bw);
1800 counter.count += try loc.write(&counter);
17991801
18001802 const adapter: struct {
18011803 wip_nav: *WipNav,
......@@ -1812,8 +1814,8 @@ pub const WipNav = struct {
18121814 try ctx.wip_nav.infoSectionOffset(.debug_info, unit, entry, 0);
18131815 }
18141816 } = .{ .wip_nav = wip_nav };
1815 try uleb128(adapter.writer(), counter.stream.bytes_written);
1816 try loc.write(adapter);
1817 try uleb128(adapter.writer(), counter.count);
1818 _ = try loc.write(adapter);
18171819 }
18181820
18191821 fn infoAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {
......@@ -1826,8 +1828,10 @@ pub const WipNav = struct {
18261828 }
18271829
18281830 fn frameExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {
1829 var counter: ExprLocCounter = .init(wip_nav.dwarf);
1830 try loc.write(&counter);
1831 var buffer: [std.atomic.cache_line]u8 = undefined;
1832 var counter_bw = std.io.Writer.null.buffered(&buffer);
1833 var counter: ExprLocCounter = .init(wip_nav.dwarf, &counter_bw);
1834 counter.count += try loc.write(&counter);
18311835
18321836 const adapter: struct {
18331837 wip_nav: *WipNav,
......@@ -1844,8 +1848,8 @@ pub const WipNav = struct {
18441848 try ctx.wip_nav.sectionOffset(.debug_frame, .debug_info, unit, entry, 0);
18451849 }
18461850 } = .{ .wip_nav = wip_nav };
1847 try uleb128(adapter.writer(), counter.stream.bytes_written);
1848 try loc.write(adapter);
1851 try uleb128(adapter.writer(), counter.count);
1852 _ = try loc.write(adapter);
18491853 }
18501854
18511855 fn frameAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {
......@@ -6015,15 +6019,21 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 {
60156019}
60166020
60176021fn uleb128Bytes(value: anytype) u32 {
6018 var cw = std.io.countingWriter(std.io.null_writer);
6019 try uleb128(cw.writer(), value);
6020 return @intCast(cw.bytes_written);
6022 var buffer: [std.atomic.cache_line]u8 = undefined;
6023 var bw: std.io.BufferedWriter = .{
6024 .unbuffered_writer = .null,
6025 .buffer = .initBuffer(&buffer),
6026 };
6027 return try std.leb.writeUleb128Count(&bw, value);
60216028}
60226029
60236030fn sleb128Bytes(value: anytype) u32 {
6024 var cw = std.io.countingWriter(std.io.null_writer);
6025 try sleb128(cw.writer(), value);
6026 return @intCast(cw.bytes_written);
6031 var buffer: [std.atomic.cache_line]u8 = undefined;
6032 var bw: std.io.BufferedWriter = .{
6033 .unbuffered_writer = .null,
6034 .buffer = .initBuffer(&buffer),
6035 };
6036 return try std.leb.writeIleb128Count(&bw, value);
60276037}
60286038
60296039/// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional