authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-22 14:02:17-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-02-22 14:02:17-08:00
log8802ec583b376101ee7fcb0a64cf937b90dc6130
treec34ce577f31b39a63d95598759628b487dd47d44
parente60d66711185fa2e3164f5af92b9786e04c4fa19
parenta5326c5ef81885821d39f843bbe83c6a3aaff170
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19032 from ianic/add_buffered_tee

propose adding BufferedTee to the std.io

3 files changed, 406 insertions(+), 143 deletions(-)

lib/std/io.zig+4
...@@ -411,6 +411,9 @@ pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAt...@@ -411,6 +411,9 @@ pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAt
411411
412pub const StreamSource = @import("io/stream_source.zig").StreamSource;412pub const StreamSource = @import("io/stream_source.zig").StreamSource;
413413
414pub const BufferedTee = @import("io/buffered_tee.zig").BufferedTee;
415pub const bufferedTee = @import("io/buffered_tee.zig").bufferedTee;
416
414pub const tty = @import("io/tty.zig");417pub const tty = @import("io/tty.zig");
415418
416/// A Writer that doesn't write to anything.419/// A Writer that doesn't write to anything.
...@@ -692,4 +695,5 @@ test {...@@ -692,4 +695,5 @@ test {
692 _ = @import("io/seekable_stream.zig");695 _ = @import("io/seekable_stream.zig");
693 _ = @import("io/stream_source.zig");696 _ = @import("io/stream_source.zig");
694 _ = @import("io/test.zig");697 _ = @import("io/test.zig");
698 _ = @import("io/buffered_tee.zig");
695}699}
lib/std/io/buffered_tee.zig created+379
...@@ -0,0 +1,379 @@
1const std = @import("std");
2const io = std.io;
3const assert = std.debug.assert;
4const testing = std.testing;
5
6/// BufferedTee provides reader interface to the consumer. Data read by consumer
7/// is also written to the output. Output is hold lookahead_size bytes behind
8/// consumer. Allowing consumer to put back some bytes to be read again. On flush
9/// all consumed bytes are flushed to the output.
10///
11/// input -> tee -> consumer
12/// |
13/// output
14///
15/// input - underlying unbuffered reader
16/// output - writer, receives data read by consumer
17/// consumer - uses provided reader interface
18///
19/// If lookahead_size is zero output always has same bytes as consumer.
20///
21pub fn BufferedTee(
22 comptime buffer_size: usize, // internal buffer size in bytes
23 comptime lookahead_size: usize, // lookahead, number of bytes to hold output behind consumer
24 comptime InputReaderType: type,
25 comptime OutputWriterType: type,
26) type {
27 comptime assert(buffer_size > lookahead_size);
28
29 return struct {
30 input: InputReaderType,
31 output: OutputWriterType,
32
33 buf: [buffer_size]u8 = undefined, // internal buffer
34 tail: usize = 0, // buffer is filled up to this position with bytes from input
35 rp: usize = 0, // reader pointer; consumer has read up to this position
36 wp: usize = 0, // writer pointer; data is sent to the output up to this position
37
38 pub const Error = InputReaderType.Error || OutputWriterType.Error;
39 pub const Reader = io.Reader(*Self, Error, read);
40
41 const Self = @This();
42
43 pub fn read(self: *Self, dest: []u8) Error!usize {
44 var dest_index: usize = 0;
45
46 while (dest_index < dest.len) {
47 const written = @min(dest.len - dest_index, self.tail - self.rp);
48 if (written == 0) {
49 try self.preserveLookahead();
50 // fill upper part of the buf
51 const n = try self.input.read(self.buf[self.tail..]);
52 if (n == 0) {
53 // reading from the unbuffered stream returned nothing
54 // so we have nothing left to read.
55 return dest_index;
56 }
57 self.tail += n;
58 } else {
59 @memcpy(dest[dest_index..][0..written], self.buf[self.rp..][0..written]);
60 self.rp += written;
61 dest_index += written;
62 try self.flush_(lookahead_size);
63 }
64 }
65 return dest.len;
66 }
67
68 /// Move lookahead_size bytes to the buffer start.
69 fn preserveLookahead(self: *Self) !void {
70 assert(self.tail == self.rp);
71 if (lookahead_size == 0) {
72 // Flush is called on each read so wp must follow rp when lookahead_size == 0.
73 assert(self.wp == self.rp);
74 // Nothing to preserve rewind pointer to the buffer start
75 self.rp = 0;
76 self.wp = 0;
77 self.tail = 0;
78 return;
79 }
80 if (self.tail <= lookahead_size) {
81 // There is still palce in the buffer, append to buffer from tail position.
82 return;
83 }
84 try self.flush_(lookahead_size);
85 const head = self.tail - lookahead_size;
86 // Preserve head..tail at the start of the buffer.
87 std.mem.copyForwards(u8, self.buf[0..lookahead_size], self.buf[head..self.tail]);
88 self.wp -= head;
89 assert(self.wp <= lookahead_size);
90 self.rp = lookahead_size;
91 self.tail = lookahead_size;
92 }
93
94 /// Flush to the output all but lookahead size bytes.
95 fn flush_(self: *Self, lookahead: usize) !void {
96 if (self.rp <= self.wp + lookahead) return;
97 const new_wp = self.rp - lookahead;
98 try self.output.writeAll(self.buf[self.wp..new_wp]);
99 self.wp = new_wp;
100 }
101
102 /// Flush to the output all consumed bytes.
103 pub fn flush(self: *Self) !void {
104 try self.flush_(0);
105 }
106
107 /// Put back some bytes to be consumed again. Usefull when we overshoot
108 /// reading and want to return that overshoot bytes. Can return maximum
109 /// of lookahead_size number of bytes.
110 pub fn putBack(self: *Self, n: usize) void {
111 assert(n <= lookahead_size and n <= self.rp);
112 self.rp -= n;
113 }
114
115 pub fn reader(self: *Self) Reader {
116 return .{ .context = self };
117 }
118 };
119}
120
121pub fn bufferedTee(
122 comptime buffer_size: usize,
123 comptime lookahead_size: usize,
124 input: anytype,
125 output: anytype,
126) BufferedTee(
127 buffer_size,
128 lookahead_size,
129 @TypeOf(input),
130 @TypeOf(output),
131) {
132 return .{ .input = input, .output = output };
133}
134
135// Running test from std.io.BufferedReader on BufferedTee
136// It should act as BufferedReader for consumer.
137
138fn BufferedReader(comptime buffer_size: usize, comptime ReaderType: type) type {
139 return BufferedTee(buffer_size, 0, ReaderType, @TypeOf(io.null_writer));
140}
141
142fn bufferedReader(reader: anytype) BufferedReader(4096, @TypeOf(reader)) {
143 return .{
144 .input = reader,
145 .output = io.null_writer,
146 };
147}
148
149test "io.BufferedTee io.BufferedReader OneByte" {
150 const OneByteReadReader = struct {
151 str: []const u8,
152 curr: usize,
153
154 const Error = error{NoError};
155 const Self = @This();
156 const Reader = io.Reader(*Self, Error, read);
157
158 fn init(str: []const u8) Self {
159 return Self{
160 .str = str,
161 .curr = 0,
162 };
163 }
164
165 fn read(self: *Self, dest: []u8) Error!usize {
166 if (self.str.len <= self.curr or dest.len == 0)
167 return 0;
168
169 dest[0] = self.str[self.curr];
170 self.curr += 1;
171 return 1;
172 }
173
174 fn reader(self: *Self) Reader {
175 return .{ .context = self };
176 }
177 };
178
179 const str = "This is a test";
180 var one_byte_stream = OneByteReadReader.init(str);
181 var buf_reader = bufferedReader(one_byte_stream.reader());
182 const stream = buf_reader.reader();
183
184 const res = try stream.readAllAlloc(testing.allocator, str.len + 1);
185 defer testing.allocator.free(res);
186 try testing.expectEqualSlices(u8, str, res);
187}
188
189test "io.BufferedTee io.BufferedReader Block" {
190 const BlockReader = struct {
191 block: []const u8,
192 reads_allowed: usize,
193 curr_read: usize,
194
195 const Error = error{NoError};
196 const Self = @This();
197 const Reader = io.Reader(*Self, Error, read);
198
199 fn init(block: []const u8, reads_allowed: usize) Self {
200 return Self{
201 .block = block,
202 .reads_allowed = reads_allowed,
203 .curr_read = 0,
204 };
205 }
206
207 fn read(self: *Self, dest: []u8) Error!usize {
208 if (self.curr_read >= self.reads_allowed) return 0;
209 @memcpy(dest[0..self.block.len], self.block);
210
211 self.curr_read += 1;
212 return self.block.len;
213 }
214
215 fn reader(self: *Self) Reader {
216 return .{ .context = self };
217 }
218 };
219
220 const block = "0123";
221
222 // len out == block
223 {
224 var test_buf_reader: BufferedReader(4, BlockReader) = .{
225 .input = BlockReader.init(block, 2),
226 .output = io.null_writer,
227 };
228 var out_buf: [4]u8 = undefined;
229 _ = try test_buf_reader.read(&out_buf);
230 try testing.expectEqualSlices(u8, &out_buf, block);
231 _ = try test_buf_reader.read(&out_buf);
232 try testing.expectEqualSlices(u8, &out_buf, block);
233 try testing.expectEqual(try test_buf_reader.read(&out_buf), 0);
234 }
235
236 // len out < block
237 {
238 var test_buf_reader: BufferedReader(4, BlockReader) = .{
239 .input = BlockReader.init(block, 2),
240 .output = io.null_writer,
241 };
242 var out_buf: [3]u8 = undefined;
243 _ = try test_buf_reader.read(&out_buf);
244 try testing.expectEqualSlices(u8, &out_buf, "012");
245 _ = try test_buf_reader.read(&out_buf);
246 try testing.expectEqualSlices(u8, &out_buf, "301");
247 const n = try test_buf_reader.read(&out_buf);
248 try testing.expectEqualSlices(u8, out_buf[0..n], "23");
249 try testing.expectEqual(try test_buf_reader.read(&out_buf), 0);
250 }
251
252 // len out > block
253 {
254 var test_buf_reader: BufferedReader(4, BlockReader) = .{
255 .input = BlockReader.init(block, 2),
256 .output = io.null_writer,
257 };
258 var out_buf: [5]u8 = undefined;
259 _ = try test_buf_reader.read(&out_buf);
260 try testing.expectEqualSlices(u8, &out_buf, "01230");
261 const n = try test_buf_reader.read(&out_buf);
262 try testing.expectEqualSlices(u8, out_buf[0..n], "123");
263 try testing.expectEqual(try test_buf_reader.read(&out_buf), 0);
264 }
265
266 // len out == 0
267 {
268 var test_buf_reader: BufferedReader(4, BlockReader) = .{
269 .input = BlockReader.init(block, 2),
270 .output = io.null_writer,
271 };
272 var out_buf: [0]u8 = undefined;
273 _ = try test_buf_reader.read(&out_buf);
274 try testing.expectEqualSlices(u8, &out_buf, "");
275 }
276
277 // len bufreader buf > block
278 {
279 var test_buf_reader: BufferedReader(5, BlockReader) = .{
280 .input = BlockReader.init(block, 2),
281 .output = io.null_writer,
282 };
283 var out_buf: [4]u8 = undefined;
284 _ = try test_buf_reader.read(&out_buf);
285 try testing.expectEqualSlices(u8, &out_buf, block);
286 _ = try test_buf_reader.read(&out_buf);
287 try testing.expectEqualSlices(u8, &out_buf, block);
288 try testing.expectEqual(try test_buf_reader.read(&out_buf), 0);
289 }
290}
291
292test "io.BufferedTee with zero lookahead" {
293 // output has same bytes as consumer
294 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 } ** 12;
295 var in = io.fixedBufferStream(&data);
296 var out = std.ArrayList(u8).init(testing.allocator);
297 defer out.deinit();
298
299 var bt = bufferedTee(8, 0, in.reader(), out.writer());
300
301 var buf: [16]u8 = undefined;
302 var read_len: usize = 0;
303 for (0..buf.len) |i| {
304 const n = try bt.read(buf[0..i]);
305 try testing.expectEqual(i, n);
306 read_len += i;
307 try testing.expectEqual(read_len, out.items.len);
308 }
309}
310
311test "io.BufferedTee with lookahead" {
312 // output is lookahead bytes behind consumer
313 inline for (1..8) |lookahead| {
314 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 } ** 12;
315 var in = io.fixedBufferStream(&data);
316 var out = std.ArrayList(u8).init(testing.allocator);
317 defer out.deinit();
318
319 var bt = bufferedTee(8, lookahead, in.reader(), out.writer());
320 var buf: [16]u8 = undefined;
321
322 var read_len: usize = 0;
323 for (1..buf.len) |i| {
324 const n = try bt.read(buf[0..i]);
325 try testing.expectEqual(i, n);
326 read_len += i;
327 const out_len = if (read_len < lookahead) 0 else read_len - lookahead;
328 try testing.expectEqual(out_len, out.items.len);
329 }
330 try testing.expectEqual(read_len, out.items.len + lookahead);
331 try bt.flush();
332 try testing.expectEqual(read_len, out.items.len);
333 }
334}
335
336test "io.BufferedTee internal state" {
337 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 } ** 2;
338 var in = io.fixedBufferStream(&data);
339 var out = std.ArrayList(u8).init(testing.allocator);
340 defer out.deinit();
341
342 var bt = bufferedTee(8, 4, in.reader(), out.writer());
343
344 var buf: [16]u8 = undefined;
345 var n = try bt.read(buf[0..3]);
346 try testing.expectEqual(3, n);
347 try testing.expectEqualSlices(u8, data[0..3], buf[0..n]);
348 try testing.expectEqual(8, bt.tail);
349 try testing.expectEqual(3, bt.rp);
350 try testing.expectEqual(0, out.items.len);
351
352 n = try bt.read(buf[0..6]);
353 try testing.expectEqual(6, n);
354 try testing.expectEqualSlices(u8, data[3..9], buf[0..n]);
355 try testing.expectEqual(8, bt.tail);
356 try testing.expectEqual(5, bt.rp);
357 try testing.expectEqualSlices(u8, data[4..12], &bt.buf);
358 try testing.expectEqual(5, out.items.len);
359
360 n = try bt.read(buf[0..9]);
361 try testing.expectEqual(9, n);
362 try testing.expectEqualSlices(u8, data[9..18], buf[0..n]);
363 try testing.expectEqual(8, bt.tail);
364 try testing.expectEqual(6, bt.rp);
365 try testing.expectEqualSlices(u8, data[12..20], &bt.buf);
366 try testing.expectEqual(14, out.items.len);
367
368 try bt.flush();
369 try testing.expectEqual(18, out.items.len);
370
371 bt.putBack(4);
372 n = try bt.read(buf[0..4]);
373 try testing.expectEqual(4, n);
374 try testing.expectEqualSlices(u8, data[14..18], buf[0..n]);
375
376 try testing.expectEqual(18, out.items.len);
377 try bt.flush();
378 try testing.expectEqual(18, out.items.len);
379}
src/Package/Fetch/git.zig+23-143
...@@ -1091,86 +1091,6 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1091,86 +1091,6 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1091 try index_writer.writeAll(&index_checksum);1091 try index_writer.writeAll(&index_checksum);
1092}1092}
10931093
1094/// A reader that stores read data in a growable internal buffer. The read
1095/// position can be rewound to allow previously read data to be read again.
1096fn AccumulatingReader(comptime ReaderType: type) type {
1097 return struct {
1098 child_reader: ReaderType,
1099 buffer: std.ArrayListUnmanaged(u8) = .{},
1100 /// The position in `buffer` from which reads should start, returning
1101 /// buffered data. If this is `buffer.items.len`, data will be read from
1102 /// `child_reader` instead.
1103 read_start: usize = 0,
1104 allocator: Allocator,
1105
1106 const Self = @This();
1107
1108 fn deinit(self: *Self) void {
1109 self.buffer.deinit(self.allocator);
1110 self.* = undefined;
1111 }
1112
1113 const ReadError = ReaderType.Error || Allocator.Error;
1114 const Reader = std.io.Reader(*Self, ReadError, read);
1115
1116 fn read(self: *Self, buf: []u8) ReadError!usize {
1117 if (self.read_start < self.buffer.items.len) {
1118 // Previously buffered data is available and should be used
1119 // before reading more data from the underlying reader.
1120 const available = self.buffer.items.len - self.read_start;
1121 const count = @min(available, buf.len);
1122 @memcpy(buf[0..count], self.buffer.items[self.read_start..][0..count]);
1123 self.read_start += count;
1124 return count;
1125 }
1126
1127 try self.buffer.ensureUnusedCapacity(self.allocator, buf.len);
1128 const read_buffer = self.buffer.unusedCapacitySlice();
1129 const count = try self.child_reader.read(read_buffer[0..buf.len]);
1130 @memcpy(buf[0..count], read_buffer[0..count]);
1131 self.buffer.items.len += count;
1132 self.read_start += count;
1133 return count;
1134 }
1135
1136 fn reader(self: *Self) Reader {
1137 return .{ .context = self };
1138 }
1139
1140 /// Returns a slice of the buffered data that has already been read,
1141 /// except the last `count_before_end` bytes.
1142 fn readDataExcept(self: Self, count_before_end: usize) []const u8 {
1143 assert(count_before_end <= self.read_start);
1144 return self.buffer.items[0 .. self.read_start - count_before_end];
1145 }
1146
1147 /// Discards the first `count` bytes of buffered data.
1148 fn discard(self: *Self, count: usize) void {
1149 assert(count <= self.buffer.items.len);
1150 const retain = self.buffer.items.len - count;
1151 mem.copyForwards(
1152 u8,
1153 self.buffer.items[0..retain],
1154 self.buffer.items[count..][0..retain],
1155 );
1156 self.buffer.items.len = retain;
1157 self.read_start -= @min(self.read_start, count);
1158 }
1159
1160 /// Rewinds the read position to the beginning of buffered data.
1161 fn rewind(self: *Self) void {
1162 self.read_start = 0;
1163 }
1164 };
1165}
1166
1167fn accumulatingReader(
1168 allocator: Allocator,
1169 reader: anytype,
1170) AccumulatingReader(@TypeOf(reader)) {
1171 return .{ .child_reader = reader, .allocator = allocator };
1172}
1173
1174/// Performs the first pass over the packfile data for index construction.1094/// Performs the first pass over the packfile data for index construction.
1175/// This will index all non-delta objects, queue delta objects for further1095/// This will index all non-delta objects, queue delta objects for further
1176/// processing, and return the pack checksum (which is part of the index1096/// processing, and return the pack checksum (which is part of the index
...@@ -1181,102 +1101,62 @@ fn indexPackFirstPass(...@@ -1181,102 +1101,62 @@ fn indexPackFirstPass(
1181 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),1101 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1182 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),1102 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1183) ![Sha1.digest_length]u8 {1103) ![Sha1.digest_length]u8 {
1184 var pack_buffered_reader = std.io.bufferedReader(pack.reader());1104 var pack_counting_writer = std.io.countingWriter(std.io.null_writer);
1185 var pack_accumulating_reader = accumulatingReader(allocator, pack_buffered_reader.reader());1105 var pack_hashed_writer = std.compress.hashedWriter(pack_counting_writer.writer(), Sha1.init(.{}));
1186 defer pack_accumulating_reader.deinit();1106 var entry_crc32_writer = std.compress.hashedWriter(pack_hashed_writer.writer(), std.hash.Crc32.init());
1187 var pack_position: usize = 0;1107 var pack_buffered_reader = std.io.bufferedTee(4096, 8, pack.reader(), entry_crc32_writer.writer());
1188 var pack_hash = Sha1.init(.{});1108 const pack_reader = pack_buffered_reader.reader();
1189 const pack_reader = pack_accumulating_reader.reader();
11901109
1191 const pack_header = try PackHeader.read(pack_reader);1110 const pack_header = try PackHeader.read(pack_reader);
1192 const pack_header_bytes = pack_accumulating_reader.readDataExcept(0);1111 try pack_buffered_reader.flush();
1193 pack_position += pack_header_bytes.len;
1194 pack_hash.update(pack_header_bytes);
1195 pack_accumulating_reader.discard(pack_header_bytes.len);
11961112
1197 var current_entry: u32 = 0;1113 var current_entry: u32 = 0;
1198 while (current_entry < pack_header.total_objects) : (current_entry += 1) {1114 while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1199 const entry_offset = pack_position;1115 const entry_offset = pack_counting_writer.bytes_written;
1200 var entry_crc32 = std.hash.Crc32.init();1116 entry_crc32_writer.hasher = std.hash.Crc32.init(); // reset hasher
1201
1202 const entry_header = try EntryHeader.read(pack_reader);1117 const entry_header = try EntryHeader.read(pack_reader);
1203 const entry_header_bytes = pack_accumulating_reader.readDataExcept(0);
1204 pack_position += entry_header_bytes.len;
1205 pack_hash.update(entry_header_bytes);
1206 entry_crc32.update(entry_header_bytes);
1207 pack_accumulating_reader.discard(entry_header_bytes.len);
12081118
1209 switch (entry_header) {1119 switch (entry_header) {
1210 .commit, .tree, .blob, .tag => |object| {1120 .commit, .tree, .blob, .tag => |object| {
1211 var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader);1121 var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader);
1212 var entry_data_size: usize = 0;1122 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1213 var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));1123 var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1214 const entry_writer = entry_hashed_writer.writer();1124 const entry_writer = entry_hashed_writer.writer();
1215
1216 // The object header is not included in the pack data but is1125 // The object header is not included in the pack data but is
1217 // part of the object's ID1126 // part of the object's ID
1218 try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });1127 try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });
12191128 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1220 while (try entry_decompress_stream.next()) |decompressed_data| {1129 try fifo.pump(entry_counting_reader.reader(), entry_writer);
1221 entry_data_size += decompressed_data.len;1130 if (entry_counting_reader.bytes_read != object.uncompressed_length) {
1222 try entry_writer.writeAll(decompressed_data);
1223
1224 const compressed_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes());
1225 pack_position += compressed_bytes.len;
1226 pack_hash.update(compressed_bytes);
1227 entry_crc32.update(compressed_bytes);
1228 pack_accumulating_reader.discard(compressed_bytes.len);
1229 }
1230 const footer_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes());
1231 pack_position += footer_bytes.len;
1232 pack_hash.update(footer_bytes);
1233 entry_crc32.update(footer_bytes);
1234 pack_accumulating_reader.discard(footer_bytes.len);
1235 pack_accumulating_reader.rewind();
1236
1237 if (entry_data_size != object.uncompressed_length) {
1238 return error.InvalidObject;1131 return error.InvalidObject;
1239 }1132 }
1240
1241 const oid = entry_hashed_writer.hasher.finalResult();1133 const oid = entry_hashed_writer.hasher.finalResult();
1134 pack_buffered_reader.putBack(entry_decompress_stream.unreadBytes());
1135 try pack_buffered_reader.flush();
1242 try index_entries.put(allocator, oid, .{1136 try index_entries.put(allocator, oid, .{
1243 .offset = entry_offset,1137 .offset = entry_offset,
1244 .crc32 = entry_crc32.final(),1138 .crc32 = entry_crc32_writer.hasher.final(),
1245 });1139 });
1246 },1140 },
1247 inline .ofs_delta, .ref_delta => |delta| {1141 inline .ofs_delta, .ref_delta => |delta| {
1248 var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader);1142 var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader);
1249 var entry_data_size: usize = 0;1143 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
12501144 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1251 while (try entry_decompress_stream.next()) |decompressed_data| {1145 try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1252 entry_data_size += decompressed_data.len;1146 if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1253
1254 const compressed_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes());
1255 pack_position += compressed_bytes.len;
1256 pack_hash.update(compressed_bytes);
1257 entry_crc32.update(compressed_bytes);
1258 pack_accumulating_reader.discard(compressed_bytes.len);
1259 }
1260 const footer_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes());
1261 pack_position += footer_bytes.len;
1262 pack_hash.update(footer_bytes);
1263 entry_crc32.update(footer_bytes);
1264 pack_accumulating_reader.discard(footer_bytes.len);
1265 pack_accumulating_reader.rewind();
1266
1267 if (entry_data_size != delta.uncompressed_length) {
1268 return error.InvalidObject;1147 return error.InvalidObject;
1269 }1148 }
12701149 pack_buffered_reader.putBack(entry_decompress_stream.unreadBytes());
1150 try pack_buffered_reader.flush();
1271 try pending_deltas.append(allocator, .{1151 try pending_deltas.append(allocator, .{
1272 .offset = entry_offset,1152 .offset = entry_offset,
1273 .crc32 = entry_crc32.final(),1153 .crc32 = entry_crc32_writer.hasher.final(),
1274 });1154 });
1275 },1155 },
1276 }1156 }
1277 }1157 }
12781158
1279 const pack_checksum = pack_hash.finalResult();1159 const pack_checksum = pack_hashed_writer.hasher.finalResult();
1280 const recorded_checksum = try pack_reader.readBytesNoEof(Sha1.digest_length);1160 const recorded_checksum = try pack_reader.readBytesNoEof(Sha1.digest_length);
1281 if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {1161 if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {
1282 return error.CorruptedPack;1162 return error.CorruptedPack;