| ... | @@ -66,6 +66,33 @@ pub const Oid = union(Format) { | ... | @@ -66,6 +66,33 @@ pub const Oid = union(Format) { |
| 66 | } | 66 | } |
| 67 | }; | 67 | }; |
| 68 | | 68 | |
| | 69 | const Hashing = union(Format) { |
| | 70 | sha1: std.Io.Writer.Hashing(Sha1), |
| | 71 | sha256: std.Io.Writer.Hashing(Sha256), |
| | 72 | |
| | 73 | fn init(oid_format: Format, buffer: []u8) Hashing { |
| | 74 | return switch (oid_format) { |
| | 75 | .sha1 => .{ .sha1 = .init(buffer) }, |
| | 76 | .sha256 => .{ .sha256 = .init(buffer) }, |
| | 77 | }; |
| | 78 | } |
| | 79 | |
| | 80 | fn writer(h: *@This()) *std.Io.Writer { |
| | 81 | return switch (h.*) { |
| | 82 | inline else => |*inner| &inner.writer, |
| | 83 | }; |
| | 84 | } |
| | 85 | |
| | 86 | fn final(h: *@This()) Oid { |
| | 87 | switch (h.*) { |
| | 88 | inline else => |*inner, tag| { |
| | 89 | inner.writer.flush() catch unreachable; // hashers cannot fail |
| | 90 | return @unionInit(Oid, @tagName(tag), inner.hasher.finalResult()); |
| | 91 | }, |
| | 92 | } |
| | 93 | } |
| | 94 | }; |
| | 95 | |
| 69 | pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { | 96 | pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { |
| 70 | assert(bytes.len == oid_format.byteLength()); | 97 | assert(bytes.len == oid_format.byteLength()); |
| 71 | return switch (oid_format) { | 98 | return switch (oid_format) { |
| ... | @@ -73,9 +100,9 @@ pub const Oid = union(Format) { | ... | @@ -73,9 +100,9 @@ pub const Oid = union(Format) { |
| 73 | }; | 100 | }; |
| 74 | } | 101 | } |
| 75 | | 102 | |
| 76 | pub fn readBytes(oid_format: Format, reader: anytype) !Oid { | 103 | pub fn readBytes(oid_format: Format, reader: *std.Io.Reader) !Oid { |
| 77 | return switch (oid_format) { | 104 | return switch (oid_format) { |
| 78 | inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), | 105 | inline else => |tag| @unionInit(Oid, @tagName(tag), (try reader.takeArray(tag.byteLength())).*), |
| 79 | }; | 106 | }; |
| 80 | } | 107 | } |
| 81 | | 108 | |
| ... | @@ -166,8 +193,15 @@ pub const Diagnostics = struct { | ... | @@ -166,8 +193,15 @@ pub const Diagnostics = struct { |
| 166 | pub const Repository = struct { | 193 | pub const Repository = struct { |
| 167 | odb: Odb, | 194 | odb: Odb, |
| 168 | | 195 | |
| 169 | pub fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Repository { | 196 | pub fn init( |
| 170 | return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; | 197 | repo: *Repository, |
| | 198 | allocator: Allocator, |
| | 199 | format: Oid.Format, |
| | 200 | pack_file: *std.fs.File.Reader, |
| | 201 | index_file: *std.fs.File.Reader, |
| | 202 | ) !void { |
| | 203 | repo.* = .{ .odb = undefined }; |
| | 204 | try repo.odb.init(allocator, format, pack_file, index_file); |
| 171 | } | 205 | } |
| 172 | | 206 | |
| 173 | pub fn deinit(repository: *Repository) void { | 207 | pub fn deinit(repository: *Repository) void { |
| ... | @@ -337,22 +371,28 @@ const Odb = struct { | ... | @@ -337,22 +371,28 @@ const Odb = struct { |
| 337 | format: Oid.Format, | 371 | format: Oid.Format, |
| 338 | pack_file: *std.fs.File.Reader, | 372 | pack_file: *std.fs.File.Reader, |
| 339 | index_header: IndexHeader, | 373 | index_header: IndexHeader, |
| 340 | index_file: std.fs.File, | 374 | index_file: *std.fs.File.Reader, |
| 341 | cache: ObjectCache = .{}, | 375 | cache: ObjectCache = .{}, |
| 342 | allocator: Allocator, | 376 | allocator: Allocator, |
| 343 | | 377 | |
| 344 | /// Initializes the database from open pack and index files. | 378 | /// Initializes the database from open pack and index files. |
| 345 | fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Odb { | 379 | fn init( |
| | 380 | odb: *Odb, |
| | 381 | allocator: Allocator, |
| | 382 | format: Oid.Format, |
| | 383 | pack_file: *std.fs.File.Reader, |
| | 384 | index_file: *std.fs.File.Reader, |
| | 385 | ) !void { |
| 346 | try pack_file.seekTo(0); | 386 | try pack_file.seekTo(0); |
| 347 | try index_file.seekTo(0); | 387 | try index_file.seekTo(0); |
| 348 | const index_header = try IndexHeader.read(index_file.deprecatedReader()); | 388 | odb.* = .{ |
| 349 | return .{ | | |
| 350 | .format = format, | 389 | .format = format, |
| 351 | .pack_file = pack_file, | 390 | .pack_file = pack_file, |
| 352 | .index_header = index_header, | 391 | .index_header = undefined, |
| 353 | .index_file = index_file, | 392 | .index_file = index_file, |
| 354 | .allocator = allocator, | 393 | .allocator = allocator, |
| 355 | }; | 394 | }; |
| | 395 | try odb.index_header.read(&index_file.interface); |
| 356 | } | 396 | } |
| 357 | | 397 | |
| 358 | fn deinit(odb: *Odb) void { | 398 | fn deinit(odb: *Odb) void { |
| ... | @@ -369,7 +409,7 @@ const Odb = struct { | ... | @@ -369,7 +409,7 @@ const Odb = struct { |
| 369 | const base_object = while (true) { | 409 | const base_object = while (true) { |
| 370 | if (odb.cache.get(base_offset)) |base_object| break base_object; | 410 | if (odb.cache.get(base_offset)) |base_object| break base_object; |
| 371 | | 411 | |
| 372 | base_header = try EntryHeader.read(odb.format, odb.pack_file.interface.adaptToOldInterface()); | 412 | base_header = try EntryHeader.read(odb.format, &odb.pack_file.interface); |
| 373 | switch (base_header) { | 413 | switch (base_header) { |
| 374 | .ofs_delta => |ofs_delta| { | 414 | .ofs_delta => |ofs_delta| { |
| 375 | try delta_offsets.append(odb.allocator, base_offset); | 415 | try delta_offsets.append(odb.allocator, base_offset); |
| ... | @@ -412,7 +452,7 @@ const Odb = struct { | ... | @@ -412,7 +452,7 @@ const Odb = struct { |
| 412 | const found_index = while (start_index < end_index) { | 452 | const found_index = while (start_index < end_index) { |
| 413 | const mid_index = start_index + (end_index - start_index) / 2; | 453 | const mid_index = start_index + (end_index - start_index) / 2; |
| 414 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); | 454 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); |
| 415 | const mid_oid = try Oid.readBytes(odb.format, odb.index_file.deprecatedReader()); | 455 | const mid_oid = try Oid.readBytes(odb.format, &odb.index_file.interface); |
| 416 | switch (mem.order(u8, mid_oid.slice(), oid.slice())) { | 456 | switch (mem.order(u8, mid_oid.slice(), oid.slice())) { |
| 417 | .lt => start_index = mid_index + 1, | 457 | .lt => start_index = mid_index + 1, |
| 418 | .gt => end_index = mid_index, | 458 | .gt => end_index = mid_index, |
| ... | @@ -423,12 +463,12 @@ const Odb = struct { | ... | @@ -423,12 +463,12 @@ const Odb = struct { |
| 423 | const n_objects = odb.index_header.fan_out_table[255]; | 463 | const n_objects = odb.index_header.fan_out_table[255]; |
| 424 | const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); | 464 | const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); |
| 425 | try odb.index_file.seekTo(offset_values_start + found_index * 4); | 465 | try odb.index_file.seekTo(offset_values_start + found_index * 4); |
| 426 | const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.deprecatedReader().readInt(u32, .big)); | 466 | const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.interface.takeInt(u32, .big)); |
| 427 | const pack_offset = pack_offset: { | 467 | const pack_offset = pack_offset: { |
| 428 | if (l1_offset.big) { | 468 | if (l1_offset.big) { |
| 429 | const l2_offset_values_start = offset_values_start + n_objects * 4; | 469 | const l2_offset_values_start = offset_values_start + n_objects * 4; |
| 430 | try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); | 470 | try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); |
| 431 | break :pack_offset try odb.index_file.deprecatedReader().readInt(u64, .big); | 471 | break :pack_offset try odb.index_file.interface.takeInt(u64, .big); |
| 432 | } else { | 472 | } else { |
| 433 | break :pack_offset l1_offset.value; | 473 | break :pack_offset l1_offset.value; |
| 434 | } | 474 | } |
| ... | @@ -1080,18 +1120,18 @@ const PackHeader = struct { | ... | @@ -1080,18 +1120,18 @@ const PackHeader = struct { |
| 1080 | const signature = "PACK"; | 1120 | const signature = "PACK"; |
| 1081 | const supported_version = 2; | 1121 | const supported_version = 2; |
| 1082 | | 1122 | |
| 1083 | fn read(reader: anytype) !PackHeader { | 1123 | fn read(reader: *std.Io.Reader) !PackHeader { |
| 1084 | const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { | 1124 | const actual_signature = reader.take(4) catch |e| switch (e) { |
| 1085 | error.EndOfStream => return error.InvalidHeader, | 1125 | error.EndOfStream => return error.InvalidHeader, |
| 1086 | else => |other| return other, | 1126 | else => |other| return other, |
| 1087 | }; | 1127 | }; |
| 1088 | if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; | 1128 | if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; |
| 1089 | const version = reader.readInt(u32, .big) catch |e| switch (e) { | 1129 | const version = reader.takeInt(u32, .big) catch |e| switch (e) { |
| 1090 | error.EndOfStream => return error.InvalidHeader, | 1130 | error.EndOfStream => return error.InvalidHeader, |
| 1091 | else => |other| return other, | 1131 | else => |other| return other, |
| 1092 | }; | 1132 | }; |
| 1093 | if (version != supported_version) return error.UnsupportedVersion; | 1133 | if (version != supported_version) return error.UnsupportedVersion; |
| 1094 | const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { | 1134 | const total_objects = reader.takeInt(u32, .big) catch |e| switch (e) { |
| 1095 | error.EndOfStream => return error.InvalidHeader, | 1135 | error.EndOfStream => return error.InvalidHeader, |
| 1096 | else => |other| return other, | 1136 | else => |other| return other, |
| 1097 | }; | 1137 | }; |
| ... | @@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) { | ... | @@ -1143,13 +1183,13 @@ const EntryHeader = union(Type) { |
| 1143 | }; | 1183 | }; |
| 1144 | } | 1184 | } |
| 1145 | | 1185 | |
| 1146 | fn read(format: Oid.Format, reader: anytype) !EntryHeader { | 1186 | fn read(format: Oid.Format, reader: *std.Io.Reader) !EntryHeader { |
| 1147 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; | 1187 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; |
| 1148 | const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { | 1188 | const initial: InitialByte = @bitCast(reader.takeByte() catch |e| switch (e) { |
| 1149 | error.EndOfStream => return error.InvalidFormat, | 1189 | error.EndOfStream => return error.InvalidFormat, |
| 1150 | else => |other| return other, | 1190 | else => |other| return other, |
| 1151 | }); | 1191 | }); |
| 1152 | const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; | 1192 | const rest_len = if (initial.has_next) try reader.takeLeb128(u64) else 0; |
| 1153 | var uncompressed_length: u64 = initial.len; | 1193 | var uncompressed_length: u64 = initial.len; |
| 1154 | uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; | 1194 | uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; |
| 1155 | const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; | 1195 | const @"type" = std.enums.fromInt(EntryHeader.Type, initial.type) orelse return error.InvalidFormat; |
| ... | @@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) { | ... | @@ -1172,25 +1212,12 @@ const EntryHeader = union(Type) { |
| 1172 | } | 1212 | } |
| 1173 | }; | 1213 | }; |
| 1174 | | 1214 | |
| 1175 | fn readSizeVarInt(r: anytype) !u64 { | 1215 | fn readOffsetVarInt(r: *std.Io.Reader) !u64 { |
| 1176 | const Byte = packed struct { value: u7, has_next: bool }; | | |
| 1177 | var b: Byte = @bitCast(try r.readByte()); | | |
| 1178 | var value: u64 = b.value; | | |
| 1179 | var shift: u6 = 0; | | |
| 1180 | while (b.has_next) { | | |
| 1181 | b = @bitCast(try r.readByte()); | | |
| 1182 | shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; | | |
| 1183 | value |= @as(u64, b.value) << shift; | | |
| 1184 | } | | |
| 1185 | return value; | | |
| 1186 | } | | |
| 1187 | | | |
| 1188 | fn readOffsetVarInt(r: anytype) !u64 { | | |
| 1189 | const Byte = packed struct { value: u7, has_next: bool }; | 1216 | const Byte = packed struct { value: u7, has_next: bool }; |
| 1190 | var b: Byte = @bitCast(try r.readByte()); | 1217 | var b: Byte = @bitCast(try r.takeByte()); |
| 1191 | var value: u64 = b.value; | 1218 | var value: u64 = b.value; |
| 1192 | while (b.has_next) { | 1219 | while (b.has_next) { |
| 1193 | b = @bitCast(try r.readByte()); | 1220 | b = @bitCast(try r.takeByte()); |
| 1194 | value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; | 1221 | value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; |
| 1195 | value |= b.value; | 1222 | value |= b.value; |
| 1196 | } | 1223 | } |
| ... | @@ -1204,19 +1231,12 @@ const IndexHeader = struct { | ... | @@ -1204,19 +1231,12 @@ const IndexHeader = struct { |
| 1204 | const supported_version = 2; | 1231 | const supported_version = 2; |
| 1205 | const size = 4 + 4 + @sizeOf([256]u32); | 1232 | const size = 4 + 4 + @sizeOf([256]u32); |
| 1206 | | 1233 | |
| 1207 | fn read(reader: anytype) !IndexHeader { | 1234 | fn read(index_header: *IndexHeader, reader: *std.Io.Reader) !void { |
| 1208 | var header_bytes = try reader.readBytesNoEof(size); | 1235 | const sig = try reader.take(4); |
| 1209 | if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; | 1236 | if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; |
| 1210 | const version = mem.readInt(u32, header_bytes[4..8], .big); | 1237 | const version = try reader.takeInt(u32, .big); |
| 1211 | if (version != supported_version) return error.UnsupportedVersion; | 1238 | if (version != supported_version) return error.UnsupportedVersion; |
| 1212 | | 1239 | try reader.readSliceEndian(u32, &index_header.fan_out_table, .big); |
| 1213 | var fan_out_table: [256]u32 = undefined; | | |
| 1214 | var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); | | |
| 1215 | const fan_out_table_reader = fan_out_table_stream.reader(); | | |
| 1216 | for (&fan_out_table) |*entry| { | | |
| 1217 | entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; | | |
| 1218 | } | | |
| 1219 | return .{ .fan_out_table = fan_out_table }; | | |
| 1220 | } | 1240 | } |
| 1221 | }; | 1241 | }; |
| 1222 | | 1242 | |
| ... | @@ -1227,7 +1247,12 @@ const IndexEntry = struct { | ... | @@ -1227,7 +1247,12 @@ const IndexEntry = struct { |
| 1227 | | 1247 | |
| 1228 | /// Writes out a version 2 index for the given packfile, as documented in | 1248 | /// Writes out a version 2 index for the given packfile, as documented in |
| 1229 | /// [pack-format](https://git-scm.com/docs/pack-format). | 1249 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 1230 | pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Reader, index_writer: anytype) !void { | 1250 | pub fn indexPack( |
| | 1251 | allocator: Allocator, |
| | 1252 | format: Oid.Format, |
| | 1253 | pack: *std.fs.File.Reader, |
| | 1254 | index_writer: *std.fs.File.Writer, |
| | 1255 | ) !void { |
| 1231 | try pack.seekTo(0); | 1256 | try pack.seekTo(0); |
| 1232 | | 1257 | |
| 1233 | var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; | 1258 | var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; |
| ... | @@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re | ... | @@ -1280,8 +1305,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re |
| 1280 | } | 1305 | } |
| 1281 | @memset(fan_out_table[fan_out_index..], count); | 1306 | @memset(fan_out_table[fan_out_index..], count); |
| 1282 | | 1307 | |
| 1283 | var index_hashed_writer = hashedWriter(index_writer, Oid.Hasher.init(format)); | 1308 | var index_hashed_writer = std.Io.Writer.hashed(&index_writer.interface, Oid.Hasher.init(format), &.{}); |
| 1284 | const writer = index_hashed_writer.writer(); | 1309 | const writer = &index_hashed_writer.writer; |
| 1285 | try writer.writeAll(IndexHeader.signature); | 1310 | try writer.writeAll(IndexHeader.signature); |
| 1286 | try writer.writeInt(u32, IndexHeader.supported_version, .big); | 1311 | try writer.writeInt(u32, IndexHeader.supported_version, .big); |
| 1287 | for (fan_out_table) |fan_out_entry| { | 1312 | for (fan_out_table) |fan_out_entry| { |
| ... | @@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re | ... | @@ -1314,7 +1339,8 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Re |
| 1314 | | 1339 | |
| 1315 | try writer.writeAll(pack_checksum.slice()); | 1340 | try writer.writeAll(pack_checksum.slice()); |
| 1316 | const index_checksum = index_hashed_writer.hasher.finalResult(); | 1341 | const index_checksum = index_hashed_writer.hasher.finalResult(); |
| 1317 | try index_writer.writeAll(index_checksum.slice()); | 1342 | try index_writer.interface.writeAll(index_checksum.slice()); |
| | 1343 | try index_writer.end(); |
| 1318 | } | 1344 | } |
| 1319 | | 1345 | |
| 1320 | /// Performs the first pass over the packfile data for index construction. | 1346 | /// Performs the first pass over the packfile data for index construction. |
| ... | @@ -1328,65 +1354,51 @@ fn indexPackFirstPass( | ... | @@ -1328,65 +1354,51 @@ fn indexPackFirstPass( |
| 1328 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), | 1354 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| 1329 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), | 1355 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), |
| 1330 | ) !Oid { | 1356 | ) !Oid { |
| 1331 | var pack_counting_reader = std.io.countingReader(pack.interface.adaptToOldInterface()); | 1357 | var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; |
| 1332 | var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); | 1358 | var entry_buffer: [1024]u8 = undefined; // Input buffer to flate. |
| 1333 | const pack_reader = pack_hashed_reader.reader(); | 1359 | var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. |
| 1334 | | 1360 | var hasher_buffer: [64]u8 = undefined; |
| 1335 | const pack_header = try PackHeader.read(pack_reader); | 1361 | var pack_hashed = pack.interface.hashed(Oid.Hasher.init(format), &pack_buffer); |
| 1336 | | 1362 | |
| 1337 | var current_entry: u32 = 0; | 1363 | const pack_header = try PackHeader.read(&pack_hashed.reader); |
| 1338 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { | 1364 | |
| 1339 | const entry_offset = pack_counting_reader.bytes_read; | 1365 | for (0..pack_header.total_objects) |_| { |
| 1340 | var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init()); | 1366 | const entry_offset = pack.logicalPos(); |
| 1341 | const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); | 1367 | var entry_crc32_stream = pack_hashed.reader.hashed(std.hash.Crc32.init(), &entry_buffer); |
| 1342 | var adapter_buffer: [1024]u8 = undefined; | 1368 | const entry_header = try EntryHeader.read(format, &entry_crc32_stream.reader); |
| 1343 | var adapter = entry_crc32_reader.reader().adaptToNewApi(&adapter_buffer); | 1369 | var entry_decompress: std.compress.flate.Decompress = .init(&entry_crc32_stream.reader, .zlib, &flate_buffer); |
| 1344 | var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined; | | |
| 1345 | var entry_decompress_stream: std.compress.flate.Decompress = .init(&adapter.new_interface, .zlib, &flate_buffer); | | |
| 1346 | const old = entry_decompress_stream.reader.adaptToOldInterface(); | | |
| 1347 | var entry_counting_reader = std.io.countingReader(old); | | |
| 1348 | switch (entry_header) { | 1370 | switch (entry_header) { |
| 1349 | .commit, .tree, .blob, .tag => |object| { | 1371 | .commit, .tree, .blob, .tag => |object| { |
| 1350 | var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); | 1372 | var oid_hasher: Oid.Hashing = .init(format, &hasher_buffer); |
| 1351 | const entry_writer = entry_hashed_writer.writer(); | 1373 | const oid_hasher_w = oid_hasher.writer(); |
| 1352 | // The object header is not included in the pack data but is | 1374 | // The object header is not included in the pack data but is |
| 1353 | // part of the object's ID | 1375 | // part of the object's ID |
| 1354 | try entry_writer.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length }); | 1376 | try oid_hasher_w.print("{t} {d}\x00", .{ entry_header, object.uncompressed_length }); |
| 1355 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | 1377 | const n = try entry_decompress.reader.streamRemaining(oid_hasher_w); |
| 1356 | try fifo.pump(entry_counting_reader.reader(), entry_writer); | 1378 | if (n != object.uncompressed_length) return error.InvalidObject; |
| 1357 | if (entry_counting_reader.bytes_read != object.uncompressed_length) { | 1379 | const oid = oid_hasher.final(); |
| 1358 | return error.InvalidObject; | | |
| 1359 | } | | |
| 1360 | const oid = entry_hashed_writer.hasher.finalResult(); | | |
| 1361 | try index_entries.put(allocator, oid, .{ | 1380 | try index_entries.put(allocator, oid, .{ |
| 1362 | .offset = entry_offset, | 1381 | .offset = entry_offset, |
| 1363 | .crc32 = entry_crc32_reader.hasher.final(), | 1382 | .crc32 = entry_crc32_stream.hasher.final(), |
| 1364 | }); | 1383 | }); |
| 1365 | }, | 1384 | }, |
| 1366 | inline .ofs_delta, .ref_delta => |delta| { | 1385 | inline .ofs_delta, .ref_delta => |delta| { |
| 1367 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | 1386 | const n = try entry_decompress.reader.discardRemaining(); |
| 1368 | try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); | 1387 | if (n != delta.uncompressed_length) return error.InvalidObject; |
| 1369 | if (entry_counting_reader.bytes_read != delta.uncompressed_length) { | | |
| 1370 | return error.InvalidObject; | | |
| 1371 | } | | |
| 1372 | try pending_deltas.append(allocator, .{ | 1388 | try pending_deltas.append(allocator, .{ |
| 1373 | .offset = entry_offset, | 1389 | .offset = entry_offset, |
| 1374 | .crc32 = entry_crc32_reader.hasher.final(), | 1390 | .crc32 = entry_crc32_stream.hasher.final(), |
| 1375 | }); | 1391 | }); |
| 1376 | }, | 1392 | }, |
| 1377 | } | 1393 | } |
| 1378 | } | 1394 | } |
| 1379 | | 1395 | |
| 1380 | const pack_checksum = pack_hashed_reader.hasher.finalResult(); | 1396 | const pack_checksum = pack_hashed.hasher.finalResult(); |
| 1381 | const recorded_checksum = try Oid.readBytes(format, pack.interface.adaptToOldInterface()); | 1397 | const recorded_checksum = try Oid.readBytes(format, &pack.interface); |
| 1382 | if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { | 1398 | if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { |
| 1383 | return error.CorruptedPack; | 1399 | return error.CorruptedPack; |
| 1384 | } | 1400 | } |
| 1385 | _ = pack_reader.readByte() catch |e| switch (e) { | 1401 | return pack_checksum; |
| 1386 | error.EndOfStream => return pack_checksum, | | |
| 1387 | else => |other| return other, | | |
| 1388 | }; | | |
| 1389 | return error.InvalidFormat; | | |
| 1390 | } | 1402 | } |
| 1391 | | 1403 | |
| 1392 | /// Attempts to determine the final object ID of the given deltified object. | 1404 | /// Attempts to determine the final object ID of the given deltified object. |
| ... | @@ -1409,7 +1421,7 @@ fn indexPackHashDelta( | ... | @@ -1409,7 +1421,7 @@ fn indexPackHashDelta( |
| 1409 | if (cache.get(base_offset)) |base_object| break base_object; | 1421 | if (cache.get(base_offset)) |base_object| break base_object; |
| 1410 | | 1422 | |
| 1411 | try pack.seekTo(base_offset); | 1423 | try pack.seekTo(base_offset); |
| 1412 | base_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); | 1424 | base_header = try EntryHeader.read(format, &pack.interface); |
| 1413 | switch (base_header) { | 1425 | switch (base_header) { |
| 1414 | .ofs_delta => |ofs_delta| { | 1426 | .ofs_delta => |ofs_delta| { |
| 1415 | try delta_offsets.append(allocator, base_offset); | 1427 | try delta_offsets.append(allocator, base_offset); |
| ... | @@ -1431,11 +1443,13 @@ fn indexPackHashDelta( | ... | @@ -1431,11 +1443,13 @@ fn indexPackHashDelta( |
| 1431 | | 1443 | |
| 1432 | const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); | 1444 | const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); |
| 1433 | | 1445 | |
| 1434 | var entry_hasher: Oid.Hasher = .init(format); | 1446 | var entry_hasher_buffer: [64]u8 = undefined; |
| 1435 | var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); | 1447 | var entry_hasher: Oid.Hashing = .init(format, &entry_hasher_buffer); |
| 1436 | try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); | 1448 | const entry_hasher_w = entry_hasher.writer(); |
| 1437 | entry_hasher.update(base_data); | 1449 | // Writes to hashers cannot fail. |
| 1438 | return entry_hasher.finalResult(); | 1450 | entry_hasher_w.print("{t} {d}\x00", .{ base_object.type, base_data.len }) catch unreachable; |
| | 1451 | entry_hasher_w.writeAll(base_data) catch unreachable; |
| | 1452 | return entry_hasher.final(); |
| 1439 | } | 1453 | } |
| 1440 | | 1454 | |
| 1441 | /// Resolves a chain of deltas, returning the final base object data. `pack` is | 1455 | /// Resolves a chain of deltas, returning the final base object data. `pack` is |
| ... | @@ -1457,21 +1471,19 @@ fn resolveDeltaChain( | ... | @@ -1457,21 +1471,19 @@ fn resolveDeltaChain( |
| 1457 | | 1471 | |
| 1458 | const delta_offset = delta_offsets[i]; | 1472 | const delta_offset = delta_offsets[i]; |
| 1459 | try pack.seekTo(delta_offset); | 1473 | try pack.seekTo(delta_offset); |
| 1460 | const delta_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface()); | 1474 | const delta_header = try EntryHeader.read(format, &pack.interface); |
| 1461 | const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); | 1475 | const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength()); |
| 1462 | defer allocator.free(delta_data); | 1476 | defer allocator.free(delta_data); |
| 1463 | var delta_stream = std.io.fixedBufferStream(delta_data); | 1477 | var delta_reader: std.Io.Reader = .fixed(delta_data); |
| 1464 | const delta_reader = delta_stream.reader(); | 1478 | _ = try delta_reader.takeLeb128(u64); // base object size |
| 1465 | _ = try readSizeVarInt(delta_reader); // base object size | 1479 | const expanded_size = try delta_reader.takeLeb128(u64); |
| 1466 | const expanded_size = try readSizeVarInt(delta_reader); | | |
| 1467 | | 1480 | |
| 1468 | const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; | 1481 | const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; |
| 1469 | const expanded_data = try allocator.alloc(u8, expanded_alloc_size); | 1482 | const expanded_data = try allocator.alloc(u8, expanded_alloc_size); |
| 1470 | errdefer allocator.free(expanded_data); | 1483 | errdefer allocator.free(expanded_data); |
| 1471 | var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); | 1484 | var expanded_delta_stream: std.Io.Writer = .fixed(expanded_data); |
| 1472 | var base_stream = std.io.fixedBufferStream(base_data); | 1485 | try expandDelta(base_data, &delta_reader, &expanded_delta_stream); |
| 1473 | try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); | 1486 | if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; |
| 1474 | if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; | | |
| 1475 | | 1487 | |
| 1476 | try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); | 1488 | try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); |
| 1477 | base_data = expanded_data; | 1489 | base_data = expanded_data; |
| ... | @@ -1497,9 +1509,10 @@ fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 | ... | @@ -1497,9 +1509,10 @@ fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 |
| 1497 | /// | 1509 | /// |
| 1498 | /// The format of the delta data is documented in | 1510 | /// The format of the delta data is documented in |
| 1499 | /// [pack-format](https://git-scm.com/docs/pack-format). | 1511 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 1500 | fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { | 1512 | fn expandDelta(base_object: []const u8, delta_reader: *std.Io.Reader, writer: *std.Io.Writer) !void { |
| | 1513 | var base_offset: u32 = 0; |
| 1501 | while (true) { | 1514 | while (true) { |
| 1502 | const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { | 1515 | const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { |
| 1503 | error.EndOfStream => return, | 1516 | error.EndOfStream => return, |
| 1504 | else => |other| return other, | 1517 | else => |other| return other, |
| 1505 | }); | 1518 | }); |
| ... | @@ -1514,27 +1527,23 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo | ... | @@ -1514,27 +1527,23 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo |
| 1514 | size3: bool, | 1527 | size3: bool, |
| 1515 | } = @bitCast(inst.value); | 1528 | } = @bitCast(inst.value); |
| 1516 | const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ | 1529 | const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ |
| 1517 | .offset1 = if (available.offset1) try delta_reader.readByte() else 0, | 1530 | .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, |
| 1518 | .offset2 = if (available.offset2) try delta_reader.readByte() else 0, | 1531 | .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, |
| 1519 | .offset3 = if (available.offset3) try delta_reader.readByte() else 0, | 1532 | .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, |
| 1520 | .offset4 = if (available.offset4) try delta_reader.readByte() else 0, | 1533 | .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, |
| 1521 | }; | 1534 | }; |
| 1522 | const offset: u32 = @bitCast(offset_parts); | 1535 | base_offset = @bitCast(offset_parts); |
| 1523 | const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ | 1536 | const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ |
| 1524 | .size1 = if (available.size1) try delta_reader.readByte() else 0, | 1537 | .size1 = if (available.size1) try delta_reader.takeByte() else 0, |
| 1525 | .size2 = if (available.size2) try delta_reader.readByte() else 0, | 1538 | .size2 = if (available.size2) try delta_reader.takeByte() else 0, |
| 1526 | .size3 = if (available.size3) try delta_reader.readByte() else 0, | 1539 | .size3 = if (available.size3) try delta_reader.takeByte() else 0, |
| 1527 | }; | 1540 | }; |
| 1528 | var size: u24 = @bitCast(size_parts); | 1541 | var size: u24 = @bitCast(size_parts); |
| 1529 | if (size == 0) size = 0x10000; | 1542 | if (size == 0) size = 0x10000; |
| 1530 | try base_object.seekTo(offset); | 1543 | try writer.writeAll(base_object[base_offset..][0..size]); |
| 1531 | var copy_reader = std.io.limitedReader(base_object.reader(), size); | 1544 | base_offset += size; |
| 1532 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | | |
| 1533 | try fifo.pump(copy_reader.reader(), writer); | | |
| 1534 | } else if (inst.value != 0) { | 1545 | } else if (inst.value != 0) { |
| 1535 | var data_reader = std.io.limitedReader(delta_reader, inst.value); | 1546 | try delta_reader.streamExact(writer, inst.value); |
| 1536 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); | | |
| 1537 | try fifo.pump(data_reader.reader(), writer); | | |
| 1538 | } else { | 1547 | } else { |
| 1539 | return error.InvalidDeltaInstruction; | 1548 | return error.InvalidDeltaInstruction; |
| 1540 | } | 1549 | } |
| ... | @@ -1564,12 +1573,14 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void | ... | @@ -1564,12 +1573,14 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void |
| 1564 | defer pack_file.close(); | 1573 | defer pack_file.close(); |
| 1565 | try pack_file.writeAll(testrepo_pack); | 1574 | try pack_file.writeAll(testrepo_pack); |
| 1566 | | 1575 | |
| 1567 | var pack_file_buffer: [4096]u8 = undefined; | 1576 | var pack_file_buffer: [2000]u8 = undefined; |
| 1568 | var pack_file_reader = pack_file.reader(&pack_file_buffer); | 1577 | var pack_file_reader = pack_file.reader(&pack_file_buffer); |
| 1569 | | 1578 | |
| 1570 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); | 1579 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); |
| 1571 | defer index_file.close(); | 1580 | defer index_file.close(); |
| 1572 | try indexPack(testing.allocator, format, &pack_file_reader, index_file.deprecatedWriter()); | 1581 | var index_file_buffer: [2000]u8 = undefined; |
| | 1582 | var index_file_writer = index_file.writer(&index_file_buffer); |
| | 1583 | try indexPack(testing.allocator, format, &pack_file_reader, &index_file_writer); |
| 1573 | | 1584 | |
| 1574 | // Arbitrary size limit on files read while checking the repository contents | 1585 | // Arbitrary size limit on files read while checking the repository contents |
| 1575 | // (all files in the test repo are known to be smaller than this) | 1586 | // (all files in the test repo are known to be smaller than this) |
| ... | @@ -1583,7 +1594,9 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void | ... | @@ -1583,7 +1594,9 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void |
| 1583 | const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); | 1594 | const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); |
| 1584 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); | 1595 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); |
| 1585 | | 1596 | |
| 1586 | var repository = try Repository.init(testing.allocator, format, &pack_file_reader, index_file); | 1597 | var index_file_reader = index_file.reader(&index_file_buffer); |
| | 1598 | var repository: Repository = undefined; |
| | 1599 | try repository.init(testing.allocator, format, &pack_file_reader, &index_file_reader); |
| 1587 | defer repository.deinit(); | 1600 | defer repository.deinit(); |
| 1588 | | 1601 | |
| 1589 | var worktree = testing.tmpDir(.{ .iterate = true }); | 1602 | var worktree = testing.tmpDir(.{ .iterate = true }); |
| ... | @@ -1704,58 +1717,3 @@ pub fn main() !void { | ... | @@ -1704,58 +1717,3 @@ pub fn main() !void { |
| 1704 | std.debug.print("Diagnostic: {}\n", .{err}); | 1717 | std.debug.print("Diagnostic: {}\n", .{err}); |
| 1705 | } | 1718 | } |
| 1706 | } | 1719 | } |
| 1707 | | | |
| 1708 | /// Deprecated | | |
| 1709 | fn hashedReader(reader: anytype, hasher: anytype) HashedReader(@TypeOf(reader), @TypeOf(hasher)) { | | |
| 1710 | return .{ .child_reader = reader, .hasher = hasher }; | | |
| 1711 | } | | |
| 1712 | | | |
| 1713 | /// Deprecated | | |
| 1714 | fn HashedReader(ReaderType: type, HasherType: type) type { | | |
| 1715 | return struct { | | |
| 1716 | child_reader: ReaderType, | | |
| 1717 | hasher: HasherType, | | |
| 1718 | | | |
| 1719 | pub const Error = ReaderType.Error; | | |
| 1720 | pub const Reader = std.io.GenericReader(*@This(), Error, read); | | |
| 1721 | | | |
| 1722 | pub fn read(self: *@This(), buf: []u8) Error!usize { | | |
| 1723 | const amt = try self.child_reader.read(buf); | | |
| 1724 | self.hasher.update(buf[0..amt]); | | |
| 1725 | return amt; | | |
| 1726 | } | | |
| 1727 | | | |
| 1728 | pub fn reader(self: *@This()) Reader { | | |
| 1729 | return .{ .context = self }; | | |
| 1730 | } | | |
| 1731 | }; | | |
| 1732 | } | | |
| 1733 | | | |
| 1734 | /// Deprecated | | |
| 1735 | pub fn HashedWriter(WriterType: type, HasherType: type) type { | | |
| 1736 | return struct { | | |
| 1737 | child_writer: WriterType, | | |
| 1738 | hasher: HasherType, | | |
| 1739 | | | |
| 1740 | pub const Error = WriterType.Error; | | |
| 1741 | pub const Writer = std.io.GenericWriter(*@This(), Error, write); | | |
| 1742 | | | |
| 1743 | pub fn write(self: *@This(), buf: []const u8) Error!usize { | | |
| 1744 | const amt = try self.child_writer.write(buf); | | |
| 1745 | self.hasher.update(buf[0..amt]); | | |
| 1746 | return amt; | | |
| 1747 | } | | |
| 1748 | | | |
| 1749 | pub fn writer(self: *@This()) Writer { | | |
| 1750 | return .{ .context = self }; | | |
| 1751 | } | | |
| 1752 | }; | | |
| 1753 | } | | |
| 1754 | | | |
| 1755 | /// Deprecated | | |
| 1756 | pub fn hashedWriter( | | |
| 1757 | writer: anytype, | | |
| 1758 | hasher: anytype, | | |
| 1759 | ) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { | | |
| 1760 | return .{ .child_writer = writer, .hasher = hasher }; | | |
| 1761 | } | | |