authorgravatar for ian@ianjohnson.devIan Johnson <ian@ianjohnson.dev> 2024-11-20 00:47:00-05:00
committergravatar for ian@ianjohnson.devIan Johnson <ian@ianjohnson.dev> 2024-12-13 08:49:45-05:00
log5217da57fc95920be93dda02bc905baffe4ee89d
tree744bdf278eb81111d036cef39d500086a3f378ad
parent62cc81a63afcc1c4bb1863c22d247169a2ab8136

zig fetch: support SHA-256 Git repositories

Closes #21888

9 files changed, 250 insertions(+), 104 deletions(-)

src/Package/Fetch.zig+12-17
...@@ -814,7 +814,7 @@ const Resource = union(enum) {...@@ -814,7 +814,7 @@ const Resource = union(enum) {
814 const Git = struct {814 const Git = struct {
815 session: git.Session,815 session: git.Session,
816 fetch_stream: git.Session.FetchStream,816 fetch_stream: git.Session.FetchStream,
817 want_oid: [git.oid_length]u8,817 want_oid: git.Oid,
818 };818 };
819819
820 fn deinit(resource: *Resource) void {820 fn deinit(resource: *Resource) void {
...@@ -976,7 +976,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -976,7 +976,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
976 const want_oid = want_oid: {976 const want_oid = want_oid: {
977 const want_ref =977 const want_ref =
978 if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD";978 if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD";
979 if (git.parseOid(want_ref)) |oid| break :want_oid oid else |_| {}979 if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {}
980980
981 const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});981 const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
982 const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});982 const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
...@@ -1018,17 +1018,13 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1018,17 +1018,13 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
1018 });1018 });
1019 const notes_start = try eb.reserveNotes(notes_len);1019 const notes_start = try eb.reserveNotes(notes_len);
1020 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{1020 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
1021 .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{1021 .msg = try eb.printString("try .url = \"{;+/}#{}\",", .{ uri, want_oid }),
1022 uri, std.fmt.fmtSliceHexLower(&want_oid),
1023 }),
1024 }));1022 }));
1025 return error.FetchFailed;1023 return error.FetchFailed;
1026 }1024 }
10271025
1028 var want_oid_buf: [git.fmt_oid_length]u8 = undefined;1026 var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
1029 _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{1027 _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{want_oid}) catch unreachable;
1030 std.fmt.fmtSliceHexLower(&want_oid),
1031 }) catch unreachable;
1032 var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| {1028 var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| {
1033 return f.fail(f.location_tok, try eb.printString(1029 return f.fail(f.location_tok, try eb.printString(
1034 "unable to create fetch stream: {s}",1030 "unable to create fetch stream: {s}",
...@@ -1163,7 +1159,7 @@ fn unpackResource(...@@ -1163,7 +1159,7 @@ fn unpackResource(
1163 });1159 });
1164 return try unpackTarball(f, tmp_directory.handle, dcp.reader());1160 return try unpackTarball(f, tmp_directory.handle, dcp.reader());
1165 },1161 },
1166 .git_pack => return unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {1162 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
1167 error.FetchFailed => return error.FetchFailed,1163 error.FetchFailed => return error.FetchFailed,
1168 error.OutOfMemory => return error.OutOfMemory,1164 error.OutOfMemory => return error.OutOfMemory,
1169 else => |e| return f.fail(f.location_tok, try eb.printString(1165 else => |e| return f.fail(f.location_tok, try eb.printString(
...@@ -1298,11 +1294,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {...@@ -1298,11 +1294,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1298 return res;1294 return res;
1299}1295}
13001296
1301fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!UnpackResult {1297fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult {
1302 const arena = f.arena.allocator();1298 const arena = f.arena.allocator();
1303 const gpa = f.arena.child_allocator;1299 const gpa = f.arena.child_allocator;
1304 const want_oid = resource.git.want_oid;1300 const object_format: git.Oid.Format = resource.want_oid;
1305 const reader = resource.git.fetch_stream.reader();
13061301
1307 var res: UnpackResult = .{};1302 var res: UnpackResult = .{};
1308 // The .git directory is used to store the packfile and associated index, but1303 // The .git directory is used to store the packfile and associated index, but
...@@ -1314,7 +1309,7 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac...@@ -1314,7 +1309,7 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac
1314 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });1309 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1315 defer pack_file.close();1310 defer pack_file.close();
1316 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();1311 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1317 try fifo.pump(reader, pack_file.writer());1312 try fifo.pump(resource.fetch_stream.reader(), pack_file.writer());
1318 try pack_file.sync();1313 try pack_file.sync();
13191314
1320 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });1315 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
...@@ -1323,7 +1318,7 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac...@@ -1323,7 +1318,7 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac
1323 const index_prog_node = f.prog_node.start("Index pack", 0);1318 const index_prog_node = f.prog_node.start("Index pack", 0);
1324 defer index_prog_node.end();1319 defer index_prog_node.end();
1325 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());1320 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1326 try git.indexPack(gpa, pack_file, index_buffered_writer.writer());1321 try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer());
1327 try index_buffered_writer.flush();1322 try index_buffered_writer.flush();
1328 try index_file.sync();1323 try index_file.sync();
1329 }1324 }
...@@ -1331,10 +1326,10 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac...@@ -1331,10 +1326,10 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!Unpac
1331 {1326 {
1332 const checkout_prog_node = f.prog_node.start("Checkout", 0);1327 const checkout_prog_node = f.prog_node.start("Checkout", 0);
1333 defer checkout_prog_node.end();1328 defer checkout_prog_node.end();
1334 var repository = try git.Repository.init(gpa, pack_file, index_file);1329 var repository = try git.Repository.init(gpa, object_format, pack_file, index_file);
1335 defer repository.deinit();1330 defer repository.deinit();
1336 var diagnostics: git.Diagnostics = .{ .allocator = arena };1331 var diagnostics: git.Diagnostics = .{ .allocator = arena };
1337 try repository.checkout(out_dir, want_oid, &diagnostics);1332 try repository.checkout(out_dir, resource.want_oid, &diagnostics);
13381333
1339 if (diagnostics.errors.items.len > 0) {1334 if (diagnostics.errors.items.len > 0) {
1340 try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");1335 try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
src/Package/Fetch/git.zig+236-85
...@@ -9,32 +9,133 @@ const mem = std.mem;...@@ -9,32 +9,133 @@ const mem = std.mem;
9const testing = std.testing;9const testing = std.testing;
10const Allocator = mem.Allocator;10const Allocator = mem.Allocator;
11const Sha1 = std.crypto.hash.Sha1;11const Sha1 = std.crypto.hash.Sha1;
12const Sha256 = std.crypto.hash.sha2.Sha256;
12const assert = std.debug.assert;13const assert = std.debug.assert;
1314
14pub const oid_length = Sha1.digest_length;15/// The ID of a Git object.
15pub const fmt_oid_length = 2 * oid_length;16pub const Oid = union(Format) {
16/// The ID of a Git object (an SHA-1 hash).17 sha1: [Sha1.digest_length]u8,
17pub const Oid = [oid_length]u8;18 sha256: [Sha256.digest_length]u8,
1819
19pub fn parseOid(s: []const u8) !Oid {20 pub const max_formatted_length = len: {
20 if (s.len != fmt_oid_length) return error.InvalidOid;21 var max: usize = 0;
21 var oid: Oid = undefined;22 for (std.enums.values(Format)) |f| {
22 for (&oid, 0..) |*b, i| {23 max = @max(max, f.formattedLength());
23 b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;24 }
25 break :len max;
26 };
27
28 pub const Format = enum {
29 sha1,
30 sha256,
31
32 pub fn byteLength(f: Format) usize {
33 return switch (f) {
34 .sha1 => Sha1.digest_length,
35 .sha256 => Sha256.digest_length,
36 };
37 }
38
39 pub fn formattedLength(f: Format) usize {
40 return 2 * f.byteLength();
41 }
42 };
43
44 const Hasher = union(Format) {
45 sha1: Sha1,
46 sha256: Sha256,
47
48 fn init(oid_format: Format) Hasher {
49 return switch (oid_format) {
50 .sha1 => .{ .sha1 = Sha1.init(.{}) },
51 .sha256 => .{ .sha256 = Sha256.init(.{}) },
52 };
53 }
54
55 // Must be public for use from HashedReader and HashedWriter.
56 pub fn update(hasher: *Hasher, b: []const u8) void {
57 switch (hasher.*) {
58 inline else => |*inner| inner.update(b),
59 }
60 }
61
62 fn finalResult(hasher: *Hasher) Oid {
63 return switch (hasher.*) {
64 inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()),
65 };
66 }
67 };
68
69 pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
70 assert(bytes.len == oid_format.byteLength());
71 return switch (oid_format) {
72 inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*),
73 };
24 }74 }
25 return oid;
26}
2775
28test parseOid {76 pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid {
29 try testing.expectEqualSlices(77 return switch (oid_format) {
30 u8,78 inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),
31 &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },79 };
32 &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"),80 }
33 );81
34 try testing.expectError(error.InvalidOid, parseOid("ce919ccf"));82 pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid {
35 try testing.expectError(error.InvalidOid, parseOid("master"));83 switch (oid_format) {
36 try testing.expectError(error.InvalidOid, parseOid("HEAD"));84 inline else => |tag| {
37}85 if (s.len != tag.formattedLength()) return error.InvalidOid;
86 var bytes: [tag.byteLength()]u8 = undefined;
87 for (&bytes, 0..) |*b, i| {
88 b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
89 }
90 return @unionInit(Oid, @tagName(tag), bytes);
91 },
92 }
93 }
94
95 test parse {
96 try testing.expectEqualSlices(
97 u8,
98 &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
99 &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1,
100 );
101 try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588"));
102 try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"));
103 try testing.expectEqualSlices(
104 u8,
105 &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A },
106 &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256,
107 );
108 try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf"));
109 try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf"));
110 try testing.expectError(error.InvalidOid, parse(.sha1, "master"));
111 try testing.expectError(error.InvalidOid, parse(.sha256, "master"));
112 try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD"));
113 try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD"));
114 }
115
116 pub fn parseAny(s: []const u8) error{InvalidOid}!Oid {
117 return for (std.enums.values(Format)) |f| {
118 if (s.len == f.formattedLength()) break parse(f, s);
119 } else error.InvalidOid;
120 }
121
122 pub fn format(
123 oid: Oid,
124 comptime fmt: []const u8,
125 options: std.fmt.FormatOptions,
126 writer: anytype,
127 ) @TypeOf(writer).Error!void {
128 _ = fmt;
129 _ = options;
130 try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())});
131 }
132
133 pub fn slice(oid: *const Oid) []const u8 {
134 return switch (oid.*) {
135 inline else => |*bytes| bytes,
136 };
137 }
138};
38139
39pub const Diagnostics = struct {140pub const Diagnostics = struct {
40 allocator: Allocator,141 allocator: Allocator,
...@@ -72,8 +173,8 @@ pub const Diagnostics = struct {...@@ -72,8 +173,8 @@ pub const Diagnostics = struct {
72pub const Repository = struct {173pub const Repository = struct {
73 odb: Odb,174 odb: Odb,
74175
75 pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository {176 pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
76 return .{ .odb = try Odb.init(allocator, pack_file, index_file) };177 return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };
77 }178 }
78179
79 pub fn deinit(repository: *Repository) void {180 pub fn deinit(repository: *Repository) void {
...@@ -92,7 +193,7 @@ pub const Repository = struct {...@@ -92,7 +193,7 @@ pub const Repository = struct {
92 const tree_oid = tree_oid: {193 const tree_oid = tree_oid: {
93 const commit_object = try repository.odb.readObject();194 const commit_object = try repository.odb.readObject();
94 if (commit_object.type != .commit) return error.NotACommit;195 if (commit_object.type != .commit) return error.NotACommit;
95 break :tree_oid try getCommitTree(commit_object.data);196 break :tree_oid try getCommitTree(repository.odb.format, commit_object.data);
96 };197 };
97 try repository.checkoutTree(worktree, tree_oid, "", diagnostics);198 try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
98 }199 }
...@@ -114,7 +215,11 @@ pub const Repository = struct {...@@ -114,7 +215,11 @@ pub const Repository = struct {
114 const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);215 const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
115 defer repository.odb.allocator.free(tree_data);216 defer repository.odb.allocator.free(tree_data);
116217
117 var tree_iter: TreeIterator = .{ .data = tree_data };218 var tree_iter: TreeIterator = .{
219 .format = repository.odb.format,
220 .data = tree_data,
221 .pos = 0,
222 };
118 while (try tree_iter.next()) |entry| {223 while (try tree_iter.next()) |entry| {
119 switch (entry.type) {224 switch (entry.type) {
120 .directory => {225 .directory => {
...@@ -170,19 +275,20 @@ pub const Repository = struct {...@@ -170,19 +275,20 @@ pub const Repository = struct {
170275
171 /// Returns the ID of the tree associated with the given commit (provided as276 /// Returns the ID of the tree associated with the given commit (provided as
172 /// raw object data).277 /// raw object data).
173 fn getCommitTree(commit_data: []const u8) !Oid {278 fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid {
174 if (!mem.startsWith(u8, commit_data, "tree ") or279 if (!mem.startsWith(u8, commit_data, "tree ") or
175 commit_data.len < "tree ".len + fmt_oid_length + "\n".len or280 commit_data.len < "tree ".len + format.formattedLength() + "\n".len or
176 commit_data["tree ".len + fmt_oid_length] != '\n')281 commit_data["tree ".len + format.formattedLength()] != '\n')
177 {282 {
178 return error.InvalidCommit;283 return error.InvalidCommit;
179 }284 }
180 return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]);285 return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]);
181 }286 }
182287
183 const TreeIterator = struct {288 const TreeIterator = struct {
289 format: Oid.Format,
184 data: []const u8,290 data: []const u8,
185 pos: usize = 0,291 pos: usize,
186292
187 const Entry = struct {293 const Entry = struct {
188 type: Type,294 type: Type,
...@@ -220,8 +326,9 @@ pub const Repository = struct {...@@ -220,8 +326,9 @@ pub const Repository = struct {
220 const name = iterator.data[iterator.pos..name_end :0];326 const name = iterator.data[iterator.pos..name_end :0];
221 iterator.pos = name_end + 1;327 iterator.pos = name_end + 1;
222328
329 const oid_length = iterator.format.byteLength();
223 if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;330 if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
224 const oid = iterator.data[iterator.pos..][0..oid_length].*;331 const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]);
225 iterator.pos += oid_length;332 iterator.pos += oid_length;
226333
227 return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };334 return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
...@@ -235,6 +342,7 @@ pub const Repository = struct {...@@ -235,6 +342,7 @@ pub const Repository = struct {
235/// The format of the packfile and its associated index are documented in342/// The format of the packfile and its associated index are documented in
236/// [pack-format](https://git-scm.com/docs/pack-format).343/// [pack-format](https://git-scm.com/docs/pack-format).
237const Odb = struct {344const Odb = struct {
345 format: Oid.Format,
238 pack_file: std.fs.File,346 pack_file: std.fs.File,
239 index_header: IndexHeader,347 index_header: IndexHeader,
240 index_file: std.fs.File,348 index_file: std.fs.File,
...@@ -242,11 +350,12 @@ const Odb = struct {...@@ -242,11 +350,12 @@ const Odb = struct {
242 allocator: Allocator,350 allocator: Allocator,
243351
244 /// Initializes the database from open pack and index files.352 /// Initializes the database from open pack and index files.
245 fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb {353 fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
246 try pack_file.seekTo(0);354 try pack_file.seekTo(0);
247 try index_file.seekTo(0);355 try index_file.seekTo(0);
248 const index_header = try IndexHeader.read(index_file.reader());356 const index_header = try IndexHeader.read(index_file.reader());
249 return .{357 return .{
358 .format = format,
250 .pack_file = pack_file,359 .pack_file = pack_file,
251 .index_header = index_header,360 .index_header = index_header,
252 .index_file = index_file,361 .index_file = index_file,
...@@ -268,7 +377,7 @@ const Odb = struct {...@@ -268,7 +377,7 @@ const Odb = struct {
268 const base_object = while (true) {377 const base_object = while (true) {
269 if (odb.cache.get(base_offset)) |base_object| break base_object;378 if (odb.cache.get(base_offset)) |base_object| break base_object;
270379
271 base_header = try EntryHeader.read(odb.pack_file.reader());380 base_header = try EntryHeader.read(odb.format, odb.pack_file.reader());
272 switch (base_header) {381 switch (base_header) {
273 .ofs_delta => |ofs_delta| {382 .ofs_delta => |ofs_delta| {
274 try delta_offsets.append(odb.allocator, base_offset);383 try delta_offsets.append(odb.allocator, base_offset);
...@@ -292,6 +401,7 @@ const Odb = struct {...@@ -292,6 +401,7 @@ const Odb = struct {
292401
293 const base_data = try resolveDeltaChain(402 const base_data = try resolveDeltaChain(
294 odb.allocator,403 odb.allocator,
404 odb.format,
295 odb.pack_file,405 odb.pack_file,
296 base_object,406 base_object,
297 delta_offsets.items,407 delta_offsets.items,
...@@ -303,14 +413,15 @@ const Odb = struct {...@@ -303,14 +413,15 @@ const Odb = struct {
303413
304 /// Seeks to the beginning of the object with the given ID.414 /// Seeks to the beginning of the object with the given ID.
305 fn seekOid(odb: *Odb, oid: Oid) !void {415 fn seekOid(odb: *Odb, oid: Oid) !void {
306 const key = oid[0];416 const oid_length = odb.format.byteLength();
417 const key = oid.slice()[0];
307 var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;418 var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
308 var end_index = odb.index_header.fan_out_table[key];419 var end_index = odb.index_header.fan_out_table[key];
309 const found_index = while (start_index < end_index) {420 const found_index = while (start_index < end_index) {
310 const mid_index = start_index + (end_index - start_index) / 2;421 const mid_index = start_index + (end_index - start_index) / 2;
311 try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);422 try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
312 const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length);423 const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader());
313 switch (mem.order(u8, &mid_oid, &oid)) {424 switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
314 .lt => start_index = mid_index + 1,425 .lt => start_index = mid_index + 1,
315 .gt => end_index = mid_index,426 .gt => end_index = mid_index,
316 .eq => break mid_index,427 .eq => break mid_index,
...@@ -495,6 +606,7 @@ pub const Session = struct {...@@ -495,6 +606,7 @@ pub const Session = struct {
495 location: Location,606 location: Location,
496 supports_agent: bool,607 supports_agent: bool,
497 supports_shallow: bool,608 supports_shallow: bool,
609 object_format: Oid.Format,
498 allocator: Allocator,610 allocator: Allocator,
499611
500 const agent = "zig/" ++ @import("builtin").zig_version_string;612 const agent = "zig/" ++ @import("builtin").zig_version_string;
...@@ -513,6 +625,7 @@ pub const Session = struct {...@@ -513,6 +625,7 @@ pub const Session = struct {
513 .location = try .init(allocator, uri),625 .location = try .init(allocator, uri),
514 .supports_agent = false,626 .supports_agent = false,
515 .supports_shallow = false,627 .supports_shallow = false,
628 .object_format = .sha1,
516 .allocator = allocator,629 .allocator = allocator,
517 };630 };
518 errdefer session.deinit();631 errdefer session.deinit();
...@@ -528,6 +641,10 @@ pub const Session = struct {...@@ -528,6 +641,10 @@ pub const Session = struct {
528 session.supports_shallow = true;641 session.supports_shallow = true;
529 }642 }
530 }643 }
644 } else if (mem.eql(u8, capability.key, "object-format")) {
645 if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| {
646 session.object_format = format;
647 }
531 }648 }
532 }649 }
533 return session;650 return session;
...@@ -708,6 +825,11 @@ pub const Session = struct {...@@ -708,6 +825,11 @@ pub const Session = struct {
708 if (session.supports_agent) {825 if (session.supports_agent) {
709 try Packet.write(.{ .data = agent_capability }, body_writer);826 try Packet.write(.{ .data = agent_capability }, body_writer);
710 }827 }
828 {
829 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});
830 defer session.allocator.free(object_format_packet);
831 try Packet.write(.{ .data = object_format_packet }, body_writer);
832 }
711 try Packet.write(.delimiter, body_writer);833 try Packet.write(.delimiter, body_writer);
712 for (options.ref_prefixes) |ref_prefix| {834 for (options.ref_prefixes) |ref_prefix| {
713 const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix});835 const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix});
...@@ -739,10 +861,14 @@ pub const Session = struct {...@@ -739,10 +861,14 @@ pub const Session = struct {
739 try request.wait();861 try request.wait();
740 if (request.response.status != .ok) return error.ProtocolError;862 if (request.response.status != .ok) return error.ProtocolError;
741863
742 return .{ .request = request };864 return .{
865 .format = session.object_format,
866 .request = request,
867 };
743 }868 }
744869
745 pub const RefIterator = struct {870 pub const RefIterator = struct {
871 format: Oid.Format,
746 request: std.http.Client.Request,872 request: std.http.Client.Request,
747 buf: [Packet.max_data_length]u8 = undefined,873 buf: [Packet.max_data_length]u8 = undefined,
748874
...@@ -764,7 +890,7 @@ pub const Session = struct {...@@ -764,7 +890,7 @@ pub const Session = struct {
764 .data => |data| {890 .data => |data| {
765 const ref_data = Packet.normalizeText(data);891 const ref_data = Packet.normalizeText(data);
766 const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;892 const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
767 const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket;893 const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
768894
769 const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;895 const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
770 const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];896 const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
...@@ -778,7 +904,7 @@ pub const Session = struct {...@@ -778,7 +904,7 @@ pub const Session = struct {
778 if (mem.startsWith(u8, attribute, "symref-target:")) {904 if (mem.startsWith(u8, attribute, "symref-target:")) {
779 symref_target = attribute["symref-target:".len..];905 symref_target = attribute["symref-target:".len..];
780 } else if (mem.startsWith(u8, attribute, "peeled:")) {906 } else if (mem.startsWith(u8, attribute, "peeled:")) {
781 peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket;907 peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
782 }908 }
783 last_sep_pos = next_sep_pos;909 last_sep_pos = next_sep_pos;
784 }910 }
...@@ -814,6 +940,11 @@ pub const Session = struct {...@@ -814,6 +940,11 @@ pub const Session = struct {
814 if (session.supports_agent) {940 if (session.supports_agent) {
815 try Packet.write(.{ .data = agent_capability }, body_writer);941 try Packet.write(.{ .data = agent_capability }, body_writer);
816 }942 }
943 {
944 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});
945 defer session.allocator.free(object_format_packet);
946 try Packet.write(.{ .data = object_format_packet }, body_writer);
947 }
817 try Packet.write(.delimiter, body_writer);948 try Packet.write(.delimiter, body_writer);
818 // Our packfile parser supports the OFS_DELTA object type949 // Our packfile parser supports the OFS_DELTA object type
819 try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);950 try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);
...@@ -997,7 +1128,7 @@ const EntryHeader = union(Type) {...@@ -997,7 +1128,7 @@ const EntryHeader = union(Type) {
997 };1128 };
998 }1129 }
9991130
1000 fn read(reader: anytype) !EntryHeader {1131 fn read(format: Oid.Format, reader: anytype) !EntryHeader {
1001 const InitialByte = packed struct { len: u4, type: u3, has_next: bool };1132 const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
1002 const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) {1133 const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) {
1003 error.EndOfStream => return error.InvalidFormat,1134 error.EndOfStream => return error.InvalidFormat,
...@@ -1016,7 +1147,7 @@ const EntryHeader = union(Type) {...@@ -1016,7 +1147,7 @@ const EntryHeader = union(Type) {
1016 .uncompressed_length = uncompressed_length,1147 .uncompressed_length = uncompressed_length,
1017 } },1148 } },
1018 .ref_delta => .{ .ref_delta = .{1149 .ref_delta => .{ .ref_delta = .{
1019 .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) {1150 .base_object = Oid.readBytes(format, reader) catch |e| switch (e) {
1020 error.EndOfStream => return error.InvalidFormat,1151 error.EndOfStream => return error.InvalidFormat,
1021 else => |other| return other,1152 else => |other| return other,
1022 },1153 },
...@@ -1081,7 +1212,7 @@ const IndexEntry = struct {...@@ -1081,7 +1212,7 @@ const IndexEntry = struct {
10811212
1082/// Writes out a version 2 index for the given packfile, as documented in1213/// Writes out a version 2 index for the given packfile, as documented in
1083/// [pack-format](https://git-scm.com/docs/pack-format).1214/// [pack-format](https://git-scm.com/docs/pack-format).
1084pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void {1215pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void {
1085 try pack.seekTo(0);1216 try pack.seekTo(0);
10861217
1087 var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;1218 var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
...@@ -1089,7 +1220,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1089,7 +1220,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1089 var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty;1220 var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty;
1090 defer pending_deltas.deinit(allocator);1221 defer pending_deltas.deinit(allocator);
10911222
1092 const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas);1223 const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas);
10931224
1094 var cache: ObjectCache = .{};1225 var cache: ObjectCache = .{};
1095 defer cache.deinit(allocator);1226 defer cache.deinit(allocator);
...@@ -1099,7 +1230,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1099,7 +1230,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1099 while (i > 0) {1230 while (i > 0) {
1100 i -= 1;1231 i -= 1;
1101 const delta = pending_deltas.items[i];1232 const delta = pending_deltas.items[i];
1102 if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| {1233 if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| {
1103 try index_entries.put(allocator, oid, delta);1234 try index_entries.put(allocator, oid, delta);
1104 _ = pending_deltas.swapRemove(i);1235 _ = pending_deltas.swapRemove(i);
1105 }1236 }
...@@ -1117,7 +1248,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1117,7 +1248,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1117 }1248 }
1118 mem.sortUnstable(Oid, oids.items, {}, struct {1249 mem.sortUnstable(Oid, oids.items, {}, struct {
1119 fn lessThan(_: void, o1: Oid, o2: Oid) bool {1250 fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1120 return mem.lessThan(u8, &o1, &o2);1251 return mem.lessThan(u8, o1.slice(), o2.slice());
1121 }1252 }
1122 }.lessThan);1253 }.lessThan);
11231254
...@@ -1125,15 +1256,16 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1125,15 +1256,16 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1125 var count: u32 = 0;1256 var count: u32 = 0;
1126 var fan_out_index: u8 = 0;1257 var fan_out_index: u8 = 0;
1127 for (oids.items) |oid| {1258 for (oids.items) |oid| {
1128 if (oid[0] > fan_out_index) {1259 const key = oid.slice()[0];
1129 @memset(fan_out_table[fan_out_index..oid[0]], count);1260 if (key > fan_out_index) {
1130 fan_out_index = oid[0];1261 @memset(fan_out_table[fan_out_index..key], count);
1262 fan_out_index = key;
1131 }1263 }
1132 count += 1;1264 count += 1;
1133 }1265 }
1134 @memset(fan_out_table[fan_out_index..], count);1266 @memset(fan_out_table[fan_out_index..], count);
11351267
1136 var index_hashed_writer = std.compress.hashedWriter(index_writer, Sha1.init(.{}));1268 var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format));
1137 const writer = index_hashed_writer.writer();1269 const writer = index_hashed_writer.writer();
1138 try writer.writeAll(IndexHeader.signature);1270 try writer.writeAll(IndexHeader.signature);
1139 try writer.writeInt(u32, IndexHeader.supported_version, .big);1271 try writer.writeInt(u32, IndexHeader.supported_version, .big);
...@@ -1142,7 +1274,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1142,7 +1274,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1142 }1274 }
11431275
1144 for (oids.items) |oid| {1276 for (oids.items) |oid| {
1145 try writer.writeAll(&oid);1277 try writer.writeAll(oid.slice());
1146 }1278 }
11471279
1148 for (oids.items) |oid| {1280 for (oids.items) |oid| {
...@@ -1165,9 +1297,9 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1165,9 +1297,9 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1165 try writer.writeInt(u64, offset, .big);1297 try writer.writeInt(u64, offset, .big);
1166 }1298 }
11671299
1168 try writer.writeAll(&pack_checksum);1300 try writer.writeAll(pack_checksum.slice());
1169 const index_checksum = index_hashed_writer.hasher.finalResult();1301 const index_checksum = index_hashed_writer.hasher.finalResult();
1170 try index_writer.writeAll(&index_checksum);1302 try index_writer.writeAll(index_checksum.slice());
1171}1303}
11721304
1173/// Performs the first pass over the packfile data for index construction.1305/// Performs the first pass over the packfile data for index construction.
...@@ -1176,13 +1308,14 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)...@@ -1176,13 +1308,14 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype)
1176/// format).1308/// format).
1177fn indexPackFirstPass(1309fn indexPackFirstPass(
1178 allocator: Allocator,1310 allocator: Allocator,
1311 format: Oid.Format,
1179 pack: std.fs.File,1312 pack: std.fs.File,
1180 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),1313 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1181 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),1314 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1182) ![Sha1.digest_length]u8 {1315) !Oid {
1183 var pack_buffered_reader = std.io.bufferedReader(pack.reader());1316 var pack_buffered_reader = std.io.bufferedReader(pack.reader());
1184 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());1317 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1185 var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{}));1318 var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1186 const pack_reader = pack_hashed_reader.reader();1319 const pack_reader = pack_hashed_reader.reader();
11871320
1188 const pack_header = try PackHeader.read(pack_reader);1321 const pack_header = try PackHeader.read(pack_reader);
...@@ -1191,12 +1324,12 @@ fn indexPackFirstPass(...@@ -1191,12 +1324,12 @@ fn indexPackFirstPass(
1191 while (current_entry < pack_header.total_objects) : (current_entry += 1) {1324 while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1192 const entry_offset = pack_counting_reader.bytes_read;1325 const entry_offset = pack_counting_reader.bytes_read;
1193 var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());1326 var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
1194 const entry_header = try EntryHeader.read(entry_crc32_reader.reader());1327 const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
1195 switch (entry_header) {1328 switch (entry_header) {
1196 .commit, .tree, .blob, .tag => |object| {1329 .commit, .tree, .blob, .tag => |object| {
1197 var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());1330 var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1198 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());1331 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1199 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Sha1.init(.{}));1332 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1200 const entry_writer = entry_hashed_writer.writer();1333 const entry_writer = entry_hashed_writer.writer();
1201 // The object header is not included in the pack data but is1334 // The object header is not included in the pack data but is
1202 // part of the object's ID1335 // part of the object's ID
...@@ -1229,8 +1362,8 @@ fn indexPackFirstPass(...@@ -1229,8 +1362,8 @@ fn indexPackFirstPass(
1229 }1362 }
12301363
1231 const pack_checksum = pack_hashed_reader.hasher.finalResult();1364 const pack_checksum = pack_hashed_reader.hasher.finalResult();
1232 const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length);1365 const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader());
1233 if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {1366 if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {
1234 return error.CorruptedPack;1367 return error.CorruptedPack;
1235 }1368 }
1236 _ = pack_reader.readByte() catch |e| switch (e) {1369 _ = pack_reader.readByte() catch |e| switch (e) {
...@@ -1245,6 +1378,7 @@ fn indexPackFirstPass(...@@ -1245,6 +1378,7 @@ fn indexPackFirstPass(
1245/// delta and we do not yet know the offset of the base object).1378/// delta and we do not yet know the offset of the base object).
1246fn indexPackHashDelta(1379fn indexPackHashDelta(
1247 allocator: Allocator,1380 allocator: Allocator,
1381 format: Oid.Format,
1248 pack: std.fs.File,1382 pack: std.fs.File,
1249 delta: IndexEntry,1383 delta: IndexEntry,
1250 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),1384 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
...@@ -1259,7 +1393,7 @@ fn indexPackHashDelta(...@@ -1259,7 +1393,7 @@ fn indexPackHashDelta(
1259 if (cache.get(base_offset)) |base_object| break base_object;1393 if (cache.get(base_offset)) |base_object| break base_object;
12601394
1261 try pack.seekTo(base_offset);1395 try pack.seekTo(base_offset);
1262 base_header = try EntryHeader.read(pack.reader());1396 base_header = try EntryHeader.read(format, pack.reader());
1263 switch (base_header) {1397 switch (base_header) {
1264 .ofs_delta => |ofs_delta| {1398 .ofs_delta => |ofs_delta| {
1265 try delta_offsets.append(allocator, base_offset);1399 try delta_offsets.append(allocator, base_offset);
...@@ -1279,9 +1413,9 @@ fn indexPackHashDelta(...@@ -1279,9 +1413,9 @@ fn indexPackHashDelta(
1279 }1413 }
1280 };1414 };
12811415
1282 const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache);1416 const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
12831417
1284 var entry_hasher = Sha1.init(.{});1418 var entry_hasher: Oid.Hasher = .init(format);
1285 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);1419 var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);
1286 try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });1420 try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1287 entry_hasher.update(base_data);1421 entry_hasher.update(base_data);
...@@ -1294,6 +1428,7 @@ fn indexPackHashDelta(...@@ -1294,6 +1428,7 @@ fn indexPackHashDelta(
1294/// to obtain the final object.1428/// to obtain the final object.
1295fn resolveDeltaChain(1429fn resolveDeltaChain(
1296 allocator: Allocator,1430 allocator: Allocator,
1431 format: Oid.Format,
1297 pack: std.fs.File,1432 pack: std.fs.File,
1298 base_object: Object,1433 base_object: Object,
1299 delta_offsets: []const u64,1434 delta_offsets: []const u64,
...@@ -1306,7 +1441,7 @@ fn resolveDeltaChain(...@@ -1306,7 +1441,7 @@ fn resolveDeltaChain(
13061441
1307 const delta_offset = delta_offsets[i];1442 const delta_offset = delta_offsets[i];
1308 try pack.seekTo(delta_offset);1443 try pack.seekTo(delta_offset);
1309 const delta_header = try EntryHeader.read(pack.reader());1444 const delta_header = try EntryHeader.read(format, pack.reader());
1310 const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());1445 const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
1311 defer allocator.free(delta_data);1446 defer allocator.free(delta_data);
1312 var delta_stream = std.io.fixedBufferStream(delta_data);1447 var delta_stream = std.io.fixedBufferStream(delta_data);
...@@ -1394,16 +1529,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo...@@ -1394,16 +1529,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo
1394 }1529 }
1395}1530}
13961531
1397test "packfile indexing and checkout" {1532/// Runs the packfile indexing and checkout test.
1398 // To verify the contents of this packfile without using the code in this1533///
1399 // file:1534/// The two testrepo repositories under testdata contain identical commit
1400 //1535/// histories and contents.
1401 // 1. Create a new empty Git repository (`git init`)1536///
1402 // 2. `git unpack-objects <path/to/testdata.pack`1537/// To verify the contents of the packfiles using Git alone, run the
1403 // 3. `git fsck` -> note the "dangling commit" ID (which matches the commit1538/// following commands in an empty directory:
1404 // checked out below)1539///
1405 // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb`1540/// 1. `git init --object-format=(sha1|sha256)`
1406 const testrepo_pack = @embedFile("git/testdata/testrepo.pack");1541/// 2. `git unpack-objects <path/to/testrepo.pack`
1542/// 3. `git fsck` - will print one "dangling commit":
1543/// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb`
1544/// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a`
1545/// 4. `git checkout $commit`
1546fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void {
1547 const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack");
14071548
1408 var git_dir = testing.tmpDir(.{});1549 var git_dir = testing.tmpDir(.{});
1409 defer git_dir.cleanup();1550 defer git_dir.cleanup();
...@@ -1413,27 +1554,27 @@ test "packfile indexing and checkout" {...@@ -1413,27 +1554,27 @@ test "packfile indexing and checkout" {
14131554
1414 var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });1555 var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1415 defer index_file.close();1556 defer index_file.close();
1416 try indexPack(testing.allocator, pack_file, index_file.writer());1557 try indexPack(testing.allocator, format, pack_file, index_file.writer());
14171558
1418 // Arbitrary size limit on files read while checking the repository contents1559 // Arbitrary size limit on files read while checking the repository contents
1419 // (all files in the test repo are known to be much smaller than this)1560 // (all files in the test repo are known to be smaller than this)
1420 const max_file_size = 4096;1561 const max_file_size = 8192;
14211562
1422 const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);1563 const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1423 defer testing.allocator.free(index_file_data);1564 defer testing.allocator.free(index_file_data);
1424 // testrepo.idx is generated by Git. The index created by this file should1565 // testrepo.idx is generated by Git. The index created by this file should
1425 // match it exactly. Running `git verify-pack -v testrepo.pack` can verify1566 // match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1426 // this.1567 // this.
1427 const testrepo_idx = @embedFile("git/testdata/testrepo.idx");1568 const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
1428 try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);1569 try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
14291570
1430 var repository = try Repository.init(testing.allocator, pack_file, index_file);1571 var repository = try Repository.init(testing.allocator, format, pack_file, index_file);
1431 defer repository.deinit();1572 defer repository.deinit();
14321573
1433 var worktree = testing.tmpDir(.{ .iterate = true });1574 var worktree = testing.tmpDir(.{ .iterate = true });
1434 defer worktree.cleanup();1575 defer worktree.cleanup();
14351576
1436 const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb");1577 const commit_id = try Oid.parse(format, head_commit);
14371578
1438 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };1579 var diagnostics: Diagnostics = .{ .allocator = testing.allocator };
1439 defer diagnostics.deinit();1580 defer diagnostics.deinit();
...@@ -1497,6 +1638,14 @@ test "packfile indexing and checkout" {...@@ -1497,6 +1638,14 @@ test "packfile indexing and checkout" {
1497 try testing.expectEqualStrings(expected_file_contents, actual_file_contents);1638 try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1498}1639}
14991640
1641test "SHA-1 packfile indexing and checkout" {
1642 try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb");
1643}
1644
1645test "SHA-256 packfile indexing and checkout" {
1646 try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a");
1647}
1648
1500/// Checks out a commit of a packfile. Intended for experimenting with and1649/// Checks out a commit of a packfile. Intended for experimenting with and
1501/// benchmarking possible optimizations to the indexing and checkout behavior.1650/// benchmarking possible optimizations to the indexing and checkout behavior.
1502pub fn main() !void {1651pub fn main() !void {
...@@ -1504,14 +1653,16 @@ pub fn main() !void {...@@ -1504,14 +1653,16 @@ pub fn main() !void {
15041653
1505 const args = try std.process.argsAlloc(allocator);1654 const args = try std.process.argsAlloc(allocator);
1506 defer std.process.argsFree(allocator, args);1655 defer std.process.argsFree(allocator, args);
1507 if (args.len != 4) {1656 if (args.len != 5) {
1508 return error.InvalidArguments; // Arguments: packfile commit worktree1657 return error.InvalidArguments; // Arguments: format packfile commit worktree
1509 }1658 }
15101659
1511 var pack_file = try std.fs.cwd().openFile(args[1], .{});1660 const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat;
1661
1662 var pack_file = try std.fs.cwd().openFile(args[2], .{});
1512 defer pack_file.close();1663 defer pack_file.close();
1513 const commit = try parseOid(args[2]);1664 const commit = try Oid.parse(format, args[3]);
1514 var worktree = try std.fs.cwd().makeOpenPath(args[3], .{});1665 var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});
1515 defer worktree.close();1666 defer worktree.close();
15161667
1517 var git_dir = try worktree.makeOpenPath(".git", .{});1668 var git_dir = try worktree.makeOpenPath(".git", .{});
...@@ -1521,12 +1672,12 @@ pub fn main() !void {...@@ -1521,12 +1672,12 @@ pub fn main() !void {
1521 var index_file = try git_dir.createFile("idx", .{ .read = true });1672 var index_file = try git_dir.createFile("idx", .{ .read = true });
1522 defer index_file.close();1673 defer index_file.close();
1523 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());1674 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1524 try indexPack(allocator, pack_file, index_buffered_writer.writer());1675 try indexPack(allocator, format, pack_file, index_buffered_writer.writer());
1525 try index_buffered_writer.flush();1676 try index_buffered_writer.flush();
1526 try index_file.sync();1677 try index_file.sync();
15271678
1528 std.debug.print("Starting checkout...\n", .{});1679 std.debug.print("Starting checkout...\n", .{});
1529 var repository = try Repository.init(allocator, pack_file, index_file);1680 var repository = try Repository.init(allocator, format, pack_file, index_file);
1530 defer repository.deinit();1681 defer repository.deinit();
1531 var diagnostics: Diagnostics = .{ .allocator = allocator };1682 var diagnostics: Diagnostics = .{ .allocator = allocator };
1532 defer diagnostics.deinit();1683 defer diagnostics.deinit();
src/Package/Fetch/git/testdata/testrepo-sha1.idx created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo-sha1.idx differ
src/Package/Fetch/git/testdata/testrepo-sha1.pack created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo-sha1.pack differ
src/Package/Fetch/git/testdata/testrepo-sha256.idx created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo-sha256.idx differ
src/Package/Fetch/git/testdata/testrepo-sha256.pack created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo-sha256.pack differ
src/Package/Fetch/git/testdata/testrepo.idx deleted
Binary files a/src/Package/Fetch/git/testdata/testrepo.idx and /dev/null differ
src/Package/Fetch/git/testdata/testrepo.pack deleted
Binary files a/src/Package/Fetch/git/testdata/testrepo.pack and /dev/null differ
src/main.zig+2-2
...@@ -7037,8 +7037,8 @@ fn cmdFetch(...@@ -7037,8 +7037,8 @@ fn cmdFetch(
70377037
7038 var saved_path_or_url = path_or_url;7038 var saved_path_or_url = path_or_url;
70397039
7040 if (fetch.latest_commit) |*latest_commit| resolved: {7040 if (fetch.latest_commit) |latest_commit| resolved: {
7041 const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{std.fmt.fmtSliceHexLower(latest_commit)});7041 const latest_commit_hex = try std.fmt.allocPrint(arena, "{}", .{latest_commit});
70427042
7043 var uri = try std.Uri.parse(path_or_url);7043 var uri = try std.Uri.parse(path_or_url);
70447044