authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 17:29:55-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 17:29:55-07:00
log6d84caf72771cf05997518ae2fa40a94de709de4
tree862543ae6537595ef67ced8d4d06da5384f29699
parent7bae6d90648e6ef0782c7f5e8a72066742feacaf

move some package management related source files around

10 files changed, 2033 insertions(+), 2035 deletions(-)

src/Manifest.zig deleted-564
...@@ -1,564 +0,0 @@
1pub const max_bytes = 10 * 1024 * 1024;
2pub const basename = "build.zig.zon";
3pub const Hash = std.crypto.hash.sha2.Sha256;
4pub const Digest = [Hash.digest_length]u8;
5pub const multihash_len = 1 + 1 + Hash.digest_length;
6pub const multihash_hex_digest_len = 2 * multihash_len;
7pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
8
9pub const Dependency = struct {
10 location: Location,
11 location_tok: Ast.TokenIndex,
12 hash: ?[]const u8,
13 hash_tok: Ast.TokenIndex,
14
15 pub const Location = union(enum) {
16 url: []const u8,
17 path: []const u8,
18 };
19};
20
21pub const ErrorMessage = struct {
22 msg: []const u8,
23 tok: Ast.TokenIndex,
24 off: u32,
25};
26
27pub const MultihashFunction = enum(u16) {
28 identity = 0x00,
29 sha1 = 0x11,
30 @"sha2-256" = 0x12,
31 @"sha2-512" = 0x13,
32 @"sha3-512" = 0x14,
33 @"sha3-384" = 0x15,
34 @"sha3-256" = 0x16,
35 @"sha3-224" = 0x17,
36 @"sha2-384" = 0x20,
37 @"sha2-256-trunc254-padded" = 0x1012,
38 @"sha2-224" = 0x1013,
39 @"sha2-512-224" = 0x1014,
40 @"sha2-512-256" = 0x1015,
41 @"blake2b-256" = 0xb220,
42 _,
43};
44
45pub const multihash_function: MultihashFunction = switch (Hash) {
46 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
47 else => @compileError("unreachable"),
48};
49comptime {
50 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
51 // values are small enough to be contained in the one-byte encoding.
52 assert(@intFromEnum(multihash_function) < 127);
53 assert(Hash.digest_length < 127);
54}
55
56name: []const u8,
57version: std.SemanticVersion,
58dependencies: std.StringArrayHashMapUnmanaged(Dependency),
59paths: std.StringArrayHashMapUnmanaged(void),
60
61errors: []ErrorMessage,
62arena_state: std.heap.ArenaAllocator.State,
63
64pub const ParseOptions = struct {
65 allow_missing_paths_field: bool = false,
66};
67
68pub const Error = Allocator.Error;
69
70pub fn parse(gpa: Allocator, ast: std.zig.Ast, options: ParseOptions) Error!Manifest {
71 const node_tags = ast.nodes.items(.tag);
72 const node_datas = ast.nodes.items(.data);
73 assert(node_tags[0] == .root);
74 const main_node_index = node_datas[0].lhs;
75
76 var arena_instance = std.heap.ArenaAllocator.init(gpa);
77 errdefer arena_instance.deinit();
78
79 var p: Parse = .{
80 .gpa = gpa,
81 .ast = ast,
82 .arena = arena_instance.allocator(),
83 .errors = .{},
84
85 .name = undefined,
86 .version = undefined,
87 .dependencies = .{},
88 .paths = .{},
89 .allow_missing_paths_field = options.allow_missing_paths_field,
90 .buf = .{},
91 };
92 defer p.buf.deinit(gpa);
93 defer p.errors.deinit(gpa);
94 defer p.dependencies.deinit(gpa);
95 defer p.paths.deinit(gpa);
96
97 p.parseRoot(main_node_index) catch |err| switch (err) {
98 error.ParseFailure => assert(p.errors.items.len > 0),
99 else => |e| return e,
100 };
101
102 return .{
103 .name = p.name,
104 .version = p.version,
105 .dependencies = try p.dependencies.clone(p.arena),
106 .paths = try p.paths.clone(p.arena),
107 .errors = try p.arena.dupe(ErrorMessage, p.errors.items),
108 .arena_state = arena_instance.state,
109 };
110}
111
112pub fn deinit(man: *Manifest, gpa: Allocator) void {
113 man.arena_state.promote(gpa).deinit();
114 man.* = undefined;
115}
116
117const hex_charset = "0123456789abcdef";
118
119pub fn hex64(x: u64) [16]u8 {
120 var result: [16]u8 = undefined;
121 var i: usize = 0;
122 while (i < 8) : (i += 1) {
123 const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i))));
124 result[i * 2 + 0] = hex_charset[byte >> 4];
125 result[i * 2 + 1] = hex_charset[byte & 15];
126 }
127 return result;
128}
129
130test hex64 {
131 const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
132 try std.testing.expectEqualStrings("[00efcdab78563412]", s);
133}
134
135pub fn hexDigest(digest: Digest) MultiHashHexDigest {
136 var result: MultiHashHexDigest = undefined;
137
138 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
139 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
140
141 result[2] = hex_charset[Hash.digest_length >> 4];
142 result[3] = hex_charset[Hash.digest_length & 15];
143
144 for (digest, 0..) |byte, i| {
145 result[4 + i * 2] = hex_charset[byte >> 4];
146 result[5 + i * 2] = hex_charset[byte & 15];
147 }
148 return result;
149}
150
151const Parse = struct {
152 gpa: Allocator,
153 ast: std.zig.Ast,
154 arena: Allocator,
155 buf: std.ArrayListUnmanaged(u8),
156 errors: std.ArrayListUnmanaged(ErrorMessage),
157
158 name: []const u8,
159 version: std.SemanticVersion,
160 dependencies: std.StringArrayHashMapUnmanaged(Dependency),
161 paths: std.StringArrayHashMapUnmanaged(void),
162 allow_missing_paths_field: bool,
163
164 const InnerError = error{ ParseFailure, OutOfMemory };
165
166 fn parseRoot(p: *Parse, node: Ast.Node.Index) !void {
167 const ast = p.ast;
168 const main_tokens = ast.nodes.items(.main_token);
169 const main_token = main_tokens[node];
170
171 var buf: [2]Ast.Node.Index = undefined;
172 const struct_init = ast.fullStructInit(&buf, node) orelse {
173 return fail(p, main_token, "expected top level expression to be a struct", .{});
174 };
175
176 var have_name = false;
177 var have_version = false;
178 var have_included_paths = false;
179
180 for (struct_init.ast.fields) |field_init| {
181 const name_token = ast.firstToken(field_init) - 2;
182 const field_name = try identifierTokenString(p, name_token);
183 // We could get fancy with reflection and comptime logic here but doing
184 // things manually provides an opportunity to do any additional verification
185 // that is desirable on a per-field basis.
186 if (mem.eql(u8, field_name, "dependencies")) {
187 try parseDependencies(p, field_init);
188 } else if (mem.eql(u8, field_name, "paths")) {
189 have_included_paths = true;
190 try parseIncludedPaths(p, field_init);
191 } else if (mem.eql(u8, field_name, "name")) {
192 p.name = try parseString(p, field_init);
193 have_name = true;
194 } else if (mem.eql(u8, field_name, "version")) {
195 const version_text = try parseString(p, field_init);
196 p.version = std.SemanticVersion.parse(version_text) catch |err| v: {
197 try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
198 break :v undefined;
199 };
200 have_version = true;
201 } else {
202 // Ignore unknown fields so that we can add fields in future zig
203 // versions without breaking older zig versions.
204 }
205 }
206
207 if (!have_name) {
208 try appendError(p, main_token, "missing top-level 'name' field", .{});
209 }
210
211 if (!have_version) {
212 try appendError(p, main_token, "missing top-level 'version' field", .{});
213 }
214
215 if (!have_included_paths) {
216 if (p.allow_missing_paths_field) {
217 try p.paths.put(p.gpa, "", {});
218 } else {
219 try appendError(p, main_token, "missing top-level 'paths' field", .{});
220 }
221 }
222 }
223
224 fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
225 const ast = p.ast;
226 const main_tokens = ast.nodes.items(.main_token);
227
228 var buf: [2]Ast.Node.Index = undefined;
229 const struct_init = ast.fullStructInit(&buf, node) orelse {
230 const tok = main_tokens[node];
231 return fail(p, tok, "expected dependencies expression to be a struct", .{});
232 };
233
234 for (struct_init.ast.fields) |field_init| {
235 const name_token = ast.firstToken(field_init) - 2;
236 const dep_name = try identifierTokenString(p, name_token);
237 const dep = try parseDependency(p, field_init);
238 try p.dependencies.put(p.gpa, dep_name, dep);
239 }
240 }
241
242 fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
243 const ast = p.ast;
244 const main_tokens = ast.nodes.items(.main_token);
245
246 var buf: [2]Ast.Node.Index = undefined;
247 const struct_init = ast.fullStructInit(&buf, node) orelse {
248 const tok = main_tokens[node];
249 return fail(p, tok, "expected dependency expression to be a struct", .{});
250 };
251
252 var dep: Dependency = .{
253 .location = undefined,
254 .location_tok = 0,
255 .hash = null,
256 .hash_tok = 0,
257 };
258 var has_location = false;
259
260 for (struct_init.ast.fields) |field_init| {
261 const name_token = ast.firstToken(field_init) - 2;
262 const field_name = try identifierTokenString(p, name_token);
263 // We could get fancy with reflection and comptime logic here but doing
264 // things manually provides an opportunity to do any additional verification
265 // that is desirable on a per-field basis.
266 if (mem.eql(u8, field_name, "url")) {
267 if (has_location) {
268 return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
269 }
270 dep.location = .{
271 .url = parseString(p, field_init) catch |err| switch (err) {
272 error.ParseFailure => continue,
273 else => |e| return e,
274 },
275 };
276 has_location = true;
277 dep.location_tok = main_tokens[field_init];
278 } else if (mem.eql(u8, field_name, "path")) {
279 if (has_location) {
280 return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
281 }
282 dep.location = .{
283 .path = parseString(p, field_init) catch |err| switch (err) {
284 error.ParseFailure => continue,
285 else => |e| return e,
286 },
287 };
288 has_location = true;
289 dep.location_tok = main_tokens[field_init];
290 } else if (mem.eql(u8, field_name, "hash")) {
291 dep.hash = parseHash(p, field_init) catch |err| switch (err) {
292 error.ParseFailure => continue,
293 else => |e| return e,
294 };
295 dep.hash_tok = main_tokens[field_init];
296 } else {
297 // Ignore unknown fields so that we can add fields in future zig
298 // versions without breaking older zig versions.
299 }
300 }
301
302 if (!has_location) {
303 try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{});
304 }
305
306 return dep;
307 }
308
309 fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
310 const ast = p.ast;
311 const main_tokens = ast.nodes.items(.main_token);
312
313 var buf: [2]Ast.Node.Index = undefined;
314 const array_init = ast.fullArrayInit(&buf, node) orelse {
315 const tok = main_tokens[node];
316 return fail(p, tok, "expected paths expression to be a struct", .{});
317 };
318
319 for (array_init.ast.elements) |elem_node| {
320 const path_string = try parseString(p, elem_node);
321 const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
322 try p.paths.put(p.gpa, normalized, {});
323 }
324 }
325
326 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
327 const ast = p.ast;
328 const node_tags = ast.nodes.items(.tag);
329 const main_tokens = ast.nodes.items(.main_token);
330 if (node_tags[node] != .string_literal) {
331 return fail(p, main_tokens[node], "expected string literal", .{});
332 }
333 const str_lit_token = main_tokens[node];
334 const token_bytes = ast.tokenSlice(str_lit_token);
335 p.buf.clearRetainingCapacity();
336 try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
337 const duped = try p.arena.dupe(u8, p.buf.items);
338 return duped;
339 }
340
341 fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
342 const ast = p.ast;
343 const main_tokens = ast.nodes.items(.main_token);
344 const tok = main_tokens[node];
345 const h = try parseString(p, node);
346
347 if (h.len >= 2) {
348 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
349 return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
350 @errorName(err),
351 });
352 };
353 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
354 return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
355 }
356 }
357
358 if (h.len != multihash_hex_digest_len) {
359 return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
360 multihash_hex_digest_len, h.len,
361 });
362 }
363
364 return h;
365 }
366
367 /// TODO: try to DRY this with AstGen.identifierTokenString
368 fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
369 const ast = p.ast;
370 const token_tags = ast.tokens.items(.tag);
371 assert(token_tags[token] == .identifier);
372 const ident_name = ast.tokenSlice(token);
373 if (!mem.startsWith(u8, ident_name, "@")) {
374 return ident_name;
375 }
376 p.buf.clearRetainingCapacity();
377 try parseStrLit(p, token, &p.buf, ident_name, 1);
378 const duped = try p.arena.dupe(u8, p.buf.items);
379 return duped;
380 }
381
382 /// TODO: try to DRY this with AstGen.parseStrLit
383 fn parseStrLit(
384 p: *Parse,
385 token: Ast.TokenIndex,
386 buf: *std.ArrayListUnmanaged(u8),
387 bytes: []const u8,
388 offset: u32,
389 ) InnerError!void {
390 const raw_string = bytes[offset..];
391 var buf_managed = buf.toManaged(p.gpa);
392 const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
393 buf.* = buf_managed.moveToUnmanaged();
394 switch (try result) {
395 .success => {},
396 .failure => |err| try p.appendStrLitError(err, token, bytes, offset),
397 }
398 }
399
400 /// TODO: try to DRY this with AstGen.failWithStrLitError
401 fn appendStrLitError(
402 p: *Parse,
403 err: std.zig.string_literal.Error,
404 token: Ast.TokenIndex,
405 bytes: []const u8,
406 offset: u32,
407 ) Allocator.Error!void {
408 const raw_string = bytes[offset..];
409 switch (err) {
410 .invalid_escape_character => |bad_index| {
411 try p.appendErrorOff(
412 token,
413 offset + @as(u32, @intCast(bad_index)),
414 "invalid escape character: '{c}'",
415 .{raw_string[bad_index]},
416 );
417 },
418 .expected_hex_digit => |bad_index| {
419 try p.appendErrorOff(
420 token,
421 offset + @as(u32, @intCast(bad_index)),
422 "expected hex digit, found '{c}'",
423 .{raw_string[bad_index]},
424 );
425 },
426 .empty_unicode_escape_sequence => |bad_index| {
427 try p.appendErrorOff(
428 token,
429 offset + @as(u32, @intCast(bad_index)),
430 "empty unicode escape sequence",
431 .{},
432 );
433 },
434 .expected_hex_digit_or_rbrace => |bad_index| {
435 try p.appendErrorOff(
436 token,
437 offset + @as(u32, @intCast(bad_index)),
438 "expected hex digit or '}}', found '{c}'",
439 .{raw_string[bad_index]},
440 );
441 },
442 .invalid_unicode_codepoint => |bad_index| {
443 try p.appendErrorOff(
444 token,
445 offset + @as(u32, @intCast(bad_index)),
446 "unicode escape does not correspond to a valid codepoint",
447 .{},
448 );
449 },
450 .expected_lbrace => |bad_index| {
451 try p.appendErrorOff(
452 token,
453 offset + @as(u32, @intCast(bad_index)),
454 "expected '{{', found '{c}",
455 .{raw_string[bad_index]},
456 );
457 },
458 .expected_rbrace => |bad_index| {
459 try p.appendErrorOff(
460 token,
461 offset + @as(u32, @intCast(bad_index)),
462 "expected '}}', found '{c}",
463 .{raw_string[bad_index]},
464 );
465 },
466 .expected_single_quote => |bad_index| {
467 try p.appendErrorOff(
468 token,
469 offset + @as(u32, @intCast(bad_index)),
470 "expected single quote ('), found '{c}",
471 .{raw_string[bad_index]},
472 );
473 },
474 .invalid_character => |bad_index| {
475 try p.appendErrorOff(
476 token,
477 offset + @as(u32, @intCast(bad_index)),
478 "invalid byte in string or character literal: '{c}'",
479 .{raw_string[bad_index]},
480 );
481 },
482 }
483 }
484
485 fn fail(
486 p: *Parse,
487 tok: Ast.TokenIndex,
488 comptime fmt: []const u8,
489 args: anytype,
490 ) InnerError {
491 try appendError(p, tok, fmt, args);
492 return error.ParseFailure;
493 }
494
495 fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
496 return appendErrorOff(p, tok, 0, fmt, args);
497 }
498
499 fn appendErrorOff(
500 p: *Parse,
501 tok: Ast.TokenIndex,
502 byte_offset: u32,
503 comptime fmt: []const u8,
504 args: anytype,
505 ) Allocator.Error!void {
506 try p.errors.append(p.gpa, .{
507 .msg = try std.fmt.allocPrint(p.arena, fmt, args),
508 .tok = tok,
509 .off = byte_offset,
510 });
511 }
512};
513
514const Manifest = @This();
515const std = @import("std");
516const mem = std.mem;
517const Allocator = std.mem.Allocator;
518const assert = std.debug.assert;
519const Ast = std.zig.Ast;
520const testing = std.testing;
521
522test "basic" {
523 const gpa = testing.allocator;
524
525 const example =
526 \\.{
527 \\ .name = "foo",
528 \\ .version = "3.2.1",
529 \\ .dependencies = .{
530 \\ .bar = .{
531 \\ .url = "https://example.com/baz.tar.gz",
532 \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
533 \\ },
534 \\ },
535 \\}
536 ;
537
538 var ast = try std.zig.Ast.parse(gpa, example, .zon);
539 defer ast.deinit(gpa);
540
541 try testing.expect(ast.errors.len == 0);
542
543 var manifest = try Manifest.parse(gpa, ast);
544 defer manifest.deinit(gpa);
545
546 try testing.expectEqualStrings("foo", manifest.name);
547
548 try testing.expectEqual(@as(std.SemanticVersion, .{
549 .major = 3,
550 .minor = 2,
551 .patch = 1,
552 }), manifest.version);
553
554 try testing.expect(manifest.dependencies.count() == 1);
555 try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
556 try testing.expectEqualStrings(
557 "https://example.com/baz.tar.gz",
558 manifest.dependencies.values()[0].url,
559 );
560 try testing.expectEqualStrings(
561 "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
562 manifest.dependencies.values()[0].hash orelse return error.TestFailed,
563 );
564}
src/Package.zig+1-1
...@@ -1,7 +1,7 @@...@@ -1,7 +1,7 @@
1pub const Module = @import("Package/Module.zig");1pub const Module = @import("Package/Module.zig");
2pub const Fetch = @import("Package/Fetch.zig");2pub const Fetch = @import("Package/Fetch.zig");
3pub const build_zig_basename = "build.zig";3pub const build_zig_basename = "build.zig";
4pub const Manifest = @import("Manifest.zig");4pub const Manifest = @import("Package/Manifest.zig");
55
6pub const Path = struct {6pub const Path = struct {
7 root_dir: Cache.Directory,7 root_dir: Cache.Directory,
src/Package/Fetch.zig+2-2
...@@ -1449,9 +1449,9 @@ const Allocator = std.mem.Allocator;...@@ -1449,9 +1449,9 @@ const Allocator = std.mem.Allocator;
1449const Cache = std.Build.Cache;1449const Cache = std.Build.Cache;
1450const ThreadPool = std.Thread.Pool;1450const ThreadPool = std.Thread.Pool;
1451const WaitGroup = std.Thread.WaitGroup;1451const WaitGroup = std.Thread.WaitGroup;
1452const Manifest = @import("../Manifest.zig");
1453const Fetch = @This();1452const Fetch = @This();
1454const main = @import("../main.zig");1453const main = @import("../main.zig");
1455const git = @import("../git.zig");1454const git = @import("Fetch/git.zig");
1456const Package = @import("../Package.zig");1455const Package = @import("../Package.zig");
1456const Manifest = Package.Manifest;
1457const ErrorBundle = std.zig.ErrorBundle;1457const ErrorBundle = std.zig.ErrorBundle;
src/Package/Fetch/git.zig created+1466
...@@ -0,0 +1,1466 @@
1//! Git support for package fetching.
2//!
3//! This is not intended to support all features of Git: it is limited to the
4//! basic functionality needed to clone a repository for the purpose of fetching
5//! a package.
6
7const std = @import("std");
8const mem = std.mem;
9const testing = std.testing;
10const Allocator = mem.Allocator;
11const Sha1 = std.crypto.hash.Sha1;
12const assert = std.debug.assert;
13
14pub const oid_length = Sha1.digest_length;
15pub const fmt_oid_length = 2 * oid_length;
16/// The ID of a Git object (an SHA-1 hash).
17pub const Oid = [oid_length]u8;
18
19pub fn parseOid(s: []const u8) !Oid {
20 if (s.len != fmt_oid_length) return error.InvalidOid;
21 var oid: Oid = undefined;
22 for (&oid, 0..) |*b, i| {
23 b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
24 }
25 return oid;
26}
27
28test parseOid {
29 try testing.expectEqualSlices(
30 u8,
31 &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
32 &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"),
33 );
34 try testing.expectError(error.InvalidOid, parseOid("ce919ccf"));
35 try testing.expectError(error.InvalidOid, parseOid("master"));
36 try testing.expectError(error.InvalidOid, parseOid("HEAD"));
37}
38
39pub const Diagnostics = struct {
40 allocator: Allocator,
41 errors: std.ArrayListUnmanaged(Error) = .{},
42
43 pub const Error = union(enum) {
44 unable_to_create_sym_link: struct {
45 code: anyerror,
46 file_name: []const u8,
47 link_name: []const u8,
48 },
49 };
50
51 pub fn deinit(d: *Diagnostics) void {
52 for (d.errors.items) |item| {
53 switch (item) {
54 .unable_to_create_sym_link => |info| {
55 d.allocator.free(info.file_name);
56 d.allocator.free(info.link_name);
57 },
58 }
59 }
60 d.errors.deinit(d.allocator);
61 d.* = undefined;
62 }
63};
64
65pub const Repository = struct {
66 odb: Odb,
67
68 pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
69 return .{ .odb = try Odb.init(allocator, pack_file, index_file) };
70 }
71
72 pub fn deinit(repository: *Repository) void {
73 repository.odb.deinit();
74 repository.* = undefined;
75 }
76
77 /// Checks out the repository at `commit_oid` to `worktree`.
78 pub fn checkout(
79 repository: *Repository,
80 worktree: std.fs.Dir,
81 commit_oid: Oid,
82 diagnostics: *Diagnostics,
83 ) !void {
84 try repository.odb.seekOid(commit_oid);
85 const tree_oid = tree_oid: {
86 var commit_object = try repository.odb.readObject();
87 if (commit_object.type != .commit) return error.NotACommit;
88 break :tree_oid try getCommitTree(commit_object.data);
89 };
90 try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
91 }
92
93 /// Checks out the tree at `tree_oid` to `worktree`.
94 fn checkoutTree(
95 repository: *Repository,
96 dir: std.fs.Dir,
97 tree_oid: Oid,
98 current_path: []const u8,
99 diagnostics: *Diagnostics,
100 ) !void {
101 try repository.odb.seekOid(tree_oid);
102 const tree_object = try repository.odb.readObject();
103 if (tree_object.type != .tree) return error.NotATree;
104 // The tree object may be evicted from the object cache while we're
105 // iterating over it, so we can make a defensive copy here to make sure
106 // it remains valid until we're done with it
107 const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
108 defer repository.odb.allocator.free(tree_data);
109
110 var tree_iter: TreeIterator = .{ .data = tree_data };
111 while (try tree_iter.next()) |entry| {
112 switch (entry.type) {
113 .directory => {
114 try dir.makeDir(entry.name);
115 var subdir = try dir.openDir(entry.name, .{});
116 defer subdir.close();
117 const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
118 defer repository.odb.allocator.free(sub_path);
119 try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
120 },
121 .file => {
122 var file = try dir.createFile(entry.name, .{});
123 defer file.close();
124 try repository.odb.seekOid(entry.oid);
125 var file_object = try repository.odb.readObject();
126 if (file_object.type != .blob) return error.InvalidFile;
127 try file.writeAll(file_object.data);
128 try file.sync();
129 },
130 .symlink => {
131 try repository.odb.seekOid(entry.oid);
132 var symlink_object = try repository.odb.readObject();
133 if (symlink_object.type != .blob) return error.InvalidFile;
134 const link_name = symlink_object.data;
135 dir.symLink(link_name, entry.name, .{}) catch |e| {
136 const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
137 errdefer diagnostics.allocator.free(file_name);
138 const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
139 errdefer diagnostics.allocator.free(link_name_dup);
140 try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
141 .code = e,
142 .file_name = file_name,
143 .link_name = link_name_dup,
144 } });
145 };
146 },
147 .gitlink => {
148 // Consistent with git archive behavior, create the directory but
149 // do nothing else
150 try dir.makeDir(entry.name);
151 },
152 }
153 }
154 }
155
156 /// Returns the ID of the tree associated with the given commit (provided as
157 /// raw object data).
158 fn getCommitTree(commit_data: []const u8) !Oid {
159 if (!mem.startsWith(u8, commit_data, "tree ") or
160 commit_data.len < "tree ".len + fmt_oid_length + "\n".len or
161 commit_data["tree ".len + fmt_oid_length] != '\n')
162 {
163 return error.InvalidCommit;
164 }
165 return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]);
166 }
167
168 const TreeIterator = struct {
169 data: []const u8,
170 pos: usize = 0,
171
172 const Entry = struct {
173 type: Type,
174 executable: bool,
175 name: [:0]const u8,
176 oid: Oid,
177
178 const Type = enum(u4) {
179 directory = 0o4,
180 file = 0o10,
181 symlink = 0o12,
182 gitlink = 0o16,
183 };
184 };
185
186 fn next(iterator: *TreeIterator) !?Entry {
187 if (iterator.pos == iterator.data.len) return null;
188
189 const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
190 const mode: packed struct {
191 permission: u9,
192 unused: u3,
193 type: u4,
194 } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
195 const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree;
196 const executable = switch (mode.permission) {
197 0 => if (@"type" == .file) return error.InvalidTree else false,
198 0o644 => if (@"type" != .file) return error.InvalidTree else false,
199 0o755 => if (@"type" != .file) return error.InvalidTree else true,
200 else => return error.InvalidTree,
201 };
202 iterator.pos = mode_end + 1;
203
204 const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
205 const name = iterator.data[iterator.pos..name_end :0];
206 iterator.pos = name_end + 1;
207
208 if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
209 const oid = iterator.data[iterator.pos..][0..oid_length].*;
210 iterator.pos += oid_length;
211
212 return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
213 }
214 };
215};
216
217/// A Git object database backed by a packfile. A packfile index is also used
218/// for efficient access to objects in the packfile.
219///
220/// The format of the packfile and its associated index are documented in
221/// [pack-format](https://git-scm.com/docs/pack-format).
222const Odb = struct {
223 pack_file: std.fs.File,
224 index_header: IndexHeader,
225 index_file: std.fs.File,
226 cache: ObjectCache = .{},
227 allocator: Allocator,
228
229 /// Initializes the database from open pack and index files.
230 fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
231 try pack_file.seekTo(0);
232 try index_file.seekTo(0);
233 const index_header = try IndexHeader.read(index_file.reader());
234 return .{
235 .pack_file = pack_file,
236 .index_header = index_header,
237 .index_file = index_file,
238 .allocator = allocator,
239 };
240 }
241
242 fn deinit(odb: *Odb) void {
243 odb.cache.deinit(odb.allocator);
244 odb.* = undefined;
245 }
246
247 /// Reads the object at the current position in the database.
248 fn readObject(odb: *Odb) !Object {
249 var base_offset = try odb.pack_file.getPos();
250 var base_header: EntryHeader = undefined;
251 var delta_offsets = std.ArrayListUnmanaged(u64){};
252 defer delta_offsets.deinit(odb.allocator);
253 const base_object = while (true) {
254 if (odb.cache.get(base_offset)) |base_object| break base_object;
255
256 base_header = try EntryHeader.read(odb.pack_file.reader());
257 switch (base_header) {
258 .ofs_delta => |ofs_delta| {
259 try delta_offsets.append(odb.allocator, base_offset);
260 base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
261 try odb.pack_file.seekTo(base_offset);
262 },
263 .ref_delta => |ref_delta| {
264 try delta_offsets.append(odb.allocator, base_offset);
265 try odb.seekOid(ref_delta.base_object);
266 base_offset = try odb.pack_file.getPos();
267 },
268 else => {
269 const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength());
270 errdefer odb.allocator.free(base_data);
271 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
272 try odb.cache.put(odb.allocator, base_offset, base_object);
273 break base_object;
274 },
275 }
276 };
277
278 const base_data = try resolveDeltaChain(
279 odb.allocator,
280 odb.pack_file,
281 base_object,
282 delta_offsets.items,
283 &odb.cache,
284 );
285
286 return .{ .type = base_object.type, .data = base_data };
287 }
288
289 /// Seeks to the beginning of the object with the given ID.
290 fn seekOid(odb: *Odb, oid: Oid) !void {
291 const key = oid[0];
292 var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
293 var end_index = odb.index_header.fan_out_table[key];
294 const found_index = while (start_index < end_index) {
295 const mid_index = start_index + (end_index - start_index) / 2;
296 try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
297 const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length);
298 switch (mem.order(u8, &mid_oid, &oid)) {
299 .lt => start_index = mid_index + 1,
300 .gt => end_index = mid_index,
301 .eq => break mid_index,
302 }
303 } else return error.ObjectNotFound;
304
305 const n_objects = odb.index_header.fan_out_table[255];
306 const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
307 try odb.index_file.seekTo(offset_values_start + found_index * 4);
308 const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32));
309 const pack_offset = pack_offset: {
310 if (l1_offset.big) {
311 const l2_offset_values_start = offset_values_start + n_objects * 4;
312 try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
313 break :pack_offset try odb.index_file.reader().readIntBig(u64);
314 } else {
315 break :pack_offset l1_offset.value;
316 }
317 };
318
319 try odb.pack_file.seekTo(pack_offset);
320 }
321};
322
323const Object = struct {
324 type: Type,
325 data: []const u8,
326
327 const Type = enum {
328 commit,
329 tree,
330 blob,
331 tag,
332 };
333};
334
335/// A cache for object data.
336///
337/// The purpose of this cache is to speed up resolution of deltas by caching the
338/// results of resolving delta objects, while maintaining a maximum cache size
339/// to avoid excessive memory usage. If the total size of the objects in the
340/// cache exceeds the maximum, the cache will begin evicting the least recently
341/// used objects: when resolving delta chains, the most recently used objects
342/// will likely be more helpful as they will be further along in the chain
343/// (skipping earlier reconstruction steps).
344///
345/// Object data stored in the cache is managed by the cache. It should not be
346/// freed by the caller at any point after inserting it into the cache. Any
347/// objects remaining in the cache will be freed when the cache itself is freed.
348const ObjectCache = struct {
349 objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{},
350 lru_nodes: LruList = .{},
351 byte_size: usize = 0,
352
353 const max_byte_size = 128 * 1024 * 1024; // 128MiB
354 /// A list of offsets stored in the cache, with the most recently used
355 /// entries at the end.
356 const LruList = std.DoublyLinkedList(u64);
357 const CacheEntry = struct { object: Object, lru_node: *LruList.Node };
358
359 fn deinit(cache: *ObjectCache, allocator: Allocator) void {
360 var object_iterator = cache.objects.iterator();
361 while (object_iterator.next()) |object| {
362 allocator.free(object.value_ptr.object.data);
363 allocator.destroy(object.value_ptr.lru_node);
364 }
365 cache.objects.deinit(allocator);
366 cache.* = undefined;
367 }
368
369 /// Gets an object from the cache, moving it to the most recently used
370 /// position if it is present.
371 fn get(cache: *ObjectCache, offset: u64) ?Object {
372 if (cache.objects.get(offset)) |entry| {
373 cache.lru_nodes.remove(entry.lru_node);
374 cache.lru_nodes.append(entry.lru_node);
375 return entry.object;
376 } else {
377 return null;
378 }
379 }
380
381 /// Puts an object in the cache, possibly evicting older entries if the
382 /// cache exceeds its maximum size. Note that, although old objects may
383 /// be evicted, the object just added to the cache with this function
384 /// will not be evicted before the next call to `put` or `deinit` even if
385 /// it exceeds the maximum cache size.
386 fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
387 const lru_node = try allocator.create(LruList.Node);
388 errdefer allocator.destroy(lru_node);
389 lru_node.data = offset;
390
391 const gop = try cache.objects.getOrPut(allocator, offset);
392 if (gop.found_existing) {
393 cache.byte_size -= gop.value_ptr.object.data.len;
394 cache.lru_nodes.remove(gop.value_ptr.lru_node);
395 allocator.destroy(gop.value_ptr.lru_node);
396 allocator.free(gop.value_ptr.object.data);
397 }
398 gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
399 cache.byte_size += object.data.len;
400 cache.lru_nodes.append(lru_node);
401
402 while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) {
403 // The > 1 check is to make sure that we don't evict the most
404 // recently added node, even if it by itself happens to exceed the
405 // maximum size of the cache.
406 const evict_node = cache.lru_nodes.popFirst().?;
407 const evict_offset = evict_node.data;
408 allocator.destroy(evict_node);
409 const evict_object = cache.objects.get(evict_offset).?.object;
410 cache.byte_size -= evict_object.data.len;
411 allocator.free(evict_object.data);
412 _ = cache.objects.remove(evict_offset);
413 }
414 }
415};
416
417/// A single pkt-line in the Git protocol.
418///
419/// The format of a pkt-line is documented in
420/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
421/// meanings of the delimiter and response-end packets are documented in
422/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
423const Packet = union(enum) {
424 flush,
425 delimiter,
426 response_end,
427 data: []const u8,
428
429 const max_data_length = 65516;
430
431 /// Reads a packet in pkt-line format.
432 fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {
433 const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;
434 switch (length) {
435 0 => return .flush,
436 1 => return .delimiter,
437 2 => return .response_end,
438 3 => return error.InvalidPacket,
439 else => if (length - 4 > max_data_length) return error.InvalidPacket,
440 }
441 const data = buf[0 .. length - 4];
442 try reader.readNoEof(data);
443 return .{ .data = data };
444 }
445
446 /// Writes a packet in pkt-line format.
447 fn write(packet: Packet, writer: anytype) !void {
448 switch (packet) {
449 .flush => try writer.writeAll("0000"),
450 .delimiter => try writer.writeAll("0001"),
451 .response_end => try writer.writeAll("0002"),
452 .data => |data| {
453 assert(data.len <= max_data_length);
454 try writer.print("{x:0>4}", .{data.len + 4});
455 try writer.writeAll(data);
456 },
457 }
458 }
459};
460
461/// A client session for the Git protocol, currently limited to an HTTP(S)
462/// transport. Only protocol version 2 is supported, as documented in
463/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
464pub const Session = struct {
465 transport: *std.http.Client,
466 uri: std.Uri,
467 supports_agent: bool = false,
468 supports_shallow: bool = false,
469
470 const agent = "zig/" ++ @import("builtin").zig_version_string;
471 const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
472
473 /// Discovers server capabilities. This should be called before using any
474 /// other client functionality, or the client will be forced to default to
475 /// the bare minimum server requirements, which may be considerably less
476 /// efficient (e.g. no shallow fetches).
477 ///
478 /// See the note on `getCapabilities` regarding `redirect_uri`.
479 pub fn discoverCapabilities(
480 session: *Session,
481 allocator: Allocator,
482 redirect_uri: *[]u8,
483 ) !void {
484 var capability_iterator = try session.getCapabilities(allocator, redirect_uri);
485 defer capability_iterator.deinit();
486 while (try capability_iterator.next()) |capability| {
487 if (mem.eql(u8, capability.key, "agent")) {
488 session.supports_agent = true;
489 } else if (mem.eql(u8, capability.key, "fetch")) {
490 var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
491 while (feature_iterator.next()) |feature| {
492 if (mem.eql(u8, feature, "shallow")) {
493 session.supports_shallow = true;
494 }
495 }
496 }
497 }
498 }
499
500 /// Returns an iterator over capabilities supported by the server.
501 ///
502 /// If the server redirects the request, `error.Redirected` is returned and
503 /// `redirect_uri` is populated with the URI resulting from the redirects.
504 /// When this occurs, the value of `redirect_uri` must be freed with
505 /// `allocator` when the caller is done with it.
506 fn getCapabilities(
507 session: Session,
508 allocator: Allocator,
509 redirect_uri: *[]u8,
510 ) !CapabilityIterator {
511 var info_refs_uri = session.uri;
512 info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
513 defer allocator.free(info_refs_uri.path);
514 info_refs_uri.query = "service=git-upload-pack";
515 info_refs_uri.fragment = null;
516
517 var headers = std.http.Headers.init(allocator);
518 defer headers.deinit();
519 try headers.append("Git-Protocol", "version=2");
520
521 var request = try session.transport.request(.GET, info_refs_uri, headers, .{
522 .max_redirects = 3,
523 });
524 errdefer request.deinit();
525 try request.start(.{});
526 try request.finish();
527
528 try request.wait();
529 if (request.response.status != .ok) return error.ProtocolError;
530 if (request.redirects_left < 3) {
531 if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;
532 var new_uri = request.uri;
533 new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];
534 new_uri.query = null;
535 redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri});
536 return error.Redirected;
537 }
538
539 const reader = request.reader();
540 var buf: [Packet.max_data_length]u8 = undefined;
541 var state: enum { response_start, response_content } = .response_start;
542 while (true) {
543 // Some Git servers (at least GitHub) include an additional
544 // '# service=git-upload-pack' informative response before sending
545 // the expected 'version 2' packet and capability information.
546 // This is not universal: SourceHut, for example, does not do this.
547 // Thus, we need to skip any such useless additional responses
548 // before we get the one we're actually looking for. The responses
549 // will be delimited by flush packets.
550 const packet = Packet.read(reader, &buf) catch |e| switch (e) {
551 error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
552 else => |other| return other,
553 };
554 switch (packet) {
555 .flush => state = .response_start,
556 .data => |data| switch (state) {
557 .response_start => if (mem.eql(u8, data, "version 2\n")) {
558 return .{ .request = request };
559 } else {
560 state = .response_content;
561 },
562 else => {},
563 },
564 else => return error.UnexpectedPacket,
565 }
566 }
567 }
568
569 const CapabilityIterator = struct {
570 request: std.http.Client.Request,
571 buf: [Packet.max_data_length]u8 = undefined,
572
573 const Capability = struct {
574 key: []const u8,
575 value: ?[]const u8 = null,
576 };
577
578 fn deinit(iterator: *CapabilityIterator) void {
579 iterator.request.deinit();
580 iterator.* = undefined;
581 }
582
583 fn next(iterator: *CapabilityIterator) !?Capability {
584 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
585 .flush => return null,
586 .data => |data| if (data.len > 0 and data[data.len - 1] == '\n') {
587 if (mem.indexOfScalar(u8, data, '=')) |separator_pos| {
588 return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] };
589 } else {
590 return .{ .key = data[0 .. data.len - 1] };
591 }
592 } else return error.UnexpectedPacket,
593 else => return error.UnexpectedPacket,
594 }
595 }
596 };
597
598 const ListRefsOptions = struct {
599 /// The ref prefixes (if any) to use to filter the refs available on the
600 /// server. Note that the client must still check the returned refs
601 /// against its desired filters itself: the server is not required to
602 /// respect these prefix filters and may return other refs as well.
603 ref_prefixes: []const []const u8 = &.{},
604 /// Whether to include symref targets for returned symbolic refs.
605 include_symrefs: bool = false,
606 /// Whether to include the peeled object ID for returned tag refs.
607 include_peeled: bool = false,
608 };
609
610 /// Returns an iterator over refs known to the server.
611 pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator {
612 var upload_pack_uri = session.uri;
613 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
614 defer allocator.free(upload_pack_uri.path);
615 upload_pack_uri.query = null;
616 upload_pack_uri.fragment = null;
617
618 var headers = std.http.Headers.init(allocator);
619 defer headers.deinit();
620 try headers.append("Content-Type", "application/x-git-upload-pack-request");
621 try headers.append("Git-Protocol", "version=2");
622
623 var body = std.ArrayListUnmanaged(u8){};
624 defer body.deinit(allocator);
625 const body_writer = body.writer(allocator);
626 try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer);
627 if (session.supports_agent) {
628 try Packet.write(.{ .data = agent_capability }, body_writer);
629 }
630 try Packet.write(.delimiter, body_writer);
631 for (options.ref_prefixes) |ref_prefix| {
632 const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix});
633 defer allocator.free(ref_prefix_packet);
634 try Packet.write(.{ .data = ref_prefix_packet }, body_writer);
635 }
636 if (options.include_symrefs) {
637 try Packet.write(.{ .data = "symrefs\n" }, body_writer);
638 }
639 if (options.include_peeled) {
640 try Packet.write(.{ .data = "peel\n" }, body_writer);
641 }
642 try Packet.write(.flush, body_writer);
643
644 var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
645 .handle_redirects = false,
646 });
647 errdefer request.deinit();
648 request.transfer_encoding = .{ .content_length = body.items.len };
649 try request.start(.{});
650 try request.writeAll(body.items);
651 try request.finish();
652
653 try request.wait();
654 if (request.response.status != .ok) return error.ProtocolError;
655
656 return .{ .request = request };
657 }
658
659 pub const RefIterator = struct {
660 request: std.http.Client.Request,
661 buf: [Packet.max_data_length]u8 = undefined,
662
663 pub const Ref = struct {
664 oid: Oid,
665 name: []const u8,
666 symref_target: ?[]const u8,
667 peeled: ?Oid,
668 };
669
670 pub fn deinit(iterator: *RefIterator) void {
671 iterator.request.deinit();
672 iterator.* = undefined;
673 }
674
675 pub fn next(iterator: *RefIterator) !?Ref {
676 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
677 .flush => return null,
678 .data => |data| {
679 const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket;
680 const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
681
682 const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
683 const name = data[oid_sep_pos + 1 .. name_sep_pos];
684
685 var symref_target: ?[]const u8 = null;
686 var peeled: ?Oid = null;
687 var last_sep_pos = name_sep_pos;
688 while (data[last_sep_pos] == ' ') {
689 const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
690 const attribute = data[last_sep_pos + 1 .. next_sep_pos];
691 if (mem.startsWith(u8, attribute, "symref-target:")) {
692 symref_target = attribute["symref-target:".len..];
693 } else if (mem.startsWith(u8, attribute, "peeled:")) {
694 peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket;
695 }
696 last_sep_pos = next_sep_pos;
697 }
698
699 return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
700 },
701 else => return error.UnexpectedPacket,
702 }
703 }
704 };
705
706 /// Fetches the given refs from the server. A shallow fetch (depth 1) is
707 /// performed if the server supports it.
708 pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {
709 var upload_pack_uri = session.uri;
710 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
711 defer allocator.free(upload_pack_uri.path);
712 upload_pack_uri.query = null;
713 upload_pack_uri.fragment = null;
714
715 var headers = std.http.Headers.init(allocator);
716 defer headers.deinit();
717 try headers.append("Content-Type", "application/x-git-upload-pack-request");
718 try headers.append("Git-Protocol", "version=2");
719
720 var body = std.ArrayListUnmanaged(u8){};
721 defer body.deinit(allocator);
722 const body_writer = body.writer(allocator);
723 try Packet.write(.{ .data = "command=fetch\n" }, body_writer);
724 if (session.supports_agent) {
725 try Packet.write(.{ .data = agent_capability }, body_writer);
726 }
727 try Packet.write(.delimiter, body_writer);
728 // Our packfile parser supports the OFS_DELTA object type
729 try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);
730 // We do not currently convey server progress information to the user
731 try Packet.write(.{ .data = "no-progress\n" }, body_writer);
732 if (session.supports_shallow) {
733 try Packet.write(.{ .data = "deepen 1\n" }, body_writer);
734 }
735 for (wants) |want| {
736 var buf: [Packet.max_data_length]u8 = undefined;
737 const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
738 try Packet.write(.{ .data = arg }, body_writer);
739 }
740 try Packet.write(.{ .data = "done\n" }, body_writer);
741 try Packet.write(.flush, body_writer);
742
743 var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
744 .handle_redirects = false,
745 });
746 errdefer request.deinit();
747 request.transfer_encoding = .{ .content_length = body.items.len };
748 try request.start(.{});
749 try request.writeAll(body.items);
750 try request.finish();
751
752 try request.wait();
753 if (request.response.status != .ok) return error.ProtocolError;
754
755 const reader = request.reader();
756 // We are not interested in any of the sections of the returned fetch
757 // data other than the packfile section, since we aren't doing anything
758 // complex like ref negotiation (this is a fresh clone).
759 var state: enum { section_start, section_content } = .section_start;
760 while (true) {
761 var buf: [Packet.max_data_length]u8 = undefined;
762 const packet = try Packet.read(reader, &buf);
763 switch (state) {
764 .section_start => switch (packet) {
765 .data => |data| if (mem.eql(u8, data, "packfile\n")) {
766 return .{ .request = request };
767 } else {
768 state = .section_content;
769 },
770 else => return error.UnexpectedPacket,
771 },
772 .section_content => switch (packet) {
773 .delimiter => state = .section_start,
774 .data => {},
775 else => return error.UnexpectedPacket,
776 },
777 }
778 }
779 }
780
781 pub const FetchStream = struct {
782 request: std.http.Client.Request,
783 buf: [Packet.max_data_length]u8 = undefined,
784 pos: usize = 0,
785 len: usize = 0,
786
787 pub fn deinit(stream: *FetchStream) void {
788 stream.request.deinit();
789 }
790
791 pub const ReadError = std.http.Client.Request.ReadError || error{
792 InvalidPacket,
793 ProtocolError,
794 UnexpectedPacket,
795 };
796 pub const Reader = std.io.Reader(*FetchStream, ReadError, read);
797
798 const StreamCode = enum(u8) {
799 pack_data = 1,
800 progress = 2,
801 fatal_error = 3,
802 _,
803 };
804
805 pub fn reader(stream: *FetchStream) Reader {
806 return .{ .context = stream };
807 }
808
809 pub fn read(stream: *FetchStream, buf: []u8) !usize {
810 if (stream.pos == stream.len) {
811 while (true) {
812 switch (try Packet.read(stream.request.reader(), &stream.buf)) {
813 .flush => return 0,
814 .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {
815 .pack_data => {
816 stream.pos = 1;
817 stream.len = data.len;
818 break;
819 },
820 .fatal_error => return error.ProtocolError,
821 else => {},
822 },
823 else => return error.UnexpectedPacket,
824 }
825 }
826 }
827
828 const size = @min(buf.len, stream.len - stream.pos);
829 @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]);
830 stream.pos += size;
831 return size;
832 }
833 };
834};
835
836const PackHeader = struct {
837 total_objects: u32,
838
839 const signature = "PACK";
840 const supported_version = 2;
841
842 fn read(reader: anytype) !PackHeader {
843 const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) {
844 error.EndOfStream => return error.InvalidHeader,
845 else => |other| return other,
846 };
847 if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
848 const version = reader.readIntBig(u32) catch |e| switch (e) {
849 error.EndOfStream => return error.InvalidHeader,
850 else => |other| return other,
851 };
852 if (version != supported_version) return error.UnsupportedVersion;
853 const total_objects = reader.readIntBig(u32) catch |e| switch (e) {
854 error.EndOfStream => return error.InvalidHeader,
855 else => |other| return other,
856 };
857 return .{ .total_objects = total_objects };
858 }
859};
860
861const EntryHeader = union(Type) {
862 commit: Undeltified,
863 tree: Undeltified,
864 blob: Undeltified,
865 tag: Undeltified,
866 ofs_delta: OfsDelta,
867 ref_delta: RefDelta,
868
869 const Type = enum(u3) {
870 commit = 1,
871 tree = 2,
872 blob = 3,
873 tag = 4,
874 ofs_delta = 6,
875 ref_delta = 7,
876 };
877
878 const Undeltified = struct {
879 uncompressed_length: u64,
880 };
881
882 const OfsDelta = struct {
883 offset: u64,
884 uncompressed_length: u64,
885 };
886
887 const RefDelta = struct {
888 base_object: Oid,
889 uncompressed_length: u64,
890 };
891
892 fn objectType(header: EntryHeader) Object.Type {
893 return switch (header) {
894 inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)),
895 else => unreachable,
896 };
897 }
898
899 fn uncompressedLength(header: EntryHeader) u64 {
900 return switch (header) {
901 inline else => |entry| entry.uncompressed_length,
902 };
903 }
904
905 fn read(reader: anytype) !EntryHeader {
906 const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
907 const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) {
908 error.EndOfStream => return error.InvalidFormat,
909 else => |other| return other,
910 });
911 const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
912 var uncompressed_length: u64 = initial.len;
913 uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
914 const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat;
915 return switch (@"type") {
916 inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{
917 .uncompressed_length = uncompressed_length,
918 }),
919 .ofs_delta => .{ .ofs_delta = .{
920 .offset = try readOffsetVarInt(reader),
921 .uncompressed_length = uncompressed_length,
922 } },
923 .ref_delta => .{ .ref_delta = .{
924 .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) {
925 error.EndOfStream => return error.InvalidFormat,
926 else => |other| return other,
927 },
928 .uncompressed_length = uncompressed_length,
929 } },
930 };
931 }
932};
933
934fn readSizeVarInt(r: anytype) !u64 {
935 const Byte = packed struct { value: u7, has_next: bool };
936 var b: Byte = @bitCast(try r.readByte());
937 var value: u64 = b.value;
938 var shift: u6 = 0;
939 while (b.has_next) {
940 b = @bitCast(try r.readByte());
941 shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
942 value |= @as(u64, b.value) << shift;
943 }
944 return value;
945}
946
947fn readOffsetVarInt(r: anytype) !u64 {
948 const Byte = packed struct { value: u7, has_next: bool };
949 var b: Byte = @bitCast(try r.readByte());
950 var value: u64 = b.value;
951 while (b.has_next) {
952 b = @bitCast(try r.readByte());
953 value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
954 value |= b.value;
955 }
956 return value;
957}
958
959const IndexHeader = struct {
960 fan_out_table: [256]u32,
961
962 const signature = "\xFFtOc";
963 const supported_version = 2;
964 const size = 4 + 4 + @sizeOf([256]u32);
965
966 fn read(reader: anytype) !IndexHeader {
967 var header_bytes = try reader.readBytesNoEof(size);
968 if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
969 const version = mem.readIntBig(u32, header_bytes[4..8]);
970 if (version != supported_version) return error.UnsupportedVersion;
971
972 var fan_out_table: [256]u32 = undefined;
973 var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
974 const fan_out_table_reader = fan_out_table_stream.reader();
975 for (&fan_out_table) |*entry| {
976 entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable;
977 }
978 return .{ .fan_out_table = fan_out_table };
979 }
980};
981
982const IndexEntry = struct {
983 offset: u64,
984 crc32: u32,
985};
986
987/// Writes out a version 2 index for the given packfile, as documented in
988/// [pack-format](https://git-scm.com/docs/pack-format).
989pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void {
990 try pack.seekTo(0);
991
992 var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){};
993 defer index_entries.deinit(allocator);
994 var pending_deltas = std.ArrayListUnmanaged(IndexEntry){};
995 defer pending_deltas.deinit(allocator);
996
997 const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas);
998
999 var cache: ObjectCache = .{};
1000 defer cache.deinit(allocator);
1001 var remaining_deltas = pending_deltas.items.len;
1002 while (remaining_deltas > 0) {
1003 var i: usize = remaining_deltas;
1004 while (i > 0) {
1005 i -= 1;
1006 const delta = pending_deltas.items[i];
1007 if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| {
1008 try index_entries.put(allocator, oid, delta);
1009 _ = pending_deltas.swapRemove(i);
1010 }
1011 }
1012 if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
1013 remaining_deltas = pending_deltas.items.len;
1014 }
1015
1016 var oids = std.ArrayListUnmanaged(Oid){};
1017 defer oids.deinit(allocator);
1018 try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
1019 var index_entries_iter = index_entries.iterator();
1020 while (index_entries_iter.next()) |entry| {
1021 oids.appendAssumeCapacity(entry.key_ptr.*);
1022 }
1023 mem.sortUnstable(Oid, oids.items, {}, struct {
1024 fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1025 return mem.lessThan(u8, &o1, &o2);
1026 }
1027 }.lessThan);
1028
1029 var fan_out_table: [256]u32 = undefined;
1030 var count: u32 = 0;
1031 var fan_out_index: u8 = 0;
1032 for (oids.items) |oid| {
1033 if (oid[0] > fan_out_index) {
1034 @memset(fan_out_table[fan_out_index..oid[0]], count);
1035 fan_out_index = oid[0];
1036 }
1037 count += 1;
1038 }
1039 @memset(fan_out_table[fan_out_index..], count);
1040
1041 var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{}));
1042 const writer = index_hashed_writer.writer();
1043 try writer.writeAll(IndexHeader.signature);
1044 try writer.writeIntBig(u32, IndexHeader.supported_version);
1045 for (fan_out_table) |fan_out_entry| {
1046 try writer.writeIntBig(u32, fan_out_entry);
1047 }
1048
1049 for (oids.items) |oid| {
1050 try writer.writeAll(&oid);
1051 }
1052
1053 for (oids.items) |oid| {
1054 try writer.writeIntBig(u32, index_entries.get(oid).?.crc32);
1055 }
1056
1057 var big_offsets = std.ArrayListUnmanaged(u64){};
1058 defer big_offsets.deinit(allocator);
1059 for (oids.items) |oid| {
1060 const offset = index_entries.get(oid).?.offset;
1061 if (offset <= std.math.maxInt(u31)) {
1062 try writer.writeIntBig(u32, @intCast(offset));
1063 } else {
1064 const index = big_offsets.items.len;
1065 try big_offsets.append(allocator, offset);
1066 try writer.writeIntBig(u32, @as(u32, @intCast(index)) | (1 << 31));
1067 }
1068 }
1069 for (big_offsets.items) |offset| {
1070 try writer.writeIntBig(u64, offset);
1071 }
1072
1073 try writer.writeAll(&pack_checksum);
1074 const index_checksum = index_hashed_writer.hasher.finalResult();
1075 try index_writer.writeAll(&index_checksum);
1076}
1077
1078/// Performs the first pass over the packfile data for index construction.
1079/// This will index all non-delta objects, queue delta objects for further
1080/// processing, and return the pack checksum (which is part of the index
1081/// format).
1082fn indexPackFirstPass(
1083 allocator: Allocator,
1084 pack: std.fs.File,
1085 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1086 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1087) ![Sha1.digest_length]u8 {
1088 var pack_buffered_reader = std.io.bufferedReader(pack.reader());
1089 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1090 var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{}));
1091 const pack_reader = pack_hashed_reader.reader();
1092
1093 const pack_header = try PackHeader.read(pack_reader);
1094
1095 var current_entry: u32 = 0;
1096 while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1097 const entry_offset = pack_counting_reader.bytes_read;
1098 var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
1099 const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
1100 switch (entry_header) {
1101 inline .commit, .tree, .blob, .tag => |object, tag| {
1102 var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1103 defer entry_decompress_stream.deinit();
1104 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1105 var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1106 const entry_writer = entry_hashed_writer.writer();
1107 // The object header is not included in the pack data but is
1108 // part of the object's ID
1109 try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length });
1110 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1111 try fifo.pump(entry_counting_reader.reader(), entry_writer);
1112 if (entry_counting_reader.bytes_read != object.uncompressed_length) {
1113 return error.InvalidObject;
1114 }
1115 const oid = entry_hashed_writer.hasher.finalResult();
1116 try index_entries.put(allocator, oid, .{
1117 .offset = entry_offset,
1118 .crc32 = entry_crc32_reader.hasher.final(),
1119 });
1120 },
1121 inline .ofs_delta, .ref_delta => |delta| {
1122 var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1123 defer entry_decompress_stream.deinit();
1124 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1125 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1126 try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1127 if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1128 return error.InvalidObject;
1129 }
1130 try pending_deltas.append(allocator, .{
1131 .offset = entry_offset,
1132 .crc32 = entry_crc32_reader.hasher.final(),
1133 });
1134 },
1135 }
1136 }
1137
1138 const pack_checksum = pack_hashed_reader.hasher.finalResult();
1139 const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length);
1140 if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {
1141 return error.CorruptedPack;
1142 }
1143 _ = pack_buffered_reader.reader().readByte() catch |e| switch (e) {
1144 error.EndOfStream => return pack_checksum,
1145 else => |other| return other,
1146 };
1147 return error.InvalidFormat;
1148}
1149
1150/// Attempts to determine the final object ID of the given deltified object.
1151/// May return null if this is not yet possible (if the delta is a ref-based
1152/// delta and we do not yet know the offset of the base object).
1153fn indexPackHashDelta(
1154 allocator: Allocator,
1155 pack: std.fs.File,
1156 delta: IndexEntry,
1157 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1158 cache: *ObjectCache,
1159) !?Oid {
1160 // Figure out the chain of deltas to resolve
1161 var base_offset = delta.offset;
1162 var base_header: EntryHeader = undefined;
1163 var delta_offsets = std.ArrayListUnmanaged(u64){};
1164 defer delta_offsets.deinit(allocator);
1165 const base_object = while (true) {
1166 if (cache.get(base_offset)) |base_object| break base_object;
1167
1168 try pack.seekTo(base_offset);
1169 base_header = try EntryHeader.read(pack.reader());
1170 switch (base_header) {
1171 .ofs_delta => |ofs_delta| {
1172 try delta_offsets.append(allocator, base_offset);
1173 base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
1174 },
1175 .ref_delta => |ref_delta| {
1176 try delta_offsets.append(allocator, base_offset);
1177 base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1178 },
1179 else => {
1180 const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength());
1181 errdefer allocator.free(base_data);
1182 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1183 try cache.put(allocator, base_offset, base_object);
1184 break base_object;
1185 },
1186 }
1187 };
1188
1189 const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache);
1190
1191 var entry_hasher = Sha1.init(.{});
1192 var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1193 try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1194 entry_hasher.update(base_data);
1195 return entry_hasher.finalResult();
1196}
1197
1198/// Resolves a chain of deltas, returning the final base object data. `pack` is
1199/// assumed to be looking at the start of the object data for the base object of
1200/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
1201/// to obtain the final object.
1202fn resolveDeltaChain(
1203 allocator: Allocator,
1204 pack: std.fs.File,
1205 base_object: Object,
1206 delta_offsets: []const u64,
1207 cache: *ObjectCache,
1208) ![]const u8 {
1209 var base_data = base_object.data;
1210 var i: usize = delta_offsets.len;
1211 while (i > 0) {
1212 i -= 1;
1213
1214 const delta_offset = delta_offsets[i];
1215 try pack.seekTo(delta_offset);
1216 const delta_header = try EntryHeader.read(pack.reader());
1217 var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
1218 defer allocator.free(delta_data);
1219 var delta_stream = std.io.fixedBufferStream(delta_data);
1220 const delta_reader = delta_stream.reader();
1221 _ = try readSizeVarInt(delta_reader); // base object size
1222 const expanded_size = try readSizeVarInt(delta_reader);
1223
1224 const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1225 var expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1226 errdefer allocator.free(expanded_data);
1227 var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
1228 var base_stream = std.io.fixedBufferStream(base_data);
1229 try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
1230 if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
1231
1232 try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1233 base_data = expanded_data;
1234 }
1235 return base_data;
1236}
1237
1238/// Reads the complete contents of an object from `reader`. This function may
1239/// read more bytes than required from `reader`, so the reader position after
1240/// returning is not reliable.
1241fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
1242 const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1243 var buffered_reader = std.io.bufferedReader(reader);
1244 var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());
1245 defer decompress_stream.deinit();
1246 var data = try allocator.alloc(u8, alloc_size);
1247 errdefer allocator.free(data);
1248 try decompress_stream.reader().readNoEof(data);
1249 _ = decompress_stream.reader().readByte() catch |e| switch (e) {
1250 error.EndOfStream => return data,
1251 else => |other| return other,
1252 };
1253 return error.InvalidFormat;
1254}
1255
1256/// Expands delta data from `delta_reader` to `writer`. `base_object` must
1257/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
1258///
1259/// The format of the delta data is documented in
1260/// [pack-format](https://git-scm.com/docs/pack-format).
1261fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {
1262 while (true) {
1263 const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) {
1264 error.EndOfStream => return,
1265 else => |other| return other,
1266 });
1267 if (inst.copy) {
1268 const available: packed struct {
1269 offset1: bool,
1270 offset2: bool,
1271 offset3: bool,
1272 offset4: bool,
1273 size1: bool,
1274 size2: bool,
1275 size3: bool,
1276 } = @bitCast(inst.value);
1277 var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1278 .offset1 = if (available.offset1) try delta_reader.readByte() else 0,
1279 .offset2 = if (available.offset2) try delta_reader.readByte() else 0,
1280 .offset3 = if (available.offset3) try delta_reader.readByte() else 0,
1281 .offset4 = if (available.offset4) try delta_reader.readByte() else 0,
1282 };
1283 const offset: u32 = @bitCast(offset_parts);
1284 var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1285 .size1 = if (available.size1) try delta_reader.readByte() else 0,
1286 .size2 = if (available.size2) try delta_reader.readByte() else 0,
1287 .size3 = if (available.size3) try delta_reader.readByte() else 0,
1288 };
1289 var size: u24 = @bitCast(size_parts);
1290 if (size == 0) size = 0x10000;
1291 try base_object.seekTo(offset);
1292 var copy_reader = std.io.limitedReader(base_object.reader(), size);
1293 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1294 try fifo.pump(copy_reader.reader(), writer);
1295 } else if (inst.value != 0) {
1296 var data_reader = std.io.limitedReader(delta_reader, inst.value);
1297 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1298 try fifo.pump(data_reader.reader(), writer);
1299 } else {
1300 return error.InvalidDeltaInstruction;
1301 }
1302 }
1303}
1304
1305fn HashedWriter(
1306 comptime WriterType: anytype,
1307 comptime HasherType: anytype,
1308) type {
1309 return struct {
1310 child_writer: WriterType,
1311 hasher: HasherType,
1312
1313 const Error = WriterType.Error;
1314 const Writer = std.io.Writer(*@This(), Error, write);
1315
1316 fn write(hashed_writer: *@This(), buf: []const u8) Error!usize {
1317 const amt = try hashed_writer.child_writer.write(buf);
1318 hashed_writer.hasher.update(buf);
1319 return amt;
1320 }
1321
1322 fn writer(hashed_writer: *@This()) Writer {
1323 return .{ .context = hashed_writer };
1324 }
1325 };
1326}
1327
1328fn hashedWriter(
1329 writer: anytype,
1330 hasher: anytype,
1331) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1332 return .{ .child_writer = writer, .hasher = hasher };
1333}
1334
1335test "packfile indexing and checkout" {
1336 // To verify the contents of this packfile without using the code in this
1337 // file:
1338 //
1339 // 1. Create a new empty Git repository (`git init`)
1340 // 2. `git unpack-objects <path/to/testdata.pack`
1341 // 3. `git fsck` -> note the "dangling commit" ID (which matches the commit
1342 // checked out below)
1343 // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb`
1344 const testrepo_pack = @embedFile("git/testdata/testrepo.pack");
1345
1346 var git_dir = testing.tmpDir(.{});
1347 defer git_dir.cleanup();
1348 var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true });
1349 defer pack_file.close();
1350 try pack_file.writeAll(testrepo_pack);
1351
1352 var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1353 defer index_file.close();
1354 try indexPack(testing.allocator, pack_file, index_file.writer());
1355
1356 // Arbitrary size limit on files read while checking the repository contents
1357 // (all files in the test repo are known to be much smaller than this)
1358 const max_file_size = 4096;
1359
1360 const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1361 defer testing.allocator.free(index_file_data);
1362 // testrepo.idx is generated by Git. The index created by this file should
1363 // match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1364 // this.
1365 const testrepo_idx = @embedFile("git/testdata/testrepo.idx");
1366 try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
1367
1368 var repository = try Repository.init(testing.allocator, pack_file, index_file);
1369 defer repository.deinit();
1370
1371 var worktree = testing.tmpIterableDir(.{});
1372 defer worktree.cleanup();
1373
1374 const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb");
1375 try repository.checkout(worktree.iterable_dir.dir, commit_id);
1376
1377 const expected_files: []const []const u8 = &.{
1378 "dir/file",
1379 "dir/subdir/file",
1380 "dir/subdir/file2",
1381 "dir2/file",
1382 "dir3/file",
1383 "dir3/file2",
1384 "file",
1385 "file2",
1386 "file3",
1387 "file4",
1388 "file5",
1389 "file6",
1390 "file7",
1391 "file8",
1392 "file9",
1393 };
1394 var actual_files: std.ArrayListUnmanaged([]u8) = .{};
1395 defer actual_files.deinit(testing.allocator);
1396 defer for (actual_files.items) |file| testing.allocator.free(file);
1397 var walker = try worktree.iterable_dir.walk(testing.allocator);
1398 defer walker.deinit();
1399 while (try walker.next()) |entry| {
1400 if (entry.kind != .file) continue;
1401 var path = try testing.allocator.dupe(u8, entry.path);
1402 errdefer testing.allocator.free(path);
1403 mem.replaceScalar(u8, path, std.fs.path.sep, '/');
1404 try actual_files.append(testing.allocator, path);
1405 }
1406 mem.sortUnstable([]u8, actual_files.items, {}, struct {
1407 fn lessThan(_: void, a: []u8, b: []u8) bool {
1408 return mem.lessThan(u8, a, b);
1409 }
1410 }.lessThan);
1411 try testing.expectEqualDeep(expected_files, actual_files.items);
1412
1413 const expected_file_contents =
1414 \\revision 1
1415 \\revision 2
1416 \\revision 4
1417 \\revision 5
1418 \\revision 7
1419 \\revision 8
1420 \\revision 9
1421 \\revision 10
1422 \\revision 12
1423 \\revision 13
1424 \\revision 14
1425 \\revision 18
1426 \\revision 19
1427 \\
1428 ;
1429 const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size);
1430 defer testing.allocator.free(actual_file_contents);
1431 try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1432}
1433
1434/// Checks out a commit of a packfile. Intended for experimenting with and
1435/// benchmarking possible optimizations to the indexing and checkout behavior.
1436pub fn main() !void {
1437 const allocator = std.heap.c_allocator;
1438
1439 const args = try std.process.argsAlloc(allocator);
1440 defer std.process.argsFree(allocator, args);
1441 if (args.len != 4) {
1442 return error.InvalidArguments; // Arguments: packfile commit worktree
1443 }
1444
1445 var pack_file = try std.fs.cwd().openFile(args[1], .{});
1446 defer pack_file.close();
1447 const commit = try parseOid(args[2]);
1448 var worktree = try std.fs.cwd().makeOpenPath(args[3], .{});
1449 defer worktree.close();
1450
1451 var git_dir = try worktree.makeOpenPath(".git", .{});
1452 defer git_dir.close();
1453
1454 std.debug.print("Starting index...\n", .{});
1455 var index_file = try git_dir.createFile("idx", .{ .read = true });
1456 defer index_file.close();
1457 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1458 try indexPack(allocator, pack_file, index_buffered_writer.writer());
1459 try index_buffered_writer.flush();
1460 try index_file.sync();
1461
1462 std.debug.print("Starting checkout...\n", .{});
1463 var repository = try Repository.init(allocator, pack_file, index_file);
1464 defer repository.deinit();
1465 try repository.checkout(worktree, commit);
1466}
src/Package/Fetch/git/testdata/testrepo.idx created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.idx differ
src/Package/Fetch/git/testdata/testrepo.pack created
Binary files /dev/null and b/src/Package/Fetch/git/testdata/testrepo.pack differ
src/Package/Manifest.zig created+564
...@@ -0,0 +1,564 @@
1pub const max_bytes = 10 * 1024 * 1024;
2pub const basename = "build.zig.zon";
3pub const Hash = std.crypto.hash.sha2.Sha256;
4pub const Digest = [Hash.digest_length]u8;
5pub const multihash_len = 1 + 1 + Hash.digest_length;
6pub const multihash_hex_digest_len = 2 * multihash_len;
7pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
8
9pub const Dependency = struct {
10 location: Location,
11 location_tok: Ast.TokenIndex,
12 hash: ?[]const u8,
13 hash_tok: Ast.TokenIndex,
14
15 pub const Location = union(enum) {
16 url: []const u8,
17 path: []const u8,
18 };
19};
20
21pub const ErrorMessage = struct {
22 msg: []const u8,
23 tok: Ast.TokenIndex,
24 off: u32,
25};
26
27pub const MultihashFunction = enum(u16) {
28 identity = 0x00,
29 sha1 = 0x11,
30 @"sha2-256" = 0x12,
31 @"sha2-512" = 0x13,
32 @"sha3-512" = 0x14,
33 @"sha3-384" = 0x15,
34 @"sha3-256" = 0x16,
35 @"sha3-224" = 0x17,
36 @"sha2-384" = 0x20,
37 @"sha2-256-trunc254-padded" = 0x1012,
38 @"sha2-224" = 0x1013,
39 @"sha2-512-224" = 0x1014,
40 @"sha2-512-256" = 0x1015,
41 @"blake2b-256" = 0xb220,
42 _,
43};
44
45pub const multihash_function: MultihashFunction = switch (Hash) {
46 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
47 else => @compileError("unreachable"),
48};
49comptime {
50 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
51 // values are small enough to be contained in the one-byte encoding.
52 assert(@intFromEnum(multihash_function) < 127);
53 assert(Hash.digest_length < 127);
54}
55
56name: []const u8,
57version: std.SemanticVersion,
58dependencies: std.StringArrayHashMapUnmanaged(Dependency),
59paths: std.StringArrayHashMapUnmanaged(void),
60
61errors: []ErrorMessage,
62arena_state: std.heap.ArenaAllocator.State,
63
64pub const ParseOptions = struct {
65 allow_missing_paths_field: bool = false,
66};
67
68pub const Error = Allocator.Error;
69
70pub fn parse(gpa: Allocator, ast: std.zig.Ast, options: ParseOptions) Error!Manifest {
71 const node_tags = ast.nodes.items(.tag);
72 const node_datas = ast.nodes.items(.data);
73 assert(node_tags[0] == .root);
74 const main_node_index = node_datas[0].lhs;
75
76 var arena_instance = std.heap.ArenaAllocator.init(gpa);
77 errdefer arena_instance.deinit();
78
79 var p: Parse = .{
80 .gpa = gpa,
81 .ast = ast,
82 .arena = arena_instance.allocator(),
83 .errors = .{},
84
85 .name = undefined,
86 .version = undefined,
87 .dependencies = .{},
88 .paths = .{},
89 .allow_missing_paths_field = options.allow_missing_paths_field,
90 .buf = .{},
91 };
92 defer p.buf.deinit(gpa);
93 defer p.errors.deinit(gpa);
94 defer p.dependencies.deinit(gpa);
95 defer p.paths.deinit(gpa);
96
97 p.parseRoot(main_node_index) catch |err| switch (err) {
98 error.ParseFailure => assert(p.errors.items.len > 0),
99 else => |e| return e,
100 };
101
102 return .{
103 .name = p.name,
104 .version = p.version,
105 .dependencies = try p.dependencies.clone(p.arena),
106 .paths = try p.paths.clone(p.arena),
107 .errors = try p.arena.dupe(ErrorMessage, p.errors.items),
108 .arena_state = arena_instance.state,
109 };
110}
111
112pub fn deinit(man: *Manifest, gpa: Allocator) void {
113 man.arena_state.promote(gpa).deinit();
114 man.* = undefined;
115}
116
117const hex_charset = "0123456789abcdef";
118
119pub fn hex64(x: u64) [16]u8 {
120 var result: [16]u8 = undefined;
121 var i: usize = 0;
122 while (i < 8) : (i += 1) {
123 const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i))));
124 result[i * 2 + 0] = hex_charset[byte >> 4];
125 result[i * 2 + 1] = hex_charset[byte & 15];
126 }
127 return result;
128}
129
130test hex64 {
131 const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
132 try std.testing.expectEqualStrings("[00efcdab78563412]", s);
133}
134
135pub fn hexDigest(digest: Digest) MultiHashHexDigest {
136 var result: MultiHashHexDigest = undefined;
137
138 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
139 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
140
141 result[2] = hex_charset[Hash.digest_length >> 4];
142 result[3] = hex_charset[Hash.digest_length & 15];
143
144 for (digest, 0..) |byte, i| {
145 result[4 + i * 2] = hex_charset[byte >> 4];
146 result[5 + i * 2] = hex_charset[byte & 15];
147 }
148 return result;
149}
150
151const Parse = struct {
152 gpa: Allocator,
153 ast: std.zig.Ast,
154 arena: Allocator,
155 buf: std.ArrayListUnmanaged(u8),
156 errors: std.ArrayListUnmanaged(ErrorMessage),
157
158 name: []const u8,
159 version: std.SemanticVersion,
160 dependencies: std.StringArrayHashMapUnmanaged(Dependency),
161 paths: std.StringArrayHashMapUnmanaged(void),
162 allow_missing_paths_field: bool,
163
164 const InnerError = error{ ParseFailure, OutOfMemory };
165
166 fn parseRoot(p: *Parse, node: Ast.Node.Index) !void {
167 const ast = p.ast;
168 const main_tokens = ast.nodes.items(.main_token);
169 const main_token = main_tokens[node];
170
171 var buf: [2]Ast.Node.Index = undefined;
172 const struct_init = ast.fullStructInit(&buf, node) orelse {
173 return fail(p, main_token, "expected top level expression to be a struct", .{});
174 };
175
176 var have_name = false;
177 var have_version = false;
178 var have_included_paths = false;
179
180 for (struct_init.ast.fields) |field_init| {
181 const name_token = ast.firstToken(field_init) - 2;
182 const field_name = try identifierTokenString(p, name_token);
183 // We could get fancy with reflection and comptime logic here but doing
184 // things manually provides an opportunity to do any additional verification
185 // that is desirable on a per-field basis.
186 if (mem.eql(u8, field_name, "dependencies")) {
187 try parseDependencies(p, field_init);
188 } else if (mem.eql(u8, field_name, "paths")) {
189 have_included_paths = true;
190 try parseIncludedPaths(p, field_init);
191 } else if (mem.eql(u8, field_name, "name")) {
192 p.name = try parseString(p, field_init);
193 have_name = true;
194 } else if (mem.eql(u8, field_name, "version")) {
195 const version_text = try parseString(p, field_init);
196 p.version = std.SemanticVersion.parse(version_text) catch |err| v: {
197 try appendError(p, main_tokens[field_init], "unable to parse semantic version: {s}", .{@errorName(err)});
198 break :v undefined;
199 };
200 have_version = true;
201 } else {
202 // Ignore unknown fields so that we can add fields in future zig
203 // versions without breaking older zig versions.
204 }
205 }
206
207 if (!have_name) {
208 try appendError(p, main_token, "missing top-level 'name' field", .{});
209 }
210
211 if (!have_version) {
212 try appendError(p, main_token, "missing top-level 'version' field", .{});
213 }
214
215 if (!have_included_paths) {
216 if (p.allow_missing_paths_field) {
217 try p.paths.put(p.gpa, "", {});
218 } else {
219 try appendError(p, main_token, "missing top-level 'paths' field", .{});
220 }
221 }
222 }
223
224 fn parseDependencies(p: *Parse, node: Ast.Node.Index) !void {
225 const ast = p.ast;
226 const main_tokens = ast.nodes.items(.main_token);
227
228 var buf: [2]Ast.Node.Index = undefined;
229 const struct_init = ast.fullStructInit(&buf, node) orelse {
230 const tok = main_tokens[node];
231 return fail(p, tok, "expected dependencies expression to be a struct", .{});
232 };
233
234 for (struct_init.ast.fields) |field_init| {
235 const name_token = ast.firstToken(field_init) - 2;
236 const dep_name = try identifierTokenString(p, name_token);
237 const dep = try parseDependency(p, field_init);
238 try p.dependencies.put(p.gpa, dep_name, dep);
239 }
240 }
241
242 fn parseDependency(p: *Parse, node: Ast.Node.Index) !Dependency {
243 const ast = p.ast;
244 const main_tokens = ast.nodes.items(.main_token);
245
246 var buf: [2]Ast.Node.Index = undefined;
247 const struct_init = ast.fullStructInit(&buf, node) orelse {
248 const tok = main_tokens[node];
249 return fail(p, tok, "expected dependency expression to be a struct", .{});
250 };
251
252 var dep: Dependency = .{
253 .location = undefined,
254 .location_tok = 0,
255 .hash = null,
256 .hash_tok = 0,
257 };
258 var has_location = false;
259
260 for (struct_init.ast.fields) |field_init| {
261 const name_token = ast.firstToken(field_init) - 2;
262 const field_name = try identifierTokenString(p, name_token);
263 // We could get fancy with reflection and comptime logic here but doing
264 // things manually provides an opportunity to do any additional verification
265 // that is desirable on a per-field basis.
266 if (mem.eql(u8, field_name, "url")) {
267 if (has_location) {
268 return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
269 }
270 dep.location = .{
271 .url = parseString(p, field_init) catch |err| switch (err) {
272 error.ParseFailure => continue,
273 else => |e| return e,
274 },
275 };
276 has_location = true;
277 dep.location_tok = main_tokens[field_init];
278 } else if (mem.eql(u8, field_name, "path")) {
279 if (has_location) {
280 return fail(p, main_tokens[field_init], "dependency should specify only one of 'url' and 'path' fields.", .{});
281 }
282 dep.location = .{
283 .path = parseString(p, field_init) catch |err| switch (err) {
284 error.ParseFailure => continue,
285 else => |e| return e,
286 },
287 };
288 has_location = true;
289 dep.location_tok = main_tokens[field_init];
290 } else if (mem.eql(u8, field_name, "hash")) {
291 dep.hash = parseHash(p, field_init) catch |err| switch (err) {
292 error.ParseFailure => continue,
293 else => |e| return e,
294 };
295 dep.hash_tok = main_tokens[field_init];
296 } else {
297 // Ignore unknown fields so that we can add fields in future zig
298 // versions without breaking older zig versions.
299 }
300 }
301
302 if (!has_location) {
303 try appendError(p, main_tokens[node], "dependency requires location field, one of 'url' or 'path'.", .{});
304 }
305
306 return dep;
307 }
308
309 fn parseIncludedPaths(p: *Parse, node: Ast.Node.Index) !void {
310 const ast = p.ast;
311 const main_tokens = ast.nodes.items(.main_token);
312
313 var buf: [2]Ast.Node.Index = undefined;
314 const array_init = ast.fullArrayInit(&buf, node) orelse {
315 const tok = main_tokens[node];
316 return fail(p, tok, "expected paths expression to be a struct", .{});
317 };
318
319 for (array_init.ast.elements) |elem_node| {
320 const path_string = try parseString(p, elem_node);
321 const normalized = try std.fs.path.resolve(p.arena, &.{path_string});
322 try p.paths.put(p.gpa, normalized, {});
323 }
324 }
325
326 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
327 const ast = p.ast;
328 const node_tags = ast.nodes.items(.tag);
329 const main_tokens = ast.nodes.items(.main_token);
330 if (node_tags[node] != .string_literal) {
331 return fail(p, main_tokens[node], "expected string literal", .{});
332 }
333 const str_lit_token = main_tokens[node];
334 const token_bytes = ast.tokenSlice(str_lit_token);
335 p.buf.clearRetainingCapacity();
336 try parseStrLit(p, str_lit_token, &p.buf, token_bytes, 0);
337 const duped = try p.arena.dupe(u8, p.buf.items);
338 return duped;
339 }
340
341 fn parseHash(p: *Parse, node: Ast.Node.Index) ![]const u8 {
342 const ast = p.ast;
343 const main_tokens = ast.nodes.items(.main_token);
344 const tok = main_tokens[node];
345 const h = try parseString(p, node);
346
347 if (h.len >= 2) {
348 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
349 return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
350 @errorName(err),
351 });
352 };
353 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
354 return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
355 }
356 }
357
358 if (h.len != multihash_hex_digest_len) {
359 return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
360 multihash_hex_digest_len, h.len,
361 });
362 }
363
364 return h;
365 }
366
367 /// TODO: try to DRY this with AstGen.identifierTokenString
368 fn identifierTokenString(p: *Parse, token: Ast.TokenIndex) InnerError![]const u8 {
369 const ast = p.ast;
370 const token_tags = ast.tokens.items(.tag);
371 assert(token_tags[token] == .identifier);
372 const ident_name = ast.tokenSlice(token);
373 if (!mem.startsWith(u8, ident_name, "@")) {
374 return ident_name;
375 }
376 p.buf.clearRetainingCapacity();
377 try parseStrLit(p, token, &p.buf, ident_name, 1);
378 const duped = try p.arena.dupe(u8, p.buf.items);
379 return duped;
380 }
381
382 /// TODO: try to DRY this with AstGen.parseStrLit
383 fn parseStrLit(
384 p: *Parse,
385 token: Ast.TokenIndex,
386 buf: *std.ArrayListUnmanaged(u8),
387 bytes: []const u8,
388 offset: u32,
389 ) InnerError!void {
390 const raw_string = bytes[offset..];
391 var buf_managed = buf.toManaged(p.gpa);
392 const result = std.zig.string_literal.parseWrite(buf_managed.writer(), raw_string);
393 buf.* = buf_managed.moveToUnmanaged();
394 switch (try result) {
395 .success => {},
396 .failure => |err| try p.appendStrLitError(err, token, bytes, offset),
397 }
398 }
399
400 /// TODO: try to DRY this with AstGen.failWithStrLitError
401 fn appendStrLitError(
402 p: *Parse,
403 err: std.zig.string_literal.Error,
404 token: Ast.TokenIndex,
405 bytes: []const u8,
406 offset: u32,
407 ) Allocator.Error!void {
408 const raw_string = bytes[offset..];
409 switch (err) {
410 .invalid_escape_character => |bad_index| {
411 try p.appendErrorOff(
412 token,
413 offset + @as(u32, @intCast(bad_index)),
414 "invalid escape character: '{c}'",
415 .{raw_string[bad_index]},
416 );
417 },
418 .expected_hex_digit => |bad_index| {
419 try p.appendErrorOff(
420 token,
421 offset + @as(u32, @intCast(bad_index)),
422 "expected hex digit, found '{c}'",
423 .{raw_string[bad_index]},
424 );
425 },
426 .empty_unicode_escape_sequence => |bad_index| {
427 try p.appendErrorOff(
428 token,
429 offset + @as(u32, @intCast(bad_index)),
430 "empty unicode escape sequence",
431 .{},
432 );
433 },
434 .expected_hex_digit_or_rbrace => |bad_index| {
435 try p.appendErrorOff(
436 token,
437 offset + @as(u32, @intCast(bad_index)),
438 "expected hex digit or '}}', found '{c}'",
439 .{raw_string[bad_index]},
440 );
441 },
442 .invalid_unicode_codepoint => |bad_index| {
443 try p.appendErrorOff(
444 token,
445 offset + @as(u32, @intCast(bad_index)),
446 "unicode escape does not correspond to a valid codepoint",
447 .{},
448 );
449 },
450 .expected_lbrace => |bad_index| {
451 try p.appendErrorOff(
452 token,
453 offset + @as(u32, @intCast(bad_index)),
454 "expected '{{', found '{c}",
455 .{raw_string[bad_index]},
456 );
457 },
458 .expected_rbrace => |bad_index| {
459 try p.appendErrorOff(
460 token,
461 offset + @as(u32, @intCast(bad_index)),
462 "expected '}}', found '{c}",
463 .{raw_string[bad_index]},
464 );
465 },
466 .expected_single_quote => |bad_index| {
467 try p.appendErrorOff(
468 token,
469 offset + @as(u32, @intCast(bad_index)),
470 "expected single quote ('), found '{c}",
471 .{raw_string[bad_index]},
472 );
473 },
474 .invalid_character => |bad_index| {
475 try p.appendErrorOff(
476 token,
477 offset + @as(u32, @intCast(bad_index)),
478 "invalid byte in string or character literal: '{c}'",
479 .{raw_string[bad_index]},
480 );
481 },
482 }
483 }
484
485 fn fail(
486 p: *Parse,
487 tok: Ast.TokenIndex,
488 comptime fmt: []const u8,
489 args: anytype,
490 ) InnerError {
491 try appendError(p, tok, fmt, args);
492 return error.ParseFailure;
493 }
494
495 fn appendError(p: *Parse, tok: Ast.TokenIndex, comptime fmt: []const u8, args: anytype) !void {
496 return appendErrorOff(p, tok, 0, fmt, args);
497 }
498
499 fn appendErrorOff(
500 p: *Parse,
501 tok: Ast.TokenIndex,
502 byte_offset: u32,
503 comptime fmt: []const u8,
504 args: anytype,
505 ) Allocator.Error!void {
506 try p.errors.append(p.gpa, .{
507 .msg = try std.fmt.allocPrint(p.arena, fmt, args),
508 .tok = tok,
509 .off = byte_offset,
510 });
511 }
512};
513
514const Manifest = @This();
515const std = @import("std");
516const mem = std.mem;
517const Allocator = std.mem.Allocator;
518const assert = std.debug.assert;
519const Ast = std.zig.Ast;
520const testing = std.testing;
521
522test "basic" {
523 const gpa = testing.allocator;
524
525 const example =
526 \\.{
527 \\ .name = "foo",
528 \\ .version = "3.2.1",
529 \\ .dependencies = .{
530 \\ .bar = .{
531 \\ .url = "https://example.com/baz.tar.gz",
532 \\ .hash = "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
533 \\ },
534 \\ },
535 \\}
536 ;
537
538 var ast = try std.zig.Ast.parse(gpa, example, .zon);
539 defer ast.deinit(gpa);
540
541 try testing.expect(ast.errors.len == 0);
542
543 var manifest = try Manifest.parse(gpa, ast);
544 defer manifest.deinit(gpa);
545
546 try testing.expectEqualStrings("foo", manifest.name);
547
548 try testing.expectEqual(@as(std.SemanticVersion, .{
549 .major = 3,
550 .minor = 2,
551 .patch = 1,
552 }), manifest.version);
553
554 try testing.expect(manifest.dependencies.count() == 1);
555 try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]);
556 try testing.expectEqualStrings(
557 "https://example.com/baz.tar.gz",
558 manifest.dependencies.values()[0].url,
559 );
560 try testing.expectEqualStrings(
561 "1220f1b680b6065fcfc94fe777f22e73bcb7e2767e5f4d99d4255fe76ded69c7a35f",
562 manifest.dependencies.values()[0].hash orelse return error.TestFailed,
563 );
564}
src/git.zig deleted-1468
...@@ -1,1468 +0,0 @@
1//! Git support for package fetching.
2//!
3//! This is not intended to support all features of Git: it is limited to the
4//! basic functionality needed to clone a repository for the purpose of fetching
5//! a package.
6
7const std = @import("std");
8const mem = std.mem;
9const testing = std.testing;
10const Allocator = mem.Allocator;
11const Sha1 = std.crypto.hash.Sha1;
12const assert = std.debug.assert;
13
14const ProgressReader = @import("Package.zig").ProgressReader;
15
16pub const oid_length = Sha1.digest_length;
17pub const fmt_oid_length = 2 * oid_length;
18/// The ID of a Git object (an SHA-1 hash).
19pub const Oid = [oid_length]u8;
20
21pub fn parseOid(s: []const u8) !Oid {
22 if (s.len != fmt_oid_length) return error.InvalidOid;
23 var oid: Oid = undefined;
24 for (&oid, 0..) |*b, i| {
25 b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid;
26 }
27 return oid;
28}
29
30test parseOid {
31 try testing.expectEqualSlices(
32 u8,
33 &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 },
34 &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"),
35 );
36 try testing.expectError(error.InvalidOid, parseOid("ce919ccf"));
37 try testing.expectError(error.InvalidOid, parseOid("master"));
38 try testing.expectError(error.InvalidOid, parseOid("HEAD"));
39}
40
41pub const Diagnostics = struct {
42 allocator: Allocator,
43 errors: std.ArrayListUnmanaged(Error) = .{},
44
45 pub const Error = union(enum) {
46 unable_to_create_sym_link: struct {
47 code: anyerror,
48 file_name: []const u8,
49 link_name: []const u8,
50 },
51 };
52
53 pub fn deinit(d: *Diagnostics) void {
54 for (d.errors.items) |item| {
55 switch (item) {
56 .unable_to_create_sym_link => |info| {
57 d.allocator.free(info.file_name);
58 d.allocator.free(info.link_name);
59 },
60 }
61 }
62 d.errors.deinit(d.allocator);
63 d.* = undefined;
64 }
65};
66
67pub const Repository = struct {
68 odb: Odb,
69
70 pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
71 return .{ .odb = try Odb.init(allocator, pack_file, index_file) };
72 }
73
74 pub fn deinit(repository: *Repository) void {
75 repository.odb.deinit();
76 repository.* = undefined;
77 }
78
79 /// Checks out the repository at `commit_oid` to `worktree`.
80 pub fn checkout(
81 repository: *Repository,
82 worktree: std.fs.Dir,
83 commit_oid: Oid,
84 diagnostics: *Diagnostics,
85 ) !void {
86 try repository.odb.seekOid(commit_oid);
87 const tree_oid = tree_oid: {
88 var commit_object = try repository.odb.readObject();
89 if (commit_object.type != .commit) return error.NotACommit;
90 break :tree_oid try getCommitTree(commit_object.data);
91 };
92 try repository.checkoutTree(worktree, tree_oid, "", diagnostics);
93 }
94
95 /// Checks out the tree at `tree_oid` to `worktree`.
96 fn checkoutTree(
97 repository: *Repository,
98 dir: std.fs.Dir,
99 tree_oid: Oid,
100 current_path: []const u8,
101 diagnostics: *Diagnostics,
102 ) !void {
103 try repository.odb.seekOid(tree_oid);
104 const tree_object = try repository.odb.readObject();
105 if (tree_object.type != .tree) return error.NotATree;
106 // The tree object may be evicted from the object cache while we're
107 // iterating over it, so we can make a defensive copy here to make sure
108 // it remains valid until we're done with it
109 const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data);
110 defer repository.odb.allocator.free(tree_data);
111
112 var tree_iter: TreeIterator = .{ .data = tree_data };
113 while (try tree_iter.next()) |entry| {
114 switch (entry.type) {
115 .directory => {
116 try dir.makeDir(entry.name);
117 var subdir = try dir.openDir(entry.name, .{});
118 defer subdir.close();
119 const sub_path = try std.fs.path.join(repository.odb.allocator, &.{ current_path, entry.name });
120 defer repository.odb.allocator.free(sub_path);
121 try repository.checkoutTree(subdir, entry.oid, sub_path, diagnostics);
122 },
123 .file => {
124 var file = try dir.createFile(entry.name, .{});
125 defer file.close();
126 try repository.odb.seekOid(entry.oid);
127 var file_object = try repository.odb.readObject();
128 if (file_object.type != .blob) return error.InvalidFile;
129 try file.writeAll(file_object.data);
130 try file.sync();
131 },
132 .symlink => {
133 try repository.odb.seekOid(entry.oid);
134 var symlink_object = try repository.odb.readObject();
135 if (symlink_object.type != .blob) return error.InvalidFile;
136 const link_name = symlink_object.data;
137 dir.symLink(link_name, entry.name, .{}) catch |e| {
138 const file_name = try std.fs.path.join(diagnostics.allocator, &.{ current_path, entry.name });
139 errdefer diagnostics.allocator.free(file_name);
140 const link_name_dup = try diagnostics.allocator.dupe(u8, link_name);
141 errdefer diagnostics.allocator.free(link_name_dup);
142 try diagnostics.errors.append(diagnostics.allocator, .{ .unable_to_create_sym_link = .{
143 .code = e,
144 .file_name = file_name,
145 .link_name = link_name_dup,
146 } });
147 };
148 },
149 .gitlink => {
150 // Consistent with git archive behavior, create the directory but
151 // do nothing else
152 try dir.makeDir(entry.name);
153 },
154 }
155 }
156 }
157
158 /// Returns the ID of the tree associated with the given commit (provided as
159 /// raw object data).
160 fn getCommitTree(commit_data: []const u8) !Oid {
161 if (!mem.startsWith(u8, commit_data, "tree ") or
162 commit_data.len < "tree ".len + fmt_oid_length + "\n".len or
163 commit_data["tree ".len + fmt_oid_length] != '\n')
164 {
165 return error.InvalidCommit;
166 }
167 return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]);
168 }
169
170 const TreeIterator = struct {
171 data: []const u8,
172 pos: usize = 0,
173
174 const Entry = struct {
175 type: Type,
176 executable: bool,
177 name: [:0]const u8,
178 oid: Oid,
179
180 const Type = enum(u4) {
181 directory = 0o4,
182 file = 0o10,
183 symlink = 0o12,
184 gitlink = 0o16,
185 };
186 };
187
188 fn next(iterator: *TreeIterator) !?Entry {
189 if (iterator.pos == iterator.data.len) return null;
190
191 const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree;
192 const mode: packed struct {
193 permission: u9,
194 unused: u3,
195 type: u4,
196 } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree);
197 const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree;
198 const executable = switch (mode.permission) {
199 0 => if (@"type" == .file) return error.InvalidTree else false,
200 0o644 => if (@"type" != .file) return error.InvalidTree else false,
201 0o755 => if (@"type" != .file) return error.InvalidTree else true,
202 else => return error.InvalidTree,
203 };
204 iterator.pos = mode_end + 1;
205
206 const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree;
207 const name = iterator.data[iterator.pos..name_end :0];
208 iterator.pos = name_end + 1;
209
210 if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree;
211 const oid = iterator.data[iterator.pos..][0..oid_length].*;
212 iterator.pos += oid_length;
213
214 return .{ .type = @"type", .executable = executable, .name = name, .oid = oid };
215 }
216 };
217};
218
219/// A Git object database backed by a packfile. A packfile index is also used
220/// for efficient access to objects in the packfile.
221///
222/// The format of the packfile and its associated index are documented in
223/// [pack-format](https://git-scm.com/docs/pack-format).
224const Odb = struct {
225 pack_file: std.fs.File,
226 index_header: IndexHeader,
227 index_file: std.fs.File,
228 cache: ObjectCache = .{},
229 allocator: Allocator,
230
231 /// Initializes the database from open pack and index files.
232 fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
233 try pack_file.seekTo(0);
234 try index_file.seekTo(0);
235 const index_header = try IndexHeader.read(index_file.reader());
236 return .{
237 .pack_file = pack_file,
238 .index_header = index_header,
239 .index_file = index_file,
240 .allocator = allocator,
241 };
242 }
243
244 fn deinit(odb: *Odb) void {
245 odb.cache.deinit(odb.allocator);
246 odb.* = undefined;
247 }
248
249 /// Reads the object at the current position in the database.
250 fn readObject(odb: *Odb) !Object {
251 var base_offset = try odb.pack_file.getPos();
252 var base_header: EntryHeader = undefined;
253 var delta_offsets = std.ArrayListUnmanaged(u64){};
254 defer delta_offsets.deinit(odb.allocator);
255 const base_object = while (true) {
256 if (odb.cache.get(base_offset)) |base_object| break base_object;
257
258 base_header = try EntryHeader.read(odb.pack_file.reader());
259 switch (base_header) {
260 .ofs_delta => |ofs_delta| {
261 try delta_offsets.append(odb.allocator, base_offset);
262 base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
263 try odb.pack_file.seekTo(base_offset);
264 },
265 .ref_delta => |ref_delta| {
266 try delta_offsets.append(odb.allocator, base_offset);
267 try odb.seekOid(ref_delta.base_object);
268 base_offset = try odb.pack_file.getPos();
269 },
270 else => {
271 const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength());
272 errdefer odb.allocator.free(base_data);
273 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
274 try odb.cache.put(odb.allocator, base_offset, base_object);
275 break base_object;
276 },
277 }
278 };
279
280 const base_data = try resolveDeltaChain(
281 odb.allocator,
282 odb.pack_file,
283 base_object,
284 delta_offsets.items,
285 &odb.cache,
286 );
287
288 return .{ .type = base_object.type, .data = base_data };
289 }
290
291 /// Seeks to the beginning of the object with the given ID.
292 fn seekOid(odb: *Odb, oid: Oid) !void {
293 const key = oid[0];
294 var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0;
295 var end_index = odb.index_header.fan_out_table[key];
296 const found_index = while (start_index < end_index) {
297 const mid_index = start_index + (end_index - start_index) / 2;
298 try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
299 const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length);
300 switch (mem.order(u8, &mid_oid, &oid)) {
301 .lt => start_index = mid_index + 1,
302 .gt => end_index = mid_index,
303 .eq => break mid_index,
304 }
305 } else return error.ObjectNotFound;
306
307 const n_objects = odb.index_header.fan_out_table[255];
308 const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
309 try odb.index_file.seekTo(offset_values_start + found_index * 4);
310 const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32));
311 const pack_offset = pack_offset: {
312 if (l1_offset.big) {
313 const l2_offset_values_start = offset_values_start + n_objects * 4;
314 try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
315 break :pack_offset try odb.index_file.reader().readIntBig(u64);
316 } else {
317 break :pack_offset l1_offset.value;
318 }
319 };
320
321 try odb.pack_file.seekTo(pack_offset);
322 }
323};
324
325const Object = struct {
326 type: Type,
327 data: []const u8,
328
329 const Type = enum {
330 commit,
331 tree,
332 blob,
333 tag,
334 };
335};
336
337/// A cache for object data.
338///
339/// The purpose of this cache is to speed up resolution of deltas by caching the
340/// results of resolving delta objects, while maintaining a maximum cache size
341/// to avoid excessive memory usage. If the total size of the objects in the
342/// cache exceeds the maximum, the cache will begin evicting the least recently
343/// used objects: when resolving delta chains, the most recently used objects
344/// will likely be more helpful as they will be further along in the chain
345/// (skipping earlier reconstruction steps).
346///
347/// Object data stored in the cache is managed by the cache. It should not be
348/// freed by the caller at any point after inserting it into the cache. Any
349/// objects remaining in the cache will be freed when the cache itself is freed.
350const ObjectCache = struct {
351 objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{},
352 lru_nodes: LruList = .{},
353 byte_size: usize = 0,
354
355 const max_byte_size = 128 * 1024 * 1024; // 128MiB
356 /// A list of offsets stored in the cache, with the most recently used
357 /// entries at the end.
358 const LruList = std.DoublyLinkedList(u64);
359 const CacheEntry = struct { object: Object, lru_node: *LruList.Node };
360
361 fn deinit(cache: *ObjectCache, allocator: Allocator) void {
362 var object_iterator = cache.objects.iterator();
363 while (object_iterator.next()) |object| {
364 allocator.free(object.value_ptr.object.data);
365 allocator.destroy(object.value_ptr.lru_node);
366 }
367 cache.objects.deinit(allocator);
368 cache.* = undefined;
369 }
370
371 /// Gets an object from the cache, moving it to the most recently used
372 /// position if it is present.
373 fn get(cache: *ObjectCache, offset: u64) ?Object {
374 if (cache.objects.get(offset)) |entry| {
375 cache.lru_nodes.remove(entry.lru_node);
376 cache.lru_nodes.append(entry.lru_node);
377 return entry.object;
378 } else {
379 return null;
380 }
381 }
382
383 /// Puts an object in the cache, possibly evicting older entries if the
384 /// cache exceeds its maximum size. Note that, although old objects may
385 /// be evicted, the object just added to the cache with this function
386 /// will not be evicted before the next call to `put` or `deinit` even if
387 /// it exceeds the maximum cache size.
388 fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void {
389 const lru_node = try allocator.create(LruList.Node);
390 errdefer allocator.destroy(lru_node);
391 lru_node.data = offset;
392
393 const gop = try cache.objects.getOrPut(allocator, offset);
394 if (gop.found_existing) {
395 cache.byte_size -= gop.value_ptr.object.data.len;
396 cache.lru_nodes.remove(gop.value_ptr.lru_node);
397 allocator.destroy(gop.value_ptr.lru_node);
398 allocator.free(gop.value_ptr.object.data);
399 }
400 gop.value_ptr.* = .{ .object = object, .lru_node = lru_node };
401 cache.byte_size += object.data.len;
402 cache.lru_nodes.append(lru_node);
403
404 while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) {
405 // The > 1 check is to make sure that we don't evict the most
406 // recently added node, even if it by itself happens to exceed the
407 // maximum size of the cache.
408 const evict_node = cache.lru_nodes.popFirst().?;
409 const evict_offset = evict_node.data;
410 allocator.destroy(evict_node);
411 const evict_object = cache.objects.get(evict_offset).?.object;
412 cache.byte_size -= evict_object.data.len;
413 allocator.free(evict_object.data);
414 _ = cache.objects.remove(evict_offset);
415 }
416 }
417};
418
419/// A single pkt-line in the Git protocol.
420///
421/// The format of a pkt-line is documented in
422/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
423/// meanings of the delimiter and response-end packets are documented in
424/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
425const Packet = union(enum) {
426 flush,
427 delimiter,
428 response_end,
429 data: []const u8,
430
431 const max_data_length = 65516;
432
433 /// Reads a packet in pkt-line format.
434 fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {
435 const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;
436 switch (length) {
437 0 => return .flush,
438 1 => return .delimiter,
439 2 => return .response_end,
440 3 => return error.InvalidPacket,
441 else => if (length - 4 > max_data_length) return error.InvalidPacket,
442 }
443 const data = buf[0 .. length - 4];
444 try reader.readNoEof(data);
445 return .{ .data = data };
446 }
447
448 /// Writes a packet in pkt-line format.
449 fn write(packet: Packet, writer: anytype) !void {
450 switch (packet) {
451 .flush => try writer.writeAll("0000"),
452 .delimiter => try writer.writeAll("0001"),
453 .response_end => try writer.writeAll("0002"),
454 .data => |data| {
455 assert(data.len <= max_data_length);
456 try writer.print("{x:0>4}", .{data.len + 4});
457 try writer.writeAll(data);
458 },
459 }
460 }
461};
462
463/// A client session for the Git protocol, currently limited to an HTTP(S)
464/// transport. Only protocol version 2 is supported, as documented in
465/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
466pub const Session = struct {
467 transport: *std.http.Client,
468 uri: std.Uri,
469 supports_agent: bool = false,
470 supports_shallow: bool = false,
471
472 const agent = "zig/" ++ @import("builtin").zig_version_string;
473 const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
474
475 /// Discovers server capabilities. This should be called before using any
476 /// other client functionality, or the client will be forced to default to
477 /// the bare minimum server requirements, which may be considerably less
478 /// efficient (e.g. no shallow fetches).
479 ///
480 /// See the note on `getCapabilities` regarding `redirect_uri`.
481 pub fn discoverCapabilities(
482 session: *Session,
483 allocator: Allocator,
484 redirect_uri: *[]u8,
485 ) !void {
486 var capability_iterator = try session.getCapabilities(allocator, redirect_uri);
487 defer capability_iterator.deinit();
488 while (try capability_iterator.next()) |capability| {
489 if (mem.eql(u8, capability.key, "agent")) {
490 session.supports_agent = true;
491 } else if (mem.eql(u8, capability.key, "fetch")) {
492 var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' ');
493 while (feature_iterator.next()) |feature| {
494 if (mem.eql(u8, feature, "shallow")) {
495 session.supports_shallow = true;
496 }
497 }
498 }
499 }
500 }
501
502 /// Returns an iterator over capabilities supported by the server.
503 ///
504 /// If the server redirects the request, `error.Redirected` is returned and
505 /// `redirect_uri` is populated with the URI resulting from the redirects.
506 /// When this occurs, the value of `redirect_uri` must be freed with
507 /// `allocator` when the caller is done with it.
508 fn getCapabilities(
509 session: Session,
510 allocator: Allocator,
511 redirect_uri: *[]u8,
512 ) !CapabilityIterator {
513 var info_refs_uri = session.uri;
514 info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" });
515 defer allocator.free(info_refs_uri.path);
516 info_refs_uri.query = "service=git-upload-pack";
517 info_refs_uri.fragment = null;
518
519 var headers = std.http.Headers.init(allocator);
520 defer headers.deinit();
521 try headers.append("Git-Protocol", "version=2");
522
523 var request = try session.transport.request(.GET, info_refs_uri, headers, .{
524 .max_redirects = 3,
525 });
526 errdefer request.deinit();
527 try request.start(.{});
528 try request.finish();
529
530 try request.wait();
531 if (request.response.status != .ok) return error.ProtocolError;
532 if (request.redirects_left < 3) {
533 if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect;
534 var new_uri = request.uri;
535 new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len];
536 new_uri.query = null;
537 redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri});
538 return error.Redirected;
539 }
540
541 const reader = request.reader();
542 var buf: [Packet.max_data_length]u8 = undefined;
543 var state: enum { response_start, response_content } = .response_start;
544 while (true) {
545 // Some Git servers (at least GitHub) include an additional
546 // '# service=git-upload-pack' informative response before sending
547 // the expected 'version 2' packet and capability information.
548 // This is not universal: SourceHut, for example, does not do this.
549 // Thus, we need to skip any such useless additional responses
550 // before we get the one we're actually looking for. The responses
551 // will be delimited by flush packets.
552 const packet = Packet.read(reader, &buf) catch |e| switch (e) {
553 error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
554 else => |other| return other,
555 };
556 switch (packet) {
557 .flush => state = .response_start,
558 .data => |data| switch (state) {
559 .response_start => if (mem.eql(u8, data, "version 2\n")) {
560 return .{ .request = request };
561 } else {
562 state = .response_content;
563 },
564 else => {},
565 },
566 else => return error.UnexpectedPacket,
567 }
568 }
569 }
570
571 const CapabilityIterator = struct {
572 request: std.http.Client.Request,
573 buf: [Packet.max_data_length]u8 = undefined,
574
575 const Capability = struct {
576 key: []const u8,
577 value: ?[]const u8 = null,
578 };
579
580 fn deinit(iterator: *CapabilityIterator) void {
581 iterator.request.deinit();
582 iterator.* = undefined;
583 }
584
585 fn next(iterator: *CapabilityIterator) !?Capability {
586 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
587 .flush => return null,
588 .data => |data| if (data.len > 0 and data[data.len - 1] == '\n') {
589 if (mem.indexOfScalar(u8, data, '=')) |separator_pos| {
590 return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] };
591 } else {
592 return .{ .key = data[0 .. data.len - 1] };
593 }
594 } else return error.UnexpectedPacket,
595 else => return error.UnexpectedPacket,
596 }
597 }
598 };
599
600 const ListRefsOptions = struct {
601 /// The ref prefixes (if any) to use to filter the refs available on the
602 /// server. Note that the client must still check the returned refs
603 /// against its desired filters itself: the server is not required to
604 /// respect these prefix filters and may return other refs as well.
605 ref_prefixes: []const []const u8 = &.{},
606 /// Whether to include symref targets for returned symbolic refs.
607 include_symrefs: bool = false,
608 /// Whether to include the peeled object ID for returned tag refs.
609 include_peeled: bool = false,
610 };
611
612 /// Returns an iterator over refs known to the server.
613 pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator {
614 var upload_pack_uri = session.uri;
615 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
616 defer allocator.free(upload_pack_uri.path);
617 upload_pack_uri.query = null;
618 upload_pack_uri.fragment = null;
619
620 var headers = std.http.Headers.init(allocator);
621 defer headers.deinit();
622 try headers.append("Content-Type", "application/x-git-upload-pack-request");
623 try headers.append("Git-Protocol", "version=2");
624
625 var body = std.ArrayListUnmanaged(u8){};
626 defer body.deinit(allocator);
627 const body_writer = body.writer(allocator);
628 try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer);
629 if (session.supports_agent) {
630 try Packet.write(.{ .data = agent_capability }, body_writer);
631 }
632 try Packet.write(.delimiter, body_writer);
633 for (options.ref_prefixes) |ref_prefix| {
634 const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix});
635 defer allocator.free(ref_prefix_packet);
636 try Packet.write(.{ .data = ref_prefix_packet }, body_writer);
637 }
638 if (options.include_symrefs) {
639 try Packet.write(.{ .data = "symrefs\n" }, body_writer);
640 }
641 if (options.include_peeled) {
642 try Packet.write(.{ .data = "peel\n" }, body_writer);
643 }
644 try Packet.write(.flush, body_writer);
645
646 var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
647 .handle_redirects = false,
648 });
649 errdefer request.deinit();
650 request.transfer_encoding = .{ .content_length = body.items.len };
651 try request.start(.{});
652 try request.writeAll(body.items);
653 try request.finish();
654
655 try request.wait();
656 if (request.response.status != .ok) return error.ProtocolError;
657
658 return .{ .request = request };
659 }
660
661 pub const RefIterator = struct {
662 request: std.http.Client.Request,
663 buf: [Packet.max_data_length]u8 = undefined,
664
665 pub const Ref = struct {
666 oid: Oid,
667 name: []const u8,
668 symref_target: ?[]const u8,
669 peeled: ?Oid,
670 };
671
672 pub fn deinit(iterator: *RefIterator) void {
673 iterator.request.deinit();
674 iterator.* = undefined;
675 }
676
677 pub fn next(iterator: *RefIterator) !?Ref {
678 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {
679 .flush => return null,
680 .data => |data| {
681 const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket;
682 const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
683
684 const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
685 const name = data[oid_sep_pos + 1 .. name_sep_pos];
686
687 var symref_target: ?[]const u8 = null;
688 var peeled: ?Oid = null;
689 var last_sep_pos = name_sep_pos;
690 while (data[last_sep_pos] == ' ') {
691 const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket;
692 const attribute = data[last_sep_pos + 1 .. next_sep_pos];
693 if (mem.startsWith(u8, attribute, "symref-target:")) {
694 symref_target = attribute["symref-target:".len..];
695 } else if (mem.startsWith(u8, attribute, "peeled:")) {
696 peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket;
697 }
698 last_sep_pos = next_sep_pos;
699 }
700
701 return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled };
702 },
703 else => return error.UnexpectedPacket,
704 }
705 }
706 };
707
708 /// Fetches the given refs from the server. A shallow fetch (depth 1) is
709 /// performed if the server supports it.
710 pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream {
711 var upload_pack_uri = session.uri;
712 upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" });
713 defer allocator.free(upload_pack_uri.path);
714 upload_pack_uri.query = null;
715 upload_pack_uri.fragment = null;
716
717 var headers = std.http.Headers.init(allocator);
718 defer headers.deinit();
719 try headers.append("Content-Type", "application/x-git-upload-pack-request");
720 try headers.append("Git-Protocol", "version=2");
721
722 var body = std.ArrayListUnmanaged(u8){};
723 defer body.deinit(allocator);
724 const body_writer = body.writer(allocator);
725 try Packet.write(.{ .data = "command=fetch\n" }, body_writer);
726 if (session.supports_agent) {
727 try Packet.write(.{ .data = agent_capability }, body_writer);
728 }
729 try Packet.write(.delimiter, body_writer);
730 // Our packfile parser supports the OFS_DELTA object type
731 try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);
732 // We do not currently convey server progress information to the user
733 try Packet.write(.{ .data = "no-progress\n" }, body_writer);
734 if (session.supports_shallow) {
735 try Packet.write(.{ .data = "deepen 1\n" }, body_writer);
736 }
737 for (wants) |want| {
738 var buf: [Packet.max_data_length]u8 = undefined;
739 const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
740 try Packet.write(.{ .data = arg }, body_writer);
741 }
742 try Packet.write(.{ .data = "done\n" }, body_writer);
743 try Packet.write(.flush, body_writer);
744
745 var request = try session.transport.request(.POST, upload_pack_uri, headers, .{
746 .handle_redirects = false,
747 });
748 errdefer request.deinit();
749 request.transfer_encoding = .{ .content_length = body.items.len };
750 try request.start(.{});
751 try request.writeAll(body.items);
752 try request.finish();
753
754 try request.wait();
755 if (request.response.status != .ok) return error.ProtocolError;
756
757 const reader = request.reader();
758 // We are not interested in any of the sections of the returned fetch
759 // data other than the packfile section, since we aren't doing anything
760 // complex like ref negotiation (this is a fresh clone).
761 var state: enum { section_start, section_content } = .section_start;
762 while (true) {
763 var buf: [Packet.max_data_length]u8 = undefined;
764 const packet = try Packet.read(reader, &buf);
765 switch (state) {
766 .section_start => switch (packet) {
767 .data => |data| if (mem.eql(u8, data, "packfile\n")) {
768 return .{ .request = request };
769 } else {
770 state = .section_content;
771 },
772 else => return error.UnexpectedPacket,
773 },
774 .section_content => switch (packet) {
775 .delimiter => state = .section_start,
776 .data => {},
777 else => return error.UnexpectedPacket,
778 },
779 }
780 }
781 }
782
783 pub const FetchStream = struct {
784 request: std.http.Client.Request,
785 buf: [Packet.max_data_length]u8 = undefined,
786 pos: usize = 0,
787 len: usize = 0,
788
789 pub fn deinit(stream: *FetchStream) void {
790 stream.request.deinit();
791 }
792
793 pub const ReadError = std.http.Client.Request.ReadError || error{
794 InvalidPacket,
795 ProtocolError,
796 UnexpectedPacket,
797 };
798 pub const Reader = std.io.Reader(*FetchStream, ReadError, read);
799
800 const StreamCode = enum(u8) {
801 pack_data = 1,
802 progress = 2,
803 fatal_error = 3,
804 _,
805 };
806
807 pub fn reader(stream: *FetchStream) Reader {
808 return .{ .context = stream };
809 }
810
811 pub fn read(stream: *FetchStream, buf: []u8) !usize {
812 if (stream.pos == stream.len) {
813 while (true) {
814 switch (try Packet.read(stream.request.reader(), &stream.buf)) {
815 .flush => return 0,
816 .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {
817 .pack_data => {
818 stream.pos = 1;
819 stream.len = data.len;
820 break;
821 },
822 .fatal_error => return error.ProtocolError,
823 else => {},
824 },
825 else => return error.UnexpectedPacket,
826 }
827 }
828 }
829
830 const size = @min(buf.len, stream.len - stream.pos);
831 @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]);
832 stream.pos += size;
833 return size;
834 }
835 };
836};
837
838const PackHeader = struct {
839 total_objects: u32,
840
841 const signature = "PACK";
842 const supported_version = 2;
843
844 fn read(reader: anytype) !PackHeader {
845 const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) {
846 error.EndOfStream => return error.InvalidHeader,
847 else => |other| return other,
848 };
849 if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
850 const version = reader.readIntBig(u32) catch |e| switch (e) {
851 error.EndOfStream => return error.InvalidHeader,
852 else => |other| return other,
853 };
854 if (version != supported_version) return error.UnsupportedVersion;
855 const total_objects = reader.readIntBig(u32) catch |e| switch (e) {
856 error.EndOfStream => return error.InvalidHeader,
857 else => |other| return other,
858 };
859 return .{ .total_objects = total_objects };
860 }
861};
862
863const EntryHeader = union(Type) {
864 commit: Undeltified,
865 tree: Undeltified,
866 blob: Undeltified,
867 tag: Undeltified,
868 ofs_delta: OfsDelta,
869 ref_delta: RefDelta,
870
871 const Type = enum(u3) {
872 commit = 1,
873 tree = 2,
874 blob = 3,
875 tag = 4,
876 ofs_delta = 6,
877 ref_delta = 7,
878 };
879
880 const Undeltified = struct {
881 uncompressed_length: u64,
882 };
883
884 const OfsDelta = struct {
885 offset: u64,
886 uncompressed_length: u64,
887 };
888
889 const RefDelta = struct {
890 base_object: Oid,
891 uncompressed_length: u64,
892 };
893
894 fn objectType(header: EntryHeader) Object.Type {
895 return switch (header) {
896 inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)),
897 else => unreachable,
898 };
899 }
900
901 fn uncompressedLength(header: EntryHeader) u64 {
902 return switch (header) {
903 inline else => |entry| entry.uncompressed_length,
904 };
905 }
906
907 fn read(reader: anytype) !EntryHeader {
908 const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
909 const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) {
910 error.EndOfStream => return error.InvalidFormat,
911 else => |other| return other,
912 });
913 const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
914 var uncompressed_length: u64 = initial.len;
915 uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
916 const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat;
917 return switch (@"type") {
918 inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{
919 .uncompressed_length = uncompressed_length,
920 }),
921 .ofs_delta => .{ .ofs_delta = .{
922 .offset = try readOffsetVarInt(reader),
923 .uncompressed_length = uncompressed_length,
924 } },
925 .ref_delta => .{ .ref_delta = .{
926 .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) {
927 error.EndOfStream => return error.InvalidFormat,
928 else => |other| return other,
929 },
930 .uncompressed_length = uncompressed_length,
931 } },
932 };
933 }
934};
935
936fn readSizeVarInt(r: anytype) !u64 {
937 const Byte = packed struct { value: u7, has_next: bool };
938 var b: Byte = @bitCast(try r.readByte());
939 var value: u64 = b.value;
940 var shift: u6 = 0;
941 while (b.has_next) {
942 b = @bitCast(try r.readByte());
943 shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
944 value |= @as(u64, b.value) << shift;
945 }
946 return value;
947}
948
949fn readOffsetVarInt(r: anytype) !u64 {
950 const Byte = packed struct { value: u7, has_next: bool };
951 var b: Byte = @bitCast(try r.readByte());
952 var value: u64 = b.value;
953 while (b.has_next) {
954 b = @bitCast(try r.readByte());
955 value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
956 value |= b.value;
957 }
958 return value;
959}
960
961const IndexHeader = struct {
962 fan_out_table: [256]u32,
963
964 const signature = "\xFFtOc";
965 const supported_version = 2;
966 const size = 4 + 4 + @sizeOf([256]u32);
967
968 fn read(reader: anytype) !IndexHeader {
969 var header_bytes = try reader.readBytesNoEof(size);
970 if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
971 const version = mem.readIntBig(u32, header_bytes[4..8]);
972 if (version != supported_version) return error.UnsupportedVersion;
973
974 var fan_out_table: [256]u32 = undefined;
975 var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
976 const fan_out_table_reader = fan_out_table_stream.reader();
977 for (&fan_out_table) |*entry| {
978 entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable;
979 }
980 return .{ .fan_out_table = fan_out_table };
981 }
982};
983
984const IndexEntry = struct {
985 offset: u64,
986 crc32: u32,
987};
988
989/// Writes out a version 2 index for the given packfile, as documented in
990/// [pack-format](https://git-scm.com/docs/pack-format).
991pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void {
992 try pack.seekTo(0);
993
994 var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){};
995 defer index_entries.deinit(allocator);
996 var pending_deltas = std.ArrayListUnmanaged(IndexEntry){};
997 defer pending_deltas.deinit(allocator);
998
999 const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas);
1000
1001 var cache: ObjectCache = .{};
1002 defer cache.deinit(allocator);
1003 var remaining_deltas = pending_deltas.items.len;
1004 while (remaining_deltas > 0) {
1005 var i: usize = remaining_deltas;
1006 while (i > 0) {
1007 i -= 1;
1008 const delta = pending_deltas.items[i];
1009 if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| {
1010 try index_entries.put(allocator, oid, delta);
1011 _ = pending_deltas.swapRemove(i);
1012 }
1013 }
1014 if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack;
1015 remaining_deltas = pending_deltas.items.len;
1016 }
1017
1018 var oids = std.ArrayListUnmanaged(Oid){};
1019 defer oids.deinit(allocator);
1020 try oids.ensureTotalCapacityPrecise(allocator, index_entries.count());
1021 var index_entries_iter = index_entries.iterator();
1022 while (index_entries_iter.next()) |entry| {
1023 oids.appendAssumeCapacity(entry.key_ptr.*);
1024 }
1025 mem.sortUnstable(Oid, oids.items, {}, struct {
1026 fn lessThan(_: void, o1: Oid, o2: Oid) bool {
1027 return mem.lessThan(u8, &o1, &o2);
1028 }
1029 }.lessThan);
1030
1031 var fan_out_table: [256]u32 = undefined;
1032 var count: u32 = 0;
1033 var fan_out_index: u8 = 0;
1034 for (oids.items) |oid| {
1035 if (oid[0] > fan_out_index) {
1036 @memset(fan_out_table[fan_out_index..oid[0]], count);
1037 fan_out_index = oid[0];
1038 }
1039 count += 1;
1040 }
1041 @memset(fan_out_table[fan_out_index..], count);
1042
1043 var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{}));
1044 const writer = index_hashed_writer.writer();
1045 try writer.writeAll(IndexHeader.signature);
1046 try writer.writeIntBig(u32, IndexHeader.supported_version);
1047 for (fan_out_table) |fan_out_entry| {
1048 try writer.writeIntBig(u32, fan_out_entry);
1049 }
1050
1051 for (oids.items) |oid| {
1052 try writer.writeAll(&oid);
1053 }
1054
1055 for (oids.items) |oid| {
1056 try writer.writeIntBig(u32, index_entries.get(oid).?.crc32);
1057 }
1058
1059 var big_offsets = std.ArrayListUnmanaged(u64){};
1060 defer big_offsets.deinit(allocator);
1061 for (oids.items) |oid| {
1062 const offset = index_entries.get(oid).?.offset;
1063 if (offset <= std.math.maxInt(u31)) {
1064 try writer.writeIntBig(u32, @intCast(offset));
1065 } else {
1066 const index = big_offsets.items.len;
1067 try big_offsets.append(allocator, offset);
1068 try writer.writeIntBig(u32, @as(u32, @intCast(index)) | (1 << 31));
1069 }
1070 }
1071 for (big_offsets.items) |offset| {
1072 try writer.writeIntBig(u64, offset);
1073 }
1074
1075 try writer.writeAll(&pack_checksum);
1076 const index_checksum = index_hashed_writer.hasher.finalResult();
1077 try index_writer.writeAll(&index_checksum);
1078}
1079
1080/// Performs the first pass over the packfile data for index construction.
1081/// This will index all non-delta objects, queue delta objects for further
1082/// processing, and return the pack checksum (which is part of the index
1083/// format).
1084fn indexPackFirstPass(
1085 allocator: Allocator,
1086 pack: std.fs.File,
1087 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1088 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1089) ![Sha1.digest_length]u8 {
1090 var pack_buffered_reader = std.io.bufferedReader(pack.reader());
1091 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1092 var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{}));
1093 const pack_reader = pack_hashed_reader.reader();
1094
1095 const pack_header = try PackHeader.read(pack_reader);
1096
1097 var current_entry: u32 = 0;
1098 while (current_entry < pack_header.total_objects) : (current_entry += 1) {
1099 const entry_offset = pack_counting_reader.bytes_read;
1100 var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
1101 const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
1102 switch (entry_header) {
1103 inline .commit, .tree, .blob, .tag => |object, tag| {
1104 var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1105 defer entry_decompress_stream.deinit();
1106 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1107 var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1108 const entry_writer = entry_hashed_writer.writer();
1109 // The object header is not included in the pack data but is
1110 // part of the object's ID
1111 try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length });
1112 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1113 try fifo.pump(entry_counting_reader.reader(), entry_writer);
1114 if (entry_counting_reader.bytes_read != object.uncompressed_length) {
1115 return error.InvalidObject;
1116 }
1117 const oid = entry_hashed_writer.hasher.finalResult();
1118 try index_entries.put(allocator, oid, .{
1119 .offset = entry_offset,
1120 .crc32 = entry_crc32_reader.hasher.final(),
1121 });
1122 },
1123 inline .ofs_delta, .ref_delta => |delta| {
1124 var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());
1125 defer entry_decompress_stream.deinit();
1126 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1127 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1128 try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1129 if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
1130 return error.InvalidObject;
1131 }
1132 try pending_deltas.append(allocator, .{
1133 .offset = entry_offset,
1134 .crc32 = entry_crc32_reader.hasher.final(),
1135 });
1136 },
1137 }
1138 }
1139
1140 const pack_checksum = pack_hashed_reader.hasher.finalResult();
1141 const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length);
1142 if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) {
1143 return error.CorruptedPack;
1144 }
1145 _ = pack_buffered_reader.reader().readByte() catch |e| switch (e) {
1146 error.EndOfStream => return pack_checksum,
1147 else => |other| return other,
1148 };
1149 return error.InvalidFormat;
1150}
1151
1152/// Attempts to determine the final object ID of the given deltified object.
1153/// May return null if this is not yet possible (if the delta is a ref-based
1154/// delta and we do not yet know the offset of the base object).
1155fn indexPackHashDelta(
1156 allocator: Allocator,
1157 pack: std.fs.File,
1158 delta: IndexEntry,
1159 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1160 cache: *ObjectCache,
1161) !?Oid {
1162 // Figure out the chain of deltas to resolve
1163 var base_offset = delta.offset;
1164 var base_header: EntryHeader = undefined;
1165 var delta_offsets = std.ArrayListUnmanaged(u64){};
1166 defer delta_offsets.deinit(allocator);
1167 const base_object = while (true) {
1168 if (cache.get(base_offset)) |base_object| break base_object;
1169
1170 try pack.seekTo(base_offset);
1171 base_header = try EntryHeader.read(pack.reader());
1172 switch (base_header) {
1173 .ofs_delta => |ofs_delta| {
1174 try delta_offsets.append(allocator, base_offset);
1175 base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject;
1176 },
1177 .ref_delta => |ref_delta| {
1178 try delta_offsets.append(allocator, base_offset);
1179 base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1180 },
1181 else => {
1182 const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength());
1183 errdefer allocator.free(base_data);
1184 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1185 try cache.put(allocator, base_offset, base_object);
1186 break base_object;
1187 },
1188 }
1189 };
1190
1191 const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache);
1192
1193 var entry_hasher = Sha1.init(.{});
1194 var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher);
1195 try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
1196 entry_hasher.update(base_data);
1197 return entry_hasher.finalResult();
1198}
1199
1200/// Resolves a chain of deltas, returning the final base object data. `pack` is
1201/// assumed to be looking at the start of the object data for the base object of
1202/// the chain, and will then apply the deltas in `delta_offsets` in reverse order
1203/// to obtain the final object.
1204fn resolveDeltaChain(
1205 allocator: Allocator,
1206 pack: std.fs.File,
1207 base_object: Object,
1208 delta_offsets: []const u64,
1209 cache: *ObjectCache,
1210) ![]const u8 {
1211 var base_data = base_object.data;
1212 var i: usize = delta_offsets.len;
1213 while (i > 0) {
1214 i -= 1;
1215
1216 const delta_offset = delta_offsets[i];
1217 try pack.seekTo(delta_offset);
1218 const delta_header = try EntryHeader.read(pack.reader());
1219 var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
1220 defer allocator.free(delta_data);
1221 var delta_stream = std.io.fixedBufferStream(delta_data);
1222 const delta_reader = delta_stream.reader();
1223 _ = try readSizeVarInt(delta_reader); // base object size
1224 const expanded_size = try readSizeVarInt(delta_reader);
1225
1226 const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
1227 var expanded_data = try allocator.alloc(u8, expanded_alloc_size);
1228 errdefer allocator.free(expanded_data);
1229 var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
1230 var base_stream = std.io.fixedBufferStream(base_data);
1231 try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
1232 if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
1233
1234 try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
1235 base_data = expanded_data;
1236 }
1237 return base_data;
1238}
1239
1240/// Reads the complete contents of an object from `reader`. This function may
1241/// read more bytes than required from `reader`, so the reader position after
1242/// returning is not reliable.
1243fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
1244 const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1245 var buffered_reader = std.io.bufferedReader(reader);
1246 var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());
1247 defer decompress_stream.deinit();
1248 var data = try allocator.alloc(u8, alloc_size);
1249 errdefer allocator.free(data);
1250 try decompress_stream.reader().readNoEof(data);
1251 _ = decompress_stream.reader().readByte() catch |e| switch (e) {
1252 error.EndOfStream => return data,
1253 else => |other| return other,
1254 };
1255 return error.InvalidFormat;
1256}
1257
1258/// Expands delta data from `delta_reader` to `writer`. `base_object` must
1259/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
1260///
1261/// The format of the delta data is documented in
1262/// [pack-format](https://git-scm.com/docs/pack-format).
1263fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void {
1264 while (true) {
1265 const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) {
1266 error.EndOfStream => return,
1267 else => |other| return other,
1268 });
1269 if (inst.copy) {
1270 const available: packed struct {
1271 offset1: bool,
1272 offset2: bool,
1273 offset3: bool,
1274 offset4: bool,
1275 size1: bool,
1276 size2: bool,
1277 size3: bool,
1278 } = @bitCast(inst.value);
1279 var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
1280 .offset1 = if (available.offset1) try delta_reader.readByte() else 0,
1281 .offset2 = if (available.offset2) try delta_reader.readByte() else 0,
1282 .offset3 = if (available.offset3) try delta_reader.readByte() else 0,
1283 .offset4 = if (available.offset4) try delta_reader.readByte() else 0,
1284 };
1285 const offset: u32 = @bitCast(offset_parts);
1286 var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
1287 .size1 = if (available.size1) try delta_reader.readByte() else 0,
1288 .size2 = if (available.size2) try delta_reader.readByte() else 0,
1289 .size3 = if (available.size3) try delta_reader.readByte() else 0,
1290 };
1291 var size: u24 = @bitCast(size_parts);
1292 if (size == 0) size = 0x10000;
1293 try base_object.seekTo(offset);
1294 var copy_reader = std.io.limitedReader(base_object.reader(), size);
1295 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1296 try fifo.pump(copy_reader.reader(), writer);
1297 } else if (inst.value != 0) {
1298 var data_reader = std.io.limitedReader(delta_reader, inst.value);
1299 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1300 try fifo.pump(data_reader.reader(), writer);
1301 } else {
1302 return error.InvalidDeltaInstruction;
1303 }
1304 }
1305}
1306
1307fn HashedWriter(
1308 comptime WriterType: anytype,
1309 comptime HasherType: anytype,
1310) type {
1311 return struct {
1312 child_writer: WriterType,
1313 hasher: HasherType,
1314
1315 const Error = WriterType.Error;
1316 const Writer = std.io.Writer(*@This(), Error, write);
1317
1318 fn write(hashed_writer: *@This(), buf: []const u8) Error!usize {
1319 const amt = try hashed_writer.child_writer.write(buf);
1320 hashed_writer.hasher.update(buf);
1321 return amt;
1322 }
1323
1324 fn writer(hashed_writer: *@This()) Writer {
1325 return .{ .context = hashed_writer };
1326 }
1327 };
1328}
1329
1330fn hashedWriter(
1331 writer: anytype,
1332 hasher: anytype,
1333) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
1334 return .{ .child_writer = writer, .hasher = hasher };
1335}
1336
1337test "packfile indexing and checkout" {
1338 // To verify the contents of this packfile without using the code in this
1339 // file:
1340 //
1341 // 1. Create a new empty Git repository (`git init`)
1342 // 2. `git unpack-objects <path/to/testdata.pack`
1343 // 3. `git fsck` -> note the "dangling commit" ID (which matches the commit
1344 // checked out below)
1345 // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb`
1346 const testrepo_pack = @embedFile("git/testdata/testrepo.pack");
1347
1348 var git_dir = testing.tmpDir(.{});
1349 defer git_dir.cleanup();
1350 var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true });
1351 defer pack_file.close();
1352 try pack_file.writeAll(testrepo_pack);
1353
1354 var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
1355 defer index_file.close();
1356 try indexPack(testing.allocator, pack_file, index_file.writer());
1357
1358 // Arbitrary size limit on files read while checking the repository contents
1359 // (all files in the test repo are known to be much smaller than this)
1360 const max_file_size = 4096;
1361
1362 const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size);
1363 defer testing.allocator.free(index_file_data);
1364 // testrepo.idx is generated by Git. The index created by this file should
1365 // match it exactly. Running `git verify-pack -v testrepo.pack` can verify
1366 // this.
1367 const testrepo_idx = @embedFile("git/testdata/testrepo.idx");
1368 try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
1369
1370 var repository = try Repository.init(testing.allocator, pack_file, index_file);
1371 defer repository.deinit();
1372
1373 var worktree = testing.tmpIterableDir(.{});
1374 defer worktree.cleanup();
1375
1376 const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb");
1377 try repository.checkout(worktree.iterable_dir.dir, commit_id);
1378
1379 const expected_files: []const []const u8 = &.{
1380 "dir/file",
1381 "dir/subdir/file",
1382 "dir/subdir/file2",
1383 "dir2/file",
1384 "dir3/file",
1385 "dir3/file2",
1386 "file",
1387 "file2",
1388 "file3",
1389 "file4",
1390 "file5",
1391 "file6",
1392 "file7",
1393 "file8",
1394 "file9",
1395 };
1396 var actual_files: std.ArrayListUnmanaged([]u8) = .{};
1397 defer actual_files.deinit(testing.allocator);
1398 defer for (actual_files.items) |file| testing.allocator.free(file);
1399 var walker = try worktree.iterable_dir.walk(testing.allocator);
1400 defer walker.deinit();
1401 while (try walker.next()) |entry| {
1402 if (entry.kind != .file) continue;
1403 var path = try testing.allocator.dupe(u8, entry.path);
1404 errdefer testing.allocator.free(path);
1405 mem.replaceScalar(u8, path, std.fs.path.sep, '/');
1406 try actual_files.append(testing.allocator, path);
1407 }
1408 mem.sortUnstable([]u8, actual_files.items, {}, struct {
1409 fn lessThan(_: void, a: []u8, b: []u8) bool {
1410 return mem.lessThan(u8, a, b);
1411 }
1412 }.lessThan);
1413 try testing.expectEqualDeep(expected_files, actual_files.items);
1414
1415 const expected_file_contents =
1416 \\revision 1
1417 \\revision 2
1418 \\revision 4
1419 \\revision 5
1420 \\revision 7
1421 \\revision 8
1422 \\revision 9
1423 \\revision 10
1424 \\revision 12
1425 \\revision 13
1426 \\revision 14
1427 \\revision 18
1428 \\revision 19
1429 \\
1430 ;
1431 const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size);
1432 defer testing.allocator.free(actual_file_contents);
1433 try testing.expectEqualStrings(expected_file_contents, actual_file_contents);
1434}
1435
1436/// Checks out a commit of a packfile. Intended for experimenting with and
1437/// benchmarking possible optimizations to the indexing and checkout behavior.
1438pub fn main() !void {
1439 const allocator = std.heap.c_allocator;
1440
1441 const args = try std.process.argsAlloc(allocator);
1442 defer std.process.argsFree(allocator, args);
1443 if (args.len != 4) {
1444 return error.InvalidArguments; // Arguments: packfile commit worktree
1445 }
1446
1447 var pack_file = try std.fs.cwd().openFile(args[1], .{});
1448 defer pack_file.close();
1449 const commit = try parseOid(args[2]);
1450 var worktree = try std.fs.cwd().makeOpenPath(args[3], .{});
1451 defer worktree.close();
1452
1453 var git_dir = try worktree.makeOpenPath(".git", .{});
1454 defer git_dir.close();
1455
1456 std.debug.print("Starting index...\n", .{});
1457 var index_file = try git_dir.createFile("idx", .{ .read = true });
1458 defer index_file.close();
1459 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
1460 try indexPack(allocator, pack_file, index_buffered_writer.writer());
1461 try index_buffered_writer.flush();
1462 try index_file.sync();
1463
1464 std.debug.print("Starting checkout...\n", .{});
1465 var repository = try Repository.init(allocator, pack_file, index_file);
1466 defer repository.deinit();
1467 try repository.checkout(worktree, commit);
1468}
src/git/testdata/testrepo.idx deleted
Binary files a/src/git/testdata/testrepo.idx and /dev/null differ
src/git/testdata/testrepo.pack deleted
Binary files a/src/git/testdata/testrepo.pack and /dev/null differ