authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-04 19:31:03-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-06 22:42:42-07:00
logb757d7f941e243d4a40fbba90e51a9f514350320
tree6b7b812d622e245d75ebf90252316abf76a5d9c4
parentfdf0e4612e4441d2f0386839078d8a7adf738a3e

fetch: update for new http API

it's not quite finished because I need to make it not copy the Resource

3 files changed, 311 insertions(+), 316 deletions(-)

lib/std/http/Client.zig+15-2
...@@ -682,7 +682,7 @@ pub const Response = struct {...@@ -682,7 +682,7 @@ pub const Response = struct {
682 ///682 ///
683 /// See also:683 /// See also:
684 /// * `readerDecompressing`684 /// * `readerDecompressing`
685 pub fn reader(response: *Response, buffer: []u8) *Reader {685 pub fn reader(response: *const Response, buffer: []u8) *Reader {
686 const req = response.request;686 const req = response.request;
687 if (!req.method.responseHasBody()) return .ending;687 if (!req.method.responseHasBody()) return .ending;
688 const head = &response.head;688 const head = &response.head;
...@@ -805,6 +805,11 @@ pub const Request = struct {...@@ -805,6 +805,11 @@ pub const Request = struct {
805 unhandled = std.math.maxInt(u16),805 unhandled = std.math.maxInt(u16),
806 _,806 _,
807807
808 pub fn init(n: u16) RedirectBehavior {
809 assert(n != std.math.maxInt(u16));
810 return @enumFromInt(n);
811 }
812
808 pub fn subtractOne(rb: *RedirectBehavior) void {813 pub fn subtractOne(rb: *RedirectBehavior) void {
809 switch (rb.*) {814 switch (rb.*) {
810 .not_allowed => unreachable,815 .not_allowed => unreachable,
...@@ -855,6 +860,14 @@ pub const Request = struct {...@@ -855,6 +860,14 @@ pub const Request = struct {
855 return result;860 return result;
856 }861 }
857862
863 /// Transfers the HTTP head and body over the connection and flushes.
864 pub fn sendBodyComplete(r: *Request, body: []u8) Writer.Error!void {
865 r.transfer_encoding = .{ .content_length = body.len };
866 var bw = try sendBodyUnflushed(r, body);
867 bw.writer.end = body.len;
868 try bw.end();
869 }
870
858 /// Transfers the HTTP head over the connection, which is not flushed until871 /// Transfers the HTTP head over the connection, which is not flushed until
859 /// `BodyWriter.flush` or `BodyWriter.end` is called.872 /// `BodyWriter.flush` or `BodyWriter.end` is called.
860 ///873 ///
...@@ -1296,7 +1309,7 @@ pub const basic_authorization = struct {...@@ -1296,7 +1309,7 @@ pub const basic_authorization = struct {
1296 pub fn value(uri: Uri, out: []u8) []u8 {1309 pub fn value(uri: Uri, out: []u8) []u8 {
1297 var bw: Writer = .fixed(out);1310 var bw: Writer = .fixed(out);
1298 write(uri, &bw) catch unreachable;1311 write(uri, &bw) catch unreachable;
1299 return bw.getWritten();1312 return bw.buffered();
1300 }1313 }
13011314
1302 pub fn write(uri: Uri, out: *Writer) Writer.Error!void {1315 pub fn write(uri: Uri, out: *Writer) Writer.Error!void {
src/Package/Fetch.zig+140-183
...@@ -385,20 +385,21 @@ pub fn run(f: *Fetch) RunError!void {...@@ -385,20 +385,21 @@ pub fn run(f: *Fetch) RunError!void {
385 var resource: Resource = .{ .dir = dir };385 var resource: Resource = .{ .dir = dir };
386 return f.runResource(path_or_url, &resource, null);386 return f.runResource(path_or_url, &resource, null);
387 } else |dir_err| {387 } else |dir_err| {
388 var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
389
388 const file_err = if (dir_err == error.NotDir) e: {390 const file_err = if (dir_err == error.NotDir) e: {
389 if (fs.cwd().openFile(path_or_url, .{})) |file| {391 if (fs.cwd().openFile(path_or_url, .{})) |file| {
390 var resource: Resource = .{ .file = file };392 var resource: Resource = .{ .file = file.reader(&server_header_buffer) };
391 return f.runResource(path_or_url, &resource, null);393 return f.runResource(path_or_url, &resource, null);
392 } else |err| break :e err;394 } else |err| break :e err;
393 } else dir_err;395 } else dir_err;
394396
395 const uri = std.Uri.parse(path_or_url) catch |uri_err| {397 const uri = std.Uri.parse(path_or_url) catch |uri_err| {
396 return f.fail(0, try eb.printString(398 return f.fail(0, try eb.printString(
397 "'{s}' could not be recognized as a file path ({s}) or an URL ({s})",399 "'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
398 .{ path_or_url, @errorName(file_err), @errorName(uri_err) },400 .{ path_or_url, file_err, uri_err },
399 ));401 ));
400 };402 };
401 var server_header_buffer: [header_buffer_size]u8 = undefined;
402 var resource = try f.initResource(uri, &server_header_buffer);403 var resource = try f.initResource(uri, &server_header_buffer);
403 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);404 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);
404 }405 }
...@@ -464,8 +465,8 @@ pub fn run(f: *Fetch) RunError!void {...@@ -464,8 +465,8 @@ pub fn run(f: *Fetch) RunError!void {
464 f.location_tok,465 f.location_tok,
465 try eb.printString("invalid URI: {s}", .{@errorName(err)}),466 try eb.printString("invalid URI: {s}", .{@errorName(err)}),
466 );467 );
467 var server_header_buffer: [header_buffer_size]u8 = undefined;468 var buffer: [init_resource_buffer_size]u8 = undefined;
468 var resource = try f.initResource(uri, &server_header_buffer);469 var resource = try f.initResource(uri, &buffer);
469 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);470 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);
470}471}
471472
...@@ -866,8 +867,8 @@ fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {...@@ -866,8 +867,8 @@ fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
866}867}
867868
868const Resource = union(enum) {869const Resource = union(enum) {
869 file: fs.File,870 file: fs.File.Reader,
870 http_request: std.http.Client.Request,871 http_request: HttpRequest,
871 git: Git,872 git: Git,
872 dir: fs.Dir,873 dir: fs.Dir,
873874
...@@ -877,10 +878,16 @@ const Resource = union(enum) {...@@ -877,10 +878,16 @@ const Resource = union(enum) {
877 want_oid: git.Oid,878 want_oid: git.Oid,
878 };879 };
879880
881 const HttpRequest = struct {
882 request: std.http.Client.Request,
883 head: std.http.Client.Response.Head,
884 buffer: []u8,
885 };
886
880 fn deinit(resource: *Resource) void {887 fn deinit(resource: *Resource) void {
881 switch (resource.*) {888 switch (resource.*) {
882 .file => |*file| file.close(),889 .file => |*file_reader| file_reader.file.close(),
883 .http_request => |*req| req.deinit(),890 .http_request => |*http_request| http_request.request.deinit(),
884 .git => |*git_resource| {891 .git => |*git_resource| {
885 git_resource.fetch_stream.deinit();892 git_resource.fetch_stream.deinit();
886 git_resource.session.deinit();893 git_resource.session.deinit();
...@@ -890,21 +897,19 @@ const Resource = union(enum) {...@@ -890,21 +897,19 @@ const Resource = union(enum) {
890 resource.* = undefined;897 resource.* = undefined;
891 }898 }
892899
893 fn reader(resource: *Resource) std.io.AnyReader {900 fn reader(resource: *Resource) *std.Io.Reader {
894 return .{901 return switch (resource.*) {
895 .context = resource,902 .file => |*file_reader| return &file_reader.interface,
896 .readFn = read,903 .http_request => |*http_request| {
897 };904 const response: std.http.Client.Response = .{
898 }905 .request = &http_request.request,
899906 .head = http_request.head,
900 fn read(context: *const anyopaque, buffer: []u8) anyerror!usize {907 };
901 const resource: *Resource = @constCast(@ptrCast(@alignCast(context)));908 return response.reader(http_request.buffer);
902 switch (resource.*) {909 },
903 .file => |*f| return f.read(buffer),910 .git => |*g| return &g.fetch_stream.reader,
904 .http_request => |*r| return r.read(buffer),
905 .git => |*g| return g.fetch_stream.read(buffer),
906 .dir => unreachable,911 .dir => unreachable,
907 }912 };
908 }913 }
909};914};
910915
...@@ -967,20 +972,21 @@ const FileType = enum {...@@ -967,20 +972,21 @@ const FileType = enum {
967 }972 }
968};973};
969974
970const header_buffer_size = 16 * 1024;975const init_resource_buffer_size = git.Packet.max_data_length;
971976
972fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Resource {977fn initResource(f: *Fetch, uri: std.Uri, reader_buffer: []u8) RunError!Resource {
973 const gpa = f.arena.child_allocator;978 const gpa = f.arena.child_allocator;
974 const arena = f.arena.allocator();979 const arena = f.arena.allocator();
975 const eb = &f.error_bundle;980 const eb = &f.error_bundle;
976981
977 if (ascii.eqlIgnoreCase(uri.scheme, "file")) {982 if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
978 const path = try uri.path.toRawMaybeAlloc(arena);983 const path = try uri.path.toRawMaybeAlloc(arena);
979 return .{ .file = f.parent_package_root.openFile(path, .{}) catch |err| {984 const file = f.parent_package_root.openFile(path, .{}) catch |err| {
980 return f.fail(f.location_tok, try eb.printString("unable to open '{f}{s}': {s}", .{985 return f.fail(f.location_tok, try eb.printString("unable to open '{f}{s}': {t}", .{
981 f.parent_package_root, path, @errorName(err),986 f.parent_package_root, path, err,
982 }));987 }));
983 } };988 };
989 return .{ .file = file.reader(reader_buffer) };
984 }990 }
985991
986 const http_client = f.job_queue.http_client;992 const http_client = f.job_queue.http_client;
...@@ -988,37 +994,27 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -988,37 +994,27 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
988 if (ascii.eqlIgnoreCase(uri.scheme, "http") or994 if (ascii.eqlIgnoreCase(uri.scheme, "http") or
989 ascii.eqlIgnoreCase(uri.scheme, "https"))995 ascii.eqlIgnoreCase(uri.scheme, "https"))
990 {996 {
991 var req = http_client.open(.GET, uri, .{997 var request = http_client.request(.GET, uri, .{}) catch |err|
992 .server_header_buffer = server_header_buffer,998 return f.fail(f.location_tok, try eb.printString("unable to connect to server: {t}", .{err}));
993 }) catch |err| {999 defer request.deinit();
994 return f.fail(f.location_tok, try eb.printString(
995 "unable to connect to server: {s}",
996 .{@errorName(err)},
997 ));
998 };
999 errdefer req.deinit(); // releases more than memory
10001000
1001 req.send() catch |err| {1001 request.sendBodiless() catch |err|
1002 return f.fail(f.location_tok, try eb.printString(1002 return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
1003 "HTTP request failed: {s}",
1004 .{@errorName(err)},
1005 ));
1006 };
1007 req.wait() catch |err| {
1008 return f.fail(f.location_tok, try eb.printString(
1009 "invalid HTTP response: {s}",
1010 .{@errorName(err)},
1011 ));
1012 };
10131003
1014 if (req.response.status != .ok) {1004 var redirect_buffer: [1024]u8 = undefined;
1015 return f.fail(f.location_tok, try eb.printString(1005 const response = request.receiveHead(&redirect_buffer) catch |err|
1016 "bad HTTP response code: '{d} {s}'",1006 return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{err}));
1017 .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" },1007
1018 ));1008 if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
1019 }1009 "bad HTTP response code: '{d} {s}'",
1010 .{ response.head.status, response.head.status.phrase() orelse "" },
1011 ));
10201012
1021 return .{ .http_request = req };1013 return .{ .http_request = .{
1014 .request = request,
1015 .head = response.head,
1016 .buffer = reader_buffer,
1017 } };
1022 }1018 }
10231019
1024 if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or1020 if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
...@@ -1026,7 +1022,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1026,7 +1022,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
1026 {1022 {
1027 var transport_uri = uri;1023 var transport_uri = uri;
1028 transport_uri.scheme = uri.scheme["git+".len..];1024 transport_uri.scheme = uri.scheme["git+".len..];
1029 var session = git.Session.init(gpa, http_client, transport_uri, server_header_buffer) catch |err| {1025 var session = git.Session.init(gpa, http_client, transport_uri, reader_buffer) catch |err| {
1030 return f.fail(f.location_tok, try eb.printString(1026 return f.fail(f.location_tok, try eb.printString(
1031 "unable to discover remote git server capabilities: {s}",1027 "unable to discover remote git server capabilities: {s}",
1032 .{@errorName(err)},1028 .{@errorName(err)},
...@@ -1042,16 +1038,12 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1042,16 +1038,12 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
1042 const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});1038 const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
1043 const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});1039 const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
10441040
1045 var ref_iterator = session.listRefs(.{1041 var ref_iterator: git.Session.RefIterator = undefined;
1042 session.listRefs(&ref_iterator, .{
1046 .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },1043 .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
1047 .include_peeled = true,1044 .include_peeled = true,
1048 .server_header_buffer = server_header_buffer,1045 .buffer = reader_buffer,
1049 }) catch |err| {1046 }) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
1050 return f.fail(f.location_tok, try eb.printString(
1051 "unable to list refs: {s}",
1052 .{@errorName(err)},
1053 ));
1054 };
1055 defer ref_iterator.deinit();1047 defer ref_iterator.deinit();
1056 while (ref_iterator.next() catch |err| {1048 while (ref_iterator.next() catch |err| {
1057 return f.fail(f.location_tok, try eb.printString(1049 return f.fail(f.location_tok, try eb.printString(
...@@ -1089,14 +1081,14 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1089,14 +1081,14 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
10891081
1090 var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;1082 var want_oid_buf: [git.Oid.max_formatted_length]u8 = undefined;
1091 _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;1083 _ = std.fmt.bufPrint(&want_oid_buf, "{f}", .{want_oid}) catch unreachable;
1092 var fetch_stream = session.fetch(&.{&want_oid_buf}, server_header_buffer) catch |err| {1084 var fetch_stream: git.Session.FetchStream = undefined;
1093 return f.fail(f.location_tok, try eb.printString(1085 session.fetch(&fetch_stream, &.{&want_oid_buf}, reader_buffer) catch |err| {
1094 "unable to create fetch stream: {s}",1086 return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
1095 .{@errorName(err)},
1096 ));
1097 };1087 };
1098 errdefer fetch_stream.deinit();1088 errdefer fetch_stream.deinit();
10991089
1090 if (true) @panic("TODO this moves fetch_stream, invalidating its reader");
1091
1100 return .{ .git = .{1092 return .{ .git = .{
1101 .session = session,1093 .session = session,
1102 .fetch_stream = fetch_stream,1094 .fetch_stream = fetch_stream,
...@@ -1104,10 +1096,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re...@@ -1104,10 +1096,7 @@ fn initResource(f: *Fetch, uri: std.Uri, server_header_buffer: []u8) RunError!Re
1104 } };1096 } };
1105 }1097 }
11061098
1107 return f.fail(f.location_tok, try eb.printString(1099 return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
1108 "unsupported URL scheme: {s}",
1109 .{uri.scheme},
1110 ));
1111}1100}
11121101
1113fn unpackResource(1102fn unpackResource(
...@@ -1121,9 +1110,11 @@ fn unpackResource(...@@ -1121,9 +1110,11 @@ fn unpackResource(
1121 .file => FileType.fromPath(uri_path) orelse1110 .file => FileType.fromPath(uri_path) orelse
1122 return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),1111 return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
11231112
1124 .http_request => |req| ft: {1113 .http_request => |*http_request| ft: {
1114 const head = &http_request.head;
1115
1125 // Content-Type takes first precedence.1116 // Content-Type takes first precedence.
1126 const content_type = req.response.content_type orelse1117 const content_type = head.content_type orelse
1127 return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));1118 return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
11281119
1129 // Extract the MIME type, ignoring charset and boundary directives1120 // Extract the MIME type, ignoring charset and boundary directives
...@@ -1165,7 +1156,7 @@ fn unpackResource(...@@ -1165,7 +1156,7 @@ fn unpackResource(
1165 }1156 }
11661157
1167 // Next, the filename from 'content-disposition: attachment' takes precedence.1158 // Next, the filename from 'content-disposition: attachment' takes precedence.
1168 if (req.response.content_disposition) |cd_header| {1159 if (head.content_disposition) |cd_header| {
1169 break :ft FileType.fromContentDisposition(cd_header) orelse {1160 break :ft FileType.fromContentDisposition(cd_header) orelse {
1170 return f.fail(f.location_tok, try eb.printString(1161 return f.fail(f.location_tok, try eb.printString(
1171 "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",1162 "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
...@@ -1176,10 +1167,7 @@ fn unpackResource(...@@ -1176,10 +1167,7 @@ fn unpackResource(
11761167
1177 // Finally, the path from the URI is used.1168 // Finally, the path from the URI is used.
1178 break :ft FileType.fromPath(uri_path) orelse {1169 break :ft FileType.fromPath(uri_path) orelse {
1179 return f.fail(f.location_tok, try eb.printString(1170 return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
1180 "unknown file type: '{s}'",
1181 .{uri_path},
1182 ));
1183 };1171 };
1184 },1172 },
11851173
...@@ -1187,10 +1175,9 @@ fn unpackResource(...@@ -1187,10 +1175,9 @@ fn unpackResource(
11871175
1188 .dir => |dir| {1176 .dir => |dir| {
1189 f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {1177 f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
1190 return f.fail(f.location_tok, try eb.printString(1178 return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
1191 "unable to copy directory '{s}': {s}",1179 uri_path, err,
1192 .{ uri_path, @errorName(err) },1180 }));
1193 ));
1194 };1181 };
1195 return .{};1182 return .{};
1196 },1183 },
...@@ -1198,15 +1185,11 @@ fn unpackResource(...@@ -1198,15 +1185,11 @@ fn unpackResource(
11981185
1199 switch (file_type) {1186 switch (file_type) {
1200 .tar => {1187 .tar => {
1201 var adapter_buffer: [1024]u8 = undefined;1188 return unpackTarball(f, tmp_directory.handle, resource.reader());
1202 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1203 return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1204 },1189 },
1205 .@"tar.gz" => {1190 .@"tar.gz" => {
1206 var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
1207 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1208 var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;1191 var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1209 var decompress: std.compress.flate.Decompress = .init(&adapter.new_interface, .gzip, &flate_buffer);1192 var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
1210 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);1193 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1211 },1194 },
1212 .@"tar.xz" => {1195 .@"tar.xz" => {
...@@ -1227,9 +1210,7 @@ fn unpackResource(...@@ -1227,9 +1210,7 @@ fn unpackResource(
1227 .@"tar.zst" => {1210 .@"tar.zst" => {
1228 const window_size = std.compress.zstd.default_window_len;1211 const window_size = std.compress.zstd.default_window_len;
1229 const window_buffer = try f.arena.allocator().create([window_size]u8);1212 const window_buffer = try f.arena.allocator().create([window_size]u8);
1230 var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;1213 var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
1231 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1232 var decompress: std.compress.zstd.Decompress = .init(&adapter.new_interface, window_buffer, .{
1233 .verify_checksum = false,1214 .verify_checksum = false,
1234 });1215 });
1235 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);1216 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
...@@ -1237,12 +1218,15 @@ fn unpackResource(...@@ -1237,12 +1218,15 @@ fn unpackResource(
1237 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {1218 .git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
1238 error.FetchFailed => return error.FetchFailed,1219 error.FetchFailed => return error.FetchFailed,
1239 error.OutOfMemory => return error.OutOfMemory,1220 error.OutOfMemory => return error.OutOfMemory,
1240 else => |e| return f.fail(f.location_tok, try eb.printString(1221 else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
1241 "unable to unpack git files: {s}",1222 },
1242 .{@errorName(e)},1223 .zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) {
1224 error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
1225 "failed reading resource: {t}",
1226 .{err},
1243 )),1227 )),
1228 else => |e| return e,
1244 },1229 },
1245 .zip => return try unzip(f, tmp_directory.handle, resource.reader()),
1246 }1230 }
1247}1231}
12481232
...@@ -1277,99 +1261,69 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *std.Io.Reader) RunError!Un...@@ -1277,99 +1261,69 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: *std.Io.Reader) RunError!Un
1277 return res;1261 return res;
1278}1262}
12791263
1280fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {1264fn unzip(f: *Fetch, out_dir: fs.Dir, reader: *std.Io.Reader) error{ ReadFailed, OutOfMemory, FetchFailed }!UnpackResult {
1281 // We write the entire contents to a file first because zip files1265 // We write the entire contents to a file first because zip files
1282 // must be processed back to front and they could be too large to1266 // must be processed back to front and they could be too large to
1283 // load into memory.1267 // load into memory.
12841268
1285 const cache_root = f.job_queue.global_cache;1269 const cache_root = f.job_queue.global_cache;
1286
1287 // TODO: the downside of this solution is if we get a failure/crash/oom/power out
1288 // during this process, we leave behind a zip file that would be
1289 // difficult to know if/when it can be cleaned up.
1290 // Might be worth it to use a mechanism that enables other processes
1291 // to see if the owning process of a file is still alive (on linux this
1292 // can be done with file locks).
1293 // Coupled with this mechansism, we could also use slots (i.e. zig-cache/tmp/0,
1294 // zig-cache/tmp/1, etc) which would mean that subsequent runs would
1295 // automatically clean up old dead files.
1296 // This could all be done with a simple TmpFile abstraction.
1297 const prefix = "tmp/";1270 const prefix = "tmp/";
1298 const suffix = ".zip";1271 const suffix = ".zip";
1299
1300 const random_bytes_count = 20;
1301 const random_path_len = comptime std.fs.base64_encoder.calcSize(random_bytes_count);
1302 var zip_path: [prefix.len + random_path_len + suffix.len]u8 = undefined;
1303 @memcpy(zip_path[0..prefix.len], prefix);
1304 @memcpy(zip_path[prefix.len + random_path_len ..], suffix);
1305 {
1306 var random_bytes: [random_bytes_count]u8 = undefined;
1307 std.crypto.random.bytes(&random_bytes);
1308 _ = std.fs.base64_encoder.encode(
1309 zip_path[prefix.len..][0..random_path_len],
1310 &random_bytes,
1311 );
1312 }
1313
1314 defer cache_root.handle.deleteFile(&zip_path) catch {};
1315
1316 const eb = &f.error_bundle;1272 const eb = &f.error_bundle;
13171273 const random_len = @sizeOf(u64) * 2;
1318 {1274
1319 var zip_file = cache_root.handle.createFile(1275 var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
1320 &zip_path,1276 zip_path[0..prefix.len].* = prefix.*;
1321 .{},1277 zip_path[prefix.len + random_len ..].* = suffix.*;
1322 ) catch |err| return f.fail(f.location_tok, try eb.printString(1278
1323 "failed to create tmp zip file: {s}",1279 var zip_file = while (true) {
1324 .{@errorName(err)},1280 const random_integer = std.crypto.random.int(u64);
1325 ));1281 zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
1326 defer zip_file.close();1282
1327 var buf: [4096]u8 = undefined;1283 break cache_root.handle.createFile(&zip_path, .{
1328 while (true) {1284 .exclusive = true,
1329 const len = reader.readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString(1285 .read = true,
1330 "read zip stream failed: {s}",1286 }) catch |err| switch (err) {
1331 .{@errorName(err)},1287 error.PathAlreadyExists => continue,
1332 ));1288 else => |e| return f.fail(
1333 if (len == 0) break;1289 f.location_tok,
1334 zip_file.deprecatedWriter().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString(1290 try eb.printString("failed to create temporary zip file: {t}", .{e}),
1335 "write temporary zip file failed: {s}",1291 ),
1336 .{@errorName(err)},1292 };
1337 ));1293 };
1338 }1294 defer zip_file.close();
1339 }1295 var zip_file_buffer: [4096]u8 = undefined;
1296 var zip_file_reader = b: {
1297 var zip_file_writer = zip_file.writer(&zip_file_buffer);
1298
1299 _ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) {
1300 error.ReadFailed => return error.ReadFailed,
1301 error.WriteFailed => return f.fail(
1302 f.location_tok,
1303 try eb.printString("failed writing temporary zip file: {t}", .{err}),
1304 ),
1305 };
1306 zip_file_writer.interface.flush() catch |err| return f.fail(
1307 f.location_tok,
1308 try eb.printString("failed writing temporary zip file: {t}", .{err}),
1309 );
1310 break :b zip_file_writer.moveToReader();
1311 };
13401312
1341 var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };1313 var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
1342 // no need to deinit since we are using an arena allocator1314 // no need to deinit since we are using an arena allocator
13431315
1344 {1316 zip_file_reader.seekTo(0) catch |err|
1345 var zip_file = cache_root.handle.openFile(1317 return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
1346 &zip_path,1318 std.zip.extract(out_dir, &zip_file_reader, .{
1347 .{},1319 .allow_backslashes = true,
1348 ) catch |err| return f.fail(f.location_tok, try eb.printString(1320 .diagnostics = &diagnostics,
1349 "failed to open temporary zip file: {s}",1321 }) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
1350 .{@errorName(err)},
1351 ));
1352 defer zip_file.close();
1353
1354 var zip_file_buffer: [1024]u8 = undefined;
1355 var zip_file_reader = zip_file.reader(&zip_file_buffer);
1356
1357 std.zip.extract(out_dir, &zip_file_reader, .{
1358 .allow_backslashes = true,
1359 .diagnostics = &diagnostics,
1360 }) catch |err| return f.fail(f.location_tok, try eb.printString(
1361 "zip extract failed: {s}",
1362 .{@errorName(err)},
1363 ));
1364 }
13651322
1366 cache_root.handle.deleteFile(&zip_path) catch |err| return f.fail(f.location_tok, try eb.printString(1323 cache_root.handle.deleteFile(&zip_path) catch |err|
1367 "delete temporary zip failed: {s}",1324 return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
1368 .{@errorName(err)},
1369 ));
13701325
1371 const res: UnpackResult = .{ .root_dir = diagnostics.root_dir };1326 return .{ .root_dir = diagnostics.root_dir };
1372 return res;
1373}1327}
13741328
1375fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult {1329fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!UnpackResult {
...@@ -1387,10 +1341,13 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U...@@ -1387,10 +1341,13 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U
1387 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });1341 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1388 defer pack_file.close();1342 defer pack_file.close();
1389 var pack_file_buffer: [4096]u8 = undefined;1343 var pack_file_buffer: [4096]u8 = undefined;
1390 var fifo = std.fifo.LinearFifo(u8, .{ .Slice = {} }).init(&pack_file_buffer);1344 var pack_file_reader = b: {
1391 try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());1345 var pack_file_writer = pack_file.writer(&pack_file_buffer);
13921346 const fetch_reader = &resource.fetch_stream.reader;
1393 var pack_file_reader = pack_file.reader(&pack_file_buffer);1347 _ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
1348 try pack_file_writer.interface.flush();
1349 break :b pack_file_writer.moveToReader();
1350 };
13941351
1395 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });1352 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
1396 defer index_file.close();1353 defer index_file.close();
src/Package/Fetch/git.zig+156-131
...@@ -585,17 +585,17 @@ const ObjectCache = struct {...@@ -585,17 +585,17 @@ const ObjectCache = struct {
585/// [protocol-common](https://git-scm.com/docs/protocol-common). The special585/// [protocol-common](https://git-scm.com/docs/protocol-common). The special
586/// meanings of the delimiter and response-end packets are documented in586/// meanings of the delimiter and response-end packets are documented in
587/// [protocol-v2](https://git-scm.com/docs/protocol-v2).587/// [protocol-v2](https://git-scm.com/docs/protocol-v2).
588const Packet = union(enum) {588pub const Packet = union(enum) {
589 flush,589 flush,
590 delimiter,590 delimiter,
591 response_end,591 response_end,
592 data: []const u8,592 data: []const u8,
593593
594 const max_data_length = 65516;594 pub const max_data_length = 65516;
595595
596 /// Reads a packet in pkt-line format.596 /// Reads a packet in pkt-line format.
597 fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {597 fn read(reader: *std.Io.Reader) !Packet {
598 const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;598 const length = std.fmt.parseUnsigned(u16, try reader.take(4), 16) catch return error.InvalidPacket;
599 switch (length) {599 switch (length) {
600 0 => return .flush,600 0 => return .flush,
601 1 => return .delimiter,601 1 => return .delimiter,
...@@ -603,13 +603,11 @@ const Packet = union(enum) {...@@ -603,13 +603,11 @@ const Packet = union(enum) {
603 3 => return error.InvalidPacket,603 3 => return error.InvalidPacket,
604 else => if (length - 4 > max_data_length) return error.InvalidPacket,604 else => if (length - 4 > max_data_length) return error.InvalidPacket,
605 }605 }
606 const data = buf[0 .. length - 4];606 return .{ .data = try reader.take(length - 4) };
607 try reader.readNoEof(data);
608 return .{ .data = data };
609 }607 }
610608
611 /// Writes a packet in pkt-line format.609 /// Writes a packet in pkt-line format.
612 fn write(packet: Packet, writer: anytype) !void {610 fn write(packet: Packet, writer: *std.Io.Writer) !void {
613 switch (packet) {611 switch (packet) {
614 .flush => try writer.writeAll("0000"),612 .flush => try writer.writeAll("0000"),
615 .delimiter => try writer.writeAll("0001"),613 .delimiter => try writer.writeAll("0001"),
...@@ -657,8 +655,10 @@ pub const Session = struct {...@@ -657,8 +655,10 @@ pub const Session = struct {
657 allocator: Allocator,655 allocator: Allocator,
658 transport: *std.http.Client,656 transport: *std.http.Client,
659 uri: std.Uri,657 uri: std.Uri,
660 http_headers_buffer: []u8,658 /// Asserted to be at least `Packet.max_data_length`
659 response_buffer: []u8,
661 ) !Session {660 ) !Session {
661 assert(response_buffer.len >= Packet.max_data_length);
662 var session: Session = .{662 var session: Session = .{
663 .transport = transport,663 .transport = transport,
664 .location = try .init(allocator, uri),664 .location = try .init(allocator, uri),
...@@ -668,7 +668,8 @@ pub const Session = struct {...@@ -668,7 +668,8 @@ pub const Session = struct {
668 .allocator = allocator,668 .allocator = allocator,
669 };669 };
670 errdefer session.deinit();670 errdefer session.deinit();
671 var capability_iterator = try session.getCapabilities(http_headers_buffer);671 var capability_iterator: CapabilityIterator = undefined;
672 try session.getCapabilities(&capability_iterator, response_buffer);
672 defer capability_iterator.deinit();673 defer capability_iterator.deinit();
673 while (try capability_iterator.next()) |capability| {674 while (try capability_iterator.next()) |capability| {
674 if (mem.eql(u8, capability.key, "agent")) {675 if (mem.eql(u8, capability.key, "agent")) {
...@@ -743,7 +744,8 @@ pub const Session = struct {...@@ -743,7 +744,8 @@ pub const Session = struct {
743 ///744 ///
744 /// The `session.location` is updated if the server returns a redirect, so745 /// The `session.location` is updated if the server returns a redirect, so
745 /// that subsequent session functions do not need to handle redirects.746 /// that subsequent session functions do not need to handle redirects.
746 fn getCapabilities(session: *Session, http_headers_buffer: []u8) !CapabilityIterator {747 fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void {
748 assert(response_buffer.len >= Packet.max_data_length);
747 var info_refs_uri = session.location.uri;749 var info_refs_uri = session.location.uri;
748 {750 {
749 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{751 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
...@@ -757,19 +759,22 @@ pub const Session = struct {...@@ -757,19 +759,22 @@ pub const Session = struct {
757 info_refs_uri.fragment = null;759 info_refs_uri.fragment = null;
758760
759 const max_redirects = 3;761 const max_redirects = 3;
760 var request = try session.transport.open(.GET, info_refs_uri, .{762 it.* = .{
761 .redirect_behavior = @enumFromInt(max_redirects),763 .request = try session.transport.request(.GET, info_refs_uri, .{
762 .server_header_buffer = http_headers_buffer,764 .redirect_behavior = .init(max_redirects),
763 .extra_headers = &.{765 .extra_headers = &.{
764 .{ .name = "Git-Protocol", .value = "version=2" },766 .{ .name = "Git-Protocol", .value = "version=2" },
765 },767 },
766 });768 }),
767 errdefer request.deinit();769 .reader = undefined,
768 try request.send();770 };
769 try request.finish();771 errdefer it.deinit();
772 const request = &it.request;
773 try request.sendBodiless();
770774
771 try request.wait();775 var redirect_buffer: [1024]u8 = undefined;
772 if (request.response.status != .ok) return error.ProtocolError;776 const response = try request.receiveHead(&redirect_buffer);
777 if (response.head.status != .ok) return error.ProtocolError;
773 const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;778 const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
774 if (any_redirects_occurred) {779 if (any_redirects_occurred) {
775 const request_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{780 const request_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
...@@ -784,8 +789,7 @@ pub const Session = struct {...@@ -784,8 +789,7 @@ pub const Session = struct {
784 session.location = new_location;789 session.location = new_location;
785 }790 }
786791
787 const reader = request.reader();792 it.reader = response.reader(response_buffer);
788 var buf: [Packet.max_data_length]u8 = undefined;
789 var state: enum { response_start, response_content } = .response_start;793 var state: enum { response_start, response_content } = .response_start;
790 while (true) {794 while (true) {
791 // Some Git servers (at least GitHub) include an additional795 // Some Git servers (at least GitHub) include an additional
...@@ -795,15 +799,15 @@ pub const Session = struct {...@@ -795,15 +799,15 @@ pub const Session = struct {
795 // Thus, we need to skip any such useless additional responses799 // Thus, we need to skip any such useless additional responses
796 // before we get the one we're actually looking for. The responses800 // before we get the one we're actually looking for. The responses
797 // will be delimited by flush packets.801 // will be delimited by flush packets.
798 const packet = Packet.read(reader, &buf) catch |e| switch (e) {802 const packet = Packet.read(it.reader) catch |err| switch (err) {
799 error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found803 error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found
800 else => |other| return other,804 else => |e| return e,
801 };805 };
802 switch (packet) {806 switch (packet) {
803 .flush => state = .response_start,807 .flush => state = .response_start,
804 .data => |data| switch (state) {808 .data => |data| switch (state) {
805 .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {809 .response_start => if (mem.eql(u8, Packet.normalizeText(data), "version 2")) {
806 return .{ .request = request };810 return;
807 } else {811 } else {
808 state = .response_content;812 state = .response_content;
809 },813 },
...@@ -816,7 +820,7 @@ pub const Session = struct {...@@ -816,7 +820,7 @@ pub const Session = struct {
816820
817 const CapabilityIterator = struct {821 const CapabilityIterator = struct {
818 request: std.http.Client.Request,822 request: std.http.Client.Request,
819 buf: [Packet.max_data_length]u8 = undefined,823 reader: *std.Io.Reader,
820824
821 const Capability = struct {825 const Capability = struct {
822 key: []const u8,826 key: []const u8,
...@@ -830,13 +834,13 @@ pub const Session = struct {...@@ -830,13 +834,13 @@ pub const Session = struct {
830 }834 }
831 };835 };
832836
833 fn deinit(iterator: *CapabilityIterator) void {837 fn deinit(it: *CapabilityIterator) void {
834 iterator.request.deinit();838 it.request.deinit();
835 iterator.* = undefined;839 it.* = undefined;
836 }840 }
837841
838 fn next(iterator: *CapabilityIterator) !?Capability {842 fn next(it: *CapabilityIterator) !?Capability {
839 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {843 switch (try Packet.read(it.reader)) {
840 .flush => return null,844 .flush => return null,
841 .data => |data| return Capability.parse(Packet.normalizeText(data)),845 .data => |data| return Capability.parse(Packet.normalizeText(data)),
842 else => return error.UnexpectedPacket,846 else => return error.UnexpectedPacket,
...@@ -854,11 +858,13 @@ pub const Session = struct {...@@ -854,11 +858,13 @@ pub const Session = struct {
854 include_symrefs: bool = false,858 include_symrefs: bool = false,
855 /// Whether to include the peeled object ID for returned tag refs.859 /// Whether to include the peeled object ID for returned tag refs.
856 include_peeled: bool = false,860 include_peeled: bool = false,
857 server_header_buffer: []u8,861 /// Asserted to be at least `Packet.max_data_length`.
862 buffer: []u8,
858 };863 };
859864
860 /// Returns an iterator over refs known to the server.865 /// Returns an iterator over refs known to the server.
861 pub fn listRefs(session: Session, options: ListRefsOptions) !RefIterator {866 pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
867 assert(options.buffer.len >= Packet.max_data_length);
862 var upload_pack_uri = session.location.uri;868 var upload_pack_uri = session.location.uri;
863 {869 {
864 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{870 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
...@@ -871,59 +877,56 @@ pub const Session = struct {...@@ -871,59 +877,56 @@ pub const Session = struct {
871 upload_pack_uri.query = null;877 upload_pack_uri.query = null;
872 upload_pack_uri.fragment = null;878 upload_pack_uri.fragment = null;
873879
874 var body: std.ArrayListUnmanaged(u8) = .empty;880 var body: std.Io.Writer = .fixed(options.buffer);
875 defer body.deinit(session.allocator);881 try Packet.write(.{ .data = "command=ls-refs\n" }, &body);
876 const body_writer = body.writer(session.allocator);
877 try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer);
878 if (session.supports_agent) {882 if (session.supports_agent) {
879 try Packet.write(.{ .data = agent_capability }, body_writer);883 try Packet.write(.{ .data = agent_capability }, &body);
880 }884 }
881 {885 {
882 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});886 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={t}\n", .{
887 session.object_format,
888 });
883 defer session.allocator.free(object_format_packet);889 defer session.allocator.free(object_format_packet);
884 try Packet.write(.{ .data = object_format_packet }, body_writer);890 try Packet.write(.{ .data = object_format_packet }, &body);
885 }891 }
886 try Packet.write(.delimiter, body_writer);892 try Packet.write(.delimiter, &body);
887 for (options.ref_prefixes) |ref_prefix| {893 for (options.ref_prefixes) |ref_prefix| {
888 const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix});894 const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix});
889 defer session.allocator.free(ref_prefix_packet);895 defer session.allocator.free(ref_prefix_packet);
890 try Packet.write(.{ .data = ref_prefix_packet }, body_writer);896 try Packet.write(.{ .data = ref_prefix_packet }, &body);
891 }897 }
892 if (options.include_symrefs) {898 if (options.include_symrefs) {
893 try Packet.write(.{ .data = "symrefs\n" }, body_writer);899 try Packet.write(.{ .data = "symrefs\n" }, &body);
894 }900 }
895 if (options.include_peeled) {901 if (options.include_peeled) {
896 try Packet.write(.{ .data = "peel\n" }, body_writer);902 try Packet.write(.{ .data = "peel\n" }, &body);
897 }903 }
898 try Packet.write(.flush, body_writer);904 try Packet.write(.flush, &body);
899905
900 var request = try session.transport.open(.POST, upload_pack_uri, .{906 it.* = .{
901 .redirect_behavior = .unhandled,907 .request = try session.transport.request(.POST, upload_pack_uri, .{
902 .server_header_buffer = options.server_header_buffer,908 .redirect_behavior = .unhandled,
903 .extra_headers = &.{909 .extra_headers = &.{
904 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },910 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
905 .{ .name = "Git-Protocol", .value = "version=2" },911 .{ .name = "Git-Protocol", .value = "version=2" },
906 },912 },
907 });913 }),
908 errdefer request.deinit();914 .reader = undefined,
909 request.transfer_encoding = .{ .content_length = body.items.len };
910 try request.send();
911 try request.writeAll(body.items);
912 try request.finish();
913
914 try request.wait();
915 if (request.response.status != .ok) return error.ProtocolError;
916
917 return .{
918 .format = session.object_format,915 .format = session.object_format,
919 .request = request,
920 };916 };
917 const request = &it.request;
918 errdefer request.deinit();
919 try request.sendBodyComplete(body.buffered());
920
921 const response = try request.receiveHead(options.buffer);
922 if (response.head.status != .ok) return error.ProtocolError;
923 it.reader = response.reader(options.buffer);
921 }924 }
922925
923 pub const RefIterator = struct {926 pub const RefIterator = struct {
924 format: Oid.Format,927 format: Oid.Format,
925 request: std.http.Client.Request,928 request: std.http.Client.Request,
926 buf: [Packet.max_data_length]u8 = undefined,929 reader: *std.Io.Reader,
927930
928 pub const Ref = struct {931 pub const Ref = struct {
929 oid: Oid,932 oid: Oid,
...@@ -937,13 +940,13 @@ pub const Session = struct {...@@ -937,13 +940,13 @@ pub const Session = struct {
937 iterator.* = undefined;940 iterator.* = undefined;
938 }941 }
939942
940 pub fn next(iterator: *RefIterator) !?Ref {943 pub fn next(it: *RefIterator) !?Ref {
941 switch (try Packet.read(iterator.request.reader(), &iterator.buf)) {944 switch (try Packet.read(it.reader)) {
942 .flush => return null,945 .flush => return null,
943 .data => |data| {946 .data => |data| {
944 const ref_data = Packet.normalizeText(data);947 const ref_data = Packet.normalizeText(data);
945 const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;948 const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket;
946 const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;949 const oid = Oid.parse(it.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket;
947950
948 const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;951 const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len;
949 const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];952 const name = ref_data[oid_sep_pos + 1 .. name_sep_pos];
...@@ -957,7 +960,7 @@ pub const Session = struct {...@@ -957,7 +960,7 @@ pub const Session = struct {
957 if (mem.startsWith(u8, attribute, "symref-target:")) {960 if (mem.startsWith(u8, attribute, "symref-target:")) {
958 symref_target = attribute["symref-target:".len..];961 symref_target = attribute["symref-target:".len..];
959 } else if (mem.startsWith(u8, attribute, "peeled:")) {962 } else if (mem.startsWith(u8, attribute, "peeled:")) {
960 peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;963 peeled = Oid.parse(it.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket;
961 }964 }
962 last_sep_pos = next_sep_pos;965 last_sep_pos = next_sep_pos;
963 }966 }
...@@ -973,9 +976,12 @@ pub const Session = struct {...@@ -973,9 +976,12 @@ pub const Session = struct {
973 /// performed if the server supports it.976 /// performed if the server supports it.
974 pub fn fetch(977 pub fn fetch(
975 session: Session,978 session: Session,
979 fs: *FetchStream,
976 wants: []const []const u8,980 wants: []const []const u8,
977 http_headers_buffer: []u8,981 /// Asserted to be at least `Packet.max_data_length`.
978 ) !FetchStream {982 response_buffer: []u8,
983 ) !void {
984 assert(response_buffer.len >= Packet.max_data_length);
979 var upload_pack_uri = session.location.uri;985 var upload_pack_uri = session.location.uri;
980 {986 {
981 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{987 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
...@@ -988,63 +994,71 @@ pub const Session = struct {...@@ -988,63 +994,71 @@ pub const Session = struct {
988 upload_pack_uri.query = null;994 upload_pack_uri.query = null;
989 upload_pack_uri.fragment = null;995 upload_pack_uri.fragment = null;
990996
991 var body: std.ArrayListUnmanaged(u8) = .empty;997 var body: std.Io.Writer = .fixed(response_buffer);
992 defer body.deinit(session.allocator);998 try Packet.write(.{ .data = "command=fetch\n" }, &body);
993 const body_writer = body.writer(session.allocator);
994 try Packet.write(.{ .data = "command=fetch\n" }, body_writer);
995 if (session.supports_agent) {999 if (session.supports_agent) {
996 try Packet.write(.{ .data = agent_capability }, body_writer);1000 try Packet.write(.{ .data = agent_capability }, &body);
997 }1001 }
998 {1002 {
999 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});1003 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});
1000 defer session.allocator.free(object_format_packet);1004 defer session.allocator.free(object_format_packet);
1001 try Packet.write(.{ .data = object_format_packet }, body_writer);1005 try Packet.write(.{ .data = object_format_packet }, &body);
1002 }1006 }
1003 try Packet.write(.delimiter, body_writer);1007 try Packet.write(.delimiter, &body);
1004 // Our packfile parser supports the OFS_DELTA object type1008 // Our packfile parser supports the OFS_DELTA object type
1005 try Packet.write(.{ .data = "ofs-delta\n" }, body_writer);1009 try Packet.write(.{ .data = "ofs-delta\n" }, &body);
1006 // We do not currently convey server progress information to the user1010 // We do not currently convey server progress information to the user
1007 try Packet.write(.{ .data = "no-progress\n" }, body_writer);1011 try Packet.write(.{ .data = "no-progress\n" }, &body);
1008 if (session.supports_shallow) {1012 if (session.supports_shallow) {
1009 try Packet.write(.{ .data = "deepen 1\n" }, body_writer);1013 try Packet.write(.{ .data = "deepen 1\n" }, &body);
1010 }1014 }
1011 for (wants) |want| {1015 for (wants) |want| {
1012 var buf: [Packet.max_data_length]u8 = undefined;1016 var buf: [Packet.max_data_length]u8 = undefined;
1013 const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;1017 const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable;
1014 try Packet.write(.{ .data = arg }, body_writer);1018 try Packet.write(.{ .data = arg }, &body);
1015 }1019 }
1016 try Packet.write(.{ .data = "done\n" }, body_writer);1020 try Packet.write(.{ .data = "done\n" }, &body);
1017 try Packet.write(.flush, body_writer);1021 try Packet.write(.flush, &body);
10181022
1019 var request = try session.transport.open(.POST, upload_pack_uri, .{1023 fs.* = .{
1020 .redirect_behavior = .not_allowed,1024 .request = try session.transport.request(.POST, upload_pack_uri, .{
1021 .server_header_buffer = http_headers_buffer,1025 .redirect_behavior = .not_allowed,
1022 .extra_headers = &.{1026 .extra_headers = &.{
1023 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },1027 .{ .name = "Content-Type", .value = "application/x-git-upload-pack-request" },
1024 .{ .name = "Git-Protocol", .value = "version=2" },1028 .{ .name = "Git-Protocol", .value = "version=2" },
1025 },1029 },
1026 });1030 }),
1031 .input = undefined,
1032 .reader = undefined,
1033 .remaining_len = undefined,
1034 };
1035 const request = &fs.request;
1027 errdefer request.deinit();1036 errdefer request.deinit();
1028 request.transfer_encoding = .{ .content_length = body.items.len };
1029 try request.send();
1030 try request.writeAll(body.items);
1031 try request.finish();
10321037
1033 try request.wait();1038 try request.sendBodyComplete(body.buffered());
1034 if (request.response.status != .ok) return error.ProtocolError;1039
1040 const response = try request.receiveHead(&.{});
1041 if (response.head.status != .ok) return error.ProtocolError;
10351042
1036 const reader = request.reader();1043 const reader = response.reader(response_buffer);
1037 // We are not interested in any of the sections of the returned fetch1044 // We are not interested in any of the sections of the returned fetch
1038 // data other than the packfile section, since we aren't doing anything1045 // data other than the packfile section, since we aren't doing anything
1039 // complex like ref negotiation (this is a fresh clone).1046 // complex like ref negotiation (this is a fresh clone).
1040 var state: enum { section_start, section_content } = .section_start;1047 var state: enum { section_start, section_content } = .section_start;
1041 while (true) {1048 while (true) {
1042 var buf: [Packet.max_data_length]u8 = undefined;1049 const packet = try Packet.read(reader);
1043 const packet = try Packet.read(reader, &buf);
1044 switch (state) {1050 switch (state) {
1045 .section_start => switch (packet) {1051 .section_start => switch (packet) {
1046 .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {1052 .data => |data| if (mem.eql(u8, Packet.normalizeText(data), "packfile")) {
1047 return .{ .request = request };1053 fs.input = reader;
1054 fs.reader = .{
1055 .buffer = &.{},
1056 .vtable = &.{ .stream = FetchStream.stream },
1057 .seek = 0,
1058 .end = 0,
1059 };
1060 fs.remaining_len = 0;
1061 return;
1048 } else {1062 } else {
1049 state = .section_content;1063 state = .section_content;
1050 },1064 },
...@@ -1061,20 +1075,23 @@ pub const Session = struct {...@@ -1061,20 +1075,23 @@ pub const Session = struct {
10611075
1062 pub const FetchStream = struct {1076 pub const FetchStream = struct {
1063 request: std.http.Client.Request,1077 request: std.http.Client.Request,
1064 buf: [Packet.max_data_length]u8 = undefined,1078 input: *std.Io.Reader,
1065 pos: usize = 0,1079 reader: std.Io.Reader,
1066 len: usize = 0,1080 err: ?Error = null,
1081 remaining_len: usize,
10671082
1068 pub fn deinit(stream: *FetchStream) void {1083 pub fn deinit(fs: *FetchStream) void {
1069 stream.request.deinit();1084 fs.request.deinit();
1070 }1085 }
10711086
1072 pub const ReadError = std.http.Client.Request.ReadError || error{1087 pub const Error = error{
1073 InvalidPacket,1088 InvalidPacket,
1074 ProtocolError,1089 ProtocolError,
1075 UnexpectedPacket,1090 UnexpectedPacket,
1091 WriteFailed,
1092 ReadFailed,
1093 EndOfStream,
1076 };1094 };
1077 pub const Reader = std.io.GenericReader(*FetchStream, ReadError, read);
10781095
1079 const StreamCode = enum(u8) {1096 const StreamCode = enum(u8) {
1080 pack_data = 1,1097 pack_data = 1,
...@@ -1083,33 +1100,41 @@ pub const Session = struct {...@@ -1083,33 +1100,41 @@ pub const Session = struct {
1083 _,1100 _,
1084 };1101 };
10851102
1086 pub fn reader(stream: *FetchStream) Reader {1103 pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize {
1087 return .{ .context = stream };1104 const fs: *FetchStream = @alignCast(@fieldParentPtr("reader", r));
1088 }1105 const input = fs.input;
10891106 if (fs.remaining_len == 0) {
1090 pub fn read(stream: *FetchStream, buf: []u8) !usize {
1091 if (stream.pos == stream.len) {
1092 while (true) {1107 while (true) {
1093 switch (try Packet.read(stream.request.reader(), &stream.buf)) {1108 switch (Packet.read(input) catch |err| {
1094 .flush => return 0,1109 fs.err = err;
1110 return error.ReadFailed;
1111 }) {
1112 .flush => return error.EndOfStream,
1095 .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {1113 .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) {
1096 .pack_data => {1114 .pack_data => {
1097 stream.pos = 1;1115 input.toss(1);
1098 stream.len = data.len;1116 fs.remaining_len = data.len;
1099 break;1117 break;
1100 },1118 },
1101 .fatal_error => return error.ProtocolError,1119 .fatal_error => {
1120 fs.err = error.ProtocolError;
1121 return error.ReadFailed;
1122 },
1102 else => {},1123 else => {},
1103 },1124 },
1104 else => return error.UnexpectedPacket,1125 else => {
1126 fs.err = error.UnexpectedPacket;
1127 return error.ReadFailed;
1128 },
1105 }1129 }
1106 }1130 }
1107 }1131 }
11081132 const buf = limit.slice(try w.writableSliceGreedy(1));
1109 const size = @min(buf.len, stream.len - stream.pos);1133 const n = @min(buf.len, fs.remaining_len);
1110 @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]);1134 @memcpy(buf[0..n], input.buffered()[0..n]);
1111 stream.pos += size;1135 input.toss(n);
1112 return size;1136 fs.remaining_len -= n;
1137 return n;
1113 }1138 }
1114 };1139 };
1115};1140};