authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-05 16:46:20-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-05 16:50:41-08:00
log1f65e7cccc3124ce2fa9a5e8107c32367a39ec29
treeb93ddf97b0ccb7607b36a9fd30015a59919eeea5
parent7246eee1e706b142abf8183e960fd692fde52bb0

fetch: recompress task integrates with std.Progress

2 files changed, 24 insertions(+), 8 deletions(-)

src/Package/Fetch.zig+22-8
...@@ -114,6 +114,7 @@ pub const JobQueue = struct {...@@ -114,6 +114,7 @@ pub const JobQueue = struct {
114 /// field contains references to all of them.114 /// field contains references to all of them.
115 /// Protected by `mutex`.115 /// Protected by `mutex`.
116 all_fetches: std.ArrayList(*Fetch) = .empty,116 all_fetches: std.ArrayList(*Fetch) = .empty,
117 prog_node: std.Progress.Node,
117118
118 http_client: *std.http.Client,119 http_client: *std.http.Client,
119 /// This tracks `Fetch` tasks as well as recompression tasks.120 /// This tracks `Fetch` tasks as well as recompression tasks.
...@@ -302,12 +303,15 @@ pub const JobQueue = struct {...@@ -302,12 +303,15 @@ pub const JobQueue = struct {
302 }303 }
303304
304 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {305 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
305 var dest_sub_path_buffer: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;306 const pkg_hash_slice = package_hash.toSlice();
307
308 const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
309 defer prog_node.end();
310
311 var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
306 const dest_path: Cache.Path = .{312 const dest_path: Cache.Path = .{
307 .root_dir = jq.global_cache,313 .root_dir = jq.global_cache,
308 .sub_path = std.fmt.bufPrint(&dest_sub_path_buffer, "p/{s}.tar.gz", .{314 .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
309 package_hash.toSlice(),
310 }) catch unreachable,
311 };315 };
312316
313 const gpa = jq.http_client.allocator;317 const gpa = jq.http_client.allocator;
...@@ -316,7 +320,7 @@ pub const JobQueue = struct {...@@ -316,7 +320,7 @@ pub const JobQueue = struct {
316 defer arena_instance.deinit();320 defer arena_instance.deinit();
317 const arena = arena_instance.allocator();321 const arena = arena_instance.allocator();
318322
319 recompressFallible(jq, arena, dest_path, package_hash.toSlice()) catch |err| switch (err) {323 recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) {
320 error.Canceled => |e| return e,324 error.Canceled => |e| return e,
321 error.ReadFailed => comptime unreachable,325 error.ReadFailed => comptime unreachable,
322 error.WriteFailed => comptime unreachable,326 error.WriteFailed => comptime unreachable,
...@@ -324,7 +328,13 @@ pub const JobQueue = struct {...@@ -324,7 +328,13 @@ pub const JobQueue = struct {
324 };328 };
325 }329 }
326330
327 fn recompressFallible(jq: *JobQueue, arena: Allocator, dest_path: Cache.Path, package_hash: []const u8) !void {331 fn recompressFallible(
332 jq: *JobQueue,
333 arena: Allocator,
334 dest_path: Cache.Path,
335 pkg_hash_slice: []const u8,
336 prog_node: std.Progress.Node,
337 ) !void {
328 const gpa = jq.http_client.allocator;338 const gpa = jq.http_client.allocator;
329 const io = jq.io;339 const io = jq.io;
330340
...@@ -337,7 +347,7 @@ pub const JobQueue = struct {...@@ -337,7 +347,7 @@ pub const JobQueue = struct {
337 var scanned_files: std.ArrayList([]const u8) = .empty;347 var scanned_files: std.ArrayList([]const u8) = .empty;
338 defer scanned_files.deinit(gpa);348 defer scanned_files.deinit(gpa);
339349
340 var pkg_dir = try jq.root_pkg_path.openDir(io, package_hash, .{ .iterate = true });350 var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });
341 defer pkg_dir.close(io);351 defer pkg_dir.close(io);
342352
343 {353 {
...@@ -359,6 +369,8 @@ pub const JobQueue = struct {...@@ -359,6 +369,8 @@ pub const JobQueue = struct {
359 std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);369 std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);
360 }370 }
361371
372 prog_node.setEstimatedTotalItems(scanned_files.items.len);
373
362 var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{374 var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
363 .make_path = true,375 .make_path = true,
364 .replace = true,376 .replace = true,
...@@ -374,7 +386,7 @@ pub const JobQueue = struct {...@@ -374,7 +386,7 @@ pub const JobQueue = struct {
374 };386 };
375387
376 var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };388 var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
377 archiver.prefix = package_hash;389 archiver.prefix = pkg_hash_slice;
378390
379 var file_read_buffer: [4096]u8 = undefined;391 var file_read_buffer: [4096]u8 = undefined;
380392
...@@ -387,6 +399,7 @@ pub const JobQueue = struct {...@@ -387,6 +399,7 @@ pub const JobQueue = struct {
387 error.WriteFailed => return file_writer.err.?,399 error.WriteFailed => return file_writer.err.?,
388 else => |e| return e,400 else => |e| return e,
389 };401 };
402 prog_node.completeOne();
390 }403 }
391404
392 // intentionally omitting the pointless trailer405 // intentionally omitting the pointless trailer
...@@ -2259,6 +2272,7 @@ const TestFetchBuilder = struct {...@@ -2259,6 +2272,7 @@ const TestFetchBuilder = struct {
2259 .read_only = false,2272 .read_only = false,
2260 .debug_hash = false,2273 .debug_hash = false,
2261 .mode = .needed,2274 .mode = .needed,
2275 .prog_node = std.Progress.Node.none,
2262 };2276 };
22632277
2264 self.fetch = .{2278 self.fetch = .{
src/main.zig+2
...@@ -5246,6 +5246,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5246,6 +5246,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5246 .debug_hash = false,5246 .debug_hash = false,
5247 .unlazy_set = unlazy_set,5247 .unlazy_set = unlazy_set,
5248 .mode = fetch_mode,5248 .mode = fetch_mode,
5249 .prog_node = fetch_prog_node,
5249 };5250 };
5250 defer job_queue.deinit();5251 defer job_queue.deinit();
52515252
...@@ -7026,6 +7027,7 @@ fn cmdFetch(...@@ -7026,6 +7027,7 @@ fn cmdFetch(
7026 .read_only = false,7027 .read_only = false,
7027 .debug_hash = debug_hash,7028 .debug_hash = debug_hash,
7028 .mode = .all,7029 .mode = .all,
7030 .prog_node = root_prog_node,
7029 };7031 };
7030 defer job_queue.deinit();7032 defer job_queue.deinit();
70317033