authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2023-07-25 14:33:19+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2023-07-25 19:17:53+01:00
log7a57f82976f2c48581ce105ca7d578e8e492b3dc
treed1d4e53a2a0a14986492fb9ecc65560061d44fc3
parent06e50e9aa7b64b0b191f4048c853dfbc02eacbed
signaturelock-open Commit is signed but in an unrecognized format.

Package: add progress indicator for package fetching

2 files changed, 108 insertions(+), 11 deletions(-)

src/Package.zig+103-11
...@@ -228,6 +228,7 @@ pub fn fetchAndAddDependencies(...@@ -228,6 +228,7 @@ pub fn fetchAndAddDependencies(
228 name_prefix: []const u8,228 name_prefix: []const u8,
229 error_bundle: *std.zig.ErrorBundle.Wip,229 error_bundle: *std.zig.ErrorBundle.Wip,
230 all_modules: *AllModules,230 all_modules: *AllModules,
231 root_prog_node: *std.Progress.Node,
231) !void {232) !void {
232 const max_bytes = 10 * 1024 * 1024;233 const max_bytes = 10 * 1024 * 1024;
233 const gpa = thread_pool.allocator;234 const gpa = thread_pool.allocator;
...@@ -272,6 +273,17 @@ pub fn fetchAndAddDependencies(...@@ -272,6 +273,17 @@ pub fn fetchAndAddDependencies(
272 .error_bundle = error_bundle,273 .error_bundle = error_bundle,
273 };274 };
274275
276 for (manifest.dependencies.values()) |dep| {
277 // If the hash is invalid, let errors happen later
278 // We only want to add these for progress reporting
279 const hash = dep.hash orelse continue;
280 if (hash.len != hex_multihash_len) continue;
281 const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*);
282 if (!gop.found_existing) gop.value_ptr.* = null;
283 }
284
285 root_prog_node.setEstimatedTotalItems(all_modules.count());
286
275 const deps_list = manifest.dependencies.values();287 const deps_list = manifest.dependencies.values();
276 for (manifest.dependencies.keys(), 0..) |name, i| {288 for (manifest.dependencies.keys(), 0..) |name, i| {
277 const dep = deps_list[i];289 const dep = deps_list[i];
...@@ -288,6 +300,7 @@ pub fn fetchAndAddDependencies(...@@ -288,6 +300,7 @@ pub fn fetchAndAddDependencies(
288 build_roots_source,300 build_roots_source,
289 fqn,301 fqn,
290 all_modules,302 all_modules,
303 root_prog_node,
291 );304 );
292305
293 if (!sub.found_existing) {306 if (!sub.found_existing) {
...@@ -304,6 +317,7 @@ pub fn fetchAndAddDependencies(...@@ -304,6 +317,7 @@ pub fn fetchAndAddDependencies(
304 sub_prefix,317 sub_prefix,
305 error_bundle,318 error_bundle,
306 all_modules,319 all_modules,
320 root_prog_node,
307 );321 );
308 }322 }
309323
...@@ -404,7 +418,51 @@ const Report = struct {...@@ -404,7 +418,51 @@ const Report = struct {
404const hex_multihash_len = 2 * Manifest.multihash_len;418const hex_multihash_len = 2 * Manifest.multihash_len;
405const MultiHashHexDigest = [hex_multihash_len]u8;419const MultiHashHexDigest = [hex_multihash_len]u8;
406/// This is to avoid creating multiple modules for the same build.zig file.420/// This is to avoid creating multiple modules for the same build.zig file.
407pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, *Package);421/// If the value is `null`, the package is a known dependency, but has not yet
422/// been fetched.
423pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?*Package);
424
425fn ProgressReader(comptime ReaderType: type) type {
426 return struct {
427 child_reader: ReaderType,
428 bytes_read: u64 = 0,
429 prog_node: *std.Progress.Node,
430 unit: enum {
431 kib,
432 mib,
433 any,
434 },
435
436 pub const Error = ReaderType.Error;
437 pub const Reader = std.io.Reader(*@This(), Error, read);
438
439 pub fn read(self: *@This(), buf: []u8) Error!usize {
440 const amt = try self.child_reader.read(buf);
441 self.bytes_read += amt;
442 const kib = self.bytes_read / 1024;
443 const mib = kib / 1024;
444 switch (self.unit) {
445 .kib => self.prog_node.setCompletedItems(@intCast(kib)),
446 .mib => self.prog_node.setCompletedItems(@intCast(mib)),
447 .any => {
448 if (mib > 0) {
449 self.prog_node.setUnit("MiB");
450 self.prog_node.setCompletedItems(@intCast(mib));
451 } else {
452 self.prog_node.setUnit("KiB");
453 self.prog_node.setCompletedItems(@intCast(kib));
454 }
455 },
456 }
457 self.prog_node.context.maybeRefresh();
458 return amt;
459 }
460
461 pub fn reader(self: *@This()) Reader {
462 return .{ .context = self };
463 }
464 };
465}
408466
409fn fetchAndUnpack(467fn fetchAndUnpack(
410 thread_pool: *ThreadPool,468 thread_pool: *ThreadPool,
...@@ -415,6 +473,7 @@ fn fetchAndUnpack(...@@ -415,6 +473,7 @@ fn fetchAndUnpack(
415 build_roots_source: *std.ArrayList(u8),473 build_roots_source: *std.ArrayList(u8),
416 fqn: []const u8,474 fqn: []const u8,
417 all_modules: *AllModules,475 all_modules: *AllModules,
476 root_prog_node: *std.Progress.Node,
418) !struct { mod: *Package, found_existing: bool } {477) !struct { mod: *Package, found_existing: bool } {
419 const gpa = http_client.allocator;478 const gpa = http_client.allocator;
420 const s = fs.path.sep_str;479 const s = fs.path.sep_str;
...@@ -442,13 +501,17 @@ fn fetchAndUnpack(...@@ -442,13 +501,17 @@ fn fetchAndUnpack(
442 // so we must detect if a module has been created for this package and reuse it.501 // so we must detect if a module has been created for this package and reuse it.
443 const gop = try all_modules.getOrPut(gpa, hex_digest.*);502 const gop = try all_modules.getOrPut(gpa, hex_digest.*);
444 if (gop.found_existing) {503 if (gop.found_existing) {
445 gpa.free(build_root);504 if (gop.value_ptr.*) |mod| {
446 return .{505 gpa.free(build_root);
447 .mod = gop.value_ptr.*,506 return .{
448 .found_existing = true,507 .mod = mod,
449 };508 .found_existing = true,
509 };
510 }
450 }511 }
451512
513 root_prog_node.completeOne();
514
452 const ptr = try gpa.create(Package);515 const ptr = try gpa.create(Package);
453 errdefer gpa.destroy(ptr);516 errdefer gpa.destroy(ptr);
454517
...@@ -471,6 +534,11 @@ fn fetchAndUnpack(...@@ -471,6 +534,11 @@ fn fetchAndUnpack(
471 };534 };
472 }535 }
473536
537 var pkg_prog_node = root_prog_node.start(fqn, 0);
538 defer pkg_prog_node.end();
539 pkg_prog_node.activate();
540 pkg_prog_node.context.refresh();
541
474 const uri = try std.Uri.parse(dep.url);542 const uri = try std.Uri.parse(dep.url);
475543
476 const rand_int = std.crypto.random.int(u64);544 const rand_int = std.crypto.random.int(u64);
...@@ -510,29 +578,53 @@ fn fetchAndUnpack(...@@ -510,29 +578,53 @@ fn fetchAndUnpack(
510 const content_type = req.response.headers.getFirstValue("Content-Type") orelse578 const content_type = req.response.headers.getFirstValue("Content-Type") orelse
511 return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});579 return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{});
512580
581 var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{
582 .child_reader = req.reader(),
583 .prog_node = &pkg_prog_node,
584 .unit = if (req.response.content_length) |content_length| unit: {
585 const kib = content_length / 1024;
586 const mib = kib / 1024;
587 if (mib > 0) {
588 pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
589 pkg_prog_node.setUnit("MiB");
590 break :unit .mib;
591 } else {
592 pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
593 pkg_prog_node.setUnit("KiB");
594 break :unit .kib;
595 }
596 } else .any,
597 };
598 pkg_prog_node.context.refresh();
599
513 if (ascii.eqlIgnoreCase(content_type, "application/gzip") or600 if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
514 ascii.eqlIgnoreCase(content_type, "application/x-gzip") or601 ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
515 ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))602 ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
516 {603 {
517 // I observed the gzip stream to read 1 byte at a time, so I am using a604 // I observed the gzip stream to read 1 byte at a time, so I am using a
518 // buffered reader on the front of it.605 // buffered reader on the front of it.
519 try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip);606 try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
520 } else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {607 } else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) {
521 // I have not checked what buffer sizes the xz decompression implementation uses608 // I have not checked what buffer sizes the xz decompression implementation uses
522 // by default, so the same logic applies for buffering the reader as for gzip.609 // by default, so the same logic applies for buffering the reader as for gzip.
523 try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz);610 try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz);
524 } else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {611 } else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
525 // support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz612 // support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
526 // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'613 // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
527 const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse614 const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
528 return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});615 return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
529 if (isTarAttachment(content_disposition)) {616 if (isTarAttachment(content_disposition)) {
530 try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip);617 try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip);
531 } else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});618 } else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
532 } else {619 } else {
533 return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});620 return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type});
534 }621 }
535622
623 // Download completed - stop showing downloaded amount as progress
624 pkg_prog_node.setEstimatedTotalItems(0);
625 pkg_prog_node.setCompletedItems(0);
626 pkg_prog_node.context.refresh();
627
536 // TODO: delete files not included in the package prior to computing the package hash.628 // TODO: delete files not included in the package prior to computing the package hash.
537 // for example, if the ini file has directives to include/not include certain files,629 // for example, if the ini file has directives to include/not include certain files,
538 // apply those rules directly to the filesystem right here. This ensures that files630 // apply those rules directly to the filesystem right here. This ensures that files
...@@ -591,11 +683,11 @@ fn fetchAndUnpack(...@@ -591,11 +683,11 @@ fn fetchAndUnpack(
591683
592fn unpackTarball(684fn unpackTarball(
593 gpa: Allocator,685 gpa: Allocator,
594 req: *std.http.Client.Request,686 req_reader: anytype,
595 out_dir: fs.Dir,687 out_dir: fs.Dir,
596 comptime compression: type,688 comptime compression: type,
597) !void {689) !void {
598 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());690 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req_reader);
599691
600 var decompress = try compression.decompress(gpa, br.reader());692 var decompress = try compression.decompress(gpa, br.reader());
601 defer decompress.deinit();693 defer decompress.deinit();
src/main.zig+5
...@@ -4433,6 +4433,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4433,6 +4433,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4433 try wip_errors.init(gpa);4433 try wip_errors.init(gpa);
4434 defer wip_errors.deinit();4434 defer wip_errors.deinit();
44354435
4436 var progress: std.Progress = .{};
4437 const root_prog_node = progress.start("Fetch Packages", 0);
4438 defer root_prog_node.end();
4439
4436 // Here we borrow main package's table and will replace it with a fresh4440 // Here we borrow main package's table and will replace it with a fresh
4437 // one after this process completes.4441 // one after this process completes.
4438 const fetch_result = build_pkg.fetchAndAddDependencies(4442 const fetch_result = build_pkg.fetchAndAddDependencies(
...@@ -4448,6 +4452,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -4448,6 +4452,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4448 "",4452 "",
4449 &wip_errors,4453 &wip_errors,
4450 &all_modules,4454 &all_modules,
4455 root_prog_node,
4451 );4456 );
4452 if (wip_errors.root_list.items.len > 0) {4457 if (wip_errors.root_list.items.len > 0) {
4453 var errors = try wip_errors.toOwnedBundle("");4458 var errors = try wip_errors.toOwnedBundle("");