authorgravatar for adambgoertz@gmail.comAdam Goertz <adambgoertz@gmail.com> 2023-09-24 01:24:49+00:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-29 00:32:43-07:00
log2f0e5b00b086981eafc0f9d2be573943495158bf
tree482693d35b3debc97bede85800e1262d64bb3c36
parentb3cad98534a4a4406d848f7cbd28165ca005bc8a

Allow only relative paths.

This commit makes the following changes: * Disallow file:/// URIs * Allow only relative paths in the .path field of build.zig.zon * Remote now-unneeded shlwapi dependency

5 files changed, 249 insertions(+), 248 deletions(-)

build.zig-2
...@@ -336,7 +336,6 @@ pub fn build(b: *std.Build) !void {...@@ -336,7 +336,6 @@ pub fn build(b: *std.Build) !void {
336 artifact.linkSystemLibrary("version");336 artifact.linkSystemLibrary("version");
337 artifact.linkSystemLibrary("uuid");337 artifact.linkSystemLibrary("uuid");
338 artifact.linkSystemLibrary("ole32");338 artifact.linkSystemLibrary("ole32");
339 artifact.linkSystemLibrary("shlwapi");
340 }339 }
341 }340 }
342 }341 }
...@@ -713,7 +712,6 @@ fn addStaticLlvmOptionsToExe(exe: *std.Build.Step.Compile) !void {...@@ -713,7 +712,6 @@ fn addStaticLlvmOptionsToExe(exe: *std.Build.Step.Compile) !void {
713 exe.linkSystemLibrary("version");712 exe.linkSystemLibrary("version");
714 exe.linkSystemLibrary("uuid");713 exe.linkSystemLibrary("uuid");
715 exe.linkSystemLibrary("ole32");714 exe.linkSystemLibrary("ole32");
716 exe.linkSystemLibrary("shlwapi");
717 }715 }
718}716}
719717
lib/std/Uri.zig+1-1
...@@ -150,7 +150,7 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {...@@ -150,7 +150,7 @@ pub fn parseWithoutScheme(text: []const u8) ParseError!Uri {
150 std.debug.assert(reader.get().? == '/');150 std.debug.assert(reader.get().? == '/');
151 std.debug.assert(reader.get().? == '/');151 std.debug.assert(reader.get().? == '/');
152152
153 var authority = reader.readUntil(isAuthoritySeparator);153 const authority = reader.readUntil(isAuthoritySeparator);
154 if (authority.len == 0) {154 if (authority.len == 0) {
155 if (reader.peekPrefix("/")) break :a else return error.InvalidFormat;155 if (reader.peekPrefix("/")) break :a else return error.InvalidFormat;
156 }156 }
lib/std/os/windows.zig-1
...@@ -30,7 +30,6 @@ pub const gdi32 = @import("windows/gdi32.zig");...@@ -30,7 +30,6 @@ pub const gdi32 = @import("windows/gdi32.zig");
30pub const winmm = @import("windows/winmm.zig");30pub const winmm = @import("windows/winmm.zig");
31pub const crypt32 = @import("windows/crypt32.zig");31pub const crypt32 = @import("windows/crypt32.zig");
32pub const nls = @import("windows/nls.zig");32pub const nls = @import("windows/nls.zig");
33pub const shlwapi = @import("windows/shlwapi.zig");
3433
35pub const self_process_handle = @as(HANDLE, @ptrFromInt(maxInt(usize)));34pub const self_process_handle = @as(HANDLE, @ptrFromInt(maxInt(usize)));
3635
lib/std/os/windows/shlwapi.zig deleted-13
...@@ -1,13 +0,0 @@
1const std = @import("../../std.zig");
2const windows = std.os.windows;
3
4const DWORD = windows.DWORD;
5const WINAPI = windows.WINAPI;
6const HRESULT = windows.HRESULT;
7const LPCSTR = windows.LPCSTR;
8const LPSTR = windows.LPSTR;
9const LPWSTR = windows.LPWSTR;
10const LPCWSTR = windows.LPCWSTR;
11
12pub extern "shlwapi" fn PathCreateFromUrlW(pszUrl: LPCWSTR, pszPath: LPWSTR, pcchPath: *DWORD, dwFlags: DWORD) callconv(WINAPI) HRESULT;
13pub extern "shlwapi" fn PathCreateFromUrlA(pszUrl: LPCSTR, pszPath: LPSTR, pcchPath: *DWORD, dwFlags: DWORD) callconv(WINAPI) HRESULT;
src/Package.zig+248-231
...@@ -245,8 +245,6 @@ pub fn fetchAndAddDependencies(...@@ -245,8 +245,6 @@ pub fn fetchAndAddDependencies(
245 error.FileNotFound => {245 error.FileNotFound => {
246 // Handle the same as no dependencies.246 // Handle the same as no dependencies.
247 if (this_hash) |hash| {247 if (this_hash) |hash| {
248 const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
249 const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
250 try dependencies_source.writer().print(248 try dependencies_source.writer().print(
251 \\ pub const {} = struct {{249 \\ pub const {} = struct {{
252 \\ pub const build_root = "{}";250 \\ pub const build_root = "{}";
...@@ -256,7 +254,7 @@ pub fn fetchAndAddDependencies(...@@ -256,7 +254,7 @@ pub fn fetchAndAddDependencies(
256 \\254 \\
257 , .{255 , .{
258 std.zig.fmtId(hash),256 std.zig.fmtId(hash),
259 std.zig.fmtEscapes(build_root),257 std.zig.fmtEscapes(pkg.root_src_directory.path.?),
260 std.zig.fmtEscapes(hash),258 std.zig.fmtEscapes(hash),
261 });259 });
262 } else {260 } else {
...@@ -312,19 +310,15 @@ pub fn fetchAndAddDependencies(...@@ -312,19 +310,15 @@ pub fn fetchAndAddDependencies(
312 try dependencies_source.writer().writeAll("pub const packages = struct {\n");310 try dependencies_source.writer().writeAll("pub const packages = struct {\n");
313 }311 }
314312
315 const deps_list = manifest.dependencies.values();313 for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, *dep| {
316 for (manifest.dependencies.keys(), 0..) |name, i| {314 const sub_mod, const found_existing = try getCachedPackage(
317 const dep = deps_list[i];315 arena,
318
319 const sub_pkg = try getCachedPackage(
320 http_client.allocator,
321 global_cache_directory,316 global_cache_directory,
322 dep,317 dep.*,
323 report,
324 all_modules,318 all_modules,
325 root_prog_node,319 root_prog_node,
326 ) orelse m: {320 ) orelse .{
327 const mod = try fetchAndUnpack(321 try fetchAndUnpack(
328 thread_pool,322 thread_pool,
329 http_client,323 http_client,
330 directory,324 directory,
...@@ -334,39 +328,58 @@ pub fn fetchAndAddDependencies(...@@ -334,39 +328,58 @@ pub fn fetchAndAddDependencies(
334 all_modules,328 all_modules,
335 root_prog_node,329 root_prog_node,
336 name,330 name,
337 );331 ),
338332 false,
339 try mod.fetchAndAddDependencies(
340 deps_pkg,
341 arena,
342 thread_pool,
343 http_client,
344 mod.root_src_directory,
345 global_cache_directory,
346 local_cache_directory,
347 dependencies_source,
348 error_bundle,
349 all_modules,
350 root_prog_node,
351 dep.hash.?,
352 );
353
354 break :m mod;
355 };333 };
356334
357 try pkg.add(gpa, name, sub_pkg);335 assert(dep.hash != null);
358 if (deps_pkg.table.get(dep.hash.?)) |other_sub| {336
359 // This should be the same package (and hence module) since it's the same hash337 switch (sub_mod) {
360 // TODO: dedup multiple versions of the same package338 .zig_pkg => |sub_pkg| {
361 assert(other_sub == sub_pkg);339 if (!found_existing) {
362 } else {340 try sub_pkg.fetchAndAddDependencies(
363 try deps_pkg.add(gpa, dep.hash.?, sub_pkg);341 deps_pkg,
342 arena,
343 thread_pool,
344 http_client,
345 sub_pkg.root_src_directory,
346 global_cache_directory,
347 local_cache_directory,
348 dependencies_source,
349 error_bundle,
350 all_modules,
351 root_prog_node,
352 dep.hash.?,
353 );
354 }
355
356 try pkg.add(gpa, name, sub_pkg);
357 if (deps_pkg.table.get(dep.hash.?)) |other_sub| {
358 // This should be the same package (and hence module) since it's the same hash
359 // TODO: dedup multiple versions of the same package
360 assert(other_sub == sub_pkg);
361 } else {
362 try deps_pkg.add(gpa, dep.hash.?, sub_pkg);
363 }
364 },
365 .non_zig_pkg => |sub_pkg| {
366 if (!found_existing) {
367 try dependencies_source.writer().print(
368 \\ pub const {} = struct {{
369 \\ pub const build_root = "{}";
370 \\ pub const deps: []const struct {{ []const u8, []const u8 }} = &.{{}};
371 \\ }};
372 \\
373 , .{
374 std.zig.fmtId(dep.hash.?),
375 std.zig.fmtEscapes(sub_pkg.root_src_directory.path.?),
376 });
377 }
378 },
364 }379 }
365 }380 }
366381
367 if (this_hash) |hash| {382 if (this_hash) |hash| {
368 const pkg_dir_sub_path = "p" ++ fs.path.sep_str ++ hash[0..hex_multihash_len];
369 const build_root = try global_cache_directory.join(arena, &.{pkg_dir_sub_path});
370 try dependencies_source.writer().print(383 try dependencies_source.writer().print(
371 \\ pub const {} = struct {{384 \\ pub const {} = struct {{
372 \\ pub const build_root = "{}";385 \\ pub const build_root = "{}";
...@@ -375,7 +388,7 @@ pub fn fetchAndAddDependencies(...@@ -375,7 +388,7 @@ pub fn fetchAndAddDependencies(
375 \\388 \\
376 , .{389 , .{
377 std.zig.fmtId(hash),390 std.zig.fmtId(hash),
378 std.zig.fmtEscapes(build_root),391 std.zig.fmtEscapes(pkg.root_src_directory.path.?),
379 std.zig.fmtEscapes(hash),392 std.zig.fmtEscapes(hash),
380 });393 });
381 for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {394 for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
...@@ -485,44 +498,40 @@ const Report = struct {...@@ -485,44 +498,40 @@ const Report = struct {
485 }498 }
486};499};
487500
488const FetchLocation = union(SourceType) {501const FetchLocation = union(enum) {
489 /// The absolute path to a file or directory.502 /// The absolute path to a file or directory.
490 /// This may be a file that requires unpacking (such as a .tar.gz),503 /// This may be a file that requires unpacking (such as a .tar.gz),
491 /// or the path to the root directory of a package.504 /// or the path to the root directory of a package.
492 file: []const u8,505 file: []const u8,
493 http_request: std.Uri,506 http_request: std.Uri,
494507
495 pub fn init(gpa: Allocator, uri: std.Uri, directory: Compilation.Directory, dep: Manifest.Dependency, report: Report) !FetchLocation {508 pub fn init(gpa: Allocator, directory: Compilation.Directory, dep: Manifest.Dependency, report: Report) !FetchLocation {
496 const source_type = getPackageSourceType(uri) catch509 switch (dep.location) {
497 return report.fail(dep.location_tok, "Unknown scheme: {s}", .{uri.scheme});510 .url => |url| {
498511 const uri = std.Uri.parse(url) catch |err| switch (err) {
499 return switch (source_type) {512 error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
500 .file => f: {513 else => return err,
501 const path = if (builtin.os.tag == .windows) p: {514 };
502 var uri_str = std.ArrayList(u8).init(gpa);515 if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
503 defer uri_str.deinit();516 return report.fail(dep.location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
504 try uri.format("+/", .{}, uri_str.writer());517 }
505 const uri_str_z = try gpa.dupeZ(u8, uri_str.items);518 return .{ .http_request = uri };
506 defer gpa.free(uri_str_z);519 },
507520 .path => |path| {
508 var buf: [std.os.windows.MAX_PATH:0]u8 = undefined;521 const unescaped = try std.Uri.unescapeString(gpa, path);
509 var buf_len: std.os.windows.DWORD = std.os.windows.MAX_PATH;522 defer gpa.free(unescaped);
510 const result = std.os.windows.shlwapi.PathCreateFromUrlA(uri_str_z, &buf, &buf_len, 0);523 const unnormalized_path = try unnormalizePath(gpa, unescaped);
511524 defer gpa.free(unnormalized_path);
512 if (result != std.os.windows.S_OK) return report.fail(dep.location_tok, "Invalid URI", .{});525
513526 if (fs.path.isAbsolute(unnormalized_path)) {
514 break :p try gpa.dupe(u8, buf[0..buf_len]);527 return report.fail(dep.location_tok, "Absolute paths are not allowed. Use a relative path instead", .{});
515 } else try std.Uri.unescapeString(gpa, uri.path);528 }
516 defer gpa.free(path);
517529
518 const new_path = try fs.path.resolve(gpa, &.{ directory.path.?, path });530 const new_path = try fs.path.resolve(gpa, &.{ directory.path.?, unnormalized_path });
519531
520 break :f .{ .file = new_path };532 return .{ .file = new_path };
521 },533 },
522 .http_request => r: {534 }
523 break :r .{ .http_request = uri };
524 },
525 };
526 }535 }
527536
528 pub fn deinit(f: *FetchLocation, gpa: Allocator) void {537 pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
...@@ -533,41 +542,6 @@ const FetchLocation = union(SourceType) {...@@ -533,41 +542,6 @@ const FetchLocation = union(SourceType) {
533 f.* = undefined;542 f.* = undefined;
534 }543 }
535544
536 const SourceType = enum {
537 file,
538 http_request,
539 };
540
541 fn getPackageSourceType(uri: std.Uri) error{UnknownScheme}!SourceType {
542 const package_source_map = std.ComptimeStringMap(
543 SourceType,
544 .{
545 .{ "file", .file },
546 .{ "http", .http_request },
547 .{ "https", .http_request },
548 },
549 );
550 return package_source_map.get(uri.scheme) orelse error.UnknownScheme;
551 }
552
553 pub fn isDirectory(path: []const u8, root_dir: Compilation.Directory) !bool {
554 return if (mem.endsWith(u8, path, std.fs.path.sep_str))
555 true
556 else if (std.fs.path.extension(path).len > 0)
557 false
558 else d: {
559 // It's common to write directories without a trailing '/'.
560 // This is some special casing logic to detect directories if
561 // the file type cannot be determined from the extension.
562 var dir = root_dir.handle.openDir(path, .{}) catch |err| switch (err) {
563 error.NotDir => break :d false,
564 else => break :d err,
565 };
566 defer dir.close();
567 break :d true;
568 };
569 }
570
571 pub fn fetch(545 pub fn fetch(
572 f: FetchLocation,546 f: FetchLocation,
573 gpa: Allocator,547 gpa: Allocator,
...@@ -578,25 +552,28 @@ const FetchLocation = union(SourceType) {...@@ -578,25 +552,28 @@ const FetchLocation = union(SourceType) {
578 ) !ReadableResource {552 ) !ReadableResource {
579 switch (f) {553 switch (f) {
580 .file => |file| {554 .file => |file| {
581 const is_dir = isDirectory(file, root_dir) catch555 const is_dir = isDirectory(root_dir, file) catch |err| switch (err) {
582 return report.fail(dep.location_tok, "File not found: {s}", .{file});556 error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{file}),
557 else => return err,
558 };
583559
584 return if (is_dir)560 const owned_path = try gpa.dupe(u8, file);
585 .{561 errdefer gpa.free(owned_path);
586 .path = try gpa.dupe(u8, file),562
587 .resource = .{ .directory = try fs.openIterableDirAbsolute(file, .{}) },563 return .{
588 }564 .path = owned_path,
589 else565 .resource = if (is_dir)
590 .{566 .{ .directory = try fs.openIterableDirAbsolute(file, .{}) }
591 .path = try gpa.dupe(u8, file),567 else
592 .resource = .{ .file = try fs.openFileAbsolute(file, .{}) },568 .{ .file = try fs.openFileAbsolute(file, .{}) },
593 };569 };
594 },570 },
595 .http_request => |uri| {571 .http_request => |uri| {
596 var h = std.http.Headers{ .allocator = gpa };572 var h = std.http.Headers{ .allocator = gpa };
597 defer h.deinit();573 defer h.deinit();
598574
599 var req = try http_client.request(.GET, uri, h, .{});575 var req = try http_client.request(.GET, uri, h, .{});
576 errdefer req.deinit();
600577
601 try req.start(.{});578 try req.start(.{});
602 try req.wait();579 try req.wait();
...@@ -638,10 +615,9 @@ const ReadableResource = struct {...@@ -638,10 +615,9 @@ const ReadableResource = struct {
638 pkg_prog_node: *std.Progress.Node,615 pkg_prog_node: *std.Progress.Node,
639 ) !PackageLocation {616 ) !PackageLocation {
640 switch (rr.resource) {617 switch (rr.resource) {
641 .directory => |dir| {618 .directory => {
642 const actual_hash = try computePackageHash(thread_pool, dir);
643 return .{619 return .{
644 .hash = actual_hash,620 .hash = computePathHash(rr.path),
645 .dir_path = try allocator.dupe(u8, rr.path),621 .dir_path = try allocator.dupe(u8, rr.path),
646 };622 };
647 },623 },
...@@ -739,11 +715,7 @@ const ReadableResource = struct {...@@ -739,11 +715,7 @@ const ReadableResource = struct {
739 pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {715 pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {
740 switch (rr.resource) {716 switch (rr.resource) {
741 .file => {717 .file => {
742 return if (mem.endsWith(u8, rr.path, ".tar.gz"))718 return fileTypeFromPath(rr.path) orelse
743 .@"tar.gz"
744 else if (mem.endsWith(u8, rr.path, ".tar.xz"))
745 .@"tar.xz"
746 else
747 return report.fail(dep.location_tok, "Unknown file type", .{});719 return report.fail(dep.location_tok, "Unknown file type", .{});
748 },720 },
749 .directory => return error.IsDir,721 .directory => return error.IsDir,
...@@ -764,16 +736,40 @@ const ReadableResource = struct {...@@ -764,16 +736,40 @@ const ReadableResource = struct {
764 // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'736 // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
765 const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse737 const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
766 return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});738 return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
767 if (mem.startsWith(u8, content_disposition, "attachment;") and739 break :ty getAttachmentType(content_disposition) orelse
768 mem.endsWith(u8, content_disposition, ".tar.gz\""))740 return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
769 {
770 break :ty .@"tar.gz";
771 } else return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
772 } else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});741 } else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});
773 },742 },
774 }743 }
775 }744 }
776745
746 fn fileTypeFromPath(file_path: []const u8) ?FileType {
747 return if (ascii.endsWithIgnoreCase(file_path, ".tar.gz"))
748 .@"tar.gz"
749 else if (ascii.endsWithIgnoreCase(file_path, ".tar.xz"))
750 .@"tar.xz"
751 else
752 null;
753 }
754
755 fn getAttachmentType(content_disposition: []const u8) ?FileType {
756 const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return null;
757
758 var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return null;
759 value_start += "filename".len;
760 if (content_disposition[value_start] == '*') {
761 value_start += 1;
762 }
763 if (content_disposition[value_start] != '=') return null;
764 value_start += 1;
765
766 var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len;
767 if (content_disposition[value_end - 1] == '\"') {
768 value_end -= 1;
769 }
770 return fileTypeFromPath(content_disposition[value_start..value_end]);
771 }
772
777 pub fn deinit(rr: *ReadableResource, gpa: Allocator) void {773 pub fn deinit(rr: *ReadableResource, gpa: Allocator) void {
778 gpa.free(rr.path);774 gpa.free(rr.path);
779 switch (rr.resource) {775 switch (rr.resource) {
...@@ -786,6 +782,8 @@ const ReadableResource = struct {...@@ -786,6 +782,8 @@ const ReadableResource = struct {
786};782};
787783
788pub const PackageLocation = struct {784pub const PackageLocation = struct {
785 /// For packages that require unpacking, this is the hash of the package contents.
786 /// For directories, this is the hash of the absolute file path.
789 hash: [Manifest.Hash.digest_length]u8,787 hash: [Manifest.Hash.digest_length]u8,
790 dir_path: []const u8,788 dir_path: []const u8,
791789
...@@ -797,13 +795,15 @@ pub const PackageLocation = struct {...@@ -797,13 +795,15 @@ pub const PackageLocation = struct {
797795
798const hex_multihash_len = 2 * Manifest.multihash_len;796const hex_multihash_len = 2 * Manifest.multihash_len;
799const MultiHashHexDigest = [hex_multihash_len]u8;797const MultiHashHexDigest = [hex_multihash_len]u8;
798
799const DependencyModule = union(enum) {
800 zig_pkg: *Package,
801 non_zig_pkg: *Package,
802};
800/// This is to avoid creating multiple modules for the same build.zig file.803/// This is to avoid creating multiple modules for the same build.zig file.
801/// If the value is `null`, the package is a known dependency, but has not yet804/// If the value is `null`, the package is a known dependency, but has not yet
802/// been fetched.805/// been fetched.
803pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?union(enum) {806pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule);
804 zig_pkg: *Package,
805 non_zig_pkg: void,
806});
807807
808fn ProgressReader(comptime ReaderType: type) type {808fn ProgressReader(comptime ReaderType: type) type {
809 return struct {809 return struct {
...@@ -847,15 +847,18 @@ fn ProgressReader(comptime ReaderType: type) type {...@@ -847,15 +847,18 @@ fn ProgressReader(comptime ReaderType: type) type {
847 };847 };
848}848}
849849
850/// Get a cached package if it exists.
851/// Returns `null` if the package has not been cached
852/// If the package exists in the cache, returns a pointer to the package and a
853/// boolean indicating whether this package has already been seen in the build
854/// (i.e. whether or not its transitive dependencies have been fetched).
850fn getCachedPackage(855fn getCachedPackage(
851 gpa: Allocator,856 gpa: Allocator,
852 global_cache_directory: Compilation.Directory,857 global_cache_directory: Compilation.Directory,
853 dep: Manifest.Dependency,858 dep: Manifest.Dependency,
854 report: Report,
855 all_modules: *AllModules,859 all_modules: *AllModules,
856 root_prog_node: *std.Progress.Node,860 root_prog_node: *std.Progress.Node,
857) !?*Package {861) !?struct { DependencyModule, bool } {
858 _ = report;
859 const s = fs.path.sep_str;862 const s = fs.path.sep_str;
860 // Check if the expected_hash is already present in the global package863 // Check if the expected_hash is already present in the global package
861 // cache, and thereby avoid both fetching and unpacking.864 // cache, and thereby avoid both fetching and unpacking.
...@@ -874,27 +877,21 @@ fn getCachedPackage(...@@ -874,27 +877,21 @@ fn getCachedPackage(
874 const gop = try all_modules.getOrPut(gpa, hex_digest.*);877 const gop = try all_modules.getOrPut(gpa, hex_digest.*);
875 if (gop.found_existing) {878 if (gop.found_existing) {
876 if (gop.value_ptr.*) |mod| {879 if (gop.value_ptr.*) |mod| {
877 return mod;880 return .{ mod, true };
878 }881 }
879 }882 }
880883
881 pkg_dir.access(build_zig_basename, .{}) catch {884 root_prog_node.completeOne();
882 gop.value_ptr.* = .non_zig_pkg;885
883 return .{886 const is_zig_mod = if (pkg_dir.access(build_zig_basename, .{})) |_| true else |_| false;
884 .mod = null,
885 .found_existing = false,
886 };
887 };
888887
889 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});888 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
890 errdefer gpa.free(build_root);889 errdefer gpa.free(build_root);
891890
892 root_prog_node.completeOne();
893
894 const ptr = try gpa.create(Package);891 const ptr = try gpa.create(Package);
895 errdefer gpa.destroy(ptr);892 errdefer gpa.destroy(ptr);
896893
897 const owned_src_path = try gpa.dupe(u8, build_zig_basename);894 const owned_src_path = if (is_zig_mod) try gpa.dupe(u8, build_zig_basename) else "";
898 errdefer gpa.free(owned_src_path);895 errdefer gpa.free(owned_src_path);
899896
900 ptr.* = .{897 ptr.* = .{
...@@ -906,8 +903,12 @@ fn getCachedPackage(...@@ -906,8 +903,12 @@ fn getCachedPackage(
906 .root_src_path = owned_src_path,903 .root_src_path = owned_src_path,
907 };904 };
908905
909 gop.value_ptr.* = ptr;906 gop.value_ptr.* = if (is_zig_mod)
910 return ptr;907 .{ .zig_pkg = ptr }
908 else
909 .{ .non_zig_pkg = ptr };
910
911 return .{ gop.value_ptr.*.?, false };
911 }912 }
912913
913 return null;914 return null;
...@@ -918,14 +919,14 @@ fn fetchAndUnpack(...@@ -918,14 +919,14 @@ fn fetchAndUnpack(
918 http_client: *std.http.Client,919 http_client: *std.http.Client,
919 directory: Compilation.Directory,920 directory: Compilation.Directory,
920 global_cache_directory: Compilation.Directory,921 global_cache_directory: Compilation.Directory,
921 dep: Manifest.Dependency,922 dep: *Manifest.Dependency,
922 report: Report,923 report: Report,
923 all_modules: *AllModules,924 all_modules: *AllModules,
924 root_prog_node: *std.Progress.Node,925 root_prog_node: *std.Progress.Node,
925 /// This does not have to be any form of canonical or fully-qualified name: it926 /// This does not have to be any form of canonical or fully-qualified name: it
926 /// is only intended to be human-readable for progress reporting.927 /// is only intended to be human-readable for progress reporting.
927 name_for_prog: []const u8,928 name_for_prog: []const u8,
928) !*Package {929) !DependencyModule {
929 const gpa = http_client.allocator;930 const gpa = http_client.allocator;
930931
931 var pkg_prog_node = root_prog_node.start(name_for_prog, 0);932 var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
...@@ -933,66 +934,65 @@ fn fetchAndUnpack(...@@ -933,66 +934,65 @@ fn fetchAndUnpack(
933 pkg_prog_node.activate();934 pkg_prog_node.activate();
934 pkg_prog_node.context.refresh();935 pkg_prog_node.context.refresh();
935936
936 const uri = switch (dep.location) {937 var fetch_location = try FetchLocation.init(gpa, directory, dep.*, report);
937 .url => |url| std.Uri.parse(url) catch |err| switch (err) {
938 error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI.", .{}),
939 else => return err,
940 },
941 .path => |path| std.Uri{
942 .scheme = "file",
943 .user = null,
944 .password = null,
945 .host = null,
946 .port = null,
947 .path = path,
948 .query = null,
949 .fragment = null,
950 },
951 };
952
953 var fetch_location = try FetchLocation.init(gpa, uri, directory, dep, report);
954 defer fetch_location.deinit(gpa);938 defer fetch_location.deinit(gpa);
955939
956 var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report);940 var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.*, report);
957 defer readable_resource.deinit(gpa);941 defer readable_resource.deinit(gpa);
958942
959 var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node);943 var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep.*, report, &pkg_prog_node);
960 defer package_location.deinit(gpa);944 defer package_location.deinit(gpa);
961945
962 const actual_hex = Manifest.hexDigest(package_location.hash);946 const actual_hex = Manifest.hexDigest(package_location.hash);
963 if (dep.hash) |h| {947 if (readable_resource.resource != .directory) {
964 if (!mem.eql(u8, h, &actual_hex)) {948 if (dep.hash) |h| {
965 return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{949 if (!mem.eql(u8, h, &actual_hex)) {
966 h, actual_hex,950 return report.fail(dep.hash_tok, "hash mismatch: expected: {s}, found: {s}", .{
951 h, actual_hex,
952 });
953 }
954 } else {
955 const file_path = try report.directory.join(gpa, &.{Manifest.basename});
956 defer gpa.free(file_path);
957
958 const eb = report.error_bundle;
959 const notes_len = 1;
960 try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
961 .tok = dep.location_tok,
962 .off = 0,
963 .msg = "dependency is missing hash field",
967 });964 });
965 const notes_start = try eb.reserveNotes(notes_len);
966 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
967 .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
968 }));
969 return error.PackageFetchFailed;
968 }970 }
969 } else {971 } else {
970 const file_path = try report.directory.join(gpa, &.{Manifest.basename});972 if (dep.hash != null) {
971 defer gpa.free(file_path);973 return report.fail(dep.hash_tok, "hash not allowed for directory package", .{});
972974 }
973 const eb = report.error_bundle;975 // Since directory dependencies don't provide a hash in build.zig.zon,
974 const notes_len = 1;976 // set the hash here to be the hash of the absolute path to the dependency.
975 try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{977 dep.hash = try gpa.dupe(u8, &actual_hex);
976 .tok = dep.location_tok,
977 .off = 0,
978 .msg = "dependency is missing hash field",
979 });
980 const notes_start = try eb.reserveNotes(notes_len);
981 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
982 .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
983 }));
984 return error.PackageFetchFailed;
985 }978 }
986979
987 const gop = try all_modules.getOrPut(gpa, actual_hex);980 const build_zig_path = try std.fs.path.join(gpa, &.{ package_location.dir_path, build_zig_basename });
981 defer gpa.free(build_zig_path);
982 assert(fs.path.isAbsolute(build_zig_path));
988983
989 if (gop.found_existing and gop.value_ptr.* != null) {984 global_cache_directory.handle.access(build_zig_path, .{}) catch |err| switch (err) {
990 return gop.value_ptr.*.?;985 error.FileNotFound => {
991 } else {986 const module = try create(gpa, package_location.dir_path, "");
992 const module = try create(gpa, package_location.dir_path, build_zig_basename);987 try all_modules.put(gpa, actual_hex, .{ .non_zig_pkg = module });
993 gop.value_ptr.* = module;988 return .{ .non_zig_pkg = module };
994 return module;989 },
995 }990 else => return err,
991 };
992
993 const module = try create(gpa, package_location.dir_path, build_zig_basename);
994 try all_modules.put(gpa, actual_hex, .{ .zig_pkg = module });
995 return .{ .zig_pkg = module };
996}996}
997997
998fn unpackTarball(998fn unpackTarball(
...@@ -1092,6 +1092,22 @@ fn computePackageHash(...@@ -1092,6 +1092,22 @@ fn computePackageHash(
1092 return hasher.finalResult();1092 return hasher.finalResult();
1093}1093}
10941094
1095/// Compute the hash of a file path.
1096fn computePathHash(path: []const u8) [Manifest.Hash.digest_length]u8 {
1097 var hasher = Manifest.Hash.init(.{});
1098 hasher.update(path);
1099 return hasher.finalResult();
1100}
1101
1102fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {
1103 var dir = root_dir.handle.openDir(path, .{}) catch |err| switch (err) {
1104 error.NotDir => return false,
1105 else => return err,
1106 };
1107 defer dir.close();
1108 return true;
1109}
1110
1095/// Make a file system path identical independently of operating system path inconsistencies.1111/// Make a file system path identical independently of operating system path inconsistencies.
1096/// This converts backslashes into forward slashes.1112/// This converts backslashes into forward slashes.
1097fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {1113fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
...@@ -1110,6 +1126,25 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {...@@ -1110,6 +1126,25 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1110 return normalized;1126 return normalized;
1111}1127}
11121128
1129/// Make a OS-specific file system path
1130/// This performs the inverse operation of normalizePath,
1131/// converting forward slashes into backslashes on Windows
1132fn unnormalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1133 const canonical_sep = '/';
1134
1135 const unnormalized = try arena.dupe(u8, fs_path);
1136 if (fs.path.sep == canonical_sep)
1137 return unnormalized;
1138
1139 for (unnormalized) |*byte| {
1140 switch (byte.*) {
1141 canonical_sep => byte.* = fs.path.sep,
1142 else => continue,
1143 }
1144 }
1145 return unnormalized;
1146}
1147
1113fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {1148fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
1114 defer wg.finish();1149 defer wg.finish();
1115 hashed_file.failure = hashFileFallible(dir, hashed_file);1150 hashed_file.failure = hashFileFallible(dir, hashed_file);
...@@ -1172,36 +1207,18 @@ fn renameTmpIntoCache(...@@ -1172,36 +1207,18 @@ fn renameTmpIntoCache(
1172 }1207 }
1173}1208}
11741209
1175fn isTarAttachment(content_disposition: []const u8) bool {1210test "getAttachmentType" {
1176 const disposition_type_end = ascii.indexOfIgnoreCase(content_disposition, "attachment;") orelse return false;1211 try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
11771212 try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; filename*=\"stuff.tar.gz\""));
1178 var value_start = ascii.indexOfIgnoreCasePos(content_disposition, disposition_type_end + 1, "filename") orelse return false;1213 try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("ATTACHMENT; filename=\"stuff.tar.xz\""));
1179 value_start += "filename".len;1214 try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.xz"), ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar.xz\""));
1180 if (content_disposition[value_start] == '*') {1215 try std.testing.expectEqual(@as(?ReadableResource.FileType, .@"tar.gz"), ReadableResource.getAttachmentType("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1181 value_start += 1;1216
1182 }1217 try std.testing.expect(ReadableResource.getAttachmentType("attachment FileName=\"stuff.tar.gz\"") == null);
1183 if (content_disposition[value_start] != '=') return false;1218 try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName=\"stuff.tar\"") == null);
1184 value_start += 1;1219 try std.testing.expect(ReadableResource.getAttachmentType("attachment; FileName\"stuff.gz\"") == null);
11851220 try std.testing.expect(ReadableResource.getAttachmentType("attachment; size=42") == null);
1186 var value_end = mem.indexOfPos(u8, content_disposition, value_start, ";") orelse content_disposition.len;1221 try std.testing.expect(ReadableResource.getAttachmentType("inline; size=42") == null);
1187 if (content_disposition[value_end - 1] == '\"') {1222 try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\"; attachment;") == null);
1188 value_end -= 1;1223 try std.testing.expect(ReadableResource.getAttachmentType("FileName=\"stuff.tar.gz\";") == null);
1189 }
1190 return ascii.endsWithIgnoreCase(content_disposition[value_start..value_end], ".tar.gz");
1191}
1192
1193test "isTarAttachment" {
1194 try std.testing.expect(isTarAttachment("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
1195 try std.testing.expect(isTarAttachment("attachment; filename*=\"stuff.tar.gz\""));
1196 try std.testing.expect(isTarAttachment("ATTACHMENT; filename=\"stuff.tar.gz\""));
1197 try std.testing.expect(isTarAttachment("attachment; FileName=\"stuff.tar.gz\""));
1198 try std.testing.expect(isTarAttachment("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1199
1200 try std.testing.expect(!isTarAttachment("attachment FileName=\"stuff.tar.gz\""));
1201 try std.testing.expect(!isTarAttachment("attachment; FileName=\"stuff.tar\""));
1202 try std.testing.expect(!isTarAttachment("attachment; FileName\"stuff.gz\""));
1203 try std.testing.expect(!isTarAttachment("attachment; size=42"));
1204 try std.testing.expect(!isTarAttachment("inline; size=42"));
1205 try std.testing.expect(!isTarAttachment("FileName=\"stuff.tar.gz\"; attachment;"));
1206 try std.testing.expect(!isTarAttachment("FileName=\"stuff.tar.gz\";"));
1207}1224}