| ... | ... | @@ -6,6 +6,7 @@ const mem = std.mem; |
| 6 | 6 | const Allocator = mem.Allocator; |
| 7 | 7 | const assert = std.debug.assert; |
| 8 | 8 | const Hash = std.crypto.hash.sha2.Sha256; |
| 9 | const log = std.log.scoped(.package); |
| 9 | 10 | |
| 10 | 11 | const Compilation = @import("Compilation.zig"); |
| 11 | 12 | const Module = @import("Module.zig"); |
| ... | ... | @@ -128,6 +129,9 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P |
| 128 | 129 | return parent.add(gpa, name, child); |
| 129 | 130 | } |
| 130 | 131 | |
| 132 | pub const build_zig_basename = "build.zig"; |
| 133 | pub const ini_basename = build_zig_basename ++ ".ini"; |
| 134 | |
| 131 | 135 | pub fn fetchAndAddDependencies( |
| 132 | 136 | pkg: *Package, |
| 133 | 137 | thread_pool: *ThreadPool, |
| ... | ... | @@ -138,7 +142,7 @@ pub fn fetchAndAddDependencies( |
| 138 | 142 | ) !void { |
| 139 | 143 | const max_bytes = 10 * 1024 * 1024; |
| 140 | 144 | const gpa = thread_pool.allocator; |
| 141 | | const build_zig_ini = directory.handle.readFileAlloc(gpa, "build.zig.ini", max_bytes) catch |err| switch (err) { |
| 145 | const build_zig_ini = directory.handle.readFileAlloc(gpa, ini_basename, max_bytes) catch |err| switch (err) { |
| 142 | 146 | error.FileNotFound => { |
| 143 | 147 | // Handle the same as no dependencies. |
| 144 | 148 | return; |
| ... | ... | @@ -154,7 +158,7 @@ pub fn fetchAndAddDependencies( |
| 154 | 158 | var line_it = mem.split(u8, dep, "\n"); |
| 155 | 159 | var opt_id: ?[]const u8 = null; |
| 156 | 160 | var opt_url: ?[]const u8 = null; |
| 157 | | var expected_hash: ?[Hash.digest_length]u8 = null; |
| 161 | var expected_hash: ?[]const u8 = null; |
| 158 | 162 | while (line_it.next()) |kv| { |
| 159 | 163 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; |
| 160 | 164 | const key = kv[0..eq_pos]; |
| ... | ... | @@ -164,8 +168,7 @@ pub fn fetchAndAddDependencies( |
| 164 | 168 | } else if (mem.eql(u8, key, "url")) { |
| 165 | 169 | opt_url = value; |
| 166 | 170 | } else if (mem.eql(u8, key, "hash")) { |
| 167 | | @panic("TODO parse hex digits of value into expected_hash"); |
| 168 | | //expected_hash = value; |
| 171 | expected_hash = value; |
| 169 | 172 | } else { |
| 170 | 173 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 171 | 174 | std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ |
| ... | ... | @@ -208,6 +211,8 @@ pub fn fetchAndAddDependencies( |
| 208 | 211 | global_cache_directory, |
| 209 | 212 | url, |
| 210 | 213 | expected_hash, |
| 214 | ini, |
| 215 | directory, |
| 211 | 216 | ); |
| 212 | 217 | |
| 213 | 218 | try sub_pkg.fetchAndAddDependencies( |
| ... | ... | @@ -229,17 +234,48 @@ fn fetchAndUnpack( |
| 229 | 234 | http_client: *std.http.Client, |
| 230 | 235 | global_cache_directory: Compilation.Directory, |
| 231 | 236 | url: []const u8, |
| 232 | | expected_hash: ?[Hash.digest_length]u8, |
| 237 | expected_hash: ?[]const u8, |
| 238 | ini: std.Ini, |
| 239 | comp_directory: Compilation.Directory, |
| 233 | 240 | ) !*Package { |
| 234 | 241 | const gpa = http_client.allocator; |
| 242 | const s = fs.path.sep_str; |
| 235 | 243 | |
| 236 | 244 | // Check if the expected_hash is already present in the global package |
| 237 | 245 | // cache, and thereby avoid both fetching and unpacking. |
| 238 | | const s = fs.path.sep_str; |
| 239 | | if (expected_hash) |h| { |
| 240 | | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(h); |
| 241 | | _ = pkg_dir_sub_path; |
| 242 | | @panic("TODO check the p dir for the package"); |
| 246 | if (expected_hash) |h| cached: { |
| 247 | if (h.len != 2 * Hash.digest_length) { |
| 248 | return reportError( |
| 249 | ini, |
| 250 | comp_directory, |
| 251 | h.ptr, |
| 252 | "wrong hash size. expected: {d}, found: {d}", |
| 253 | .{ Hash.digest_length, h.len }, |
| 254 | ); |
| 255 | } |
| 256 | const hex_digest = h[0 .. 2 * Hash.digest_length]; |
| 257 | const pkg_dir_sub_path = "p" ++ s ++ hex_digest; |
| 258 | var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { |
| 259 | error.FileNotFound => break :cached, |
| 260 | else => |e| return e, |
| 261 | }; |
| 262 | errdefer pkg_dir.close(); |
| 263 | |
| 264 | const ptr = try gpa.create(Package); |
| 265 | errdefer gpa.destroy(ptr); |
| 266 | |
| 267 | const owned_src_path = try gpa.dupe(u8, build_zig_basename); |
| 268 | errdefer gpa.free(owned_src_path); |
| 269 | |
| 270 | ptr.* = .{ |
| 271 | .root_src_directory = .{ |
| 272 | .path = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}), |
| 273 | .handle = pkg_dir, |
| 274 | }, |
| 275 | .root_src_directory_owned = true, |
| 276 | .root_src_path = owned_src_path, |
| 277 | }; |
| 278 | return ptr; |
| 243 | 279 | } |
| 244 | 280 | |
| 245 | 281 | const uri = try std.Uri.parse(url); |
| ... | ... | @@ -277,9 +313,13 @@ fn fetchAndUnpack( |
| 277 | 313 | .strip_components = 1, |
| 278 | 314 | }); |
| 279 | 315 | } else { |
| 280 | | // TODO: show the build.zig.ini file and line number |
| 281 | | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| 282 | | return error.UnknownPackageExtension; |
| 316 | return reportError( |
| 317 | ini, |
| 318 | comp_directory, |
| 319 | uri.path.ptr, |
| 320 | "unknown file extension for path '{s}'", |
| 321 | .{uri.path}, |
| 322 | ); |
| 283 | 323 | } |
| 284 | 324 | |
| 285 | 325 | // TODO: delete files not included in the package prior to computing the package hash. |
| ... | ... | @@ -287,24 +327,13 @@ fn fetchAndUnpack( |
| 287 | 327 | // apply those rules directly to the filesystem right here. This ensures that files |
| 288 | 328 | // not protected by the hash are not present on the file system. |
| 289 | 329 | |
| 290 | | const actual_hash = try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); |
| 291 | | |
| 292 | | if (expected_hash) |h| { |
| 293 | | if (!mem.eql(u8, &h, &actual_hash)) { |
| 294 | | // TODO: show the build.zig.ini file and line number |
| 295 | | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| 296 | | url, h, actual_hash, |
| 297 | | }); |
| 298 | | return error.PackageHashMismatch; |
| 299 | | } |
| 300 | | } |
| 301 | | |
| 302 | | break :a actual_hash; |
| 330 | break :a try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); |
| 303 | 331 | }; |
| 304 | 332 | |
| 333 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| 334 | |
| 305 | 335 | { |
| 306 | 336 | // Rename the temporary directory into the global package cache. |
| 307 | | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| 308 | 337 | var handled_missing_dir = false; |
| 309 | 338 | while (true) { |
| 310 | 339 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { |
| ... | ... | @@ -316,27 +345,60 @@ fn fetchAndUnpack( |
| 316 | 345 | }; |
| 317 | 346 | continue; |
| 318 | 347 | }, |
| 348 | error.PathAlreadyExists => { |
| 349 | // Package has been already downloaded and may already be in use on the system. |
| 350 | global_cache_directory.handle.deleteTree(tmp_dir_sub_path) catch |del_err| { |
| 351 | std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); |
| 352 | }; |
| 353 | }, |
| 319 | 354 | else => |e| return e, |
| 320 | 355 | }; |
| 321 | 356 | break; |
| 322 | 357 | } |
| 323 | 358 | } |
| 324 | 359 | |
| 325 | | if (expected_hash == null) { |
| 326 | | // TODO: show the build.zig.ini file and line number |
| 327 | | std.log.err("{s}: missing hash:\nhash={s}", .{ |
| 328 | | url, std.fmt.fmtSliceHexLower(&actual_hash), |
| 329 | | }); |
| 330 | | return error.PackageDependencyMissingHash; |
| 360 | if (expected_hash) |h| { |
| 361 | const actual_hex = hexDigest(actual_hash); |
| 362 | if (!mem.eql(u8, h, &actual_hex)) { |
| 363 | return reportError( |
| 364 | ini, |
| 365 | comp_directory, |
| 366 | h.ptr, |
| 367 | "hash mismatch: expected: {s}, found: {s}", |
| 368 | .{ h, actual_hex }, |
| 369 | ); |
| 370 | } |
| 371 | } else { |
| 372 | return reportError( |
| 373 | ini, |
| 374 | comp_directory, |
| 375 | url.ptr, |
| 376 | "url field is missing corresponding hash field: hash={s}", |
| 377 | .{std.fmt.fmtSliceHexLower(&actual_hash)}, |
| 378 | ); |
| 331 | 379 | } |
| 332 | 380 | |
| 333 | | @panic("TODO create package and set root_src_directory"); |
| 334 | | //return create(gpa, root_src |
| 335 | | //gpa: Allocator, |
| 336 | | ///// Null indicates the current working directory |
| 337 | | //root_src_dir_path: ?[]const u8, |
| 338 | | ///// Relative to root_src_dir_path |
| 339 | | //root_src_path: []const u8, |
| 381 | return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); |
| 382 | } |
| 383 | |
| 384 | fn reportError( |
| 385 | ini: std.Ini, |
| 386 | comp_directory: Compilation.Directory, |
| 387 | src_ptr: [*]const u8, |
| 388 | comptime fmt_string: []const u8, |
| 389 | fmt_args: anytype, |
| 390 | ) error{PackageFetchFailed} { |
| 391 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(src_ptr) - @ptrToInt(ini.bytes.ptr)); |
| 392 | if (comp_directory.path) |p| { |
| 393 | std.debug.print("{s}{c}{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ |
| 394 | p, fs.path.sep, ini_basename, loc.line + 1, loc.column + 1, |
| 395 | } ++ fmt_args); |
| 396 | } else { |
| 397 | std.debug.print("{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ |
| 398 | ini_basename, loc.line + 1, loc.column + 1, |
| 399 | } ++ fmt_args); |
| 400 | } |
| 401 | return error.PackageFetchFailed; |
| 340 | 402 | } |
| 341 | 403 | |
| 342 | 404 | const HashedFile = struct { |
| ... | ... | @@ -389,9 +451,10 @@ fn computePackageHash( |
| 389 | 451 | .hash = undefined, // to be populated by the worker |
| 390 | 452 | .failure = undefined, // to be populated by the worker |
| 391 | 453 | }; |
| 392 | | |
| 393 | 454 | wait_group.start(); |
| 394 | 455 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); |
| 456 | |
| 457 | try all_files.append(hashed_file); |
| 395 | 458 | } |
| 396 | 459 | } |
| 397 | 460 | |
Andrew Kelley <andrew@ziglang.org> 2023-01-10 16:26:27-07:00