| ... | ... | @@ -0,0 +1,738 @@ |
| 1 | const std = @import("std"); |
| 2 | const fs = std.fs; |
| 3 | const io = std.io; |
| 4 | const mem = std.mem; |
| 5 | const process = std.process; |
| 6 | const assert = std.debug.assert; |
| 7 | const tmpDir = std.testing.tmpDir; |
| 8 | |
| 9 | const Allocator = mem.Allocator; |
| 10 | const Blake3 = std.crypto.hash.Blake3; |
| 11 | const OsTag = std.Target.Os.Tag; |
| 12 | |
| 13 | var general_purpose_allocator = std.heap.GeneralPurposeAllocator(.{}){}; |
| 14 | const gpa = general_purpose_allocator.allocator(); |
| 15 | |
| 16 | const Arch = enum { |
| 17 | any, |
| 18 | aarch64, |
| 19 | x86_64, |
| 20 | }; |
| 21 | |
| 22 | const Abi = enum { any, none }; |
| 23 | |
| 24 | const OsVer = enum(u32) { |
| 25 | any = 0, |
| 26 | catalina = 10, |
| 27 | big_sur = 11, |
| 28 | monterey = 12, |
| 29 | ventura = 13, |
| 30 | sonoma = 14, |
| 31 | sequoia = 15, |
| 32 | }; |
| 33 | |
| 34 | const Target = struct { |
| 35 | arch: Arch, |
| 36 | os: OsTag = .macos, |
| 37 | os_ver: OsVer, |
| 38 | abi: Abi = .none, |
| 39 | |
| 40 | fn hash(a: Target) u32 { |
| 41 | var hasher = std.hash.Wyhash.init(0); |
| 42 | std.hash.autoHash(&hasher, a.arch); |
| 43 | std.hash.autoHash(&hasher, a.os); |
| 44 | std.hash.autoHash(&hasher, a.os_ver); |
| 45 | std.hash.autoHash(&hasher, a.abi); |
| 46 | return @as(u32, @truncate(hasher.final())); |
| 47 | } |
| 48 | |
| 49 | fn eql(a: Target, b: Target) bool { |
| 50 | return a.arch == b.arch and |
| 51 | a.os == b.os and |
| 52 | a.os_ver == b.os_ver and |
| 53 | a.abi == b.abi; |
| 54 | } |
| 55 | |
| 56 | fn name(self: Target, allocator: Allocator) ![]const u8 { |
| 57 | return std.fmt.allocPrint(allocator, "{s}-{s}-{s}", .{ |
| 58 | @tagName(self.arch), |
| 59 | @tagName(self.os), |
| 60 | @tagName(self.abi), |
| 61 | }); |
| 62 | } |
| 63 | |
| 64 | fn fullName(self: Target, allocator: Allocator) ![]const u8 { |
| 65 | if (self.os_ver == .any) return self.name(allocator); |
| 66 | return std.fmt.allocPrint(allocator, "{s}-{s}.{d}-{s}", .{ |
| 67 | @tagName(self.arch), |
| 68 | @tagName(self.os), |
| 69 | @intFromEnum(self.os_ver), |
| 70 | @tagName(self.abi), |
| 71 | }); |
| 72 | } |
| 73 | }; |
| 74 | |
| 75 | const targets = [_]Target{ |
| 76 | Target{ |
| 77 | .arch = .any, |
| 78 | .abi = .any, |
| 79 | .os_ver = .any, |
| 80 | }, |
| 81 | Target{ |
| 82 | .arch = .aarch64, |
| 83 | .os_ver = .any, |
| 84 | }, |
| 85 | Target{ |
| 86 | .arch = .x86_64, |
| 87 | .os_ver = .any, |
| 88 | }, |
| 89 | Target{ |
| 90 | .arch = .x86_64, |
| 91 | .os_ver = .catalina, |
| 92 | }, |
| 93 | Target{ |
| 94 | .arch = .x86_64, |
| 95 | .os_ver = .big_sur, |
| 96 | }, |
| 97 | Target{ |
| 98 | .arch = .x86_64, |
| 99 | .os_ver = .monterey, |
| 100 | }, |
| 101 | Target{ |
| 102 | .arch = .x86_64, |
| 103 | .os_ver = .ventura, |
| 104 | }, |
| 105 | Target{ |
| 106 | .arch = .x86_64, |
| 107 | .os_ver = .sonoma, |
| 108 | }, |
| 109 | Target{ |
| 110 | .arch = .x86_64, |
| 111 | .os_ver = .sequoia, |
| 112 | }, |
| 113 | Target{ |
| 114 | .arch = .aarch64, |
| 115 | .os_ver = .big_sur, |
| 116 | }, |
| 117 | Target{ |
| 118 | .arch = .aarch64, |
| 119 | .os_ver = .monterey, |
| 120 | }, |
| 121 | Target{ |
| 122 | .arch = .aarch64, |
| 123 | .os_ver = .ventura, |
| 124 | }, |
| 125 | Target{ |
| 126 | .arch = .aarch64, |
| 127 | .os_ver = .sonoma, |
| 128 | }, |
| 129 | Target{ |
| 130 | .arch = .aarch64, |
| 131 | .os_ver = .sequoia, |
| 132 | }, |
| 133 | }; |
| 134 | |
| 135 | const headers_source_prefix: []const u8 = "headers"; |
| 136 | |
| 137 | const Contents = struct { |
| 138 | bytes: []const u8, |
| 139 | hit_count: usize, |
| 140 | hash: []const u8, |
| 141 | is_generic: bool, |
| 142 | |
| 143 | fn hitCountLessThan(context: void, lhs: *const Contents, rhs: *const Contents) bool { |
| 144 | _ = context; |
| 145 | return lhs.hit_count < rhs.hit_count; |
| 146 | } |
| 147 | }; |
| 148 | |
| 149 | const TargetToHashContext = struct { |
| 150 | pub fn hash(self: @This(), target: Target) u32 { |
| 151 | _ = self; |
| 152 | return target.hash(); |
| 153 | } |
| 154 | pub fn eql(self: @This(), a: Target, b: Target, b_index: usize) bool { |
| 155 | _ = self; |
| 156 | _ = b_index; |
| 157 | return a.eql(b); |
| 158 | } |
| 159 | }; |
| 160 | const TargetToHash = std.ArrayHashMap(Target, []const u8, TargetToHashContext, true); |
| 161 | |
| 162 | const HashToContents = std.StringHashMap(Contents); |
| 163 | const PathTable = std.StringHashMap(*TargetToHash); |
| 164 | |
| 165 | /// The don't-dedup-list contains file paths with known problematic headers |
| 166 | /// which while contain the same contents between architectures, should not be |
| 167 | /// deduped since they contain includes, etc. which are relative and thus cannot be separated |
| 168 | /// into a shared include dir such as `any-macos-any`. |
| 169 | const dont_dedup_list = &[_][]const u8{ |
| 170 | "libkern/OSAtomic.h", |
| 171 | "libkern/OSAtomicDeprecated.h", |
| 172 | "libkern/OSSpinLockDeprecated.h", |
| 173 | "libkern/OSAtomicQueue.h", |
| 174 | }; |
| 175 | |
| 176 | fn generateDontDedupMap(arena: Allocator) !std.StringHashMap(void) { |
| 177 | var map = std.StringHashMap(void).init(arena); |
| 178 | try map.ensureTotalCapacity(dont_dedup_list.len); |
| 179 | for (dont_dedup_list) |path| { |
| 180 | map.putAssumeCapacityNoClobber(path, {}); |
| 181 | } |
| 182 | return map; |
| 183 | } |
| 184 | |
| 185 | const usage = |
| 186 | \\fetch_them_macos_headers fetch |
| 187 | \\fetch_them_macos_headers dedup |
| 188 | \\ |
| 189 | \\Commands: |
| 190 | \\ fetch Fetch libc headers into headers/<arch>-macos.<os_ver> dir |
| 191 | \\ dedup Generate deduplicated dirs into a given <destination> path |
| 192 | \\ |
| 193 | \\General Options: |
| 194 | \\-h, --help Print this help and exit |
| 195 | ; |
| 196 | |
| 197 | pub fn main() anyerror!void { |
| 198 | var arena = std.heap.ArenaAllocator.init(gpa); |
| 199 | defer arena.deinit(); |
| 200 | |
| 201 | const all_args = try std.process.argsAlloc(arena.allocator()); |
| 202 | const args = all_args[1..]; |
| 203 | if (args.len == 0) fatal("no command or option specified", .{}); |
| 204 | |
| 205 | const cmd = args[0]; |
| 206 | if (mem.eql(u8, cmd, "--help") or mem.eql(u8, cmd, "-h")) { |
| 207 | return info(usage, .{}); |
| 208 | } else if (mem.eql(u8, cmd, "dedup")) { |
| 209 | return dedup(arena.allocator(), args[1..]); |
| 210 | } else if (mem.eql(u8, cmd, "fetch")) { |
| 211 | return fetch(arena.allocator(), args[1..]); |
| 212 | } else fatal("unknown command or option: {s}", .{cmd}); |
| 213 | } |
| 214 | |
| 215 | const ArgsIterator = struct { |
| 216 | args: []const []const u8, |
| 217 | i: usize = 0, |
| 218 | |
| 219 | fn next(it: *@This()) ?[]const u8 { |
| 220 | if (it.i >= it.args.len) { |
| 221 | return null; |
| 222 | } |
| 223 | defer it.i += 1; |
| 224 | return it.args[it.i]; |
| 225 | } |
| 226 | |
| 227 | fn nextOrFatal(it: *@This()) []const u8 { |
| 228 | const arg = it.next() orelse fatal("expected parameter after '{s}'", .{it.args[it.i - 1]}); |
| 229 | return arg; |
| 230 | } |
| 231 | }; |
| 232 | |
| 233 | fn info(comptime format: []const u8, args: anytype) void { |
| 234 | const msg = std.fmt.allocPrint(gpa, "info: " ++ format ++ "\n", args) catch return; |
| 235 | std.io.getStdOut().writeAll(msg) catch {}; |
| 236 | } |
| 237 | |
| 238 | fn fatal(comptime format: []const u8, args: anytype) noreturn { |
| 239 | ret: { |
| 240 | const msg = std.fmt.allocPrint(gpa, "fatal: " ++ format ++ "\n", args) catch break :ret; |
| 241 | std.io.getStdErr().writeAll(msg) catch {}; |
| 242 | } |
| 243 | std.process.exit(1); |
| 244 | } |
| 245 | |
| 246 | const fetch_usage = |
| 247 | \\fetch_them_macos_headers fetch |
| 248 | \\ |
| 249 | \\Options: |
| 250 | \\ --sysroot Path to macOS SDK |
| 251 | \\ |
| 252 | \\General Options: |
| 253 | \\-h, --help Print this help and exit |
| 254 | ; |
| 255 | |
| 256 | fn fetch(arena: Allocator, args: []const []const u8) !void { |
| 257 | var argv = std.ArrayList([]const u8).init(arena); |
| 258 | var sysroot: ?[]const u8 = null; |
| 259 | |
| 260 | var args_iter = ArgsIterator{ .args = args }; |
| 261 | while (args_iter.next()) |arg| { |
| 262 | if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) { |
| 263 | return info(fetch_usage, .{}); |
| 264 | } else if (mem.eql(u8, arg, "--sysroot")) { |
| 265 | sysroot = args_iter.nextOrFatal(); |
| 266 | } else try argv.append(arg); |
| 267 | } |
| 268 | |
| 269 | const sysroot_path = sysroot orelse blk: { |
| 270 | const target = try std.zig.system.resolveTargetQuery(.{}); |
| 271 | break :blk std.zig.system.darwin.getSdk(arena, target) orelse |
| 272 | fatal("no SDK found; you can provide one explicitly with '--sysroot' flag", .{}); |
| 273 | }; |
| 274 | |
| 275 | var sdk_dir = try std.fs.cwd().openDir(sysroot_path, .{}); |
| 276 | defer sdk_dir.close(); |
| 277 | const sdk_info = try sdk_dir.readFileAlloc(arena, "SDKSettings.json", std.math.maxInt(u32)); |
| 278 | |
| 279 | const parsed_json = try std.json.parseFromSlice(struct { |
| 280 | DefaultProperties: struct { MACOSX_DEPLOYMENT_TARGET: []const u8 }, |
| 281 | }, arena, sdk_info, .{ .ignore_unknown_fields = true }); |
| 282 | |
| 283 | const version = Version.parse(parsed_json.value.DefaultProperties.MACOSX_DEPLOYMENT_TARGET) orelse |
| 284 | fatal("don't know how to parse SDK version: {s}", .{ |
| 285 | parsed_json.value.DefaultProperties.MACOSX_DEPLOYMENT_TARGET, |
| 286 | }); |
| 287 | const os_ver: OsVer = switch (version.major) { |
| 288 | 10 => .catalina, |
| 289 | 11 => .big_sur, |
| 290 | 12 => .monterey, |
| 291 | 13 => .ventura, |
| 292 | 14 => .sonoma, |
| 293 | 15 => .sequoia, |
| 294 | else => unreachable, |
| 295 | }; |
| 296 | info("found SDK deployment target macOS {} aka '{s}'", .{ version, @tagName(os_ver) }); |
| 297 | |
| 298 | var tmp = tmpDir(.{}); |
| 299 | defer tmp.cleanup(); |
| 300 | |
| 301 | for (&[_]Arch{ .aarch64, .x86_64 }) |arch| { |
| 302 | const target: Target = .{ |
| 303 | .arch = arch, |
| 304 | .os_ver = os_ver, |
| 305 | }; |
| 306 | try fetchTarget(arena, argv.items, sysroot_path, target, version, tmp); |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | fn fetchTarget( |
| 311 | arena: Allocator, |
| 312 | args: []const []const u8, |
| 313 | sysroot: []const u8, |
| 314 | target: Target, |
| 315 | ver: Version, |
| 316 | tmp: std.testing.TmpDir, |
| 317 | ) !void { |
| 318 | const tmp_filename = "headers"; |
| 319 | const headers_list_filename = "headers.o.d"; |
| 320 | const tmp_path = try tmp.dir.realpathAlloc(arena, "."); |
| 321 | const tmp_file_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, tmp_filename }); |
| 322 | const headers_list_path = try fs.path.join(arena, &[_][]const u8{ tmp_path, headers_list_filename }); |
| 323 | |
| 324 | const macos_version = try std.fmt.allocPrint(arena, "-mmacosx-version-min={d}.{d}", .{ |
| 325 | ver.major, |
| 326 | ver.minor, |
| 327 | }); |
| 328 | |
| 329 | var cc_argv = std.ArrayList([]const u8).init(arena); |
| 330 | try cc_argv.appendSlice(&[_][]const u8{ |
| 331 | "cc", |
| 332 | "-arch", |
| 333 | switch (target.arch) { |
| 334 | .x86_64 => "x86_64", |
| 335 | .aarch64 => "arm64", |
| 336 | else => unreachable, |
| 337 | }, |
| 338 | macos_version, |
| 339 | "-isysroot", |
| 340 | sysroot, |
| 341 | "-iwithsysroot", |
| 342 | "/usr/include", |
| 343 | "-o", |
| 344 | tmp_file_path, |
| 345 | "macos-headers.c", |
| 346 | "-MD", |
| 347 | "-MV", |
| 348 | "-MF", |
| 349 | headers_list_path, |
| 350 | }); |
| 351 | try cc_argv.appendSlice(args); |
| 352 | |
| 353 | // TODO instead of calling `cc` as a child process here, |
| 354 | // hook in directly to `zig cc` API. |
| 355 | const res = try std.process.Child.run(.{ |
| 356 | .allocator = arena, |
| 357 | .argv = cc_argv.items, |
| 358 | }); |
| 359 | |
| 360 | if (res.stderr.len != 0) { |
| 361 | std.log.err("{s}", .{res.stderr}); |
| 362 | } |
| 363 | |
| 364 | // Read in the contents of `upgrade.o.d` |
| 365 | const headers_list_file = try tmp.dir.openFile(headers_list_filename, .{}); |
| 366 | defer headers_list_file.close(); |
| 367 | |
| 368 | var headers_dir = fs.cwd().openDir(headers_source_prefix, .{}) catch |err| switch (err) { |
| 369 | error.FileNotFound, |
| 370 | error.NotDir, |
| 371 | => fatal("path '{s}' not found or not a directory. Did you accidentally delete it?", .{ |
| 372 | headers_source_prefix, |
| 373 | }), |
| 374 | else => return err, |
| 375 | }; |
| 376 | defer headers_dir.close(); |
| 377 | |
| 378 | const dest_path = try target.fullName(arena); |
| 379 | try headers_dir.deleteTree(dest_path); |
| 380 | |
| 381 | var dest_dir = try headers_dir.makeOpenPath(dest_path, .{}); |
| 382 | var dirs = std.StringHashMap(fs.Dir).init(arena); |
| 383 | try dirs.putNoClobber(".", dest_dir); |
| 384 | |
| 385 | const headers_list_str = try headers_list_file.reader().readAllAlloc(arena, std.math.maxInt(usize)); |
| 386 | const prefix = "/usr/include"; |
| 387 | |
| 388 | var it = mem.splitScalar(u8, headers_list_str, '\n'); |
| 389 | while (it.next()) |line| { |
| 390 | if (mem.lastIndexOf(u8, line, "clang") != null) continue; |
| 391 | if (mem.lastIndexOf(u8, line, prefix[0..])) |idx| { |
| 392 | const out_rel_path = line[idx + prefix.len + 1 ..]; |
| 393 | const out_rel_path_stripped = mem.trim(u8, out_rel_path, " \\"); |
| 394 | const dirname = fs.path.dirname(out_rel_path_stripped) orelse "."; |
| 395 | const maybe_dir = try dirs.getOrPut(dirname); |
| 396 | if (!maybe_dir.found_existing) { |
| 397 | maybe_dir.value_ptr.* = try dest_dir.makeOpenPath(dirname, .{}); |
| 398 | } |
| 399 | const basename = fs.path.basename(out_rel_path_stripped); |
| 400 | |
| 401 | const line_stripped = mem.trim(u8, line, " \\"); |
| 402 | const abs_dirname = fs.path.dirname(line_stripped).?; |
| 403 | var orig_subdir = try fs.cwd().openDir(abs_dirname, .{}); |
| 404 | defer orig_subdir.close(); |
| 405 | |
| 406 | try orig_subdir.copyFile(basename, maybe_dir.value_ptr.*, basename, .{}); |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | var dir_it = dirs.iterator(); |
| 411 | while (dir_it.next()) |entry| { |
| 412 | entry.value_ptr.close(); |
| 413 | } |
| 414 | } |
| 415 | |
| 416 | const dedup_usage = |
| 417 | \\fetch_them_macos_headers dedup [path] |
| 418 | \\ |
| 419 | \\General Options: |
| 420 | \\-h, --help Print this help and exit |
| 421 | ; |
| 422 | |
| 423 | /// Dedups libs headers assuming the following layered structure: |
| 424 | /// layer 1: x86_64-macos.10 x86_64-macos.11 x86_64-macos.12 aarch64-macos.11 aarch64-macos.12 |
| 425 | /// layer 2: any-macos.10 any-macos.11 any-macos.12 |
| 426 | /// layer 3: any-macos |
| 427 | /// |
| 428 | /// The first layer consists of headers specific to a CPU architecture AND macOS version. The second |
| 429 | /// layer consists of headers common to a macOS version across CPU architectures, and the final |
| 430 | /// layer consists of headers common to all libc headers. |
| 431 | fn dedup(arena: Allocator, args: []const []const u8) !void { |
| 432 | var path: ?[]const u8 = null; |
| 433 | var args_iter = ArgsIterator{ .args = args }; |
| 434 | while (args_iter.next()) |arg| { |
| 435 | if (mem.eql(u8, arg, "--help") or mem.eql(u8, arg, "-h")) { |
| 436 | return info(dedup_usage, .{}); |
| 437 | } else { |
| 438 | if (path != null) fatal("too many arguments", .{}); |
| 439 | path = arg; |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | const dest_path = path orelse fatal("no destination path specified", .{}); |
| 444 | var dest_dir = fs.cwd().makeOpenPath(dest_path, .{}) catch |err| switch (err) { |
| 445 | error.NotDir => fatal("path '{s}' not a directory", .{dest_path}), |
| 446 | else => return err, |
| 447 | }; |
| 448 | defer dest_dir.close(); |
| 449 | |
| 450 | var dont_dedup_map = try generateDontDedupMap(arena); |
| 451 | var layer_2_targets = std.ArrayList(TargetWithPrefix).init(arena); |
| 452 | |
| 453 | for (&[_]OsVer{ .catalina, .big_sur, .monterey, .ventura, .sonoma, .sequoia }) |os_ver| { |
| 454 | var layer_1_targets = std.ArrayList(TargetWithPrefix).init(arena); |
| 455 | |
| 456 | for (targets) |target| { |
| 457 | if (target.os_ver != os_ver) continue; |
| 458 | try layer_1_targets.append(.{ |
| 459 | .prefix = headers_source_prefix, |
| 460 | .target = target, |
| 461 | }); |
| 462 | } |
| 463 | |
| 464 | if (layer_1_targets.items.len < 2) { |
| 465 | try layer_2_targets.appendSlice(layer_1_targets.items); |
| 466 | continue; |
| 467 | } |
| 468 | |
| 469 | const layer_2_target = try dedupDirs(arena, .{ |
| 470 | .os_ver = os_ver, |
| 471 | .dest_path = dest_path, |
| 472 | .dest_dir = dest_dir, |
| 473 | .targets = layer_1_targets.items, |
| 474 | .dont_dedup_map = &dont_dedup_map, |
| 475 | }); |
| 476 | try layer_2_targets.append(layer_2_target); |
| 477 | } |
| 478 | |
| 479 | const layer_3_target = try dedupDirs(arena, .{ |
| 480 | .os_ver = .any, |
| 481 | .dest_path = dest_path, |
| 482 | .dest_dir = dest_dir, |
| 483 | .targets = layer_2_targets.items, |
| 484 | .dont_dedup_map = &dont_dedup_map, |
| 485 | }); |
| 486 | assert(layer_3_target.target.eql(targets[0])); |
| 487 | } |
| 488 | |
| 489 | const TargetWithPrefix = struct { |
| 490 | prefix: []const u8, |
| 491 | target: Target, |
| 492 | }; |
| 493 | |
| 494 | const DedupDirsArgs = struct { |
| 495 | os_ver: OsVer, |
| 496 | dest_path: []const u8, |
| 497 | dest_dir: fs.Dir, |
| 498 | targets: []const TargetWithPrefix, |
| 499 | dont_dedup_map: *const std.StringHashMap(void), |
| 500 | }; |
| 501 | |
| 502 | fn dedupDirs(arena: Allocator, args: DedupDirsArgs) !TargetWithPrefix { |
| 503 | var tmp = tmpDir(.{ .iterate = true }); |
| 504 | defer tmp.cleanup(); |
| 505 | |
| 506 | var path_table = PathTable.init(arena); |
| 507 | var hash_to_contents = HashToContents.init(arena); |
| 508 | |
| 509 | var savings = FindResult{}; |
| 510 | for (args.targets) |target| { |
| 511 | const res = try findDuplicates(target.target, arena, target.prefix, &path_table, &hash_to_contents); |
| 512 | savings.max_bytes_saved += res.max_bytes_saved; |
| 513 | savings.total_bytes += res.total_bytes; |
| 514 | } |
| 515 | |
| 516 | info("summary: {} could be reduced to {}", .{ |
| 517 | std.fmt.fmtIntSizeBin(savings.total_bytes), |
| 518 | std.fmt.fmtIntSizeBin(savings.total_bytes - savings.max_bytes_saved), |
| 519 | }); |
| 520 | |
| 521 | const output_target = Target{ |
| 522 | .arch = .any, |
| 523 | .abi = .any, |
| 524 | .os_ver = args.os_ver, |
| 525 | }; |
| 526 | const common_name = try output_target.fullName(arena); |
| 527 | |
| 528 | var missed_opportunity_bytes: usize = 0; |
| 529 | // Iterate path_table. For each path, put all the hashes into a list. Sort by hit_count. |
| 530 | // The hash with the highest hit_count gets to be the "generic" one. Everybody else |
| 531 | // gets their header in a separate arch directory. |
| 532 | var path_it = path_table.iterator(); |
| 533 | while (path_it.next()) |path_kv| { |
| 534 | if (!args.dont_dedup_map.contains(path_kv.key_ptr.*)) { |
| 535 | var contents_list = std.ArrayList(*Contents).init(arena); |
| 536 | { |
| 537 | var hash_it = path_kv.value_ptr.*.iterator(); |
| 538 | while (hash_it.next()) |hash_kv| { |
| 539 | const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*; |
| 540 | try contents_list.append(contents); |
| 541 | } |
| 542 | } |
| 543 | std.mem.sort(*Contents, contents_list.items, {}, Contents.hitCountLessThan); |
| 544 | const best_contents = contents_list.popOrNull().?; |
| 545 | if (best_contents.hit_count > 1) { |
| 546 | // Put it in `any-macos-none`. |
| 547 | const full_path = try fs.path.join(arena, &[_][]const u8{ common_name, path_kv.key_ptr.* }); |
| 548 | try tmp.dir.makePath(fs.path.dirname(full_path).?); |
| 549 | try tmp.dir.writeFile(.{ .sub_path = full_path, .data = best_contents.bytes }); |
| 550 | best_contents.is_generic = true; |
| 551 | while (contents_list.popOrNull()) |contender| { |
| 552 | if (contender.hit_count > 1) { |
| 553 | const this_missed_bytes = contender.hit_count * contender.bytes.len; |
| 554 | missed_opportunity_bytes += this_missed_bytes; |
| 555 | info("Missed opportunity ({}): {s}", .{ |
| 556 | std.fmt.fmtIntSizeBin(this_missed_bytes), |
| 557 | path_kv.key_ptr.*, |
| 558 | }); |
| 559 | } else break; |
| 560 | } |
| 561 | } |
| 562 | } |
| 563 | var hash_it = path_kv.value_ptr.*.iterator(); |
| 564 | while (hash_it.next()) |hash_kv| { |
| 565 | const contents = &hash_to_contents.getEntry(hash_kv.value_ptr.*).?.value_ptr.*; |
| 566 | if (contents.is_generic) continue; |
| 567 | |
| 568 | const target = hash_kv.key_ptr.*; |
| 569 | const target_name = try target.fullName(arena); |
| 570 | const full_path = try fs.path.join(arena, &[_][]const u8{ target_name, path_kv.key_ptr.* }); |
| 571 | try tmp.dir.makePath(fs.path.dirname(full_path).?); |
| 572 | try tmp.dir.writeFile(.{ .sub_path = full_path, .data = contents.bytes }); |
| 573 | } |
| 574 | } |
| 575 | |
| 576 | for (args.targets) |target| { |
| 577 | const target_name = try target.target.fullName(arena); |
| 578 | try args.dest_dir.deleteTree(target_name); |
| 579 | } |
| 580 | try args.dest_dir.deleteTree(common_name); |
| 581 | |
| 582 | var tmp_it = tmp.dir.iterate(); |
| 583 | while (try tmp_it.next()) |entry| { |
| 584 | switch (entry.kind) { |
| 585 | .directory => { |
| 586 | const sub_dir = try tmp.dir.openDir(entry.name, .{ .iterate = true }); |
| 587 | const dest_sub_dir = try args.dest_dir.makeOpenPath(entry.name, .{}); |
| 588 | try copyDirAll(sub_dir, dest_sub_dir); |
| 589 | }, |
| 590 | else => info("unexpected file format: not a directory: '{s}'", .{entry.name}), |
| 591 | } |
| 592 | } |
| 593 | |
| 594 | return TargetWithPrefix{ |
| 595 | .prefix = args.dest_path, |
| 596 | .target = output_target, |
| 597 | }; |
| 598 | } |
| 599 | |
| 600 | const FindResult = struct { |
| 601 | max_bytes_saved: usize = 0, |
| 602 | total_bytes: usize = 0, |
| 603 | }; |
| 604 | |
| 605 | fn findDuplicates( |
| 606 | target: Target, |
| 607 | arena: Allocator, |
| 608 | dest_path: []const u8, |
| 609 | path_table: *PathTable, |
| 610 | hash_to_contents: *HashToContents, |
| 611 | ) !FindResult { |
| 612 | var result = FindResult{}; |
| 613 | |
| 614 | const target_name = try target.fullName(arena); |
| 615 | const target_include_dir = try fs.path.join(arena, &[_][]const u8{ dest_path, target_name }); |
| 616 | var dir_stack = std.ArrayList([]const u8).init(arena); |
| 617 | try dir_stack.append(target_include_dir); |
| 618 | |
| 619 | while (dir_stack.popOrNull()) |full_dir_name| { |
| 620 | var dir = fs.cwd().openDir(full_dir_name, .{ .iterate = true }) catch |err| switch (err) { |
| 621 | error.FileNotFound => break, |
| 622 | error.AccessDenied => break, |
| 623 | else => return err, |
| 624 | }; |
| 625 | defer dir.close(); |
| 626 | |
| 627 | var dir_it = dir.iterate(); |
| 628 | |
| 629 | while (try dir_it.next()) |entry| { |
| 630 | const full_path = try fs.path.join(arena, &[_][]const u8{ full_dir_name, entry.name }); |
| 631 | switch (entry.kind) { |
| 632 | .directory => try dir_stack.append(full_path), |
| 633 | .file => { |
| 634 | const rel_path = try fs.path.relative(arena, target_include_dir, full_path); |
| 635 | const max_size = 2 * 1024 * 1024 * 1024; |
| 636 | const raw_bytes = try fs.cwd().readFileAlloc(arena, full_path, max_size); |
| 637 | const trimmed = mem.trim(u8, raw_bytes, " \r\n\t"); |
| 638 | result.total_bytes += raw_bytes.len; |
| 639 | const hash = try arena.alloc(u8, 32); |
| 640 | var hasher = Blake3.init(.{}); |
| 641 | hasher.update(rel_path); |
| 642 | hasher.update(trimmed); |
| 643 | hasher.final(hash); |
| 644 | const gop = try hash_to_contents.getOrPut(hash); |
| 645 | if (gop.found_existing) { |
| 646 | result.max_bytes_saved += raw_bytes.len; |
| 647 | gop.value_ptr.hit_count += 1; |
| 648 | info("duplicate: {s} {s} ({})", .{ |
| 649 | target_name, |
| 650 | rel_path, |
| 651 | std.fmt.fmtIntSizeBin(raw_bytes.len), |
| 652 | }); |
| 653 | } else { |
| 654 | gop.value_ptr.* = Contents{ |
| 655 | .bytes = trimmed, |
| 656 | .hit_count = 1, |
| 657 | .hash = hash, |
| 658 | .is_generic = false, |
| 659 | }; |
| 660 | } |
| 661 | const path_gop = try path_table.getOrPut(rel_path); |
| 662 | const target_to_hash = if (path_gop.found_existing) path_gop.value_ptr.* else blk: { |
| 663 | const ptr = try arena.create(TargetToHash); |
| 664 | ptr.* = TargetToHash.init(arena); |
| 665 | path_gop.value_ptr.* = ptr; |
| 666 | break :blk ptr; |
| 667 | }; |
| 668 | try target_to_hash.putNoClobber(target, hash); |
| 669 | }, |
| 670 | else => info("unexpected file: {s}", .{full_path}), |
| 671 | } |
| 672 | } |
| 673 | } |
| 674 | |
| 675 | return result; |
| 676 | } |
| 677 | |
| 678 | fn copyDirAll(source: fs.Dir, dest: fs.Dir) anyerror!void { |
| 679 | var it = source.iterate(); |
| 680 | while (try it.next()) |next| { |
| 681 | switch (next.kind) { |
| 682 | .directory => { |
| 683 | var sub_dir = try dest.makeOpenPath(next.name, .{}); |
| 684 | var sub_source = try source.openDir(next.name, .{ .iterate = true }); |
| 685 | defer { |
| 686 | sub_dir.close(); |
| 687 | sub_source.close(); |
| 688 | } |
| 689 | try copyDirAll(sub_source, sub_dir); |
| 690 | }, |
| 691 | .file => { |
| 692 | var source_file = try source.openFile(next.name, .{}); |
| 693 | var dest_file = try dest.createFile(next.name, .{}); |
| 694 | defer { |
| 695 | source_file.close(); |
| 696 | dest_file.close(); |
| 697 | } |
| 698 | const stat = try source_file.stat(); |
| 699 | const ncopied = try source_file.copyRangeAll(0, dest_file, 0, stat.size); |
| 700 | assert(ncopied == stat.size); |
| 701 | }, |
| 702 | else => |kind| info("unexpected file kind '{s}' will be ignored", .{@tagName(kind)}), |
| 703 | } |
| 704 | } |
| 705 | } |
| 706 | |
| 707 | const Version = struct { |
| 708 | major: u16, |
| 709 | minor: u8, |
| 710 | patch: u8, |
| 711 | |
| 712 | fn parse(raw: []const u8) ?Version { |
| 713 | var parsed: [3]u16 = [_]u16{0} ** 3; |
| 714 | var count: usize = 0; |
| 715 | var it = std.mem.splitAny(u8, raw, "."); |
| 716 | while (it.next()) |comp| { |
| 717 | if (count >= 3) return null; |
| 718 | parsed[count] = std.fmt.parseInt(u16, comp, 10) catch return null; |
| 719 | count += 1; |
| 720 | } |
| 721 | if (count == 0) return null; |
| 722 | const major = parsed[0]; |
| 723 | const minor = std.math.cast(u8, parsed[1]) orelse return null; |
| 724 | const patch = std.math.cast(u8, parsed[2]) orelse return null; |
| 725 | return .{ .major = major, .minor = minor, .patch = patch }; |
| 726 | } |
| 727 | |
| 728 | pub fn format( |
| 729 | v: Version, |
| 730 | comptime unused_fmt_string: []const u8, |
| 731 | options: std.fmt.FormatOptions, |
| 732 | writer: anytype, |
| 733 | ) !void { |
| 734 | _ = unused_fmt_string; |
| 735 | _ = options; |
| 736 | try writer.print("{d}.{d}.{d}", .{ v.major, v.minor, v.patch }); |
| 737 | } |
| 738 | }; |
Alex Rønne Petersen <alex@alexrp.com> 2024-12-13 04:41:58+01:00
test