authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-02 22:15:42-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-03-02 22:15:42-05:00
log00a8742bbf71b5175b52ecdf736fb4a53ab8177f
treedd54188333e02876d57a6f0a8ecff33d8782b0e7
parent0367d684fccf8bf011fe8ac1a984820c824871a8
parentd741be512b5c13bec77bde5cc0fd70fbd3535bc0
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #22982 from mlugg/cache-mode

compiler: default to `.whole` cache mode for self-hosted backends

6 files changed, 93 insertions(+), 63 deletions(-)

src/Compilation.zig+12-5
...@@ -2397,7 +2397,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2397,7 +2397,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
23972397
2398 // Work around windows `AccessDenied` if any files within this2398 // Work around windows `AccessDenied` if any files within this
2399 // directory are open by closing and reopening the file handles.2399 // directory are open by closing and reopening the file handles.
2400 const need_writable_dance = w: {2400 const need_writable_dance: enum { no, lf_only, lf_and_debug } = w: {
2401 if (builtin.os.tag == .windows) {2401 if (builtin.os.tag == .windows) {
2402 if (comp.bin_file) |lf| {2402 if (comp.bin_file) |lf| {
2403 // We cannot just call `makeExecutable` as it makes a false2403 // We cannot just call `makeExecutable` as it makes a false
...@@ -2410,11 +2410,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2410,11 +2410,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2410 if (lf.file) |f| {2410 if (lf.file) |f| {
2411 f.close();2411 f.close();
2412 lf.file = null;2412 lf.file = null;
2413 break :w true;2413
2414 if (lf.closeDebugInfo()) break :w .lf_and_debug;
2415 break :w .lf_only;
2414 }2416 }
2415 }2417 }
2416 }2418 }
2417 break :w false;2419 break :w .no;
2418 };2420 };
24192421
2420 renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path) catch |err| {2422 renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path) catch |err| {
...@@ -2441,8 +2443,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2441,8 +2443,13 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2441 };2443 };
24422444
2443 // Has to be after the `wholeCacheModeSetBinFilePath` above.2445 // Has to be after the `wholeCacheModeSetBinFilePath` above.
2444 if (need_writable_dance) {2446 switch (need_writable_dance) {
2445 try lf.makeWritable();2447 .no => {},
2448 .lf_only => try lf.makeWritable(),
2449 .lf_and_debug => {
2450 try lf.makeWritable();
2451 try lf.reopenDebugInfo();
2452 },
2446 }2453 }
2447 }2454 }
24482455
src/link.zig+14
...@@ -600,6 +600,20 @@ pub const File = struct {...@@ -600,6 +600,20 @@ pub const File = struct {
600 }600 }
601 }601 }
602602
603 /// Some linkers create a separate file for debug info, which we might need to temporarily close
604 /// when moving the compilation result directory due to the host OS not allowing moving a
605 /// file/directory while a handle remains open.
606 /// Returns `true` if a debug info file was closed. In that case, `reopenDebugInfo` may be called.
607 pub fn closeDebugInfo(base: *File) bool {
608 const macho = base.cast(.macho) orelse return false;
609 return macho.closeDebugInfo();
610 }
611
612 pub fn reopenDebugInfo(base: *File) !void {
613 const macho = base.cast(.macho).?;
614 return macho.reopenDebugInfo();
615 }
616
603 pub fn makeExecutable(base: *File) !void {617 pub fn makeExecutable(base: *File) !void {
604 dev.check(.make_executable);618 dev.check(.make_executable);
605 const comp = base.comp;619 const comp = base.comp;
src/link/MachO.zig+33-19
...@@ -3277,6 +3277,37 @@ const InitMetadataOptions = struct {...@@ -3277,6 +3277,37 @@ const InitMetadataOptions = struct {
3277 program_code_size_hint: u64,3277 program_code_size_hint: u64,
3278};3278};
32793279
3280pub fn closeDebugInfo(self: *MachO) bool {
3281 const d_sym = &(self.d_sym orelse return false);
3282 d_sym.file.?.close();
3283 d_sym.file = null;
3284 return true;
3285}
3286
3287pub fn reopenDebugInfo(self: *MachO) !void {
3288 assert(self.d_sym.?.file == null);
3289
3290 assert(!self.base.comp.config.use_llvm);
3291 assert(self.base.comp.config.debug_format == .dwarf);
3292
3293 const gpa = self.base.comp.gpa;
3294 const sep = fs.path.sep_str;
3295 const d_sym_path = try std.fmt.allocPrint(
3296 gpa,
3297 "{s}.dSYM" ++ sep ++ "Contents" ++ sep ++ "Resources" ++ sep ++ "DWARF",
3298 .{self.base.emit.sub_path},
3299 );
3300 defer gpa.free(d_sym_path);
3301
3302 var d_sym_bundle = try self.base.emit.root_dir.handle.makeOpenPath(d_sym_path, .{});
3303 defer d_sym_bundle.close();
3304
3305 self.d_sym.?.file = try d_sym_bundle.createFile(fs.path.basename(self.base.emit.sub_path), .{
3306 .truncate = false,
3307 .read = true,
3308 });
3309}
3310
3280// TODO: move to ZigObject3311// TODO: move to ZigObject
3281fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {3312fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {
3282 if (!self.base.isRelocatable()) {3313 if (!self.base.isRelocatable()) {
...@@ -3333,25 +3364,8 @@ fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {...@@ -3333,25 +3364,8 @@ fn initMetadata(self: *MachO, options: InitMetadataOptions) !void {
3333 if (options.zo.dwarf) |*dwarf| {3364 if (options.zo.dwarf) |*dwarf| {
3334 // Create dSYM bundle.3365 // Create dSYM bundle.
3335 log.debug("creating {s}.dSYM bundle", .{options.emit.sub_path});3366 log.debug("creating {s}.dSYM bundle", .{options.emit.sub_path});
33363367 self.d_sym = .{ .allocator = self.base.comp.gpa, .file = null };
3337 const gpa = self.base.comp.gpa;3368 try self.reopenDebugInfo();
3338 const sep = fs.path.sep_str;
3339 const d_sym_path = try std.fmt.allocPrint(
3340 gpa,
3341 "{s}.dSYM" ++ sep ++ "Contents" ++ sep ++ "Resources" ++ sep ++ "DWARF",
3342 .{options.emit.sub_path},
3343 );
3344 defer gpa.free(d_sym_path);
3345
3346 var d_sym_bundle = try options.emit.root_dir.handle.makeOpenPath(d_sym_path, .{});
3347 defer d_sym_bundle.close();
3348
3349 const d_sym_file = try d_sym_bundle.createFile(options.emit.sub_path, .{
3350 .truncate = false,
3351 .read = true,
3352 });
3353
3354 self.d_sym = .{ .allocator = gpa, .file = d_sym_file };
3355 try self.d_sym.?.initMetadata(self);3369 try self.d_sym.?.initMetadata(self);
3356 try dwarf.initMetadata();3370 try dwarf.initMetadata();
3357 }3371 }
src/link/MachO/DebugSymbols.zig+11-11
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1allocator: Allocator,1allocator: Allocator,
2file: fs.File,2file: ?fs.File,
33
4symtab_cmd: macho.symtab_command = .{},4symtab_cmd: macho.symtab_command = .{},
5uuid_cmd: macho.uuid_command = .{ .uuid = [_]u8{0} ** 16 },5uuid_cmd: macho.uuid_command = .{ .uuid = [_]u8{0} ** 16 },
...@@ -118,9 +118,9 @@ pub fn growSection(...@@ -118,9 +118,9 @@ pub fn growSection(
118 });118 });
119119
120 if (requires_file_copy) {120 if (requires_file_copy) {
121 const amt = try self.file.copyRangeAll(121 const amt = try self.file.?.copyRangeAll(
122 sect.offset,122 sect.offset,
123 self.file,123 self.file.?,
124 new_offset,124 new_offset,
125 existing_size,125 existing_size,
126 );126 );
...@@ -129,7 +129,7 @@ pub fn growSection(...@@ -129,7 +129,7 @@ pub fn growSection(
129129
130 sect.offset = @intCast(new_offset);130 sect.offset = @intCast(new_offset);
131 } else if (sect.offset + allocated_size == std.math.maxInt(u64)) {131 } else if (sect.offset + allocated_size == std.math.maxInt(u64)) {
132 try self.file.setEndPos(sect.offset + needed_size);132 try self.file.?.setEndPos(sect.offset + needed_size);
133 }133 }
134134
135 sect.size = needed_size;135 sect.size = needed_size;
...@@ -165,7 +165,7 @@ fn detectAllocCollision(self: *DebugSymbols, start: u64, size: u64) !?u64 {...@@ -165,7 +165,7 @@ fn detectAllocCollision(self: *DebugSymbols, start: u64, size: u64) !?u64 {
165 }165 }
166 }166 }
167167
168 if (at_end) try self.file.setEndPos(end);168 if (at_end) try self.file.?.setEndPos(end);
169 return null;169 return null;
170}170}
171171
...@@ -195,7 +195,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {...@@ -195,7 +195,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {
195 sym_name,195 sym_name,
196 file_offset,196 file_offset,
197 });197 });
198 try self.file.pwriteAll(mem.asBytes(&addr), file_offset);198 try self.file.?.pwriteAll(mem.asBytes(&addr), file_offset);
199 }199 }
200200
201 self.finalizeDwarfSegment(macho_file);201 self.finalizeDwarfSegment(macho_file);
...@@ -208,7 +208,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {...@@ -208,7 +208,7 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void {
208208
209pub fn deinit(self: *DebugSymbols) void {209pub fn deinit(self: *DebugSymbols) void {
210 const gpa = self.allocator;210 const gpa = self.allocator;
211 self.file.close();211 if (self.file) |file| file.close();
212 self.segments.deinit(gpa);212 self.segments.deinit(gpa);
213 self.sections.deinit(gpa);213 self.sections.deinit(gpa);
214 self.relocs.deinit(gpa);214 self.relocs.deinit(gpa);
...@@ -320,7 +320,7 @@ fn writeLoadCommands(self: *DebugSymbols, macho_file: *MachO) !struct { usize, u...@@ -320,7 +320,7 @@ fn writeLoadCommands(self: *DebugSymbols, macho_file: *MachO) !struct { usize, u
320320
321 assert(stream.pos == needed_size);321 assert(stream.pos == needed_size);
322322
323 try self.file.pwriteAll(buffer, @sizeOf(macho.mach_header_64));323 try self.file.?.pwriteAll(buffer, @sizeOf(macho.mach_header_64));
324324
325 return .{ ncmds, buffer.len };325 return .{ ncmds, buffer.len };
326}326}
...@@ -346,7 +346,7 @@ fn writeHeader(self: *DebugSymbols, macho_file: *MachO, ncmds: usize, sizeofcmds...@@ -346,7 +346,7 @@ fn writeHeader(self: *DebugSymbols, macho_file: *MachO, ncmds: usize, sizeofcmds
346346
347 log.debug("writing Mach-O header {}", .{header});347 log.debug("writing Mach-O header {}", .{header});
348348
349 try self.file.pwriteAll(mem.asBytes(&header), 0);349 try self.file.?.pwriteAll(mem.asBytes(&header), 0);
350}350}
351351
352fn allocatedSize(self: *DebugSymbols, start: u64) u64 {352fn allocatedSize(self: *DebugSymbols, start: u64) u64 {
...@@ -404,7 +404,7 @@ pub fn writeSymtab(self: *DebugSymbols, off: u32, macho_file: *MachO) !u32 {...@@ -404,7 +404,7 @@ pub fn writeSymtab(self: *DebugSymbols, off: u32, macho_file: *MachO) !u32 {
404 internal.writeSymtab(macho_file, self);404 internal.writeSymtab(macho_file, self);
405 }405 }
406406
407 try self.file.pwriteAll(mem.sliceAsBytes(self.symtab.items), cmd.symoff);407 try self.file.?.pwriteAll(mem.sliceAsBytes(self.symtab.items), cmd.symoff);
408408
409 return off + cmd.nsyms * @sizeOf(macho.nlist_64);409 return off + cmd.nsyms * @sizeOf(macho.nlist_64);
410}410}
...@@ -412,7 +412,7 @@ pub fn writeSymtab(self: *DebugSymbols, off: u32, macho_file: *MachO) !u32 {...@@ -412,7 +412,7 @@ pub fn writeSymtab(self: *DebugSymbols, off: u32, macho_file: *MachO) !u32 {
412pub fn writeStrtab(self: *DebugSymbols, off: u32) !u32 {412pub fn writeStrtab(self: *DebugSymbols, off: u32) !u32 {
413 const cmd = &self.symtab_cmd;413 const cmd = &self.symtab_cmd;
414 cmd.stroff = off;414 cmd.stroff = off;
415 try self.file.pwriteAll(self.strtab.items, cmd.stroff);415 try self.file.?.pwriteAll(self.strtab.items, cmd.stroff);
416 return off + cmd.strsize;416 return off + cmd.strsize;
417}417}
418418
src/link/SpirV.zig+18-27
...@@ -61,6 +61,18 @@ pub fn createEmpty(...@@ -61,6 +61,18 @@ pub fn createEmpty(
61 const gpa = comp.gpa;61 const gpa = comp.gpa;
62 const target = comp.root_mod.resolved_target.result;62 const target = comp.root_mod.resolved_target.result;
6363
64 assert(!comp.config.use_lld); // Caught by Compilation.Config.resolve
65 assert(!comp.config.use_llvm); // Caught by Compilation.Config.resolve
66 assert(target.ofmt == .spirv); // Caught by Compilation.Config.resolve
67 switch (target.cpu.arch) {
68 .spirv, .spirv32, .spirv64 => {},
69 else => unreachable, // Caught by Compilation.Config.resolve.
70 }
71 switch (target.os.tag) {
72 .opencl, .opengl, .vulkan => {},
73 else => unreachable, // Caught by Compilation.Config.resolve.
74 }
75
64 const self = try arena.create(SpirV);76 const self = try arena.create(SpirV);
65 self.* = .{77 self.* = .{
66 .base = .{78 .base = .{
...@@ -79,15 +91,11 @@ pub fn createEmpty(...@@ -79,15 +91,11 @@ pub fn createEmpty(
79 };91 };
80 errdefer self.deinit();92 errdefer self.deinit();
8193
82 switch (target.cpu.arch) {94 // TODO: read the file and keep valid parts instead of truncating
83 .spirv, .spirv32, .spirv64 => {},95 self.base.file = try emit.root_dir.handle.createFile(emit.sub_path, .{
84 else => unreachable, // Caught by Compilation.Config.resolve.96 .truncate = true,
85 }97 .read = true,
8698 });
87 switch (target.os.tag) {
88 .opencl, .opengl, .vulkan => {},
89 else => unreachable, // Caught by Compilation.Config.resolve.
90 }
9199
92 return self;100 return self;
93}101}
...@@ -98,24 +106,7 @@ pub fn open(...@@ -98,24 +106,7 @@ pub fn open(
98 emit: Path,106 emit: Path,
99 options: link.File.OpenOptions,107 options: link.File.OpenOptions,
100) !*SpirV {108) !*SpirV {
101 const target = comp.root_mod.resolved_target.result;109 return createEmpty(arena, comp, emit, options);
102 const use_lld = build_options.have_llvm and comp.config.use_lld;
103 const use_llvm = comp.config.use_llvm;
104
105 assert(!use_llvm); // Caught by Compilation.Config.resolve.
106 assert(!use_lld); // Caught by Compilation.Config.resolve.
107 assert(target.ofmt == .spirv); // Caught by Compilation.Config.resolve.
108
109 const spirv = try createEmpty(arena, comp, emit, options);
110 errdefer spirv.base.destroy();
111
112 // TODO: read the file and keep valid parts instead of truncating
113 const file = try emit.root_dir.handle.createFile(emit.sub_path, .{
114 .truncate = true,
115 .read = true,
116 });
117 spirv.base.file = file;
118 return spirv;
119}110}
120111
121pub fn deinit(self: *SpirV) void {112pub fn deinit(self: *SpirV) void {
src/main.zig+5-1
...@@ -3470,7 +3470,11 @@ fn buildOutputType(...@@ -3470,7 +3470,11 @@ fn buildOutputType(
3470 // incremental cache mode is used for LLVM backend too.3470 // incremental cache mode is used for LLVM backend too.
3471 if (create_module.resolved_options.use_llvm) break :b .whole;3471 if (create_module.resolved_options.use_llvm) break :b .whole;
34723472
3473 break :b .incremental;3473 // Eventually, this default should be `.incremental`. However, since incremental
3474 // compilation is currently an opt-in feature, it makes a strictly worse default cache mode
3475 // than `.whole`.
3476 // https://github.com/ziglang/zig/issues/21165
3477 break :b .whole;
3474 };3478 };
34753479
3476 process.raiseFileDescriptorLimit();3480 process.raiseFileDescriptorLimit();