authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-15 17:42:36+01:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-02-16 08:38:28+01:00
log5283a52af565521094bcd1186efc179b4be7ee8c
tree28f00732496d6204284ff0631a0b0f9f3dd55784
parent0b22b6b70e012e5881cdbef3c2b2b1fef56c24b3

macho: handle binary updates in dSYM companion file

* move DWARF in file if LINKEDIT spilled in dSYM * update VM addresses for all segments * introduce copyFileRangeOverlappedAll instead of File.copyRangeAll since we make lots of overlapping writes in MachO linker

2 files changed, 185 insertions(+), 37 deletions(-)

src/link/MachO.zig+27-13
...@@ -5082,22 +5082,18 @@ fn growSegment(self: *MachO, seg_id: u16, new_size: u64) !void {...@@ -5082,22 +5082,18 @@ fn growSegment(self: *MachO, seg_id: u16, new_size: u64) !void {
5082 seg.inner.vmaddr + seg.inner.vmsize,5082 seg.inner.vmaddr + seg.inner.vmsize,
5083 });5083 });
50845084
5085 // TODO We should probably nop the expanded by distance, or put 0s.
5086
5087 // TODO copyRangeAll doesn't automatically extend the file on macOS.
5088 const ledit_seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
5089 const new_filesize = offset_amt + ledit_seg.inner.fileoff + ledit_seg.inner.filesize;
5090 try self.base.file.?.pwriteAll(&[_]u8{0}, new_filesize - 1);
5091
5092 var next: usize = seg_id + 1;5085 var next: usize = seg_id + 1;
5093 while (next < self.linkedit_segment_cmd_index.? + 1) : (next += 1) {5086 while (next < self.linkedit_segment_cmd_index.? + 1) : (next += 1) {
5094 const next_seg = &self.load_commands.items[next].segment;5087 const next_seg = &self.load_commands.items[next].segment;
5095 _ = try self.base.file.?.copyRangeAll(5088
5096 next_seg.inner.fileoff,5089 try MachO.copyRangeAllOverlappingAlloc(
5090 self.base.allocator,
5097 self.base.file.?,5091 self.base.file.?,
5092 next_seg.inner.fileoff,
5098 next_seg.inner.fileoff + offset_amt,5093 next_seg.inner.fileoff + offset_amt,
5099 next_seg.inner.filesize,5094 try math.cast(usize, next_seg.inner.filesize),
5100 );5095 );
5096
5101 next_seg.inner.fileoff += offset_amt;5097 next_seg.inner.fileoff += offset_amt;
5102 next_seg.inner.vmaddr += offset_amt;5098 next_seg.inner.vmaddr += offset_amt;
51035099
...@@ -5174,11 +5170,13 @@ fn growSection(self: *MachO, match: MatchingSection, new_size: u32) !void {...@@ -5174,11 +5170,13 @@ fn growSection(self: *MachO, match: MatchingSection, new_size: u32) !void {
5174 // the required amount and update their header offsets.5170 // the required amount and update their header offsets.
5175 const next_sect = seg.sections.items[match.sect + 1];5171 const next_sect = seg.sections.items[match.sect + 1];
5176 const total_size = last_sect_off - next_sect.offset;5172 const total_size = last_sect_off - next_sect.offset;
5177 _ = try self.base.file.?.copyRangeAll(5173
5178 next_sect.offset,5174 try MachO.copyRangeAllOverlappingAlloc(
5175 self.base.allocator,
5179 self.base.file.?,5176 self.base.file.?,
5177 next_sect.offset,
5180 next_sect.offset + offset_amt,5178 next_sect.offset + offset_amt,
5181 total_size,5179 try math.cast(usize, total_size),
5182 );5180 );
51835181
5184 var next = match.sect + 1;5182 var next = match.sect + 1;
...@@ -6738,3 +6736,19 @@ fn logSectionOrdinals(self: MachO) void {...@@ -6738,3 +6736,19 @@ fn logSectionOrdinals(self: MachO) void {
6738 });6736 });
6739 }6737 }
6740}6738}
6739
6740/// Since `os.copy_file_range` cannot be used when copying overlapping ranges within the same file,
6741/// and since `File.copyRangeAll` uses `os.copy_file_range` under-the-hood, we use heap allocated
6742/// buffers on all hosts except Linux (if `copy_file_range` syscall is available).
6743pub fn copyRangeAllOverlappingAlloc(
6744 allocator: Allocator,
6745 file: std.fs.File,
6746 in_offset: u64,
6747 out_offset: u64,
6748 len: usize,
6749) !void {
6750 const buf = try allocator.alloc(u8, len);
6751 defer allocator.free(buf);
6752 _ = try file.preadAll(buf, in_offset);
6753 try file.pwriteAll(buf, out_offset);
6754}
src/link/MachO/DebugSymbols.zig+158-24
...@@ -634,8 +634,9 @@ pub fn flushModule(self: *DebugSymbols, allocator: Allocator, options: link.Opti...@@ -634,8 +634,9 @@ pub fn flushModule(self: *DebugSymbols, allocator: Allocator, options: link.Opti
634 }634 }
635 }635 }
636636
637 try self.writeLinkeditSegment();
638 self.updateDwarfSegment();637 self.updateDwarfSegment();
638 try self.writeLinkeditSegment();
639 try self.updateVirtualMemoryMapping();
639 try self.writeLoadCommands(allocator);640 try self.writeLoadCommands(allocator);
640 try self.writeHeader();641 try self.writeHeader();
641642
...@@ -706,10 +707,25 @@ fn copySegmentCommand(...@@ -706,10 +707,25 @@ fn copySegmentCommand(
706707
707fn updateDwarfSegment(self: *DebugSymbols) void {708fn updateDwarfSegment(self: *DebugSymbols) void {
708 const dwarf_segment = &self.load_commands.items[self.dwarf_segment_cmd_index.?].segment;709 const dwarf_segment = &self.load_commands.items[self.dwarf_segment_cmd_index.?].segment;
709 var file_size: u64 = 0;710
711 var max_offset: u64 = 0;
710 for (dwarf_segment.sections.items) |sect| {712 for (dwarf_segment.sections.items) |sect| {
711 file_size += sect.size;713 log.debug(" {s},{s} - 0x{x}-0x{x} - 0x{x}-0x{x}", .{
714 sect.segName(),
715 sect.sectName(),
716 sect.offset,
717 sect.offset + sect.size,
718 sect.addr,
719 sect.addr + sect.size,
720 });
721 if (sect.offset + sect.size > max_offset) {
722 max_offset = sect.offset + sect.size;
723 }
712 }724 }
725
726 const file_size = max_offset - dwarf_segment.inner.fileoff;
727 log.debug("__DWARF size 0x{x}", .{file_size});
728
713 if (file_size != dwarf_segment.inner.filesize) {729 if (file_size != dwarf_segment.inner.filesize) {
714 dwarf_segment.inner.filesize = file_size;730 dwarf_segment.inner.filesize = file_size;
715 if (dwarf_segment.inner.vmsize < dwarf_segment.inner.filesize) {731 if (dwarf_segment.inner.vmsize < dwarf_segment.inner.filesize) {
...@@ -780,15 +796,59 @@ fn allocatedSize(self: *DebugSymbols, start: u64) u64 {...@@ -780,15 +796,59 @@ fn allocatedSize(self: *DebugSymbols, start: u64) u64 {
780 return min_pos - start;796 return min_pos - start;
781}797}
782798
799fn updateVirtualMemoryMapping(self: *DebugSymbols) !void {
800 const macho_file = self.base;
801 const allocator = macho_file.base.allocator;
802
803 const IndexTuple = std.meta.Tuple(&[_]type{ *?u16, *?u16 });
804 const indices = &[_]IndexTuple{
805 .{ &macho_file.text_segment_cmd_index, &self.text_segment_cmd_index },
806 .{ &macho_file.data_const_segment_cmd_index, &self.data_const_segment_cmd_index },
807 .{ &macho_file.data_segment_cmd_index, &self.data_segment_cmd_index },
808 };
809
810 for (indices) |tuple| {
811 const orig_cmd = macho_file.load_commands.items[tuple[0].*.?].segment;
812 const cmd = try self.copySegmentCommand(allocator, orig_cmd);
813 const comp_cmd = &self.load_commands.items[tuple[1].*.?];
814 comp_cmd.deinit(allocator);
815 self.load_commands.items[tuple[1].*.?] = .{ .segment = cmd };
816 }
817
818 // TODO should we set the linkedit vmsize to that of the binary?
819 const orig_cmd = macho_file.load_commands.items[macho_file.linkedit_segment_cmd_index.?].segment;
820 const orig_vmaddr = orig_cmd.inner.vmaddr;
821 const linkedit_cmd = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
822 linkedit_cmd.inner.vmaddr = orig_vmaddr;
823
824 // Update VM address for the DWARF segment and sections including re-running relocations.
825 // TODO re-run relocations
826 const dwarf_cmd = &self.load_commands.items[self.dwarf_segment_cmd_index.?].segment;
827 const new_start_aligned = orig_vmaddr + linkedit_cmd.inner.vmsize;
828 const old_start_aligned = dwarf_cmd.inner.vmaddr;
829 const diff = new_start_aligned - old_start_aligned;
830 if (diff > 0) {
831 dwarf_cmd.inner.vmaddr = new_start_aligned;
832
833 for (dwarf_cmd.sections.items) |*sect| {
834 sect.addr += (new_start_aligned - old_start_aligned);
835 }
836 }
837
838 self.load_commands_dirty = true;
839}
840
783fn writeLinkeditSegment(self: *DebugSymbols) !void {841fn writeLinkeditSegment(self: *DebugSymbols) !void {
784 const tracy = trace(@src());842 const tracy = trace(@src());
785 defer tracy.end();843 defer tracy.end();
786844
787 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
788 seg.inner.filesize = 0;
789
790 try self.writeSymbolTable();845 try self.writeSymbolTable();
791 try self.writeStringTable();846 try self.writeStringTable();
847
848 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
849 const aligned_size = mem.alignForwardGeneric(u64, seg.inner.filesize, self.base.page_size);
850 seg.inner.filesize = aligned_size;
851 seg.inner.vmsize = aligned_size;
792}852}
793853
794fn writeSymbolTable(self: *DebugSymbols) !void {854fn writeSymbolTable(self: *DebugSymbols) !void {
...@@ -797,7 +857,7 @@ fn writeSymbolTable(self: *DebugSymbols) !void {...@@ -797,7 +857,7 @@ fn writeSymbolTable(self: *DebugSymbols) !void {
797857
798 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;858 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
799 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].symtab;859 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].symtab;
800 symtab.symoff = @intCast(u32, seg.inner.fileoff + seg.inner.filesize);860 symtab.symoff = @intCast(u32, seg.inner.fileoff);
801861
802 var locals = std.ArrayList(macho.nlist_64).init(self.base.base.allocator);862 var locals = std.ArrayList(macho.nlist_64).init(self.base.base.allocator);
803 defer locals.deinit();863 defer locals.deinit();
...@@ -810,20 +870,52 @@ fn writeSymbolTable(self: *DebugSymbols) !void {...@@ -810,20 +870,52 @@ fn writeSymbolTable(self: *DebugSymbols) !void {
810870
811 const nlocals = locals.items.len;871 const nlocals = locals.items.len;
812 const nexports = self.base.globals.items.len;872 const nexports = self.base.globals.items.len;
813
814 const locals_off = symtab.symoff;873 const locals_off = symtab.symoff;
815 const locals_size = nlocals * @sizeOf(macho.nlist_64);874 const locals_size = nlocals * @sizeOf(macho.nlist_64);
875 const exports_off = locals_off + locals_size;
876 const exports_size = nexports * @sizeOf(macho.nlist_64);
877
878 symtab.nsyms = @intCast(u32, nlocals + nexports);
879 const needed_size = (nlocals + nexports) * @sizeOf(macho.nlist_64);
880
881 if (needed_size > seg.inner.filesize) {
882 const aligned_size = mem.alignForwardGeneric(u64, needed_size, self.base.page_size);
883 const diff = @intCast(u32, aligned_size - seg.inner.filesize);
884 const dwarf_seg = &self.load_commands.items[self.dwarf_segment_cmd_index.?].segment;
885 seg.inner.filesize = aligned_size;
886
887 try MachO.copyRangeAllOverlappingAlloc(
888 self.base.base.allocator,
889 self.file,
890 dwarf_seg.inner.fileoff,
891 dwarf_seg.inner.fileoff + diff,
892 try math.cast(usize, dwarf_seg.inner.filesize),
893 );
894
895 const old_seg_fileoff = dwarf_seg.inner.fileoff;
896 dwarf_seg.inner.fileoff += diff;
897
898 log.debug(" (moving __DWARF segment from 0x{x} to 0x{x})", .{ old_seg_fileoff, dwarf_seg.inner.fileoff });
899
900 for (dwarf_seg.sections.items) |*sect| {
901 const old_offset = sect.offset;
902 sect.offset += diff;
903
904 log.debug(" (moving {s},{s} from 0x{x} to 0x{x})", .{
905 sect.segName(),
906 sect.sectName(),
907 old_offset,
908 sect.offset,
909 });
910 }
911 }
912
816 log.debug("writing local symbols from 0x{x} to 0x{x}", .{ locals_off, locals_size + locals_off });913 log.debug("writing local symbols from 0x{x} to 0x{x}", .{ locals_off, locals_size + locals_off });
817 try self.file.pwriteAll(mem.sliceAsBytes(locals.items), locals_off);914 try self.file.pwriteAll(mem.sliceAsBytes(locals.items), locals_off);
818915
819 const exports_off = locals_off + locals_size;
820 const exports_size = nexports * @sizeOf(macho.nlist_64);
821 log.debug("writing exported symbols from 0x{x} to 0x{x}", .{ exports_off, exports_size + exports_off });916 log.debug("writing exported symbols from 0x{x} to 0x{x}", .{ exports_off, exports_size + exports_off });
822 try self.file.pwriteAll(mem.sliceAsBytes(self.base.globals.items), exports_off);917 try self.file.pwriteAll(mem.sliceAsBytes(self.base.globals.items), exports_off);
823918
824 symtab.nsyms = @intCast(u32, nlocals + nexports);
825 seg.inner.filesize += locals_size + exports_size;
826
827 self.load_commands_dirty = true;919 self.load_commands_dirty = true;
828}920}
829921
...@@ -833,18 +925,48 @@ fn writeStringTable(self: *DebugSymbols) !void {...@@ -833,18 +925,48 @@ fn writeStringTable(self: *DebugSymbols) !void {
833925
834 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;926 const seg = &self.load_commands.items[self.linkedit_segment_cmd_index.?].segment;
835 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].symtab;927 const symtab = &self.load_commands.items[self.symtab_cmd_index.?].symtab;
836 symtab.stroff = @intCast(u32, seg.inner.fileoff + seg.inner.filesize);928 const symtab_size = @intCast(u32, symtab.nsyms * @sizeOf(macho.nlist_64));
837 symtab.strsize = @intCast(u32, mem.alignForwardGeneric(u64, self.base.strtab.items.len, @alignOf(u64)));929 symtab.stroff = symtab.symoff + symtab_size;
838 seg.inner.filesize += symtab.strsize;930
931 const needed_size = mem.alignForwardGeneric(u64, self.base.strtab.items.len, @alignOf(u64));
932 symtab.strsize = @intCast(u32, needed_size);
933
934 if (symtab_size + needed_size > seg.inner.filesize) {
935 const aligned_size = mem.alignForwardGeneric(u64, symtab_size + needed_size, self.base.page_size);
936 const diff = @intCast(u32, aligned_size - seg.inner.filesize);
937 const dwarf_seg = &self.load_commands.items[self.dwarf_segment_cmd_index.?].segment;
938 seg.inner.filesize = aligned_size;
939
940 try MachO.copyRangeAllOverlappingAlloc(
941 self.base.base.allocator,
942 self.file,
943 dwarf_seg.inner.fileoff,
944 dwarf_seg.inner.fileoff + diff,
945 try math.cast(usize, dwarf_seg.inner.filesize),
946 );
947
948 const old_seg_fileoff = dwarf_seg.inner.fileoff;
949 dwarf_seg.inner.fileoff += diff;
950
951 log.debug(" (moving __DWARF segment from 0x{x} to 0x{x})", .{ old_seg_fileoff, dwarf_seg.inner.fileoff });
952
953 for (dwarf_seg.sections.items) |*sect| {
954 const old_offset = sect.offset;
955 sect.offset += diff;
956
957 log.debug(" (moving {s},{s} from 0x{x} to 0x{x})", .{
958 sect.segName(),
959 sect.sectName(),
960 old_offset,
961 sect.offset,
962 });
963 }
964 }
839965
840 log.debug("writing string table from 0x{x} to 0x{x}", .{ symtab.stroff, symtab.stroff + symtab.strsize });966 log.debug("writing string table from 0x{x} to 0x{x}", .{ symtab.stroff, symtab.stroff + symtab.strsize });
841967
842 try self.file.pwriteAll(self.base.strtab.items, symtab.stroff);968 try self.file.pwriteAll(self.base.strtab.items, symtab.stroff);
843969
844 if (symtab.strsize > self.base.strtab.items.len) {
845 // This is potentially the last section, so we need to pad it out.
846 try self.file.pwriteAll(&[_]u8{0}, seg.inner.fileoff + seg.inner.filesize - 1);
847 }
848 self.load_commands_dirty = true;970 self.load_commands_dirty = true;
849}971}
850972
...@@ -1088,8 +1210,14 @@ pub fn commitDeclDebugInfo(...@@ -1088,8 +1210,14 @@ pub fn commitDeclDebugInfo(
1088 new_offset,1210 new_offset,
1089 });1211 });
10901212
1091 const amt = try self.file.copyRangeAll(debug_line_sect.offset, self.file, new_offset, existing_size);1213 try MachO.copyRangeAllOverlappingAlloc(
1092 if (amt != existing_size) return error.InputOutput;1214 self.base.base.allocator,
1215 self.file,
1216 debug_line_sect.offset,
1217 new_offset,
1218 existing_size,
1219 );
1220
1093 debug_line_sect.offset = @intCast(u32, new_offset);1221 debug_line_sect.offset = @intCast(u32, new_offset);
1094 debug_line_sect.addr = dwarf_segment.inner.vmaddr + new_offset - dwarf_segment.inner.fileoff;1222 debug_line_sect.addr = dwarf_segment.inner.vmaddr + new_offset - dwarf_segment.inner.fileoff;
1095 }1223 }
...@@ -1390,8 +1518,14 @@ fn writeDeclDebugInfo(self: *DebugSymbols, text_block: *TextBlock, dbg_info_buf:...@@ -1390,8 +1518,14 @@ fn writeDeclDebugInfo(self: *DebugSymbols, text_block: *TextBlock, dbg_info_buf:
1390 new_offset,1518 new_offset,
1391 });1519 });
13921520
1393 const amt = try self.file.copyRangeAll(debug_info_sect.offset, self.file, new_offset, existing_size);1521 try MachO.copyRangeAllOverlappingAlloc(
1394 if (amt != existing_size) return error.InputOutput;1522 self.base.base.allocator,
1523 self.file,
1524 debug_info_sect.offset,
1525 new_offset,
1526 existing_size,
1527 );
1528
1395 debug_info_sect.offset = @intCast(u32, new_offset);1529 debug_info_sect.offset = @intCast(u32, new_offset);
1396 debug_info_sect.addr = dwarf_segment.inner.vmaddr + new_offset - dwarf_segment.inner.fileoff;1530 debug_info_sect.addr = dwarf_segment.inner.vmaddr + new_offset - dwarf_segment.inner.fileoff;
1397 }1531 }