| author |  Jakub Konka <kubkon@jakubkonka.com> 2024-08-27 12:53:05+02:00 |
| committer | Jakub Konka <kubkon@jakubkonka.com> 2024-09-04 13:34:25+02:00 |
| log | b44dd599ad4a0f8a6684acd6e3c5a02f3c7b8f04 |
| tree | 8b6df85ad68443e229429d30ef9dd3f24484d3cf |
| parent | f87dd43c1285d38d7a0f3092f6487bf1e1f4faa6 |
elf: split Atom.allocate into Atom-independent parts
3 files changed, 136 insertions(+), 141 deletions(-)
src
link
src/link/Elf.zig+77
| ... | @@ -675,6 +675,83 @@ pub fn markDirty(self: *Elf, shdr_index: u32) void { | ... | @@ -675,6 +675,83 @@ pub fn markDirty(self: *Elf, shdr_index: u32) void { |
| 675 | } | 675 | } |
| 676 | } | 676 | } |
| 677 | 677 | ||
| 678 | const AllocateChunkResult = struct { | ||
| 679 | value: u64, | ||
| 680 | placement: Ref, | ||
| 681 | }; | ||
| 682 | |||
| 683 | pub fn allocateChunk(self: *Elf, shndx: u32, size: u64, alignment: Atom.Alignment) !AllocateChunkResult { | ||
| 684 | const slice = self.sections.slice(); | ||
| 685 | const shdr = &slice.items(.shdr)[shndx]; | ||
| 686 | const free_list = &slice.items(.free_list)[shndx]; | ||
| 687 | const last_atom_ref = &slice.items(.last_atom)[shndx]; | ||
| 688 | const new_atom_ideal_capacity = padToIdeal(size); | ||
| 689 | |||
| 690 | // First we look for an appropriately sized free list node. | ||
| 691 | // The list is unordered. We'll just take the first thing that works. | ||
| 692 | const res: AllocateChunkResult = blk: { | ||
| 693 | var i: usize = if (self.base.child_pid == null) 0 else free_list.items.len; | ||
| 694 | while (i < free_list.items.len) { | ||
| 695 | const big_atom_ref = free_list.items[i]; | ||
| 696 | const big_atom = self.atom(big_atom_ref).?; | ||
| 697 | // We now have a pointer to a live atom that has too much capacity. | ||
| 698 | // Is it enough that we could fit this new atom? | ||
| 699 | const cap = big_atom.capacity(self); | ||
| 700 | const ideal_capacity = padToIdeal(cap); | ||
| 701 | const ideal_capacity_end_vaddr = std.math.add(u64, @intCast(big_atom.value), ideal_capacity) catch ideal_capacity; | ||
| 702 | const capacity_end_vaddr = @as(u64, @intCast(big_atom.value)) + cap; | ||
| 703 | const new_start_vaddr_unaligned = capacity_end_vaddr - new_atom_ideal_capacity; | ||
| 704 | const new_start_vaddr = alignment.backward(new_start_vaddr_unaligned); | ||
| 705 | if (new_start_vaddr < ideal_capacity_end_vaddr) { | ||
| 706 | // Additional bookkeeping here to notice if this free list node | ||
| 707 | // should be deleted because the block that it points to has grown to take up | ||
| 708 | // more of the extra capacity. | ||
| 709 | if (!big_atom.freeListEligible(self)) { | ||
| 710 | _ = free_list.swapRemove(i); | ||
| 711 | } else { | ||
| 712 | i += 1; | ||
| 713 | } | ||
| 714 | continue; | ||
| 715 | } | ||
| 716 | // At this point we know that we will place the new block here. But the | ||
| 717 | // remaining question is whether there is still yet enough capacity left | ||
| 718 | // over for there to still be a free list node. | ||
| 719 | const remaining_capacity = new_start_vaddr - ideal_capacity_end_vaddr; | ||
| 720 | const keep_free_list_node = remaining_capacity >= min_text_capacity; | ||
| 721 | |||
| 722 | if (!keep_free_list_node) { | ||
| 723 | _ = free_list.swapRemove(i); | ||
| 724 | } | ||
| 725 | break :blk .{ .value = new_start_vaddr, .placement = big_atom_ref }; | ||
| 726 | } else if (self.atom(last_atom_ref.*)) |last_atom| { | ||
| 727 | const ideal_capacity = padToIdeal(last_atom.size); | ||
| 728 | const ideal_capacity_end_vaddr = @as(u64, @intCast(last_atom.value)) + ideal_capacity; | ||
| 729 | const new_start_vaddr = alignment.forward(ideal_capacity_end_vaddr); | ||
| 730 | break :blk .{ .value = new_start_vaddr, .placement = last_atom.ref() }; | ||
| 731 | } else { | ||
| 732 | break :blk .{ .value = 0, .placement = .{} }; | ||
| 733 | } | ||
| 734 | }; | ||
| 735 | |||
| 736 | log.debug("allocated chunk (size({x}),align({x})) at 0x{x} (file(0x{x}))", .{ | ||
| 737 | size, | ||
| 738 | alignment.toByteUnits().?, | ||
| 739 | shdr.sh_addr + res.value, | ||
| 740 | shdr.sh_offset + res.value, | ||
| 741 | }); | ||
| 742 | |||
| 743 | const expand_section = if (self.atom(res.placement)) |placement_atom| | ||
| 744 | placement_atom.nextAtom(self) == null | ||
| 745 | else | ||
| 746 | true; | ||
| 747 | if (expand_section) { | ||
| 748 | const needed_size = res.value + size; | ||
| 749 | try self.growAllocSection(shndx, needed_size); | ||
| 750 | } | ||
| 751 | |||
| 752 | return res; | ||
| 753 | } | ||
| 754 | |||
| 678 | pub fn flush(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) link.File.FlushError!void { | 755 | pub fn flush(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) link.File.FlushError!void { |
| 679 | const use_lld = build_options.have_llvm and self.base.comp.config.use_lld; | 756 | const use_lld = build_options.have_llvm and self.base.comp.config.use_lld; |
| 680 | if (use_lld) { | 757 | if (use_lld) { |
src/link/Elf/Atom.zig-134
| ... | @@ -123,140 +123,6 @@ pub fn freeListEligible(self: Atom, elf_file: *Elf) bool { | ... | @@ -123,140 +123,6 @@ pub fn freeListEligible(self: Atom, elf_file: *Elf) bool { |
| 123 | return surplus >= Elf.min_text_capacity; | 123 | return surplus >= Elf.min_text_capacity; |
| 124 | } | 124 | } |
| 125 | 125 | ||
| 126 | pub fn allocate(self: *Atom, elf_file: *Elf) !void { | ||
| 127 | const slice = elf_file.sections.slice(); | ||
| 128 | const shdr = &slice.items(.shdr)[self.output_section_index]; | ||
| 129 | const free_list = &slice.items(.free_list)[self.output_section_index]; | ||
| 130 | const last_atom_ref = &slice.items(.last_atom)[self.output_section_index]; | ||
| 131 | const new_atom_ideal_capacity = Elf.padToIdeal(self.size); | ||
| 132 | |||
| 133 | // We use these to indicate our intention to update metadata, placing the new atom, | ||
| 134 | // and possibly removing a free list node. | ||
| 135 | // It would be simpler to do it inside the for loop below, but that would cause a | ||
| 136 | // problem if an error was returned later in the function. So this action | ||
| 137 | // is actually carried out at the end of the function, when errors are no longer possible. | ||
| 138 | var atom_placement: ?Elf.Ref = null; | ||
| 139 | var free_list_removal: ?usize = null; | ||
| 140 | |||
| 141 | // First we look for an appropriately sized free list node. | ||
| 142 | // The list is unordered. We'll just take the first thing that works. | ||
| 143 | self.value = blk: { | ||
| 144 | var i: usize = if (elf_file.base.child_pid == null) 0 else free_list.items.len; | ||
| 145 | while (i < free_list.items.len) { | ||
| 146 | const big_atom_ref = free_list.items[i]; | ||
| 147 | const big_atom = elf_file.atom(big_atom_ref).?; | ||
| 148 | // We now have a pointer to a live atom that has too much capacity. | ||
| 149 | // Is it enough that we could fit this new atom? | ||
| 150 | const cap = big_atom.capacity(elf_file); | ||
| 151 | const ideal_capacity = Elf.padToIdeal(cap); | ||
| 152 | const ideal_capacity_end_vaddr = std.math.add(u64, @intCast(big_atom.value), ideal_capacity) catch ideal_capacity; | ||
| 153 | const capacity_end_vaddr = @as(u64, @intCast(big_atom.value)) + cap; | ||
| 154 | const new_start_vaddr_unaligned = capacity_end_vaddr - new_atom_ideal_capacity; | ||
| 155 | const new_start_vaddr = self.alignment.backward(new_start_vaddr_unaligned); | ||
| 156 | if (new_start_vaddr < ideal_capacity_end_vaddr) { | ||
| 157 | // Additional bookkeeping here to notice if this free list node | ||
| 158 | // should be deleted because the block that it points to has grown to take up | ||
| 159 | // more of the extra capacity. | ||
| 160 | if (!big_atom.freeListEligible(elf_file)) { | ||
| 161 | _ = free_list.swapRemove(i); | ||
| 162 | } else { | ||
| 163 | i += 1; | ||
| 164 | } | ||
| 165 | continue; | ||
| 166 | } | ||
| 167 | // At this point we know that we will place the new block here. But the | ||
| 168 | // remaining question is whether there is still yet enough capacity left | ||
| 169 | // over for there to still be a free list node. | ||
| 170 | const remaining_capacity = new_start_vaddr - ideal_capacity_end_vaddr; | ||
| 171 | const keep_free_list_node = remaining_capacity >= Elf.min_text_capacity; | ||
| 172 | |||
| 173 | // Set up the metadata to be updated, after errors are no longer possible. | ||
| 174 | atom_placement = big_atom_ref; | ||
| 175 | if (!keep_free_list_node) { | ||
| 176 | free_list_removal = i; | ||
| 177 | } | ||
| 178 | break :blk @intCast(new_start_vaddr); | ||
| 179 | } else if (elf_file.atom(last_atom_ref.*)) |last_atom| { | ||
| 180 | const ideal_capacity = Elf.padToIdeal(last_atom.size); | ||
| 181 | const ideal_capacity_end_vaddr = @as(u64, @intCast(last_atom.value)) + ideal_capacity; | ||
| 182 | const new_start_vaddr = self.alignment.forward(ideal_capacity_end_vaddr); | ||
| 183 | // Set up the metadata to be updated, after errors are no longer possible. | ||
| 184 | atom_placement = last_atom.ref(); | ||
| 185 | break :blk @intCast(new_start_vaddr); | ||
| 186 | } else { | ||
| 187 | break :blk 0; | ||
| 188 | } | ||
| 189 | }; | ||
| 190 | |||
| 191 | log.debug("allocated atom({}) : '{s}' at 0x{x} to 0x{x}", .{ | ||
| 192 | self.ref(), | ||
| 193 | self.name(elf_file), | ||
| 194 | self.address(elf_file), | ||
| 195 | self.address(elf_file) + @as(i64, @intCast(self.size)), | ||
| 196 | }); | ||
| 197 | |||
| 198 | const expand_section = if (atom_placement) |placement_ref| | ||
| 199 | elf_file.atom(placement_ref).?.nextAtom(elf_file) == null | ||
| 200 | else | ||
| 201 | true; | ||
| 202 | if (expand_section) { | ||
| 203 | const needed_size: u64 = @intCast(self.value + @as(i64, @intCast(self.size))); | ||
| 204 | try elf_file.growAllocSection(self.output_section_index, needed_size); | ||
| 205 | last_atom_ref.* = self.ref(); | ||
| 206 | |||
| 207 | switch (self.file(elf_file).?) { | ||
| 208 | .zig_object => |zo| if (zo.dwarf) |_| { | ||
| 209 | // The .debug_info section has `low_pc` and `high_pc` values which is the virtual address | ||
| 210 | // range of the compilation unit. When we expand the text section, this range changes, | ||
| 211 | // so the DW_TAG.compile_unit tag of the .debug_info section becomes dirty. | ||
| 212 | zo.debug_info_section_dirty = true; | ||
| 213 | // This becomes dirty for the same reason. We could potentially make this more | ||
| 214 | // fine-grained with the addition of support for more compilation units. It is planned to | ||
| 215 | // model each package as a different compilation unit. | ||
| 216 | zo.debug_aranges_section_dirty = true; | ||
| 217 | zo.debug_rnglists_section_dirty = true; | ||
| 218 | }, | ||
| 219 | else => {}, | ||
| 220 | } | ||
| 221 | } | ||
| 222 | shdr.sh_addralign = @max(shdr.sh_addralign, self.alignment.toByteUnits().?); | ||
| 223 | |||
| 224 | // This function can also reallocate an atom. | ||
| 225 | // In this case we need to "unplug" it from its previous location before | ||
| 226 | // plugging it in to its new location. | ||
| 227 | if (self.prevAtom(elf_file)) |prev| { | ||
| 228 | prev.next_atom_ref = self.next_atom_ref; | ||
| 229 | } | ||
| 230 | if (self.nextAtom(elf_file)) |next| { | ||
| 231 | next.prev_atom_ref = self.prev_atom_ref; | ||
| 232 | } | ||
| 233 | |||
| 234 | if (atom_placement) |big_atom_ref| { | ||
| 235 | const big_atom = elf_file.atom(big_atom_ref).?; | ||
| 236 | self.prev_atom_ref = big_atom_ref; | ||
| 237 | self.next_atom_ref = big_atom.next_atom_ref; | ||
| 238 | big_atom.next_atom_ref = self.ref(); | ||
| 239 | } else { | ||
| 240 | self.prev_atom_ref = .{ .index = 0, .file = 0 }; | ||
| 241 | self.next_atom_ref = .{ .index = 0, .file = 0 }; | ||
| 242 | } | ||
| 243 | if (free_list_removal) |i| { | ||
| 244 | _ = free_list.swapRemove(i); | ||
| 245 | } | ||
| 246 | |||
| 247 | self.alive = true; | ||
| 248 | } | ||
| 249 | |||
| 250 | pub fn shrink(self: *Atom, elf_file: *Elf) void { | ||
| 251 | _ = self; | ||
| 252 | _ = elf_file; | ||
| 253 | } | ||
| 254 | |||
| 255 | pub fn grow(self: *Atom, elf_file: *Elf) !void { | ||
| 256 | if (!self.alignment.check(@intCast(self.value)) or self.size > self.capacity(elf_file)) | ||
| 257 | try self.allocate(elf_file); | ||
| 258 | } | ||
| 259 | |||
| 260 | pub fn free(self: *Atom, elf_file: *Elf) void { | 126 | pub fn free(self: *Atom, elf_file: *Elf) void { |
| 261 | log.debug("freeAtom atom({}) ({s})", .{ self.ref(), self.name(elf_file) }); | 127 | log.debug("freeAtom atom({}) ({s})", .{ self.ref(), self.name(elf_file) }); |
| 262 | 128 |
src/link/Elf/ZigObject.zig+59-7
| ... | @@ -1362,19 +1362,18 @@ fn updateNavCode( | ... | @@ -1362,19 +1362,18 @@ fn updateNavCode( |
| 1362 | const capacity = atom_ptr.capacity(elf_file); | 1362 | const capacity = atom_ptr.capacity(elf_file); |
| 1363 | const need_realloc = code.len > capacity or !required_alignment.check(@intCast(atom_ptr.value)); | 1363 | const need_realloc = code.len > capacity or !required_alignment.check(@intCast(atom_ptr.value)); |
| 1364 | if (need_realloc) { | 1364 | if (need_realloc) { |
| 1365 | try atom_ptr.grow(elf_file); | 1365 | try self.growAtom(atom_ptr, elf_file); |
| 1366 | log.debug("growing {} from 0x{x} to 0x{x}", .{ nav.fqn.fmt(ip), old_vaddr, atom_ptr.value }); | 1366 | log.debug("growing {} from 0x{x} to 0x{x}", .{ nav.fqn.fmt(ip), old_vaddr, atom_ptr.value }); |
| 1367 | if (old_vaddr != atom_ptr.value) { | 1367 | if (old_vaddr != atom_ptr.value) { |
| 1368 | sym.value = 0; | 1368 | sym.value = 0; |
| 1369 | esym.st_value = 0; | 1369 | esym.st_value = 0; |
| 1370 | } | 1370 | } |
| 1371 | } else if (code.len < old_size) { | 1371 | } else if (code.len < old_size) { |
| 1372 | atom_ptr.shrink(elf_file); | 1372 | // TODO shrink section size |
| 1373 | } | 1373 | } |
| 1374 | } else { | 1374 | } else { |
| 1375 | try atom_ptr.allocate(elf_file); | 1375 | try self.allocateAtom(atom_ptr, elf_file); |
| 1376 | errdefer self.freeNavMetadata(elf_file, sym_index); | 1376 | errdefer self.freeNavMetadata(elf_file, sym_index); |
| 1377 | |||
| 1378 | sym.value = 0; | 1377 | sym.value = 0; |
| 1379 | esym.st_value = 0; | 1378 | esym.st_value = 0; |
| 1380 | } | 1379 | } |
| ... | @@ -1739,7 +1738,7 @@ fn updateLazySymbol( | ... | @@ -1739,7 +1738,7 @@ fn updateLazySymbol( |
| 1739 | atom_ptr.size = code.len; | 1738 | atom_ptr.size = code.len; |
| 1740 | atom_ptr.output_section_index = output_section_index; | 1739 | atom_ptr.output_section_index = output_section_index; |
| 1741 | 1740 | ||
| 1742 | try atom_ptr.allocate(elf_file); | 1741 | try self.allocateAtom(atom_ptr, elf_file); |
| 1743 | errdefer self.freeNavMetadata(elf_file, symbol_index); | 1742 | errdefer self.freeNavMetadata(elf_file, symbol_index); |
| 1744 | 1743 | ||
| 1745 | local_sym.value = 0; | 1744 | local_sym.value = 0; |
| ... | @@ -1797,8 +1796,7 @@ fn lowerConst( | ... | @@ -1797,8 +1796,7 @@ fn lowerConst( |
| 1797 | atom_ptr.size = code.len; | 1796 | atom_ptr.size = code.len; |
| 1798 | atom_ptr.output_section_index = output_section_index; | 1797 | atom_ptr.output_section_index = output_section_index; |
| 1799 | 1798 | ||
| 1800 | try atom_ptr.allocate(elf_file); | 1799 | try self.allocateAtom(atom_ptr, elf_file); |
| 1801 | // TODO rename and re-audit this method | ||
| 1802 | errdefer self.freeNavMetadata(elf_file, sym_index); | 1800 | errdefer self.freeNavMetadata(elf_file, sym_index); |
| 1803 | 1801 | ||
| 1804 | const shdr = elf_file.sections.items(.shdr)[output_section_index]; | 1802 | const shdr = elf_file.sections.items(.shdr)[output_section_index]; |
| ... | @@ -1998,6 +1996,60 @@ fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void { | ... | @@ -1998,6 +1996,60 @@ fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void { |
| 1998 | } | 1996 | } |
| 1999 | } | 1997 | } |
| 2000 | 1998 | ||
| 1999 | fn allocateAtom(self: *ZigObject, atom_ptr: *Atom, elf_file: *Elf) !void { | ||
| 2000 | const alloc_res = try elf_file.allocateChunk(atom_ptr.output_section_index, atom_ptr.size, atom_ptr.alignment); | ||
| 2001 | atom_ptr.value = @intCast(alloc_res.value); | ||
| 2002 | |||
| 2003 | const slice = elf_file.sections.slice(); | ||
| 2004 | const shdr = &slice.items(.shdr)[atom_ptr.output_section_index]; | ||
| 2005 | const last_atom_ref = &slice.items(.last_atom)[atom_ptr.output_section_index]; | ||
| 2006 | |||
| 2007 | const expand_section = if (elf_file.atom(alloc_res.placement)) |placement_atom| | ||
| 2008 | placement_atom.nextAtom(elf_file) == null | ||
| 2009 | else | ||
| 2010 | true; | ||
| 2011 | if (expand_section) { | ||
| 2012 | last_atom_ref.* = atom_ptr.ref(); | ||
| 2013 | if (self.dwarf) |_| { | ||
| 2014 | // The .debug_info section has `low_pc` and `high_pc` values which is the virtual address | ||
| 2015 | // range of the compilation unit. When we expand the text section, this range changes, | ||
| 2016 | // so the DW_TAG.compile_unit tag of the .debug_info section becomes dirty. | ||
| 2017 | self.debug_info_section_dirty = true; | ||
| 2018 | // This becomes dirty for the same reason. We could potentially make this more | ||
| 2019 | // fine-grained with the addition of support for more compilation units. It is planned to | ||
| 2020 | // model each package as a different compilation unit. | ||
| 2021 | self.debug_aranges_section_dirty = true; | ||
| 2022 | self.debug_rnglists_section_dirty = true; | ||
| 2023 | } | ||
| 2024 | } | ||
| 2025 | shdr.sh_addralign = @max(shdr.sh_addralign, atom_ptr.alignment.toByteUnits().?); | ||
| 2026 | |||
| 2027 | // This function can also reallocate an atom. | ||
| 2028 | // In this case we need to "unplug" it from its previous location before | ||
| 2029 | // plugging it in to its new location. | ||
| 2030 | if (atom_ptr.prevAtom(elf_file)) |prev| { | ||
| 2031 | prev.next_atom_ref = atom_ptr.next_atom_ref; | ||
| 2032 | } | ||
| 2033 | if (atom_ptr.nextAtom(elf_file)) |next| { | ||
| 2034 | next.prev_atom_ref = atom_ptr.prev_atom_ref; | ||
| 2035 | } | ||
| 2036 | |||
| 2037 | if (elf_file.atom(alloc_res.placement)) |big_atom| { | ||
| 2038 | atom_ptr.prev_atom_ref = alloc_res.placement; | ||
| 2039 | atom_ptr.next_atom_ref = big_atom.next_atom_ref; | ||
| 2040 | big_atom.next_atom_ref = atom_ptr.ref(); | ||
| 2041 | } else { | ||
| 2042 | atom_ptr.prev_atom_ref = .{ .index = 0, .file = 0 }; | ||
| 2043 | atom_ptr.next_atom_ref = .{ .index = 0, .file = 0 }; | ||
| 2044 | } | ||
| 2045 | } | ||
| 2046 | |||
| 2047 | fn growAtom(self: *ZigObject, atom_ptr: *Atom, elf_file: *Elf) !void { | ||
| 2048 | if (!atom_ptr.alignment.check(@intCast(atom_ptr.value)) or atom_ptr.size > atom_ptr.capacity(elf_file)) { | ||
| 2049 | try self.allocateAtom(atom_ptr, elf_file); | ||
| 2050 | } | ||
| 2051 | } | ||
| 2052 | |||
| 2001 | pub fn asFile(self: *ZigObject) File { | 2053 | pub fn asFile(self: *ZigObject) File { |
| 2002 | return .{ .zig_object = self }; | 2054 | return .{ .zig_object = self }; |
| 2003 | } | 2055 | } |