From a6ab86acd92b81dcdfa3f8192690de7a26fa13d7 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 29 Aug 2026 11:30:58 +0100 Subject: [PATCH] MappedFile: handle footers more efficiently A previous commit reworked how `Elf2` implements archives to make the main ZCU object file a footer of the root node. In theory this is fairly efficient, and this held in practice when `FALLOCATE_FL_INSERT_RANGE` was available, but when it wasn't (e.g. certain filesystems, non-Linux host), the behavior of growing a footer was very inefficient. Previously, to grow a footer, we would grow the parent (thereby moving all of its footers forwards in the file) to ensure free space before the footers, and then move the footer backwards to insert space where we need it. This meant that in the `Elf2` case, every single resize of the `.elf` node was effectively guaranteed to perform two `@memmove`s of the node's entire content. This is *extremely* inefficient, because the only operation which is actually necessary to grow the last footer node on the file itself is increasing the file size! I could have special-cased the situation `Elf2` finds itself in (a footer node which is the last child of the root), but instead, I opted to spend time on something a bit more generally applicable. We now have a new strategy for resizing footer nodes, where we ask the parent node to resize itself *without* moving its footers forwards (so the padding is inserted after its footers instead of between its floating nodes and footers), and move only any footers which actually need to move (any footers which follow the growing node, plus any nested footers). This strategy is not always possible, and it also has the downside of being incapable of reclaiming free space in the parent node, so in some cases the old strategy is still chosen. I tested this commit by disabling the `FALLOCATE_FL_INSERT_RANGE` code path and running the `MappedFile` fuzz test. Also, to verify that I had actually improved efficiency, I tried using `Elf2` to build some static libraries (the situation in which it uses footer nodes) and counted how many bytes' worth of `moveRange` occured before and after this patch. It seems that across the duration of such a compilation, the total number of bytes given to `moveRange` has decreased by around a factor of 10: for instance, the number when building compiler-rt (in debug mode) went from around 70.3 MB to 6.9 MB. --- src/link/MappedFile.zig | 622 ++++++++++++++++++++++++++-------------- 1 file changed, 405 insertions(+), 217 deletions(-) diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index 0dfb6527f3f6768db0ba24e791ef560178d263b9..937b0989a8d40dabcf74ea8c920ac47a2a5d1f67 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -648,7 +648,10 @@ pub const Node = extern struct { _, const current_size = ni.location(mf).resolve(mf); if (current_size >= min_size) return; const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); - try mf.growNode(gpa, ni, new_size, .minimum); + try mf.growNode(gpa, ni, new_size, .{ + .exact_size = false, + .move_footers = true, + }); mf.updateWriters(); } @@ -664,7 +667,10 @@ pub const Node = extern struct { switch (std.math.order(size, old_size)) { .lt => try mf.shrinkLeafNode(gpa, ni, size), .eq => {}, // `old_size` must be well-aligned, so `size` is too - .gt => try mf.growNode(gpa, ni, size, .exact), + .gt => try mf.growNode(gpa, ni, size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }), } mf.updateWriters(); } @@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { try mf.realignNode(gpa, new_ni, opts.add_options.alignment); if (opts.add_options.size > 0) { - try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); + try mf.growNode(gpa, new_ni, opts.add_options.size, .{ + .exact_size = true, + .move_footers = false, // irrelevant, since we have no footers + }); } mf.updateWriters(); @@ -1070,12 +1079,21 @@ fn shrinkLeafNode( } } -const GrowMode = enum { exact, minimum }; +const GrowOptions = struct { + /// If `true`, the node size must be set to exactly the given size. + /// + /// If `false`, the given size is a minimum, and the actual new node size may be larger. + exact_size: bool, + /// If `true`, footers within the resized node will be moved forwards to its new end. + /// + /// If `false`, footers will all remain at their current offsets (so the nodes are in a + /// temporarily invalid state), and moving them is the responsibility of the *caller*. + move_footers: bool, +}; -/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. -/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// Increases the size of a node. /// -/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`. /// /// Asserts that `new_size` is greater than the current size of `ni`. fn growNode( @@ -1083,7 +1101,7 @@ fn growNode( gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1098,7 +1116,7 @@ fn growNode( const parent_ni = node.parent.unwrap() orelse { assert(ni == .root); - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1120,21 +1138,23 @@ fn growNode( }; try mf.ensureTotalCapacityPrecise(@intCast(new_size)); try ni.setLocation(mf, gpa, old_offset, new_size); - // We need to move any footers to be at the *new* end of the file. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = old_size - old_footers_offset; - try mf.moveRange( - old_footers_offset, - old_footers_offset + (new_size - old_size), - footers_size, - ); - // Also update the footers' locations. - var cur_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } } return; @@ -1142,7 +1162,7 @@ fn growNode( switch (node.flags.position) { .header => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } @@ -1163,7 +1183,13 @@ fn growNode( const old_headers_size = last_header_offset + last_header_size; // This is the first footer *inside* of `ni`. - const first_sub_footer_oni = ni.firstFooter(mf); + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); + }; const sub_footers_size = size: { const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); @@ -1215,92 +1241,264 @@ fn growNode( return; }, .floating => { - try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options); }, .footer => { - if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) { return; } - try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); - - const first_footer_ni: Node.Index = first_footer: { - var footer_ni = ni; - while (true) { - const prev_ni = footer_ni.prev(mf).unwrap() orelse break; - if (prev_ni.position(mf) != .footer) break; - footer_ni = prev_ni; - } - break :first_footer footer_ni; - }; - // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself - // a footer within its parent). - const first_sub_footer_oni = ni.firstFooter(mf); - const sub_footers_size = size: { - const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; - const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); - break :size old_size - first_sub_footer_offset; + // a footer within its parent). We'll need this later in any case, so just find it now. + const first_sub_footer_oni: Node.Index.Optional = footer: { + if (!grow_options.move_footers) { + // Pretend there are no nested footers so as to not move them. + break :footer .none; + } + break :footer ni.firstFooter(mf); }; - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; - const new_footers_size = old_footers_size - old_size + new_size; - - // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node - // itself needs to shift its contents, because our offset was shifted backwards by - // `new_size - old_size`, and the added bytes should go at the end of this footer node. - // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so - // we *shouldn't* shift *that* data. - const old_footers_start = parent_size - old_footers_size; - const new_footers_start = parent_size - new_footers_size; - const end_offset = node.location().resolve(mf)[0] + old_size; - const parent_file_offset = parent_ni.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_footers_start, - parent_file_offset + new_footers_start, - end_offset - old_footers_start - sub_footers_size, - ); - - // Update our own offset and size: - try ni.setLocation(mf, gpa, end_offset - new_size, new_size); - - // Any footers inside of us have had their offsets changed due to us growing: - if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { - var cur_ni = first_sub_footer_ni; - while (true) { - const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation( + // We have two different strategies for growing a footer node, with different advantages + // and disadvantages; so first we must decide which to use. + const strat: union(enum) { + /// Expand into pre-footer padding space in the parent node (growing the parent if + /// necessary). This strategy has the benefit that it can reclaim padding bytes in + /// the parent, but it has the disadvantage that it requires moving this node's + /// existing content backwards in the file, which may be expensive (particularly + /// since the src and dest ranges are likely to overlap). + grow_backwards, + + /// Grow the parent node with `GrowOptions.move_footers` set to `false`, and + /// implicitly grow ourselves into the newly available space. This usually requires + /// a lot less moving of bytes, but never reclaims unused space before the parent's + /// footers, and is sometimes straight-up impossible. + grow_parent_at_end: struct { + add_size: u64, + exact_size: bool, + }, + } = strat: { + // If this node is small, the move overhead is trivial, so prefer `.grow_backwards` + // to avoid unnecessary growth of the parent node. + if (old_size <= mf.flags.block_size.toByteUnits() * 2) { + break :strat .grow_backwards; + } + + // It may also be worth doing `.grow_backwards` if the parent has a *lot* of space + // we could grow into. More specifically, if "free space we can grow into" makes up + // a significant proportion of the parent's total size, then that implies the parent + // has quite poor utilization of space, *and* that we can significantly improve that + // statistic by growing into that space. + if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) { + break :strat .grow_backwards; + } + + if (grow_options.exact_size) { + const add_size = new_size - old_size; + if (parent_ni.alignment(mf).check(add_size)) { + break :strat .{ .grow_parent_at_end = .{ + .add_size = add_size, + .exact_size = true, + } }; + } else { + // We *can't* ask the parent to grow by this much, so we have no choice. + break :strat .grow_backwards; + } + } + + if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) { + // Because the parent's alignment is less than our own, if we gave them the + // freedom to pick a size, they might choose one which results in *us* having a + // size incompatible with our alignment. Therefore, to prevent that, we need to + // request an *exact* size from the parent in this case. + break :strat .{ .grow_parent_at_end = .{ + .add_size = new_size - old_size, + .exact_size = true, + } }; + } + + // The parent's alignment is greater than or equal to our own, so we only need to + // give the parent a *minimum* size (although we need to ensure it matches their + // alignment since it could be greater than our own). + break :strat .{ .grow_parent_at_end = .{ + .add_size = parent_ni.alignment(mf).forward(new_size - old_size), + .exact_size = false, + } }; + }; + + switch (strat) { + .grow_backwards => { + // First, we might need to grow the parent to make enough space. + { + const available_size = mf.availableFooterCapacity(parent_ni); + if (old_size + available_size < new_size) { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const min_parent_size = old_parent_size + (new_size - old_size - available_size); + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); + assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size); + } + } + + // Now we need to grow! To do that, we must move `ni` itself, and every footer + // before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in + // the shift, because the bytes we're adding need to go at the *end* of `ni` + // rather than its start. + + // This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to + // start at `ni` than to start at `parent_ni.last(mf)`. + const first_parent_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + const shift = new_size - old_size; + + // Update our own offset and size: + try ni.setLocation( mf, gpa, - old_sub_footer_offset + (new_size - old_size), - sub_footer_size, + node.location().resolve(mf)[0] - shift, + new_size, ); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - } - // Finally, update the offsets of every footer before us: - if (node.prev.unwrap()) |prev_ni| { - var maybe_footer_ni = prev_ni; - while (true) { - switch (maybe_footer_ni.position(mf)) { - .header, .floating => break, - .footer => {}, + // Any footers *inside* of `ni` have had their offsets changed, because they are + // now positioned at the *new* end of `ni`: + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size); + } + } + + // Any footers *before* `ni` (in `parent_ni`) have been shifted backwards. We'll + // also be moving their actual bytes in a moment, so track whether they have + // content (if nothing does then we'll be able to skip the `moveRange`). That + // flag is initially whether `ni` has content because we're shifting our own + // bytes backwards too. + var moved_has_content: bool = node.flags.has_content; + { + var footer_ni = first_parent_footer_ni; + while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) { + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one thing containing initialized bytes, so we need to + // move the actual data. However, we should *not* move the bytes of any + // nested footers inside of `ni`, because they've been "moved" to the end + // of our new size, which is the same file location as before. + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size new_size - first_sub_footer_offset; + }; + const new_offset: u64, _ = node.location().resolve(mf); + const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf); + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + new_footers_offset + shift, + parent_file_offset + new_footers_offset, + (new_offset - new_footers_offset) + // accounts for every footer before `ni` + (old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers + ); } - const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); - try maybe_footer_ni.setLocation( + }, + .grow_parent_at_end => |grow_parent| { + _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf); + try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{ + .exact_size = grow_parent.exact_size, + .move_footers = false, + }); + _, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf); + const shift = new_parent_size - old_parent_size; + + // Here's what we have left to do: + // + // * Increase our own size by `shift` to absorb the added space. + // + // * If there are any footers *inside* `ni`, increase their offsets by `shift`. + // + // * If there are any footers *after* `ni` (inside `parent_ni`), increase their + // offsets by `shift`. + // + // * Do a `moveRange` corresponding to those offset changes. This is a single + // range which starts at the footers *inside* `ni`. + + const actual_new_size = old_size + shift; + if (grow_options.exact_size) { + assert(actual_new_size == new_size); + } + + try ni.setLocation( mf, gpa, - moved_footer_offset + old_size - new_size, - moved_footer_size, + node.location().resolve(mf)[0], + actual_new_size, ); - maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; - } - } - return; + // This will track whether any node with a changed offset actually contains + // initialized bytes. If not, there'll be no need to call `moveRange`. + var moved_has_content: bool = false; + + // Set any nested footers' offsets (and include them in `moved_has_content`). + { + var footer_oni = first_sub_footer_oni; + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + // Now set offsets for footers after `ni` inside of `parent_ni`. + { + var footer_oni = ni.next(mf); + while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) { + assert(footer_ni.position(mf) == .footer); + moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content; + const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size); + } + } + + if (moved_has_content) { + // We moved at least one footer containing initialized bytes, so we need to + // move the actual data. Compute how big the footers inside `ni` are... + const sub_footers_size: u64 = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + // `actual_new_size` is used here since we already updated the nested footers' offsets above. + break :size actual_new_size - first_sub_footer_offset; + }; + // ...and how big the footers *after* `ni`, inside `parent_ni`, are... + const post_footers_size: u64 = old_parent_size - (old_offset + old_size); + // ...and move them both. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const total_move_size = sub_footers_size + post_footers_size; + assert(total_move_size != 0); + try mf.moveRange( + parent_file_off + old_parent_size - total_move_size, + parent_file_off + new_parent_size - total_move_size, + total_move_size, + ); + } + }, + } }, } } @@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment( ni: Node.Index, new_alignment: ?Alignment, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment( } // Great, we can grow this node without changing its offset or moving any siblings. try ni.setLocation(mf, gpa, old_offset, new_size); - // If we have any footers, we need to move them to the end of our new size, and update their - // offsets accordingly. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var cur_ni = first_footer_ni; - var footers_have_content = false; - while (true) { - footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; - const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); - try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); - cur_ni = cur_ni.next(mf).unwrap() orelse break; - } - if (footers_have_content) { - const parent_file_off = parent_ni.fileLocation(mf, false).offset; - // This gets the *new* offset because we already updated the offsets above. - const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); - const footers_size = new_size - new_footers_offset; - try mf.moveRange( - parent_file_off + old_offset + old_size - footers_size, - parent_file_off + old_offset + new_size - footers_size, - footers_size, - ); + if (grow_options.move_footers) { + // If we have any footers, we need to move them to the end of our new size, and update + // their offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, + ); + } } } return; @@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment( // that, let's first try the Linux "insert range" fast path. We didn't try it before now // because it would have been more efficient to just move ourselves into existing space. // - // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the // "insert range" path, because that function is unaware of `new_alignment`. - const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + const insert_range_grow_options: GrowOptions = .{ + .exact_size = grow_options.exact_size or new_alignment != null, + .move_footers = grow_options.move_footers, + }; if (alignment.check(old_offset) and - try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options)) { // The Linux fast path did our job for us! return; @@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } break :new_loc .{ @@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment( // Footers need to move to a different place than the rest of our content. const footers_size: u64, const footers_have_content: bool = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ 0, false }; + } const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { break :footers .{ 0, false }; }; @@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment( /// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` /// without changing any nodes' locations or invalidating any slices. /// -/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node -/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, -/// the actual new size of `ni` may be greater than `new_size`. +/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`), +/// updates the location of `ni` and every node whose offset has changed, and returns `true`. fn growNodeViaInsertRange( mf: *MappedFile, gpa: Allocator, ni: Node.Index, new_size: u64, - grow_mode: GrowMode, + grow_options: GrowOptions, ) Error!bool { if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { return false; @@ -1541,24 +1750,22 @@ fn growNodeViaInsertRange( _, const old_size = ni.location(mf).resolve(mf); - // We don't compute the size of the range yet, because depending on `grow_mode` we might want to - // bump it based on our sibling and parent nodes' alignments. However, we can do an early check - // for cases where we should obviously exit. + // We don't compute the size of the range yet, because depending on `grow_options` we might want + // to bump it based on our sibling and parent nodes' alignments. However, we can do an early + // check for cases where we should obviously exit. const min_range_size: u64 = s: { - const exact_size = new_size - old_size; - if (mf.flags.block_size.check(exact_size)) { - break :s exact_size; + const requested_size = new_size - old_size; + if (mf.flags.block_size.check(requested_size)) { + break :s requested_size; } - switch (grow_mode) { - .exact => return false, - .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { - // We're growing by at least a few blocks, so allow ourselves to bump the size - // slightly to give it the needed alignment. - break :s mf.flags.block_size.forward(exact_size); - } else { - return false; - }, + if (!grow_options.exact_size and + requested_size >= mf.flags.block_size.toByteUnits() * 2) + { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(requested_size); } + return false; }; assert(min_range_size > 0); assert(mf.flags.block_size.check(min_range_size)); @@ -1573,14 +1780,17 @@ fn growNodeViaInsertRange( } break :range_file_offset range_file_offset; }; - const first_footer_oni = ni.firstFooter(mf); - const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: { + if (!grow_options.move_footers) { + // Pretend there are no footers so as to not move them. + break :footers .{ .wrap(last_ni), 0 }; + } + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ .wrap(last_ni), 0 }; + }; const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); - break :size old_size - first_footer_offset; - } else 0; - const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { - break :pre_footer first_footer_ni.prev(mf); - } else .wrap(last_ni); + break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset }; + }; const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); break :end pre_footer_off + pre_footer_size; @@ -1634,18 +1844,14 @@ fn growNodeViaInsertRange( break :range_size min_range_size; } // Perhaps we're allowed to grow by more than `min_range_size`? - switch (grow_mode) { - .exact => return false, - .minimum => { - const candidate_range_size = need_range_align.forward(min_range_size); - // Allow growing by up to 50% more than was requested. - if (candidate_range_size <= min_range_size +| min_range_size / 2) { - break :range_size candidate_range_size; - } else { - return false; - } - }, + const candidate_range_size = need_range_align.forward(min_range_size); + if (!grow_options.exact_size and + // Allow growing by up to 50% more than was requested. + candidate_range_size <= min_range_size +| min_range_size / 2) + { + break :range_size candidate_range_size; } + return false; }; // This `range_size` is compatible with everyone's alignment requirements, and we won't move too @@ -1723,13 +1929,15 @@ fn growNodeViaInsertRange( cur_ni = cur_ni.parent(mf).unwrap() orelse break; } - // The only thing left is to update the offsets of any footers inside of `ni`. - if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { - var footer_ni = first_footer_ni; - while (true) { - const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); - try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); - footer_ni = footer_ni.next(mf).unwrap() orelse break; + if (grow_options.move_footers) { + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; + } } } @@ -1783,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } return; }; @@ -1865,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity( const new_parent_size = parent_ni.alignment(mf).forward( min_parent_size +| min_parent_size / growth_factor, ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + try mf.growNode(gpa, parent_ni, new_parent_size, .{ + .exact_size = false, + .move_footers = true, + }); } if (moving_has_content) { @@ -1895,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity( } } -/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current -/// footers, so that the footers can grow into that space. -fn ensureAdditionalFooterCapacity( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - extra_capacity: u64, -) Error!void { - // This is way easier than the header case, because we don't need to actually move anything; we - // just need to expand the parent if there isn't space, and that will add padding after the - // parent's floating children, which is exactly where we want it. - +/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which +/// footers can therefore grow into. +fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 { const first_footer_oni = parent_ni.firstFooter(mf); - _, const parent_size = parent_ni.location(mf).resolve(mf); - - const footers_size: u64 = footers_size: { - const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: { + const first_footer_ni = first_footer_oni.unwrap() orelse { + _, const parent_size = parent_ni.location(mf).resolve(mf); + break :footers .{ parent_ni.last(mf), parent_size }; + }; const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); - break :footers_size parent_size - first_footer_off; + break :footers .{ first_footer_ni.prev(mf), first_footer_off }; }; const header_and_floating_end: u64 = end: { - const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { - break :before_footers first_footer_ni.prev(mf); - } else before_footers: { - break :before_footers parent_ni.last(mf); - }; const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; const offset, const size = before_footers_ni.location(mf).resolve(mf); break :end offset + size; }; - assert(header_and_floating_end + footers_size <= parent_size); - - const min_parent_size = header_and_floating_end + footers_size + extra_capacity; - if (parent_size < min_parent_size) { - const new_parent_size = parent_ni.alignment(mf).forward( - min_parent_size +| min_parent_size / growth_factor, - ); - try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); - } + return footers_off - header_and_floating_end; } fn removeNodesFromChildList( @@ -2030,7 +2223,7 @@ fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: Alignment, + new_align: Alignment, ) Error!void { mf.nodes_lock.assertUnlocked(); @@ -2038,30 +2231,25 @@ fn realignNode( if (ni == .root or ni.position(mf) != .floating) { // Only this node's size is aligned, not its offset. - if (!new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growNode( - gpa, - ni, - new_alignment.forward(old_size), - .exact, // because `growNode` is not aware that the size needs to match `new_alignment` - ); + if (!new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growNode(gpa, ni, new_align.forward(old_size), .{ + .exact_size = true, // because `growNode` is not aware that the size needs to match `new_align` + .move_footers = true, + }); } } else { // This is a floating node, so its size and offset are both aligned. - if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { - assert(new_alignment.compare(.gt, ni.alignment(mf))); - try mf.growFloatingNodeWithAlignment( - gpa, - ni, - new_alignment, - new_alignment.forward(old_size), - .minimum, - ); + if (!new_align.check(old_offset) or !new_align.check(old_size)) { + assert(new_align.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{ + .exact_size = false, + .move_footers = true, + }); } } - ni.get(mf).flags.alignment = new_alignment; + ni.get(mf).flags.alignment = new_align; } fn updateWriters(mf: *MappedFile) void { -- 2.54.0