authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-08-29 11:30:58+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-08-29 13:33:14+01:00
loga6ab86acd92b81dcdfa3f8192690de7a26fa13d7
treeb354d21036b36d07a0bd2644ba39083db0d7e1d2
parent78d44e41ef84f9968072914fe288a57e55c683bd
signaturelock-open Commit is signed but in an unrecognized format.

MappedFile: handle footers more efficiently

A previous commit reworked how `Elf2` implements archives to make the main ZCU object file a footer of the root node. In theory this is fairly efficient, and this held in practice when `FALLOCATE_FL_INSERT_RANGE` was available, but when it wasn't (e.g. certain filesystems, non-Linux host), the behavior of growing a footer was very inefficient. Previously, to grow a footer, we would grow the parent (thereby moving all of its footers forwards in the file) to ensure free space before the footers, and then move the footer backwards to insert space where we need it. This meant that in the `Elf2` case, every single resize of the `.elf` node was effectively guaranteed to perform two `@memmove`s of the node's entire content. This is *extremely* inefficient, because the only operation which is actually necessary to grow the last footer node on the file itself is increasing the file size! I could have special-cased the situation `Elf2` finds itself in (a footer node which is the last child of the root), but instead, I opted to spend time on something a bit more generally applicable. We now have a new strategy for resizing footer nodes, where we ask the parent node to resize itself *without* moving its footers forwards (so the padding is inserted after its footers instead of between its floating nodes and footers), and move only any footers which actually need to move (any footers which follow the growing node, plus any nested footers). This strategy is not always possible, and it also has the downside of being incapable of reclaiming free space in the parent node, so in some cases the old strategy is still chosen. I tested this commit by disabling the `FALLOCATE_FL_INSERT_RANGE` code path and running the `MappedFile` fuzz test. Also, to verify that I had actually improved efficiency, I tried using `Elf2` to build some static libraries (the situation in which it uses footer nodes) and counted how many bytes' worth of `moveRange` occured before and after this patch. It seems that across the duration of such a compilation, the total number of bytes given to `moveRange` has decreased by around a factor of 10: for instance, the number when building compiler-rt (in debug mode) went from around 70.3 MB to 6.9 MB.

1 files changed, 401 insertions(+), 213 deletions(-)

src/link/MappedFile.zig+401-213
...@@ -648,7 +648,10 @@ pub const Node = extern struct {...@@ -648,7 +648,10 @@ pub const Node = extern struct {
648 _, const current_size = ni.location(mf).resolve(mf);648 _, const current_size = ni.location(mf).resolve(mf);
649 if (current_size >= min_size) return;649 if (current_size >= min_size) return;
650 const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor);650 const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor);
651 try mf.growNode(gpa, ni, new_size, .minimum);651 try mf.growNode(gpa, ni, new_size, .{
652 .exact_size = false,
653 .move_footers = true,
654 });
652 mf.updateWriters();655 mf.updateWriters();
653 }656 }
654657
...@@ -664,7 +667,10 @@ pub const Node = extern struct {...@@ -664,7 +667,10 @@ pub const Node = extern struct {
664 switch (std.math.order(size, old_size)) {667 switch (std.math.order(size, old_size)) {
665 .lt => try mf.shrinkLeafNode(gpa, ni, size),668 .lt => try mf.shrinkLeafNode(gpa, ni, size),
666 .eq => {}, // `old_size` must be well-aligned, so `size` is too669 .eq => {}, // `old_size` must be well-aligned, so `size` is too
667 .gt => try mf.growNode(gpa, ni, size, .exact),670 .gt => try mf.growNode(gpa, ni, size, .{
671 .exact_size = true,
672 .move_footers = false, // irrelevant, since we have no footers
673 }),
668 }674 }
669 mf.updateWriters();675 mf.updateWriters();
670 }676 }
...@@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {...@@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {
935941
936 try mf.realignNode(gpa, new_ni, opts.add_options.alignment);942 try mf.realignNode(gpa, new_ni, opts.add_options.alignment);
937 if (opts.add_options.size > 0) {943 if (opts.add_options.size > 0) {
938 try mf.growNode(gpa, new_ni, opts.add_options.size, .exact);944 try mf.growNode(gpa, new_ni, opts.add_options.size, .{
945 .exact_size = true,
946 .move_footers = false, // irrelevant, since we have no footers
947 });
939 }948 }
940 mf.updateWriters();949 mf.updateWriters();
941950
...@@ -1070,12 +1079,21 @@ fn shrinkLeafNode(...@@ -1070,12 +1079,21 @@ fn shrinkLeafNode(
1070 }1079 }
1071}1080}
10721081
1073const GrowMode = enum { exact, minimum };1082const GrowOptions = struct {
1083 /// If `true`, the node size must be set to exactly the given size.
1084 ///
1085 /// If `false`, the given size is a minimum, and the actual new node size may be larger.
1086 exact_size: bool,
1087 /// If `true`, footers within the resized node will be moved forwards to its new end.
1088 ///
1089 /// If `false`, footers will all remain at their current offsets (so the nodes are in a
1090 /// temporarily invalid state), and moving them is the responsibility of the *caller*.
1091 move_footers: bool,
1092};
10741093
1075/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`.1094/// Increases the size of a node.
1076/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`.
1077///1095///
1078/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!).1096/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`.
1079///1097///
1080/// Asserts that `new_size` is greater than the current size of `ni`.1098/// Asserts that `new_size` is greater than the current size of `ni`.
1081fn growNode(1099fn growNode(
...@@ -1083,7 +1101,7 @@ fn growNode(...@@ -1083,7 +1101,7 @@ fn growNode(
1083 gpa: Allocator,1101 gpa: Allocator,
1084 ni: Node.Index,1102 ni: Node.Index,
1085 new_size: u64,1103 new_size: u64,
1086 grow_mode: GrowMode,1104 grow_options: GrowOptions,
1087) Error!void {1105) Error!void {
1088 mf.nodes_lock.assertUnlocked();1106 mf.nodes_lock.assertUnlocked();
10891107
...@@ -1098,7 +1116,7 @@ fn growNode(...@@ -1098,7 +1116,7 @@ fn growNode(
1098 const parent_ni = node.parent.unwrap() orelse {1116 const parent_ni = node.parent.unwrap() orelse {
1099 assert(ni == .root);1117 assert(ni == .root);
11001118
1101 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1119 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1102 return;1120 return;
1103 }1121 }
11041122
...@@ -1120,21 +1138,23 @@ fn growNode(...@@ -1120,21 +1138,23 @@ fn growNode(
1120 };1138 };
1121 try mf.ensureTotalCapacityPrecise(@intCast(new_size));1139 try mf.ensureTotalCapacityPrecise(@intCast(new_size));
1122 try ni.setLocation(mf, gpa, old_offset, new_size);1140 try ni.setLocation(mf, gpa, old_offset, new_size);
1123 // We need to move any footers to be at the *new* end of the file.1141 if (grow_options.move_footers) {
1124 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1142 // We need to move any footers to be at the *new* end of the file.
1125 const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);1143 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1126 const footers_size = old_size - old_footers_offset;1144 const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1127 try mf.moveRange(1145 const footers_size = old_size - old_footers_offset;
1128 old_footers_offset,1146 try mf.moveRange(
1129 old_footers_offset + (new_size - old_size),1147 old_footers_offset,
1130 footers_size,1148 old_footers_offset + (new_size - old_size),
1131 );1149 footers_size,
1132 // Also update the footers' locations.1150 );
1133 var cur_ni = first_footer_ni;1151 // Also update the footers' locations.
1134 while (true) {1152 var cur_ni = first_footer_ni;
1135 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);1153 while (true) {
1136 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);1154 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1137 cur_ni = cur_ni.next(mf).unwrap() orelse break;1155 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1156 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1157 }
1138 }1158 }
1139 }1159 }
1140 return;1160 return;
...@@ -1142,7 +1162,7 @@ fn growNode(...@@ -1142,7 +1162,7 @@ fn growNode(
11421162
1143 switch (node.flags.position) {1163 switch (node.flags.position) {
1144 .header => {1164 .header => {
1145 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1165 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1146 return;1166 return;
1147 }1167 }
11481168
...@@ -1163,7 +1183,13 @@ fn growNode(...@@ -1163,7 +1183,13 @@ fn growNode(
1163 const old_headers_size = last_header_offset + last_header_size;1183 const old_headers_size = last_header_offset + last_header_size;
11641184
1165 // This is the first footer *inside* of `ni`.1185 // This is the first footer *inside* of `ni`.
1166 const first_sub_footer_oni = ni.firstFooter(mf);1186 const first_sub_footer_oni: Node.Index.Optional = footer: {
1187 if (!grow_options.move_footers) {
1188 // Pretend there are no footers so as to not move them.
1189 break :footer .none;
1190 }
1191 break :footer ni.firstFooter(mf);
1192 };
1167 const sub_footers_size = size: {1193 const sub_footers_size = size: {
1168 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;1194 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1169 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);1195 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
...@@ -1215,92 +1241,264 @@ fn growNode(...@@ -1215,92 +1241,264 @@ fn growNode(
1215 return;1241 return;
1216 },1242 },
1217 .floating => {1243 .floating => {
1218 try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode);1244 try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options);
1219 },1245 },
1220 .footer => {1246 .footer => {
1221 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1247 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1222 return;1248 return;
1223 }1249 }
12241250
1225 try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size);1251 // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself
12261252 // a footer within its parent). We'll need this later in any case, so just find it now.
1227 const first_footer_ni: Node.Index = first_footer: {1253 const first_sub_footer_oni: Node.Index.Optional = footer: {
1228 var footer_ni = ni;1254 if (!grow_options.move_footers) {
1229 while (true) {1255 // Pretend there are no nested footers so as to not move them.
1230 const prev_ni = footer_ni.prev(mf).unwrap() orelse break;1256 break :footer .none;
1231 if (prev_ni.position(mf) != .footer) break;
1232 footer_ni = prev_ni;
1233 }1257 }
1234 break :first_footer footer_ni;1258 break :footer ni.firstFooter(mf);
1235 };1259 };
12361260
1237 // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself1261 // We have two different strategies for growing a footer node, with different advantages
1238 // a footer within its parent).1262 // and disadvantages; so first we must decide which to use.
1239 const first_sub_footer_oni = ni.firstFooter(mf);1263 const strat: union(enum) {
1240 const sub_footers_size = size: {1264 /// Expand into pre-footer padding space in the parent node (growing the parent if
1241 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;1265 /// necessary). This strategy has the benefit that it can reclaim padding bytes in
1242 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);1266 /// the parent, but it has the disadvantage that it requires moving this node's
1243 break :size old_size - first_sub_footer_offset;1267 /// existing content backwards in the file, which may be expensive (particularly
1268 /// since the src and dest ranges are likely to overlap).
1269 grow_backwards,
1270
1271 /// Grow the parent node with `GrowOptions.move_footers` set to `false`, and
1272 /// implicitly grow ourselves into the newly available space. This usually requires
1273 /// a lot less moving of bytes, but never reclaims unused space before the parent's
1274 /// footers, and is sometimes straight-up impossible.
1275 grow_parent_at_end: struct {
1276 add_size: u64,
1277 exact_size: bool,
1278 },
1279 } = strat: {
1280 // If this node is small, the move overhead is trivial, so prefer `.grow_backwards`
1281 // to avoid unnecessary growth of the parent node.
1282 if (old_size <= mf.flags.block_size.toByteUnits() * 2) {
1283 break :strat .grow_backwards;
1284 }
1285
1286 // It may also be worth doing `.grow_backwards` if the parent has a *lot* of space
1287 // we could grow into. More specifically, if "free space we can grow into" makes up
1288 // a significant proportion of the parent's total size, then that implies the parent
1289 // has quite poor utilization of space, *and* that we can significantly improve that
1290 // statistic by growing into that space.
1291 if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) {
1292 break :strat .grow_backwards;
1293 }
1294
1295 if (grow_options.exact_size) {
1296 const add_size = new_size - old_size;
1297 if (parent_ni.alignment(mf).check(add_size)) {
1298 break :strat .{ .grow_parent_at_end = .{
1299 .add_size = add_size,
1300 .exact_size = true,
1301 } };
1302 } else {
1303 // We *can't* ask the parent to grow by this much, so we have no choice.
1304 break :strat .grow_backwards;
1305 }
1306 }
1307
1308 if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) {
1309 // Because the parent's alignment is less than our own, if we gave them the
1310 // freedom to pick a size, they might choose one which results in *us* having a
1311 // size incompatible with our alignment. Therefore, to prevent that, we need to
1312 // request an *exact* size from the parent in this case.
1313 break :strat .{ .grow_parent_at_end = .{
1314 .add_size = new_size - old_size,
1315 .exact_size = true,
1316 } };
1317 }
1318
1319 // The parent's alignment is greater than or equal to our own, so we only need to
1320 // give the parent a *minimum* size (although we need to ensure it matches their
1321 // alignment since it could be greater than our own).
1322 break :strat .{ .grow_parent_at_end = .{
1323 .add_size = parent_ni.alignment(mf).forward(new_size - old_size),
1324 .exact_size = false,
1325 } };
1244 };1326 };
12451327
1246 _, const parent_size = parent_ni.location(mf).resolve(mf);1328 switch (strat) {
1329 .grow_backwards => {
1330 // First, we might need to grow the parent to make enough space.
1331 {
1332 const available_size = mf.availableFooterCapacity(parent_ni);
1333 if (old_size + available_size < new_size) {
1334 _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1335 const min_parent_size = old_parent_size + (new_size - old_size - available_size);
1336 const new_parent_size = parent_ni.alignment(mf).forward(
1337 min_parent_size +| min_parent_size / growth_factor,
1338 );
1339 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1340 .exact_size = false,
1341 .move_footers = true,
1342 });
1343 assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size);
1344 }
1345 }
12471346
1248 const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0];1347 // Now we need to grow! To do that, we must move `ni` itself, and every footer
1249 const new_footers_size = old_footers_size - old_size + new_size;1348 // before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in
12501349 // the shift, because the bytes we're adding need to go at the *end* of `ni`
1251 // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node1350 // rather than its start.
1252 // itself needs to shift its contents, because our offset was shifted backwards by1351
1253 // `new_size - old_size`, and the added bytes should go at the end of this footer node.1352 // This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to
1254 // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so1353 // start at `ni` than to start at `parent_ni.last(mf)`.
1255 // we *shouldn't* shift *that* data.1354 const first_parent_footer_ni: Node.Index = first_footer: {
1256 const old_footers_start = parent_size - old_footers_size;1355 var footer_ni = ni;
1257 const new_footers_start = parent_size - new_footers_size;1356 while (true) {
1258 const end_offset = node.location().resolve(mf)[0] + old_size;1357 const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
1259 const parent_file_offset = parent_ni.fileLocation(mf, false).offset;1358 if (prev_ni.position(mf) != .footer) break;
1260 try mf.moveRange(1359 footer_ni = prev_ni;
1261 parent_file_offset + old_footers_start,1360 }
1262 parent_file_offset + new_footers_start,1361 break :first_footer footer_ni;
1263 end_offset - old_footers_start - sub_footers_size,1362 };
1264 );
12651363
1266 // Update our own offset and size:1364 const shift = new_size - old_size;
1267 try ni.setLocation(mf, gpa, end_offset - new_size, new_size);
12681365
1269 // Any footers inside of us have had their offsets changed due to us growing:1366 // Update our own offset and size:
1270 if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| {1367 try ni.setLocation(
1271 var cur_ni = first_sub_footer_ni;
1272 while (true) {
1273 const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf);
1274 try cur_ni.setLocation(
1275 mf,1368 mf,
1276 gpa,1369 gpa,
1277 old_sub_footer_offset + (new_size - old_size),1370 node.location().resolve(mf)[0] - shift,
1278 sub_footer_size,1371 new_size,
1279 );1372 );
1280 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1281 }
1282 }
12831373
1284 // Finally, update the offsets of every footer before us:1374 // Any footers *inside* of `ni` have had their offsets changed, because they are
1285 if (node.prev.unwrap()) |prev_ni| {1375 // now positioned at the *new* end of `ni`:
1286 var maybe_footer_ni = prev_ni;1376 {
1287 while (true) {1377 var footer_oni = first_sub_footer_oni;
1288 switch (maybe_footer_ni.position(mf)) {1378 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1289 .header, .floating => break,1379 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1290 .footer => {},1380 try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size);
1381 }
1291 }1382 }
1292 const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf);1383
1293 try maybe_footer_ni.setLocation(1384 // Any footers *before* `ni` (in `parent_ni`) have been shifted backwards. We'll
1385 // also be moving their actual bytes in a moment, so track whether they have
1386 // content (if nothing does then we'll be able to skip the `moveRange`). That
1387 // flag is initially whether `ni` has content because we're shifting our own
1388 // bytes backwards too.
1389 var moved_has_content: bool = node.flags.has_content;
1390 {
1391 var footer_ni = first_parent_footer_ni;
1392 while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) {
1393 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1394 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1395 try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size);
1396 }
1397 }
1398
1399 if (moved_has_content) {
1400 // We moved at least one thing containing initialized bytes, so we need to
1401 // move the actual data. However, we should *not* move the bytes of any
1402 // nested footers inside of `ni`, because they've been "moved" to the end
1403 // of our new size, which is the same file location as before.
1404 const sub_footers_size = size: {
1405 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1406 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
1407 break :size new_size - first_sub_footer_offset;
1408 };
1409 const new_offset: u64, _ = node.location().resolve(mf);
1410 const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf);
1411 const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
1412 try mf.moveRange(
1413 parent_file_offset + new_footers_offset + shift,
1414 parent_file_offset + new_footers_offset,
1415 (new_offset - new_footers_offset) + // accounts for every footer before `ni`
1416 (old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers
1417 );
1418 }
1419 },
1420 .grow_parent_at_end => |grow_parent| {
1421 _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1422 try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{
1423 .exact_size = grow_parent.exact_size,
1424 .move_footers = false,
1425 });
1426 _, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1427 const shift = new_parent_size - old_parent_size;
1428
1429 // Here's what we have left to do:
1430 //
1431 // * Increase our own size by `shift` to absorb the added space.
1432 //
1433 // * If there are any footers *inside* `ni`, increase their offsets by `shift`.
1434 //
1435 // * If there are any footers *after* `ni` (inside `parent_ni`), increase their
1436 // offsets by `shift`.
1437 //
1438 // * Do a `moveRange` corresponding to those offset changes. This is a single
1439 // range which starts at the footers *inside* `ni`.
1440
1441 const actual_new_size = old_size + shift;
1442 if (grow_options.exact_size) {
1443 assert(actual_new_size == new_size);
1444 }
1445
1446 try ni.setLocation(
1294 mf,1447 mf,
1295 gpa,1448 gpa,
1296 moved_footer_offset + old_size - new_size,1449 node.location().resolve(mf)[0],
1297 moved_footer_size,1450 actual_new_size,
1298 );1451 );
1299 maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break;
1300 }
1301 }
13021452
1303 return;1453 // This will track whether any node with a changed offset actually contains
1454 // initialized bytes. If not, there'll be no need to call `moveRange`.
1455 var moved_has_content: bool = false;
1456
1457 // Set any nested footers' offsets (and include them in `moved_has_content`).
1458 {
1459 var footer_oni = first_sub_footer_oni;
1460 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1461 assert(footer_ni.position(mf) == .footer);
1462 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1463 const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
1464 try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
1465 }
1466 }
1467
1468 // Now set offsets for footers after `ni` inside of `parent_ni`.
1469 {
1470 var footer_oni = ni.next(mf);
1471 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1472 assert(footer_ni.position(mf) == .footer);
1473 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1474 const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
1475 try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
1476 }
1477 }
1478
1479 if (moved_has_content) {
1480 // We moved at least one footer containing initialized bytes, so we need to
1481 // move the actual data. Compute how big the footers inside `ni` are...
1482 const sub_footers_size: u64 = size: {
1483 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1484 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
1485 // `actual_new_size` is used here since we already updated the nested footers' offsets above.
1486 break :size actual_new_size - first_sub_footer_offset;
1487 };
1488 // ...and how big the footers *after* `ni`, inside `parent_ni`, are...
1489 const post_footers_size: u64 = old_parent_size - (old_offset + old_size);
1490 // ...and move them both.
1491 const parent_file_off = parent_ni.fileLocation(mf, false).offset;
1492 const total_move_size = sub_footers_size + post_footers_size;
1493 assert(total_move_size != 0);
1494 try mf.moveRange(
1495 parent_file_off + old_parent_size - total_move_size,
1496 parent_file_off + new_parent_size - total_move_size,
1497 total_move_size,
1498 );
1499 }
1500 },
1501 }
1304 },1502 },
1305 }1503 }
1306}1504}
...@@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment(...@@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment(
1320 ni: Node.Index,1518 ni: Node.Index,
1321 new_alignment: ?Alignment,1519 new_alignment: ?Alignment,
1322 new_size: u64,1520 new_size: u64,
1323 grow_mode: GrowMode,1521 grow_options: GrowOptions,
1324) Error!void {1522) Error!void {
1325 mf.nodes_lock.assertUnlocked();1523 mf.nodes_lock.assertUnlocked();
13261524
...@@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment(...@@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment(
1347 }1545 }
1348 // Great, we can grow this node without changing its offset or moving any siblings.1546 // Great, we can grow this node without changing its offset or moving any siblings.
1349 try ni.setLocation(mf, gpa, old_offset, new_size);1547 try ni.setLocation(mf, gpa, old_offset, new_size);
1350 // If we have any footers, we need to move them to the end of our new size, and update their1548 if (grow_options.move_footers) {
1351 // offsets accordingly.1549 // If we have any footers, we need to move them to the end of our new size, and update
1352 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1550 // their offsets accordingly.
1353 var cur_ni = first_footer_ni;1551 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1354 var footers_have_content = false;1552 var cur_ni = first_footer_ni;
1355 while (true) {1553 var footers_have_content = false;
1356 footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;1554 while (true) {
1357 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);1555 footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
1358 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);1556 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1359 cur_ni = cur_ni.next(mf).unwrap() orelse break;1557 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1360 }1558 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1361 if (footers_have_content) {1559 }
1362 const parent_file_off = parent_ni.fileLocation(mf, false).offset;1560 if (footers_have_content) {
1363 // This gets the *new* offset because we already updated the offsets above.1561 const parent_file_off = parent_ni.fileLocation(mf, false).offset;
1364 const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);1562 // This gets the *new* offset because we already updated the offsets above.
1365 const footers_size = new_size - new_footers_offset;1563 const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1366 try mf.moveRange(1564 const footers_size = new_size - new_footers_offset;
1367 parent_file_off + old_offset + old_size - footers_size,1565 try mf.moveRange(
1368 parent_file_off + old_offset + new_size - footers_size,1566 parent_file_off + old_offset + old_size - footers_size,
1369 footers_size,1567 parent_file_off + old_offset + new_size - footers_size,
1370 );1568 footers_size,
1569 );
1570 }
1371 }1571 }
1372 }1572 }
1373 return;1573 return;
...@@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment(...@@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment(
1444 // that, let's first try the Linux "insert range" fast path. We didn't try it before now1644 // that, let's first try the Linux "insert range" fast path. We didn't try it before now
1445 // because it would have been more efficient to just move ourselves into existing space.1645 // because it would have been more efficient to just move ourselves into existing space.
1446 //1646 //
1447 // If we were given a custom alignment, we cannot pass `grow_mode` directly into the1647 // If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the
1448 // "insert range" path, because that function is unaware of `new_alignment`.1648 // "insert range" path, because that function is unaware of `new_alignment`.
1449 const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact;1649 const insert_range_grow_options: GrowOptions = .{
1650 .exact_size = grow_options.exact_size or new_alignment != null,
1651 .move_footers = grow_options.move_footers,
1652 };
1450 if (alignment.check(old_offset) and1653 if (alignment.check(old_offset) and
1451 try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode))1654 try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options))
1452 {1655 {
1453 // The Linux fast path did our job for us!1656 // The Linux fast path did our job for us!
1454 return;1657 return;
...@@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment(...@@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment(
1458 const new_parent_size = parent_ni.alignment(mf).forward(1661 const new_parent_size = parent_ni.alignment(mf).forward(
1459 min_parent_size +| min_parent_size / growth_factor,1662 min_parent_size +| min_parent_size / growth_factor,
1460 );1663 );
1461 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);1664 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1665 .exact_size = false,
1666 .move_footers = true,
1667 });
1462 }1668 }
14631669
1464 break :new_loc .{1670 break :new_loc .{
...@@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment(...@@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment(
14711677
1472 // Footers need to move to a different place than the rest of our content.1678 // Footers need to move to a different place than the rest of our content.
1473 const footers_size: u64, const footers_have_content: bool = footers: {1679 const footers_size: u64, const footers_have_content: bool = footers: {
1680 if (!grow_options.move_footers) {
1681 // Pretend there are no footers so as to not move them.
1682 break :footers .{ 0, false };
1683 }
1474 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {1684 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
1475 break :footers .{ 0, false };1685 break :footers .{ 0, false };
1476 };1686 };
...@@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment(...@@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment(
1525/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`1735/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`
1526/// without changing any nodes' locations or invalidating any slices.1736/// without changing any nodes' locations or invalidating any slices.
1527///1737///
1528/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node1738/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`),
1529/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`,1739/// updates the location of `ni` and every node whose offset has changed, and returns `true`.
1530/// the actual new size of `ni` may be greater than `new_size`.
1531fn growNodeViaInsertRange(1740fn growNodeViaInsertRange(
1532 mf: *MappedFile,1741 mf: *MappedFile,
1533 gpa: Allocator,1742 gpa: Allocator,
1534 ni: Node.Index,1743 ni: Node.Index,
1535 new_size: u64,1744 new_size: u64,
1536 grow_mode: GrowMode,1745 grow_options: GrowOptions,
1537) Error!bool {1746) Error!bool {
1538 if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {1747 if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {
1539 return false;1748 return false;
...@@ -1541,24 +1750,22 @@ fn growNodeViaInsertRange(...@@ -1541,24 +1750,22 @@ fn growNodeViaInsertRange(
15411750
1542 _, const old_size = ni.location(mf).resolve(mf);1751 _, const old_size = ni.location(mf).resolve(mf);
15431752
1544 // We don't compute the size of the range yet, because depending on `grow_mode` we might want to1753 // We don't compute the size of the range yet, because depending on `grow_options` we might want
1545 // bump it based on our sibling and parent nodes' alignments. However, we can do an early check1754 // to bump it based on our sibling and parent nodes' alignments. However, we can do an early
1546 // for cases where we should obviously exit.1755 // check for cases where we should obviously exit.
1547 const min_range_size: u64 = s: {1756 const min_range_size: u64 = s: {
1548 const exact_size = new_size - old_size;1757 const requested_size = new_size - old_size;
1549 if (mf.flags.block_size.check(exact_size)) {1758 if (mf.flags.block_size.check(requested_size)) {
1550 break :s exact_size;1759 break :s requested_size;
1551 }1760 }
1552 switch (grow_mode) {1761 if (!grow_options.exact_size and
1553 .exact => return false,1762 requested_size >= mf.flags.block_size.toByteUnits() * 2)
1554 .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) {1763 {
1555 // We're growing by at least a few blocks, so allow ourselves to bump the size1764 // We're growing by at least a few blocks, so allow ourselves to bump the size
1556 // slightly to give it the needed alignment.1765 // slightly to give it the needed alignment.
1557 break :s mf.flags.block_size.forward(exact_size);1766 break :s mf.flags.block_size.forward(requested_size);
1558 } else {
1559 return false;
1560 },
1561 }1767 }
1768 return false;
1562 };1769 };
1563 assert(min_range_size > 0);1770 assert(min_range_size > 0);
1564 assert(mf.flags.block_size.check(min_range_size));1771 assert(mf.flags.block_size.check(min_range_size));
...@@ -1573,14 +1780,17 @@ fn growNodeViaInsertRange(...@@ -1573,14 +1780,17 @@ fn growNodeViaInsertRange(
1573 }1780 }
1574 break :range_file_offset range_file_offset;1781 break :range_file_offset range_file_offset;
1575 };1782 };
1576 const first_footer_oni = ni.firstFooter(mf);1783 const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: {
1577 const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: {1784 if (!grow_options.move_footers) {
1785 // Pretend there are no footers so as to not move them.
1786 break :footers .{ .wrap(last_ni), 0 };
1787 }
1788 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
1789 break :footers .{ .wrap(last_ni), 0 };
1790 };
1578 const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);1791 const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);
1579 break :size old_size - first_footer_offset;1792 break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset };
1580 } else 0;1793 };
1581 const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: {
1582 break :pre_footer first_footer_ni.prev(mf);
1583 } else .wrap(last_ni);
1584 const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: {1794 const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: {
1585 const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);1795 const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);
1586 break :end pre_footer_off + pre_footer_size;1796 break :end pre_footer_off + pre_footer_size;
...@@ -1634,18 +1844,14 @@ fn growNodeViaInsertRange(...@@ -1634,18 +1844,14 @@ fn growNodeViaInsertRange(
1634 break :range_size min_range_size;1844 break :range_size min_range_size;
1635 }1845 }
1636 // Perhaps we're allowed to grow by more than `min_range_size`?1846 // Perhaps we're allowed to grow by more than `min_range_size`?
1637 switch (grow_mode) {1847 const candidate_range_size = need_range_align.forward(min_range_size);
1638 .exact => return false,1848 if (!grow_options.exact_size and
1639 .minimum => {1849 // Allow growing by up to 50% more than was requested.
1640 const candidate_range_size = need_range_align.forward(min_range_size);1850 candidate_range_size <= min_range_size +| min_range_size / 2)
1641 // Allow growing by up to 50% more than was requested.1851 {
1642 if (candidate_range_size <= min_range_size +| min_range_size / 2) {1852 break :range_size candidate_range_size;
1643 break :range_size candidate_range_size;
1644 } else {
1645 return false;
1646 }
1647 },
1648 }1853 }
1854 return false;
1649 };1855 };
16501856
1651 // This `range_size` is compatible with everyone's alignment requirements, and we won't move too1857 // This `range_size` is compatible with everyone's alignment requirements, and we won't move too
...@@ -1723,13 +1929,15 @@ fn growNodeViaInsertRange(...@@ -1723,13 +1929,15 @@ fn growNodeViaInsertRange(
1723 cur_ni = cur_ni.parent(mf).unwrap() orelse break;1929 cur_ni = cur_ni.parent(mf).unwrap() orelse break;
1724 }1930 }
17251931
1726 // The only thing left is to update the offsets of any footers inside of `ni`.1932 if (grow_options.move_footers) {
1727 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1933 // The only thing left is to update the offsets of any footers inside of `ni`.
1728 var footer_ni = first_footer_ni;1934 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1729 while (true) {1935 var footer_ni = first_footer_ni;
1730 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);1936 while (true) {
1731 try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);1937 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1732 footer_ni = footer_ni.next(mf).unwrap() orelse break;1938 try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);
1939 footer_ni = footer_ni.next(mf).unwrap() orelse break;
1940 }
1733 }1941 }
1734 }1942 }
17351943
...@@ -1783,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity(...@@ -1783,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity(
1783 const new_parent_size = parent_ni.alignment(mf).forward(1991 const new_parent_size = parent_ni.alignment(mf).forward(
1784 min_parent_size +| min_parent_size / growth_factor,1992 min_parent_size +| min_parent_size / growth_factor,
1785 );1993 );
1786 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);1994 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1995 .exact_size = false,
1996 .move_footers = true,
1997 });
1787 }1998 }
1788 return;1999 return;
1789 };2000 };
...@@ -1865,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity(...@@ -1865,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity(
1865 const new_parent_size = parent_ni.alignment(mf).forward(2076 const new_parent_size = parent_ni.alignment(mf).forward(
1866 min_parent_size +| min_parent_size / growth_factor,2077 min_parent_size +| min_parent_size / growth_factor,
1867 );2078 );
1868 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);2079 try mf.growNode(gpa, parent_ni, new_parent_size, .{
2080 .exact_size = false,
2081 .move_footers = true,
2082 });
1869 }2083 }
18702084
1871 if (moving_has_content) {2085 if (moving_has_content) {
...@@ -1895,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity(...@@ -1895,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity(
1895 }2109 }
1896}2110}
18972111
1898/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current2112/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which
1899/// footers, so that the footers can grow into that space.2113/// footers can therefore grow into.
1900fn ensureAdditionalFooterCapacity(2114fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 {
1901 mf: *MappedFile,
1902 gpa: Allocator,
1903 parent_ni: Node.Index,
1904 extra_capacity: u64,
1905) Error!void {
1906 // This is way easier than the header case, because we don't need to actually move anything; we
1907 // just need to expand the parent if there isn't space, and that will add padding after the
1908 // parent's floating children, which is exactly where we want it.
1909
1910 const first_footer_oni = parent_ni.firstFooter(mf);2115 const first_footer_oni = parent_ni.firstFooter(mf);
19112116
1912 _, const parent_size = parent_ni.location(mf).resolve(mf);2117 const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: {
19132118 const first_footer_ni = first_footer_oni.unwrap() orelse {
1914 const footers_size: u64 = footers_size: {2119 _, const parent_size = parent_ni.location(mf).resolve(mf);
1915 const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0;2120 break :footers .{ parent_ni.last(mf), parent_size };
2121 };
1916 const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);2122 const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);
1917 break :footers_size parent_size - first_footer_off;2123 break :footers .{ first_footer_ni.prev(mf), first_footer_off };
1918 };2124 };
19192125
1920 const header_and_floating_end: u64 = end: {2126 const header_and_floating_end: u64 = end: {
1921 const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: {
1922 break :before_footers first_footer_ni.prev(mf);
1923 } else before_footers: {
1924 break :before_footers parent_ni.last(mf);
1925 };
1926 const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;2127 const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;
1927 const offset, const size = before_footers_ni.location(mf).resolve(mf);2128 const offset, const size = before_footers_ni.location(mf).resolve(mf);
1928 break :end offset + size;2129 break :end offset + size;
1929 };2130 };
19302131
1931 assert(header_and_floating_end + footers_size <= parent_size);2132 return footers_off - header_and_floating_end;
1932
1933 const min_parent_size = header_and_floating_end + footers_size + extra_capacity;
1934 if (parent_size < min_parent_size) {
1935 const new_parent_size = parent_ni.alignment(mf).forward(
1936 min_parent_size +| min_parent_size / growth_factor,
1937 );
1938 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
1939 }
1940}2133}
19412134
1942fn removeNodesFromChildList(2135fn removeNodesFromChildList(
...@@ -2030,7 +2223,7 @@ fn realignNode(...@@ -2030,7 +2223,7 @@ fn realignNode(
2030 mf: *MappedFile,2223 mf: *MappedFile,
2031 gpa: Allocator,2224 gpa: Allocator,
2032 ni: Node.Index,2225 ni: Node.Index,
2033 new_alignment: Alignment,2226 new_align: Alignment,
2034) Error!void {2227) Error!void {
2035 mf.nodes_lock.assertUnlocked();2228 mf.nodes_lock.assertUnlocked();
20362229
...@@ -2038,30 +2231,25 @@ fn realignNode(...@@ -2038,30 +2231,25 @@ fn realignNode(
20382231
2039 if (ni == .root or ni.position(mf) != .floating) {2232 if (ni == .root or ni.position(mf) != .floating) {
2040 // Only this node's size is aligned, not its offset.2233 // Only this node's size is aligned, not its offset.
2041 if (!new_alignment.check(old_size)) {2234 if (!new_align.check(old_size)) {
2042 assert(new_alignment.compare(.gt, ni.alignment(mf)));2235 assert(new_align.compare(.gt, ni.alignment(mf)));
2043 try mf.growNode(2236 try mf.growNode(gpa, ni, new_align.forward(old_size), .{
2044 gpa,2237 .exact_size = true, // because `growNode` is not aware that the size needs to match `new_align`
2045 ni,2238 .move_footers = true,
2046 new_alignment.forward(old_size),2239 });
2047 .exact, // because `growNode` is not aware that the size needs to match `new_alignment`
2048 );
2049 }2240 }
2050 } else {2241 } else {
2051 // This is a floating node, so its size and offset are both aligned.2242 // This is a floating node, so its size and offset are both aligned.
2052 if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) {2243 if (!new_align.check(old_offset) or !new_align.check(old_size)) {
2053 assert(new_alignment.compare(.gt, ni.alignment(mf)));2244 assert(new_align.compare(.gt, ni.alignment(mf)));
2054 try mf.growFloatingNodeWithAlignment(2245 try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{
2055 gpa,2246 .exact_size = false,
2056 ni,2247 .move_footers = true,
2057 new_alignment,2248 });
2058 new_alignment.forward(old_size),
2059 .minimum,
2060 );
2061 }2249 }
2062 }2250 }
20632251
2064 ni.get(mf).flags.alignment = new_alignment;2252 ni.get(mf).flags.alignment = new_align;
2065}2253}
20662254
2067fn updateWriters(mf: *MappedFile) void {2255fn updateWriters(mf: *MappedFile) void {