authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-10 09:30:45+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 10:20:52+01:00
logfa3a9fcdfaee42ef304f662d65decbb685c705a0
tree6839b76688eda3e09f99af6f7fc81078f28e523a
parentf26cdb2771a4bb4d5f1d5acc446ec51c3e177f75
signaturelock-open Commit is signed but in an unrecognized format.

llvm: only load/store ABI-sized integers to/from memory

In theory, for any integer type containing padding bits (e.g. `i3`), LLVM *does* support storing and loading that type to and from memory, and the documented semantics for this in the LLVM langref are a valid implementation of Zig's integer semantics. However, in practice, this support is frequently buggy, because Clang never emits such operations and so they are poorly tested in LLVM. The most recent example of such a bug is https://codeberg.org/ziglang/zig/issues/35560. In addition, LLVM's semantics here lead to suboptimal codegen, because loads/stores are only rounded up to the next byte (rather than the next ABI integer type) and padding bits are unspecified (or, on LLVM master, padding bits are all zero, even for a negative signed integer). We can both mitigate LLVM bugs and get better codegen by employing the strategy used by the self-hosted backends, wherein we zero- or sign-extend to an integer of bit size `@sizeOf(WeirdInt) * 8`. This type has the same size in memory but has no padding bits. When loading, we load the extended type and then use `trunc nuw` or `trunc nsw` (depending on the signedness) to convert to the correct value type. The name of the game here is to minimize occurrences of `wip.load` and `wip.store` in the LLVM backend, with most accesses instead going through higher-level functions which handle this extension/truncation for us. Those higher-level functions are `codegen.llvm.FuncGen.load` and `codegen.llvm.FuncGen.store`. I changed their signatures slightly to make them consistent and more generally useful, and changed the majority of calls to `wip.load`/`wip.store` to use them. They are the only functions which include the integer extension/truncation logic. The remaining occurrences of `wip.load` and `wip.store` outside of those two functions are all in ABI handling. These code paths cannot occur for non-ABI types due to how we lower `CallingConvention.auto`, therefore they cannot be reached for non-ABI-sized integer types. This diff was not purely mechanical---I had to make a handful of non-trivial logic changes and refactors. The biggest logic change is in lowering a "select" operation to a manual loop: we now use phi nodes to track the loop iterator and accumulator instead of an `alloca`. (To be honest, this wasn't really necessary, but I felt dirty migrating the old logic when this approach is clearly better.) The most significant refactor is that the logic for extracting function call arguments has been moved into `FuncGen`: the `codegen.llvm.Object.updateFunc` implementation will not initialize the three fields which depend on function arguments, and a new function `FuncGen.genMainBody` will populate those fields before calling `genBody` on the main AIR body. This change was made so that I could use member functions on `FuncGen` in this logic without jumping through hoops. The reproduction given in https://codeberg.org/ziglang/zig/issues/35560 is apparently quite delicate, so I haven't added a test case because it realistically would not be very useful. However, I have manually tested the reproduction, and can confirm that the bug *does* trigger before this commit (targeting both baseline and native CPU), and does *not* trigger after this commit (again, for both baseline and native CPU). Resolves: https://codeberg.org/ziglang/zig/issues/35560

5 files changed, 762 insertions(+), 807 deletions(-)

lib/std/zig/llvm/Builder.zig+48-2
...@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {...@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {
2061 };2061 };
2062 }2062 }
20632063
2064 /// Asserts that neither `a` nor `b` is `.default`.
2065 pub fn max(a: Alignment, b: Alignment) Alignment {
2066 assert(a != .default);
2067 assert(b != .default);
2068 return @enumFromInt(@max(@intFromEnum(a), @intFromEnum(b)));
2069 }
2070
2064 pub fn toLlvm(self: Alignment) u6 {2071 pub fn toLlvm(self: Alignment) u6 {
2065 return switch (self) {2072 return switch (self) {
2066 .default => 0,2073 .default => 0,
...@@ -4314,6 +4321,9 @@ pub const Function = struct {...@@ -4314,6 +4321,9 @@ pub const Function = struct {
4314 @"tail call",4321 @"tail call",
4315 @"tail call fast",4322 @"tail call fast",
4316 trunc,4323 trunc,
4324 @"trunc nuw",
4325 @"trunc nsw",
4326 @"trunc nuw nsw",
4317 udiv,4327 udiv,
4318 @"udiv exact",4328 @"udiv exact",
4319 urem,4329 urem,
...@@ -4377,7 +4387,10 @@ pub const Function = struct {...@@ -4377,7 +4387,10 @@ pub const Function = struct {
4377 };4387 };
4378 }4388 }
43794389
4380 pub fn toCastOpcode(self: Tag) CastOpcode {4390 /// Does not accept `.@"trunc nuw"`, `.@"trunc nsw"`, or `.@"trunc nuw nsw"`, because
4391 /// they do not have distinct `CastOpcode` values, and are instead encoded in bitcode
4392 /// using flags on a normal `trunc` operation.
4393 fn toCastOpcode(self: Tag) CastOpcode {
4381 return switch (self) {4394 return switch (self) {
4382 .trunc => .trunc,4395 .trunc => .trunc,
4383 .zext => .zext,4396 .zext => .zext,
...@@ -4572,6 +4585,9 @@ pub const Function = struct {...@@ -4572,6 +4585,9 @@ pub const Function = struct {
4572 .sext,4585 .sext,
4573 .sitofp,4586 .sitofp,
4574 .trunc,4587 .trunc,
4588 .@"trunc nuw",
4589 .@"trunc nsw",
4590 .@"trunc nuw nsw",
4575 .uitofp,4591 .uitofp,
4576 .zext,4592 .zext,
4577 => wip.extraData(Cast, instruction.data).type,4593 => wip.extraData(Cast, instruction.data).type,
...@@ -4758,6 +4774,9 @@ pub const Function = struct {...@@ -4758,6 +4774,9 @@ pub const Function = struct {
4758 .sext,4774 .sext,
4759 .sitofp,4775 .sitofp,
4760 .trunc,4776 .trunc,
4777 .@"trunc nuw",
4778 .@"trunc nsw",
4779 .@"trunc nuw nsw",
4761 .uitofp,4780 .uitofp,
4762 .zext,4781 .zext,
4763 => function.extraData(Cast, instruction.data).type,4782 => function.extraData(Cast, instruction.data).type,
...@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {...@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {
5975 .sext,5994 .sext,
5976 .sitofp,5995 .sitofp,
5977 .trunc,5996 .trunc,
5997 .@"trunc nuw",
5998 .@"trunc nsw",
5999 .@"trunc nuw nsw",
5978 .uitofp,6000 .uitofp,
5979 .zext,6001 .zext,
5980 => {},6002 => {},
...@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {...@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {
6583 .sext,6605 .sext,
6584 .sitofp,6606 .sitofp,
6585 .trunc,6607 .trunc,
6608 .@"trunc nuw",
6609 .@"trunc nsw",
6610 .@"trunc nuw nsw",
6586 .uitofp,6611 .uitofp,
6587 .zext,6612 .zext,
6588 => {6613 => {
...@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void...@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void
9975 .sext,10000 .sext,
9976 .sitofp,10001 .sitofp,
9977 .trunc,10002 .trunc,
10003 .@"trunc nuw",
10004 .@"trunc nsw",
10005 .@"trunc nuw nsw",
9978 .uitofp,10006 .uitofp,
9979 .zext,10007 .zext,
9980 => |tag| {10008 => |tag| {
...@@ -11649,7 +11677,11 @@ fn convTag(...@@ -11649,7 +11677,11 @@ fn convTag(
11649 .unneeded => unreachable,11677 .unneeded => unreachable,
11650 },11678 },
11651 .eq => unreachable,11679 .eq => unreachable,
11652 .gt => .trunc,11680 .gt => switch (signedness) {
11681 .unsigned => .@"trunc nuw",
11682 .signed => .@"trunc nsw",
11683 .unneeded => .trunc,
11684 },
11653 },11685 },
11654 .pointer => .inttoptr,11686 .pointer => .inttoptr,
11655 else => unreachable,11687 else => unreachable,
...@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco...@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
14962 .opcode = kind.toCastOpcode(),14994 .opcode = kind.toCastOpcode(),
14963 });14995 });
14964 },14996 },
14997 .@"trunc nuw",
14998 .@"trunc nsw",
14999 .@"trunc nuw nsw",
15000 => |kind| {
15001 const extra = func.extraData(Function.Instruction.Cast, data);
15002 try function_block.writeAbbrev(FunctionBlock.TruncNoWrap{
15003 .val = adapter.getOffsetValueIndex(extra.val),
15004 .type_index = extra.type,
15005 .flags = .{
15006 .no_unsigned_wrap = kind == .@"trunc nuw" or kind == .@"trunc nuw nsw",
15007 .no_signed_wrap = kind == .@"trunc nsw" or kind == .@"trunc nuw nsw",
15008 },
15009 });
15010 },
14965 .@"fcmp false",15011 .@"fcmp false",
14966 .@"fcmp oeq",15012 .@"fcmp oeq",
14967 .@"fcmp oge",15013 .@"fcmp oge",
lib/std/zig/llvm/ir.zig+19
...@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {...@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {
696 ModuleBlock.FunctionBlock.Select,696 ModuleBlock.FunctionBlock.Select,
697 ModuleBlock.FunctionBlock.SelectFast,697 ModuleBlock.FunctionBlock.SelectFast,
698 ModuleBlock.FunctionBlock.Cast,698 ModuleBlock.FunctionBlock.Cast,
699 ModuleBlock.FunctionBlock.TruncNoWrap,
699 ModuleBlock.FunctionBlock.Alloca,700 ModuleBlock.FunctionBlock.Alloca,
700 ModuleBlock.FunctionBlock.GetElementPtr,701 ModuleBlock.FunctionBlock.GetElementPtr,
701 ModuleBlock.FunctionBlock.ExtractValue,702 ModuleBlock.FunctionBlock.ExtractValue,
...@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {...@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {
1086 opcode: CastOpcode,1087 opcode: CastOpcode,
1087 };1088 };
10881089
1090 pub const TruncNoWrap = struct {
1091 pub const Flags = packed struct(u2) {
1092 no_unsigned_wrap: bool,
1093 no_signed_wrap: bool,
1094 };
1095 pub const ops = [_]AbbrevOp{
1096 .{ .literal = @intFromEnum(ModuleBlock.FunctionBlock.Code.INST_CAST) },
1097 ValueAbbrev,
1098 .{ .fixed_runtime = Builder.Type },
1099 .{ .literal = @intFromEnum(Builder.CastOpcode.trunc) },
1100 .{ .fixed = @bitSizeOf(Flags) },
1101 };
1102
1103 val: u32,
1104 type_index: Builder.Type,
1105 flags: Flags,
1106 };
1107
1089 pub const Alloca = struct {1108 pub const Alloca = struct {
1090 pub const Flags = packed struct(u11) {1109 pub const Flags = packed struct(u11) {
1091 align_lower: u5,1110 align_lower: u5,
src/InternPool.zig+1-7
...@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {...@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {
6007 return r;6007 return r;
6008 }6008 }
60096009
6010 const LlvmBuilderAlignment = std.zig.llvm.Builder.Alignment;6010 pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
6011
6012 pub fn toLlvm(a: Alignment) LlvmBuilderAlignment {
6013 return @enumFromInt(@intFromEnum(a));
6014 }
6015
6016 pub fn fromLlvm(a: LlvmBuilderAlignment) Alignment {
6017 return @enumFromInt(@intFromEnum(a));6011 return @enumFromInt(@intFromEnum(a));
6018 }6012 }
6019};6013};
src/codegen/llvm.zig+15-175
...@@ -20,7 +20,6 @@ const Value = @import("../Value.zig");...@@ -20,7 +20,6 @@ const Value = @import("../Value.zig");
20const Zcu = @import("../Zcu.zig");20const Zcu = @import("../Zcu.zig");
21const aarch64_c_abi = @import("aarch64/abi.zig");21const aarch64_c_abi = @import("aarch64/abi.zig");
22const FuncGen = @import("llvm/FuncGen.zig");22const FuncGen = @import("llvm/FuncGen.zig");
23const buildAllocaInner = FuncGen.buildAllocaInner;
24const isByRef = FuncGen.isByRef;23const isByRef = FuncGen.isByRef;
25const firstParamSRet = FuncGen.firstParamSRet;24const firstParamSRet = FuncGen.firstParamSRet;
26const lowerFnRetTy = FuncGen.lowerFnRetTy;25const lowerFnRetTy = FuncGen.lowerFnRetTy;
...@@ -1274,165 +1273,7 @@ pub const Object = struct {...@@ -1274,165 +1273,7 @@ pub const Object = struct {
1274 } }, &o.builder);1273 } }, &o.builder);
1275 }1274 }
12761275
1277 var deinit_wip = true;1276 const file, const subprogram = if (!owner_mod.strip) debug_info: {
1278 var wip = try Builder.WipFunction.init(&o.builder, .{
1279 .function = llvm_function,
1280 .strip = owner_mod.strip,
1281 });
1282 defer if (deinit_wip) wip.deinit();
1283 wip.cursor = .{ .block = try wip.block(0, "Entry") };
1284
1285 // This is the list of args we will use that correspond directly to the AIR arg
1286 // instructions. Depending on the calling convention, this list is not necessarily
1287 // a bijection with the actual LLVM parameters of the function.
1288 var args: std.ArrayList(Builder.Value) = .empty;
1289 defer args.deinit(gpa);
1290
1291 const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
1292 var it = iterateParamTypes(o, fn_info);
1293
1294 const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
1295 const param = wip.arg(it.llvm_index);
1296 it.llvm_index += 1;
1297 break :param param;
1298 } else .none;
1299
1300 const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1301 const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1302 const param = wip.arg(it.llvm_index);
1303 it.llvm_index += 1;
1304 break :param param;
1305 } else .none;
1306
1307 while (try it.next()) |lowering| {
1308 try args.ensureUnusedCapacity(gpa, 1);
1309
1310 switch (lowering) {
1311 .no_bits => continue,
1312 .byval => {
1313 assert(!it.byval_attr);
1314 const param_index = it.zig_index - 1;
1315 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1316 const param = wip.arg(it.llvm_index - 1);
1317
1318 if (isByRef(param_ty, zcu)) {
1319 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1320 const param_llvm_ty = param.typeOfWip(&wip);
1321 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1322 _ = try wip.store(.normal, param, arg_ptr, alignment);
1323 args.appendAssumeCapacity(arg_ptr);
1324 } else {
1325 args.appendAssumeCapacity(param);
1326 }
1327 },
1328 .byref => {
1329 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1330 const param = wip.arg(it.llvm_index - 1);
1331
1332 if (isByRef(param_ty, zcu)) {
1333 args.appendAssumeCapacity(param);
1334 } else {
1335 const param_llvm_ty = try o.lowerType(param_ty);
1336 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1337 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1338 }
1339 },
1340 .byref_mut => {
1341 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1342 const param = wip.arg(it.llvm_index - 1);
1343
1344 if (isByRef(param_ty, zcu)) {
1345 args.appendAssumeCapacity(param);
1346 } else {
1347 const param_llvm_ty = try o.lowerType(param_ty);
1348 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1349 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1350 }
1351 },
1352 .abi_sized_int => {
1353 assert(!it.byval_attr);
1354 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1355 const param = wip.arg(it.llvm_index - 1);
1356
1357 const param_llvm_ty = try o.lowerType(param_ty);
1358 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1359 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1360 _ = try wip.store(.normal, param, arg_ptr, alignment);
1361
1362 if (isByRef(param_ty, zcu)) {
1363 args.appendAssumeCapacity(arg_ptr);
1364 } else {
1365 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1366 }
1367 },
1368 .slice => {
1369 assert(!it.byval_attr);
1370 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1371 assert(!isByRef(param_ty, zcu));
1372 const slice_val = try wip.buildAggregate(
1373 try o.lowerType(param_ty),
1374 &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
1375 "",
1376 );
1377 args.appendAssumeCapacity(slice_val);
1378 },
1379 .multiple_llvm_types => {
1380 assert(!it.byval_attr);
1381 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1382 const param_llvm_ty = try o.lowerType(param_ty);
1383 const param_alignment = param_ty.abiAlignment(zcu);
1384 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
1385 const arg_ptr = try buildAllocaInner(&wip, llvm_ty, param_alignment.toLlvm(), target);
1386 const llvm_args_start = it.llvm_index - it.types_len;
1387 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
1388 const param = wip.arg(@intCast(llvm_arg_index));
1389 const part_ptr = try o.ptraddConst(&wip, arg_ptr, offset);
1390 _ = try wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
1391 }
1392
1393 if (isByRef(param_ty, zcu)) {
1394 args.appendAssumeCapacity(arg_ptr);
1395 } else {
1396 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment.toLlvm(), ""));
1397 }
1398 },
1399 .float_array => {
1400 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1401 const param_llvm_ty = try o.lowerType(param_ty);
1402 const param = wip.arg(it.llvm_index - 1);
1403
1404 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1405 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1406 _ = try wip.store(.normal, param, arg_ptr, alignment);
1407
1408 if (isByRef(param_ty, zcu)) {
1409 args.appendAssumeCapacity(arg_ptr);
1410 } else {
1411 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1412 }
1413 },
1414 .i32_array, .i64_array => {
1415 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1416 const param_llvm_ty = try o.lowerType(param_ty);
1417 const param = wip.arg(it.llvm_index - 1);
1418
1419 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1420 const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1421 _ = try wip.store(.normal, param, arg_ptr, alignment);
1422
1423 if (isByRef(param_ty, zcu)) {
1424 args.appendAssumeCapacity(arg_ptr);
1425 } else {
1426 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1427 }
1428 },
1429 }
1430 }
1431
1432 break :implicit_args .{ ret_ptr, err_ret_trace };
1433 };
1434
1435 const file, const subprogram = if (!wip.strip) debug_info: {
1436 const file = try o.getDebugFile(file_scope);1277 const file = try o.getDebugFile(file_scope);
14371278
1438 const line_number = zcu.navSrcLine(func.owner_nav) + 1;1279 const line_number = zcu.navSrcLine(func.owner_nav) + 1;
...@@ -1498,11 +1339,12 @@ pub const Object = struct {...@@ -1498,11 +1339,12 @@ pub const Object = struct {
1498 .gpa = gpa,1339 .gpa = gpa,
1499 .air = air.*,1340 .air = air.*,
1500 .liveness = liveness.*.?,1341 .liveness = liveness.*.?,
1501 .wip = wip,1342 .wip = try .init(&o.builder, .{
1343 .function = llvm_function,
1344 .strip = owner_mod.strip,
1345 }),
1502 .is_naked = fn_info.cc == .naked,1346 .is_naked = fn_info.cc == .naked,
1503 .fuzz = fuzz,1347 .fuzz = fuzz,
1504 .ret_ptr = ret_ptr,
1505 .args = args.items,
1506 .arg_index = 0,1348 .arg_index = 0,
1507 .arg_inline_index = 0,1349 .arg_inline_index = 0,
1508 .func_inst_table = .empty,1350 .func_inst_table = .empty,
...@@ -1516,14 +1358,18 @@ pub const Object = struct {...@@ -1516,14 +1358,18 @@ pub const Object = struct {
1516 .base_line = zcu.navSrcLine(func.owner_nav),1358 .base_line = zcu.navSrcLine(func.owner_nav),
1517 .prev_dbg_line = 0,1359 .prev_dbg_line = 0,
1518 .prev_dbg_column = 0,1360 .prev_dbg_column = 0,
1519 .err_ret_trace = err_ret_trace,
1520 .disable_intrinsics = disable_intrinsics,1361 .disable_intrinsics = disable_intrinsics,
1521 .allowzero_access = false,1362 .allowzero_access = false,
1363
1364 .ret_ptr = undefined, // populated by `genMainBody`
1365 .err_ret_trace = undefined, // populated by `genMainBody`
1366 .args = undefined, // populated by `genMainBody`
1522 };1367 };
1523 defer fg.deinit();1368 defer fg.deinit();
1524 deinit_wip = false;
15251369
1526 try fg.genBody(air.getMainBody(), .poi);1370 fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
1371
1372 try fg.genMainBody();
15271373
1528 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole1374 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
1529 // function as considering null pointers valid so that LLVM's optimizers don't remove these1375 // function as considering null pointers valid so that LLVM's optimizers don't remove these
...@@ -4074,8 +3920,9 @@ pub const Object = struct {...@@ -4074,8 +3920,9 @@ pub const Object = struct {
4074 if (gop.found_existing) {3920 if (gop.found_existing) {
4075 // Keep the greater of the two alignments.3921 // Keep the greater of the two alignments.
4076 const llvm_variable = gop.value_ptr.*;3922 const llvm_variable = gop.value_ptr.*;
4077 const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));3923 const llvm_old_align = llvm_variable.getAlignment(&o.builder);
4078 llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);3924 const llvm_new_align = llvm_old_align.max(@"align".toLlvm());
3925 llvm_variable.setAlignment(llvm_new_align, &o.builder);
4079 return llvm_variable.ptrConst(&o.builder).global.toConst();3926 return llvm_variable.ptrConst(&o.builder).global.toConst();
4080 }3927 }
4081 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));3928 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
...@@ -4411,13 +4258,6 @@ pub const Object = struct {...@@ -4411,13 +4258,6 @@ pub const Object = struct {
4411 toLlvmAddressSpace(.generic, o.zcu.getTarget()),4258 toLlvmAddressSpace(.generic, o.zcu.getTarget()),
4412 );4259 );
4413 }4260 }
4414
4415 pub fn ptraddConst(o: *Object, wip: *Builder.WipFunction, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
4416 if (offset == 0) return ptr;
4417 const llvm_usize_ty = try o.lowerType(.usize);
4418 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
4419 return wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
4420 }
4421};4261};
44224262
4423const CallingConventionInfo = struct {4263const CallingConventionInfo = struct {
src/codegen/llvm/FuncGen.zig+679-623
...@@ -175,7 +175,158 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {...@@ -175,7 +175,158 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
175 }175 }
176}176}
177177
178pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {178/// Populates `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` based on the parameters of the
179/// function type, then generates the entire function body.
180///
181/// The caller may initialize `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` to undefined.
182pub fn genMainBody(fg: *FuncGen) TodoError!void {
183 const o = fg.object;
184 const zcu = o.zcu;
185 const ip = &zcu.intern_pool;
186 const comp = zcu.comp;
187 const gpa = comp.gpa;
188
189 const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type);
190 const fn_info = zcu.typeToFunc(fn_ty).?;
191 const param_types = fn_info.param_types.get(ip);
192
193 var it = iterateParamTypes(o, fn_info);
194
195 // Populate `fg.ret_ptr`...
196 if (firstParamSRet(fn_info, zcu, zcu.getTarget())) {
197 fg.ret_ptr = fg.wip.arg(it.llvm_index);
198 it.llvm_index += 1;
199 } else {
200 fg.ret_ptr = .none;
201 }
202 // ...and `fg.err_ret_trace`...
203 if (fn_info.cc == .auto and comp.config.any_error_tracing) {
204 fg.err_ret_trace = fg.wip.arg(it.llvm_index);
205 it.llvm_index += 1;
206 } else {
207 fg.err_ret_trace = .none;
208 }
209 // ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's
210 // done we'll use its buffer as `fg.args`.
211 var args: std.ArrayList(Builder.Value) = .empty;
212 defer args.deinit(gpa);
213
214 while (try it.next()) |lowering| {
215 try args.ensureUnusedCapacity(gpa, 1);
216
217 switch (lowering) {
218 .no_bits => continue,
219 .byval => {
220 assert(!it.byval_attr);
221 const param_index = it.zig_index - 1;
222 const param_ty: Type = .fromInterned(param_types[param_index]);
223 const param = fg.wip.arg(it.llvm_index - 1);
224
225 if (isByRef(param_ty, zcu)) {
226 const alignment = param_ty.abiAlignment(zcu).toLlvm();
227 const arg_ptr = try fg.buildAlloca(try o.lowerType(param_ty), alignment);
228 // We don't need to handle non-ABI-sized integer types in memory here since they
229 // are never by-ref.
230 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
231 args.appendAssumeCapacity(arg_ptr);
232 } else {
233 args.appendAssumeCapacity(param);
234 }
235 },
236 .byref, .byref_mut => {
237 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
238 const param = fg.wip.arg(it.llvm_index - 1);
239
240 if (isByRef(param_ty, zcu)) {
241 args.appendAssumeCapacity(param);
242 } else {
243 args.appendAssumeCapacity(try fg.load(param, .none, param_ty, .normal));
244 }
245 },
246 .abi_sized_int => {
247 assert(!it.byval_attr);
248 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
249 const param = fg.wip.arg(it.llvm_index - 1);
250
251 const param_llvm_ty = try o.lowerType(param_ty);
252 const alignment = param_ty.abiAlignment(zcu).toLlvm();
253 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
254 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
255
256 if (isByRef(param_ty, zcu)) {
257 args.appendAssumeCapacity(arg_ptr);
258 } else {
259 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
260 }
261 },
262 .slice => {
263 assert(!it.byval_attr);
264 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
265 assert(!isByRef(param_ty, zcu));
266 const slice_val = try fg.wip.buildAggregate(
267 try o.lowerType(param_ty),
268 &.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
269 "",
270 );
271 args.appendAssumeCapacity(slice_val);
272 },
273 .multiple_llvm_types => {
274 assert(!it.byval_attr);
275 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
276 const param_alignment = param_ty.abiAlignment(zcu);
277 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
278 const arg_ptr = try fg.buildAlloca(llvm_ty, param_alignment.toLlvm());
279 const llvm_args_start = it.llvm_index - it.types_len;
280 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
281 const param = fg.wip.arg(@intCast(llvm_arg_index));
282 const part_ptr = try fg.ptraddConst(arg_ptr, offset);
283 _ = try fg.wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
284 }
285
286 if (isByRef(param_ty, zcu)) {
287 args.appendAssumeCapacity(arg_ptr);
288 } else {
289 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
290 }
291 },
292 .float_array => {
293 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
294 const param_llvm_ty = try o.lowerType(param_ty);
295 const param = fg.wip.arg(it.llvm_index - 1);
296
297 const alignment = param_ty.abiAlignment(zcu).toLlvm();
298 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
299 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
300
301 if (isByRef(param_ty, zcu)) {
302 args.appendAssumeCapacity(arg_ptr);
303 } else {
304 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
305 }
306 },
307 .i32_array, .i64_array => {
308 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
309 const param = fg.wip.arg(it.llvm_index - 1);
310
311 const alignment = param_ty.abiAlignment(zcu).toLlvm();
312 const arg_ptr = try fg.buildAlloca(param.typeOfWip(&fg.wip), alignment);
313 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
314
315 if (isByRef(param_ty, zcu)) {
316 args.appendAssumeCapacity(arg_ptr);
317 } else {
318 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
319 }
320 },
321 }
322 }
323
324 fg.args = args.items;
325
326 try fg.genBody(fg.air.getMainBody(), .poi);
327}
328
329fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
179 const o = self.object;330 const o = self.object;
180 const zcu = self.object.zcu;331 const zcu = self.object.zcu;
181 const ip = &zcu.intern_pool;332 const ip = &zcu.intern_pool;
...@@ -400,8 +551,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -400,8 +551,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
400 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),551 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
401 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),552 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
402553
403 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),554 .unwrap_errunion_payload => try self.airErrUnionPayload(inst),
404 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),555 .unwrap_errunion_payload_ptr => try self.airErrUnionPayloadPtr(inst),
405 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),556 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
406 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),557 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
407 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),558 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
...@@ -644,6 +795,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -644,6 +795,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
644 const llvm_param_ty = try o.lowerType(param_ty);795 const llvm_param_ty = try o.lowerType(param_ty);
645 if (isByRef(param_ty, zcu)) {796 if (isByRef(param_ty, zcu)) {
646 const alignment = param_ty.abiAlignment(zcu).toLlvm();797 const alignment = param_ty.abiAlignment(zcu).toLlvm();
798 // We don't need to handle non-ABI-sized integer types in memory here since they are
799 // never by-ref.
647 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");800 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
648 try llvm_args.append(loaded);801 try llvm_args.append(loaded);
649 } else {802 } else {
...@@ -660,7 +813,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -660,7 +813,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
660 const alignment = param_ty.abiAlignment(zcu).toLlvm();813 const alignment = param_ty.abiAlignment(zcu).toLlvm();
661 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);814 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
662 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);815 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
663 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);816 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
664 try llvm_args.append(arg_ptr);817 try llvm_args.append(arg_ptr);
665 }818 }
666 },819 },
...@@ -672,12 +825,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -672,12 +825,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
672 const alignment = param_ty.abiAlignment(zcu).toLlvm();825 const alignment = param_ty.abiAlignment(zcu).toLlvm();
673 const param_llvm_ty = try o.lowerType(param_ty);826 const param_llvm_ty = try o.lowerType(param_ty);
674 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);827 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
675 if (isByRef(param_ty, zcu)) {828 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
676 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
677 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
678 } else {
679 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
680 }
681 try llvm_args.append(arg_ptr);829 try llvm_args.append(arg_ptr);
682 },830 },
683 .abi_sized_int => {831 .abi_sized_int => {
...@@ -694,9 +842,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -694,9 +842,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
694 // LLVM does not allow bitcasting structs so we must allocate842 // LLVM does not allow bitcasting structs so we must allocate
695 // a local, store as one type, and then load as another type.843 // a local, store as one type, and then load as another type.
696 const alignment = param_ty.abiAlignment(zcu).toLlvm();844 const alignment = param_ty.abiAlignment(zcu).toLlvm();
697 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);845 const ptr = try self.buildAlloca(int_llvm_ty, alignment);
698 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);846 try self.store(ptr, .none, llvm_arg, param_ty, .normal);
699 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");847 const loaded = try self.wip.load(.normal, int_llvm_ty, ptr, alignment, "");
700 try llvm_args.append(loaded);848 try llvm_args.append(loaded);
701 }849 }
702 },850 },
...@@ -711,19 +859,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -711,19 +859,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
711 const arg = args[it.zig_index - 1];859 const arg = args[it.zig_index - 1];
712 const param_ty = self.typeOf(arg);860 const param_ty = self.typeOf(arg);
713 const llvm_arg = try self.resolveInst(arg);861 const llvm_arg = try self.resolveInst(arg);
714 const is_by_ref = isByRef(param_ty, zcu);
715 const param_alignment = param_ty.abiAlignment(zcu);862 const param_alignment = param_ty.abiAlignment(zcu);
716 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);863 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
717 const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());864 const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());
718 if (is_by_ref) _ = try self.wip.callMemCpy(865 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
719 arg_ptr,
720 param_alignment.toLlvm(),
721 llvm_arg,
722 param_alignment.toLlvm(),
723 try o.builder.intValue(try o.lowerType(.usize), param_ty.abiSize(zcu)),
724 .normal,
725 self.disable_intrinsics,
726 ) else _ = try self.wip.store(.normal, llvm_arg, arg_ptr, param_alignment.toLlvm());
727866
728 try llvm_args.ensureUnusedCapacity(it.types_len);867 try llvm_args.ensureUnusedCapacity(it.types_len);
729 for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {868 for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {
...@@ -735,35 +874,38 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -735,35 +874,38 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
735 .float_array => |count| {874 .float_array => |count| {
736 const arg = args[it.zig_index - 1];875 const arg = args[it.zig_index - 1];
737 const arg_ty = self.typeOf(arg);876 const arg_ty = self.typeOf(arg);
738 var llvm_arg = try self.resolveInst(arg);877 const arg_val = try self.resolveInst(arg);
739 const alignment = arg_ty.abiAlignment(zcu).toLlvm();878
740 if (!isByRef(arg_ty, zcu)) {879 const arg_align = arg_ty.abiAlignment(zcu);
741 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);880
742 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);881 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
743 llvm_arg = ptr;882 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
744 }883 try self.store(ptr, .none, arg_val, arg_ty, .normal);
884 break :ptr ptr;
885 } else arg_val;
745886
746 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);887 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
747 const array_ty = try o.builder.arrayType(count, float_ty);888 const array_ty = try o.builder.arrayType(count, float_ty);
748889
749 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");890 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
750 try llvm_args.append(loaded);891 try llvm_args.append(loaded);
751 },892 },
752 .i32_array, .i64_array => |arr_len| {893 .i32_array, .i64_array => |arr_len| {
753 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;894 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
754 const arg = args[it.zig_index - 1];895 const arg = args[it.zig_index - 1];
755 const arg_ty = self.typeOf(arg);896 const arg_ty = self.typeOf(arg);
756 var llvm_arg = try self.resolveInst(arg);897 const arg_val = try self.resolveInst(arg);
757 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
758 if (!isByRef(arg_ty, zcu)) {
759 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
760 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
761 llvm_arg = ptr;
762 }
763898
764 const array_ty =899 const arg_align = arg_ty.abiAlignment(zcu);
765 try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));900
766 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");901 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
902 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
903 try self.store(ptr, .none, arg_val, arg_ty, .normal);
904 break :ptr ptr;
905 } else arg_val;
906
907 const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
908 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
767 try llvm_args.append(loaded);909 try llvm_args.append(loaded);
768 },910 },
769 };911 };
...@@ -875,8 +1017,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -875,8 +1017,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
875 return rp;1017 return rp;
876 } else {1018 } else {
877 // our by-ref status disagrees with sret so we must load.1019 // our by-ref status disagrees with sret so we must load.
878 const return_alignment = return_type.abiAlignment(zcu).toLlvm();1020 return self.load(rp, .none, return_type, .normal);
879 return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
880 }1021 }
881 }1022 }
8821023
...@@ -888,11 +1029,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -888,11 +1029,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
888 // by using our canonical type, then loading it if necessary.1029 // by using our canonical type, then loading it if necessary.
889 const alignment = return_type.abiAlignment(zcu).toLlvm();1030 const alignment = return_type.abiAlignment(zcu).toLlvm();
890 const rp = try self.buildAlloca(abi_ret_ty, alignment);1031 const rp = try self.buildAlloca(abi_ret_ty, alignment);
1032 // We don't need to handle non-ABI-sized integer types in memory here since they can only be
1033 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1034 // `llvm_ret_ty` anyway.
891 _ = try self.wip.store(.normal, call, rp, alignment);1035 _ = try self.wip.store(.normal, call, rp, alignment);
892 return if (isByRef(return_type, zcu))1036 return if (isByRef(return_type, zcu))
893 rp1037 rp
894 else1038 else
895 try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");1039 try self.load(rp, .none, return_type, .normal);
896 }1040 }
8971041
898 if (isByRef(return_type, zcu)) {1042 if (isByRef(return_type, zcu)) {
...@@ -900,6 +1044,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -900,6 +1044,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
900 // and return the allocation pointer.1044 // and return the allocation pointer.
901 const alignment = return_type.abiAlignment(zcu).toLlvm();1045 const alignment = return_type.abiAlignment(zcu).toLlvm();
902 const rp = try self.buildAlloca(llvm_ret_ty, alignment);1046 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1047 // We don't need to handle non-ABI-sized integer types here since they are never by-ref.
903 _ = try self.wip.store(.normal, call, rp, alignment);1048 _ = try self.wip.store(.normal, call, rp, alignment);
904 return rp;1049 return rp;
905 } else {1050 } else {
...@@ -969,12 +1114,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -969,12 +1114,7 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
969 return;1114 return;
970 }1115 }
9711116
972 try self.store(1117 try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
973 self.ret_ptr,
974 .none,
975 operand,
976 ret_ty,
977 );
978 _ = try self.wip.retVoid();1118 _ = try self.wip.retVoid();
979 return;1119 return;
980 }1120 }
...@@ -991,18 +1131,18 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -991,18 +1131,18 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
991 return;1131 return;
992 }1132 }
9931133
1134 const llvm_ret_ty = try o.lowerType(ret_ty);
994 const abi_ret_ty = try lowerFnRetTy(o, fn_info);1135 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
995 const operand = try self.resolveInst(un_op);1136 const operand = try self.resolveInst(un_op);
996 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;1137 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
997 const alignment = ret_ty.abiAlignment(zcu).toLlvm();1138 const ret_ty_align = ret_ty.abiAlignment(zcu);
9981139
999 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {1140 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1000 const llvm_ret_ty = operand.typeOfWip(&self.wip);1141 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
1001 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1002 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));1142 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
1003 _ = try self.wip.callMemSet(1143 _ = try self.wip.callMemSet(
1004 rp,1144 rp,
1005 alignment,1145 ret_ty_align.toLlvm(),
1006 try o.builder.intValue(.i8, 0xaa),1146 try o.builder.intValue(.i8, 0xaa),
1007 len,1147 len,
1008 .normal,1148 .normal,
...@@ -1012,27 +1152,37 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -1012,27 +1152,37 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1012 if (owner_mod.valgrind) {1152 if (owner_mod.valgrind) {
1013 try self.valgrindMarkUndef(rp, len);1153 try self.valgrindMarkUndef(rp, len);
1014 }1154 }
1015 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));1155 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1156 assert(!isByRef(ret_ty, zcu));
1157 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure
1158 // we load it from memory correctly.
1159 const loaded = try self.load(rp, .none, ret_ty, .normal);
1160 _ = try self.wip.ret(loaded);
1161 } else {
1162 const loaded = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1163 _ = try self.wip.ret(loaded);
1164 }
1016 return;1165 return;
1017 }1166 }
10181167
1019 if (isByRef(ret_ty, zcu)) {1168 if (isByRef(ret_ty, zcu)) {
1020 // operand is a pointer however self.ret_ptr is null so that means1169 // operand is a pointer however self.ret_ptr is null so that means we need to return a value.
1021 // we need to return a value.1170 // No need to handle non-ABI-sized integer types in memory here since they are never by-ref.
1022 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));1171 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, ret_ty_align.toLlvm(), ""));
1023 return;1172 return;
1024 }1173 }
10251174
1026 const llvm_ret_ty = operand.typeOfWip(&self.wip);
1027 if (abi_ret_ty == llvm_ret_ty) {1175 if (abi_ret_ty == llvm_ret_ty) {
1028 _ = try self.wip.ret(operand);1176 _ = try self.wip.ret(operand);
1029 return;1177 } else {
1178 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
1179 try self.store(rp, .none, operand, ret_ty, .normal);
1180 // No need to handle non-ABI-sized integer types in memory here since they can only be
1181 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1182 // `llvm_ret_ty` anyway.
1183 const ret_val = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1184 _ = try self.wip.ret(ret_val);
1030 }1185 }
1031
1032 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1033 _ = try self.wip.store(.normal, operand, rp, alignment);
1034 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1035 return;
1036}1186}
10371187
1038fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {1188fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
...@@ -1043,26 +1193,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {...@@ -1043,26 +1193,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1043 const ptr_ty = self.typeOf(un_op);1193 const ptr_ty = self.typeOf(un_op);
1044 const ret_ty = ptr_ty.childType(zcu);1194 const ret_ty = ptr_ty.childType(zcu);
1045 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;1195 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1046 if (!ret_ty.hasRuntimeBits(zcu)) {1196 if (!ret_ty.hasRuntimeBits(zcu) or self.ret_ptr != .none) {
1047 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1048 // Functions with an empty error set are emitted with an error code
1049 // return type and return zero so they can be function pointers coerced
1050 // to functions that return anyerror.
1051 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1052 } else {
1053 _ = try self.wip.retVoid();
1054 }
1055 return;
1056 }
1057 if (self.ret_ptr != .none) {
1058 _ = try self.wip.retVoid();1197 _ = try self.wip.retVoid();
1059 return;1198 return;
1060 }1199 }
1061 const ptr = try self.resolveInst(un_op);1200 const ptr = try self.resolveInst(un_op);
1201 const llvm_ret_ty = try o.lowerType(ret_ty);
1062 const abi_ret_ty = try lowerFnRetTy(o, fn_info);1202 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1063 const alignment = ret_ty.abiAlignment(zcu).toLlvm();1203 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1064 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));1204 assert(!isByRef(ret_ty, zcu));
1065 return;1205 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure we
1206 // load it from memory correctly.
1207 const loaded = try self.load(ptr, .none, ret_ty, .normal);
1208 _ = try self.wip.ret(loaded);
1209 } else {
1210 const ret_ty_align = ret_ty.abiAlignment(zcu);
1211 const loaded = try self.wip.load(.normal, abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1212 _ = try self.wip.ret(loaded);
1213 }
1066}1214}
10671215
1068fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {1216fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -1089,7 +1237,7 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -1089,7 +1237,7 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
1089 return if (isByRef(va_list_ty, zcu))1237 return if (isByRef(va_list_ty, zcu))
1090 dest_list1238 dest_list
1091 else1239 else
1092 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");1240 try self.load(dest_list, .none, va_list_ty, .normal);
1093}1241}
10941242
1095fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {1243fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -1113,7 +1261,7 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -1113,7 +1261,7 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
1113 return if (isByRef(va_list_ty, zcu))1261 return if (isByRef(va_list_ty, zcu))
1114 dest_list1262 dest_list
1115 else1263 else
1116 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");1264 try self.load(dest_list, .none, va_list_ty, .normal);
1117}1265}
11181266
1119fn airCmp(1267fn airCmp(
...@@ -1147,13 +1295,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -1147,13 +1295,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
1147 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;1295 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1148 const operand = try self.resolveInst(un_op);1296 const operand = try self.resolveInst(un_op);
1149 const errors_len_ptr = try o.getErrorsLen();1297 const errors_len_ptr = try o.getErrorsLen();
1150 const errors_len_val = try self.wip.load(1298 const errors_len_val = try self.load(errors_len_ptr.toValue(&o.builder), .none, .anyerror, .normal);
1151 .normal,
1152 try o.errorIntType(),
1153 errors_len_ptr.toValue(&o.builder),
1154 Type.errorAbiAlignment(o.zcu).toLlvm(),
1155 "",
1156 );
1157 return self.wip.icmp(.ule, operand, errors_len_val, "");1299 return self.wip.icmp(.ule, operand, errors_len_val, "");
1158}1300}
11591301
...@@ -1630,32 +1772,23 @@ fn lowerTry(...@@ -1630,32 +1772,23 @@ fn lowerTry(
1630 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);1772 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1631 const error_type = try o.errorIntType();1773 const error_type = try o.errorIntType();
16321774
1633 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{1775 const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
1634 operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
1635 operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
1636 } else .{ .none, .none };
16371776
1638 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {1777 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
1639 const loaded = loaded: {1778 const loaded = loaded: {
1640 const access_kind: Builder.MemoryAccessKind =1779 if (payload_has_bits) {
1641 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;1780 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
16421781 } else if (!operand_is_ptr) {
1643 if (!payload_has_bits) {1782 break :loaded err_union;
1644 break :loaded if (operand_is_ptr)
1645 try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
1646 else
1647 err_union;
1648 }1783 }
16491784
1650 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1651 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);1785 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1652 const err_field_ptr = try fg.ptraddConst(err_union, offset);1786 const err_field_ptr = try fg.ptraddConst(err_union, offset);
1653 break :loaded try fg.wip.load(1787 break :loaded try fg.load(
1654 if (operand_is_ptr) access_kind else .normal,
1655 error_type,
1656 err_field_ptr,1788 err_field_ptr,
1657 err_set_align.toLlvm(),1789 operand_align.offset(offset),
1658 "",1790 .anyerror,
1791 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1659 );1792 );
1660 };1793 };
1661 const zero = try o.builder.intValue(error_type, 0);1794 const zero = try o.builder.intValue(error_type, 0);
...@@ -1672,15 +1805,18 @@ fn lowerTry(...@@ -1672,15 +1805,18 @@ fn lowerTry(
1672 fg.wip.cursor = .{ .block = continue_block };1805 fg.wip.cursor = .{ .block = continue_block };
1673 }1806 }
1674 if (is_unused) return .none;1807 if (is_unused) return .none;
1675 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;1808
1676 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits1809 if (!operand_is_ptr) {
1677 const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));1810 assert(payload_has_bits); // otherwise the result should be comptime-known
1811 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1812 }
1813
1814 const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
1815 const payload_ptr = try fg.ptraddConst(err_union, offset);
1678 if (operand_is_ptr) {1816 if (operand_is_ptr) {
1679 return payload_ptr;1817 return payload_ptr;
1680 } else if (isByRef(payload_ty, zcu)) {
1681 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1682 } else {1818 } else {
1683 return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), "");1819 return fg.load(payload_ptr, operand_align.offset(offset), payload_ty, .normal);
1684 }1820 }
1685}1821}
16861822
...@@ -2144,11 +2280,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -2144,11 +2280,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2144 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));2280 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2145 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;2281 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2146 self.maybeMarkAllowZeroAccess(slice_info);2282 self.maybeMarkAllowZeroAccess(slice_info);
2147 if (isByRef(elem_ty, zcu)) {2283 return self.load(ptr, elem_align, elem_ty, access_kind);
2148 return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
2149 } else {
2150 return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
2151 }
2152}2284}
21532285
2154fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {2286fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -2173,12 +2305,7 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -2173,12 +2305,7 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2173 const elem_ty = array_ty.childType(zcu);2305 const elem_ty = array_ty.childType(zcu);
2174 if (isByRef(array_ty, zcu)) {2306 if (isByRef(array_ty, zcu)) {
2175 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));2307 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2176 if (isByRef(elem_ty, zcu)) {2308 return self.load(elem_ptr, .none, elem_ty, .normal);
2177 const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2178 return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
2179 } else {
2180 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2181 }
2182 }2309 }
21832310
2184 // This branch can be reached for vectors, which are always by-value.2311 // This branch can be reached for vectors, which are always by-value.
...@@ -2197,8 +2324,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -2197,8 +2324,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
21972324
2198 return self.load(2325 return self.load(
2199 try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),2326 try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
2327 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)),
2200 elem_ty,2328 elem_ty,
2201 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(),
2202 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,2329 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2203 );2330 );
2204}2331}
...@@ -2294,11 +2421,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2294,11 +2421,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2294 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);2421 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2295 const field_ptr_align = struct_ptr_align.offset(offset);2422 const field_ptr_align = struct_ptr_align.offset(offset);
22962423
2297 if (isByRef(field_ty, zcu)) {2424 return self.load(field_ptr, field_ptr_align, field_ty, .normal);
2298 return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2299 } else {
2300 return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
2301 }
2302}2425}
23032426
2304fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {2427fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -2439,8 +2562,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er...@@ -2439,8 +2562,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
2439 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.2562 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
24402563
2441 const alignment = operand_ty.abiAlignment(zcu).toLlvm();2564 const alignment = operand_ty.abiAlignment(zcu).toLlvm();
2442 const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);2565 const alloca = try self.buildAlloca(try o.lowerType(operand_ty), alignment);
2443 _ = try self.wip.store(.normal, operand, alloca, alignment);2566 try self.store(alloca, .none, operand, operand_ty, .normal);
2444 _ = try self.wip.callIntrinsic(2567 _ = try self.wip.callIntrinsic(
2445 .normal,2568 .normal,
2446 .none,2569 .none,
...@@ -2609,8 +2732,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2609,8 +2732,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2609 } else {2732 } else {
2610 const alignment = arg_ty.abiAlignment(zcu).toLlvm();2733 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2611 const arg_llvm_ty = try o.lowerType(arg_ty);2734 const arg_llvm_ty = try o.lowerType(arg_ty);
2612 const load_inst =2735 const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2613 try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2614 llvm_param_values[llvm_param_i] = load_inst;2736 llvm_param_values[llvm_param_i] = load_inst;
2615 llvm_param_types[llvm_param_i] = arg_llvm_ty;2737 llvm_param_types[llvm_param_i] = arg_llvm_ty;
2616 }2738 }
...@@ -2621,7 +2743,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2621,7 +2743,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2621 } else {2743 } else {
2622 const alignment = arg_ty.abiAlignment(zcu).toLlvm();2744 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2623 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);2745 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2624 _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);2746 try self.store(arg_ptr, .none, arg_llvm_value, arg_ty, .normal);
2625 llvm_param_values[llvm_param_i] = arg_ptr;2747 llvm_param_values[llvm_param_i] = arg_ptr;
2626 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);2748 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
2627 }2749 }
...@@ -2668,14 +2790,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2668,14 +2790,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2668 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];2790 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2669 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);2791 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
2670 } else {2792 } else {
2671 const alignment = rw_ty.abiAlignment(zcu).toLlvm();2793 const access_kind: Builder.MemoryAccessKind = if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2672 const loaded = try self.wip.load(2794 const loaded = try self.load(llvm_rw_vals[output.index], .none, rw_ty.childType(zcu), access_kind);
2673 if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2674 llvm_elem_ty,
2675 llvm_rw_vals[output.index],
2676 alignment,
2677 "",
2678 );
2679 llvm_param_values[llvm_param_i] = loaded;2795 llvm_param_values[llvm_param_i] = loaded;
2680 llvm_param_types[llvm_param_i] = llvm_elem_ty;2796 llvm_param_types[llvm_param_i] = llvm_elem_ty;
2681 }2797 }
...@@ -2835,12 +2951,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2835,12 +2951,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2835 if (output != .none) {2951 if (output != .none) {
2836 const output_ptr = try self.resolveInst(output);2952 const output_ptr = try self.resolveInst(output);
2837 const output_ptr_ty = self.typeOf(output);2953 const output_ptr_ty = self.typeOf(output);
2838 const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();2954 try self.store(
2839 _ = try self.wip.store(
2840 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2841 output_value,
2842 output_ptr,2955 output_ptr,
2843 alignment,2956 output_ptr_ty.ptrAlignment(zcu),
2957 output_value,
2958 output_ptr_ty.childType(zcu),
2959 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2844 );2960 );
2845 } else {2961 } else {
2846 ret_val = output_value;2962 ret_val = output_value;
...@@ -2863,7 +2979,6 @@ fn airIsNonNull(...@@ -2863,7 +2979,6 @@ fn airIsNonNull(
2863 const operand = try self.resolveInst(un_op);2979 const operand = try self.resolveInst(un_op);
2864 const operand_ty = self.typeOf(un_op);2980 const operand_ty = self.typeOf(un_op);
2865 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;2981 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2866 const optional_llvm_ty = try o.lowerType(optional_ty);
2867 const payload_ty = optional_ty.optionalChild(zcu);2982 const payload_ty = optional_ty.optionalChild(zcu);
28682983
2869 const access_kind: Builder.MemoryAccessKind =2984 const access_kind: Builder.MemoryAccessKind =
...@@ -2873,7 +2988,7 @@ fn airIsNonNull(...@@ -2873,7 +2988,7 @@ fn airIsNonNull(
28732988
2874 if (optional_ty.optionalReprIsPayload(zcu)) {2989 if (optional_ty.optionalReprIsPayload(zcu)) {
2875 const loaded = if (operand_is_ptr)2990 const loaded = if (operand_is_ptr)
2876 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")2991 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2877 else2992 else
2878 operand;2993 operand;
2879 if (payload_ty.isSlice(zcu)) {2994 if (payload_ty.isSlice(zcu)) {
...@@ -2884,14 +2999,14 @@ fn airIsNonNull(...@@ -2884,14 +2999,14 @@ fn airIsNonNull(
2884 ));2999 ));
2885 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");3000 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
2886 }3001 }
2887 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");3002 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty)), "");
2888 }3003 }
28893004
2890 comptime assert(optional_layout_version == 3);3005 comptime assert(optional_layout_version == 3);
28913006
2892 if (!payload_ty.hasRuntimeBits(zcu)) {3007 if (!payload_ty.hasRuntimeBits(zcu)) {
2893 const loaded = if (operand_is_ptr)3008 const loaded = if (operand_is_ptr)
2894 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")3009 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2895 else3010 else
2896 operand;3011 operand;
2897 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");3012 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
...@@ -2932,7 +3047,7 @@ fn airIsErr(...@@ -2932,7 +3047,7 @@ fn airIsErr(
29323047
2933 if (!payload_ty.hasRuntimeBits(zcu)) {3048 if (!payload_ty.hasRuntimeBits(zcu)) {
2934 const loaded = if (operand_is_ptr)3049 const loaded = if (operand_is_ptr)
2935 try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")3050 try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
2936 else3051 else
2937 operand;3052 operand;
2938 return self.wip.icmp(cond, loaded, zero, "");3053 return self.wip.icmp(cond, loaded, zero, "");
...@@ -2944,7 +3059,7 @@ fn airIsErr(...@@ -2944,7 +3059,7 @@ fn airIsErr(
2944 else3059 else
2945 .none;3060 .none;
2946 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3061 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
2947 const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");3062 const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
2948 return self.wip.icmp(cond, loaded, zero, "");3063 return self.wip.icmp(cond, loaded, zero, "");
2949}3064}
29503065
...@@ -2966,7 +3081,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2966,7 +3081,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2966 const optional_ptr_ty = self.typeOf(ty_op.operand);3081 const optional_ptr_ty = self.typeOf(ty_op.operand);
2967 const optional_ty = optional_ptr_ty.childType(zcu);3082 const optional_ty = optional_ptr_ty.childType(zcu);
2968 const payload_ty = optional_ty.optionalChild(zcu);3083 const payload_ty = optional_ty.optionalChild(zcu);
2969 const non_null_bit = try o.builder.intValue(.i8, 1);
29703084
2971 const access_kind: Builder.MemoryAccessKind =3085 const access_kind: Builder.MemoryAccessKind =
2972 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3086 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -2976,7 +3090,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2976,7 +3090,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
29763090
2977 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.3091 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
2978 // Default alignment store because align of the non null bit is 1 anyway.3092 // Default alignment store because align of the non null bit is 1 anyway.
2979 _ = try self.wip.store(access_kind, non_null_bit, operand, .default);3093 try self.store(operand, .@"1", .true, .bool, access_kind);
2980 return operand;3094 return operand;
2981 }3095 }
2982 if (optional_ty.optionalReprIsPayload(zcu)) {3096 if (optional_ty.optionalReprIsPayload(zcu)) {
...@@ -2992,7 +3106,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2992,7 +3106,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2992 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));3106 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
29933107
2994 // Default alignment store because align of the non null bit is 1 anyway.3108 // Default alignment store because align of the non null bit is 1 anyway.
2995 _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);3109 try self.store(non_null_ptr, .@"1", .true, .bool, access_kind);
29963110
2997 // Then return the payload pointer (only if it's used).3111 // Then return the payload pointer (only if it's used).
2998 if (self.liveness.isUnused(inst)) return .none;3112 if (self.liveness.isUnused(inst)) return .none;
...@@ -3016,31 +3130,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -3016,31 +3130,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
3016 return self.optPayloadHandle(operand, optional_ty, false);3130 return self.optPayloadHandle(operand, optional_ty, false);
3017}3131}
30183132
3019fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value {3133fn airErrUnionPayload(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3020 const o = self.object;3134 const o = fg.object;
3021 const zcu = o.zcu;3135 const zcu = o.zcu;
3022 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3136 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3023 const operand = try self.resolveInst(ty_op.operand);3137 const operand = try fg.resolveInst(ty_op.operand);
3024 const operand_ty = self.typeOf(ty_op.operand);3138 const err_union_ty = fg.typeOf(ty_op.operand);
3025 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;3139 const payload_ty = fg.typeOfIndex(inst);
3026 const result_ty = self.typeOfIndex(inst);
3027 const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
30283140
3029 if (!payload_ty.hasRuntimeBits(zcu)) {3141 assert(payload_ty.hasRuntimeBits(zcu));
3030 return if (operand_is_ptr) operand else .none;
3031 }
3032 const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3033 if (operand_is_ptr) {
3034 return payload_ptr;
3035 }
3036 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits3142 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3037 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();3143
3038 if (isByRef(payload_ty, zcu)) {3144 const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
3039 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);3145 const payload_ptr = try fg.ptraddConst(operand, payload_offset);
3040 } else {3146 return fg.load(payload_ptr, err_union_ty.abiAlignment(zcu).offset(payload_offset), payload_ty, .normal);
3041 const payload_llvm_ty = try o.lowerType(payload_ty);3147}
3042 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");3148
3043 }3149fn airErrUnionPayloadPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3150 const o = fg.object;
3151 const zcu = o.zcu;
3152 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3153 const operand = try fg.resolveInst(ty_op.operand);
3154 const payload_ty = fg.typeOfIndex(inst).childType(zcu);
3155 return fg.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3044}3156}
30453157
3046fn airErrUnionErr(3158fn airErrUnionErr(
...@@ -3053,40 +3165,28 @@ fn airErrUnionErr(...@@ -3053,40 +3165,28 @@ fn airErrUnionErr(
3053 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3165 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3054 const operand = try self.resolveInst(ty_op.operand);3166 const operand = try self.resolveInst(ty_op.operand);
3055 const operand_ty = self.typeOf(ty_op.operand);3167 const operand_ty = self.typeOf(ty_op.operand);
3056 const error_type = try o.errorIntType();
3057 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;3168 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3058 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
3059 if (operand_is_ptr) {
3060 return operand;
3061 } else {
3062 return o.builder.intValue(error_type, 0);
3063 }
3064 }
30653169
3066 const access_kind: Builder.MemoryAccessKind =3170 const access_kind: Builder.MemoryAccessKind =
3067 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3171 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
30683172
3069 const payload_ty = err_union_ty.errorUnionPayload(zcu);3173 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3070 if (!payload_ty.hasRuntimeBits(zcu)) {
3071 if (!operand_is_ptr) return operand;
3072
3073 self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
30743174
3075 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");3175 if (payload_ty.hasRuntimeBits(zcu)) {
3176 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3177 } else if (!operand_is_ptr) {
3178 return operand;
3076 }3179 }
30773180
3078 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3079
3080 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));3181 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
30813182
3082 const err_align: InternPool.Alignment = a: {3183 const ptr_align = if (operand_is_ptr) operand_ty.ptrAlignment(zcu) else err_union_ty.abiAlignment(zcu);
3083 const err_abi_align = Type.anyerror.abiAlignment(zcu);
3084 if (!operand_is_ptr) break :a err_abi_align;
3085 break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3086 };
30873184
3088 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3185 const err_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
3089 return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");3186 const err_align = ptr_align.offset(err_offset);
3187 const err_ptr = try self.ptraddConst(operand, err_offset);
3188
3189 return self.load(err_ptr, err_align, .anyerror, access_kind);
3090}3190}
30913191
3092fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {3192fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -3107,10 +3207,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -3107,10 +3207,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3107 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));3207 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
31083208
3109 {3209 {
3110 const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3111 // First set the non-error value.3210 // First set the non-error value.
3112 const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3211 const error_off = codegen.errUnionErrorOffset(payload_ty, zcu);
3113 _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);3212 const error_ptr = try self.ptraddConst(operand, error_off);
3213 try self.store(error_ptr, err_union_ptr_align.offset(error_off), non_error_val, .anyerror, access_kind);
3114 }3214 }
31153215
3116 // Then return the payload pointer (only if it is used).3216 // Then return the payload pointer (only if it is used).
...@@ -3142,7 +3242,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er...@@ -3142,7 +3242,7 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
3142 const field_offset = struct_ty.structFieldOffset(field_index, zcu);3242 const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3143 const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);3243 const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);
3144 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);3244 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3145 return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal);3245 return self.load(field_ptr, field_align, field_ty, .normal);
3146}3246}
31473247
3148/// As an optimization, we want to avoid unnecessary copies of3248/// As an optimization, we want to avoid unnecessary copies of
...@@ -3174,7 +3274,6 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator....@@ -3174,7 +3274,6 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3174 const inst = body_tail[0];3274 const inst = body_tail[0];
3175 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3275 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3176 const payload_ty = self.typeOf(ty_op.operand);3276 const payload_ty = self.typeOf(ty_op.operand);
3177 const non_null_bit = try o.builder.intValue(.i8, 1);
3178 comptime assert(optional_layout_version == 3);3277 comptime assert(optional_layout_version == 3);
3179 assert(payload_ty.hasRuntimeBits(zcu));3278 assert(payload_ty.hasRuntimeBits(zcu));
3180 const operand = try self.resolveInst(ty_op.operand);3279 const operand = try self.resolveInst(ty_op.operand);
...@@ -3191,15 +3290,12 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator....@@ -3191,15 +3290,12 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3191 };3290 };
31923291
3193 const payload_ptr = optional_ptr; // payload always at offset 03292 const payload_ptr = optional_ptr; // payload always at offset 0
3194 try self.store(3293 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3195 payload_ptr,3294
3196 .none,
3197 operand,
3198 payload_ty,
3199 );
3200 // Non-null bit immediately after payload (no padding because the bit has alignment 1).3295 // Non-null bit immediately after payload (no padding because the bit has alignment 1).
3201 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));3296 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3202 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);3297 try self.store(non_null_ptr, .none, .true, .bool, .normal);
3298
3203 return optional_ptr;3299 return optional_ptr;
3204}3300}
32053301
...@@ -3225,15 +3321,11 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All...@@ -3225,15 +3321,11 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All
3225 };3321 };
32263322
3227 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3323 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3228 const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm();3324 try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
3229 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);3325
3230 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));3326 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3231 try self.store(3327 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3232 payload_ptr,3328
3233 .none,
3234 operand,
3235 payload_ty,
3236 );
3237 return result_ptr;3329 return result_ptr;
3238}3330}
32393331
...@@ -3258,11 +3350,12 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocat...@@ -3258,11 +3350,12 @@ fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocat
3258 };3350 };
32593351
3260 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3352 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3261 const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();3353 try self.store(err_ptr, .none, operand, .anyerror, .normal);
3262 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);3354
3263 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));3355 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3264 // TODO store undef to payload_ptr3356 // TODO store undef to payload_ptr
3265 _ = payload_ptr;3357 _ = payload_ptr;
3358
3266 return result_ptr;3359 return result_ptr;
3267}3360}
32683361
...@@ -3723,19 +3816,21 @@ fn airOverflow(...@@ -3723,19 +3816,21 @@ fn airOverflow(
3723 const result_val = try self.wip.extractValue(results, &.{0}, "");3816 const result_val = try self.wip.extractValue(results, &.{0}, "");
3724 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");3817 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
37253818
3726 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();3819 const result_alignment = inst_ty.abiAlignment(zcu);
3727 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);3820 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment.toLlvm());
37283821
3729 {3822 {
3730 // Store to 'result: IntType' field3823 // Store to 'result: IntType' field
3731 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));3824 const field_off = inst_ty.structFieldOffset(0, zcu);
3732 _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());3825 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3826 try self.store(field_ptr, result_alignment.offset(field_off), result_val, lhs_ty, .normal);
3733 }3827 }
37343828
3735 {3829 {
3736 // Store to 'overflow: u1' field3830 // Store to 'overflow: u1' field
3737 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));3831 const field_off = inst_ty.structFieldOffset(1, zcu);
3738 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));3832 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3833 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, inst_ty.fieldType(1, zcu), .normal);
3739 }3834 }
37403835
3741 return alloca_inst;3836 return alloca_inst;
...@@ -4064,19 +4159,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -4064,19 +4159,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
40644159
4065 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");4160 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
40664161
4067 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();4162 const result_alignment = dest_ty.abiAlignment(zcu);
4068 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);4163 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment.toLlvm());
40694164
4070 {4165 {
4071 // Store to 'result: IntType' field4166 // Store to 'result: IntType' field
4072 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));4167 const field_off = dest_ty.structFieldOffset(0, zcu);
4073 _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());4168 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4169 try self.store(field_ptr, result_alignment.offset(field_off), result, lhs_ty, .normal);
4074 }4170 }
40754171
4076 {4172 {
4077 // Store to 'overflow: u1' field4173 // Store to 'overflow: u1' field
4078 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));4174 const field_off = dest_ty.structFieldOffset(1, zcu);
4079 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));4175 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4176 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, dest_ty.fieldType(1, zcu), .normal);
4080 }4177 }
40814178
4082 return alloca_inst;4179 return alloca_inst;
...@@ -4264,7 +4361,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4264,7 +4361,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4264 .none,4361 .none,
4265 .abs,4362 .abs,
4266 &.{try o.lowerType(operand_ty)},4363 &.{try o.lowerType(operand_ty)},
4267 &.{ operand, try o.builder.intValue(.i1, 0) },4364 &.{ operand, .false },
4268 "",4365 "",
4269 ),4366 ),
4270 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),4367 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
...@@ -4483,7 +4580,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4483,7 +4580,8 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4483 }4580 }
44844581
4485 if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {4582 if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {
4486 return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");4583 assert(inst_ty.bitSize(zcu) == operand_ty.bitSize(zcu));
4584 return operand;
4487 }4585 }
44884586
4489 const operand_scalar_ty = operand_ty.scalarType(zcu);4587 const operand_scalar_ty = operand_ty.scalarType(zcu);
...@@ -4498,11 +4596,11 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4498,11 +4596,11 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4498 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {4596 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4499 const elem_ty = operand_scalar_ty;4597 const elem_ty = operand_scalar_ty;
4500 assert(result_is_ref); // arrays are always by-ref provided they have runtime bits4598 assert(result_is_ref); // arrays are always by-ref provided they have runtime bits
4501 const alignment = inst_ty.abiAlignment(zcu).toLlvm();4599 const alignment = inst_ty.abiAlignment(zcu);
4502 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);4600 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm());
4503 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;4601 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4504 if (bitcast_ok) {4602 if (bitcast_ok) {
4505 _ = try self.wip.store(.normal, operand, array_ptr, alignment);4603 try self.store(array_ptr, alignment, operand, operand_ty, .normal);
4506 } else {4604 } else {
4507 // If the ABI size of the element type is not evenly divisible by size in bits;4605 // If the ABI size of the element type is not evenly divisible by size in bits;
4508 // a simple bitcast will not work, and we fall back to extractelement.4606 // a simple bitcast will not work, and we fall back to extractelement.
...@@ -4512,7 +4610,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4512,7 +4610,7 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4512 while (i < vector_len) : (i += 1) {4610 while (i < vector_len) : (i += 1) {
4513 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);4611 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
4514 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");4612 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4515 _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);4613 try self.store(arr_elem_ptr, .none, vec_elem, elem_ty, .normal);
4516 }4614 }
4517 }4615 }
4518 return array_ptr;4616 return array_ptr;
...@@ -4525,19 +4623,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4525,19 +4623,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4525 if (bitcast_ok) {4623 if (bitcast_ok) {
4526 // The array is aligned to the element's alignment, while the vector might have a completely4624 // The array is aligned to the element's alignment, while the vector might have a completely
4527 // different alignment. This means we need to enforce the alignment of this load.4625 // different alignment. This means we need to enforce the alignment of this load.
4528 const alignment = elem_ty.abiAlignment(zcu).toLlvm();4626 return self.load(operand, elem_ty.abiAlignment(zcu), inst_ty, .normal);
4529 return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
4530 } else {4627 } else {
4531 // If the ABI size of the element type is not evenly divisible by size in bits;4628 // If the ABI size of the element type is not evenly divisible by size in bits;
4532 // a simple bitcast will not work, and we fall back to extractelement.4629 // a simple bitcast will not work, and we fall back to extractelement.
4533 const elem_llvm_ty = try o.lowerType(elem_ty);
4534 const elem_size = elem_ty.abiSize(zcu);4630 const elem_size = elem_ty.abiSize(zcu);
4535 const vector_len = operand_ty.arrayLen(zcu);4631 const vector_len = operand_ty.arrayLen(zcu);
4536 var vector = try o.builder.poisonValue(llvm_vector_ty);4632 var vector = try o.builder.poisonValue(llvm_vector_ty);
4537 var i: u64 = 0;4633 var i: u64 = 0;
4538 while (i < vector_len) : (i += 1) {4634 while (i < vector_len) : (i += 1) {
4539 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);4635 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4540 const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");4636 const arr_elem = try self.load(arr_elem_ptr, .none, elem_ty, .normal);
4541 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");4637 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
4542 }4638 }
4543 return vector;4639 return vector;
...@@ -4545,14 +4641,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4545,14 +4641,17 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4545 }4641 }
45464642
4547 if (operand_is_ref) {4643 if (operand_is_ref) {
4548 const alignment = operand_ty.abiAlignment(zcu).toLlvm();4644 return self.load(operand, operand_ty.abiAlignment(zcu), inst_ty, .normal);
4549 return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
4550 }4645 }
45514646
4552 if (result_is_ref) {4647 if (result_is_ref) {
4553 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();4648 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu));
4554 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);4649 const llvm_alloc_ty = if (operand_ty.abiSize(zcu) > inst_ty.abiSize(zcu))
4555 _ = try self.wip.store(.normal, operand, result_ptr, alignment);4650 try o.lowerType(operand_ty)
4651 else
4652 llvm_dest_ty;
4653 const result_ptr = try self.buildAlloca(llvm_alloc_ty, alignment.toLlvm());
4654 try self.store(result_ptr, alignment, operand, operand_ty, .normal);
4556 return result_ptr;4655 return result_ptr;
4557 }4656 }
45584657
...@@ -4563,10 +4662,10 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty...@@ -4563,10 +4662,10 @@ fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Ty
4563 // Both our operand and our result are values, not pointers,4662 // Both our operand and our result are values, not pointers,
4564 // but LLVM won't let us bitcast struct values or vectors with padding bits.4663 // but LLVM won't let us bitcast struct values or vectors with padding bits.
4565 // Therefore, we store operand to alloca, then load for result.4664 // Therefore, we store operand to alloca, then load for result.
4566 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();4665 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu));
4567 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);4666 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment.toLlvm());
4568 _ = try self.wip.store(.normal, operand, result_ptr, alignment);4667 try self.store(result_ptr, alignment, operand, operand_ty, .normal);
4569 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");4668 return self.load(result_ptr, alignment, inst_ty, .normal);
4570 }4669 }
45714670
4572 return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");4671 return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
...@@ -4629,8 +4728,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4629,8 +4728,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4629 );4728 );
4630 } else if (mod.optimize_mode == .Debug) {4729 } else if (mod.optimize_mode == .Debug) {
4631 const alignment = inst_ty.abiAlignment(zcu).toLlvm();4730 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4632 const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);4731 const alloca = try self.buildAlloca(try o.lowerType(inst_ty), alignment);
4633 _ = try self.wip.store(.normal, arg_val, alloca, alignment);4732 try self.store(alloca, .none, arg_val, inst_ty, .normal);
4634 _ = try self.wip.callIntrinsic(4733 _ = try self.wip.callIntrinsic(
4635 .normal,4734 .normal,
4636 .none,4735 .none,
...@@ -4689,28 +4788,56 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4689,28 +4788,56 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4689 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());4788 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
4690}4789}
46914790
4692/// Use this instead of builder.buildAlloca, because this function makes sure to4791/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
4693/// put the alloca instruction at the top of the function!
4694fn buildAlloca(4792fn buildAlloca(
4695 self: *FuncGen,4793 fg: *FuncGen,
4696 llvm_ty: Builder.Type,4794 llvm_ty: Builder.Type,
4697 alignment: Builder.Alignment,4795 alignment: Builder.Alignment,
4698) Allocator.Error!Builder.Value {4796) Allocator.Error!Builder.Value {
4699 const target = self.object.zcu.getTarget();4797 const wip = &fg.wip;
4700 return buildAllocaInner(&self.wip, llvm_ty, alignment, target);4798
4799 const alloca = blk: {
4800 const prev_cursor = wip.cursor;
4801 const prev_debug_location = wip.debug_location;
4802 defer {
4803 wip.cursor = prev_cursor;
4804 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
4805 wip.debug_location = prev_debug_location;
4806 }
4807
4808 wip.cursor = .{ .block = .entry };
4809 wip.debug_location = .no_location;
4810 const address_space = llvmAllocaAddressSpace(fg.object.zcu.getTarget());
4811 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
4812 };
4813
4814 // The pointer returned from this function should have the generic address space,
4815 // if this isn't the case then cast it to the generic address space.
4816 return fg.wip.conv(.unneeded, alloca, .ptr, "");
4701}4817}
47024818
4703fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {4819fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4704 const o = self.object;4820 const o = fg.object;
4705 const zcu = o.zcu;4821 const zcu = o.zcu;
4706 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4822 const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4707 const dest_ptr = try self.resolveInst(bin_op.lhs);4823 const ptr = try fg.resolveInst(bin_op.lhs);
4708 const ptr_ty = self.typeOf(bin_op.lhs);4824 const ptr_ty = fg.typeOf(bin_op.lhs);
4709 const operand_ty = ptr_ty.childType(zcu);4825 const ptr_info = ptr_ty.ptrInfo(zcu);
4826 const ptr_alignment = ptr_ty.ptrAlignment(zcu);
4827
4828 const elem_ty = fg.typeOf(bin_op.rhs);
4829 assert(elem_ty.hasRuntimeBits(zcu));
4830
4831 fg.maybeMarkAllowZeroAccess(ptr_info);
4832
4833 const access_kind: Builder.MemoryAccessKind = switch (ptr_info.flags.is_volatile) {
4834 true => .@"volatile",
4835 false => .normal,
4836 };
47104837
4711 const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;4838 const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
4712 if (val_is_undef and !self.needMemsetWorkaround(operand_ty.abiSize(zcu))) {4839 if (val_is_undef and !fg.needMemsetWorkaround(elem_ty.abiSize(zcu))) {
4713 const owner_mod = self.ownerModule();4840 const owner_mod = fg.ownerModule();
47144841
4715 // Even if safety is disabled, we still emit a memset to undefined since it conveys4842 // Even if safety is disabled, we still emit a memset to undefined since it conveys
4716 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference4843 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
...@@ -4725,7 +4852,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4725,7 +4852,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4725 return .none;4852 return .none;
4726 }4853 }
47274854
4728 const ptr_info = ptr_ty.ptrInfo(zcu);
4729 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);4855 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
4730 if (needs_bitmask) {4856 if (needs_bitmask) {
4731 // TODO: only some bits are to be undef, we cannot write with a simple memset.4857 // TODO: only some bits are to be undef, we cannot write with a simple memset.
...@@ -4734,27 +4860,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4734,27 +4860,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4734 return .none;4860 return .none;
4735 }4861 }
47364862
4737 self.maybeMarkAllowZeroAccess(ptr_info);4863 const len = try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu));
47384864 _ = try fg.wip.callMemSet(
4739 const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu));4865 ptr,
4740 _ = try self.wip.callMemSet(4866 ptr_alignment.toLlvm(),
4741 dest_ptr,
4742 ptr_ty.ptrAlignment(zcu).toLlvm(),
4743 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),4867 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
4744 len,4868 len,
4745 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,4869 access_kind,
4746 self.disable_intrinsics,4870 fg.disable_intrinsics,
4747 );4871 );
4748 if (safety and owner_mod.valgrind) {4872 if (safety and owner_mod.valgrind) {
4749 try self.valgrindMarkUndef(dest_ptr, len);4873 try fg.valgrindMarkUndef(ptr, len);
4750 }4874 }
4751 return .none;4875 return .none;
4752 }4876 }
47534877
4754 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));4878 const elem = try fg.resolveInst(bin_op.rhs);
4879
4880 if (ptr_info.flags.vector_index != .none) {
4881 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4882 const vec_ty = try fg.pt.vectorType(.{
4883 .len = ptr_info.packed_offset.host_size,
4884 .child = elem_ty.toIntern(),
4885 });
4886
4887 const loaded_vector = try fg.load(ptr, ptr_alignment, vec_ty, access_kind);
4888 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4889 const modified_vector = try fg.wip.insertElement(loaded_vector, elem, index_val, "");
4890
4891 try fg.store(ptr, ptr_alignment, modified_vector, vec_ty, access_kind);
4892 return .none;
4893 }
4894
4895 if (ptr_info.packed_offset.host_size != 0) {
4896 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4897 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4898 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
4899
4900 const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
47554901
4756 const src_operand = try self.resolveInst(bin_op.rhs);4902 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4757 try self.storeFull(dest_ptr, ptr_ty, src_operand, .none);4903 const shift_amt = try o.builder.intConst(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4904
4905 // Convert to equally-sized integer type in order to perform the bit
4906 // operations on the value to store
4907 const new_val_bits_type = try o.builder.intType(@intCast(elem_bits));
4908 const new_val_bits = if (elem_ty.isPtrAtRuntime(zcu))
4909 try fg.wip.cast(.ptrtoint, elem, new_val_bits_type, "")
4910 else
4911 try fg.wip.cast(.bitcast, elem, new_val_bits_type, "");
4912
4913 const mask_val = blk: {
4914 const zext = try fg.wip.cast(
4915 .zext,
4916 try o.builder.intValue(new_val_bits_type, -1),
4917 llvm_backing_int_ty,
4918 "",
4919 );
4920 const shl = try fg.wip.bin(.shl, zext, shift_amt.toValue(), "");
4921 break :blk try fg.wip.bin(
4922 .xor,
4923 shl,
4924 try o.builder.intValue(llvm_backing_int_ty, -1),
4925 "",
4926 );
4927 };
4928
4929 const masked_backing_int_val = try fg.wip.bin(.@"and", backing_int_val, mask_val, "");
4930 const extended_new_val = try fg.wip.cast(.zext, new_val_bits, llvm_backing_int_ty, "");
4931 const shifted_new_val = try fg.wip.bin(.shl, extended_new_val, shift_amt.toValue(), "");
4932 const new_backing_int_val = try fg.wip.bin(.@"or", shifted_new_val, masked_backing_int_val, "");
4933
4934 try fg.store(ptr, ptr_alignment, new_backing_int_val, backing_int_ty, access_kind);
4935 return .none;
4936 }
4937
4938 try fg.store(ptr, ptr_alignment, elem, elem_ty, access_kind);
4758 return .none;4939 return .none;
4759}4940}
47604941
...@@ -4766,7 +4947,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4766,7 +4947,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4766 const ptr_info = ptr_ty.ptrInfo(zcu);4947 const ptr_info = ptr_ty.ptrInfo(zcu);
4767 const ptr = try fg.resolveInst(ty_op.operand);4948 const ptr = try fg.resolveInst(ty_op.operand);
4768 const elem_ty = ptr_ty.childType(zcu);4949 const elem_ty = ptr_ty.childType(zcu);
4769 const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();4950 const ptr_align = ptr_ty.ptrAlignment(zcu);
47704951
4771 fg.maybeMarkAllowZeroAccess(ptr_info);4952 fg.maybeMarkAllowZeroAccess(ptr_info);
47724953
...@@ -4774,36 +4955,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4774,36 +4955,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4774 if (ptr_info.flags.is_volatile) .@"volatile" else .normal;4955 if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
47754956
4776 if (ptr_info.flags.vector_index != .none) {4957 if (ptr_info.flags.vector_index != .none) {
4777 const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index);4958 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4778 const vec_elem_ty = try o.lowerType(elem_ty);4959 const vec_ty = try fg.pt.vectorType(.{
4779 const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty);4960 .len = ptr_info.packed_offset.host_size,
47804961 .child = elem_ty.toIntern(),
4781 const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, "");4962 });
4782 return fg.wip.extractElement(loaded_vector, index_u32, "");4963 const vector_val = try fg.load(ptr, ptr_align, vec_ty, access_kind);
4964 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4965 return fg.wip.extractElement(vector_val, index_val, "");
4783 }4966 }
47844967
4785 if (ptr_info.packed_offset.host_size == 0) {4968 if (ptr_info.packed_offset.host_size == 0) {
4786 return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind);4969 return fg.load(ptr, ptr_align, elem_ty, access_kind);
4787 }4970 }
47884971
4789 const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8));4972 assert(!isByRef(elem_ty, zcu)); // all packable types are by-val
4790 const containing_int =
4791 try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, "");
47924973
4793 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);4974 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4794 const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset);4975 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4795 const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, "");4976 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
4796 const elem_llvm_ty = try o.lowerType(elem_ty);
47974977
4798 if (isByRef(elem_ty, zcu)) {4978 const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
4799 const result_align = elem_ty.abiAlignment(zcu).toLlvm();
4800 const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align);
48014979
4802 const same_size_int = try o.builder.intType(@intCast(elem_bits));4980 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4803 const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");4981 const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4804 _ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align);4982 const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
4805 return result_ptr;4983 const elem_llvm_ty = try o.lowerType(elem_ty);
4806 }
48074984
4808 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {4985 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
4809 const same_size_int = try o.builder.intType(@intCast(elem_bits));4986 const same_size_int = try o.builder.intType(@intCast(elem_bits));
...@@ -4918,21 +5095,22 @@ fn airCmpxchg(...@@ -4918,21 +5095,22 @@ fn airCmpxchg(
4918 return self.wip.select(.normal, success_bit, zero, payload, "");5095 return self.wip.select(.normal, success_bit, zero, payload, "");
4919 }5096 }
49205097
4921 assert(isByRef(optional_ty, zcu));5098 assert(!isByRef(operand_ty, zcu)); // can only cmpxchg non-by-ref types
5099 assert(isByRef(optional_ty, zcu)); // all optionals are by-ref
49225100
4923 comptime assert(optional_layout_version == 3);5101 comptime assert(optional_layout_version == 3);
49245102
4925 const non_null_bit = try self.wip.not(success_bit, "");5103 const non_null_bit = try self.wip.not(success_bit, "");
49265104
4927 const payload_align = operand_ty.abiAlignment(zcu).toLlvm();5105 const payload_align = operand_ty.abiAlignment(zcu);
4928 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align);5106 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align.toLlvm());
49295107
4930 // Payload is always the first field at offset 0, so address is `alloca_inst`5108 // Payload is always the first field at offset 0, so address is `alloca_inst`
4931 _ = try self.wip.store(.normal, payload, alloca_inst, payload_align);5109 try self.store(alloca_inst, .none, payload, operand_ty, .normal);
49325110
4933 // Non-null bit is after payload with no padding because it has alignment 15111 // Non-null bit is after payload with no padding because it has alignment 1
4934 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));5112 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4935 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));5113 try self.store(non_null_ptr, payload_align, non_null_bit, .bool, .normal);
49365114
4937 return alloca_inst;5115 return alloca_inst;
4938}5116}
...@@ -5073,7 +5251,17 @@ fn airAtomicStore(...@@ -5073,7 +5251,17 @@ fn airAtomicStore(
50735251
5074 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));5252 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
50755253
5076 try self.storeFull(ptr, ptr_ty, element, ordering);5254 assert(!isByRef(operand_ty, zcu));
5255
5256 _ = try self.wip.storeAtomic(
5257 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5258 element,
5259 ptr,
5260 self.sync_scope,
5261 ordering,
5262 ptr_ty.ptrAlignment(zcu).toLlvm(),
5263 );
5264
5077 return .none;5265 return .none;
5078}5266}
50795267
...@@ -5084,7 +5272,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5084,7 +5272,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5084 const dest_slice = try self.resolveInst(bin_op.lhs);5272 const dest_slice = try self.resolveInst(bin_op.lhs);
5085 const ptr_ty = self.typeOf(bin_op.lhs);5273 const ptr_ty = self.typeOf(bin_op.lhs);
5086 const elem_ty = self.typeOf(bin_op.rhs);5274 const elem_ty = self.typeOf(bin_op.rhs);
5087 const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();5275 const dest_ptr_align = ptr_ty.ptrAlignment(zcu);
5088 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);5276 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
5089 const access_kind: Builder.MemoryAccessKind =5277 const access_kind: Builder.MemoryAccessKind =
5090 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;5278 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -5110,7 +5298,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5110,7 +5298,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5110 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);5298 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5111 _ = try self.wip.callMemSet(5299 _ = try self.wip.callMemSet(
5112 dest_ptr,5300 dest_ptr,
5113 dest_ptr_align,5301 dest_ptr_align.toLlvm(),
5114 fill_byte,5302 fill_byte,
5115 len,5303 len,
5116 access_kind,5304 access_kind,
...@@ -5132,7 +5320,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5132,7 +5320,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5132 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);5320 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5133 _ = try self.wip.callMemSet(5321 _ = try self.wip.callMemSet(
5134 dest_ptr,5322 dest_ptr,
5135 dest_ptr_align,5323 dest_ptr_align.toLlvm(),
5136 fill_byte,5324 fill_byte,
5137 len,5325 len,
5138 access_kind,5326 access_kind,
...@@ -5152,7 +5340,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5152,7 +5340,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
51525340
5153 _ = try self.wip.callMemSet(5341 _ = try self.wip.callMemSet(
5154 dest_ptr,5342 dest_ptr,
5155 dest_ptr_align,5343 dest_ptr_align.toLlvm(),
5156 fill_byte,5344 fill_byte,
5157 len,5345 len,
5158 access_kind,5346 access_kind,
...@@ -5182,7 +5370,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5182,7 +5370,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5182 const body_block = try self.wip.block(1, "InlineMemsetBody");5370 const body_block = try self.wip.block(1, "InlineMemsetBody");
5183 const end_block = try self.wip.block(1, "InlineMemsetEnd");5371 const end_block = try self.wip.block(1, "InlineMemsetEnd");
51845372
5185 const llvm_usize_ty = try o.lowerType(.usize);
5186 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {5373 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5187 .slice => try self.ptraddScaled(5374 .slice => try self.ptraddScaled(
5188 dest_ptr,5375 dest_ptr,
...@@ -5201,18 +5388,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5201,18 +5388,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
52015388
5202 self.wip.cursor = .{ .block = body_block };5389 self.wip.cursor = .{ .block = body_block };
5203 const elem_abi_align = elem_ty.abiAlignment(zcu);5390 const elem_abi_align = elem_ty.abiAlignment(zcu);
5204 const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();5391 const it_ptr_align: InternPool.Alignment = dest_ptr_align.min(elem_abi_align);
5205 if (isByRef(elem_ty, zcu)) {5392 try self.store(it_ptr.toValue(), it_ptr_align, value, elem_ty, access_kind);
5206 _ = try self.wip.callMemCpy(
5207 it_ptr.toValue(),
5208 it_ptr_align,
5209 value,
5210 elem_abi_align.toLlvm(),
5211 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
5212 access_kind,
5213 self.disable_intrinsics,
5214 );
5215 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5216 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);5393 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
5217 _ = try self.wip.br(loop_block);5394 _ = try self.wip.br(loop_block);
52185395
...@@ -5289,14 +5466,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder....@@ -5289,14 +5466,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
52895466
5290 const union_ptr = try self.resolveInst(bin_op.lhs);5467 const union_ptr = try self.resolveInst(bin_op.lhs);
5291 const new_tag = try self.resolveInst(bin_op.rhs);5468 const new_tag = try self.resolveInst(bin_op.rhs);
5469 const tag_ty = self.typeOf(bin_op.rhs);
5292 const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);5470 const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5293 if (layout.payload_size == 0) {
5294 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5295 return .none;
5296 }
5297 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());5471 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5298 const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());5472 const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());
5299 _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());5473 try self.store(tag_field_ptr, tag_ptr_align, new_tag, tag_ty, access_kind);
5300 return .none;5474 return .none;
5301}5475}
53025476
...@@ -5309,9 +5483,8 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder....@@ -5309,9 +5483,8 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5309 assert(layout.tag_size != 0);5483 assert(layout.tag_size != 0);
5310 const operand = try self.resolveInst(ty_op.operand);5484 const operand = try self.resolveInst(ty_op.operand);
5311 if (isByRef(un_ty, zcu)) {5485 if (isByRef(un_ty, zcu)) {
5312 const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?);
5313 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());5486 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5314 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");5487 return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5315 } else {5488 } else {
5316 // This is only possible if all fields are zero-bit, in which case `operand` is already an5489 // This is only possible if all fields are zero-bit, in which case `operand` is already an
5317 // integer value (the union is lowered as its enum tag).5490 // integer value (the union is lowered as its enum tag).
...@@ -5480,14 +5653,13 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5480,14 +5653,13 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5480 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;5653 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5481 const operand = try self.resolveInst(un_op);5654 const operand = try self.resolveInst(un_op);
5482 const slice_ty = self.typeOfIndex(inst);5655 const slice_ty = self.typeOfIndex(inst);
5483 const slice_llvm_ty = try o.lowerType(slice_ty);
54845656
5485 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.5657 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5486 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");5658 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");
54875659
5488 const error_name_table_ptr = try o.getErrorNameTable();5660 const error_name_table_ptr = try o.getErrorNameTable();
5489 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));5661 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
5490 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");5662 return self.load(error_name_ptr, .none, slice_ty, .normal);
5491}5663}
54925664
5493fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5665fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5696,15 +5868,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5696,15 +5868,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5696/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.5868/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
5697///5869///
5698/// Equivalent to:5870/// Equivalent to:
5699/// reduce: {5871/// ```
5700/// var i: usize = 0;5872/// var accum: T = init;
5701/// var accum: T = init;5873/// for (0..i) |i| {
5702/// while (i < vec.len) : (i += 1) {5874/// accum = llvm_fn(accum, vec[i]);
5703/// accum = llvm_fn(accum, vec[i]);5875/// }
5704/// }5876/// // result is 'accum'
5705/// break :reduce accum;5877/// ```
5706/// }
5707///
5708fn buildReducedCall(5878fn buildReducedCall(
5709 self: *FuncGen,5879 self: *FuncGen,
5710 llvm_fn: Builder.Function.Index,5880 llvm_fn: Builder.Function.Index,
...@@ -5713,56 +5883,54 @@ fn buildReducedCall(...@@ -5713,56 +5883,54 @@ fn buildReducedCall(
5713 accum_init: Builder.Value,5883 accum_init: Builder.Value,
5714) Allocator.Error!Builder.Value {5884) Allocator.Error!Builder.Value {
5715 const o = self.object;5885 const o = self.object;
5716 const usize_ty = try o.lowerType(.usize);5886 const llvm_usize_ty = try o.lowerType(.usize);
5717 const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);5887 const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
5718 const llvm_result_ty = accum_init.typeOfWip(&self.wip);5888 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
57195889
5720 // Allocate and initialize our mutable variables5890 const entry_block = self.wip.cursor.block;
5721 const i_ptr = try self.buildAlloca(usize_ty, .default);
5722 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
5723 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
5724 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
5725
5726 // Setup the loop
5727 const loop = try self.wip.block(2, "ReduceLoop");
5728 const loop_exit = try self.wip.block(1, "AfterReduce");
5729 _ = try self.wip.br(loop);
5730 {
5731 self.wip.cursor = .{ .block = loop };
5732
5733 // while (i < vec.len)
5734 const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
5735 const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
5736 const loop_then = try self.wip.block(1, "ReduceLoopThen");
5737
5738 _ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
5739
5740 {
5741 self.wip.cursor = .{ .block = loop_then };
57425891
5743 // accum = f(accum, vec[i]);5892 const cond_block = try self.wip.block(2, "ReduceLoopCond");
5744 const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");5893 const body_block = try self.wip.block(1, "ReduceLoopBody");
5745 const element = try self.wip.extractElement(operand_vector, i, "");5894 const exit_block = try self.wip.block(1, "ReduceLoopExit");
5746 const new_accum = try self.wip.call(5895
5747 .normal,5896 _ = try self.wip.br(cond_block);
5748 .ccc,5897
5749 .none,5898 // ReduceLoopCond:
5750 llvm_fn.typeOf(&o.builder),5899 // %index = phi iN [0, %Entry], [%new_index, %ReduceLoopBody]
5751 llvm_fn.toValue(&o.builder),5900 // %accum = phi T [%accum_init, %Entry], [%new_accum, %ReduceLoopBody]
5752 &.{ accum, element },5901 // %cond = icmp ult iN %index, %vector_len
5753 "",5902 // br i1 %cond, label %ReduceLoopBody, label %ReduceLoopExit
5754 );5903 self.wip.cursor = .{ .block = cond_block };
5755 _ = try self.wip.store(.normal, new_accum, accum_ptr, .default);5904 const index = try self.wip.phi(llvm_usize_ty, "");
5905 const accum = try self.wip.phi(llvm_result_ty, "");
5906 const cond = try self.wip.icmp(.ult, index.toValue(), llvm_vector_len, "");
5907 _ = try self.wip.brCond(cond, body_block, exit_block, .none);
5908
5909 // ReduceLoopBody:
5910 // %elem = extractelement <n x T> %operand_vec, iN %index
5911 // %new_accum = call T @llvm_fn(T %accum, T %elem)
5912 // %new_index = add nuw iN %index, 1
5913 // br label %ReduceLoopCond
5914 self.wip.cursor = .{ .block = body_block };
5915 const elem = try self.wip.extractElement(operand_vector, index.toValue(), "");
5916 const new_accum = try self.wip.call(
5917 .normal,
5918 .ccc,
5919 .none,
5920 llvm_fn.typeOf(&o.builder),
5921 llvm_fn.toValue(&o.builder),
5922 &.{ accum.toValue(), elem },
5923 "",
5924 );
5925 const new_index = try self.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(llvm_usize_ty, 1), "");
5926 _ = try self.wip.br(cond_block);
57565927
5757 // i += 15928 const index_init = try o.builder.intValue(llvm_usize_ty, 0);
5758 const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");5929 index.finish(&.{ index_init, new_index }, &.{ entry_block, body_block }, &self.wip);
5759 _ = try self.wip.store(.normal, new_i, i_ptr, .default);5930 accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
5760 _ = try self.wip.br(loop);
5761 }
5762 }
57635931
5764 self.wip.cursor = .{ .block = loop_exit };5932 self.wip.cursor = .{ .block = exit_block };
5765 return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");5933 return new_accum;
5766}5934}
57675935
5768fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {5936fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...@@ -5939,24 +6107,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5939,24 +6107,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
59396107
5940 const llvm_field_val = try self.resolveInst(elem);6108 const llvm_field_val = try self.resolveInst(elem);
59416109
5942 if (isByRef(field_ty, zcu)) {6110 try self.store(field_ptr, field_ptr_align, llvm_field_val, field_ty, .normal);
5943 _ = try self.wip.callMemCpy(
5944 field_ptr,
5945 field_ptr_align.toLlvm(),
5946 llvm_field_val,
5947 field_ty.abiAlignment(zcu).toLlvm(),
5948 try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)),
5949 .normal,
5950 self.disable_intrinsics,
5951 );
5952 } else {
5953 _ = try self.wip.store(
5954 .normal,
5955 llvm_field_val,
5956 field_ptr,
5957 field_ptr_align.toLlvm(),
5958 );
5959 }
5960 }6111 }
59616112
5962 return alloca_inst;6113 return alloca_inst;
...@@ -5975,12 +6126,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5975,12 +6126,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5975 for (elements, 0..) |elem, i| {6126 for (elements, 0..) |elem, i| {
5976 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);6127 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
5977 const llvm_elem = try self.resolveInst(elem);6128 const llvm_elem = try self.resolveInst(elem);
5978 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type);6129 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type, .normal);
5979 }6130 }
5980 if (array_info.sentinel) |sent_val| {6131 if (array_info.sentinel) |sent_val| {
5981 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);6132 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
5982 const llvm_elem = try self.resolveValue(sent_val);6133 const llvm_elem = try self.resolveValue(sent_val);
5983 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type);6134 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type, .normal);
5984 }6135 }
59856136
5986 return alloca_inst;6137 return alloca_inst;
...@@ -6014,11 +6165,12 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6014,11 +6165,12 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
60146165
6015 {6166 {
6016 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());6167 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6017 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty);6168 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty, .normal);
6018 }6169 }
60196170
6020 if (layout.tag_size != 0) {6171 if (layout.tag_size != 0) {
6021 const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type);6172 const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6173 const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6022 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {6174 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6023 .none => try o.builder.intConst(6175 .none => try o.builder.intConst(
6024 try o.lowerType(.fromInterned(union_obj.enum_tag_type)),6176 try o.lowerType(.fromInterned(union_obj.enum_tag_type)),
...@@ -6027,7 +6179,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6027,7 +6179,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6027 else => |tag_val_ip| try o.lowerValue(tag_val_ip),6179 else => |tag_val_ip| try o.lowerValue(tag_val_ip),
6028 };6180 };
6029 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());6181 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6030 _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());6182 try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
6031 }6183 }
60326184
6033 return result_ptr;6185 return result_ptr;
...@@ -6134,7 +6286,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6134,7 +6286,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6134 // Load the work_group_* member from the struct as u16.6286 // Load the work_group_* member from the struct as u16.
6135 // Just treat the dispatch pointer as an array of u16 to keep things simple.6287 // Just treat the dispatch pointer as an array of u16 to keep things simple.
6136 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);6288 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6137 return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");6289 return self.load(workgroup_size_ptr, .@"2", .u16, .normal);
6138 },6290 },
6139 .nvptx, .nvptx64 => {6291 .nvptx, .nvptx64 => {
6140 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");6292 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
...@@ -6169,8 +6321,8 @@ fn optCmpNull(...@@ -6169,8 +6321,8 @@ fn optCmpNull(
6169 comptime assert(optional_layout_version == 3);6321 comptime assert(optional_layout_version == 3);
6170 // Non-null bit is always after the payload, with no padding because it has alignment 1.6322 // Non-null bit is always after the payload, with no padding because it has alignment 1.
6171 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));6323 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6172 const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");6324 const non_null = try self.load(non_null_ptr, .@"1", .bool, access_kind);
6173 return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");6325 return self.wip.icmp(cond, non_null, .false, "");
6174}6326}
61756327
6176/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.6328/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
...@@ -6187,13 +6339,9 @@ fn optPayloadHandle(...@@ -6187,13 +6339,9 @@ fn optPayloadHandle(
6187 // Payload is first field so always at the same address as the optional itself.6339 // Payload is first field so always at the same address as the optional itself.
6188 const payload_ptr = opt_ptr;6340 const payload_ptr = opt_ptr;
61896341
6190 const payload_align = payload_ty.abiAlignment(zcu).toLlvm();6342 if (can_elide_load and isByRef(payload_ty, zcu)) return payload_ptr;
6191 if (isByRef(payload_ty, zcu)) {6343
6192 if (can_elide_load) return payload_ptr;6344 return fg.load(payload_ptr, .none, payload_ty, .normal);
6193 return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
6194 } else {
6195 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
6196 }
6197}6345}
61986346
6199fn fieldPtr(6347fn fieldPtr(
...@@ -6217,214 +6365,145 @@ fn fieldPtr(...@@ -6217,214 +6365,145 @@ fn fieldPtr(
6217 return self.ptraddConst(aggregate_ptr, offset);6365 return self.ptraddConst(aggregate_ptr, offset);
6218}6366}
62196367
6220/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.6368/// Non-atomic, non-bitpacked load of type `load_ty` from pointer `ptr`.
6221fn loadTruncate(6369///
6222 fg: *FuncGen,6370/// `ptr` has alignment `ptr_align`, or `load_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6223 access_kind: Builder.MemoryAccessKind,6371///
6224 payload_ty: Type,6372/// If `load_ty` is a by-ref type, then the value is copied to a new alloca with a memcpy, and a
6225 payload_ptr: Builder.Value,6373/// pointer to that alloca is returned.
6226 payload_alignment: Builder.Alignment,
6227) Allocator.Error!Builder.Value {
6228 // from https://llvm.org/docs/LangRef.html#load-instruction :
6229 // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
6230 // => so load the byte aligned value and trunc the unwanted bits.
6231
6232 const o = fg.object;
6233 const zcu = o.zcu;
6234 const payload_llvm_ty = try o.lowerType(payload_ty);
6235 const abi_size = payload_ty.abiSize(zcu);
6236
6237 const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
6238 try o.builder.intType(@intCast(abi_size * 8))
6239 else
6240 payload_llvm_ty;
6241 const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
6242 const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big)
6243 try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
6244 load_llvm_ty,
6245 (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
6246 ), "")
6247 else
6248 loaded;
6249
6250 return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
6251}
6252
6253/// Load a by-ref type by constructing a new alloca and performing a memcpy.
6254fn loadByRef(
6255 fg: *FuncGen,
6256 ptr: Builder.Value,
6257 pointee_type: Type,
6258 ptr_alignment: Builder.Alignment,
6259 access_kind: Builder.MemoryAccessKind,
6260) Allocator.Error!Builder.Value {
6261 const o = fg.object;
6262 const pointee_llvm_ty = try o.lowerType(pointee_type);
6263 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
6264 .max(pointee_type.abiAlignment(o.zcu)).toLlvm();
6265 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
6266 const size_bytes = pointee_type.abiSize(o.zcu);
6267 _ = try fg.wip.callMemCpy(
6268 result_ptr,
6269 result_align,
6270 ptr,
6271 ptr_alignment,
6272 try o.builder.intValue(try o.lowerType(.usize), size_bytes),
6273 access_kind,
6274 fg.disable_intrinsics,
6275 );
6276 return result_ptr;
6277}
6278
6279/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value
6280/// into a new alloca.
6281fn load(6374fn load(
6282 fg: *FuncGen,6375 fg: *FuncGen,
6283 ptr: Builder.Value,6376 ptr: Builder.Value,
6284 elem_ty: Type,6377 ptr_align: InternPool.Alignment,
6285 ptr_alignment: Builder.Alignment,6378 load_ty: Type,
6286 access_kind: Builder.MemoryAccessKind,6379 access_kind: Builder.MemoryAccessKind,
6287) Allocator.Error!Builder.Value {6380) Allocator.Error!Builder.Value {
6288 const zcu = fg.object.zcu;6381 const o = fg.object;
6289 if (isByRef(elem_ty, zcu)) {
6290 return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
6291 } else {
6292 return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
6293 }
6294}
6295
6296fn storeFull(
6297 self: *FuncGen,
6298 ptr: Builder.Value,
6299 ptr_ty: Type,
6300 elem: Builder.Value,
6301 ordering: Builder.AtomicOrdering,
6302) Allocator.Error!void {
6303 const o = self.object;
6304 const zcu = o.zcu;6382 const zcu = o.zcu;
6305 const info = ptr_ty.ptrInfo(zcu);
6306 const elem_ty = Type.fromInterned(info.child);
6307 if (!elem_ty.hasRuntimeBits(zcu)) {
6308 return;
6309 }
6310 const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
6311 const access_kind: Builder.MemoryAccessKind =
6312 if (info.flags.is_volatile) .@"volatile" else .normal;
6313
6314 if (info.flags.vector_index != .none) {
6315 const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6316 const vec_elem_ty = try o.lowerType(elem_ty);
6317 const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
63186383
6319 const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");6384 const abi_align = load_ty.abiAlignment(zcu);
6385 const abi_size = load_ty.abiSize(zcu);
63206386
6321 const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");6387 const llvm_load_ty = try o.lowerType(load_ty);
63226388 const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6323 assert(ordering == .none);6389 .none => abi_align.toLlvm(),
6324 _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);6390 else => |a| a.toLlvm(),
6325 return;6391 };
6326 }
6327
6328 if (info.packed_offset.host_size != 0) {
6329 const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
6330 assert(ordering == .none);
6331 const containing_int =
6332 try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
6333 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6334 const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
6335 // Convert to equally-sized integer type in order to perform the bit
6336 // operations on the value to store
6337 const value_bits_type = try o.builder.intType(@intCast(elem_bits));
6338 const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
6339 try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
6340 else
6341 try self.wip.cast(.bitcast, elem, value_bits_type, "");
6342
6343 const mask_val = blk: {
6344 const zext = try self.wip.cast(
6345 .zext,
6346 try o.builder.intValue(value_bits_type, -1),
6347 containing_int_ty,
6348 "",
6349 );
6350 const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
6351 break :blk try self.wip.bin(
6352 .xor,
6353 shl,
6354 try o.builder.intValue(containing_int_ty, -1),
6355 "",
6356 );
6357 };
6358
6359 const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
6360 const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
6361 const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
6362 const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
63636392
6364 assert(ordering == .none);6393 if (isByRef(load_ty, zcu)) {
6365 _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);6394 const llvm_usize_ty = try o.lowerType(.usize);
6366 return;6395 const result_ptr = try fg.buildAlloca(llvm_load_ty, abi_align.toLlvm());
6367 }6396 _ = try fg.wip.callMemCpy(
6368 if (!isByRef(elem_ty, zcu)) {6397 result_ptr,
6369 _ = try self.wip.storeAtomic(6398 abi_align.toLlvm(),
6370 access_kind,
6371 elem,
6372 ptr,6399 ptr,
6373 self.sync_scope,6400 llvm_ptr_align,
6374 ordering,6401 try o.builder.intValue(llvm_usize_ty, abi_size),
6375 ptr_alignment,6402 access_kind,
6403 fg.disable_intrinsics,
6376 );6404 );
6377 return;6405 return result_ptr;
6378 }6406 }
6379 assert(ordering == .none);6407
6380 _ = try self.wip.callMemCpy(6408 if (load_ty.isAbiInt(zcu) and load_ty.bitSize(zcu) != abi_size * 8) {
6381 ptr,6409 // `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6382 ptr_alignment,6410 // handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6383 elem,6411 // semantics. However:
6384 elem_ty.abiAlignment(zcu).toLlvm(),6412 //
6385 try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)),6413 // * LLVM's lowering for these integer types generally leads to poor codegen, as integers
6386 access_kind,6414 // are only extended to the next byte, instead of to the next "natural" integer type.
6387 self.disable_intrinsics,6415 //
6388 );6416 // * Clang never emits loads or stores of these types, so LLVM's support for them is rather
6417 // flaky---we have encountered several LLVM bugs caused by incorrect handling of them.
6418 //
6419 // Therefore, we handle these memory accesses specially: in this case we will actually load
6420 // the next-largest "natural" integer type and then truncate to `load_ty`.
6421 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6422 const loaded = try fg.wip.load(access_kind, llvm_abi_ty, ptr, llvm_ptr_align, "");
6423 // For packed structs, current Zig semantics don't really allow us to make the padding bits
6424 // well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6425 // implemented, but until then, do a normal trunc for packed types.
6426 return fg.wip.cast(switch (load_ty.zigTypeTag(zcu)) {
6427 .@"struct", .@"union" => .trunc,
6428 else => switch (load_ty.intInfo(zcu).signedness) {
6429 .unsigned => .@"trunc nuw",
6430 .signed => .@"trunc nsw",
6431 },
6432 }, loaded, llvm_load_ty, "");
6433 }
6434
6435 // `load_ty` is a simple by-val type which requires no special handling.
6436 return fg.wip.load(access_kind, llvm_load_ty, ptr, llvm_ptr_align, "");
6389}6437}
63906438
6391/// Non-atomic, non-volatile, non-packed store.6439/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
6440///
6441/// `ptr` has alignment `ptr_align`, or `elem_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6442///
6443/// If `elem_ty` is a by-ref type, then `elem` is itself a pointer, and a memcpy is emitted.
6392fn store(6444fn store(
6393 fg: *FuncGen,6445 fg: *FuncGen,
6394 ptr: Builder.Value,6446 ptr: Builder.Value,
6395 ptr_align: InternPool.Alignment,6447 ptr_align: InternPool.Alignment,
6396 elem: Builder.Value,6448 elem: Builder.Value,
6397 elem_ty: Type,6449 elem_ty: Type,
6450 access_kind: Builder.MemoryAccessKind,
6398) Allocator.Error!void {6451) Allocator.Error!void {
6399 const o = fg.object;6452 const o = fg.object;
6400 const zcu = o.zcu;6453 const zcu = o.zcu;
6454
6455 const abi_align = elem_ty.abiAlignment(zcu);
6456 const abi_size = elem_ty.abiSize(zcu);
6457
6401 const llvm_ptr_align = switch (ptr_align) {6458 const llvm_ptr_align = switch (ptr_align) {
6402 .none => elem_ty.abiAlignment(zcu).toLlvm(),6459 .none => abi_align.toLlvm(),
6403 else => ptr_align.toLlvm(),6460 else => ptr_align.toLlvm(),
6404 };6461 };
6462
6405 if (isByRef(elem_ty, zcu)) {6463 if (isByRef(elem_ty, zcu)) {
6464 const llvm_usize_ty = try o.lowerType(.usize);
6406 _ = try fg.wip.callMemCpy(6465 _ = try fg.wip.callMemCpy(
6407 ptr,6466 ptr,
6408 llvm_ptr_align,6467 llvm_ptr_align,
6409 elem,6468 elem,
6410 elem_ty.abiAlignment(zcu).toLlvm(),6469 abi_align.toLlvm(),
6411 try o.builder.intValue(6470 try o.builder.intValue(llvm_usize_ty, abi_size),
6412 try o.lowerType(.usize),6471 access_kind,
6413 elem_ty.abiSize(zcu),
6414 ),
6415 .normal,
6416 fg.disable_intrinsics,6472 fg.disable_intrinsics,
6417 );6473 );
6418 } else {6474 return;
6475 }
6476
6477 assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty));
6478
6479 if (elem_ty.isAbiInt(zcu) and elem_ty.bitSize(zcu) != abi_size * 8) {
6480 // `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6481 // the corresponding comment in `FuncGen.load` for more details.
6482 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6483 const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
6484 .unsigned => .zext,
6485 .signed => .sext,
6486 }, elem, llvm_abi_ty, "");
6419 _ = try fg.wip.storeAtomic(6487 _ = try fg.wip.storeAtomic(
6420 .normal,6488 access_kind,
6421 elem,6489 extended,
6422 ptr,6490 ptr,
6423 fg.sync_scope,6491 fg.sync_scope,
6424 .none,6492 .none,
6425 llvm_ptr_align,6493 llvm_ptr_align,
6426 );6494 );
6495 return;
6427 }6496 }
6497
6498 // `elem_ty` is a simple by-val type which requires no special handling.
6499 _ = try fg.wip.storeAtomic(
6500 access_kind,
6501 elem,
6502 ptr,
6503 fg.sync_scope,
6504 .none,
6505 llvm_ptr_align,
6506 );
6428}6507}
64296508
6430fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {6509fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
...@@ -6453,18 +6532,18 @@ fn valgrindClientRequest(...@@ -6453,18 +6532,18 @@ fn valgrindClientRequest(
6453 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;6532 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
64546533
6455 const llvm_usize = try o.lowerType(.usize);6534 const llvm_usize = try o.lowerType(.usize);
6456 const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();6535 const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
64576536
6458 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);6537 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
6459 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {6538 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6460 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);6539 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_align);
6461 fg.valgrind_client_request_array = array_ptr;6540 fg.valgrind_client_request_array = array_ptr;
6462 break :a array_ptr;6541 break :a array_ptr;
6463 } else fg.valgrind_client_request_array;6542 } else fg.valgrind_client_request_array;
6464 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };6543 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6465 for (array_elements, 0..) |elem, i| {6544 for (array_elements, 0..) |elem, i| {
6466 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));6545 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6467 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);6546 try fg.store(elem_ptr, .none, elem, .usize, .normal);
6468 }6547 }
64696548
6470 const arch_specific: struct {6549 const arch_specific: struct {
...@@ -7283,33 +7362,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {...@@ -7283,33 +7362,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {
7283 };7362 };
7284}7363}
72857364
7286pub fn buildAllocaInner(
7287 wip: *Builder.WipFunction,
7288 llvm_ty: Builder.Type,
7289 alignment: Builder.Alignment,
7290 target: *const std.Target,
7291) Allocator.Error!Builder.Value {
7292 const address_space = llvmAllocaAddressSpace(target);
7293
7294 const alloca = blk: {
7295 const prev_cursor = wip.cursor;
7296 const prev_debug_location = wip.debug_location;
7297 defer {
7298 wip.cursor = prev_cursor;
7299 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
7300 wip.debug_location = prev_debug_location;
7301 }
7302
7303 wip.cursor = .{ .block = .entry };
7304 wip.debug_location = .no_location;
7305 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
7306 };
7307
7308 // The pointer returned from this function should have the generic address space,
7309 // if this isn't the case then cast it to the generic address space.
7310 return wip.conv(.unneeded, alloca, .ptr, "");
7311}
7312
7313/// This is the one source of truth for whether a type is passed around as an LLVM pointer,7365/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7314/// or as an LLVM value.7366/// or as an LLVM value.
7315pub fn isByRef(ty: Type, zcu: *const Zcu) bool {7367pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
...@@ -7380,7 +7432,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E...@@ -7380,7 +7432,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7380}7432}
73817433
7382fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {7434fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7383 return fg.object.ptraddConst(&fg.wip, ptr, offset);7435 if (offset == 0) return ptr;
7436 const o = fg.object;
7437 const llvm_usize_ty = try o.lowerType(.usize);
7438 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7439 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7384}7440}
7385fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {7441fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7386 if (scale == 0) return ptr;7442 if (scale == 0) return ptr;