authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 17:28:01+02:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 17:28:01+02:00
log39a9a4b2cae0ef94eb4aea482dabf275fdd0a910
tree8fbb6febdf32fcd315de6bd279ee8b9000da6e7f
parentefe73c787a89f71de37bf876c3ef7bcfddebebba
parentcbef1f5a013c664979e68d3ea8fa3ba378e3f0f6

Merge pull request 'llvm: only load/store ABI-sized integers to/from memory' (#35711) from mlugg/llvm-weird-int-in-memory into master

Resolves: https://github.com/ziglang/zig/issues/17768 Resolves: https://github.com/ziglang/zig/issues/18936 Resolves: https://github.com/ziglang/zig/issues/19755 Resolves: https://github.com/ziglang/zig/issues/24282 Resolves: https://github.com/ziglang/zig/issues/24883 Resolves: https://github.com/ziglang/zig/issues/25042 Resolves: https://github.com/ziglang/zig/issues/25555 Resolves: https://codeberg.org/ziglang/zig/issues/32036 Resolves: https://codeberg.org/ziglang/zig/issues/35560 Reviewed-on: https://codeberg.org/ziglang/zig/pulls/35711 Reviewed-by: Andrew Kelley <andrew@ziglang.org>

69 files changed, 4646 insertions(+), 4049 deletions(-)

CMakeLists.txt+1-1
...@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES...@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES
342 src/Package/Module.zig342 src/Package/Module.zig
343 src/RangeSet.zig343 src/RangeSet.zig
344 src/Sema.zig344 src/Sema.zig
345 src/Sema/bitcast.zig345 src/Sema/reinterpret.zig
346 src/Sema/comptime_ptr_access.zig346 src/Sema/comptime_ptr_access.zig
347 src/Sema/type_resolution.zig347 src/Sema/type_resolution.zig
348 src/Type.zig348 src/Type.zig
doc/langref/test_packed_structs.zig+2-10
...@@ -26,16 +26,8 @@ fn doTheTest() !void {...@@ -26,16 +26,8 @@ fn doTheTest() !void {
26 try expectEqual(0x1, divided.quarter4);26 try expectEqual(0x1, divided.quarter4);
2727
28 const ordered: [2]u8 = @bitCast(full);28 const ordered: [2]u8 = @bitCast(full);
29 switch (native_endian) {29 try expectEqual(0x34, ordered[0]);
30 .big => {30 try expectEqual(0x12, ordered[1]);
31 try expectEqual(0x12, ordered[0]);
32 try expectEqual(0x34, ordered[1]);
33 },
34 .little => {
35 try expectEqual(0x34, ordered[0]);
36 try expectEqual(0x12, ordered[1]);
37 },
38 }
39}31}
4032
41// test33// test
doc/langref/test_pointer_casting.zig+12-8
...@@ -1,18 +1,22 @@...@@ -1,18 +1,22 @@
1const std = @import("std");1const std = @import("std");
2const native_endian = @import("builtin").target.cpu.arch.endian();
2const expectEqual = std.testing.expectEqual;3const expectEqual = std.testing.expectEqual;
34
4test "pointer casting" {5test "pointer casting" {
5 const bytes align(@alignOf(u32)) = [_]u8{ 0x12, 0x12, 0x12, 0x12 };6 const bytes: [4]u8 align(@alignOf(u32)) = .{ 0x10, 0x20, 0x30, 0x40 };
6 const u32_ptr: *const u32 = @ptrCast(&bytes);7 const u32_ptr: *const u32 = @ptrCast(&bytes);
7 try expectEqual(0x12121212, u32_ptr.*);
88
9 // Even this example is contrived - there are better ways to do the above than9 // Because we directly reinterpreted bytes of memory, the `u32` value we
10 // pointer casting. For example, using a slice narrowing cast:10 // load from `u32_ptr` depends on the target endian:
11 const u32_value = std.mem.bytesAsSlice(u32, bytes[0..])[0];11 switch (native_endian) {
12 try expectEqual(0x12121212, u32_value);12 .little => try expectEqual(0x40302010, u32_ptr.*),
13 .big => try expectEqual(0x10203040, u32_ptr.*),
14 }
1315
14 // And even another way, the most straightforward way to do it:16 // To instead reinterpret the logical bit representation of `bytes` with no
15 try expectEqual(0x12121212, @as(u32, @bitCast(bytes)));17 // dependency on the target endian, use `@bitCast`, which always places
18 // earlier array elements into less-significant bits:
19 try expectEqual(0x40302010, @as(u32, @bitCast(bytes)));
16}20}
1721
18test "pointer child type" {22test "pointer child type" {
lib/compiler_rt/float_from_int.zig+15-4
...@@ -64,10 +64,21 @@ inline fn limb(limbs: []const u32, index: usize) u32 {...@@ -64,10 +64,21 @@ inline fn limb(limbs: []const u32, index: usize) u32 {
64pub inline fn floatFromBigInt(comptime T: type, comptime signedness: std.builtin.Signedness, x: []const u32) T {64pub inline fn floatFromBigInt(comptime T: type, comptime signedness: std.builtin.Signedness, x: []const u32) T {
65 switch (x.len) {65 switch (x.len) {
66 0 => return 0,66 0 => return 0,
67 inline 1...4 => |limbs_len| return @floatFromInt(@as(67 inline 1...4 => |limbs_len| {
68 @Int(signedness, 32 * limbs_len),68 const low_to_high: [limbs_len]u32 = switch (@import("builtin").cpu.arch.endian()) {
69 @bitCast(x[0..limbs_len].*),69 .little => x[0..limbs_len].*,
70 )),70 .big => switch (limbs_len) {
71 1 => .{x[0]},
72 2 => .{ x[1], x[0] },
73 3 => .{ x[2], x[1], x[0] },
74 4 => .{ x[3], x[2], x[1], x[0] },
75 else => comptime unreachable,
76 },
77 };
78 const I = @Int(signedness, 32 * limbs_len);
79 const int: I = @bitCast(low_to_high);
80 return @floatFromInt(int);
81 },
71 else => {},82 else => {},
72 }83 }
7384
lib/compiler_rt/int_from_float.zig+12-4
...@@ -80,10 +80,18 @@ pub inline fn bigIntFromFloat(comptime signedness: std.builtin.Signedness, resul...@@ -80,10 +80,18 @@ pub inline fn bigIntFromFloat(comptime signedness: std.builtin.Signedness, resul
80 switch (result.len) {80 switch (result.len) {
81 0 => return,81 0 => return,
82 inline 1...4 => |limbs_len| {82 inline 1...4 => |limbs_len| {
83 result[0..limbs_len].* = @bitCast(@as(83 const I = @Int(signedness, 32 * limbs_len);
84 @Int(signedness, 32 * limbs_len),84 const low_to_high: [limbs_len]u32 = @bitCast(@as(I, @intFromFloat(a)));
85 @intFromFloat(a),85 result[0..limbs_len].* = switch (@import("builtin").cpu.arch.endian()) {
86 ));86 .little => low_to_high,
87 .big => switch (limbs_len) {
88 1 => .{low_to_high[0]},
89 2 => .{ low_to_high[1], low_to_high[0] },
90 3 => .{ low_to_high[2], low_to_high[1], low_to_high[0] },
91 4 => .{ low_to_high[3], low_to_high[2], low_to_high[1], low_to_high[0] },
92 else => comptime unreachable,
93 },
94 };
87 return;95 return;
88 },96 },
89 else => {},97 else => {},
lib/compiler_rt/limb64.zig+12-6
...@@ -75,7 +75,17 @@ fn asLimbs(v: anytype) Limbs(@TypeOf(v)) {...@@ -75,7 +75,17 @@ fn asLimbs(v: anytype) Limbs(@TypeOf(v)) {
75 const int_info = @typeInfo(T).int;75 const int_info = @typeInfo(T).int;
76 const limb_cnt = comptime limbCount(int_info.bits);76 const limb_cnt = comptime limbCount(int_info.bits);
77 const ET = @Int(int_info.signedness, limb_cnt * 64);77 const ET = @Int(int_info.signedness, limb_cnt * 64);
78 return @bitCast(@as(ET, v));78 const low_to_high: Limbs(T) = @bitCast(@as(ET, v));
79 switch (endian) {
80 .little => return low_to_high,
81 .big => {
82 var swapped: Limbs(T) = undefined;
83 for (low_to_high, 0..) |x, i| {
84 swapped[limb_cnt - i - 1] = x;
85 }
86 return swapped;
87 },
88 }
79}89}
8090
81fn limbWrap(limb: u64, is_signed: bool, bits: u16) u64 {91fn limbWrap(limb: u64, is_signed: bool, bits: u16) u64 {
...@@ -944,11 +954,7 @@ inline fn add3(x: *[3]u64, start: usize, v0: u64) void {...@@ -944,11 +954,7 @@ inline fn add3(x: *[3]u64, start: usize, v0: u64) void {
944954
945fn mulwide(a: u64, b: u64) [2]u64 {955fn mulwide(a: u64, b: u64) [2]u64 {
946 const muldXi = @import("mulXi3.zig").muldXi;956 const muldXi = @import("mulXi3.zig").muldXi;
947 const limbs: [2]u64 = @bitCast(muldXi(u64, a, b));957 return @bitCast(muldXi(u64, a, b));
948 return switch (endian) {
949 .little => limbs,
950 .big => .{ limbs[1], limbs[0] },
951 };
952}958}
953959
954fn __mulo_limb64(out_ptr: [*]u64, a_ptr: [*]const u64, b_ptr: [*]const u64, is_signed: bool, bits: u16) callconv(.c) bool {960fn __mulo_limb64(out_ptr: [*]u64, a_ptr: [*]const u64, b_ptr: [*]const u64, is_signed: bool, bits: u16) callconv(.c) bool {
lib/compiler_rt/udivmod.zig+32-38
...@@ -61,12 +61,6 @@ pub fn __umodti3(a: u128, b: u128) callconv(.c) u128 {...@@ -61,12 +61,6 @@ pub fn __umodti3(a: u128, b: u128) callconv(.c) u128 {
61 return r;61 return r;
62}62}
6363
64const lo = switch (builtin.cpu.arch.endian()) {
65 .big => 1,
66 .little => 0,
67};
68const hi = 1 - lo;
69
70// Let _u1 and _u0 be the high and low limbs of U respectively.64// Let _u1 and _u0 be the high and low limbs of U respectively.
71// Returns U / v_ and sets r = U % v_.65// Returns U / v_ and sets r = U % v_.
72fn divwide_generic(comptime T: type, _u1: T, _u0: T, v_: T, r: *T) T {66fn divwide_generic(comptime T: type, _u1: T, _u0: T, v_: T, r: *T) T {
...@@ -158,22 +152,22 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {...@@ -158,22 +152,22 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
158 return 0;152 return 0;
159 }153 }
160154
161 const a: [2]HalfT = @bitCast(a_);155 const a: [2]HalfT = @bitCast(a_); // [0] is low bits, [1] is high bits
162 const b: [2]HalfT = @bitCast(b_);156 const b: [2]HalfT = @bitCast(b_); // [0] is low bits, [1] is high bits
163 var q: [2]HalfT = undefined;157 var q: [2]HalfT = undefined;
164 var r: [2]HalfT = undefined;158 var r: [2]HalfT = undefined;
165159
166 // When the divisor fits in 64 bits, we can use an optimized path160 // When the divisor fits in 64 bits, we can use an optimized path
167 if (b[hi] == 0) {161 if (b[1] == 0) {
168 r[hi] = 0;162 r[1] = 0;
169 if (a[hi] < b[lo]) {163 if (a[1] < b[0]) {
170 // The result fits in 64 bits164 // The result fits in 64 bits
171 q[hi] = 0;165 q[1] = 0;
172 q[lo] = divwide(HalfT, a[hi], a[lo], b[lo], &r[lo]);166 q[0] = divwide(HalfT, a[1], a[0], b[0], &r[0]);
173 } else {167 } else {
174 // First, divide with the high part to get the remainder. After that a_hi < b_lo.168 // First, divide with the high part to get the remainder. After that a_hi < b_lo.
175 q[hi] = a[hi] / b[lo];169 q[1] = a[1] / b[0];
176 q[lo] = divwide(HalfT, a[hi] % b[lo], a[lo], b[lo], &r[lo]);170 q[0] = divwide(HalfT, a[1] % b[0], a[0], b[0], &r[0]);
177 }171 }
178 if (maybe_rem) |rem| {172 if (maybe_rem) |rem| {
179 rem.* = @bitCast(r);173 rem.* = @bitCast(r);
...@@ -181,21 +175,21 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {...@@ -181,21 +175,21 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
181 return @bitCast(q);175 return @bitCast(q);
182 }176 }
183177
184 // Large-divisor case: b[hi] != 0, so the quotient fits in one HalfT word.178 // Large-divisor case: b[1] != 0, so the quotient fits in one HalfT word.
185 //179 //
186 // Trial quotient via divwide (Knuth Vol 2, Section 4.3.1):180 // Trial quotient via divwide (Knuth Vol 2, Section 4.3.1):
187 // Normalize the divisor so its high half has the MSB set, then use divwide181 // Normalize the divisor so its high half has the MSB set, then use divwide
188 // on the top bits to get a trial quotient that is at most 1 too large.182 // on the top bits to get a trial quotient that is at most 1 too large.
189 // This replaces the O(shift) bit-by-bit loop with O(1) operations.183 // This replaces the O(shift) bit-by-bit loop with O(1) operations.
190 const s: Log2Int(HalfT) = @intCast(@clz(b[hi]));184 const s: Log2Int(HalfT) = @intCast(@clz(b[1]));
191185
192 if (s == 0) {186 if (s == 0) {
193 // b[hi] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b187 // b[1] already has its MSB set, so b >= 2^(T_bits - 1). Since a >= b
194 // (we passed the b_ > a_ check), a >= 2^(T_bits - 1) too, meaning188 // (we passed the b_ > a_ check), a >= 2^(T_bits - 1) too, meaning
195 // a[hi] also has its MSB set. Therefore a / b < 2, and the quotient189 // a[1] also has its MSB set. Therefore a / b < 2, and the quotient
196 // is exactly 1.190 // is exactly 1.
197 q = @bitCast(@as(T, 0));191 q = @bitCast(@as(T, 0));
198 q[lo] = 1;192 q[0] = 1;
199 if (maybe_rem) |rem| {193 if (maybe_rem) |rem| {
200 rem.* = a_ - b_;194 rem.* = a_ - b_;
201 }195 }
...@@ -207,12 +201,12 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {...@@ -207,12 +201,12 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
207 std.math.IntFittingRange(0, half_bits),201 std.math.IntFittingRange(0, half_bits),
208 @intCast(s),202 @intCast(s),
209 ));203 ));
210 const bn_hi: HalfT = (b[hi] << s) | (b[lo] >> sr);204 const bn_hi: HalfT = (b[1] << s) | (b[0] >> sr);
211205
212 // Trial numerator: the top (half_bits + s) bits of (a << s), as [a2:a1].206 // Trial numerator: the top (half_bits + s) bits of (a << s), as [a2:a1].
213 // a2 < bn_hi is guaranteed since a2 < 2^s and bn_hi >= 2^(half_bits - 1).207 // a2 < bn_hi is guaranteed since a2 < 2^s and bn_hi >= 2^(half_bits - 1).
214 const a2: HalfT = a[hi] >> sr;208 const a2: HalfT = a[1] >> sr;
215 const a1: HalfT = (a[hi] << s) | (a[lo] >> sr);209 const a1: HalfT = (a[1] << s) | (a[0] >> sr);
216210
217 // Trial quotient via divwide: q_hat = floor([a2:a1] / bn_hi).211 // Trial quotient via divwide: q_hat = floor([a2:a1] / bn_hi).
218 // By Knuth's theorem (normalized divisor), q <= q_hat <= q + 1.212 // By Knuth's theorem (normalized divisor), q <= q_hat <= q + 1.
...@@ -223,42 +217,42 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {...@@ -223,42 +217,42 @@ pub fn udivmod(comptime T: type, a_: T, b_: T, maybe_rem: ?*T) T {
223 // Compute the product using HalfT * HalfT -> T widening multiplications,217 // Compute the product using HalfT * HalfT -> T widening multiplications,
224 // which are native single-instruction ops when HalfT fits in a register218 // which are native single-instruction ops when HalfT fits in a register
225 // (e.g. u64 * u64 -> u128 via mulq on x86_64, mul on aarch64).219 // (e.g. u64 * u64 -> u128 via mulq on x86_64, mul on aarch64).
226 // product = q_hat * [b[hi]:b[lo]] = [p_top : p_mid : p_lo] (3 half-words)220 // product = q_hat * [b[1]:b[0]] = [p_top : p_mid : p_lo] (3 half-words)
227 const prod_lo: T = @as(T, q_hat) * @as(T, b[lo]);221 const prod_lo: T = @as(T, q_hat) * @as(T, b[0]);
228 const prod_hi: T = @as(T, q_hat) * @as(T, b[hi]);222 const prod_hi: T = @as(T, q_hat) * @as(T, b[1]);
229223
230 const prod_lo_parts: [2]HalfT = @bitCast(prod_lo);224 const prod_lo_parts: [2]HalfT = @bitCast(prod_lo);
231 const prod_hi_parts: [2]HalfT = @bitCast(prod_hi);225 const prod_hi_parts: [2]HalfT = @bitCast(prod_hi);
232226
233 const mid_add = @addWithOverflow(prod_hi_parts[lo], prod_lo_parts[hi]);227 const mid_add = @addWithOverflow(prod_hi_parts[0], prod_lo_parts[1]);
234 var p_mid: HalfT = mid_add[0];228 var p_mid: HalfT = mid_add[0];
235 const p_top: HalfT = prod_hi_parts[hi] +% @as(HalfT, mid_add[1]);229 const p_top: HalfT = prod_hi_parts[1] +% @as(HalfT, mid_add[1]);
236 var p_lo: HalfT = prod_lo_parts[lo];230 var p_lo: HalfT = prod_lo_parts[0];
237231
238 // If product > a, decrement q_hat (at most once, guaranteed by Knuth).232 // If product > a, decrement q_hat (at most once, guaranteed by Knuth).
239 if (p_top > 0 or p_mid > a[hi] or (p_mid == a[hi] and p_lo > a[lo])) {233 if (p_top > 0 or p_mid > a[1] or (p_mid == a[1] and p_lo > a[0])) {
240 q_hat -= 1;234 q_hat -= 1;
241 // Subtract b from the product for correct remainder computation.235 // Subtract b from the product for correct remainder computation.
242 // After correction, (q_hat * b) fits in T bits, so borrows into236 // After correction, (q_hat * b) fits in T bits, so borrows into
243 // p_top cancel it to zero -- we only need [p_mid:p_lo].237 // p_top cancel it to zero -- we only need [p_mid:p_lo].
244 const sub_lo = @subWithOverflow(p_lo, b[lo]);238 const sub_lo = @subWithOverflow(p_lo, b[0]);
245 p_lo = sub_lo[0];239 p_lo = sub_lo[0];
246 const sub_mid = @subWithOverflow(p_mid, b[hi]);240 const sub_mid = @subWithOverflow(p_mid, b[1]);
247 const sub_mid2 = @subWithOverflow(sub_mid[0], @as(HalfT, sub_lo[1]));241 const sub_mid2 = @subWithOverflow(sub_mid[0], @as(HalfT, sub_lo[1]));
248 p_mid = sub_mid2[0];242 p_mid = sub_mid2[0];
249 }243 }
250244
251 q = @bitCast(@as(T, 0));245 q = @bitCast(@as(T, 0));
252 q[lo] = q_hat;246 q[0] = q_hat;
253247
254 if (maybe_rem) |rem| {248 if (maybe_rem) |rem| {
255 // remainder = a - q_hat * b = [a[hi]:a[lo]] - [p_mid:p_lo]249 // remainder = a - q_hat * b = [a[1]:a[0]] - [p_mid:p_lo]
256 // This subtraction is non-negative since q_hat <= true quotient.250 // This subtraction is non-negative since q_hat <= true quotient.
257 const rem_lo = @subWithOverflow(a[lo], p_lo);251 const rem_lo = @subWithOverflow(a[0], p_lo);
258 r[lo] = rem_lo[0];252 r[0] = rem_lo[0];
259 const rem_hi = @subWithOverflow(a[hi], p_mid);253 const rem_hi = @subWithOverflow(a[1], p_mid);
260 const rem_hi2 = @subWithOverflow(rem_hi[0], @as(HalfT, rem_lo[1]));254 const rem_hi2 = @subWithOverflow(rem_hi[0], @as(HalfT, rem_lo[1]));
261 r[hi] = rem_hi2[0];255 r[1] = rem_hi2[0];
262 rem.* = @bitCast(r);256 rem.* = @bitCast(r);
263 }257 }
264 return @bitCast(q);258 return @bitCast(q);
lib/std/Build/Configuration.zig+4-2
...@@ -3215,7 +3215,8 @@ pub const Storage = enum {...@@ -3215,7 +3215,8 @@ pub const Storage = enum {
3215 .@"extern" => {3215 .@"extern" => {
3216 const n = @divExact(@sizeOf(Field), @sizeOf(u32));3216 const n = @divExact(@sizeOf(Field), @sizeOf(u32));
3217 defer i.* += n;3217 defer i.* += n;
3218 return @bitCast(buffer[i.*..][0..n].*);3218 const ptr: *align(@alignOf(u32)) const Field = @ptrCast(buffer[i.*..][0..n]);
3219 return ptr.*;
3219 },3220 },
3220 },3221 },
3221 else => comptime unreachable,3222 else => comptime unreachable,
...@@ -3381,7 +3382,8 @@ pub const Storage = enum {...@@ -3381,7 +3382,8 @@ pub const Storage = enum {
3381 },3382 },
3382 .@"extern" => {3383 .@"extern" => {
3383 const n = @divExact(@sizeOf(Field), @sizeOf(u32));3384 const n = @divExact(@sizeOf(Field), @sizeOf(u32));
3384 buffer[i..][0..n].* = @bitCast(value);3385 const ptr: *align(@alignOf(Field)) const [n]u32 = @ptrCast(&value);
3386 buffer[i..][0..n].* = ptr.*;
3385 return n;3387 return n;
3386 },3388 },
3387 },3389 },
lib/std/Io/Threaded.zig+6-2
...@@ -14188,9 +14188,11 @@ fn addressUnixToPosix(a: *const net.UnixAddress, storage: *UnixAddress) posix.so...@@ -14188,9 +14188,11 @@ fn addressUnixToPosix(a: *const net.UnixAddress, storage: *UnixAddress) posix.so
14188}14188}
1418914189
14190fn address4FromPosix(in: *const posix.sockaddr.in) net.Ip4Address {14190fn address4FromPosix(in: *const posix.sockaddr.in) net.Ip4Address {
14191 // The network byte order address in `in.addr` is already the byte order we want.
14192 const addr_bytes: *const [4]u8 = @ptrCast(&in.addr);
14191 return .{14193 return .{
14192 .port = std.mem.bigToNative(u16, in.port),14194 .port = std.mem.bigToNative(u16, in.port),
14193 .bytes = @bitCast(in.addr),14195 .bytes = addr_bytes.*,
14194 };14196 };
14195}14197}
1419614198
...@@ -14204,9 +14206,11 @@ fn address6FromPosix(in6: *const posix.sockaddr.in6) net.Ip6Address {...@@ -14204,9 +14206,11 @@ fn address6FromPosix(in6: *const posix.sockaddr.in6) net.Ip6Address {
14204}14206}
1420514207
14206fn address4ToPosix(a: net.Ip4Address) posix.sockaddr.in {14208fn address4ToPosix(a: net.Ip4Address) posix.sockaddr.in {
14209 // The byte order of `a.bytes` is already equivalent to a network byte order address.
14210 const addr_raw: *align(1) const u32 = @ptrCast(&a.bytes);
14207 return .{14211 return .{
14208 .port = std.mem.nativeToBig(u16, a.port),14212 .port = std.mem.nativeToBig(u16, a.port),
14209 .addr = @bitCast(a.bytes),14213 .addr = addr_raw.*,
14210 };14214 };
14211}14215}
1421214216
lib/std/Io/net.zig+1-5
...@@ -592,11 +592,7 @@ pub const Ip6Address = struct {...@@ -592,11 +592,7 @@ pub const Ip6Address = struct {
592 if (remaining != 0) return .incomplete;592 if (remaining != 0) return .incomplete;
593 }593 }
594594
595 // Workaround that can be removed when this proposal is595 for (&parts) |*part| part.* = @byteSwap(part.*);
596 // implemented https://github.com/ziglang/zig/issues/19755
597 if ((comptime @import("builtin").cpu.arch.endian()) != .big) {
598 for (&parts) |*part| part.* = @byteSwap(part.*);
599 }
600596
601 return .{ .success = .{597 return .{ .success = .{
602 .bytes = @bitCast(parts),598 .bytes = @bitCast(parts),
lib/std/Random/RomuTrio.zig+4-2
...@@ -5,6 +5,7 @@...@@ -5,6 +5,7 @@
5const std = @import("std");5const std = @import("std");
6const math = std.math;6const math = std.math;
7const RomuTrio = @This();7const RomuTrio = @This();
8const builtin = @import("builtin");
89
9x_state: u64,10x_state: u64,
10y_state: u64,11y_state: u64,
...@@ -33,7 +34,7 @@ fn next(self: *RomuTrio) u64 {...@@ -33,7 +34,7 @@ fn next(self: *RomuTrio) u64 {
33}34}
3435
35pub fn seedWithBuf(self: *RomuTrio, buf: [24]u8) void {36pub fn seedWithBuf(self: *RomuTrio, buf: [24]u8) void {
36 const seed_buf = @as([3]u64, @bitCast(buf));37 const seed_buf: [3]u64 = @bitCast(buf);
37 self.x_state = seed_buf[0];38 self.x_state = seed_buf[0];
38 self.y_state = seed_buf[1];39 self.y_state = seed_buf[1];
39 self.z_state = seed_buf[2];40 self.z_state = seed_buf[2];
...@@ -121,7 +122,8 @@ test fill {...@@ -121,7 +122,8 @@ test fill {
121}122}
122123
123test "buf seeding test" {124test "buf seeding test" {
124 const buf0 = @as([24]u8, @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 }));125 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
126 const buf0: [24]u8 = @bitCast([3]u64{ 16294208416658607535, 13964609475759908645, 4703697494102998476 });
125 const resulting_state = .{ .x = 16294208416658607535, .y = 13964609475759908645, .z = 4703697494102998476 };127 const resulting_state = .{ .x = 16294208416658607535, .y = 13964609475759908645, .z = 4703697494102998476 };
126 var r = RomuTrio.init(0);128 var r = RomuTrio.init(0);
127 r.seedWithBuf(buf0);129 r.seedWithBuf(buf0);
lib/std/Random/Xoshiro256.zig+2-2
...@@ -46,12 +46,12 @@ pub fn jump(self: *Xoshiro256) void {...@@ -46,12 +46,12 @@ pub fn jump(self: *Xoshiro256) void {
4646
47 while (table != 0) : (table >>= 1) {47 while (table != 0) : (table >>= 1) {
48 if (@as(u1, @truncate(table)) != 0) {48 if (@as(u1, @truncate(table)) != 0) {
49 s ^= @as(u256, @bitCast(self.s));49 s ^= @bitCast(self.s);
50 }50 }
51 _ = self.next();51 _ = self.next();
52 }52 }
5353
54 self.s = @as([4]u64, @bitCast(s));54 self.s = @bitCast(s);
55}55}
5656
57pub fn seed(self: *Xoshiro256, init_s: u64) void {57pub fn seed(self: *Xoshiro256, init_s: u64) void {
lib/std/crypto/tls/Client.zig+5-5
...@@ -376,7 +376,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client...@@ -376,7 +376,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
376 const nonce = nonce: {376 const nonce = nonce: {
377 const V = @Vector(P.AEAD.nonce_length, u8);377 const V = @Vector(P.AEAD.nonce_length, u8);
378 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);378 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
379 const operand: V = pad ++ @as([8]u8, @bitCast(big(read_seq)));379 const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(read_seq)));
380 break :nonce @as(V, pv.server_handshake_iv) ^ operand;380 break :nonce @as(V, pv.server_handshake_iv) ^ operand;
381 };381 };
382 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, record_header, nonce, pv.server_handshake_key) catch382 P.AEAD.decrypt(cleartext, ciphertext, auth_tag, record_header, nonce, pv.server_handshake_key) catch
...@@ -416,7 +416,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client...@@ -416,7 +416,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
416 const nonce: [P.AEAD.nonce_length]u8 = nonce: {416 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
417 const V = @Vector(P.AEAD.nonce_length, u8);417 const V = @Vector(P.AEAD.nonce_length, u8);
418 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);418 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
419 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));419 const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
420 break :nonce @as(V, pv.app_cipher.server_write_IV ++ record_iv) ^ operand;420 break :nonce @as(V, pv.app_cipher.server_write_IV ++ record_iv) ^ operand;
421 };421 };
422 const ciphertext = record_decoder.slice(message_len);422 const ciphertext = record_decoder.slice(message_len);
...@@ -792,7 +792,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client...@@ -792,7 +792,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client
792 const nonce: [P.AEAD.nonce_length]u8 = nonce: {792 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
793 const V = @Vector(P.AEAD.nonce_length, u8);793 const V = @Vector(P.AEAD.nonce_length, u8);
794 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);794 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
795 const operand: V = pad ++ @as([8]u8, @bitCast(big(write_seq)));795 const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(write_seq)));
796 break :nonce @as(V, pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt) ^ operand;796 break :nonce @as(V, pv.app_cipher.client_write_IV ++ pv.app_cipher.client_salt) ^ operand;
797 };797 };
798 var client_verify_msg = .{@intFromEnum(tls.ContentType.handshake)} ++798 var client_verify_msg = .{@intFromEnum(tls.ContentType.handshake)} ++
...@@ -1105,7 +1105,7 @@ fn prepareCiphertextRecord(...@@ -1105,7 +1105,7 @@ fn prepareCiphertextRecord(
1105 const nonce: [P.AEAD.nonce_length]u8 = nonce: {1105 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1106 const V = @Vector(P.AEAD.nonce_length, u8);1106 const V = @Vector(P.AEAD.nonce_length, u8);
1107 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);1107 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
1108 const operand: V = pad ++ @as([8]u8, @bitCast(big(c.write_seq)));1108 const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(c.write_seq)));
1109 break :nonce @as(V, pv.client_write_IV ++ pv.client_salt) ^ operand;1109 break :nonce @as(V, pv.client_write_IV ++ pv.client_salt) ^ operand;
1110 };1110 };
1111 record_iv.* = nonce[P.fixed_iv_length..].*;1111 record_iv.* = nonce[P.fixed_iv_length..].*;
...@@ -1214,7 +1214,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {...@@ -1214,7 +1214,7 @@ fn readIndirect(c: *Client) Reader.Error!usize {
1214 const nonce: [P.AEAD.nonce_length]u8 = nonce: {1214 const nonce: [P.AEAD.nonce_length]u8 = nonce: {
1215 const V = @Vector(P.AEAD.nonce_length, u8);1215 const V = @Vector(P.AEAD.nonce_length, u8);
1216 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);1216 const pad: [P.AEAD.nonce_length - 8]u8 = @splat(0);
1217 const operand: V = pad ++ @as([8]u8, @bitCast(big(masked_read_seq)));1217 const operand: V = pad ++ @as([8]u8, @bitCast(@byteSwap(masked_read_seq)));
1218 break :nonce @as(V, pv.server_write_IV ++ record_iv) ^ operand;1218 break :nonce @as(V, pv.server_write_IV ++ record_iv) ^ operand;
1219 };1219 };
1220 const ciphertext = input.take(message_len) catch unreachable; // already peeked1220 const ciphertext = input.take(message_len) catch unreachable; // already peeked
lib/std/hash/xxhash.zig+68-57
...@@ -2,7 +2,6 @@ const std = @import("std");...@@ -2,7 +2,6 @@ const std = @import("std");
2const builtin = @import("builtin");2const builtin = @import("builtin");
3const mem = std.mem;3const mem = std.mem;
4const expectEqual = std.testing.expectEqual;4const expectEqual = std.testing.expectEqual;
5const native_endian = builtin.cpu.arch.endian();
65
7const rotl = std.math.rotl;6const rotl = std.math.rotl;
87
...@@ -421,7 +420,18 @@ pub const XxHash32 = struct {...@@ -421,7 +420,18 @@ pub const XxHash32 = struct {
421};420};
422421
423pub const XxHash3 = struct {422pub const XxHash3 = struct {
423 const block_bytes = 64;
424 const Block = @Vector(8, u64);424 const Block = @Vector(8, u64);
425 const InputBlock = extern struct {
426 raw: [block_bytes]u8,
427 inline fn load(ptr: *const InputBlock) Block {
428 return @bitCast(ptr.raw);
429 }
430 inline fn store(ptr: *InputBlock, val: Block) void {
431 ptr.raw = @bitCast(val);
432 }
433 };
434
425 const default_secret: [192]u8 = .{435 const default_secret: [192]u8 = .{
426 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,436 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
427 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,437 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
...@@ -464,10 +474,6 @@ pub const XxHash3 = struct {...@@ -464,10 +474,6 @@ pub const XxHash3 = struct {
464 return wide[0] ^ wide[1];474 return wide[0] ^ wide[1];
465 }475 }
466476
467 inline fn swap(x: anytype) @TypeOf(x) {
468 return if (native_endian == .big) @byteSwap(x) else x;
469 }
470
471 inline fn disableAutoVectorization(x: anytype) void {477 inline fn disableAutoVectorization(x: anytype) void {
472 if (!@inComptime()) asm volatile (""478 if (!@inComptime()) asm volatile (""
473 :479 :
...@@ -476,20 +482,20 @@ pub const XxHash3 = struct {...@@ -476,20 +482,20 @@ pub const XxHash3 = struct {
476 }482 }
477483
478 inline fn mix16(seed: u64, input: []const u8, secret: []const u8) u64 {484 inline fn mix16(seed: u64, input: []const u8, secret: []const u8) u64 {
479 const blk: [4]u64 = @bitCast([_][16]u8{ input[0..16].*, secret[0..16].* });485 const blk: [4]u64 = @bitCast([2][16]u8{ input[0..16].*, secret[0..16].* });
480 disableAutoVectorization(seed);486 disableAutoVectorization(seed);
481487
482 return fold(488 return fold(
483 swap(blk[0]) ^ (swap(blk[2]) +% seed),489 blk[0] ^ (blk[2] +% seed),
484 swap(blk[1]) ^ (swap(blk[3]) -% seed),490 blk[1] ^ (blk[3] -% seed),
485 );491 );
486 }492 }
487493
488 const Accumulator = extern struct {494 const Accumulator = struct {
489 consumed: usize = 0,495 consumed: usize = 0,
490 seed: u64,496 seed: u64,
491 secret: [192]u8 = undefined,497 secret: [192]u8,
492 state: Block = Block{498 state: Block = .{
493 XxHash32.prime_3,499 XxHash32.prime_3,
494 XxHash64.prime_1,500 XxHash64.prime_1,
495 XxHash64.prime_2,501 XxHash64.prime_2,
...@@ -501,33 +507,35 @@ pub const XxHash3 = struct {...@@ -501,33 +507,35 @@ pub const XxHash3 = struct {
501 },507 },
502508
503 inline fn init(seed: u64) Accumulator {509 inline fn init(seed: u64) Accumulator {
504 var self = Accumulator{ .seed = seed };510 const seed_block: Block = .{
505 for (511 seed, @as(u64, 0) -% seed,
506 std.mem.bytesAsSlice(Block, &self.secret),512 seed, @as(u64, 0) -% seed,
507 std.mem.bytesAsSlice(Block, &default_secret),513 seed, @as(u64, 0) -% seed,
508 ) |*dst, src| {514 seed, @as(u64, 0) -% seed,
509 dst.* = swap(swap(src) +% Block{515 };
510 seed, @as(u64, 0) -% seed,516
511 seed, @as(u64, 0) -% seed,517 var secret: [192]u8 = undefined;
512 seed, @as(u64, 0) -% seed,518 const secret_blocks: []InputBlock = @ptrCast(&secret);
513 seed, @as(u64, 0) -% seed,519 const default_secret_blocks: []const InputBlock = @ptrCast(&default_secret);
514 });520 for (secret_blocks, default_secret_blocks) |*dst, *src| {
521 dst.store(src.load() +% seed_block);
515 }522 }
516 return self;523
524 return .{ .seed = seed, .secret = secret };
517 }525 }
518526
519 inline fn round(527 inline fn round(
520 noalias state: *Block,528 noalias state: *Block,
521 noalias input_block: *align(1) const Block,529 noalias input_block: *const InputBlock,
522 noalias secret_block: *align(1) const Block,530 noalias secret_block: *const InputBlock,
523 ) void {531 ) void {
524 const data = swap(input_block.*);532 const data = input_block.load();
525 const mixed = data ^ swap(secret_block.*);533 const mixed = data ^ secret_block.load();
526 state.* +%= (mixed & @as(Block, @splat(0xffffffff))) *% (mixed >> @splat(32));534 state.* +%= (mixed & @as(Block, @splat(0xffffffff))) *% (mixed >> @splat(32));
527 state.* +%= @shuffle(u64, data, undefined, [_]i32{ 1, 0, 3, 2, 5, 4, 7, 6 });535 state.* +%= @shuffle(u64, data, undefined, [_]i32{ 1, 0, 3, 2, 5, 4, 7, 6 });
528 }536 }
529537
530 fn accumulate(noalias self: *Accumulator, blocks: []align(1) const Block) void {538 fn accumulate(noalias self: *Accumulator, blocks: []const InputBlock) void {
531 const secret = std.mem.bytesAsSlice(u64, self.secret[self.consumed * 8 ..]);539 const secret = std.mem.bytesAsSlice(u64, self.secret[self.consumed * 8 ..]);
532 for (blocks, secret[0..blocks.len]) |*input_block, *secret_block| {540 for (blocks, secret[0..blocks.len]) |*input_block, *secret_block| {
533 @prefetch(@as([*]const u8, @ptrCast(input_block)) + 320, .{});541 @prefetch(@as([*]const u8, @ptrCast(input_block)) + 320, .{});
...@@ -536,14 +544,14 @@ pub const XxHash3 = struct {...@@ -536,14 +544,14 @@ pub const XxHash3 = struct {
536 }544 }
537545
538 fn scramble(self: *Accumulator) void {546 fn scramble(self: *Accumulator) void {
539 const secret_block: Block = @bitCast(self.secret[192 - @sizeOf(Block) .. 192].*);547 const secret_block: Block = @bitCast(self.secret[192 - block_bytes .. 192].*);
540 self.state ^= self.state >> @splat(47);548 self.state ^= self.state >> @splat(47);
541 self.state ^= swap(secret_block);549 self.state ^= secret_block;
542 self.state *%= @as(Block, @splat(XxHash32.prime_1));550 self.state *%= @as(Block, @splat(XxHash32.prime_1));
543 }551 }
544552
545 fn consume(noalias self: *Accumulator, input_blocks: []align(1) const Block) void {553 fn consume(noalias self: *Accumulator, input_blocks: []const InputBlock) void {
546 const blocks_per_scramble = 1024 / @sizeOf(Block);554 const blocks_per_scramble = 1024 / block_bytes;
547 std.debug.assert(self.consumed <= blocks_per_scramble);555 std.debug.assert(self.consumed <= blocks_per_scramble);
548556
549 var blocks = input_blocks;557 var blocks = input_blocks;
...@@ -561,12 +569,12 @@ pub const XxHash3 = struct {...@@ -561,12 +569,12 @@ pub const XxHash3 = struct {
561 self.consumed += blocks.len;569 self.consumed += blocks.len;
562 }570 }
563571
564 fn digest(noalias self: *Accumulator, total_len: u64, noalias last_block: *align(1) const Block) u64 {572 fn digest(noalias self: *Accumulator, total_len: u64, noalias last_block: *const InputBlock) u64 {
565 const secret_block = self.secret[192 - @sizeOf(Block) - 7 ..][0..@sizeOf(Block)];573 const secret_block = self.secret[192 - block_bytes - 7 ..][0..block_bytes];
566 round(&self.state, last_block, @ptrCast(secret_block));574 round(&self.state, last_block, @ptrCast(secret_block));
567575
568 const merge_block: Block = @bitCast(self.secret[11 .. 11 + @sizeOf(Block)].*);576 const merge_block: Block = @bitCast(self.secret[11 .. 11 + block_bytes].*);
569 self.state ^= swap(merge_block);577 self.state ^= merge_block;
570578
571 var result = XxHash64.prime_1 *% total_len;579 var result = XxHash64.prime_1 *% total_len;
572 inline for (0..4) |i| {580 inline for (0..4) |i| {
...@@ -588,7 +596,7 @@ pub const XxHash3 = struct {...@@ -588,7 +596,7 @@ pub const XxHash3 = struct {
588 if (input.len > 0) return hash3(seed, input, secret);596 if (input.len > 0) return hash3(seed, input, secret);
589597
590 const flip: [2]u64 = @bitCast(secret[56..72].*);598 const flip: [2]u64 = @bitCast(secret[56..72].*);
591 const key = swap(flip[0]) ^ swap(flip[1]);599 const key = flip[0] ^ flip[1];
592 return avalanche(.h64, seed ^ key);600 return avalanche(.h64, seed ^ key);
593 }601 }
594602
...@@ -604,8 +612,8 @@ pub const XxHash3 = struct {...@@ -604,8 +612,8 @@ pub const XxHash3 = struct {
604 input[input.len / 2],612 input[input.len / 2],
605 });613 });
606614
607 const key = @as(u64, swap(flip[0]) ^ swap(flip[1])) +% seed;615 const key = @as(u64, flip[0] ^ flip[1]) +% seed;
608 return avalanche(.h64, key ^ swap(blk));616 return avalanche(.h64, key ^ blk);
609 }617 }
610618
611 fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {619 fn hash8(seed: u64, input: anytype, noalias secret: *const [192]u8) u64 {
...@@ -619,8 +627,8 @@ pub const XxHash3 = struct {...@@ -619,8 +627,8 @@ pub const XxHash3 = struct {
619 });627 });
620628
621 const mixed = seed ^ (@as(u64, @byteSwap(@as(u32, @truncate(seed)))) << 32);629 const mixed = seed ^ (@as(u64, @byteSwap(@as(u32, @truncate(seed)))) << 32);
622 const key = (swap(flip[0]) ^ swap(flip[1])) -% mixed;630 const key = (flip[0] ^ flip[1]) -% mixed;
623 const combined = (@as(u64, swap(blk[0])) << 32) +% swap(blk[1]);631 const combined = (@as(u64, blk[0]) << 32) +% blk[1];
624 return avalanche(.{ .rrmxmx = input.len }, key ^ combined);632 return avalanche(.{ .rrmxmx = input.len }, key ^ combined);
625 }633 }
626634
...@@ -634,8 +642,8 @@ pub const XxHash3 = struct {...@@ -634,8 +642,8 @@ pub const XxHash3 = struct {
634 input[input.len - 8 ..][0..8].*,642 input[input.len - 8 ..][0..8].*,
635 });643 });
636644
637 const lo = swap(blk[0]) ^ ((swap(flip[0]) ^ swap(flip[1])) +% seed);645 const lo = blk[0] ^ ((flip[0] ^ flip[1]) +% seed);
638 const hi = swap(blk[1]) ^ ((swap(flip[2]) ^ swap(flip[3])) -% seed);646 const hi = blk[1] ^ ((flip[2] ^ flip[3]) -% seed);
639 const combined = @as(u64, input.len) +% @byteSwap(lo) +% hi +% fold(lo, hi);647 const combined = @as(u64, input.len) +% @byteSwap(lo) +% hi +% fold(lo, hi);
640 return avalanche(.h3, combined);648 return avalanche(.h3, combined);
641 }649 }
...@@ -679,11 +687,11 @@ pub const XxHash3 = struct {...@@ -679,11 +687,11 @@ pub const XxHash3 = struct {
679 @branchHint(.unlikely);687 @branchHint(.unlikely);
680 std.debug.assert(input.len >= 240);688 std.debug.assert(input.len >= 240);
681689
682 const block_count = ((input.len - 1) / @sizeOf(Block)) * @sizeOf(Block);690 const block_count = ((input.len - 1) / block_bytes) * block_bytes;
683 const last_block = input[input.len - @sizeOf(Block) ..][0..@sizeOf(Block)];691 const last_block = input[input.len - block_bytes ..][0..block_bytes];
684692
685 var acc = Accumulator.init(seed);693 var acc = Accumulator.init(seed);
686 acc.consume(std.mem.bytesAsSlice(Block, input[0..block_count]));694 acc.consume(std.mem.bytesAsSlice(InputBlock, input[0..block_count]));
687 return acc.digest(input.len, @ptrCast(last_block));695 return acc.digest(input.len, @ptrCast(last_block));
688 }696 }
689697
...@@ -716,21 +724,21 @@ pub const XxHash3 = struct {...@@ -716,21 +724,21 @@ pub const XxHash3 = struct {
716 @memcpy(self.buffer[self.buffered..], consumable[0..remaining]);724 @memcpy(self.buffer[self.buffered..], consumable[0..remaining]);
717 consumable = consumable[remaining..];725 consumable = consumable[remaining..];
718726
719 self.accumulator.consume(std.mem.bytesAsSlice(Block, &self.buffer));727 self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, &self.buffer));
720 self.buffered = 0;728 self.buffered = 0;
721 }729 }
722730
723 // The input isn't small enough to fit in the buffer. Consume it directly.731 // The input isn't small enough to fit in the buffer. Consume it directly.
724 if (consumable.len > self.buffer.len) {732 if (consumable.len > self.buffer.len) {
725 const block_count = ((consumable.len - 1) / @sizeOf(Block)) * @sizeOf(Block);733 const block_count = ((consumable.len - 1) / block_bytes) * block_bytes;
726 self.accumulator.consume(std.mem.bytesAsSlice(Block, consumable[0..block_count]));734 self.accumulator.consume(std.mem.bytesAsSlice(InputBlock, consumable[0..block_count]));
727 consumable = consumable[block_count..];735 consumable = consumable[block_count..];
728736
729 // In case we consume all remaining input, write the last block to end of the buffer737 // In case we consume all remaining input, write the last block to end of the buffer
730 // to populate the last_block_copy in final() similar to hashLong()'s last_block.738 // to populate the last_block_copy in final() similar to hashLong()'s last_block.
731 @memcpy(739 @memcpy(
732 self.buffer[self.buffer.len - @sizeOf(Block) .. self.buffer.len],740 self.buffer[self.buffer.len - block_bytes .. self.buffer.len],
733 (consumable.ptr - @sizeOf(Block))[0..@sizeOf(Block)],741 (consumable.ptr - block_bytes)[0..block_bytes],
734 );742 );
735 }743 }
736744
...@@ -751,16 +759,16 @@ pub const XxHash3 = struct {...@@ -751,16 +759,16 @@ pub const XxHash3 = struct {
751759
752 // Make a copy of the Accumulator state in case `self` needs to update() / be used later.760 // Make a copy of the Accumulator state in case `self` needs to update() / be used later.
753 var accumulator_copy = self.accumulator;761 var accumulator_copy = self.accumulator;
754 var last_block_copy: [@sizeOf(Block)]u8 = undefined;762 var last_block_copy: [block_bytes]u8 = undefined;
755763
756 // Digest the last block onthe Accumulator copy.764 // Digest the last block onthe Accumulator copy.
757 return accumulator_copy.digest(self.total_len, last_block: {765 return accumulator_copy.digest(self.total_len, last_block: {
758 if (self.buffered >= @sizeOf(Block)) {766 if (self.buffered >= block_bytes) {
759 const block_count = ((self.buffered - 1) / @sizeOf(Block)) * @sizeOf(Block);767 const block_count = ((self.buffered - 1) / block_bytes) * block_bytes;
760 accumulator_copy.consume(std.mem.bytesAsSlice(Block, self.buffer[0..block_count]));768 accumulator_copy.consume(std.mem.bytesAsSlice(InputBlock, self.buffer[0..block_count]));
761 break :last_block @ptrCast(self.buffer[self.buffered - @sizeOf(Block) ..][0..@sizeOf(Block)]);769 break :last_block @ptrCast(self.buffer[self.buffered - block_bytes ..][0..block_bytes]);
762 } else {770 } else {
763 const remaining = @sizeOf(Block) - self.buffered;771 const remaining = block_bytes - self.buffered;
764 @memcpy(last_block_copy[0..remaining], self.buffer[self.buffer.len - remaining ..][0..remaining]);772 @memcpy(last_block_copy[0..remaining], self.buffer[self.buffer.len - remaining ..][0..remaining]);
765 @memcpy(last_block_copy[remaining..][0..self.buffered], self.buffer[0..self.buffered]);773 @memcpy(last_block_copy[remaining..][0..self.buffered], self.buffer[0..self.buffered]);
766 break :last_block @ptrCast(&last_block_copy);774 break :last_block @ptrCast(&last_block_copy);
...@@ -780,6 +788,7 @@ fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64)...@@ -780,6 +788,7 @@ fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64)
780}788}
781789
782test "xxhash3" {790test "xxhash3" {
791 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
783 const H = XxHash3;792 const H = XxHash3;
784 // Non-Seeded Tests793 // Non-Seeded Tests
785 try testExpect(H, 0, "", 0x2d06800538d394c2);794 try testExpect(H, 0, "", 0x2d06800538d394c2);
...@@ -811,6 +820,7 @@ test "xxhash3" {...@@ -811,6 +820,7 @@ test "xxhash3" {
811}820}
812821
813test "xxhash3 smhasher" {822test "xxhash3 smhasher" {
823 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
814 const Test = struct {824 const Test = struct {
815 fn do() !void {825 fn do() !void {
816 try expectEqual(verify.smhasher(XxHash3.hash), 0x9a636405);826 try expectEqual(verify.smhasher(XxHash3.hash), 0x9a636405);
...@@ -822,6 +832,7 @@ test "xxhash3 smhasher" {...@@ -822,6 +832,7 @@ test "xxhash3 smhasher" {
822}832}
823833
824test "xxhash3 iterative api" {834test "xxhash3 iterative api" {
835 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
825 const Test = struct {836 const Test = struct {
826 fn do() !void {837 fn do() !void {
827 try verify.iterativeApi(XxHash3);838 try verify.iterativeApi(XxHash3);
lib/std/hash_map.zig+2-2
...@@ -593,8 +593,8 @@ fn Custom(...@@ -593,8 +593,8 @@ fn Custom(
593 fingerprint: FingerPrint = free,593 fingerprint: FingerPrint = free,
594 used: u1 = 0,594 used: u1 = 0,
595595
596 const slot_free = @as(u8, @bitCast(Metadata{ .fingerprint = free }));596 const slot_free: u8 = @bitCast(Metadata{ .fingerprint = free });
597 const slot_tombstone = @as(u8, @bitCast(Metadata{ .fingerprint = tombstone }));597 const slot_tombstone: u8 = @bitCast(Metadata{ .fingerprint = tombstone });
598598
599 pub fn isUsed(self: Metadata) bool {599 pub fn isUsed(self: Metadata) bool {
600 return self.used == 1;600 return self.used == 1;
lib/std/http/HeadParser.zig+5-8
...@@ -116,11 +116,8 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {...@@ -116,11 +116,8 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
116116
117 const chunk = bytes[index..][0..vector_len];117 const chunk = bytes[index..][0..vector_len];
118 const v: Vector = chunk.*;118 const v: Vector = chunk.*;
119 // depends on https://github.com/ziglang/zig/issues/19755119 const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));
120 // const matches_r: BitVector = @bitCast(v == @as(Vector, @splat('\r')));120 const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
121 // const matches_n: BitVector = @bitCast(v == @as(Vector, @splat('\n')));
122 const matches_r: BitVector = @select(u1, v == @as(Vector, @splat('\r')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
123 const matches_n: BitVector = @select(u1, v == @as(Vector, @splat('\n')), @as(Vector, @splat(1)), @as(Vector, @splat(0)));
124 const matches_or: SizeVector = matches_r | matches_n;121 const matches_or: SizeVector = matches_r | matches_n;
125122
126 const matches = @reduce(.Add, matches_or);123 const matches = @reduce(.Add, matches_or);
...@@ -331,15 +328,15 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {...@@ -331,15 +328,15 @@ pub fn feed(p: *HeadParser, bytes: []const u8) usize {
331}328}
332329
333inline fn int16(array: *const [2]u8) u16 {330inline fn int16(array: *const [2]u8) u16 {
334 return @bitCast(array.*);331 return std.mem.toNative(u16, @bitCast(array.*), .little);
335}332}
336333
337inline fn int24(array: *const [3]u8) u24 {334inline fn int24(array: *const [3]u8) u24 {
338 return @bitCast(array.*);335 return std.mem.toNative(u24, @bitCast(array.*), .little);
339}336}
340337
341inline fn int32(array: *const [4]u8) u32 {338inline fn int32(array: *const [4]u8) u32 {
342 return @bitCast(array.*);339 return std.mem.toNative(u32, @bitCast(array.*), .little);
343}340}
344341
345inline fn intShift(comptime T: type, x: anytype) T {342inline fn intShift(comptime T: type, x: anytype) T {
lib/std/http/Server.zig+10-5
...@@ -726,7 +726,7 @@ pub const WebSocket = struct {...@@ -726,7 +726,7 @@ pub const WebSocket = struct {
726 else => @intFromEnum(h1.payload_len),726 else => @intFromEnum(h1.payload_len),
727 };727 };
728 if (len > in.buffer.len) return error.MessageOversize;728 if (len > in.buffer.len) return error.MessageOversize;
729 const mask: u32 = @bitCast((try in.takeArray(4)).*);729 const mask: [4]u8 = (try in.takeArray(4)).*;
730 const payload = try in.take(len);730 const payload = try in.take(len);
731731
732 // Skip pongs.732 // Skip pongs.
...@@ -734,11 +734,16 @@ pub const WebSocket = struct {...@@ -734,11 +734,16 @@ pub const WebSocket = struct {
734734
735 // The last item may contain a partial word of unused data.735 // The last item may contain a partial word of unused data.
736 const floored_len = (payload.len / 4) * 4;736 const floored_len = (payload.len / 4) * 4;
737 const u32_payload: []align(1) u32 = @ptrCast(payload[0..floored_len]);737
738 for (u32_payload) |*elem| elem.* ^= mask;738 const payload_chunks: [][4]u8 = @ptrCast(payload[0..floored_len]);
739 const mask_bytes: []const u8 = @ptrCast(&mask);739 for (payload_chunks) |*chunk| {
740 for (payload[floored_len..], mask_bytes[0 .. payload.len - floored_len]) |*leftover, m|740 const mask_i: u32 = @bitCast(mask);
741 const chunk_i: u32 = @bitCast(chunk.*);
742 chunk.* = @bitCast(chunk_i ^ mask_i);
743 }
744 for (payload[floored_len..], mask[0 .. payload.len - floored_len]) |*leftover, m| {
741 leftover.* ^= m;745 leftover.* ^= m;
746 }
742747
743 return .{748 return .{
744 .opcode = h0.opcode,749 .opcode = h0.opcode,
lib/std/mem.zig+80-58
...@@ -1848,18 +1848,18 @@ pub fn readVarPackedInt(...@@ -1848,18 +1848,18 @@ pub fn readVarPackedInt(
1848 if (@bitSizeOf(T) <= 8) {1848 if (@bitSizeOf(T) <= 8) {
1849 // These are the same shifts/masks we perform below, but adds `@truncate`/`@intCast`1849 // These are the same shifts/masks we perform below, but adds `@truncate`/`@intCast`
1850 // where needed since int is smaller than a byte.1850 // where needed since int is smaller than a byte.
1851 const value = if (read_size == 1) b: {1851 const value: uN = if (read_size == 1) b: {
1852 break :b @as(uN, @truncate(read_bytes[0] >> bit_shift));1852 break :b @truncate(read_bytes[0] >> bit_shift);
1853 } else b: {1853 } else b: {
1854 const i: u1 = @intFromBool(endian == .big);1854 const i: u1 = @intFromBool(endian == .big);
1855 const head = @as(uN, @truncate(read_bytes[i] >> bit_shift));1855 const head: uN = @truncate(read_bytes[i] >> bit_shift);
1856 const tail_shift = @as(Log2N, @intCast(@as(u4, 8) - bit_shift));1856 const tail_shift: Log2N = @intCast(@as(u4, 8) - bit_shift);
1857 const tail = @as(uN, @truncate(read_bytes[1 - i]));1857 const tail: uN = @truncate(read_bytes[1 - i]);
1858 break :b (tail << tail_shift) | head;1858 break :b (tail << tail_shift) | head;
1859 };1859 };
1860 switch (signedness) {1860 switch (signedness) {
1861 .signed => return @as(T, @intCast((@as(iN, @bitCast(value)) << pad) >> pad)),1861 .signed => return @intCast((@as(iN, @bitCast(value)) << pad) >> pad),
1862 .unsigned => return @as(T, @intCast((@as(uN, @bitCast(value)) << pad) >> pad)),1862 .unsigned => return @intCast((value << pad) >> pad),
1863 }1863 }
1864 }1864 }
18651865
...@@ -1880,8 +1880,8 @@ pub fn readVarPackedInt(...@@ -1880,8 +1880,8 @@ pub fn readVarPackedInt(
1880 },1880 },
1881 }1881 }
1882 switch (signedness) {1882 switch (signedness) {
1883 .signed => return @as(T, @intCast((@as(iN, @bitCast(int)) << pad) >> pad)),1883 .signed => return @intCast((@as(iN, @bitCast(int)) << pad) >> pad),
1884 .unsigned => return @as(T, @intCast((@as(uN, @bitCast(int)) << pad) >> pad)),1884 .unsigned => return @intCast((int << pad) >> pad),
1885 }1885 }
1886}1886}
18871887
...@@ -1896,8 +1896,13 @@ test readVarPackedInt {...@@ -1896,8 +1896,13 @@ test readVarPackedInt {
1896/// The bit count of T must be evenly divisible by 8.1896/// The bit count of T must be evenly divisible by 8.
1897/// This function cannot fail and cannot cause undefined behavior.1897/// This function cannot fail and cannot cause undefined behavior.
1898pub inline fn readInt(comptime T: type, buffer: *const [@divExact(@typeInfo(T).int.bits, 8)]u8, endian: Endian) T {1898pub inline fn readInt(comptime T: type, buffer: *const [@divExact(@typeInfo(T).int.bits, 8)]u8, endian: Endian) T {
1899 const value: T = @bitCast(buffer.*);1899 // Zig's logical bit order aligns with a little-endian byte array, so when reading in big-endian
1900 return if (endian == native_endian) value else @byteSwap(value);1900 // we must `@byteSwap` the int after we `@bitCast` to it.
1901 const little_val: T = @bitCast(buffer.*);
1902 return switch (endian) {
1903 .little => little_val,
1904 .big => @byteSwap(little_val),
1905 };
1901}1906}
19021907
1903test readInt {1908test readInt {
...@@ -1940,13 +1945,15 @@ fn readPackedIntLittle(comptime T: type, bytes: []const u8, bit_offset: usize) T...@@ -1940,13 +1945,15 @@ fn readPackedIntLittle(comptime T: type, bytes: []const u8, bit_offset: usize) T
1940 // Read by loading a LoadInt, and then follow it up with a 1-byte read1945 // Read by loading a LoadInt, and then follow it up with a 1-byte read
1941 // of the tail if bit_offset pushed us over a byte boundary.1946 // of the tail if bit_offset pushed us over a byte boundary.
1942 const read_bytes = bytes[bit_offset / 8 ..];1947 const read_bytes = bytes[bit_offset / 8 ..];
1943 const val = @as(uN, @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift));1948 const val: uN = @truncate(readInt(LoadInt, read_bytes[0..load_size], .little) >> bit_shift);
1944 if (bit_shift > load_tail_bits) {1949 if (bit_shift > load_tail_bits) {
1945 const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));1950 const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
1946 const tail_byte = read_bytes[load_size];1951 const tail_byte = read_bytes[load_size];
1947 const tail_truncated = if (bit_count < 8) @as(uN, @truncate(tail_byte)) else @as(uN, tail_byte);1952 const tail_truncated = if (bit_count < 8) @as(uN, @truncate(tail_byte)) else @as(uN, tail_byte);
1948 return @as(T, @bitCast(val | (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));1953 return @bitCast(val | (tail_truncated << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
1949 } else return @as(T, @bitCast(val));1954 } else {
1955 return @bitCast(val);
1956 }
1950}1957}
19511958
1952fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {1959fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
...@@ -1972,8 +1979,10 @@ fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {...@@ -1972,8 +1979,10 @@ fn readPackedIntBig(comptime T: type, bytes: []const u8, bit_offset: usize) T {
1972 if (bit_shift > load_tail_bits) {1979 if (bit_shift > load_tail_bits) {
1973 const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));1980 const tail_bits = @as(Log2N, @intCast(bit_shift - load_tail_bits));
1974 const tail_byte = if (bit_count < 8) @as(uN, @truncate(read_bytes[0])) else @as(uN, read_bytes[0]);1981 const tail_byte = if (bit_count < 8) @as(uN, @truncate(read_bytes[0])) else @as(uN, read_bytes[0]);
1975 return @as(T, @bitCast(val | (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits))));1982 return @bitCast(val | (tail_byte << (@as(Log2N, @truncate(bit_count)) -% tail_bits)));
1976 } else return @as(T, @bitCast(val));1983 } else {
1984 return @bitCast(val);
1985 }
1977}1986}
19781987
1979/// Loads an integer from packed memory.1988/// Loads an integer from packed memory.
...@@ -2011,7 +2020,12 @@ test "comptime read/write int" {...@@ -2011,7 +2020,12 @@ test "comptime read/write int" {
2011/// This function always succeeds, has defined behavior for all inputs, but2020/// This function always succeeds, has defined behavior for all inputs, but
2012/// the integer bit width must be divisible by 8.2021/// the integer bit width must be divisible by 8.
2013pub inline fn writeInt(comptime T: type, buffer: *[@divExact(@typeInfo(T).int.bits, 8)]u8, value: T, endian: Endian) void {2022pub inline fn writeInt(comptime T: type, buffer: *[@divExact(@typeInfo(T).int.bits, 8)]u8, value: T, endian: Endian) void {
2014 buffer.* = @bitCast(if (endian == native_endian) value else @byteSwap(value));2023 // Zig's logical bit order aligns with a little-endian byte array, so when writing in big-endian
2024 // we must `@byteSwap` the int before we `@bitCast` to an array.
2025 buffer.* = switch (endian) {
2026 .little => @bitCast(value),
2027 .big => @bitCast(@byteSwap(value)),
2028 };
2015}2029}
20162030
2017test writeInt {2031test writeInt {
...@@ -2209,10 +2223,10 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {...@@ -2209,10 +2223,10 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
2209/// (Changing their endianness)2223/// (Changing their endianness)
2210pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *align(a.toByteUnits()) S) void {2224pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *align(a.toByteUnits()) S) void {
2211 switch (@typeInfo(S)) {2225 switch (@typeInfo(S)) {
2212 .@"struct" => |struct_info| {2226 .@"struct" => |@"struct"| {
2213 if (struct_info.backing_integer) |Int| {2227 if (@"struct".backing_integer) |Int| {
2214 ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));2228 ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
2215 } else inline for (struct_info.field_types, struct_info.field_names, struct_info.field_attrs) |f_type, f_name, f_attr| {2229 } else inline for (@"struct".field_types, @"struct".field_names, @"struct".field_attrs) |f_type, f_name, f_attr| {
2216 switch (@typeInfo(f_type)) {2230 switch (@typeInfo(f_type)) {
2217 .@"struct" => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),2231 .@"struct" => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
2218 .@"union", .array => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),2232 .@"union", .array => byteSwapAllFieldsAligned(f_type, .fromByteUnits(f_attr.@"align" orelse @alignOf(f_type)), &@field(ptr, f_name)),
...@@ -2220,8 +2234,8 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a...@@ -2220,8 +2234,8 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2220 @field(ptr, f_name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f_name))));2234 @field(ptr, f_name) = @enumFromInt(@byteSwap(@intFromEnum(@field(ptr, f_name))));
2221 },2235 },
2222 .bool => {},2236 .bool => {},
2223 .float => |float_info| {2237 .float => |float| {
2224 @field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(@field(ptr, f_name)))));2238 @field(ptr, f_name) = @bitCast(@byteSwap(@as(@Int(.unsigned, float.bits), @bitCast(@field(ptr, f_name)))));
2225 },2239 },
2226 else => {2240 else => {
2227 @field(ptr, f_name) = @byteSwap(@field(ptr, f_name));2241 @field(ptr, f_name) = @byteSwap(@field(ptr, f_name));
...@@ -2229,23 +2243,26 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a...@@ -2229,23 +2243,26 @@ pub fn byteSwapAllFieldsAligned(comptime S: type, comptime a: Alignment, ptr: *a
2229 }2243 }
2230 }2244 }
2231 },2245 },
2232 .@"union" => |union_info| {2246 .@"union" => |@"union"| if (@"union".backing_integer) |Int| {
2233 if (union_info.tag_type != null) {2247 ptr.* = @bitCast(@byteSwap(@as(Int, @bitCast(ptr.*))));
2234 @compileError("byteSwapAllFields expects an untagged union");2248 } else {
2249 if (@"union".layout != .@"extern") {
2250 @compileError("byteSwapAllFields expects a packed or extern union");
2235 }2251 }
22362252
2237 const first_size = @bitSizeOf(union_info.field_types[0]);2253 const first_size = @bitSizeOf(@"union".field_types[0]);
2238 inline for (union_info.field_types) |field_type| {2254 inline for (@"union".field_types) |field_type| {
2239 if (@bitSizeOf(field_type) != first_size) {2255 if (@bitSizeOf(field_type) != first_size) {
2240 @compileError("Unable to byte-swap unions with varying field sizes");2256 @compileError("Unable to byte-swap unions with varying field sizes");
2241 }2257 }
2242 }2258 }
22432259
2244 const BackingInt = @Int(.unsigned, @bitSizeOf(S));2260 const FieldInt = @Int(.unsigned, first_size);
2245 ptr.* = @bitCast(@byteSwap(@as(BackingInt, @bitCast(ptr.*))));2261 const field_ptr = &@field(ptr, @"union".field_names[0]);
2262 field_ptr.* = @bitCast(@byteSwap(@as(FieldInt, @bitCast(field_ptr.*))));
2246 },2263 },
2247 .array => |info| {2264 .array => |array| {
2248 byteSwapAllElements(info.child, ptr);2265 byteSwapAllElements(array.child, ptr);
2249 },2266 },
2250 else => {2267 else => {
2251 ptr.* = @byteSwap(ptr.*);2268 ptr.* = @byteSwap(ptr.*);
...@@ -2291,7 +2308,7 @@ test byteSwapAllFields {...@@ -2291,7 +2308,7 @@ test byteSwapAllFields {
2291 .f2 = 0x12345678,2308 .f2 = 0x12345678,
2292 .f3 = .{0x12},2309 .f3 = .{0x12},
2293 .f4 = true,2310 .f4 = true,
2294 .f5 = @as(f32, @bitCast(@as(u32, 0x4640e400))),2311 .f5 = @bitCast(@as(u32, 0x4640e400)),
2295 .f6 = .{ .f0 = 0x1234 },2312 .f6 = .{ .f0 = 0x1234 },
2296 };2313 };
2297 var k = K{2314 var k = K{
...@@ -2300,7 +2317,7 @@ test byteSwapAllFields {...@@ -2300,7 +2317,7 @@ test byteSwapAllFields {
2300 .f2 = 0x1234,2317 .f2 = 0x1234,
2301 .f3 = .{0x12},2318 .f3 = .{0x12},
2302 .f4 = false,2319 .f4 = false,
2303 .f5 = @as(f32, @bitCast(@as(u32, 0x45d42800))),2320 .f5 = @bitCast(@as(u32, 0x45d42800)),
2304 };2321 };
2305 var p: P = @bitCast(@as(u32, 0x01234567));2322 var p: P = @bitCast(@as(u32, 0x01234567));
2306 var a: A = A{2323 var a: A = A{
...@@ -2318,7 +2335,7 @@ test byteSwapAllFields {...@@ -2318,7 +2335,7 @@ test byteSwapAllFields {
2318 .f2 = 0x78563412,2335 .f2 = 0x78563412,
2319 .f3 = .{0x12},2336 .f3 = .{0x12},
2320 .f4 = true,2337 .f4 = true,
2321 .f5 = @as(f32, @bitCast(@as(u32, 0x00e44046))),2338 .f5 = @bitCast(@as(u32, 0x00e44046)),
2322 .f6 = .{ .f0 = 0x3412 },2339 .f6 = .{ .f0 = 0x3412 },
2323 }, s);2340 }, s);
2324 try std.testing.expectEqual(K{2341 try std.testing.expectEqual(K{
...@@ -2327,7 +2344,7 @@ test byteSwapAllFields {...@@ -2327,7 +2344,7 @@ test byteSwapAllFields {
2327 .f2 = 0x3412,2344 .f2 = 0x3412,
2328 .f3 = .{0x12},2345 .f3 = .{0x12},
2329 .f4 = false,2346 .f4 = false,
2330 .f5 = @as(f32, @bitCast(@as(u32, 0x0028d445))),2347 .f5 = @bitCast(@as(u32, 0x0028d445)),
2331 }, k);2348 }, k);
2332 try std.testing.expectEqual(@as(P, @bitCast(@as(u32, 0x67452301))), p);2349 try std.testing.expectEqual(@as(P, @bitCast(@as(u32, 0x67452301))), p);
2333 try std.testing.expectEqual(A{2350 try std.testing.expectEqual(A{
...@@ -2348,8 +2365,9 @@ pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void {...@@ -2348,8 +2365,9 @@ pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void {
2348 elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*)));2365 elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*)));
2349 },2366 },
2350 .bool => {},2367 .bool => {},
2351 .float => |float_info| {2368 .float => |float| {
2352 elem.* = @bitCast(@byteSwap(@as(@Int(.unsigned, float_info.bits), @bitCast(elem.*))));2369 const int_repr: @Int(.unsigned, float.bits) = @bitCast(elem.*);
2370 elem.* = @bitCast(@byteSwap(int_repr));
2353 },2371 },
2354 else => {2372 else => {
2355 elem.* = @byteSwap(elem.*);2373 elem.* = @byteSwap(elem.*);
...@@ -3870,25 +3888,29 @@ inline fn reverseVector(comptime N: usize, comptime T: type, a: []T) [N]T {...@@ -3870,25 +3888,29 @@ inline fn reverseVector(comptime N: usize, comptime T: type, a: []T) [N]T {
3870pub fn reverse(comptime T: type, items: []T) void {3888pub fn reverse(comptime T: type, items: []T) void {
3871 var i: usize = 0;3889 var i: usize = 0;
3872 const end = items.len / 2;3890 const end = items.len / 2;
3873 if (use_vectors and3891
3874 !@inComptime() and3892 vec: {
3875 @bitSizeOf(T) > 0 and3893 if (!use_vectors) break :vec;
3876 std.math.isPowerOfTwo(@bitSizeOf(T)))3894 if (@inComptime()) break :vec;
3877 {3895 switch (@typeInfo(T)) {
3878 if (std.simd.suggestVectorLength(T)) |simd_size| {3896 .int, .float => {},
3879 if (simd_size <= end) {3897 .pointer => |pointer| if (pointer.size == .slice) break :vec,
3880 const simd_end = end - (simd_size - 1);3898 else => break :vec,
3881 while (i < simd_end) : (i += simd_size) {3899 }
3882 const left_slice = items[i .. i + simd_size];3900 if (@bitSizeOf(T) == 0 or !comptime std.math.isPowerOfTwo(@bitSizeOf(T))) break :vec;
3883 const right_slice = items[items.len - i - simd_size .. items.len - i];3901 const simd_size = std.simd.suggestVectorLength(T) orelse break :vec;
38843902 if (simd_size > end) break :vec;
3885 const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);3903
3886 const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);3904 const simd_end = end - (simd_size - 1);
38873905 while (i < simd_end) : (i += simd_size) {
3888 @memcpy(right_slice, &left_shuffled);3906 const left_slice = items[i .. i + simd_size];
3889 @memcpy(left_slice, &right_shuffled);3907 const right_slice = items[items.len - i - simd_size .. items.len - i];
3890 }3908
3891 }3909 const left_shuffled: [simd_size]T = reverseVector(simd_size, T, left_slice);
3910 const right_shuffled: [simd_size]T = reverseVector(simd_size, T, right_slice);
3911
3912 @memcpy(right_slice, &left_shuffled);
3913 @memcpy(left_slice, &right_shuffled);
3892 }3914 }
3893 }3915 }
38943916
...@@ -5009,8 +5031,8 @@ test "read/write(Var)PackedInt" {...@@ -5009,8 +5031,8 @@ test "read/write(Var)PackedInt" {
5009 for ([_]PackedType{5031 for ([_]PackedType{
5010 ~@as(PackedType, 0), // all ones: -1 iN / maxInt uN5032 ~@as(PackedType, 0), // all ones: -1 iN / maxInt uN
5011 @as(PackedType, 0), // all zeros: 0 iN / 0 uN5033 @as(PackedType, 0), // all zeros: 0 iN / 0 uN
5012 @as(PackedType, @bitCast(@as(iPackedType, math.maxInt(iPackedType)))), // maxInt iN5034 @bitCast(@as(iPackedType, math.maxInt(iPackedType))), // maxInt iN
5013 @as(PackedType, @bitCast(@as(iPackedType, math.minInt(iPackedType)))), // maxInt iN5035 @bitCast(@as(iPackedType, math.minInt(iPackedType))), // maxInt iN
5014 random.int(PackedType), // random5036 random.int(PackedType), // random
5015 random.int(PackedType), // random5037 random.int(PackedType), // random
5016 }) |write_value| {5038 }) |write_value| {
lib/std/os/linux/IoUring/test.zig+24-8
...@@ -526,7 +526,9 @@ test "sendmsg/recvmsg" {...@@ -526,7 +526,9 @@ test "sendmsg/recvmsg" {
526526
527 var address_server: linux.sockaddr.in = .{527 var address_server: linux.sockaddr.in = .{
528 .port = 0,528 .port = 0,
529 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),529 .addr = @as(*align(1) const u32, @ptrCast(
530 &@as([4]u8, .{ 127, 0, 0, 1 }),
531 )).*,
530 };532 };
531533
532 const server = try socket(address_server.family, posix.SOCK.DGRAM, 0);534 const server = try socket(address_server.family, posix.SOCK.DGRAM, 0);
...@@ -1028,7 +1030,9 @@ test "shutdown" {...@@ -1028,7 +1030,9 @@ test "shutdown" {
10281030
1029 var address: linux.sockaddr.in = .{1031 var address: linux.sockaddr.in = .{
1030 .port = 0,1032 .port = 0,
1031 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),1033 .addr = @as(*align(1) const u32, @ptrCast(
1034 &@as([4]u8, .{ 127, 0, 0, 1 }),
1035 )).*,
1032 };1036 };
10331037
1034 // Socket bound, expect shutdown to work1038 // Socket bound, expect shutdown to work
...@@ -1740,7 +1744,9 @@ test "accept multishot" {...@@ -1740,7 +1744,9 @@ test "accept multishot" {
17401744
1741 var address: linux.sockaddr.in = .{1745 var address: linux.sockaddr.in = .{
1742 .port = 0,1746 .port = 0,
1743 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),1747 .addr = @as(*align(1) const u32, @ptrCast(
1748 &@as([4]u8, .{ 127, 0, 0, 1 }),
1749 )).*,
1744 };1750 };
1745 const listener_socket = try createListenerSocket(&address);1751 const listener_socket = try createListenerSocket(&address);
1746 defer _ = linux.close(listener_socket);1752 defer _ = linux.close(listener_socket);
...@@ -1842,7 +1848,9 @@ test "accept_direct" {...@@ -1842,7 +1848,9 @@ test "accept_direct" {
1842 defer ring.deinit();1848 defer ring.deinit();
1843 var address: linux.sockaddr.in = .{1849 var address: linux.sockaddr.in = .{
1844 .port = 0,1850 .port = 0,
1845 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),1851 .addr = @as(*align(1) const u32, @ptrCast(
1852 &@as([4]u8, .{ 127, 0, 0, 1 }),
1853 )).*,
1846 };1854 };
18471855
1848 // register direct file descriptors1856 // register direct file descriptors
...@@ -1931,7 +1939,9 @@ test "accept_multishot_direct" {...@@ -1931,7 +1939,9 @@ test "accept_multishot_direct" {
19311939
1932 var address: linux.sockaddr.in = .{1940 var address: linux.sockaddr.in = .{
1933 .port = 0,1941 .port = 0,
1934 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),1942 .addr = @as(*align(1) const u32, @ptrCast(
1943 &@as([4]u8, .{ 127, 0, 0, 1 }),
1944 )).*,
1935 };1945 };
19361946
1937 var registered_fds: [2]linux.fd_t = @splat(-1);1947 var registered_fds: [2]linux.fd_t = @splat(-1);
...@@ -2041,7 +2051,9 @@ test "socket_direct/socket_direct_alloc/close_direct" {...@@ -2041,7 +2051,9 @@ test "socket_direct/socket_direct_alloc/close_direct" {
2041 // use sockets from registered_fds in connect operation2051 // use sockets from registered_fds in connect operation
2042 var address: linux.sockaddr.in = .{2052 var address: linux.sockaddr.in = .{
2043 .port = 0,2053 .port = 0,
2044 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),2054 .addr = @as(*align(1) const u32, @ptrCast(
2055 &@as([4]u8, .{ 127, 0, 0, 1 }),
2056 )).*,
2045 };2057 };
2046 const listener_socket = try createListenerSocket(&address);2058 const listener_socket = try createListenerSocket(&address);
2047 defer _ = linux.close(listener_socket);2059 defer _ = linux.close(listener_socket);
...@@ -2426,7 +2438,9 @@ test "bind/listen/connect" {...@@ -2426,7 +2438,9 @@ test "bind/listen/connect" {
24262438
2427 var addr: linux.sockaddr.in = .{2439 var addr: linux.sockaddr.in = .{
2428 .port = 0,2440 .port = 0,
2429 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),2441 .addr = @as(*align(1) const u32, @ptrCast(
2442 &@as([4]u8, .{ 127, 0, 0, 1 }),
2443 )).*,
2430 };2444 };
2431 const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;2445 const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP;
24322446
...@@ -2614,7 +2628,9 @@ pub fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {...@@ -2614,7 +2628,9 @@ pub fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
2614 // Create a TCP server socket2628 // Create a TCP server socket
2615 var address: linux.sockaddr.in = .{2629 var address: linux.sockaddr.in = .{
2616 .port = 0,2630 .port = 0,
2617 .addr = @bitCast([4]u8{ 127, 0, 0, 1 }),2631 .addr = @as(*align(1) const u32, @ptrCast(
2632 &@as([4]u8, .{ 127, 0, 0, 1 }),
2633 )).*,
2618 };2634 };
2619 const listener_socket = try createListenerSocket(&address);2635 const listener_socket = try createListenerSocket(&address);
2620 errdefer _ = linux.close(listener_socket);2636 errdefer _ = linux.close(listener_socket);
lib/std/os/uefi.zig+1-1
...@@ -218,7 +218,7 @@ pub const TimeCapabilities = extern struct {...@@ -218,7 +218,7 @@ pub const TimeCapabilities = extern struct {
218pub const FileHandle = *opaque {};218pub const FileHandle = *opaque {};
219219
220test "GUID formatting" {220test "GUID formatting" {
221 const bytes = [_]u8{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };221 const bytes: [16]u8 = .{ 137, 60, 203, 50, 128, 128, 124, 66, 186, 19, 80, 73, 135, 59, 194, 135 };
222 const guid: Guid = @bitCast(bytes);222 const guid: Guid = @bitCast(bytes);
223223
224 const str = try std.fmt.allocPrint(std.testing.allocator, "{f}", .{guid});224 const str = try std.fmt.allocPrint(std.testing.allocator, "{f}", .{guid});
lib/std/os/windows.zig+37-36
...@@ -4189,19 +4189,11 @@ pub const GUID = extern struct {...@@ -4189,19 +4189,11 @@ pub const GUID = extern struct {
4189 Data3: u16,4189 Data3: u16,
4190 Data4: [8]u8,4190 Data4: [8]u8,
41914191
4192 const hex_offsets = switch (builtin.target.cpu.arch.endian()) {4192 const hex_offsets: [16]u6 = .{
4193 .big => [16]u6{4193 6, 4, 2, 0,
4194 0, 2, 4, 6,4194 11, 9, 16, 14,
4195 9, 11, 14, 16,4195 19, 21, 24, 26,
4196 19, 21, 24, 26,4196 28, 30, 32, 34,
4197 28, 30, 32, 34,
4198 },
4199 .little => [16]u6{
4200 6, 4, 2, 0,
4201 11, 9, 16, 14,
4202 19, 21, 24, 26,
4203 28, 30, 32, 34,
4204 },
4205 };4197 };
42064198
4207 pub fn parse(s: []const u8) GUID {4199 pub fn parse(s: []const u8) GUID {
...@@ -4216,12 +4208,21 @@ pub const GUID = extern struct {...@@ -4216,12 +4208,21 @@ pub const GUID = extern struct {
4216 assert(s[13] == '-');4208 assert(s[13] == '-');
4217 assert(s[18] == '-');4209 assert(s[18] == '-');
4218 assert(s[23] == '-');4210 assert(s[23] == '-');
4219 var bytes: [16]u8 = undefined;4211 var raw1: [4]u8 = undefined;
4220 for (hex_offsets, 0..) |hex_offset, i| {4212 var raw2: [2]u8 = undefined;
4221 bytes[i] = (try std.fmt.charToDigit(s[hex_offset], 16)) << 4 |4213 var raw3: [2]u8 = undefined;
4222 try std.fmt.charToDigit(s[hex_offset + 1], 16);4214 var raw4: [8]u8 = undefined;
4223 }4215 assert((try std.fmt.hexToBytes(&raw1, s[0..8])).len == raw1.len);
4224 return @as(GUID, @bitCast(bytes));4216 assert((try std.fmt.hexToBytes(&raw2, s[9..13])).len == raw2.len);
4217 assert((try std.fmt.hexToBytes(&raw3, s[14..18])).len == raw3.len);
4218 assert((try std.fmt.hexToBytes(raw4[0..2], s[19..23])).len == 2);
4219 assert((try std.fmt.hexToBytes(raw4[2..8], s[24..36])).len == 6);
4220 return .{
4221 .Data1 = @byteSwap(@as(u32, @bitCast(raw1))),
4222 .Data2 = @byteSwap(@as(u16, @bitCast(raw2))),
4223 .Data3 = @byteSwap(@as(u16, @bitCast(raw3))),
4224 .Data4 = raw4,
4225 };
4225 }4226 }
42264227
4227 pub fn format(self: GUID, w: *std.Io.Writer) std.Io.Writer.Error!void {4228 pub fn format(self: GUID, w: *std.Io.Writer) std.Io.Writer.Error!void {
...@@ -4233,28 +4234,28 @@ pub const GUID = extern struct {...@@ -4233,28 +4234,28 @@ pub const GUID = extern struct {
4233 self.Data4[2..8],4234 self.Data4[2..8],
4234 });4235 });
4235 }4236 }
4236};
42374237
4238test GUID {4238 test parse {
4239 try std.testing.expectEqual(4239 const expected: GUID = .{
4240 GUID{
4241 .Data1 = 0x01234567,4240 .Data1 = 0x01234567,
4242 .Data2 = 0x89ab,4241 .Data2 = 0x89ab,
4243 .Data3 = 0xef10,4242 .Data3 = 0xef10,
4244 .Data4 = "\x32\x54\x76\x98\xba\xdc\xfe\x91".*,4243 .Data4 = "\x32\x54\x76\x98\xba\xdc\xfe\x91".*,
4245 },4244 };
4246 GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"),4245 try std.testing.expectEqual(expected, GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}"));
4247 );4246 }
4248 try std.testing.expectFmt(4247
4249 "{01234567-89ab-ef10-3254-7698badcfe91}",4248 test format {
4250 "{f}",4249 const guid0: GUID = .{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = .{ 0, 1, 0, 0, 0, 0, 0, 1 } };
4251 .{GUID.parse("{01234567-89AB-EF10-3254-7698badcfe91}")},4250 try std.testing.expectFmt("{00000001-0001-0001-0001-000000000001}", "{f}", .{guid0});
4252 );4251
4253 try std.testing.expectFmt(4252 const guid1: GUID = .parse("{01234567-89AB-EF10-3254-7698badcfe91}");
4254 "{00000001-0001-0001-0001-000000000001}",4253 try std.testing.expectFmt("{01234567-89ab-ef10-3254-7698badcfe91}", "{f}", .{guid1});
4255 "{f}",4254 }
4256 .{GUID{ .Data1 = 1, .Data2 = 1, .Data3 = 1, .Data4 = [_]u8{ 0, 1, 0, 0, 0, 0, 0, 1 } }},4255};
4257 );4256
4257test {
4258 _ = GUID;
4258}4259}
42594260
4260pub const COORD = extern struct {4261pub const COORD = extern struct {
lib/std/testing.zig+30-24
...@@ -141,39 +141,45 @@ fn expectEqualInner(comptime T: type, expected: T, actual: T) !void {...@@ -141,39 +141,45 @@ fn expectEqualInner(comptime T: type, expected: T, actual: T) !void {
141 try expectEqualSlices(info.child, &expect_array, &actual_array);141 try expectEqualSlices(info.child, &expect_array, &actual_array);
142 },142 },
143143
144 .@"struct" => |structType| {144 .@"struct" => |@"struct"| {
145 inline for (structType.field_names) |field_name| {145 inline for (@"struct".field_names) |field_name| {
146 try expectEqual(@field(expected, field_name), @field(actual, field_name));146 try expectEqual(@field(expected, field_name), @field(actual, field_name));
147 }147 }
148 },148 },
149149
150 .@"union" => |union_info| {150 .@"union" => |@"union"| if (@"union".backing_integer) |Int| {
151 if (union_info.tag_type == null) {151 try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
152 const first_size = @bitSizeOf(union_info.field_types[0]);152 } else switch (@"union".layout) {
153 inline for (union_info.field_types) |field_type| {153 .@"packed" => {
154 const Int = @Int(.unsigned, @bitSizeOf(T));
155 try expectEqual(@as(Int, @bitCast(expected)), @as(Int, @bitCast(actual)));
156 },
157 .@"extern" => {
158 const first_size = @bitSizeOf(@"union".field_types[0]);
159 inline for (@"union".field_types) |field_type| {
154 if (@bitSizeOf(field_type) != first_size) {160 if (@bitSizeOf(field_type) != first_size) {
155 @compileError("Unable to compare untagged unions with varying field sizes for type " ++ @typeName(@TypeOf(actual)));161 @compileError("Unable to compare extern unions with varying field sizes for type " ++ @typeName(T));
156 }162 }
157 }163 }
158164 const FieldInt = @Int(.unsigned, first_size);
159 const BackingInt = @Int(.unsigned, @bitSizeOf(T));165 const expected_field = @field(expected, @"union".field_names[0]);
166 const actual_field = @field(actual, @"union".field_names[0]);
160 return expectEqual(167 return expectEqual(
161 @as(BackingInt, @bitCast(expected)),168 @as(FieldInt, @bitCast(expected_field)),
162 @as(BackingInt, @bitCast(actual)),169 @as(FieldInt, @bitCast(actual_field)),
163 );170 );
164 }171 },
165172 .auto => {
166 const Tag = std.meta.Tag(@TypeOf(expected));173 const Tag = @"union".tag_type orelse @compileError("byteSwapAllFields expects packed, extern, or tagged union");
167174
168 const expectedTag = @as(Tag, expected);175 try expectEqual(@as(Tag, expected), @as(Tag, actual));
169 const actualTag = @as(Tag, actual);176 switch (expected) {
170177 inline else => |expected_payload, tag| {
171 try expectEqual(expectedTag, actualTag);178 const actual_payload = @field(actual, @tagName(tag));
172179 try expectEqual(expected_payload, actual_payload);
173 // we only reach this switch if the tags are equal180 },
174 switch (expected) {181 }
175 inline else => |val, tag| try expectEqual(val, @field(actual, @tagName(tag))),182 },
176 }
177 },183 },
178184
179 .optional => {185 .optional => {
lib/std/zig/llvm/Builder.zig+50-4
...@@ -1816,7 +1816,7 @@ pub const Linkage = enum(u4) {...@@ -1816,7 +1816,7 @@ pub const Linkage = enum(u4) {
1816 }1816 }
1817};1817};
18181818
1819pub const Preemption = enum {1819pub const Preemption = enum(u2) {
1820 dso_preemptable,1820 dso_preemptable,
1821 dso_local,1821 dso_local,
1822 implicit_dso_local,1822 implicit_dso_local,
...@@ -2011,7 +2011,7 @@ pub const AddrSpace = enum(u24) {...@@ -2011,7 +2011,7 @@ pub const AddrSpace = enum(u24) {
2011 }2011 }
2012};2012};
20132013
2014pub const ExternallyInitialized = enum {2014pub const ExternallyInitialized = enum(u1) {
2015 default,2015 default,
2016 externally_initialized,2016 externally_initialized,
20172017
...@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {...@@ -2061,6 +2061,13 @@ pub const Alignment = enum(u6) {
2061 };2061 };
2062 }2062 }
20632063
2064 /// Asserts that neither `a` nor `b` is `.default`.
2065 pub fn max(a: Alignment, b: Alignment) Alignment {
2066 assert(a != .default);
2067 assert(b != .default);
2068 return @enumFromInt(@max(@intFromEnum(a), @intFromEnum(b)));
2069 }
2070
2064 pub fn toLlvm(self: Alignment) u6 {2071 pub fn toLlvm(self: Alignment) u6 {
2065 return switch (self) {2072 return switch (self) {
2066 .default => 0,2073 .default => 0,
...@@ -4314,6 +4321,9 @@ pub const Function = struct {...@@ -4314,6 +4321,9 @@ pub const Function = struct {
4314 @"tail call",4321 @"tail call",
4315 @"tail call fast",4322 @"tail call fast",
4316 trunc,4323 trunc,
4324 @"trunc nuw",
4325 @"trunc nsw",
4326 @"trunc nuw nsw",
4317 udiv,4327 udiv,
4318 @"udiv exact",4328 @"udiv exact",
4319 urem,4329 urem,
...@@ -4377,7 +4387,10 @@ pub const Function = struct {...@@ -4377,7 +4387,10 @@ pub const Function = struct {
4377 };4387 };
4378 }4388 }
43794389
4380 pub fn toCastOpcode(self: Tag) CastOpcode {4390 /// Does not accept `.@"trunc nuw"`, `.@"trunc nsw"`, or `.@"trunc nuw nsw"`, because
4391 /// they do not have distinct `CastOpcode` values, and are instead encoded in bitcode
4392 /// using flags on a normal `trunc` operation.
4393 fn toCastOpcode(self: Tag) CastOpcode {
4381 return switch (self) {4394 return switch (self) {
4382 .trunc => .trunc,4395 .trunc => .trunc,
4383 .zext => .zext,4396 .zext => .zext,
...@@ -4572,6 +4585,9 @@ pub const Function = struct {...@@ -4572,6 +4585,9 @@ pub const Function = struct {
4572 .sext,4585 .sext,
4573 .sitofp,4586 .sitofp,
4574 .trunc,4587 .trunc,
4588 .@"trunc nuw",
4589 .@"trunc nsw",
4590 .@"trunc nuw nsw",
4575 .uitofp,4591 .uitofp,
4576 .zext,4592 .zext,
4577 => wip.extraData(Cast, instruction.data).type,4593 => wip.extraData(Cast, instruction.data).type,
...@@ -4758,6 +4774,9 @@ pub const Function = struct {...@@ -4758,6 +4774,9 @@ pub const Function = struct {
4758 .sext,4774 .sext,
4759 .sitofp,4775 .sitofp,
4760 .trunc,4776 .trunc,
4777 .@"trunc nuw",
4778 .@"trunc nsw",
4779 .@"trunc nuw nsw",
4761 .uitofp,4780 .uitofp,
4762 .zext,4781 .zext,
4763 => function.extraData(Cast, instruction.data).type,4782 => function.extraData(Cast, instruction.data).type,
...@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {...@@ -5975,6 +5994,9 @@ pub const WipFunction = struct {
5975 .sext,5994 .sext,
5976 .sitofp,5995 .sitofp,
5977 .trunc,5996 .trunc,
5997 .@"trunc nuw",
5998 .@"trunc nsw",
5999 .@"trunc nuw nsw",
5978 .uitofp,6000 .uitofp,
5979 .zext,6001 .zext,
5980 => {},6002 => {},
...@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {...@@ -6583,6 +6605,9 @@ pub const WipFunction = struct {
6583 .sext,6605 .sext,
6584 .sitofp,6606 .sitofp,
6585 .trunc,6607 .trunc,
6608 .@"trunc nuw",
6609 .@"trunc nsw",
6610 .@"trunc nuw nsw",
6586 .uitofp,6611 .uitofp,
6587 .zext,6612 .zext,
6588 => {6613 => {
...@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void...@@ -9975,6 +10000,9 @@ pub fn print(self: *Builder, w: *Writer) (Writer.Error || Allocator.Error)!void
9975 .sext,10000 .sext,
9976 .sitofp,10001 .sitofp,
9977 .trunc,10002 .trunc,
10003 .@"trunc nuw",
10004 .@"trunc nsw",
10005 .@"trunc nuw nsw",
9978 .uitofp,10006 .uitofp,
9979 .zext,10007 .zext,
9980 => |tag| {10008 => |tag| {
...@@ -11649,7 +11677,11 @@ fn convTag(...@@ -11649,7 +11677,11 @@ fn convTag(
11649 .unneeded => unreachable,11677 .unneeded => unreachable,
11650 },11678 },
11651 .eq => unreachable,11679 .eq => unreachable,
11652 .gt => .trunc,11680 .gt => switch (signedness) {
11681 .unsigned => .@"trunc nuw",
11682 .signed => .@"trunc nsw",
11683 .unneeded => .trunc,
11684 },
11653 },11685 },
11654 .pointer => .inttoptr,11686 .pointer => .inttoptr,
11655 else => unreachable,11687 else => unreachable,
...@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco...@@ -14962,6 +14994,20 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
14962 .opcode = kind.toCastOpcode(),14994 .opcode = kind.toCastOpcode(),
14963 });14995 });
14964 },14996 },
14997 .@"trunc nuw",
14998 .@"trunc nsw",
14999 .@"trunc nuw nsw",
15000 => |kind| {
15001 const extra = func.extraData(Function.Instruction.Cast, data);
15002 try function_block.writeAbbrev(FunctionBlock.TruncNoWrap{
15003 .val = adapter.getOffsetValueIndex(extra.val),
15004 .type_index = extra.type,
15005 .flags = .{
15006 .no_unsigned_wrap = kind == .@"trunc nuw" or kind == .@"trunc nuw nsw",
15007 .no_signed_wrap = kind == .@"trunc nsw" or kind == .@"trunc nuw nsw",
15008 },
15009 });
15010 },
14965 .@"fcmp false",15011 .@"fcmp false",
14966 .@"fcmp oeq",15012 .@"fcmp oeq",
14967 .@"fcmp oge",15013 .@"fcmp oge",
lib/std/zig/llvm/ir.zig+19
...@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {...@@ -696,6 +696,7 @@ pub const ModuleBlock = struct {
696 ModuleBlock.FunctionBlock.Select,696 ModuleBlock.FunctionBlock.Select,
697 ModuleBlock.FunctionBlock.SelectFast,697 ModuleBlock.FunctionBlock.SelectFast,
698 ModuleBlock.FunctionBlock.Cast,698 ModuleBlock.FunctionBlock.Cast,
699 ModuleBlock.FunctionBlock.TruncNoWrap,
699 ModuleBlock.FunctionBlock.Alloca,700 ModuleBlock.FunctionBlock.Alloca,
700 ModuleBlock.FunctionBlock.GetElementPtr,701 ModuleBlock.FunctionBlock.GetElementPtr,
701 ModuleBlock.FunctionBlock.ExtractValue,702 ModuleBlock.FunctionBlock.ExtractValue,
...@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {...@@ -1086,6 +1087,24 @@ pub const ModuleBlock = struct {
1086 opcode: CastOpcode,1087 opcode: CastOpcode,
1087 };1088 };
10881089
1090 pub const TruncNoWrap = struct {
1091 pub const Flags = packed struct(u2) {
1092 no_unsigned_wrap: bool,
1093 no_signed_wrap: bool,
1094 };
1095 pub const ops = [_]AbbrevOp{
1096 .{ .literal = @intFromEnum(ModuleBlock.FunctionBlock.Code.INST_CAST) },
1097 ValueAbbrev,
1098 .{ .fixed_runtime = Builder.Type },
1099 .{ .literal = @intFromEnum(Builder.CastOpcode.trunc) },
1100 .{ .fixed = @bitSizeOf(Flags) },
1101 };
1102
1103 val: u32,
1104 type_index: Builder.Type,
1105 flags: Flags,
1106 };
1107
1089 pub const Alloca = struct {1108 pub const Alloca = struct {
1090 pub const Flags = packed struct(u11) {1109 pub const Flags = packed struct(u11) {
1091 align_lower: u5,1110 align_lower: u5,
src/Air.zig+68-13
...@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");...@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");
1717
18pub const Legalize = @import("Air/Legalize.zig");18pub const Legalize = @import("Air/Legalize.zig");
19pub const Liveness = @import("Air/Liveness.zig");19pub const Liveness = @import("Air/Liveness.zig");
20pub const Verify = @import("Air/Verify.zig");
2021
21instructions: std.MultiArrayList(Inst).Slice,22instructions: std.MultiArrayList(Inst).Slice,
22/// The meaning of this data is determined by `Inst.Tag` value.23/// The meaning of this data is determined by `Inst.Tag` value.
...@@ -276,10 +277,50 @@ pub const Inst = struct {...@@ -276,10 +277,50 @@ pub const Inst = struct {
276 /// Boolean or binary NOT.277 /// Boolean or binary NOT.
277 /// Uses the `ty_op` field.278 /// Uses the `ty_op` field.
278 not,279 not,
279 /// Reinterpret the bits of a value as a different type. This is like `@bitCast` but280 /// Implements `@bitCast`.
280 /// also supports enums and pointers.281 ///
282 /// Uses the `ty_op` field.
283 bit_cast,
284 /// Cast a pointer to a different pointer type. The result type is a slice iff the operand
285 /// type is a slice (the length of the slice does not change). All other pointer attributes
286 /// except for the address space may change.
287 ///
288 /// Supports vectors of pointers.
289 ///
290 /// Uses the `ty_op` field.
291 ptr_cast,
292 /// Cast an integer to a pointer (not a slice). Operand type is always `usize`.
293 ///
294 /// Supports vectors of integers.
295 ///
296 /// Uses the `ty_op` field.
297 ptr_from_int,
298 /// Cast a pointer (not a slice) to an integer. Result type is always `usize`.
299 ///
300 /// Supports vectors of pointers.
301 ///
302 /// Uses the `ty_op` field.
303 int_from_ptr,
304 /// Cast an error set `E1` to a different error set `E2`, or cast an error union `E1!T` to
305 /// an error union `E2!T` with the same payload type but a different error set type.
306 ///
307 /// Uses the `ty_op` field.
308 error_cast,
309 /// Cast an integer to an error set type. The integer operand type is unsigned and has bit
310 /// width equal to `zcu.errorSetBits()`.
311 ///
312 /// Uses the `ty_op` field.
313 error_from_int,
314 /// Cast an error set to an integer type. The integer destination type is unsigned and has
315 /// bit width equal to `zcu.errorSetBits()`.
316 ///
317 /// Uses the `ty_op` field.
318 int_from_error,
319 /// Cast an enum value to a tagged union, whose tag type is that enum, and which has no
320 /// payload bits (i.e. all payloads are equivalent to `void`).
321 ///
281 /// Uses the `ty_op` field.322 /// Uses the `ty_op` field.
282 bitcast,323 union_from_enum,
283 /// A block runs its body which always ends with a `noreturn` instruction,324 /// A block runs its body which always ends with a `noreturn` instruction,
284 /// so the only way to proceed to the code after the `block` is to encounter a `br`325 /// so the only way to proceed to the code after the `block` is to encounter a `br`
285 /// that targets this `block`. If the `block` type is `noreturn`,326 /// that targets this `block`. If the `block` type is `noreturn`,
...@@ -589,13 +630,13 @@ pub const Inst = struct {...@@ -589,13 +630,13 @@ pub const Inst = struct {
589 /// the integer tag type of the enum.630 /// the integer tag type of the enum.
590 /// See `trunc` for integer truncation.631 /// See `trunc` for integer truncation.
591 /// Uses the `ty_op` field.632 /// Uses the `ty_op` field.
592 intcast,633 int_cast,
593 /// Like `intcast`, but includes two safety checks:634 /// Like `int_cast`, but includes two safety checks:
594 /// * triggers a safety panic if the cast truncates bits635 /// * triggers a safety panic if the cast truncates bits
595 /// * triggers a safety panic if the destination type is an exhaustive enum636 /// * triggers a safety panic if the destination type is an exhaustive enum
596 /// and the operand is not a valid value of this type; i.e. equivalent to637 /// and the operand is not a valid value of this type; i.e. equivalent to
597 /// a safety check based on `.is_named_enum_value`638 /// a safety check based on `.is_named_enum_value`
598 intcast_safe,639 int_cast_safe,
599 /// Truncate higher bits from an integer, resulting in an integer type with the same640 /// Truncate higher bits from an integer, resulting in an integer type with the same
600 /// sign but an equal or smaller number of bits.641 /// sign but an equal or smaller number of bits.
601 /// Uses the `ty_op` field.642 /// Uses the `ty_op` field.
...@@ -955,7 +996,7 @@ pub const Inst = struct {...@@ -955,7 +996,7 @@ pub const Inst = struct {
955 /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit996 /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
956 /// this instruction when scalarizing vector operations.997 /// this instruction when scalarizing vector operations.
957 ///998 ///
958 /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result999 /// Uses the `bin_op` field. `lhs` is the vector value. `rhs` is the element index. Result
959 /// type is the vector element type.1000 /// type is the vector element type.
960 legalize_vec_elem_val,1001 legalize_vec_elem_val,
9611002
...@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1667 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),1708 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
16681709
1669 .not,1710 .not,
1670 .bitcast,1711 .bit_cast,
1712 .ptr_cast,
1713 .ptr_from_int,
1714 .int_from_ptr,
1715 .error_cast,
1716 .error_from_int,
1717 .int_from_error,
1718 .union_from_enum,
1671 .load,1719 .load,
1672 .fpext,1720 .fpext,
1673 .fptrunc,1721 .fptrunc,
1674 .intcast,1722 .int_cast,
1675 .intcast_safe,1723 .int_cast_safe,
1676 .trunc,1724 .trunc,
1677 .optional_payload,1725 .optional_payload,
1678 .optional_payload_ptr,1726 .optional_payload_ptr,
...@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1913 .add_safe,1961 .add_safe,
1914 .sub_safe,1962 .sub_safe,
1915 .mul_safe,1963 .mul_safe,
1916 .intcast_safe,1964 .int_cast_safe,
1917 .int_from_float_safe,1965 .int_from_float_safe,
1918 .int_from_float_optimized_safe,1966 .int_from_float_optimized_safe,
1919 .legalize_vec_store_elem,1967 .legalize_vec_store_elem,
...@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1965 .shl_sat,2013 .shl_sat,
1966 .xor,2014 .xor,
1967 .not,2015 .not,
1968 .bitcast,2016 .bit_cast,
2017 .ptr_cast,
2018 .ptr_from_int,
2019 .int_from_ptr,
2020 .error_cast,
2021 .error_from_int,
2022 .int_from_error,
2023 .union_from_enum,
1969 .ret_addr,2024 .ret_addr,
1970 .frame_addr,2025 .frame_addr,
1971 .clz,2026 .clz,
...@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2009 .is_non_err,2064 .is_non_err,
2010 .fptrunc,2065 .fptrunc,
2011 .fpext,2066 .fpext,
2012 .intcast,2067 .int_cast,
2013 .trunc,2068 .trunc,
2014 .optional_payload,2069 .optional_payload,
2015 .optional_payload_ptr,2070 .optional_payload_ptr,
src/Air/Legalize.zig+238-95
...@@ -75,13 +75,9 @@ pub const Feature = enum {...@@ -75,13 +75,9 @@ pub const Feature = enum {
75 scalarize_shl_sat,75 scalarize_shl_sat,
76 scalarize_xor,76 scalarize_xor,
77 scalarize_not,77 scalarize_not,
78 /// Scalarize `bitcast` from or to an array or vector type to `bitcast`s of the elements.78 scalarize_ptr_cast,
79 /// This does not apply if `@bitSizeOf(Elem) == 8 * @sizeOf(Elem)`.79 scalarize_ptr_from_int,
80 /// When this feature is enabled, all remaining `bitcast`s can be lowered using the old bitcast80 scalarize_int_from_ptr,
81 /// semantics (reinterpret memory) instead of the new bitcast semantics (copy logical bits) and
82 /// the behavior will be equivalent. However, the behavior of `@bitSize` on arrays must be
83 /// changed in `Type.zig` before enabling this feature to conform to the new bitcast semantics.
84 scalarize_bitcast,
85 scalarize_clz,81 scalarize_clz,
86 scalarize_ctz,82 scalarize_ctz,
87 scalarize_popcount,83 scalarize_popcount,
...@@ -107,8 +103,8 @@ pub const Feature = enum {...@@ -107,8 +103,8 @@ pub const Feature = enum {
107 scalarize_cmp_vector_optimized,103 scalarize_cmp_vector_optimized,
108 scalarize_fptrunc,104 scalarize_fptrunc,
109 scalarize_fpext,105 scalarize_fpext,
110 scalarize_intcast,106 scalarize_int_cast,
111 scalarize_intcast_safe,107 scalarize_int_cast_safe,
112 scalarize_trunc,108 scalarize_trunc,
113 scalarize_int_from_float,109 scalarize_int_from_float,
114 scalarize_int_from_float_optimized,110 scalarize_int_from_float_optimized,
...@@ -122,16 +118,45 @@ pub const Feature = enum {...@@ -122,16 +118,45 @@ pub const Feature = enum {
122 scalarize_select,118 scalarize_select,
123 scalarize_mul_add,119 scalarize_mul_add,
124120
121 // Below are several different features for scalarizing `bit_cast` in different scenarios. It is
122 // valid to enable any combination of these features.
123
124 /// Scalarize `bit_cast` where the operand or result type is an array.
125 scalarize_bit_cast_array,
126 /// Scalarize `bit_cast` where either:
127 ///
128 /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
129 /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
130 ///
131 /// This effectively scalarizes any `bit_cast` to/from a vector, *unless* the operation can be
132 /// performed by bitcasting each vector element and returning a vector of the results.
133 ///
134 /// If this feature is enabled, the following AIR instruction tags may be emitted:
135 /// * `.legalize_vec_elem_val`
136 /// * `.legalize_vec_store_elem`
137 scalarize_bit_cast_vector_non_elementwise,
138 /// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
139 /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
140 /// need to sign- or zero-extend multiple elements to populate "padding" bits.
141 ///
142 /// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig`
143 /// to conform to the new bitcast semantics.
144 ///
145 /// If this feature is enabled, the following AIR instruction tags may be emitted:
146 /// * `.legalize_vec_elem_val`
147 /// * `.legalize_vec_store_elem`
148 scalarize_bit_cast_padded_elems,
149
125 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)150 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
126 unsplat_shift_rhs,151 unsplat_shift_rhs,
127 /// Legalize reduce of a one element vector to a bitcast.152 /// Legalize reduce of a one element vector to a bitcast.
128 reduce_one_elem_to_bitcast,153 reduce_one_elem_to_bit_cast,
129 /// Legalize splat to a one element vector to a bitcast.154 /// Legalize splat to a one element vector to a bitcast.
130 splat_one_elem_to_bitcast,155 splat_one_elem_to_bit_cast,
131156
132 /// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.157 /// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
133 /// Not compatible with `scalarize_intcast_safe`.158 /// Not compatible with `scalarize_int_cast_safe`.
134 expand_intcast_safe,159 expand_int_cast_safe,
135 /// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.160 /// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
136 /// Not compatible with `scalarize_int_from_float_safe`.161 /// Not compatible with `scalarize_int_from_float_safe`.
137 expand_int_from_float_safe,162 expand_int_from_float_safe,
...@@ -156,9 +181,9 @@ pub const Feature = enum {...@@ -156,9 +181,9 @@ pub const Feature = enum {
156 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the181 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
157 /// first byte of memory until bit pointers know their backing type.182 /// first byte of memory until bit pointers know their backing type.
158 expand_packed_store,183 expand_packed_store,
159 /// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.184 /// Replace `struct_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
160 expand_packed_struct_field_val,185 expand_packed_struct_field_val,
161 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.186 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
162 expand_packed_aggregate_init,187 expand_packed_aggregate_init,
163188
164 /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float189 /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
...@@ -227,7 +252,6 @@ pub const Feature = enum {...@@ -227,7 +252,6 @@ pub const Feature = enum {
227 .shl_sat => .scalarize_shl_sat,252 .shl_sat => .scalarize_shl_sat,
228 .xor => .scalarize_xor,253 .xor => .scalarize_xor,
229 .not => .scalarize_not,254 .not => .scalarize_not,
230 .bitcast => .scalarize_bitcast,
231 .clz => .scalarize_clz,255 .clz => .scalarize_clz,
232 .ctz => .scalarize_ctz,256 .ctz => .scalarize_ctz,
233 .popcount => .scalarize_popcount,257 .popcount => .scalarize_popcount,
...@@ -253,8 +277,11 @@ pub const Feature = enum {...@@ -253,8 +277,11 @@ pub const Feature = enum {
253 .cmp_vector_optimized => .scalarize_cmp_vector_optimized,277 .cmp_vector_optimized => .scalarize_cmp_vector_optimized,
254 .fptrunc => .scalarize_fptrunc,278 .fptrunc => .scalarize_fptrunc,
255 .fpext => .scalarize_fpext,279 .fpext => .scalarize_fpext,
256 .intcast => .scalarize_intcast,280 .int_cast => .scalarize_int_cast,
257 .intcast_safe => .scalarize_intcast_safe,281 .int_cast_safe => .scalarize_int_cast_safe,
282 .ptr_cast => .scalarize_ptr_cast,
283 .ptr_from_int => .scalarize_ptr_from_int,
284 .int_from_ptr => .scalarize_int_from_ptr,
258 .trunc => .scalarize_trunc,285 .trunc => .scalarize_trunc,
259 .int_from_float => .scalarize_int_from_float,286 .int_from_float => .scalarize_int_from_float,
260 .int_from_float_optimized => .scalarize_int_from_float_optimized,287 .int_from_float_optimized => .scalarize_int_from_float_optimized,
...@@ -474,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -474,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
474 .popcount,501 .popcount,
475 .byte_swap,502 .byte_swap,
476 .bit_reverse,503 .bit_reverse,
477 .intcast,504 .int_cast,
505 .ptr_cast,
506 .ptr_from_int,
507 .int_from_ptr,
478 .trunc,508 .trunc,
479 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {509 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
480 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;510 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -548,15 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -548,15 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
548 },578 },
549 }579 }
550 },580 },
551 .bitcast => if (l.features.has(.scalarize_bitcast)) {581 .bit_cast => if (l.features.hasAny(&.{
582 .scalarize_bit_cast_array,
583 .scalarize_bit_cast_vector_non_elementwise,
584 .scalarize_bit_cast_padded_elems,
585 })) {
552 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {586 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
553 continue :inst l.replaceInst(inst, .block, payload);587 continue :inst l.replaceInst(inst, .block, payload);
554 }588 }
555 },589 },
556 .intcast_safe => if (l.features.has(.expand_intcast_safe)) {590 .int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
557 assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both591 assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
558 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));592 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
559 } else if (l.features.has(.scalarize_intcast_safe)) {593 } else if (l.features.has(.scalarize_int_cast_safe)) {
560 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;594 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
561 if (ty_op.ty.toType().isVector(zcu)) {595 if (ty_op.ty.toType().isVector(zcu)) {
562 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));596 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
...@@ -772,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -772,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
772 inline .reduce, .reduce_optimized => |air_tag| {806 inline .reduce, .reduce_optimized => |air_tag| {
773 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;807 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
774 const vector_ty = l.typeOf(reduce.operand);808 const vector_ty = l.typeOf(reduce.operand);
775 if (l.features.has(.reduce_one_elem_to_bitcast)) {809 if (l.features.has(.reduce_one_elem_to_bit_cast)) {
776 switch (vector_ty.vectorLen(zcu)) {810 switch (vector_ty.vectorLen(zcu)) {
777 0 => unreachable,811 0 => unreachable,
778 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{812 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
779 .ty = .fromType(vector_ty.childType(zcu)),813 .ty = .fromType(vector_ty.childType(zcu)),
780 .operand = reduce.operand,814 .operand = reduce.operand,
781 } }),815 } }),
...@@ -792,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -792,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
792 .soft_float => unreachable, // the operand is not a scalar826 .soft_float => unreachable, // the operand is not a scalar
793 }827 }
794 },828 },
795 .splat => if (l.features.has(.splat_one_elem_to_bitcast)) {829 .splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
796 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;830 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
797 switch (ty_op.ty.toType().vectorLen(zcu)) {831 switch (ty_op.ty.toType().vectorLen(zcu)) {
798 0 => unreachable,832 0 => unreachable,
799 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{833 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
800 .ty = ty_op.ty,834 .ty = ty_op.ty,
801 .operand = ty_op.operand,835 .operand = ty_op.operand,
802 } }),836 } }),
...@@ -862,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -862,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
862 const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);896 const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
863 if (field_bits == struct_bits) {897 if (field_bits == struct_bits) {
864 // Just bitcast this field.898 // Just bitcast this field.
865 continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{899 continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
866 .ty = .fromType(agg_ty),900 .ty = .fromType(agg_ty),
867 .operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),901 .operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),
868 } });902 } });
...@@ -909,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -909,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
909 .legalize_vec_store_elem,943 .legalize_vec_store_elem,
910 .legalize_compiler_rt_call,944 .legalize_compiler_rt_call,
911 .spirv_runtime_array_len,945 .spirv_runtime_array_len,
946 .error_cast,
947 .error_from_int,
948 .int_from_error,
949 .union_from_enum,
912 => {},950 => {},
913 }951 }
914 }952 }
...@@ -931,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz...@@ -931,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz
931969
932 if (result_is_array) {970 if (result_is_array) {
933 // This is only allowed when legalizing an elementwise bitcast.971 // This is only allowed when legalizing an elementwise bitcast.
934 assert(orig.tag == .bitcast);972 assert(orig.tag == .bit_cast);
935 assert(form == .ty_op);973 assert(form == .ty_op);
936 }974 }
937975
...@@ -1423,35 +1461,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1423,35 +1461,94 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1423 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;1461 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
14241462
1425 const dest_ty = ty_op.ty.toType();1463 const dest_ty = ty_op.ty.toType();
1426 const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
1427 else => true,
1428 .array, .vector => legal: {
1429 if (dest_ty.arrayLen(zcu) == 1) break :legal true;
1430 const dest_elem_ty = dest_ty.childType(zcu);
1431 break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
1432 },
1433 };
1434
1435 const operand_ty = l.typeOf(ty_op.operand);1464 const operand_ty = l.typeOf(ty_op.operand);
1436 const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
1437 else => true,
1438 .array, .vector => legal: {
1439 if (operand_ty.arrayLen(zcu) == 1) break :legal true;
1440 const operand_elem_ty = operand_ty.childType(zcu);
1441 break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
1442 },
1443 };
14441465
1445 if (dest_legal and operand_legal) return null;1466 // We exit this block only if the scalarization is actually necessary. Otherwise we will return
1467 // `null` from within the block.
1468 const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: {
1469 const operand_tag = operand_ty.zigTypeTag(zcu);
1470 const dest_tag = dest_ty.zigTypeTag(zcu);
1471
1472 if (operand_tag != .array and
1473 operand_tag != .vector and
1474 dest_tag != .array and
1475 dest_tag != .vector)
1476 {
1477 return null;
1478 }
1479
1480 // We track the validity of 3 different bitcast operations:
1481 // * operand -> dest
1482 // * operand -> uint
1483 // * uint -> dest
1484 // If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing
1485 // the validity of the other operations helps us lower the scalarization efficiently.
1486 var operand_to_dest: bool = true;
1487 var operand_to_int: bool = true;
1488 var int_to_dest: bool = true;
1489
1490 if (l.features.has(.scalarize_bit_cast_array)) {
1491 if (operand_tag == .array) {
1492 operand_to_dest = false;
1493 operand_to_int = false;
1494 }
1495 if (dest_tag == .array) {
1496 operand_to_dest = false;
1497 int_to_dest = false;
1498 }
1499 }
14461500
1447 if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {1501 if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
1448 // from_ty and to_ty are both arrays or vectors of types with the same bit size,1502 if (operand_tag == .vector) operand_to_int = false;
1449 // so we can do an elementwise bitcast.1503 if (dest_tag == .vector) int_to_dest = false;
1450 return try l.scalarizeBlockPayload(orig_inst, .ty_op);
1451 }
14521504
1453 // Fallback path. Our strategy is to use an unsigned integer type as an intermediate1505 if (operand_tag == .vector or dest_tag == .vector) {
1454 // "bag of bits" representation which can be manipulated by bitwise operations.1506 if (operand_tag != .vector or
1507 dest_tag != .vector or
1508 operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu))
1509 {
1510 operand_to_dest = false;
1511 }
1512 }
1513 }
1514
1515 if (l.features.has(.scalarize_bit_cast_padded_elems)) {
1516 if (operand_tag == .array or operand_tag == .vector) {
1517 const elem_ty = operand_ty.childType(zcu);
1518 if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
1519 operand_to_int = false;
1520 operand_to_dest = false;
1521 }
1522 }
1523 if (dest_tag == .array or dest_tag == .vector) {
1524 const elem_ty = dest_ty.childType(zcu);
1525 if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
1526 int_to_dest = false;
1527 operand_to_dest = false;
1528 }
1529 }
1530 }
1531
1532 if (operand_to_dest) {
1533 return null; // no scalarization needed!
1534 }
1535
1536 // We need a scalarization, but before breaking from the block, check if we can do it
1537 // elementwise---if we can, that's preferable to the generic lowering.
1538 if ((operand_tag == .array or operand_tag == .vector) and
1539 (dest_tag == .array or dest_tag == .vector) and
1540 operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu))
1541 {
1542 // Operand and result types are both arrays/vectors whose element types have the same
1543 // bit size, so we can do an elementwise bitcast.
1544 return try l.scalarizeBlockPayload(orig_inst, .ty_op);
1545 }
1546
1547 break :int_ok .{ operand_to_int, int_to_dest };
1548 };
1549
1550 // Generic scalarization implementation. Our strategy is to use an unsigned integer type as an
1551 // intermediate "bag of bits" representation which can be manipulated by bitwise operations.
14551552
1456 const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));1553 const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
1457 assert(operand_ty.bitSize(zcu) == num_bits);1554 assert(operand_ty.bitSize(zcu) == num_bits);
...@@ -1465,11 +1562,17 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1465,11 +1562,17 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1465 // First, convert `operand_ty` to `uint_ty` (`uN`).1562 // First, convert `operand_ty` to `uint_ty` (`uN`).
14661563
1467 const uint_val: Air.Inst.Ref = uint_val: {1564 const uint_val: Air.Inst.Ref = uint_val: {
1468 if (operand_legal) {1565 if (operand_to_int_ok) {
1469 _ = main_block.stealCapacity(19);1566 _ = main_block.stealCapacity(19);
1470 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);1567 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1471 }1568 }
14721569
1570 if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
1571 _ = main_block.stealCapacity(18);
1572 const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
1573 break :uint_val main_block.addBitCast(l, uint_ty, elem);
1574 }
1575
1473 // %1 = block({1576 // %1 = block({
1474 // %2 = alloc(*usize)1577 // %2 = alloc(*usize)
1475 // %3 = alloc(*uN)1578 // %3 = alloc(*uN)
...@@ -1478,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1478,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1478 // %6 = loop({1581 // %6 = loop({
1479 // %7 = load(%2)1582 // %7 = load(%2)
1480 // %8 = array_elem_val(orig_operand, %7)1583 // %8 = array_elem_val(orig_operand, %7)
1481 // %9 = bitcast(uE, %8)1584 // %9 = bit_cast(uE, %8)
1482 // %10 = intcast(uN, %9)1585 // %10 = int_cast(uN, %9)
1483 // %11 = load(%3)1586 // %11 = load(%3)
1484 // %12 = shl_exact(%11, <uS, E>)1587 // %12 = shl_exact(%11, <uS, E>)
1485 // %13 = bit_or(%12, %10)1588 // %13 = bit_or(%12, %10)
...@@ -1529,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1529,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1529 index_val,1632 index_val,
1530 ).toRef();1633 ).toRef();
1531 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);1634 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1532 const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();1635 const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
1533 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();1636 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1534 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();1637 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1535 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();1638 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
...@@ -1560,10 +1663,23 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1560,10 +1663,23 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
15601663
1561 // Now convert `uint_ty` (`uN`) to `dest_ty`.1664 // Now convert `uint_ty` (`uN`) to `dest_ty`.
15621665
1563 if (dest_legal) {1666 if (int_to_dest_ok) {
1564 _ = main_block.stealCapacity(17);1667 _ = main_block.stealCapacity(17);
1565 const result = main_block.addBitCast(l, dest_ty, uint_val);1668 const result = main_block.addBitCast(l, dest_ty, uint_val);
1566 main_block.addBr(l, orig_inst, result);1669 main_block.addBr(l, orig_inst, result);
1670 } else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
1671 _ = main_block.stealCapacity(16);
1672 const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
1673 const aggregate_init_payload_start = l.air_extra.items.len;
1674 try l.air_extra.append(zcu.gpa, @intFromEnum(elem));
1675 const result = main_block.add(l, .{
1676 .tag = .aggregate_init,
1677 .data = .{ .ty_pl = .{
1678 .ty = .fromType(dest_ty),
1679 .payload = @intCast(aggregate_init_payload_start),
1680 } },
1681 }).toRef();
1682 main_block.addBr(l, orig_inst, result);
1567 } else {1683 } else {
1568 // %1 = alloc(*usize)1684 // %1 = alloc(*usize)
1569 // %2 = alloc(*@Vector(N, Result))1685 // %2 = alloc(*@Vector(N, Result))
...@@ -1571,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1571,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1571 // %4 = loop({1687 // %4 = loop({
1572 // %5 = load(%1)1688 // %5 = load(%1)
1573 // %6 = mul(%5, <usize, E>)1689 // %6 = mul(%5, <usize, E>)
1574 // %7 = intcast(uS, %6)1690 // %7 = int_cast(uS, %6)
1575 // %8 = shr(uint_val, %7)1691 // %8 = shr(uint_val, %7)
1576 // %9 = trunc(uE, %8)1692 // %9 = trunc(uE, %8)
1577 // %10 = bitcast(Result, %9)1693 // %10 = bit_cast(Result, %9)
1578 // %11 = legalize_vec_store_elem(%2, %5, %10)1694 // %11 = legalize_vec_store_elem(%2, %5, %10)
1579 // %12 = cmp_eq(%5, <usize, vec_len>)1695 // %12 = cmp_eq(%5, <usize, vec_len>)
1580 // %13 = cond_br(%12, {1696 // %13 = cond_br(%12, {
...@@ -1603,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1603,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
16031719
1604 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();1720 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1605 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();1721 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1606 const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();1722 const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
1607 const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();1723 const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
1608 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();1724 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1609 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);1725 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
...@@ -1981,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -1981,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
1981 // %5 = call(@panic.invalidEnumValue, [])2097 // %5 = call(@panic.invalidEnumValue, [])
1982 // %6 = unreach()2098 // %6 = unreach()
1983 // }, {2099 // }, {
1984 // %7 = intcast(@res_ty, %y)2100 // %7 = int_cast(@res_ty, %y)
1985 // %8 = is_named_enum_value(%7)2101 // %8 = is_named_enum_value(%7)
1986 // %9 = cond_br(%8, {2102 // %9 = cond_br(%8, {
1987 // %10 = br(%x, %7)2103 // %10 = br(%x, %7)
...@@ -2003,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2003,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2003 // %6 = call(@panic.invalidEnumValue, [])2119 // %6 = call(@panic.invalidEnumValue, [])
2004 // %7 = unreach()2120 // %7 = unreach()
2005 // }, {2121 // }, {
2006 // %8 = intcast(@res_ty, %y)2122 // %8 = int_cast(@res_ty, %y)
2007 // %9 = br(%x, %8)2123 // %9 = br(%x, %8)
2008 // })2124 // })
2009 // })2125 // })
...@@ -2056,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2056,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2056 cur_block = &condbr.else_block;2172 cur_block = &condbr.else_block;
2057 }2173 }
20582174
2059 // Now we know we're in-range, we can intcast:2175 // Now we know we're in-range, we can int_cast:
2060 const cast_inst = cur_block.add(l, .{2176 const cast_inst = cur_block.add(l, .{
2061 .tag = .intcast,2177 .tag = .int_cast,
2062 .data = .{ .ty_op = .{2178 .data = .{ .ty_op = .{
2063 .ty = Air.internedToRef(dest_ty.toIntern()),2179 .ty = Air.internedToRef(dest_ty.toIntern()),
2064 .operand = operand_ref,2180 .operand = operand_ref,
...@@ -2229,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_...@@ -2229,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2229 // %1 = add_with_overflow(%x, %y)2345 // %1 = add_with_overflow(%x, %y)
2230 // %2 = struct_field_val(%1, .@"1")2346 // %2 = struct_field_val(%1, .@"1")
2231 // %3 = reduce(%2, .@"or")2347 // %3 = reduce(%2, .@"or")
2232 // %4 = bitcast(%3, @bool_type)2348 // %4 = bit_cast(%3, @bool_type)
2233 // %5 = cond_br(%4, {2349 // %5 = cond_br(%4, {
2234 // %6 = call(@panic.integerOverflow, [])2350 // %6 = call(@panic.integerOverflow, [])
2235 // %7 = unreach()2351 // %7 = unreach()
...@@ -2335,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins...@@ -2335,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins
2335 .tag = .load,2451 .tag = .load,
2336 .data = .{ .ty_op = .{2452 .data = .{ .ty_op = .{
2337 .ty = Air.internedToRef(load_ty.toIntern()),2453 .ty = Air.internedToRef(load_ty.toIntern()),
2338 .operand = res_block.addBitCast(l, load_ptr_ty: {2454 .operand = res_block.addPtrCast(l, load_ptr_ty: {
2339 var load_ptr_info = ptr_info;2455 var load_ptr_info = ptr_info;
2340 load_ptr_info.child = load_ty.toIntern();2456 load_ptr_info.child = load_ty.toIntern();
2341 load_ptr_info.flags.vector_index = .none;2457 load_ptr_info.flags.vector_index = .none;
...@@ -2378,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2378,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
23782494
2379 var res_block: Block = .init(&inst_buf);2495 var res_block: Block = .init(&inst_buf);
2380 {2496 {
2381 const backing_ptr_inst = res_block.add(l, .{2497 const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
2382 .tag = .bitcast,2498 var load_ptr_info = ptr_info;
2383 .data = .{ .ty_op = .{2499 load_ptr_info.child = load_store_ty.toIntern();
2384 .ty = Air.internedToRef((load_store_ptr_ty: {2500 load_ptr_info.flags.vector_index = .none;
2385 var load_ptr_info = ptr_info;2501 load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2386 load_ptr_info.child = load_store_ty.toIntern();2502 break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2387 load_ptr_info.flags.vector_index = .none;2503 }, orig_bin_op.lhs);
2388 load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2389 break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2390 }).toIntern()),
2391 .operand = orig_bin_op.lhs,
2392 } },
2393 });
2394 _ = res_block.add(l, .{2504 _ = res_block.add(l, .{
2395 .tag = .store,2505 .tag = .store,
2396 .data = .{ .bin_op = .{2506 .data = .{ .bin_op = .{
2397 .lhs = backing_ptr_inst.toRef(),2507 .lhs = backing_ptr,
2398 .rhs = res_block.add(l, .{2508 .rhs = res_block.add(l, .{
2399 .tag = .bit_or,2509 .tag = .bit_or,
2400 .data = .{ .bin_op = .{2510 .data = .{ .bin_op = .{
...@@ -2405,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2405,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2405 .tag = .load,2515 .tag = .load,
2406 .data = .{ .ty_op = .{2516 .data = .{ .ty_op = .{
2407 .ty = Air.internedToRef(load_store_ty.toIntern()),2517 .ty = Air.internedToRef(load_store_ty.toIntern()),
2408 .operand = backing_ptr_inst.toRef(),2518 .operand = backing_ptr,
2409 } },2519 } },
2410 }).toRef(),2520 }).toRef(),
2411 .rhs = Air.internedToRef((keep_mask: {2521 .rhs = Air.internedToRef((keep_mask: {
...@@ -2434,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2434,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2434 .tag = .shl_exact,2544 .tag = .shl_exact,
2435 .data = .{ .bin_op = .{2545 .data = .{ .bin_op = .{
2436 .lhs = res_block.add(l, .{2546 .lhs = res_block.add(l, .{
2437 .tag = .intcast,2547 .tag = .int_cast,
2438 .data = .{ .ty_op = .{2548 .data = .{ .ty_op = .{
2439 .ty = Air.internedToRef(load_store_ty.toIntern()),2549 .ty = Air.internedToRef(load_store_ty.toIntern()),
2440 .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),2550 .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
...@@ -2532,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro...@@ -2532,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro
25322642
2533 const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();2643 const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2534 const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);2644 const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2535 const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();2645 const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
2536 cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();2646 cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2537 }2647 }
25382648
...@@ -2721,18 +2831,51 @@ const Block = struct {...@@ -2721,18 +2831,51 @@ const Block = struct {
2721 });2831 });
2722 }2832 }
27232833
2724 /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for2834 /// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2725 /// no-op casts.2835 /// no-op casts.
2726 fn addBitCast(2836 fn addBitCast(
2727 b: *Block,2837 b: *Block,
2728 l: *Legalize,2838 l: *Legalize,
2729 ty: Type,2839 result_ty: Type,
2840 operand: Air.Inst.Ref,
2841 ) Air.Inst.Ref {
2842 const zcu = l.pt.zcu;
2843 const operand_ty = l.typeOf(operand);
2844 assert(!operand_ty.isPtrAtRuntime(zcu));
2845 assert(!operand_ty.isSliceAtRuntime(zcu));
2846 assert(!result_ty.isPtrAtRuntime(zcu));
2847 assert(!result_ty.isSliceAtRuntime(zcu));
2848 if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
2849 .tag = .bit_cast,
2850 .data = .{ .ty_op = .{
2851 .ty = .fromType(result_ty),
2852 .operand = operand,
2853 } },
2854 }).toRef();
2855 _ = b.stealCapacity(1);
2856 return operand;
2857 }
2858
2859 /// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2860 /// no-op casts.
2861 fn addPtrCast(
2862 b: *Block,
2863 l: *Legalize,
2864 result_ty: Type,
2730 operand: Air.Inst.Ref,2865 operand: Air.Inst.Ref,
2731 ) Air.Inst.Ref {2866 ) Air.Inst.Ref {
2732 if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{2867 const zcu = l.pt.zcu;
2733 .tag = .bitcast,2868 const operand_ty = l.typeOf(operand);
2869 if (operand_ty.isSliceAtRuntime(zcu)) {
2870 assert(result_ty.isSliceAtRuntime(zcu));
2871 } else {
2872 assert(operand_ty.isPtrAtRuntime(zcu));
2873 assert(result_ty.isPtrAtRuntime(zcu));
2874 }
2875 if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
2876 .tag = .ptr_cast,
2734 .data = .{ .ty_op = .{2877 .data = .{ .ty_op = .{
2735 .ty = Air.internedToRef(ty.toIntern()),2878 .ty = .fromType(result_ty),
2736 .operand = operand,2879 .operand = operand,
2737 } },2880 } },
2738 }).toRef();2881 }).toRef();
...@@ -3073,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3073,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3073 var main_block: Block = .init(&inst_buf);3216 var main_block: Block = .init(&inst_buf);
3074 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3217 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
30753218
3076 const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();3219 const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3077 const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});3220 const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3078 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());3221 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3079 main_block.addBr(l, orig_inst, casted_result);3222 main_block.addBr(l, orig_inst, casted_result);
...@@ -3100,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3100,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3100 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3243 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
31013244
3102 const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {3245 const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3103 break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();3246 break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3104 } else ext: {3247 } else ext: {
3105 _ = main_block.stealCapacity(1);3248 _ = main_block.stealCapacity(1);
3106 break :ext ty_op.operand;3249 break :ext ty_op.operand;
...@@ -3165,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3165,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3165 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3308 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
31663309
3167 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});3310 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3168 const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();3311 const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
3169 main_block.addBr(l, orig_inst, casted_val);3312 main_block.addBr(l, orig_inst, casted_val);
31703313
3171 return .{ .block_payload = .{ .ty_pl = .{3314 return .{ .block_payload = .{ .ty_pl = .{
...@@ -3189,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3189,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3189 const bits_val = try pt.intValue(.usize, dest_info.bits);3332 const bits_val = try pt.intValue(.usize, dest_info.bits);
3190 _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });3333 _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3191 const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();3334 const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3192 const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();3335 const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
3193 main_block.addBr(l, orig_inst, result_val);3336 main_block.addBr(l, orig_inst, result_val);
31943337
3195 return .{ .block_payload = .{ .ty_pl = .{3338 return .{ .block_payload = .{ .ty_pl = .{
src/Air/Liveness.zig+10-3
...@@ -488,12 +488,19 @@ fn analyzeInst(...@@ -488,12 +488,19 @@ fn analyzeInst(
488 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),488 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
489489
490 .not,490 .not,
491 .bitcast,491 .bit_cast,
492 .ptr_cast,
493 .ptr_from_int,
494 .int_from_ptr,
495 .error_cast,
496 .error_from_int,
497 .int_from_error,
498 .union_from_enum,
492 .load,499 .load,
493 .fpext,500 .fpext,
494 .fptrunc,501 .fptrunc,
495 .intcast,502 .int_cast,
496 .intcast_safe,503 .int_cast_safe,
497 .trunc,504 .trunc,
498 .optional_payload,505 .optional_payload,
499 .optional_payload_ptr,506 .optional_payload_ptr,
src/Air/Liveness/Verify.zig+10-3
...@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
7878
79 // unary79 // unary
80 .not,80 .not,
81 .bitcast,81 .bit_cast,
82 .ptr_cast,
83 .ptr_from_int,
84 .int_from_ptr,
85 .error_cast,
86 .error_from_int,
87 .int_from_error,
88 .union_from_enum,
82 .load,89 .load,
83 .fpext,90 .fpext,
84 .fptrunc,91 .fptrunc,
85 .intcast,92 .int_cast,
86 .intcast_safe,93 .int_cast_safe,
87 .trunc,94 .trunc,
88 .optional_payload,95 .optional_payload,
89 .optional_payload_ptr,96 .optional_payload_ptr,
src/Air/Verify.zig created+465
...@@ -0,0 +1,465 @@
1/// Verifies that AIR is valid, in that every instruction has valid operands and types. In compiler
2/// builds with debug extensions, this is run on all AIR, both before `Air.Legalize` is run and (if
3/// it is run) after it.
4///
5/// This verification pass is currently highly incomplete---expand it as needed.
6const Verify = @This();
7
8zcu: *Zcu,
9func_index: InternPool.Index,
10ret_ty: Type,
11air: *const Air,
12cur_inst: Air.Inst.Index,
13
14pub fn run(pt: Zcu.PerThread, func_index: InternPool.Index, air: *const Air) void {
15 if (!@import("build_options").enable_debug_extensions) {
16 // `Air.Verify` is a debugging feature---it should not be used in release builds because it
17 // has little benefit and negatively affects compiler performance.
18 return;
19 }
20
21 const zcu = pt.zcu;
22
23 const func_ty: Type = Value.fromInterned(func_index).typeOf(zcu);
24 const ret_ty = func_ty.fnReturnType(zcu);
25
26 var verify: Verify = .{
27 .zcu = zcu,
28 .func_index = func_index,
29 .ret_ty = ret_ty,
30 .air = air,
31 .cur_inst = undefined, // populated by `body(...)`
32 };
33 verify.body(air.getMainBody()) catch |verify_err| switch (verify_err) {
34 error.VerifyFail => {
35 const ip = &zcu.intern_pool;
36 const func_nav = ip.indexToKey(func_index).func.owner_nav;
37 const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
38 log.info("AIR for '{s}':", .{func_fqn});
39 const io = zcu.comp.io;
40 const stderr = io.lockStderr(&.{}, null) catch |err| switch (err) {
41 error.Canceled => return io.recancel(),
42 };
43 defer io.unlockStderr();
44 air.write(&stderr.file_writer.interface, pt, null) catch |err| switch (err) {
45 error.WriteFailed => switch (stderr.file_writer.err.?) {
46 error.Canceled => return io.recancel(),
47 else => {},
48 },
49 };
50 },
51 };
52}
53
54const Error = error{VerifyFail};
55
56fn fail(verify: *Verify, msg: []const u8) Error {
57 const ip = &verify.zcu.intern_pool;
58 const func_nav = ip.indexToKey(verify.func_index).func.owner_nav;
59 const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
60 log.err("'{s}', %{d}: {s}", .{ func_fqn, verify.cur_inst, msg });
61 return error.VerifyFail;
62}
63
64fn body(verify: *Verify, body_insts: []const Air.Inst.Index) Error!void {
65 const zcu = verify.zcu;
66 const ip = &zcu.intern_pool;
67 const air = verify.air;
68 const tags = air.instructions.items(.tag);
69 const data = air.instructions.items(.data);
70 for (body_insts, 0..) |inst, body_index| {
71 verify.cur_inst = inst;
72 switch (tags[@intFromEnum(inst)]) {
73 .block => {
74 const block = air.unwrapBlock(inst);
75 try verify.body(block.body);
76 },
77 .dbg_inline_block => {
78 const block = air.unwrapDbgBlock(inst);
79 try verify.body(block.body);
80 },
81 .@"try", .try_cold => {
82 const @"try" = air.unwrapTry(inst);
83 try verify.body(@"try".else_body);
84 },
85 .try_ptr, .try_ptr_cold => {
86 const try_ptr = air.unwrapTryPtr(inst);
87 try verify.body(try_ptr.else_body);
88 },
89 .loop => {
90 const block = air.unwrapBlock(inst);
91 try verify.body(block.body);
92 },
93 .cond_br => {
94 const cond_br = air.unwrapCondBr(inst);
95 try verify.body(cond_br.then_body);
96 try verify.body(cond_br.else_body);
97 },
98 .switch_br, .loop_switch_br => {
99 const switch_br = air.unwrapSwitch(inst);
100 var it = switch_br.iterateCases();
101 while (it.next()) |case| {
102 try verify.body(case.body);
103 }
104 const else_body = it.elseBody();
105 if (else_body.len > 0) {
106 try verify.body(else_body);
107 }
108 },
109 .ret, .ret_safe => {
110 const operand = data[@intFromEnum(inst)].un_op;
111 if (air.typeOf(operand, ip).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
112 },
113 .ret_load => {
114 const operand = data[@intFromEnum(inst)].un_op;
115 const ptr_ty = air.typeOf(operand, ip);
116 if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("operand is not a pointer");
117 if (ptr_ty.ptrSize(zcu) != .one) return verify.fail("pointer size is not '.one'");
118 if (ptr_ty.childType(zcu).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
119 },
120
121 .bit_cast => {
122 const ty_op = data[@intFromEnum(inst)].ty_op;
123 const operand_ty = air.typeOf(ty_op.operand, ip);
124 const result_ty = ty_op.ty.toType();
125 // Enums are allowed here even if their backing type is implicit.
126 if (!operand_ty.hasBitRepresentation(zcu) and operand_ty.zigTypeTag(zcu) != .@"enum") {
127 return verify.fail("bad operand type");
128 }
129 if (!result_ty.hasBitRepresentation(zcu) and result_ty.zigTypeTag(zcu) != .@"enum") {
130 return verify.fail("bad result type");
131 }
132 if (operand_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type (pointer)");
133 if (result_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type (pointer)");
134 if (operand_ty.bitSize(zcu) != result_ty.bitSize(zcu)) return verify.fail("bit size mismatch");
135 },
136 .ptr_cast => {
137 const ty_op = data[@intFromEnum(inst)].ty_op;
138 const operand_ty = air.typeOf(ty_op.operand, ip);
139 const result_ty = ty_op.ty.toType();
140 const operand_scalar_ty = operand_ty.scalarType(zcu);
141 const result_scalar_ty = result_ty.scalarType(zcu);
142 if (operand_ty.isSliceAtRuntime(zcu)) {
143 if (!result_ty.isSliceAtRuntime(zcu)) return verify.fail("operand is slice, but result is not");
144 } else {
145 if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
146 if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("operand is pointer, but result is not");
147 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
148 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
149 }
150 if (operand_scalar_ty.ptrAddressSpace(zcu) != result_scalar_ty.ptrAddressSpace(zcu)) {
151 return verify.fail("illegal change to address space");
152 }
153 },
154 .ptr_from_int => {
155 const ty_op = data[@intFromEnum(inst)].ty_op;
156 const operand_ty = air.typeOf(ty_op.operand, ip);
157 const result_ty = ty_op.ty.toType();
158 const operand_scalar_ty = operand_ty.scalarType(zcu);
159 const result_scalar_ty = result_ty.scalarType(zcu);
160 if (operand_scalar_ty.toIntern() != .usize_type) return verify.fail("bad operand type");
161 if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type");
162 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
163 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
164 },
165 .int_from_ptr => {
166 const ty_op = data[@intFromEnum(inst)].ty_op;
167 const operand_ty = air.typeOf(ty_op.operand, ip);
168 const result_ty = ty_op.ty.toType();
169 const operand_scalar_ty = operand_ty.scalarType(zcu);
170 const result_scalar_ty = result_ty.scalarType(zcu);
171 if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
172 if (result_scalar_ty.toIntern() != .usize_type) return verify.fail("bad result type");
173 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
174 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
175 },
176 .error_cast => {
177 const ty_op = data[@intFromEnum(inst)].ty_op;
178 const operand_ty = air.typeOf(ty_op.operand, ip);
179 const result_ty = ty_op.ty.toType();
180 switch (operand_ty.zigTypeTag(zcu)) {
181 else => return verify.fail("bad operand type"),
182 .error_union => {
183 if (result_ty.zigTypeTag(zcu) != .error_union) {
184 return verify.fail("operand is error union, but result is not");
185 }
186 if (operand_ty.errorUnionPayload(zcu).toIntern() != result_ty.errorUnionPayload(zcu).toIntern()) {
187 return verify.fail("error union payload type differs");
188 }
189 },
190 .error_set => if (result_ty.zigTypeTag(zcu) != .error_set) {
191 return verify.fail("operand is error set, but result is not");
192 },
193 }
194 },
195 .error_from_int => {
196 const ty_op = data[@intFromEnum(inst)].ty_op;
197 const operand_ty = air.typeOf(ty_op.operand, ip);
198 const result_ty = ty_op.ty.toType();
199 if (!operand_ty.isUnsignedInt(zcu)) return verify.fail("bad operand type");
200 if (operand_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad operand bit size");
201 if (result_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad result type");
202 },
203 .int_from_error => {
204 const ty_op = data[@intFromEnum(inst)].ty_op;
205 const operand_ty = air.typeOf(ty_op.operand, ip);
206 const result_ty = ty_op.ty.toType();
207 if (operand_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad operand type");
208 if (!result_ty.isUnsignedInt(zcu)) return verify.fail("bad result type");
209 if (result_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad result bit size");
210 },
211 .union_from_enum => {
212 const ty_op = data[@intFromEnum(inst)].ty_op;
213 const operand_ty = air.typeOf(ty_op.operand, ip);
214 const result_ty = ty_op.ty.toType();
215 if (operand_ty.zigTypeTag(zcu) != .@"enum") return verify.fail("bad operand type");
216 if (result_ty.zigTypeTag(zcu) != .@"union") return verify.fail("bad result type");
217 const union_tag_ty = result_ty.unionTagType(zcu) orelse return verify.fail("union type is not tagged");
218 if (union_tag_ty.toIntern() != operand_ty.toIntern()) return verify.fail("union tag type does not match operand type");
219 },
220
221 .ptr_elem_ptr => {
222 const ty_pl = data[@intFromEnum(inst)].ty_pl;
223 const bin_op = air.extraData(Air.Bin, ty_pl.payload).data;
224 const ptr_ty = air.typeOf(bin_op.lhs, ip);
225 const result_ty = ty_pl.ty.toType();
226 if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad pointer type");
227 if (result_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad result type");
228 const ptr_info = ptr_ty.ptrInfo(zcu);
229 const result_ptr_info = result_ty.ptrInfo(zcu);
230 if (ptr_info.packed_offset.host_size != 0) return verify.fail("pointer type is bitpacked pointer");
231 if (result_ptr_info.packed_offset.host_size != 0) return verify.fail("result type is bitpacked pointer");
232 },
233
234 .arg,
235 .add,
236 .add_safe,
237 .add_optimized,
238 .add_wrap,
239 .add_sat,
240 .sub,
241 .sub_safe,
242 .sub_optimized,
243 .sub_wrap,
244 .sub_sat,
245 .mul,
246 .mul_safe,
247 .mul_optimized,
248 .mul_wrap,
249 .mul_sat,
250 .div_float,
251 .div_float_optimized,
252 .div_trunc,
253 .div_trunc_optimized,
254 .div_floor,
255 .div_floor_optimized,
256 .div_exact,
257 .div_exact_optimized,
258 .rem,
259 .rem_optimized,
260 .mod,
261 .mod_optimized,
262 .ptr_add,
263 .ptr_sub,
264 .max,
265 .min,
266 .add_with_overflow,
267 .sub_with_overflow,
268 .mul_with_overflow,
269 .shl_with_overflow,
270 .alloc,
271 .inferred_alloc,
272 .inferred_alloc_comptime,
273 .ret_ptr,
274 .assembly,
275 .bit_and,
276 .bit_or,
277 .shr,
278 .shr_exact,
279 .shl,
280 .shl_exact,
281 .shl_sat,
282 .xor,
283 .not,
284 .repeat,
285 .br,
286 .trap,
287 .breakpoint,
288 .ret_addr,
289 .frame_addr,
290 .call,
291 .call_always_tail,
292 .call_never_tail,
293 .call_never_inline,
294 .clz,
295 .ctz,
296 .popcount,
297 .byte_swap,
298 .bit_reverse,
299 .sqrt,
300 .sin,
301 .cos,
302 .tan,
303 .exp,
304 .exp2,
305 .log,
306 .log2,
307 .log10,
308 .abs,
309 .floor,
310 .ceil,
311 .round,
312 .trunc_float,
313 .neg,
314 .neg_optimized,
315 .cmp_lt,
316 .cmp_lt_optimized,
317 .cmp_lte,
318 .cmp_lte_optimized,
319 .cmp_eq,
320 .cmp_eq_optimized,
321 .cmp_gte,
322 .cmp_gte_optimized,
323 .cmp_gt,
324 .cmp_gt_optimized,
325 .cmp_neq,
326 .cmp_neq_optimized,
327 .cmp_vector,
328 .cmp_vector_optimized,
329 .switch_dispatch,
330 .dbg_stmt,
331 .dbg_empty_stmt,
332 .dbg_var_ptr,
333 .dbg_var_val,
334 .dbg_arg_inline,
335 .is_null,
336 .is_non_null,
337 .is_null_ptr,
338 .is_non_null_ptr,
339 .is_err,
340 .is_non_err,
341 .is_err_ptr,
342 .is_non_err_ptr,
343 .load,
344 .store,
345 .store_safe,
346 .unreach,
347 .fptrunc,
348 .fpext,
349 .int_cast,
350 .int_cast_safe,
351 .trunc,
352 .optional_payload,
353 .optional_payload_ptr,
354 .optional_payload_ptr_set,
355 .wrap_optional,
356 .unwrap_errunion_payload,
357 .unwrap_errunion_err,
358 .unwrap_errunion_payload_ptr,
359 .unwrap_errunion_err_ptr,
360 .errunion_payload_ptr_set,
361 .wrap_errunion_payload,
362 .wrap_errunion_err,
363 .struct_field_ptr,
364 .struct_field_ptr_index_0,
365 .struct_field_ptr_index_1,
366 .struct_field_ptr_index_2,
367 .struct_field_ptr_index_3,
368 .struct_field_val,
369 .set_union_tag,
370 .get_union_tag,
371 .slice,
372 .slice_len,
373 .slice_ptr,
374 .ptr_slice_len_ptr,
375 .ptr_slice_ptr_ptr,
376 .array_elem_val,
377 .slice_elem_val,
378 .slice_elem_ptr,
379 .ptr_elem_val,
380 .array_to_slice,
381 .int_from_float,
382 .int_from_float_optimized,
383 .int_from_float_safe,
384 .int_from_float_optimized_safe,
385 .float_from_int,
386 .reduce,
387 .reduce_optimized,
388 .splat,
389 .shuffle_one,
390 .shuffle_two,
391 .select,
392 .memset,
393 .memset_safe,
394 .memcpy,
395 .memmove,
396 .cmpxchg_weak,
397 .cmpxchg_strong,
398 .atomic_load,
399 .atomic_store_unordered,
400 .atomic_store_monotonic,
401 .atomic_store_release,
402 .atomic_store_seq_cst,
403 .atomic_rmw,
404 .is_named_enum_value,
405 .tag_name,
406 .error_name,
407 .error_set_has_value,
408 .aggregate_init,
409 .union_init,
410 .prefetch,
411 .mul_add,
412 .field_parent_ptr,
413 .wasm_memory_size,
414 .wasm_memory_grow,
415 .cmp_lte_errors_len,
416 .err_return_trace,
417 .set_err_return_trace,
418 .addrspace_cast,
419 .save_err_return_trace_index,
420 .runtime_nav_ptr,
421 .c_va_arg,
422 .c_va_copy,
423 .c_va_end,
424 .c_va_start,
425 .spirv_runtime_array_len,
426 .work_item_id,
427 .work_group_size,
428 .work_group_id,
429 .legalize_vec_store_elem,
430 .legalize_vec_elem_val,
431 .legalize_compiler_rt_call,
432 => {},
433 }
434 if (air.typeOfIndex(inst, ip).isNoReturn(zcu)) {
435 if (body_index == body_insts.len - 1) return;
436
437 // HACK: right now, we emit the safety check for noreturn functions returning in a weird
438 // way, where the `call` instruction is `noreturn` but there are still instructions
439 // following it. We need to figure out a better way to represent that! That safety check
440 // probably just needs to live exclusively in backends; putting AIR instructions after a
441 // call implies that we have e.g. a valid stack at that point, which we can't actually
442 // assume when the user has gotten a function's ABI wrong.
443 switch (tags[@intFromEnum(inst)]) {
444 .call,
445 .call_always_tail,
446 .call_never_tail,
447 .call_never_inline,
448 => continue,
449 else => {},
450 }
451
452 return verify.fail("body contains instructions after noreturn");
453 }
454 }
455 return verify.fail("body does not terminate noreturn");
456}
457
458const std = @import("std");
459const log = std.log.scoped(.air_verify);
460
461const Zcu = @import("../Zcu.zig");
462const InternPool = @import("../InternPool.zig");
463const Air = @import("../Air.zig");
464const Type = @import("../Type.zig");
465const Value = @import("../Value.zig");
src/Air/print.zig+10-3
...@@ -232,12 +232,19 @@ const Writer = struct {...@@ -232,12 +232,19 @@ const Writer = struct {
232 .arg => try w.writeArg(s, inst),232 .arg => try w.writeArg(s, inst),
233233
234 .not,234 .not,
235 .bitcast,235 .bit_cast,
236 .ptr_cast,
237 .ptr_from_int,
238 .int_from_ptr,
239 .error_cast,
240 .error_from_int,
241 .int_from_error,
242 .union_from_enum,
236 .load,243 .load,
237 .fptrunc,244 .fptrunc,
238 .fpext,245 .fpext,
239 .intcast,246 .int_cast,
240 .intcast_safe,247 .int_cast_safe,
241 .trunc,248 .trunc,
242 .optional_payload,249 .optional_payload,
243 .optional_payload_ptr,250 .optional_payload_ptr,
src/InternPool.zig+1-7
...@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {...@@ -6007,13 +6007,7 @@ pub const Alignment = enum(u6) {
6007 return r;6007 return r;
6008 }6008 }
60096009
6010 const LlvmBuilderAlignment = std.zig.llvm.Builder.Alignment;6010 pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
6011
6012 pub fn toLlvm(a: Alignment) LlvmBuilderAlignment {
6013 return @enumFromInt(@intFromEnum(a));
6014 }
6015
6016 pub fn fromLlvm(a: LlvmBuilderAlignment) Alignment {
6017 return @enumFromInt(@intFromEnum(a));6011 return @enumFromInt(@intFromEnum(a));
6018 }6012 }
6019};6013};
src/Sema.zig+376-478
...@@ -583,16 +583,6 @@ pub const Block = struct {...@@ -583,16 +583,6 @@ pub const Block = struct {
583 });583 });
584 }584 }
585585
586 fn addBitCast(block: *Block, ty: Type, operand: Air.Inst.Ref) Allocator.Error!Air.Inst.Ref {
587 return block.addInst(.{
588 .tag = .bitcast,
589 .data = .{ .ty_op = .{
590 .ty = Air.internedToRef(ty.toIntern()),
591 .operand = operand,
592 } },
593 });
594 }
595
596 fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {586 fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {
597 return block.addInst(.{587 return block.addInst(.{
598 .tag = tag,588 .tag = tag,
...@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
3113 // https://github.com/ziglang/zig/issues/65973103 // https://github.com/ziglang/zig/issues/6597
3114 if (sema.resolveValue(operand)) |operand_val| {3104 if (sema.resolveValue(operand)) |operand_val| {
3115 if (!operand_val.isNull(zcu)) {3105 if (!operand_val.isNull(zcu)) {
3116 break :single_ptr try sema.coerceInMemory(operand_val, single_ptr_ty);3106 break :single_ptr .fromValue(try pt.getCoerced(operand_val, single_ptr_ty));
3117 }3107 }
3118 }3108 }
3119 if (block.wantSafety()) {3109 if (block.wantSafety()) {
3120 const is_non_null = try block.addUnOp(.is_non_null, operand);3110 const is_non_null = try block.addUnOp(.is_non_null, operand);
3121 try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);3111 try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);
3122 }3112 }
3123 const single_ptr = try block.addBitCast(single_ptr_ty, operand);3113 const single_ptr = try block.addTyOp(.ptr_cast, single_ptr_ty, operand);
3124 try sema.checkKnownAllocPtr(block, operand, single_ptr);3114 try sema.checkKnownAllocPtr(block, operand, single_ptr);
3125 break :single_ptr single_ptr;3115 break :single_ptr single_ptr;
3126 },3116 },
...@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: *Sema, block: *Block, alloc: Air.Inst.Ref,...@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: *Sema, block: *Block, alloc: Air.Inst.Ref,
3586 .{ .elem = idx_val.toUnsignedInt(zcu) },3576 .{ .elem = idx_val.toUnsignedInt(zcu) },
3587 };3577 };
3588 },3578 },
3589 .bitcast => .{3579 .ptr_cast => .{
3590 tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,3580 tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,
3591 .same_addr,3581 .same_addr,
3592 },3582 },
...@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(...@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(
3729 // This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to3719 // This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to
3730 // this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this3720 // this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this
3731 // too because it doesn't really matter what they become.3721 // too because it doesn't really matter what they become.
3732 const nop_inst: Air.Inst = .{ .tag = .bitcast, .data = .{ .ty_op = .{3722 const nop_inst: Air.Inst = .{ .tag = .ptr_from_int, .data = .{ .ty_op = .{
3733 .ty = .fromIntern(alloc_ty.toIntern()),3723 .ty = .fromIntern(alloc_ty.toIntern()),
3734 .operand = .zero_usize,3724 .operand = .zero_usize,
3735 } } };3725 } } };
...@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: *Sema, block: *Block, alloc: Air.Inst.Ref) CompileError!Ai...@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: *Sema, block: *Block, alloc: Air.Inst.Ref) CompileError!Ai
3779 return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());3769 return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());
3780 }3770 }
37813771
3782 return block.addBitCast(const_ptr_ty, alloc);3772 return block.addTyOp(.ptr_cast, const_ptr_ty, alloc);
3783}3773}
37843774
3785fn zirAllocInferredComptime(3775fn zirAllocInferredComptime(
...@@ -7330,7 +7320,7 @@ fn analyzeCall(...@@ -7330,7 +7320,7 @@ fn analyzeCall(
7330 if (resolved_ty == .none) break :r result_raw;7320 if (resolved_ty == .none) break :r result_raw;
7331 // TODO: mutate in place the previous instruction if possible7321 // TODO: mutate in place the previous instruction if possible
7332 // rather than adding a bitcast instruction.7322 // rather than adding a bitcast instruction.
7333 break :r try block.addBitCast(.fromInterned(resolved_ty), result_raw);7323 break :r try block.addTyOp(.error_cast, .fromInterned(resolved_ty), result_raw);
7334 };7324 };
73357325
7336 if (block.isComptime()) {7326 if (block.isComptime()) {
...@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD...@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD
7636 }7626 }
76377627
7638 try sema.requireRuntimeBlock(block, src, operand_src);7628 try sema.requireRuntimeBlock(block, src, operand_src);
7639 return block.addBitCast(err_int_ty, operand);7629 return block.addTyOp(.int_from_error, err_int_ty, operand);
7640}7630}
76417631
7642fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {7632fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
...@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD...@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD
7674 const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);7664 const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);
7675 try sema.addSafetyCheck(block, src, ok, .invalid_error_code);7665 try sema.addSafetyCheck(block, src, ok, .invalid_error_code);
7676 }7666 }
7677 return block.addInst(.{7667 return block.addTyOp(.error_from_int, .anyerror, operand);
7678 .tag = .bitcast,
7679 .data = .{ .ty_op = .{
7680 .ty = .anyerror_type,
7681 .operand = operand,
7682 } },
7683 });
7684}7668}
76857669
7686fn zirMergeErrorSets(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {7670fn zirMergeErrorSets(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
7863 }7847 }
78647848
7865 try sema.requireRuntimeBlock(block, src, operand_src);7849 try sema.requireRuntimeBlock(block, src, operand_src);
7866 return block.addBitCast(int_tag_ty, enum_tag);7850 return block.addTyOp(.bit_cast, int_tag_ty, enum_tag);
7867}7851}
78687852
7869fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {7853fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
7922 try sema.requireRuntimeBlock(block, src, operand_src);7906 try sema.requireRuntimeBlock(block, src, operand_src);
7923 if (block.wantSafety()) {7907 if (block.wantSafety()) {
7924 try sema.preparePanicId(src, .invalid_enum_value);7908 try sema.preparePanicId(src, .invalid_enum_value);
7925 return block.addTyOp(.intcast_safe, dest_ty, operand);7909 return block.addTyOp(.int_cast_safe, dest_ty, operand);
7926 }7910 }
7927 return block.addTyOp(.intcast, dest_ty, operand);7911 return block.addTyOp(.int_cast, dest_ty, operand);
7928}7912}
79297913
7930/// Pointer in, pointer out.7914/// Pointer in, pointer out.
...@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!...@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
9152 try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);9136 try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);
9153 try sema.validateRuntimeValue(block, ptr_src, operand);9137 try sema.validateRuntimeValue(block, ptr_src, operand);
9154 try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);9138 try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);
9155 return block.addBitCast(dest_ty, operand);9139 return block.addTyOp(.int_from_ptr, dest_ty, operand);
9156}9140}
91579141
9158fn zirFieldPtrLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9142fn zirFieldPtrLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -9314,9 +9298,9 @@ fn intCast(...@@ -9314,9 +9298,9 @@ fn intCast(
9314 try sema.requireRuntimeBlock(block, src, operand_src);9298 try sema.requireRuntimeBlock(block, src, operand_src);
9315 if (block.wantSafety()) {9299 if (block.wantSafety()) {
9316 try sema.preparePanicId(src, .integer_out_of_bounds);9300 try sema.preparePanicId(src, .integer_out_of_bounds);
9317 return block.addTyOp(.intcast_safe, dest_ty, operand);9301 return block.addTyOp(.int_cast_safe, dest_ty, operand);
9318 }9302 }
9319 return block.addTyOp(.intcast, dest_ty, operand);9303 return block.addTyOp(.int_cast, dest_ty, operand);
9320}9304}
93219305
9322fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9306fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
9330 const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");9314 const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");
9331 const operand = sema.resolveInst(extra.rhs);9315 const operand = sema.resolveInst(extra.rhs);
9332 const operand_ty = sema.typeOf(operand);9316 const operand_ty = sema.typeOf(operand);
9333 switch (dest_ty.zigTypeTag(zcu)) {
9334 .@"anyframe",
9335 .comptime_float,
9336 .comptime_int,
9337 .enum_literal,
9338 .error_set,
9339 .error_union,
9340 .@"fn",
9341 .frame,
9342 .noreturn,
9343 .null,
9344 .@"opaque",
9345 .spirv,
9346 .optional,
9347 .type,
9348 .undefined,
9349 .void,
9350 => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9351
9352 .@"enum" => {
9353 const msg = msg: {
9354 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9355 errdefer msg.destroy(sema.gpa);
9356 switch (operand_ty.zigTypeTag(zcu)) {
9357 .int, .comptime_int => try sema.errNote(src, msg, "use @enumFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9358 else => {},
9359 }
9360
9361 break :msg msg;
9362 };
9363 return sema.failWithOwnedErrorMsg(block, msg);
9364 },
93659317
9366 .pointer => {9318 // Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
9367 const msg = msg: {9319 switch (dest_ty.scalarType(zcu).zigTypeTag(zcu)) {
9368 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});9320 .pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
9369 errdefer msg.destroy(sema.gpa);9321 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9370 switch (operand_ty.zigTypeTag(zcu)) {9322 errdefer msg.destroy(sema.gpa);
9371 .int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),9323 switch (operand_ty.zigTypeTag(zcu)) {
9372 .pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),9324 .int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9373 else => {},9325 .pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
9374 }9326 else => {},
9375
9376 break :msg msg;
9377 };
9378 return sema.failWithOwnedErrorMsg(block, msg);
9379 },
9380 .@"struct", .@"union" => if (dest_ty.containerLayout(zcu) == .auto) {
9381 const container = switch (dest_ty.zigTypeTag(zcu)) {
9382 .@"struct" => "struct",
9383 .@"union" => "union",
9384 else => unreachable,
9385 };
9386 return sema.fail(block, src, "cannot @bitCast to '{f}'; {s} does not have a guaranteed in-memory layout", .{
9387 dest_ty.fmt(pt), container,
9388 });
9389 },
9390 .array => {
9391 const elem_ty = dest_ty.childType(zcu);
9392 if (!elem_ty.hasWellDefinedLayout(zcu)) {
9393 const msg = msg: {
9394 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9395 errdefer msg.destroy(sema.gpa);
9396 try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9397 break :msg msg;
9398 };
9399 return sema.failWithOwnedErrorMsg(block, msg);
9400 }9327 }
9401 },
94029328
9403 .bool,9329 break :msg msg;
9404 .float,9330 }),
9405 .int,9331 .array => switch (dest_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
9406 .vector,9332 .pointer, .optional => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9407 => {},9333 else => {},
9408 }
9409 switch (operand_ty.zigTypeTag(zcu)) {
9410 .@"anyframe",
9411 .comptime_float,
9412 .comptime_int,
9413 .enum_literal,
9414 .error_set,
9415 .error_union,
9416 .@"fn",
9417 .frame,
9418 .noreturn,
9419 .null,
9420 .@"opaque",
9421 .spirv,
9422 .optional,
9423 .type,
9424 .undefined,
9425 .void,
9426 => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)}),
9427
9428 .@"enum" => {
9429 const msg = msg: {
9430 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9431 errdefer msg.destroy(sema.gpa);
9432 switch (dest_ty.zigTypeTag(zcu)) {
9433 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromEnum to cast to '{f}'", .{dest_ty.fmt(pt)}),
9434 else => {},
9435 }
9436
9437 break :msg msg;
9438 };
9439 return sema.failWithOwnedErrorMsg(block, msg);
9440 },9334 },
9441 .pointer => {9335 else => {},
9442 const msg = msg: {9336 }
9443 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});9337 if (!dest_ty.hasBitRepresentation(zcu)) {
9444 errdefer msg.destroy(sema.gpa);9338 return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9445 switch (dest_ty.zigTypeTag(zcu)) {9339 }
9446 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9447 .pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9448 else => {},
9449 }
94509340
9451 break :msg msg;9341 // Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
9452 };9342 switch (operand_ty.scalarType(zcu).zigTypeTag(zcu)) {
9453 return sema.failWithOwnedErrorMsg(block, msg);9343 .pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
9454 },9344 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9455 .@"struct", .@"union" => if (operand_ty.containerLayout(zcu) == .auto) {9345 errdefer msg.destroy(sema.gpa);
9456 const container = switch (operand_ty.zigTypeTag(zcu)) {9346 switch (dest_ty.zigTypeTag(zcu)) {
9457 .@"struct" => "struct",9347 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9458 .@"union" => "union",9348 .pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9459 else => unreachable,9349 else => {},
9460 };
9461 return sema.fail(block, operand_src, "cannot @bitCast from '{f}'; {s} does not have a guaranteed in-memory layout", .{
9462 operand_ty.fmt(pt), container,
9463 });
9464 },
9465 .array => {
9466 const elem_ty = operand_ty.childType(zcu);
9467 if (!elem_ty.hasWellDefinedLayout(zcu)) {
9468 const msg = msg: {
9469 const msg = try sema.errMsg(src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9470 errdefer msg.destroy(sema.gpa);
9471 try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9472 break :msg msg;
9473 };
9474 return sema.failWithOwnedErrorMsg(block, msg);
9475 }9350 }
9351 break :msg msg;
9352 }),
9353 .array => switch (operand_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
9354 .pointer, .optional => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{dest_ty.fmt(pt)}),
9355 else => {},
9476 },9356 },
94779357 else => {},
9478 .bool,9358 }
9479 .float,9359 if (!operand_ty.hasBitRepresentation(zcu)) {
9480 .int,9360 return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9481 .vector,
9482 => {},
9483 }9361 }
9484 return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node), operand_src);9362
9363 return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node));
9485}9364}
94869365
9487fn zirFloatCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9366fn zirFloatCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(...@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(
12026 .@"inline" => unreachable, // handled above11905 .@"inline" => unreachable, // handled above
12027 .has_ranges => unreachable, // not possible for error set11906 .has_ranges => unreachable, // not possible for error set
12028 .special => {11907 .special => {
12029 if (else_err_ty) |err_ty| {11908 const capture_err_ty = else_err_ty orelse {
12030 break :payload_ref try sema.bitCast(case_block, err_ty, loaded_operand, operand_src, null);
12031 } else {
12032 try sema.analyzeUnreachable(case_block, operand_src, false);11909 try sema.analyzeUnreachable(case_block, operand_src, false);
12033 break :payload_ref .unreachable_value;11910 break :payload_ref .unreachable_value;
11911 };
11912 if (sema.resolveValue(loaded_operand)) |err_val| {
11913 break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
11914 .ty = capture_err_ty.toIntern(),
11915 .name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
11916 } }));
11917 } else {
11918 break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
12034 }11919 }
12035 },11920 },
12036 .item_refs => |item_refs| {11921 .item_refs => |item_refs| {
...@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(...@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(
12040 const item_val = sema.resolveValue(item_ref).?;11925 const item_val = sema.resolveValue(item_ref).?;
12041 names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});11926 names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});
12042 }11927 }
12043 const narrowed_ty = try pt.errorSetFromUnsortedNames(names.keys());11928 const capture_err_ty = try pt.errorSetFromUnsortedNames(names.keys());
12044 break :payload_ref try sema.bitCast(case_block, narrowed_ty, loaded_operand, operand_src, null);11929 if (sema.resolveValue(loaded_operand)) |err_val| {
11930 break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
11931 .ty = capture_err_ty.toIntern(),
11932 .name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
11933 } }));
11934 } else {
11935 break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
11936 }
12045 },11937 },
12046 }11938 }
12047 }11939 }
...@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12259 return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);12151 return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);
12260 }12152 }
1226112153
12262 // We may have to emit a switch block which coerces the operand to the capture type.
12263 // If we can, try to avoid that using in-memory coercions.
12264 const first_non_imc = in_mem: {
12265 for (field_indices, 0..) |field_idx, i| {
12266 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12267 if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12268 break :in_mem i;
12269 }
12270 }
12271 // All fields are in-memory coercible to the resolved type!
12272 // Just take the first field and bitcast the result.
12273 const uncoerced = try case_block.addStructFieldVal(loaded_operand, first_field_index, first_field_ty);
12274 return case_block.addBitCast(capture_ty, uncoerced);
12275 };
12276
12277 // By-val capture with heterogeneous types which are not all in-memory coercible to12154 // By-val capture with heterogeneous types which are not all in-memory coercible to
12278 // the resolved capture type. We finally have to fall back to the ugly method.12155 // the resolved capture type. We finally have to fall back to the ugly method.
1227912156
12280 // However, let's first track which operands are in-memory coercible. There may well
12281 // be several, and we can squash all of these cases into the same switch prong using
12282 // a simple bitcast. We'll make this the 'else' prong.
12283
12284 var in_mem_coercible: std.bit_set.Dynamic = try .initFull(sema.arena, field_indices.len);
12285 in_mem_coercible.unset(first_non_imc);
12286 {
12287 const next = first_non_imc + 1;
12288 for (field_indices[next..], next..) |field_idx, i| {
12289 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12290 if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12291 in_mem_coercible.unset(i);
12292 }
12293 }
12294 }
12295
12296 const capture_block_inst = try case_block.addInstAsIndex(.{12157 const capture_block_inst = try case_block.addInstAsIndex(.{
12297 .tag = .block,12158 .tag = .block,
12298 .data = .{12159 .data = .{
...@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12303 },12164 },
12304 });12165 });
1230512166
12306 const prong_count = field_indices.len - in_mem_coercible.count();12167 const estimated_extra = field_indices.len * 6 + (field_indices.len / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12307
12308 const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12309 var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);12168 var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);
12310 defer cases_extra.deinit(gpa);12169 defer cases_extra.deinit(gpa);
1231112170
12312 {12171 {
12313 // All branch hints are `.none`, so just add zero elems.12172 // All branch hints are `.none`, so just add zero elems.
12314 comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);12173 comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);
12315 const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;12174 const need_elems = std.math.divCeil(usize, field_indices.len + 1, 10) catch unreachable;
12316 try cases_extra.appendNTimes(gpa, 0, need_elems);12175 try cases_extra.appendNTimes(gpa, 0, need_elems);
12317 }12176 }
1231812177
12319 {12178 {
12320 // Non-bitcast cases12179 for (field_indices, item_refs, 0..) |field_index, item, item_index| {
12321 var it = in_mem_coercible.iterator(.{ .kind = .unset });
12322 while (it.next()) |idx| {
12323 var coerce_block = case_block.makeSubBlock();12180 var coerce_block = case_block.makeSubBlock();
12324 defer coerce_block.instructions.deinit(sema.gpa);12181 defer coerce_block.instructions.deinit(sema.gpa);
1232512182
...@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12328 .offset = .{ .switch_case_item = .{12185 .offset = .{ .switch_case_item = .{
12329 .switch_node_offset = switch_node_offset,12186 .switch_node_offset = switch_node_offset,
12330 .case_idx = capture_src.offset.switch_capture.case_idx,12187 .case_idx = capture_src.offset.switch_capture.case_idx,
12331 .item_idx = .{ .kind = .single, .value = @intCast(idx) },12188 .item_idx = .{ .kind = .single, .value = @intCast(item_index) },
12332 } },12189 } },
12333 };12190 };
1233412191
12335 const field_idx = field_indices[idx];12192 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_index]);
12336 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);12193 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_index, field_ty);
12337 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_idx, field_ty);
12338 const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);12194 const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);
12339 _ = try coerce_block.addBr(capture_block_inst, coerced);12195 _ = try coerce_block.addBr(capture_block_inst, coerced);
1234012196
...@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12346 .ranges_len = 0,12202 .ranges_len = 0,
12347 .body_len = @intCast(coerce_block.instructions.items.len),12203 .body_len = @intCast(coerce_block.instructions.items.len),
12348 }));12204 }));
12349 cases_extra.appendAssumeCapacity(@intFromEnum(item_refs[idx])); // item12205 cases_extra.appendAssumeCapacity(@intFromEnum(item)); // item
12350 cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body12206 cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body
12351 }12207 }
12352 }12208 }
12353 const else_body_len = len: {12209 const else_body_len = len: {
12354 // 'else' prong uses a bitcast12210 // 'else' prong is unreachable
12355 var coerce_block = case_block.makeSubBlock();12211 const result_index: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
12356 defer coerce_block.instructions.deinit(sema.gpa);12212 try sema.air_instructions.append(gpa, .{ .tag = .unreach, .data = .{ .no_op = {} } });
1235712213 try cases_extra.append(gpa, @intFromEnum(result_index));
12358 const first_imc_item_idx = in_mem_coercible.findFirstSet().?;12214 break :len 1;
12359 const first_imc_field_idx = field_indices[first_imc_item_idx];
12360 const first_imc_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[first_imc_field_idx]);
12361 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, first_imc_field_idx, first_imc_field_ty);
12362 const coerced = try coerce_block.addBitCast(capture_ty, uncoerced);
12363 _ = try coerce_block.addBr(capture_block_inst, coerced);
12364
12365 try cases_extra.appendSlice(gpa, @ptrCast(coerce_block.instructions.items));
12366 break :len coerce_block.instructions.items.len;
12367 };12215 };
1236812216
12369 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +12217 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +
...@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12378 .pl_op = .{12226 .pl_op = .{
12379 .operand = undefined, // set by switch below12227 .operand = undefined, // set by switch below
12380 .payload = sema.addExtraAssumeCapacity(Air.SwitchBr{12228 .payload = sema.addExtraAssumeCapacity(Air.SwitchBr{
12381 .cases_len = @intCast(prong_count),12229 .cases_len = @intCast(field_indices.len),
12382 .else_body_len = @intCast(else_body_len),12230 .else_body_len = @intCast(else_body_len),
12383 }),12231 }),
12384 },12232 },
...@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(...@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(
12488 // being switched on if their prong body is `=> comptime unreachable,`.12336 // being switched on if their prong body is `=> comptime unreachable,`.
12489 switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {12337 switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {
12490 .ok => if (sema.resolveValue(uncoerced)) |uncoerced_val| {12338 .ok => if (sema.resolveValue(uncoerced)) |uncoerced_val| {
12491 break :item_ref try sema.coerceInMemory(uncoerced_val, item_ty);12339 break :item_ref .fromValue(try pt.getCoerced(uncoerced_val, item_ty));
12492 },12340 },
12493 .missing_error => if (prong_is_comptime_unreach) {12341 .missing_error => if (prong_is_comptime_unreach) {
12494 break :item_ref uncoerced;12342 break :item_ref uncoerced;
...@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13394 defer trash_block.instructions.deinit(sema.gpa);13242 defer trash_block.instructions.deinit(sema.gpa);
1339513243
13396 const instructions = [_]Air.Inst.Ref{13244 const instructions = [_]Air.Inst.Ref{
13397 try trash_block.addBitCast(lhs_info.elem_type, .void_value),13245 try trash_block.addTyOp(.bit_cast, lhs_info.elem_type, .void_value),
13398 try trash_block.addBitCast(rhs_info.elem_type, .void_value),13246 try trash_block.addTyOp(.bit_cast, rhs_info.elem_type, .void_value),
13399 };13247 };
13400 break :t try sema.resolvePeerTypes(block, src, &instructions, .{13248 break :t try sema.resolvePeerTypes(block, src, &instructions, .{
13401 .override = &[_]?LazySrcLoc{ lhs_src, rhs_src },13249 .override = &[_]?LazySrcLoc{ lhs_src, rhs_src },
...@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13552 });13400 });
1355313401
13554 const many_ty = slice_ty.slicePtrFieldType(zcu);13402 const many_ty = slice_ty.slicePtrFieldType(zcu);
13555 const many_alloc = try block.addBitCast(many_ty, mutable_alloc);13403 const many_alloc = try block.addTyOp(.ptr_cast, many_ty, mutable_alloc);
1355613404
13557 // lhs_dest_slice = dest[0..lhs.len]13405 // lhs_dest_slice = dest[0..lhs.len]
13558 if (lhs_len > 0) {13406 if (lhs_len > 0) {
...@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13601 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);13449 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13602 }13450 }
1360313451
13604 return block.addBitCast(constant_alloc_ty, mutable_alloc);13452 return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13605 }13453 }
1360613454
13607 var elem_i: u32 = 0;13455 var elem_i: u32 = 0;
...@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13634 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);13482 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13635 }13483 }
1363613484
13637 return block.addBitCast(constant_alloc_ty, mutable_alloc);13485 return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13638 }13486 }
1363913487
13640 const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);13488 const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);
...@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(...@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(
14917 try sema.requireRuntimeBlock(block, src, runtime_src);14765 try sema.requireRuntimeBlock(block, src, runtime_src);
14918 try sema.checkLogicalPtrOperation(block, src, lhs_ty);14766 try sema.checkLogicalPtrOperation(block, src, lhs_ty);
14919 try sema.checkLogicalPtrOperation(block, src, rhs_ty);14767 try sema.checkLogicalPtrOperation(block, src, rhs_ty);
14920 const lhs_int = try block.addBitCast(.usize, lhs);14768 const lhs_int = try block.addTyOp(.int_from_ptr, .usize, lhs);
14921 const rhs_int = try block.addBitCast(.usize, rhs);14769 const rhs_int = try block.addTyOp(.int_from_ptr, .usize, rhs);
14922 const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);14770 const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);
14923 return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));14771 return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));
14924 }14772 }
...@@ -15186,15 +15034,76 @@ fn zirAsm(...@@ -15186,15 +15034,76 @@ fn zirAsm(
15186 break :out_ty sema.typeOf(inst).childType(zcu);15034 break :out_ty sema.typeOf(inst).childType(zcu);
15187 }15035 }
15188 };15036 };
15189 if (!out_ty.hasWellDefinedLayout(zcu)) {15037 switch (out_ty.zigTypeTag(zcu)) {
15190 return sema.failWithOwnedErrorMsg(block, msg: {15038 .int, .float, .bool, .vector => {},
15191 const msg = try sema.errMsg(output_src, "invalid inline assembly output type; '{f}' does not have a guaranteed in-memory layout", .{15039
15192 out_ty.fmt(pt),15040 .pointer => if (out_ty.isSlice(zcu)) return sema.failWithOwnedErrorMsg(block, msg: {
15193 });15041 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15194 errdefer msg.destroy(gpa);15042 errdefer msg.destroy(gpa);
15195 try sema.addDeclaredHereNote(msg, out_ty);15043 try sema.errNote(output_src, msg, "consider separate outputs for 'ptr' and 'len'", .{});
15196 break :msg msg;15044 break :msg msg;
15197 });15045 }),
15046
15047 .optional => if (!out_ty.isPtrLikeOptional(zcu)) {
15048 return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15049 },
15050
15051 .@"enum" => switch (ip.loadEnumType(out_ty.toIntern()).int_tag_mode) {
15052 .explicit => {},
15053 .auto => return sema.failWithOwnedErrorMsg(block, msg: {
15054 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15055 errdefer msg.destroy(gpa);
15056 try sema.errNote(out_ty.srcLoc(zcu), msg, "integer tag type of enum is inferred", .{});
15057 try sema.errNote(out_ty.srcLoc(zcu), msg, "consider explicitly specifying the integer tag type", .{});
15058 break :msg msg;
15059 }),
15060 },
15061
15062 .@"struct" => switch (out_ty.containerLayout(zcu)) {
15063 .@"packed" => {},
15064 .auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
15065 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15066 errdefer msg.destroy(gpa);
15067 try sema.errNote(output_src, msg, "struct types cannot be passed to inline assembly", .{});
15068 try sema.addDeclaredHereNote(msg, out_ty);
15069 break :msg msg;
15070 }),
15071 },
15072
15073 .@"union" => switch (out_ty.containerLayout(zcu)) {
15074 .@"packed" => {},
15075 .auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
15076 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15077 errdefer msg.destroy(gpa);
15078 try sema.errNote(output_src, msg, "union types cannot be passed to inline assembly", .{});
15079 try sema.addDeclaredHereNote(msg, out_ty);
15080 break :msg msg;
15081 }),
15082 },
15083
15084 .array => return sema.failWithOwnedErrorMsg(block, msg: {
15085 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15086 errdefer msg.destroy(gpa);
15087 try sema.errNote(output_src, msg, "array types cannot be passed to inline assembly", .{});
15088 break :msg msg;
15089 }),
15090
15091 .void,
15092 .type,
15093 .noreturn,
15094 .comptime_float,
15095 .comptime_int,
15096 .undefined,
15097 .null,
15098 .error_union,
15099 .error_set,
15100 .@"fn",
15101 .@"opaque",
15102 .frame,
15103 .@"anyframe",
15104 .enum_literal,
15105 .spirv,
15106 => return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)}),
15198 }15107 }
1519915108
15200 const constraint = sema.code.nullTerminatedString(output.data.constraint);15109 const constraint = sema.code.nullTerminatedString(output.data.constraint);
...@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15598 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;15507 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
15599 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);15508 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
15600 const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);15509 const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);
15601 switch (operand_ty.zigTypeTag(zcu)) {15510 if (!operand_ty.hasBitRepresentation(zcu) and
15602 .@"fn",15511 // TODO: allow these types too for now because this is used in some places. We need to
15603 .noreturn,15512 // figure out whether we think errors and auto-enums have bit representations!
15604 .undefined,15513 operand_ty.zigTypeTag(zcu) != .error_set and
15605 .null,15514 operand_ty.zigTypeTag(zcu) != .@"enum")
15606 .@"opaque",15515 {
15607 .spirv,15516 return sema.fail(block, operand_src, "no bit size available for type '{f}'", .{operand_ty.fmt(pt)});
15608 .type,
15609 .enum_literal,
15610 .comptime_float,
15611 .comptime_int,
15612 => return sema.fail(block, operand_src, "no size available for type '{f}'", .{operand_ty.fmt(pt)}),
15613
15614 .void,
15615 => return .zero,
15616
15617 .bool,
15618 .int,
15619 .float,
15620 .pointer,
15621 .array,
15622 .@"struct",
15623 .optional,
15624 .error_union,
15625 .error_set,
15626 .@"enum",
15627 .@"union",
15628 .vector,
15629 .frame,
15630 .@"anyframe",
15631 => {},
15632 }15517 }
15633 try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);15518 try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);
15634 return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));15519 return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));
...@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
18179 } else 0;18064 } else 0;
1818018065
18181 if (host_size != 0) {18066 if (host_size != 0) {
18067 try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18068 if (elem_ty.unpackable(zcu)) |reason| return sema.failWithOwnedErrorMsg(block, msg: {
18069 const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18070 errdefer msg.destroy(sema.gpa);
18071 try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18072 break :msg msg;
18073 });
18074 const elem_bit_size = elem_ty.bitSize(zcu);
18182 if (bit_offset >= host_size * 8) {18075 if (bit_offset >= host_size * 8) {
18183 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{18076 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{
18184 elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,18077 elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,
18185 });18078 });
18186 }18079 }
18187 try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18188 const elem_bit_size = elem_ty.bitSize(zcu);
18189 if (elem_bit_size > host_size * 8 - bit_offset) {18080 if (elem_bit_size > host_size * 8 - bit_offset) {
18190 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{18081 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{
18191 elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,18082 elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,
...@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
18201 return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});18092 return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});
18202 }18093 }
1820318094
18204 if (host_size != 0) {
18205 if (elem_ty.unpackable(zcu)) |reason| return sema.failWithOwnedErrorMsg(block, msg: {
18206 const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18207 errdefer msg.destroy(sema.gpa);
18208 try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18209 break :msg msg;
18210 });
18211 }
18212
18213 const ty = try pt.ptrType(.{18095 const ty = try pt.ptrType(.{
18214 .child = elem_ty.toIntern(),18096 .child = elem_ty.toIntern(),
18215 .sentinel = sentinel,18097 .sentinel = sentinel,
...@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
18379 const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);18261 const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);
1838018262
18381 if (union_ty.containerLayout(zcu) == .@"packed") {18263 if (union_ty.containerLayout(zcu) == .@"packed") {
18382 return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node), payload_src);18264 return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node));
18383 }18265 }
1838418266
18385 if (sema.resolveValue(payload)) |payload_val| {18267 if (sema.resolveValue(payload)) |payload_val| {
...@@ -18516,7 +18398,7 @@ fn zirStructInit(...@@ -18516,7 +18398,7 @@ fn zirStructInit(
18516 const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);18398 const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);
1851718399
18518 if (resolved_ty.containerLayout(zcu) == .@"packed") {18400 if (resolved_ty.containerLayout(zcu) == .@"packed") {
18519 const union_val = try sema.bitCast(block, resolved_ty, init_inst, src, field_src);18401 const union_val = try sema.bitCast(block, resolved_ty, init_inst, src);
18520 const result_val = try sema.coerce(block, result_ty, union_val, src);18402 const result_val = try sema.coerce(block, result_ty, union_val, src);
18521 if (is_ref) {18403 if (is_ref) {
18522 return sema.analyzeRef(block, src, result_val, .none);18404 return sema.analyzeRef(block, src, result_val, .none);
...@@ -18680,20 +18562,15 @@ fn finishStructInit(...@@ -18680,20 +18562,15 @@ fn finishStructInit(
18680 },18562 },
18681 .@"packed" => {18563 .@"packed" => {
18682 const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));18564 const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));
18565 @memset(buf, 0);
18683 var bit_offset: u16 = 0;18566 var bit_offset: u16 = 0;
18684 for (field_inits) |field_init| {18567 for (field_inits) |field_init| {
18685 const field_val = sema.resolveValue(field_init).?;18568 const field_val = sema.resolveValue(field_init).?;
18686 field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) {18569 field_val.writeToPackedMemory(zcu, buf, bit_offset);
18687 error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
18688 error.OutOfMemory => |e| return e,
18689 };
18690 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));18570 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
18691 }18571 }
18692 assert(bit_offset == struct_ty.bitSize(zcu));18572 assert(bit_offset == struct_ty.bitSize(zcu));
18693 const struct_val = Value.readFromPackedMemory(struct_ty, pt, buf, 0, sema.arena) catch |err| switch (err) {18573 const struct_val: Value = try .readFromPackedMemory(struct_ty, pt, buf, 0);
18694 error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
18695 error.OutOfMemory => |e| return e,
18696 };
18697 const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);18574 const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);
18698 return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);18575 return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);
18699 },18576 },
...@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
19387 }19264 }
19388 return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());19265 return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());
19389 }19266 }
19390 return block.addBitCast(dest_ty, operand);19267 return block.addTyOp(.bit_cast, dest_ty, operand);
19391}19268}
1939219269
19393fn zirErrorName(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {19270fn zirErrorName(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!...@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
21245 try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);21122 try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);
21246 }21123 }
21247 }21124 }
21248 return block.addBitCast(dest_ty, operand_coerced);21125 return block.addTyOp(.ptr_from_int, dest_ty, operand_coerced);
21249}21126}
2125021127
21251fn ptrFromIntVal(21128fn ptrFromIntVal(
...@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData...@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData
21444 .error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),21321 .error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),
21445 else => unreachable,21322 else => unreachable,
21446 };21323 };
21447 const err_int_inst = try block.addBitCast(err_int_ty, err_code_inst);21324 const err_int_inst = try block.addTyOp(.int_from_error, err_int_ty, err_code_inst);
21448 if (dest_tag == .error_union) {21325 if (dest_tag == .error_union) {
21449 const zero_err = try pt.intRef(err_int_ty, 0);21326 const zero_err = try pt.intRef(err_int_ty, 0);
21450 const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);21327 const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);
...@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData...@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData
21464 }21341 }
2146521342
21466 if (operand_tag == .error_set and dest_tag == .error_union) {21343 if (operand_tag == .error_set and dest_tag == .error_union) {
21467 const err_val = try block.addBitCast(dest_err_ty, operand);21344 const err_val = try block.addTyOp(.error_cast, dest_err_ty, operand);
21468 return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);21345 return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);
21469 } else {21346 } else {
21470 return block.addBitCast(dest_ty, operand);21347 return block.addTyOp(.error_cast, dest_ty, operand);
21471 }21348 }
21472}21349}
2147321350
...@@ -22015,7 +21892,7 @@ fn ptrCastFull(...@@ -22015,7 +21892,7 @@ fn ptrCastFull(
22015 // `operand_ptr` converted to an integer, for safety checks.21892 // `operand_ptr` converted to an integer, for safety checks.
22016 const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {21893 const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {
22017 assert(need_operand_ptr);21894 assert(need_operand_ptr);
22018 break :i try block.addBitCast(.usize, operand_ptr);21895 break :i try block.addTyOp(.int_from_ptr, .usize, operand_ptr);
22019 } else .none;21896 } else .none;
2202021897
22021 if (need_null_check) {21898 if (need_null_check) {
...@@ -22042,8 +21919,8 @@ fn ptrCastFull(...@@ -22042,8 +21919,8 @@ fn ptrCastFull(
2204221919
22043 if (dest_info.flags.size == .slice) {21920 if (dest_info.flags.size == .slice) {
22044 if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {21921 if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {
22045 // Fast path: just bitcast!21922 // Fast path: just pointer cast!
22046 return block.addBitCast(dest_ty, operand);21923 return block.addTyOp(.ptr_cast, dest_ty, operand);
22047 }21924 }
2204821925
22049 // We need to deconstruct the slice (if applicable) and reconstruct it.21926 // We need to deconstruct the slice (if applicable) and reconstruct it.
...@@ -22101,7 +21978,7 @@ fn ptrCastFull(...@@ -22101,7 +21978,7 @@ fn ptrCastFull(
22101 else => unreachable,21978 else => unreachable,
22102 };21979 };
22103 const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {21980 const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {
22104 break :ptr try block.addBitCast(want_ptr_ty, operand_ptr);21981 break :ptr try block.addTyOp(.ptr_cast, want_ptr_ty, operand_ptr);
22105 } else operand_ptr;21982 } else operand_ptr;
2210621983
22107 return block.addInst(.{21984 return block.addInst(.{
...@@ -22116,12 +21993,11 @@ fn ptrCastFull(...@@ -22116,12 +21993,11 @@ fn ptrCastFull(
22116 });21993 });
22117 } else {21994 } else {
22118 assert(need_operand_ptr);21995 assert(need_operand_ptr);
22119 // We just need to bitcast the pointer, if necessary.21996 // We just need a ptr_cast, if even that (we might only have needed the `addrspace_cast`).
22120 // It might not be necessary, since we might have just needed the `addrspace_cast`.
22121 const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())21997 const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())
22122 operand_ptr21998 operand_ptr
22123 else21999 else
22124 try block.addBitCast(dest_ty, operand_ptr);22000 try block.addTyOp(.ptr_cast, dest_ty, operand_ptr);
2212522001
22126 try sema.checkKnownAllocPtr(block, operand, result);22002 try sema.checkKnownAllocPtr(block, operand, result);
22127 return result;22003 return result;
...@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst...@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst
22157 }22033 }
2215822034
22159 try sema.requireRuntimeBlock(block, src, null);22035 try sema.requireRuntimeBlock(block, src, null);
22160 const new_ptr = try block.addBitCast(dest_ty, operand);22036 const new_ptr = try block.addTyOp(.ptr_cast, dest_ty, operand);
22161 try sema.checkKnownAllocPtr(block, operand, new_ptr);22037 try sema.checkKnownAllocPtr(block, operand, new_ptr);
22162 return new_ptr;22038 return new_ptr;
22163}22039}
...@@ -24259,7 +24135,7 @@ fn analyzeMinMax(...@@ -24259,7 +24135,7 @@ fn analyzeMinMax(
24259 // where we have refined the range, so we should be doing an intcast.24135 // where we have refined the range, so we should be doing an intcast.
24260 assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);24136 assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);
24261 assert(result_scalar_ty.zigTypeTag(zcu) == .int);24137 assert(result_scalar_ty.zigTypeTag(zcu) == .int);
24262 return block.addTyOp(.intcast, result_ty, cur_result);24138 return block.addTyOp(.int_cast, result_ty, cur_result);
24263}24139}
2426424140
24265fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {24141fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {
...@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !A...@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !A
24289 try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)24165 try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)
24290 else24166 else
24291 ptr;24167 ptr;
24292 return block.addBitCast(new_ty, non_slice_ptr);24168 return block.addTyOp(.ptr_cast, new_ty, non_slice_ptr);
24293}24169}
2429424170
24295fn zirMemcpy(24171fn zirMemcpy(
...@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(...@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(
25087 const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);24963 const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);
25088 return Air.internedToRef(casted_ptr_val.toIntern());24964 return Air.internedToRef(casted_ptr_val.toIntern());
25089 } else {24965 } else {
25090 return block.addBitCast(result_ptr_ty, uncasted_ptr);24966 return block.addTyOp(.ptr_cast, result_ptr_ty, uncasted_ptr);
25091 }24967 }
25092}24968}
2509324969
...@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(...@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(
25533 .param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),25409 .param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),
25534 else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),25410 else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
25535 },25411 },
25536 .vector => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),25412 .vector => try sema.errNote(src_loc, msg, "vectors have no guaranteed in-memory representation", .{}),
25537 .optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),25413 .optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),
25538 }25414 }
25539}25415}
...@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(...@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(
25801 .address_space = ptr_info.flags.address_space,25677 .address_space = ptr_info.flags.address_space,
25802 },25678 },
25803 });25679 });
25804 const many_ptr = try parent_block.addBitCast(many_ptr_ty, ptr);25680 const many_ptr = try parent_block.addTyOp(.ptr_cast, many_ptr_ty, ptr);
25805 break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);25681 break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);
25806 },25682 },
25807 .many => unreachable,25683 .many => unreachable,
...@@ -26278,7 +26154,7 @@ fn fieldPtr(...@@ -26278,7 +26154,7 @@ fn fieldPtr(
26278 },26154 },
26279 .packed_offset = ptr_ptr_info.packed_offset,26155 .packed_offset = ptr_ptr_info.packed_offset,
26280 });26156 });
26281 return sema.bitCast(block, result_ty, object_ptr, src, null);26157 return block.addTyOp(.ptr_cast, result_ty, object_ptr);
26282 } else {26158 } else {
26283 return sema.fail(26159 return sema.fail(
26284 block,26160 block,
...@@ -26996,16 +26872,13 @@ fn unionFieldVal(...@@ -26996,16 +26872,13 @@ fn unionFieldVal(
26996 break :msg msg;26872 break :msg msg;
26997 });26873 });
26998 },26874 },
26999 .@"extern" => if (try sema.bitCastVal(union_val, field_ty, 0, 0, 0)) |field_val| {26875 .@"extern" => if (try sema.castMemory(union_val, field_ty, 0)) |field_val| {
27000 return .fromValue(field_val);26876 return .fromValue(field_val);
27001 } else {26877 } else {
27002 // Runtime-known due to a pointer-to-integer conversion.26878 // Runtime-known due to a pointer-to-integer conversion.
27003 },26879 },
27004 .@"packed" => {26880 .@"packed" => {
27005 const field_val = try sema.bitCastVal(union_val, field_ty, 0, union_ty.bitSize(zcu), 0) orelse {26881 return .fromValue(try sema.bitCastVal(union_val, field_ty));
27006 unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
27007 };
27008 return .fromValue(field_val);
27009 },26882 },
27010 }26883 }
27011 }26884 }
...@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(...@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(
2710426977
27105 if (child_ty.abiSize(zcu) == 0) {26978 if (child_ty.abiSize(zcu) == 0) {
27106 // zero-bit child type; just bitcast the pointer26979 // zero-bit child type; just bitcast the pointer
27107 return block.addBitCast(result_ty, indexable);26980 return block.addTyOp(.ptr_cast, result_ty, indexable);
27108 }26981 }
2710926982
27110 return block.addPtrElemPtr(indexable, elem_index, result_ty);26983 return block.addPtrElemPtr(indexable, elem_index, result_ty);
...@@ -27401,68 +27274,36 @@ fn elemPtrVector(...@@ -27401,68 +27274,36 @@ fn elemPtrVector(
27401 }27274 }
2740227275
27403 const elem_ty = vector_ty.childType(zcu);27276 const elem_ty = vector_ty.childType(zcu);
27404 const elem_bits = elem_ty.bitSize(zcu);27277
27405 // Exiting this block means the operation is a runtime one.27278 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27406 const elem_ptr_ty: Type = if (elem_bits < 8 or !std.math.isPowerOfTwo(elem_bits)) elem_ptr_ty: {27279 const elem_ptr_ty = try pt.ptrType(.{
27407 // Use a packed pointer (i.e. vector_index != 0)27280 .child = elem_ty.toIntern(),
27408 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);27281 .flags = .{
27409 const elem_ptr_ty = try pt.ptrType(.{27282 .size = .one,
27410 .child = elem_ty.toIntern(),27283 .alignment = vector_ptr_info.flags.alignment,
27411 .flags = .{27284 .is_const = vector_ptr_info.flags.is_const,
27412 .size = .one,27285 .is_volatile = vector_ptr_info.flags.is_volatile,
27413 .alignment = vector_ptr_info.flags.alignment,27286 .is_allowzero = vector_ptr_info.flags.is_allowzero,
27414 .is_const = vector_ptr_info.flags.is_const,27287 .address_space = vector_ptr_info.flags.address_space,
27415 .is_volatile = vector_ptr_info.flags.is_volatile,27288 .vector_index = @enumFromInt(index),
27416 .is_allowzero = vector_ptr_info.flags.is_allowzero,27289 },
27417 .address_space = vector_ptr_info.flags.address_space,27290 .packed_offset = .{
27418 .vector_index = @enumFromInt(index),27291 .host_size = @intCast(vector_len),
27419 },27292 .bit_offset = 0,
27420 .packed_offset = .{27293 },
27421 .host_size = @intCast(vector_len),27294 });
27422 .bit_offset = 0,27295
27423 },27296 if (maybe_vector_ptr_val) |ptr_val| {
27424 });27297 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27425 if (maybe_vector_ptr_val) |ptr_val| {27298 return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27426 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);27299 }
27427 return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27428 }
27429 break :elem_ptr_ty elem_ptr_ty;
27430 } else elem_ptr_ty: {
27431 // Use a normal pointer (i.e. vector_index == 0)
27432 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27433 const elem_ptr_ty = try pt.ptrType(.{
27434 .child = elem_ty.toIntern(),
27435 .flags = .{
27436 .size = .one,
27437 // TODO: this logic was ported from old code, but it's bogus. This entire block will
27438 // go away when https://github.com/ziglang/zig/issues/24061 is implemented anyway.
27439 .alignment = switch (vector_ptr_info.flags.alignment) {
27440 .none => .none,
27441 else => |vec_align| switch (index * elem_ty.abiSize(zcu)) {
27442 0 => vec_align,
27443 else => |byte_offset| .minStrict(vec_align, .fromLog2Units(@ctz(byte_offset))),
27444 },
27445 },
27446 .is_const = vector_ptr_info.flags.is_const,
27447 .is_volatile = vector_ptr_info.flags.is_volatile,
27448 .is_allowzero = vector_ptr_info.flags.is_allowzero,
27449 .address_space = vector_ptr_info.flags.address_space,
27450 },
27451 });
27452 if (maybe_vector_ptr_val) |ptr_val| {
27453 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27454 const bit_offset = index * @divExact(elem_ty.bitSize(zcu), 8);
27455 return .fromValue(try ptr_val.getOffsetPtr(bit_offset, elem_ptr_ty, pt));
27456 }
27457 break :elem_ptr_ty elem_ptr_ty;
27458 };
2745927300
27460 if (!init) {27301 if (!init) {
27461 try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);27302 try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);
27462 try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);27303 try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);
27463 }27304 }
2746427305
27465 return block.addPtrElemPtr(vector_ptr, elem_index, elem_ptr_ty);27306 return block.addTyOp(.ptr_cast, elem_ptr_ty, vector_ptr);
27466}27307}
2746727308
27468fn elemPtrSpirvRuntimeArray(27309fn elemPtrSpirvRuntimeArray(
...@@ -27544,7 +27385,7 @@ fn elemPtrArray(...@@ -27544,7 +27385,7 @@ fn elemPtrArray(
2754427385
27545 if (array_ty.childType(zcu).abiSize(zcu) == 0) {27386 if (array_ty.childType(zcu).abiSize(zcu) == 0) {
27546 // zero-bit child type; just bitcast the pointer27387 // zero-bit child type; just bitcast the pointer
27547 return block.addBitCast(elem_ptr_ty, array_ptr);27388 return block.addTyOp(.ptr_cast, elem_ptr_ty, array_ptr);
27548 }27389 }
2754927390
27550 return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);27391 return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);
...@@ -27670,7 +27511,7 @@ fn elemPtrSlice(...@@ -27670,7 +27511,7 @@ fn elemPtrSlice(
27670 if (elem_ty.abiSize(zcu) == 0) {27511 if (elem_ty.abiSize(zcu) == 0) {
27671 // zero-bit child type; just extract the pointer and bitcast it27512 // zero-bit child type; just extract the pointer and bitcast it
27672 const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);27513 const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);
27673 return block.addBitCast(elem_ptr_ty, slice_ptr);27514 return block.addTyOp(.ptr_cast, elem_ptr_ty, slice_ptr);
27674 }27515 }
27675 return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);27516 return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);
27676}27517}
...@@ -27753,12 +27594,31 @@ fn coerceExtra(...@@ -27753,12 +27594,31 @@ fn coerceExtra(
27753 var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);27594 var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);
27754 if (in_memory_result == .ok) {27595 if (in_memory_result == .ok) {
27755 if (maybe_inst_val) |val| {27596 if (maybe_inst_val) |val| {
27756 return sema.coerceInMemory(val, dest_ty);27597 return .fromValue(try pt.getCoerced(val, dest_ty));
27757 }27598 }
27758 try sema.requireRuntimeBlock(block, inst_src, null);27599 const coerced: Air.Inst.Ref = switch (in_memory_result.ok) {
27759 const new_val = try block.addBitCast(dest_ty, inst);27600 .none => coerced: {
27760 try sema.checkKnownAllocPtr(block, inst, new_val);27601 const @"addrspace" = target_util.defaultAddressSpace(zcu.getTarget(), .local);
27761 return new_val;27602 const src_ptr_ty = try pt.ptrType(.{
27603 .child = inst_ty.toIntern(),
27604 .flags = .{ .size = .one, .address_space = @"addrspace" },
27605 });
27606 const dest_ptr_ty = try pt.ptrType(.{
27607 .child = dest_ty.toIntern(),
27608 .flags = .{ .size = .one, .address_space = @"addrspace" },
27609 });
27610 const ptr = try block.addTy(.alloc, src_ptr_ty);
27611 _ = try block.addBinOp(.store_safe, ptr, inst);
27612 const casted_ptr = try block.addTyOp(.ptr_cast, dest_ptr_ty, ptr);
27613 break :coerced try block.addTyOp(.load, dest_ty, casted_ptr);
27614 },
27615 .same_type => unreachable, // we checked for equal types just above
27616 .bit_cast => try block.addTyOp(.bit_cast, dest_ty, inst),
27617 .ptr_cast => try block.addTyOp(.ptr_cast, dest_ty, inst),
27618 .error_cast => try block.addTyOp(.error_cast, dest_ty, inst),
27619 };
27620 try sema.checkKnownAllocPtr(block, inst, coerced);
27621 return coerced;
27762 }27622 }
2776327623
27764 switch (dest_ty.zigTypeTag(zcu)) {27624 switch (dest_ty.zigTypeTag(zcu)) {
...@@ -27872,8 +27732,8 @@ fn coerceExtra(...@@ -27872,8 +27732,8 @@ fn coerceExtra(
2787227732
27873 if (dest_info.sentinel != .none) {27733 if (dest_info.sentinel != .none) {
27874 if (array_ty.sentinel(zcu)) |inst_sent| {27734 if (array_ty.sentinel(zcu)) |inst_sent| {
27875 if (Air.internedToRef(dest_info.sentinel) !=27735 if (dest_info.sentinel !=
27876 try sema.coerceInMemory(inst_sent, dst_elem_type))27736 (try pt.getCoerced(inst_sent, dst_elem_type)).toIntern())
27877 {27737 {
27878 in_memory_result = .{ .ptr_sentinel = .{27738 in_memory_result = .{ .ptr_sentinel = .{
27879 .actual = inst_sent,27739 .actual = inst_sent,
...@@ -28067,8 +27927,8 @@ fn coerceExtra(...@@ -28067,8 +27927,8 @@ fn coerceExtra(
28067 }27927 }
2806827928
28069 if (dest_info.sentinel == .none or inst_info.sentinel == .none or27929 if (dest_info.sentinel == .none or inst_info.sentinel == .none or
28070 Air.internedToRef(dest_info.sentinel) !=27930 dest_info.sentinel !=
28071 try sema.coerceInMemory(Value.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child)))27931 (try pt.getCoerced(.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child))).toIntern())
28072 break :p;27932 break :p;
2807327933
28074 const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);27934 const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);
...@@ -28117,7 +27977,7 @@ fn coerceExtra(...@@ -28117,7 +27977,7 @@ fn coerceExtra(
28117 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))27977 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))
28118 {27978 {
28119 try sema.requireRuntimeBlock(block, inst_src, null);27979 try sema.requireRuntimeBlock(block, inst_src, null);
28120 return block.addTyOp(.intcast, dest_ty, inst);27980 return block.addTyOp(.int_cast, dest_ty, inst);
28121 }27981 }
28122 },27982 },
28123 else => {},27983 else => {},
...@@ -28379,16 +28239,8 @@ fn coerceExtra(...@@ -28379,16 +28239,8 @@ fn coerceExtra(
28379 return sema.failWithOwnedErrorMsg(block, msg);28239 return sema.failWithOwnedErrorMsg(block, msg);
28380}28240}
2838128241
28382fn coerceInMemory(
28383 sema: *Sema,
28384 val: Value,
28385 dst_ty: Type,
28386) CompileError!Air.Inst.Ref {
28387 return Air.internedToRef((try sema.pt.getCoerced(val, dst_ty)).toIntern());
28388}
28389
28390const InMemoryCoercionResult = union(enum) {28242const InMemoryCoercionResult = union(enum) {
28391 ok,28243 ok: Strategy,
28392 no_match: Pair,28244 no_match: Pair,
28393 int_not_coercible: Int,28245 int_not_coercible: Int,
28394 comptime_int_not_coercible: TypeValuePair,28246 comptime_int_not_coercible: TypeValuePair,
...@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {...@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {
28424 double_ptr_to_anyopaque: Pair,28276 double_ptr_to_anyopaque: Pair,
28425 slice_to_anyopaque: Pair,28277 slice_to_anyopaque: Pair,
2842628278
28279 const Strategy = enum {
28280 /// There isn't a special strategy for this particular coercion---we'll just need to
28281 /// reinterpret the bytes in memory.
28282 none,
28283
28284 /// The source and destination types are equal, so no explicit cast operation is necessary.
28285 same_type,
28286 /// The coercion can be lowered to `Air.Inst.Tag.bit_cast`.
28287 bit_cast,
28288 /// The coercion can be lowered to `Air.Inst.Tag.ptr_cast`.
28289 ptr_cast,
28290 /// The coercion can be lowered to `Air.Inst.Tag.error_cast`.
28291 error_cast,
28292 };
28293
28427 const Pair = struct {28294 const Pair = struct {
28428 actual: Type,28295 actual: Type,
28429 wanted: Type,28296 wanted: Type,
...@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(...@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(
28797 }28664 }
2879828665
28799 if (dest_ty.eql(src_ty))28666 if (dest_ty.eql(src_ty))
28800 return .ok;28667 return .{ .ok = .same_type };
2880128668
28802 const dest_tag = dest_ty.zigTypeTag(zcu);28669 const dest_tag = dest_ty.zigTypeTag(zcu);
28803 const src_tag = src_ty.zigTypeTag(zcu);28670 const src_tag = src_ty.zigTypeTag(zcu);
...@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(...@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(
28810 if (dest_info.signedness == src_info.signedness and28677 if (dest_info.signedness == src_info.signedness and
28811 dest_info.bits == src_info.bits)28678 dest_info.bits == src_info.bits)
28812 {28679 {
28813 return .ok;28680 return .{ .ok = .bit_cast };
28814 }28681 }
2881528682
28816 if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or28683 if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or
...@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(...@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(
28818 (dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or28685 (dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or
28819 (dest_info.signedness == .unsigned and src_info.signedness == .signed))28686 (dest_info.signedness == .unsigned and src_info.signedness == .signed))
28820 {28687 {
28821 return InMemoryCoercionResult{ .int_not_coercible = .{28688 return .{ .int_not_coercible = .{
28822 .actual_signedness = src_info.signedness,28689 .actual_signedness = src_info.signedness,
28823 .wanted_signedness = dest_info.signedness,28690 .wanted_signedness = dest_info.signedness,
28824 .actual_bits = src_info.bits,28691 .actual_bits = src_info.bits,
...@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(...@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(
28841 const dest_bits = dest_ty.floatBits(target);28708 const dest_bits = dest_ty.floatBits(target);
28842 const src_bits = src_ty.floatBits(target);28709 const src_bits = src_ty.floatBits(target);
28843 if (dest_bits == src_bits) {28710 if (dest_bits == src_bits) {
28844 return .ok;28711 return .{ .ok = .bit_cast };
28845 }28712 }
28846 }28713 }
2884728714
...@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(...@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(
28864 if (dest_tag == .error_union and src_tag == .error_union) {28731 if (dest_tag == .error_union and src_tag == .error_union) {
28865 const dest_payload = dest_ty.errorUnionPayload(zcu);28732 const dest_payload = dest_ty.errorUnionPayload(zcu);
28866 const src_payload = src_ty.errorUnionPayload(zcu);28733 const src_payload = src_ty.errorUnionPayload(zcu);
28867 const child = try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null);28734 const payload_strat = switch (try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null)) {
28868 if (child != .ok) {28735 .ok => |strat| strat,
28869 return .{ .error_union_payload = .{28736 else => |payload_result| return .{ .error_union_payload = .{
28870 .child = try child.dupe(sema.arena),28737 .child = try payload_result.dupe(sema.arena),
28871 .actual = src_payload,28738 .actual = src_payload,
28872 .wanted = dest_payload,28739 .wanted = dest_payload,
28873 } };28740 } },
28741 };
28742 switch (try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null)) {
28743 .ok => {},
28744 else => |err_set_result| return err_set_result,
28874 }28745 }
28875 return try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null);28746 return switch (payload_strat) {
28747 .same_type => .{ .ok = .error_cast },
28748 else => .{ .ok = .none },
28749 };
28876 }28750 }
2887728751
28878 // Error Sets28752 // Error Sets
28879 if (dest_tag == .error_set and src_tag == .error_set) {28753 if (dest_tag == .error_set and src_tag == .error_set) {
28880 const res1 = try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src);28754 switch (try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src)) {
28881 if (!dest_is_mut or res1 != .ok) return res1;28755 .ok => |strat| assert(strat == .error_cast),
28882 // src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.28756 else => |result| return result,
28883 const res2 = try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src);28757 }
28884 return res2;28758 if (dest_is_mut) {
28759 // src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
28760 switch (try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src)) {
28761 .ok => |strat| assert(strat == .error_cast),
28762 else => |result| return result,
28763 }
28764 }
28765 return .{ .ok = .error_cast };
28885 }28766 }
2888628767
28887 // Arrays28768 // Arrays
...@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(...@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(
28896 }28777 }
2889728778
28898 const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);28779 const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);
28899 switch (child) {28780 const child_strat = switch (child) {
28900 .ok => {},28781 .ok => |strat| strat,
28901 .no_match => return child,28782 .no_match => |no_match| return .{ .no_match = no_match },
28902 else => {28783 else => {
28903 return .{ .array_elem = .{28784 return .{ .array_elem = .{
28904 .child = try child.dupe(sema.arena),28785 .child = try child.dupe(sema.arena),
...@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(...@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(
28906 .wanted = dest_info.elem_type,28787 .wanted = dest_info.elem_type,
28907 } };28788 } };
28908 },28789 },
28909 }28790 };
28910 const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or28791 const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or
28911 (src_info.sentinel != null and28792 (src_info.sentinel != null and
28912 dest_info.sentinel != null and28793 dest_info.sentinel != null and
...@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(...@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(
28922 .ty = dest_info.elem_type,28803 .ty = dest_info.elem_type,
28923 } };28804 } };
28924 }28805 }
28925 return .ok;28806 return .{ .ok = switch (child_strat) {
28807 .bit_cast => .bit_cast,
28808 else => .none,
28809 } };
28926 }28810 }
2892728811
28928 // Vectors28812 // Vectors
...@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(...@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(
2893828822
28939 const dest_elem_ty = dest_ty.scalarType(zcu);28823 const dest_elem_ty = dest_ty.scalarType(zcu);
28940 const src_elem_ty = src_ty.scalarType(zcu);28824 const src_elem_ty = src_ty.scalarType(zcu);
28941 const child = try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null);28825 switch (try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null)) {
28942 if (child != .ok) {28826 .ok => |child_strat| return .{ .ok = switch (child_strat) {
28943 return .{ .vector_elem = .{28827 .bit_cast => .bit_cast,
28944 .child = try child.dupe(sema.arena),28828 .ptr_cast => .ptr_cast,
28829 else => .none,
28830 } },
28831 else => |child_result| return .{ .vector_elem = .{
28832 .child = try child_result.dupe(sema.arena),
28945 .actual = src_elem_ty,28833 .actual = src_elem_ty,
28946 .wanted = dest_elem_ty,28834 .wanted = dest_elem_ty,
28947 } };28835 } },
28948 }28836 }
28949
28950 return .ok;
28951 }28837 }
2895228838
28953 // Optionals28839 // Optionals
...@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(...@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(
28971 } };28857 } };
28972 }28858 }
2897328859
28974 return .ok;28860 return .{ .ok = .none };
28975 }28861 }
2897628862
28977 // Tuples (with in-memory-coercible fields)28863 // Tuples (with in-memory-coercible fields)
...@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(...@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(
28985 const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);28871 const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);
28986 if (field != .ok) break :tuple;28872 if (field != .ok) break :tuple;
28987 }28873 }
28988 return .ok;28874 return .{ .ok = .none };
28989 }28875 }
2899028876
28991 return .{ .no_match = .{28877 return .{ .no_match = .{
...@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(
29008 const ip = &zcu.intern_pool;28894 const ip = &zcu.intern_pool;
2900928895
29010 const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {28896 const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {
29011 .anyerror_type => return .ok,28897 .anyerror_type => return .{ .ok = .error_cast },
29012 .adhoc_inferred_error_set_type => {28898 .adhoc_inferred_error_set_type => {
29013 // We are trying to coerce an error set to the current function's28899 // We are trying to coerce an error set to the current function's
29014 // inferred error set.28900 // inferred error set.
29015 const dst_ies = sema.fn_ret_ty_ies.?;28901 const dst_ies = sema.fn_ret_ty_ies.?;
29016 try dst_ies.addErrorSet(src_ty, ip, sema.arena);28902 try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29017 return .ok;28903 return .{ .ok = .error_cast };
29018 },28904 },
29019 else => |err_set_ty| switch (ip.indexToKey(err_set_ty)) {28905 else => |err_set_ty| switch (ip.indexToKey(err_set_ty)) {
29020 .inferred_error_set_type => |func_index| {28906 .inferred_error_set_type => |func_index| {
...@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(
29023 // We are trying to coerce an error set to the current function's28909 // We are trying to coerce an error set to the current function's
29024 // inferred error set.28910 // inferred error set.
29025 try dst_ies.addErrorSet(src_ty, ip, sema.arena);28911 try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29026 return .ok;28912 return .{ .ok = .error_cast };
29027 }28913 }
29028 }28914 }
29029 try sema.ensureFuncIesResolved(block, dest_src, func_index);28915 try sema.ensureFuncIesResolved(block, dest_src, func_index);
...@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(
29062 ) };28948 ) };
29063 }28949 }
2906428950
29065 return .ok;28951 return .{ .ok = .error_cast };
29066}28952}
2906728953
29068fn coerceInMemoryAllowedFns(28954fn coerceInMemoryAllowedFns(
...@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(...@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(
29179 }29065 }
29180 }29066 }
2918129067
29182 return .ok;29068 return .{ .ok = .none };
29183}29069}
2918429070
29185fn callconvCoerceAllowed(29071fn callconvCoerceAllowed(
...@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(...@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(
29256 const ok_ptr_size = src_info.flags.size == dest_info.flags.size or29142 const ok_ptr_size = src_info.flags.size == dest_info.flags.size or
29257 src_info.flags.size == .c or dest_info.flags.size == .c;29143 src_info.flags.size == .c or dest_info.flags.size == .c;
29258 if (!ok_ptr_size) {29144 if (!ok_ptr_size) {
29259 return InMemoryCoercionResult{ .ptr_size = .{29145 return .{ .ptr_size = .{
29260 .actual = src_info.flags.size,29146 .actual = src_info.flags.size,
29261 .wanted = dest_info.flags.size,29147 .wanted = dest_info.flags.size,
29262 } };29148 } };
...@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(...@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(
29392 break :a dest_child.abiAlignment(zcu);29278 break :a dest_child.abiAlignment(zcu);
29393 } else dest_info.flags.alignment;29279 } else dest_info.flags.alignment;
29394 if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {29280 if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {
29395 return InMemoryCoercionResult{ .ptr_alignment = .{29281 return .{ .ptr_alignment = .{
29396 .actual = src_align,29282 .actual = src_align,
29397 .wanted = dest_align,29283 .wanted = dest_align,
29398 } };29284 } };
29399 }29285 }
29400 }29286 }
2940129287
29402 return .ok;29288 return .{ .ok = .ptr_cast };
29403}29289}
2940429290
29405fn coerceVarArgParam(29291fn coerceVarArgParam(
...@@ -29703,7 +29589,6 @@ fn bitCast(...@@ -29703,7 +29589,6 @@ fn bitCast(
29703 dest_ty: Type,29589 dest_ty: Type,
29704 inst: Air.Inst.Ref,29590 inst: Air.Inst.Ref,
29705 inst_src: LazySrcLoc,29591 inst_src: LazySrcLoc,
29706 operand_src: ?LazySrcLoc,
29707) CompileError!Air.Inst.Ref {29592) CompileError!Air.Inst.Ref {
29708 const pt = sema.pt;29593 const pt = sema.pt;
29709 const zcu = pt.zcu;29594 const zcu = pt.zcu;
...@@ -29712,6 +29597,11 @@ fn bitCast(...@@ -29712,6 +29597,11 @@ fn bitCast(
29712 old_ty.assertHasLayout(zcu);29597 old_ty.assertHasLayout(zcu);
29713 try sema.ensureLayoutResolved(dest_ty, inst_src, .init);29598 try sema.ensureLayoutResolved(dest_ty, inst_src, .init);
2971429599
29600 assert(old_ty.hasBitRepresentation(zcu));
29601 assert(dest_ty.hasBitRepresentation(zcu));
29602 assert(old_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
29603 assert(dest_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
29604
29715 const dest_bits = dest_ty.bitSize(zcu);29605 const dest_bits = dest_ty.bitSize(zcu);
29716 const old_bits = old_ty.bitSize(zcu);29606 const old_bits = old_ty.bitSize(zcu);
2971729607
...@@ -29725,20 +29615,30 @@ fn bitCast(...@@ -29725,20 +29615,30 @@ fn bitCast(
29725 }29615 }
2972629616
29727 if (sema.resolveValue(inst)) |val| {29617 if (sema.resolveValue(inst)) |val| {
29728 if (val.isUndef(zcu))29618 return .fromValue(try sema.bitCastVal(val, dest_ty));
29729 return pt.undefRef(dest_ty);
29730 if (old_ty.zigTypeTag(zcu) == .error_set and dest_ty.zigTypeTag(zcu) == .error_set) {
29731 // Special case: we sometimes call `bitCast` on error set values, but they
29732 // don't have a well-defined layout, so we can't use `bitCastVal` on them.
29733 return Air.internedToRef((try pt.getCoerced(val, dest_ty)).toIntern());
29734 }
29735 if (try sema.bitCastVal(val, dest_ty, 0, 0, 0)) |result_val| {
29736 return Air.internedToRef(result_val.toIntern());
29737 }
29738 }29619 }
29739 try sema.requireRuntimeBlock(block, inst_src, operand_src);
29740 try sema.validateRuntimeValue(block, inst_src, inst);29620 try sema.validateRuntimeValue(block, inst_src, inst);
29741 return block.addBitCast(dest_ty, inst);29621 return block.addTyOp(.bit_cast, dest_ty, inst);
29622}
29623
29624/// Supports only types which `@bitCast` supports, so pointers are *not* supported.
29625pub fn bitCastVal(
29626 sema: *Sema,
29627 val: Value,
29628 dest_ty: Type,
29629) Allocator.Error!Value {
29630 const pt = sema.pt;
29631 const zcu = pt.zcu;
29632 const bit_size = dest_ty.bitSize(zcu);
29633 assert(val.typeOf(zcu).bitSize(zcu) == bit_size);
29634 if (val.isUndef(zcu)) {
29635 return pt.undefValue(dest_ty);
29636 } else {
29637 const buf = try sema.arena.alloc(u8, @intCast((bit_size + 7) / 8));
29638 @memset(buf, 0);
29639 val.writeToPackedMemory(zcu, buf, 0);
29640 return .readFromPackedMemory(dest_ty, pt, buf, 0);
29641 }
29742}29642}
2974329643
29744fn coerceArrayPtrToSlice(29644fn coerceArrayPtrToSlice(
...@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(...@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(
29855 );29755 );
29856 }29756 }
29857 try sema.requireRuntimeBlock(block, inst_src, null);29757 try sema.requireRuntimeBlock(block, inst_src, null);
29858 const inst_allows_zero = inst_ty.zigTypeTag(zcu) != .pointer or inst_ty.ptrAllowsZero(zcu);29758 const maybe_zero: bool = switch (inst_ty.toIntern()) {
29859 if (block.wantSafety() and inst_allows_zero and !dest_ty.ptrAllowsZero(zcu)) {29759 .usize_type, .isize_type => true,
29760 else => inst_ty.ptrAllowsZero(zcu),
29761 };
29762 if (block.wantSafety() and maybe_zero and !dest_ty.ptrAllowsZero(zcu)) {
29860 try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);29763 try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);
29861 const actual_ptr = if (inst_ty.isSlice(zcu))29764 const actual_ptr = if (inst_ty.isSlice(zcu))
29862 try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)29765 try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)
29863 else29766 else
29864 inst;29767 inst;
29865 const ptr_int = try block.addBitCast(.usize, actual_ptr);29768 const ptr_int = try block.addTyOp(.int_from_ptr, .usize, actual_ptr);
29866 const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);29769 const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);
29867 const ok = if (inst_ty.isSlice(zcu)) ok: {29770 const ok = if (inst_ty.isSlice(zcu)) ok: {
29868 const len = try sema.analyzeSliceLen(block, inst_src, inst);29771 const len = try sema.analyzeSliceLen(block, inst_src, inst);
...@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(...@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(
29871 } else is_non_zero;29774 } else is_non_zero;
29872 try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);29775 try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);
29873 }29776 }
29874 const new_ptr = try sema.bitCast(block, dest_ty, inst, inst_src, null);29777 const new_ptr: Air.Inst.Ref = switch (inst_ty.toIntern()) {
29778 .usize_type => try block.addTyOp(.ptr_from_int, dest_ty, inst),
29779 .isize_type => new_ptr: {
29780 const usize_inst = try block.addTyOp(.bit_cast, .usize, inst);
29781 break :new_ptr try block.addTyOp(.ptr_from_int, dest_ty, usize_inst);
29782 },
29783 else => try block.addTyOp(.ptr_cast, dest_ty, inst),
29784 };
29875 try sema.checkKnownAllocPtr(block, inst, new_ptr);29785 try sema.checkKnownAllocPtr(block, inst, new_ptr);
29876 return new_ptr;29786 return new_ptr;
29877}29787}
...@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(...@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(
29968 return .fromValue(opv);29878 return .fromValue(opv);
29969 } else {29879 } else {
29970 // The union layout is just the tag, so we can bitcast the enum straight to the union.29880 // The union layout is just the tag, so we can bitcast the enum straight to the union.
29971 return block.addBitCast(union_ty, enum_tag);29881 return block.addTyOp(.union_from_enum, union_ty, enum_tag);
29972 }29882 }
29973 }29883 }
2997429884
...@@ -30017,18 +29927,6 @@ fn coerceArrayLike(...@@ -30017,18 +29927,6 @@ fn coerceArrayLike(
30017 const inst_ty = sema.typeOf(inst);29927 const inst_ty = sema.typeOf(inst);
30018 const target = zcu.getTarget();29928 const target = zcu.getTarget();
3001929929
30020 // try coercion of the whole array
30021 const in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, null);
30022 if (in_memory_result == .ok) {
30023 if (sema.resolveValue(inst)) |inst_val| {
30024 // These types share the same comptime value representation.
30025 return sema.coerceInMemory(inst_val, dest_ty);
30026 }
30027 try sema.requireRuntimeBlock(block, inst_src, null);
30028 return block.addBitCast(dest_ty, inst);
30029 }
30030
30031 // otherwise, try element by element
30032 const inst_len = inst_ty.arrayLen(zcu);29930 const inst_len = inst_ty.arrayLen(zcu);
30033 const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));29931 const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));
30034 if (dest_len != inst_len) {29932 if (dest_len != inst_len) {
...@@ -30055,7 +29953,7 @@ fn coerceArrayLike(...@@ -30055,7 +29953,7 @@ fn coerceArrayLike(
30055 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))29953 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))
30056 {29954 {
30057 try sema.requireRuntimeBlock(block, inst_src, null);29955 try sema.requireRuntimeBlock(block, inst_src, null);
30058 return block.addTyOp(.intcast, dest_ty, inst);29956 return block.addTyOp(.int_cast, dest_ty, inst);
30059 }29957 }
30060 },29958 },
30061 .float => if (inst_elem_ty.isRuntimeFloat()) {29959 .float => if (inst_elem_ty.isRuntimeFloat()) {
...@@ -30582,7 +30480,7 @@ fn analyzeRef(...@@ -30582,7 +30480,7 @@ fn analyzeRef(
3058230480
30583 // Cast to the constant pointer type. We do this directly rather than going via `coerce` to30481 // Cast to the constant pointer type. We do this directly rather than going via `coerce` to
30584 // avoid errors in the `block.isComptime()` case.30482 // avoid errors in the `block.isComptime()` case.
30585 return block.addBitCast(ptr_type, alloc);30483 return block.addTyOp(.ptr_cast, ptr_type, alloc);
30586}30484}
3058730485
30588fn analyzeLoad(30486fn analyzeLoad(
...@@ -31334,7 +31232,7 @@ fn analyzeSlice(...@@ -31334,7 +31232,7 @@ fn analyzeSlice(
3133431232
31335 const opt_new_ptr_val = sema.resolveValue(new_ptr);31233 const opt_new_ptr_val = sema.resolveValue(new_ptr);
31336 const new_ptr_val = opt_new_ptr_val orelse {31234 const new_ptr_val = opt_new_ptr_val orelse {
31337 const result = try block.addBitCast(return_ty, new_ptr);31235 const result = try block.addTyOp(.ptr_cast, return_ty, new_ptr);
31338 if (block.wantSafety()) {31236 if (block.wantSafety()) {
31339 // requirement: slicing C ptr is non-null31237 // requirement: slicing C ptr is non-null
31340 if (ptr_ptr_child_ty.isCPtr(zcu)) {31238 if (ptr_ptr_child_ty.isCPtr(zcu)) {
...@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {...@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {
34366 }34264 }
34367}34265}
3436834266
34369pub const bitCastVal = @import("Sema/bitcast.zig").bitCast;34267pub const castMemory = @import("Sema/reinterpret.zig").castMemory;
34370pub const bitCastSpliceVal = @import("Sema/bitcast.zig").bitCastSplice;34268pub const spliceMemory = @import("Sema/reinterpret.zig").spliceMemory;
3437134269
34372const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;34270const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;
34373const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;34271const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;
src/Sema/LowerZon.zig+4-11
...@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool...@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool
815 .@"packed" => result: {815 .@"packed" => result: {
816 const arena = self.sema.arena;816 const arena = self.sema.arena;
817 const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));817 const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));
818 @memset(buf, 0);
818 var bit_offset: u16 = 0;819 var bit_offset: u16 = 0;
819 for (field_values) |field_ip| {820 for (field_values) |field_ip| {
820 const field_val: Value = .fromInterned(field_ip);821 const field_val: Value = .fromInterned(field_ip);
821 field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) {822 field_val.writeToPackedMemory(zcu, buf, bit_offset);
822 error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
823 error.OutOfMemory => |e| return e,
824 };
825 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));823 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
826 }824 }
827 assert(bit_offset == res_ty.bitSize(zcu));825 assert(bit_offset == res_ty.bitSize(zcu));
828 break :result Value.readFromPackedMemory(res_ty, pt, buf, 0, arena) catch |err| switch (err) {826 break :result try .readFromPackedMemory(res_ty, pt, buf, 0);
829 error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
830 error.OutOfMemory => |e| return e,
831 };
832 },827 },
833 };828 };
834 return result.toIntern();829 return result.toIntern();
...@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool....@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool.
981 };976 };
982 const result: Value = switch (union_info.layout) {977 const result: Value = switch (union_info.layout) {
983 .auto, .@"extern" => try pt.unionValue(res_ty, tag, val),978 .auto, .@"extern" => try pt.unionValue(res_ty, tag, val),
984 .@"packed" => try self.sema.bitCastVal(val, res_ty, 0, 0, 0) orelse {979 .@"packed" => try self.sema.bitCastVal(val, res_ty),
985 unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
986 },
987 };980 };
988 return result.toIntern();981 return result.toIntern();
989}982}
src/Sema/bitcast.zig deleted-774
...@@ -1,774 +0,0 @@
1//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3//! a sequence of values in *packed* memory, and then unflatten through a combination of special
4//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5//!
6//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7//! have their fields reversed when represented as packed memory on such targets.
8
9/// If `host_bits` is `0`, attempts to convert the memory at offset
10/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11/// ignoring `bit_offset`.
12///
13/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14/// non-packed value consisting of `host_bits` bits. A value of type
15/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16/// into this value.
17///
18/// Returns `null` if the operation must be performed at runtime.
19pub fn bitCast(
20 sema: *Sema,
21 val: Value,
22 dest_ty: Type,
23 byte_offset: u64,
24 host_bits: u64,
25 bit_offset: u64,
26) CompileError!?Value {
27 return bitCastInner(sema, val, dest_ty, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
28 error.ReinterpretDeclRef => return null,
29 error.IllDefinedMemoryLayout => unreachable,
30 error.Unimplemented => @panic("unimplemented bitcast"),
31 else => |e| return e,
32 };
33}
34
35/// Uses bitcasting to splice the value `splice_val` into `val`,
36/// replacing overlapping bits and returning the modified value.
37///
38/// If `host_bits` is `0`, splices `splice_val` at an offset
39/// `byte_offset` bytes into the virtual memory of `val`, ignoring
40/// `bit_offset`.
41///
42/// Otherwise, `byte_offset` is an offset into bytes into `val` to
43/// a non-packed value consisting of `host_bits` bits. The value
44/// `splice_val` will be placed at a packed offset of `bit_offset`
45/// into this value.
46pub fn bitCastSplice(
47 sema: *Sema,
48 val: Value,
49 splice_val: Value,
50 byte_offset: u64,
51 host_bits: u64,
52 bit_offset: u64,
53) CompileError!?Value {
54 return bitCastSpliceInner(sema, val, splice_val, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
55 error.ReinterpretDeclRef => return null,
56 error.IllDefinedMemoryLayout => unreachable,
57 error.Unimplemented => @panic("unimplemented bitcast"),
58 else => |e| return e,
59 };
60}
61
62const BitCastError = CompileError || error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented };
63
64fn bitCastInner(
65 sema: *Sema,
66 val: Value,
67 dest_ty: Type,
68 byte_offset: u64,
69 host_bits: u64,
70 bit_offset: u64,
71) BitCastError!Value {
72 const pt = sema.pt;
73 const zcu = pt.zcu;
74 const endian = zcu.getTarget().cpu.arch.endian();
75
76 if (dest_ty.toIntern() == val.typeOf(zcu).toIntern() and bit_offset == 0) {
77 return val;
78 }
79
80 const val_ty = val.typeOf(zcu);
81
82 val_ty.assertHasLayout(zcu);
83 dest_ty.assertHasLayout(zcu);
84
85 assert(val_ty.hasWellDefinedLayout(zcu));
86
87 const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
88 .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
89 else
90 .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
91
92 const skip_bits = switch (endian) {
93 .little => bit_offset + byte_offset * 8,
94 .big => if (host_bits > 0)
95 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
96 else
97 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - dest_ty.bitSize(zcu),
98 };
99
100 var unpack: UnpackValueBits = .{
101 .pt = sema.pt,
102 .arena = sema.arena,
103 .skip_bits = skip_bits,
104 .remaining_bits = dest_ty.bitSize(zcu),
105 .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
106 };
107 switch (endian) {
108 .little => {
109 try unpack.add(val);
110 try unpack.padding(abi_pad_bits);
111 },
112 .big => {
113 try unpack.padding(abi_pad_bits);
114 try unpack.add(val);
115 },
116 }
117 try unpack.padding(host_pad_bits);
118
119 var pack: PackValueBits = .{
120 .pt = sema.pt,
121 .arena = sema.arena,
122 .unpacked = unpack.unpacked.items,
123 };
124 return pack.get(dest_ty);
125}
126
127fn bitCastSpliceInner(
128 sema: *Sema,
129 val: Value,
130 splice_val: Value,
131 byte_offset: u64,
132 host_bits: u64,
133 bit_offset: u64,
134) BitCastError!Value {
135 const pt = sema.pt;
136 const zcu = pt.zcu;
137 const endian = zcu.getTarget().cpu.arch.endian();
138 const val_ty = val.typeOf(zcu);
139 const splice_val_ty = splice_val.typeOf(zcu);
140
141 val_ty.assertHasLayout(zcu);
142 splice_val_ty.assertHasLayout(zcu);
143
144 const splice_bits = splice_val_ty.bitSize(zcu);
145
146 const splice_offset = switch (endian) {
147 .little => bit_offset + byte_offset * 8,
148 .big => if (host_bits > 0)
149 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
150 else
151 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - splice_bits,
152 };
153
154 assert(splice_offset + splice_bits <= val_ty.abiSize(zcu) * 8);
155
156 const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
157 .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
158 else
159 .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
160
161 var unpack: UnpackValueBits = .{
162 .pt = pt,
163 .arena = sema.arena,
164 .skip_bits = 0,
165 .remaining_bits = splice_offset,
166 .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
167 };
168 switch (endian) {
169 .little => {
170 try unpack.add(val);
171 try unpack.padding(abi_pad_bits);
172 },
173 .big => {
174 try unpack.padding(abi_pad_bits);
175 try unpack.add(val);
176 },
177 }
178 try unpack.padding(host_pad_bits);
179
180 unpack.remaining_bits = splice_bits;
181 try unpack.add(splice_val);
182
183 unpack.skip_bits = splice_offset + splice_bits;
184 unpack.remaining_bits = val_ty.abiSize(zcu) * 8 - splice_offset - splice_bits;
185 switch (endian) {
186 .little => {
187 try unpack.add(val);
188 try unpack.padding(abi_pad_bits);
189 },
190 .big => {
191 try unpack.padding(abi_pad_bits);
192 try unpack.add(val);
193 },
194 }
195 try unpack.padding(host_pad_bits);
196
197 var pack: PackValueBits = .{
198 .pt = pt,
199 .arena = sema.arena,
200 .unpacked = unpack.unpacked.items,
201 };
202 switch (endian) {
203 .little => {},
204 .big => try pack.padding(abi_pad_bits),
205 }
206 return pack.get(val_ty);
207}
208
209/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
210/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
211/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
212/// list contains bits from the *final* byte of the value.
213const UnpackValueBits = struct {
214 pt: Zcu.PerThread,
215 arena: Allocator,
216 skip_bits: u64,
217 remaining_bits: u64,
218 extra_bits: u64 = undefined,
219 unpacked: std.array_list.Managed(InternPool.Index),
220
221 fn add(unpack: *UnpackValueBits, val: Value) BitCastError!void {
222 const pt = unpack.pt;
223 const zcu = pt.zcu;
224 const endian = zcu.getTarget().cpu.arch.endian();
225 const ip = &zcu.intern_pool;
226
227 if (unpack.remaining_bits == 0) {
228 return;
229 }
230
231 const ty = val.typeOf(zcu);
232 const bit_size = ty.bitSize(zcu);
233
234 if (unpack.skip_bits >= bit_size) {
235 unpack.skip_bits -= bit_size;
236 return;
237 }
238
239 switch (ip.indexToKey(val.toIntern())) {
240 .int_type,
241 .ptr_type,
242 .array_type,
243 .vector_type,
244 .opt_type,
245 .anyframe_type,
246 .error_union_type,
247 .simple_type,
248 .struct_type,
249 .tuple_type,
250 .union_type,
251 .opaque_type,
252 .spirv_type,
253 .enum_type,
254 .func_type,
255 .error_set_type,
256 .inferred_error_set_type,
257 .@"extern",
258 .func,
259 .err,
260 .error_union,
261 .enum_literal,
262 .slice,
263 .memoized_call,
264 => unreachable, // ill-defined layout or not real values
265
266 .undef,
267 .int,
268 .enum_tag,
269 .simple_value,
270 .float,
271 .ptr,
272 .opt,
273 => try unpack.primitive(val),
274
275 .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
276
277 .aggregate => switch (ty.zigTypeTag(zcu)) {
278 .vector => {
279 const len: usize = @intCast(ty.arrayLen(zcu));
280 for (0..len) |i| {
281 // We reverse vector elements in packed memory on BE targets.
282 const real_idx = switch (endian) {
283 .little => i,
284 .big => len - i - 1,
285 };
286 const elem_val = try val.elemValue(pt, real_idx);
287 try unpack.add(elem_val);
288 }
289 },
290 .array => {
291 // Each element is padded up to its ABI size. Padding bits are undefined.
292 // The final element does not have trailing padding.
293 // Elements are reversed in packed memory on BE targets.
294 const elem_ty = ty.childType(zcu);
295 const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
296 const len = ty.arrayLen(zcu);
297 const maybe_sent = ty.sentinel(zcu);
298
299 if (endian == .big) if (maybe_sent) |s| {
300 try unpack.add(s);
301 if (len != 0) try unpack.padding(pad_bits);
302 };
303
304 for (0..@intCast(len)) |i| {
305 // We reverse array elements in packed memory on BE targets.
306 const real_idx = switch (endian) {
307 .little => i,
308 .big => len - i - 1,
309 };
310 const elem_val = try val.elemValue(pt, @intCast(real_idx));
311 try unpack.add(elem_val);
312 if (i != len - 1) try unpack.padding(pad_bits);
313 }
314
315 if (endian == .little) if (maybe_sent) |s| {
316 if (len != 0) try unpack.padding(pad_bits);
317 try unpack.add(s);
318 };
319 },
320 .@"struct" => switch (ty.containerLayout(zcu)) {
321 .auto => unreachable, // ill-defined layout
322 .@"extern" => switch (endian) {
323 .little => {
324 var cur_bit_off: u64 = 0;
325 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
326 while (it.next()) |field_idx| {
327 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
328 const pad_bits = want_bit_off - cur_bit_off;
329 const field_val = try val.fieldValue(pt, field_idx);
330 try unpack.padding(pad_bits);
331 try unpack.add(field_val);
332 cur_bit_off = want_bit_off + field_val.typeOf(zcu).bitSize(zcu);
333 }
334 // Add trailing padding bits.
335 try unpack.padding(bit_size - cur_bit_off);
336 },
337 .big => {
338 var cur_bit_off: u64 = bit_size;
339 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
340 while (it.next()) |field_idx| {
341 const field_val = try val.fieldValue(pt, field_idx);
342 const field_ty = field_val.typeOf(zcu);
343 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
344 const pad_bits = cur_bit_off - want_bit_off;
345 try unpack.padding(pad_bits);
346 try unpack.add(field_val);
347 cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
348 }
349 assert(cur_bit_off == 0);
350 },
351 },
352 .@"packed" => {
353 // Just add all fields in order. There are no padding bits.
354 // This is identical between LE and BE targets.
355 for (0..ty.structFieldCount(zcu)) |i| {
356 const field_val = try val.fieldValue(pt, i);
357 try unpack.add(field_val);
358 }
359 },
360 },
361 else => unreachable,
362 },
363
364 .un => |un| {
365 // We actually don't care about the tag here!
366 // Instead, we just need to write the payload value, plus any necessary padding.
367 // This correctly handles the case where `tag == .none`, since the payload is then
368 // either an integer or a byte array, both of which we can unpack.
369 const payload_val = Value.fromInterned(un.val);
370 const pad_bits = bit_size - payload_val.typeOf(zcu).bitSize(zcu);
371 if (endian == .little or ty.containerLayout(zcu) == .@"packed") {
372 try unpack.add(payload_val);
373 try unpack.padding(pad_bits);
374 } else {
375 try unpack.padding(pad_bits);
376 try unpack.add(payload_val);
377 }
378 },
379 }
380 }
381
382 fn padding(unpack: *UnpackValueBits, pad_bits: u64) BitCastError!void {
383 if (pad_bits == 0) return;
384 const pt = unpack.pt;
385 // Figure out how many full bytes and leftover bits there are.
386 const bytes = pad_bits / 8;
387 const bits = pad_bits % 8;
388 // Add undef u8 values for the bytes...
389 const undef_u8 = try pt.undefValue(Type.u8);
390 for (0..@intCast(bytes)) |_| {
391 try unpack.primitive(undef_u8);
392 }
393 // ...and an undef int for the leftover bits.
394 if (bits == 0) return;
395 const bits_ty = try pt.intType(.unsigned, @intCast(bits));
396 const bits_val = try pt.undefValue(bits_ty);
397 try unpack.primitive(bits_val);
398 }
399
400 fn primitive(unpack: *UnpackValueBits, val: Value) BitCastError!void {
401 const pt = unpack.pt;
402 const zcu = pt.zcu;
403
404 if (unpack.remaining_bits == 0) {
405 return;
406 }
407
408 const ty = val.typeOf(pt.zcu);
409 const bit_size = ty.bitSize(zcu);
410
411 // Note that this skips all zero-bit types.
412 if (unpack.skip_bits >= bit_size) {
413 unpack.skip_bits -= bit_size;
414 return;
415 }
416
417 if (unpack.skip_bits > 0) {
418 const skip = unpack.skip_bits;
419 unpack.skip_bits = 0;
420 return unpack.splitPrimitive(val, skip, bit_size - skip);
421 }
422
423 if (unpack.remaining_bits < bit_size) {
424 return unpack.splitPrimitive(val, 0, unpack.remaining_bits);
425 }
426
427 unpack.remaining_bits -|= bit_size;
428
429 try unpack.unpacked.append(val.toIntern());
430 }
431
432 fn splitPrimitive(unpack: *UnpackValueBits, val: Value, bit_offset: u64, bit_count: u64) BitCastError!void {
433 const pt = unpack.pt;
434 const zcu = pt.zcu;
435 const ty = val.typeOf(pt.zcu);
436
437 const val_bits = ty.bitSize(zcu);
438 assert(bit_offset + bit_count <= val_bits);
439
440 switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
441 // In the `ptr` case, this will return `error.ReinterpretDeclRef`
442 // if we're trying to split a non-integer pointer value.
443 .int, .float, .enum_tag, .ptr, .opt => {
444 // This @intCast is okay because no primitive can exceed the size of a u16.
445 const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
446 const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
447 try val.writeToPackedMemory(zcu, buf, 0);
448 const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
449 try unpack.primitive(sub_val);
450 },
451 .undef => try unpack.padding(bit_count),
452 // The only values here with runtime bits are `true` and `false.
453 // These are both 1 bit, so will never need truncating.
454 .simple_value => unreachable,
455 else => unreachable, // zero-bit or not primitives
456 }
457 }
458};
459
460/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBits`),
461/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
462/// values and of pointers which align in virtual memory.
463const PackValueBits = struct {
464 pt: Zcu.PerThread,
465 arena: Allocator,
466 bit_offset: u64 = 0,
467 unpacked: []const InternPool.Index,
468
469 fn get(pack: *PackValueBits, ty: Type) BitCastError!Value {
470 const pt = pack.pt;
471 const zcu = pt.zcu;
472 const endian = zcu.getTarget().cpu.arch.endian();
473 const ip = &zcu.intern_pool;
474 const arena = pack.arena;
475 switch (ty.zigTypeTag(zcu)) {
476 .vector => {
477 // Elements are bit-packed.
478 const len = ty.arrayLen(zcu);
479 const elem_ty = ty.childType(zcu);
480 const elems = try arena.alloc(InternPool.Index, @intCast(len));
481 // We reverse vector elements in packed memory on BE targets.
482 switch (endian) {
483 .little => for (elems) |*elem| {
484 elem.* = (try pack.get(elem_ty)).toIntern();
485 },
486 .big => {
487 var i = elems.len;
488 while (i > 0) {
489 i -= 1;
490 elems[i] = (try pack.get(elem_ty)).toIntern();
491 }
492 },
493 }
494 return pt.aggregateValue(ty, elems);
495 },
496 .array => {
497 // Each element is padded up to its ABI size. The final element does not have trailing padding.
498 const len = ty.arrayLen(zcu);
499 const elem_ty = ty.childType(zcu);
500 const maybe_sent = ty.sentinel(zcu);
501 const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
502 const elems = try arena.alloc(InternPool.Index, @intCast(len));
503
504 if (endian == .big and maybe_sent != null) {
505 // TODO: validate sentinel was preserved!
506 try pack.padding(elem_ty.bitSize(zcu));
507 if (len != 0) try pack.padding(pad_bits);
508 }
509
510 for (0..elems.len) |i| {
511 const real_idx = switch (endian) {
512 .little => i,
513 .big => len - i - 1,
514 };
515 elems[@intCast(real_idx)] = (try pack.get(elem_ty)).toIntern();
516 if (i != len - 1) try pack.padding(pad_bits);
517 }
518
519 if (endian == .little and maybe_sent != null) {
520 // TODO: validate sentinel was preserved!
521 if (len != 0) try pack.padding(pad_bits);
522 try pack.padding(elem_ty.bitSize(zcu));
523 }
524
525 return pt.aggregateValue(ty, elems);
526 },
527 .@"struct" => switch (ty.containerLayout(zcu)) {
528 .auto => unreachable, // ill-defined layout
529 .@"extern" => {
530 const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
531 @memset(elems, .none);
532 switch (endian) {
533 .little => {
534 var cur_bit_off: u64 = 0;
535 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
536 while (it.next()) |field_idx| {
537 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
538 try pack.padding(want_bit_off - cur_bit_off);
539 const field_ty = ty.fieldType(field_idx, zcu);
540 elems[field_idx] = (try pack.get(field_ty)).toIntern();
541 cur_bit_off = want_bit_off + field_ty.bitSize(zcu);
542 }
543 try pack.padding(ty.bitSize(zcu) - cur_bit_off);
544 },
545 .big => {
546 var cur_bit_off: u64 = ty.bitSize(zcu);
547 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
548 while (it.next()) |field_idx| {
549 const field_ty = ty.fieldType(field_idx, zcu);
550 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
551 try pack.padding(cur_bit_off - want_bit_off);
552 elems[field_idx] = (try pack.get(field_ty)).toIntern();
553 cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
554 }
555 assert(cur_bit_off == 0);
556 },
557 }
558 // Any fields which do not have runtime bits should be OPV or comptime fields.
559 // Fill those values now.
560 for (elems, 0..) |*elem, field_idx| {
561 if (elem.* != .none) continue;
562 const val = (try ty.structFieldValueComptime(pt, field_idx)).?;
563 elem.* = val.toIntern();
564 }
565 return pt.aggregateValue(ty, elems);
566 },
567 .@"packed" => {
568 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
569 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
570 return pt.bitpackValue(ty, backing_int_val);
571 },
572 },
573 .@"union" => switch (ty.containerLayout(zcu)) {
574 .auto => unreachable, // ill-defined layout
575 .@"extern" => {
576 // We will attempt to read as the backing representation. If this emits
577 // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
578 // We will also attempt smaller fields when we get `undefined`, as if some bits are
579 // defined we want to include them.
580 // TODO: this is very very bad. We need a more sophisticated union representation.
581
582 const prev_unpacked = pack.unpacked;
583 const prev_bit_offset = pack.bit_offset;
584
585 const backing_ty = try ty.externUnionBackingType(pt);
586
587 backing: {
588 const backing_val = pack.get(backing_ty) catch |err| switch (err) {
589 error.ReinterpretDeclRef => {
590 pack.unpacked = prev_unpacked;
591 pack.bit_offset = prev_bit_offset;
592 break :backing;
593 },
594 else => |e| return e,
595 };
596 if (backing_val.isUndef(zcu)) {
597 pack.unpacked = prev_unpacked;
598 pack.bit_offset = prev_bit_offset;
599 break :backing;
600 }
601 return Value.fromInterned(try pt.internUnion(.{
602 .ty = ty.toIntern(),
603 .tag = .none,
604 .val = backing_val.toIntern(),
605 }));
606 }
607
608 const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
609 for (field_order, 0..) |*f, i| f.* = @intCast(i);
610 // Sort `field_order` to put the fields with the largest bit sizes first.
611 const SizeSortCtx = struct {
612 zcu: *Zcu,
613 field_types: []const InternPool.Index,
614 fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
615 const a_ty = Type.fromInterned(ctx.field_types[a_idx]);
616 const b_ty = Type.fromInterned(ctx.field_types[b_idx]);
617 return a_ty.bitSize(ctx.zcu) > b_ty.bitSize(ctx.zcu);
618 }
619 };
620 std.mem.sortUnstable(u32, field_order, SizeSortCtx{
621 .zcu = zcu,
622 .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
623 }, SizeSortCtx.lessThan);
624
625 const padding_after = endian == .little or ty.containerLayout(zcu) == .@"packed";
626
627 for (field_order) |field_idx| {
628 const field_ty = Type.fromInterned(zcu.typeToUnion(ty).?.field_types.get(ip)[field_idx]);
629 const pad_bits = ty.bitSize(zcu) - field_ty.bitSize(zcu);
630 if (!padding_after) try pack.padding(pad_bits);
631 const field_val = pack.get(field_ty) catch |err| switch (err) {
632 error.ReinterpretDeclRef => {
633 pack.unpacked = prev_unpacked;
634 pack.bit_offset = prev_bit_offset;
635 continue;
636 },
637 else => |e| return e,
638 };
639 if (padding_after) try pack.padding(pad_bits);
640 if (field_val.isUndef(zcu)) {
641 pack.unpacked = prev_unpacked;
642 pack.bit_offset = prev_bit_offset;
643 continue;
644 }
645 const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_idx);
646 return Value.fromInterned(try pt.internUnion(.{
647 .ty = ty.toIntern(),
648 .tag = tag_val.toIntern(),
649 .val = field_val.toIntern(),
650 }));
651 }
652
653 // No field could represent the value. Just do whatever happens when we try to read
654 // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
655 const backing_val = try pack.get(backing_ty);
656 return Value.fromInterned(try pt.internUnion(.{
657 .ty = ty.toIntern(),
658 .tag = .none,
659 .val = backing_val.toIntern(),
660 }));
661 },
662 .@"packed" => {
663 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
664 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
665 return pt.bitpackValue(ty, backing_int_val);
666 },
667 },
668 else => return pack.primitive(ty),
669 }
670 }
671
672 fn padding(pack: *PackValueBits, pad_bits: u64) BitCastError!void {
673 _ = pack.prepareBits(pad_bits);
674 }
675
676 fn primitive(pack: *PackValueBits, want_ty: Type) BitCastError!Value {
677 const pt = pack.pt;
678 const zcu = pt.zcu;
679
680 if (try want_ty.onePossibleValue(pt)) |opv| return opv;
681
682 const vals, const bit_offset = pack.prepareBits(want_ty.bitSize(zcu));
683
684 for (vals) |val| {
685 if (!Value.fromInterned(val).isUndef(zcu)) break;
686 } else {
687 // All bits of the value are `undefined`.
688 return pt.undefValue(want_ty);
689 }
690
691 // TODO: we need to decide how to handle partially-undef values here.
692 // Currently, a value with some undefined bits becomes `0xAA` so that we
693 // preserve the well-defined bits, because we can't currently represent
694 // a partially-undefined primitive (e.g. an int with some undef bits).
695 // In future, we probably want to take one of these two routes:
696 // * Define that if any bits are `undefined`, the entire value is `undefined`.
697 // This is a major breaking change, and probably a footgun.
698 // * Introduce tracking for partially-undef values at comptime.
699 // This would complicate a lot of operations in Sema, such as basic
700 // arithmetic.
701 // This design complexity is tracked by #19634.
702
703 ptr_cast: {
704 if (vals.len != 1) break :ptr_cast;
705 const val = Value.fromInterned(vals[0]);
706 if (!val.typeOf(zcu).isPtrAtRuntime(zcu)) break :ptr_cast;
707 if (!want_ty.isPtrAtRuntime(zcu)) break :ptr_cast;
708 return pt.getCoerced(val, want_ty);
709 }
710
711 // Reinterpret via an in-memory buffer.
712
713 var buf_bits: u64 = 0;
714 for (vals) |ip_val| {
715 const val = Value.fromInterned(ip_val);
716 const ty = val.typeOf(pt.zcu);
717 buf_bits += ty.bitSize(zcu);
718 }
719
720 const buf = try pack.arena.alloc(u8, @intCast((buf_bits + 7) / 8));
721 // We will skip writing undefined values, so mark the buffer as `0xAA` so we get "undefined" bits.
722 @memset(buf, 0xAA);
723 var cur_bit_off: usize = 0;
724 for (vals) |ip_val| {
725 const val = Value.fromInterned(ip_val);
726 const ty = val.typeOf(zcu);
727 if (!val.isUndef(zcu)) {
728 try val.writeToPackedMemory(zcu, buf, cur_bit_off);
729 }
730 cur_bit_off += @intCast(ty.bitSize(zcu));
731 }
732
733 return Value.readFromPackedMemory(want_ty, pt, buf, @intCast(bit_offset), pack.arena);
734 }
735
736 fn prepareBits(pack: *PackValueBits, need_bits: u64) struct { []const InternPool.Index, u64 } {
737 if (need_bits == 0) return .{ &.{}, 0 };
738
739 const pt = pack.pt;
740 const zcu = pt.zcu;
741
742 var bits: u64 = 0;
743 var len: usize = 0;
744 while (bits < pack.bit_offset + need_bits) {
745 bits += Value.fromInterned(pack.unpacked[len]).typeOf(pt.zcu).bitSize(zcu);
746 len += 1;
747 }
748
749 const result_vals = pack.unpacked[0..len];
750 const result_offset = pack.bit_offset;
751
752 const extra_bits = bits - pack.bit_offset - need_bits;
753 if (extra_bits == 0) {
754 pack.unpacked = pack.unpacked[len..];
755 pack.bit_offset = 0;
756 } else {
757 pack.unpacked = pack.unpacked[len - 1 ..];
758 pack.bit_offset = Value.fromInterned(pack.unpacked[0]).typeOf(pt.zcu).bitSize(zcu) - extra_bits;
759 }
760
761 return .{ result_vals, result_offset };
762 }
763};
764
765const std = @import("std");
766const Allocator = std.mem.Allocator;
767const assert = std.debug.assert;
768
769const Sema = @import("../Sema.zig");
770const Zcu = @import("../Zcu.zig");
771const InternPool = @import("../InternPool.zig");
772const Type = @import("../Type.zig");
773const Value = @import("../Value.zig");
774const CompileError = Zcu.CompileError;
src/Sema/comptime_ptr_access.zig+173-148
...@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {...@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {
14pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {14pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {
15 const pt = sema.pt;15 const pt = sema.pt;
16 const zcu = pt.zcu;16 const zcu = pt.zcu;
17
17 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);18 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
18 // TODO: host size for vectors is terrible19 const elem_ty: Type = .fromInterned(ptr_info.child);
19 const host_bits = switch (ptr_info.flags.vector_index) {20 const host_size = ptr_info.packed_offset.host_size;
20 .none => ptr_info.packed_offset.host_size * 8,21
21 else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),22 if (host_size == 0) {
22 };23 return loadComptimePtrInner(sema, block, src, ptr, elem_ty, 0);
23 const bit_offset = if (host_bits != 0) bit_offset: {24 }
24 const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);25
25 const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {26 assert(elem_ty.hasBitRepresentation(zcu));
26 .none => 0,27 if (ptr_info.flags.vector_index == .none) {
27 else => |idx| switch (pt.zcu.getTarget().cpu.arch.endian()) {28 if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
28 .little => child_bits * @intFromEnum(idx),
29 .big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
30 },
31 };
32 if (child_bits + bit_offset > host_bits) {
33 return .exceeds_host_size;29 return .exceeds_host_size;
34 }30 }
35 break :bit_offset bit_offset;31 const load_ty: Type = try pt.intType(.unsigned, host_size * 8);
36 } else 0;32 const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
37 return loadComptimePtrInner(sema, block, src, ptr, bit_offset, host_bits, Type.fromInterned(ptr_info.child), 0);33 else => |result| return result,
34 .success => |mv| mv,
35 };
36 const backing_int_val = try backing_int_mv.intern(pt, sema.arena);
37 const buf = try sema.arena.alloc(u8, host_size);
38 @memset(buf, 0);
39 backing_int_val.writeToPackedMemory(zcu, buf, 0);
40 const result_val: Value = try .readFromPackedMemory(elem_ty, pt, buf, ptr_info.packed_offset.bit_offset);
41 return .{ .success = .{ .interned = result_val.toIntern() } };
42 }
43 if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
44 return .exceeds_host_size;
45 }
46 const load_ty: Type = try pt.vectorType(.{
47 .len = host_size,
48 .child = elem_ty.toIntern(),
49 });
50 const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
51 else => |result| return result,
52 .success => |mv| mv,
53 };
54 const vector_val = try vector_mv.intern(pt, sema.arena);
55 const result_val = try vector_val.elemValue(pt, @intFromEnum(ptr_info.flags.vector_index));
56 return .{ .success = .{ .interned = result_val.toIntern() } };
38}57}
3958
40pub const ComptimeStoreResult = union(enum) {59pub const ComptimeStoreResult = union(enum) {
...@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {...@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {
52};71};
5372
54/// Perform a comptime load of value `store_val` to a pointer.73/// Perform a comptime load of value `store_val` to a pointer.
55/// The pointer's type is ignored.74///
75/// Asserts that the type of `store_val` equals the element type of the pointer type.
56pub fn storeComptimePtr(76pub fn storeComptimePtr(
57 sema: *Sema,77 sema: *Sema,
58 block: *Block,78 block: *Block,
...@@ -62,42 +82,84 @@ pub fn storeComptimePtr(...@@ -62,42 +82,84 @@ pub fn storeComptimePtr(
62) !ComptimeStoreResult {82) !ComptimeStoreResult {
63 const pt = sema.pt;83 const pt = sema.pt;
64 const zcu = pt.zcu;84 const zcu = pt.zcu;
65 const ptr_info = ptr.typeOf(zcu).ptrInfo(zcu);
66 assert(store_val.typeOf(zcu).toIntern() == ptr_info.child);
6785
68 {86 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
69 const store_ty: Type = .fromInterned(ptr_info.child);87 const elem_ty: Type = .fromInterned(ptr_info.child);
70 if (!store_ty.comptimeOnly(zcu) and !store_ty.hasRuntimeBits(zcu)) {88 const host_size = ptr_info.packed_offset.host_size;
71 // zero-bit store; nothing to do89 assert(store_val.typeOf(zcu).toIntern() == elem_ty.toIntern());
72 return .success;90
73 }91 if (host_size == 0) {
92 return storeComptimePtrInner(sema, block, src, ptr, store_val);
74 }93 }
7594
76 // TODO: host size for vectors is terrible95 assert(elem_ty.hasBitRepresentation(zcu));
77 const host_bits = switch (ptr_info.flags.vector_index) {96 if (ptr_info.flags.vector_index == .none) {
78 .none => ptr_info.packed_offset.host_size * 8,97 if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
79 else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
80 };
81 const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
82 .none => 0,
83 else => |idx| switch (zcu.getTarget().cpu.arch.endian()) {
84 .little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
85 .big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian
86 },
87 };
88 const pseudo_store_ty = if (host_bits > 0) t: {
89 const need_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
90 if (need_bits + bit_offset > host_bits) {
91 return .exceeds_host_size;98 return .exceeds_host_size;
92 }99 }
93 break :t try sema.pt.intType(.unsigned, @intCast(host_bits));100 const backing_ty: Type = try pt.intType(.unsigned, host_size * 8);
94 } else Type.fromInterned(ptr_info.child);101 const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, backing_ty, 0)) {
102 .success => |mv| mv,
103 .runtime_load => return .runtime_store,
104 inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
105 };
106 const old_backing_int_val = try backing_int_mv.intern(pt, sema.arena);
107 const buf = try sema.arena.alloc(u8, host_size);
108 @memset(buf, 0);
109 old_backing_int_val.writeToPackedMemory(zcu, buf, 0);
110 // Write the new element...
111 store_val.writeToPackedMemory(zcu, buf, ptr_info.packed_offset.bit_offset);
112 // ...then read the resulting backing integer value...
113 const new_backing_int_val: Value = try .readFromPackedMemory(backing_ty, pt, buf, 0);
114 // ...and store that back into memory
115 return storeComptimePtrInner(sema, block, src, ptr, new_backing_int_val);
116 }
117
118 if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
119 return .exceeds_host_size;
120 }
121 const vec_ty: Type = try pt.vectorType(.{
122 .len = host_size,
123 .child = elem_ty.toIntern(),
124 });
125 const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, vec_ty, 0)) {
126 .success => |mv| mv,
127 .runtime_load => return .runtime_store,
128 inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
129 };
130 const old_vector_val = try vector_mv.intern(pt, sema.arena);
131 const elems_buf = try sema.arena.alloc(InternPool.Index, host_size);
132 for (elems_buf, 0..) |*elem, elem_index| {
133 const elem_val = try old_vector_val.elemValue(pt, elem_index);
134 elem.* = elem_val.toIntern();
135 }
136 elems_buf[@intFromEnum(ptr_info.flags.vector_index)] = store_val.toIntern();
137 const new_vector_val = try pt.aggregateValue(vec_ty, elems_buf);
138 return storeComptimePtrInner(sema, block, src, ptr, new_vector_val);
139}
95140
96 const strat = try prepareComptimePtrStore(sema, block, src, ptr, pseudo_store_ty, 0);141/// Like `storeComptimePtr`, except ignores the type of `ptr`, instead treating it as a single-item
142/// pointer to `store_val.typeOf(zcu)`.
143fn storeComptimePtrInner(
144 sema: *Sema,
145 block: *Block,
146 src: LazySrcLoc,
147 ptr: Value,
148 store_val: Value,
149) !ComptimeStoreResult {
150 const pt = sema.pt;
151 const zcu = pt.zcu;
152 const store_ty = store_val.typeOf(zcu);
153
154 if (store_ty.classify(zcu) == .one_possible_value) {
155 // zero-bit store; nothing to do
156 return .success;
157 }
158
159 const strat = try prepareComptimePtrStore(sema, block, src, ptr, store_ty, 0);
97160
98 // Propagate errors and handle comptime fields.161 // Propagate errors and handle comptime fields.
99 switch (strat) {162 switch (strat) {
100 .direct, .index, .flat_index, .reinterpret => {},
101 .comptime_field => {163 .comptime_field => {
102 // To "store" to a comptime field, just perform a load of the field164 // To "store" to a comptime field, just perform a load of the field
103 // and see if the store value matches.165 // and see if the store value matches.
...@@ -125,79 +187,60 @@ pub fn storeComptimePtr(...@@ -125,79 +187,60 @@ pub fn storeComptimePtr(
125 .inactive_union_field => return .inactive_union_field,187 .inactive_union_field => return .inactive_union_field,
126 .needed_well_defined => |ty| return .{ .needed_well_defined = ty },188 .needed_well_defined => |ty| return .{ .needed_well_defined = ty },
127 .out_of_bounds => |ty| return .{ .out_of_bounds = ty },189 .out_of_bounds => |ty| return .{ .out_of_bounds = ty },
128 }
129
130 // Check the store is not inside a runtime condition
131 try checkComptimeVarStore(sema, block, src, strat.alloc());
132
133 if (host_bits == 0) {
134 // We can attempt a direct store depending on the strategy.
135 switch (strat) {
136 .direct => |direct| {
137 const want_ty = direct.val.typeOf(zcu);
138 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
139 direct.val.* = .{ .interned = coerced_store_val.toIntern() };
140 return .success;
141 },
142 .index => |index| {
143 const want_ty = index.val.typeOf(zcu).childType(zcu);
144 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
145 try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
146 return .success;
147 },
148 .flat_index => |flat| {
149 const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
150 const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
151 {
152 var next_idx: u64 = 0;
153 var skip: u64 = 0;
154 try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
155 }
156 for (flat_elems, 0..) |elem, idx| {
157 // TODO: recursiveIndex in a loop does a lot of redundant work!
158 // Better would be to gather all the store targets into an array.
159 var index: u64 = flat.flat_elem_index + idx;
160 const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
161 try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
162 }
163 return .success;
164 },
165 .reinterpret => {},
166 else => unreachable,
167 }
168 }
169190
170 // Either there is a bit offset, or the strategy required reinterpreting.191 .direct => |direct| {
171 // Therefore, we must perform a bitcast.192 try checkComptimeVarStore(sema, block, src, direct.alloc);
193 const want_ty = direct.val.typeOf(zcu);
194 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
195 direct.val.* = .{ .interned = coerced_store_val.toIntern() };
196 return .success;
197 },
172198
173 const val_ptr: *MutableValue, const byte_offset: u64 = switch (strat) {199 .index => |index| {
174 .direct => |direct| .{ direct.val, 0 },200 try checkComptimeVarStore(sema, block, src, index.alloc);
175 .index => |index| .{201 const want_ty = index.val.typeOf(zcu).childType(zcu);
176 index.val,202 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
177 index.elem_index * index.val.typeOf(zcu).childType(zcu).abiSize(zcu),203 try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
204 return .success;
178 },205 },
179 .flat_index => |flat| .{ flat.val, flat.flat_elem_index * flat.val.typeOf(zcu).arrayBase(zcu)[0].abiSize(zcu) },
180 .reinterpret => |reinterpret| .{ reinterpret.val, reinterpret.byte_offset },
181 else => unreachable,
182 };
183206
184 if (!val_ptr.typeOf(zcu).hasWellDefinedLayout(zcu)) {207 .flat_index => |flat| {
185 return .{ .needed_well_defined = val_ptr.typeOf(zcu) };208 try checkComptimeVarStore(sema, block, src, flat.alloc);
186 }209 const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
210 const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
211 {
212 var next_idx: u64 = 0;
213 var skip: u64 = 0;
214 try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
215 }
216 for (flat_elems, 0..) |elem, idx| {
217 // TODO: recursiveIndex in a loop does a lot of redundant work!
218 // Better would be to gather all the store targets into an array.
219 var index: u64 = flat.flat_elem_index + idx;
220 const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
221 try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
222 }
223 return .success;
224 },
187225
188 if (!store_val.typeOf(zcu).hasWellDefinedLayout(zcu)) {226 .reinterpret => |reinterpret| {
189 return .{ .needed_well_defined = store_val.typeOf(zcu) };227 try checkComptimeVarStore(sema, block, src, reinterpret.alloc);
228 if (!reinterpret.val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
229 return .{ .needed_well_defined = reinterpret.val.typeOf(zcu) };
230 }
231 if (!store_ty.hasWellDefinedLayout(zcu)) {
232 return .{ .needed_well_defined = store_ty };
233 }
234 const old_val = try reinterpret.val.intern(pt, sema.arena);
235 const new_val = try sema.spliceMemory(
236 old_val,
237 store_val,
238 reinterpret.byte_offset,
239 ) orelse return .runtime_store;
240 reinterpret.val.* = .{ .interned = new_val.toIntern() };
241 return .success;
242 },
190 }243 }
191
192 const new_val = try sema.bitCastSpliceVal(
193 try val_ptr.intern(pt, sema.arena),
194 store_val,
195 byte_offset,
196 host_bits,
197 bit_offset,
198 ) orelse return .runtime_store;
199 val_ptr.* = .{ .interned = new_val.toIntern() };
200 return .success;
201}244}
202245
203/// Perform a comptime load of type `load_ty` from a pointer.246/// Perform a comptime load of type `load_ty` from a pointer.
...@@ -207,8 +250,6 @@ fn loadComptimePtrInner(...@@ -207,8 +250,6 @@ fn loadComptimePtrInner(
207 block: *Block,250 block: *Block,
208 src: LazySrcLoc,251 src: LazySrcLoc,
209 ptr_val: Value,252 ptr_val: Value,
210 bit_offset: u64,
211 host_bits: u64,
212 load_ty: Type,253 load_ty: Type,
213 /// If `load_ty` is an array, this is the number of array elements to skip254 /// If `load_ty` is an array, this is the number of array elements to skip
214 /// before `load_ty`. Otherwise, it is ignored and may be `undefined`.255 /// before `load_ty`. Otherwise, it is ignored and may be `undefined`.
...@@ -244,7 +285,7 @@ fn loadComptimePtrInner(...@@ -244,7 +285,7 @@ fn loadComptimePtrInner(
244 .eu_payload => |base_ptr_ip| val: {285 .eu_payload => |base_ptr_ip| val: {
245 const base_ptr = Value.fromInterned(base_ptr_ip);286 const base_ptr = Value.fromInterned(base_ptr_ip);
246 const base_ty = base_ptr.typeOf(zcu).childType(zcu);287 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
247 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {288 switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
248 .success => |eu_val| switch (eu_val.unpackErrorUnion(zcu)) {289 .success => |eu_val| switch (eu_val.unpackErrorUnion(zcu)) {
249 .undef => return .undef,290 .undef => return .undef,
250 .err => |err| return .{ .err_payload = err },291 .err => |err| return .{ .err_payload = err },
...@@ -256,7 +297,7 @@ fn loadComptimePtrInner(...@@ -256,7 +297,7 @@ fn loadComptimePtrInner(
256 .opt_payload => |base_ptr_ip| val: {297 .opt_payload => |base_ptr_ip| val: {
257 const base_ptr = Value.fromInterned(base_ptr_ip);298 const base_ptr = Value.fromInterned(base_ptr_ip);
258 const base_ty = base_ptr.typeOf(zcu).childType(zcu);299 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
259 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {300 switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
260 .success => |eu_val| switch (eu_val.unpackOptional(zcu)) {301 .success => |eu_val| switch (eu_val.unpackOptional(zcu)) {
261 .undef => return .undef,302 .undef => return .undef,
262 .null => return .null_payload,303 .null => return .null_payload,
...@@ -283,7 +324,7 @@ fn loadComptimePtrInner(...@@ -283,7 +324,7 @@ fn loadComptimePtrInner(
283 .child = base_ty.toIntern(),324 .child = base_ty.toIntern(),
284 });325 });
285326
286 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, want_ty, base_index.index)) {327 switch (try loadComptimePtrInner(sema, block, src, base_ptr, want_ty, base_index.index)) {
287 .success => |arr_val| break :val arr_val,328 .success => |arr_val| break :val arr_val,
288 else => |err| return err,329 else => |err| return err,
289 }330 }
...@@ -293,7 +334,7 @@ fn loadComptimePtrInner(...@@ -293,7 +334,7 @@ fn loadComptimePtrInner(
293 const base_ty = base_ptr.typeOf(zcu).childType(zcu);334 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
294335
295 // Field of a slice, or of an auto-layout struct or union.336 // Field of a slice, or of an auto-layout struct or union.
296 const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {337 const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
297 .success => |val| val,338 .success => |val| val,
298 else => |err| return err,339 else => |err| return err,
299 };340 };
...@@ -324,7 +365,7 @@ fn loadComptimePtrInner(...@@ -324,7 +365,7 @@ fn loadComptimePtrInner(
324 },365 },
325 };366 };
326367
327 if (ptr.byte_offset == 0 and host_bits == 0) {368 if (ptr.byte_offset == 0) {
328 if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {369 if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {
329 if (.ok == try sema.coerceInMemoryAllowed(370 if (.ok == try sema.coerceInMemoryAllowed(
330 block,371 block,
...@@ -343,8 +384,6 @@ fn loadComptimePtrInner(...@@ -343,8 +384,6 @@ fn loadComptimePtrInner(
343 }384 }
344385
345 restructure_array: {386 restructure_array: {
346 if (host_bits != 0) break :restructure_array;
347
348 // We might also be changing the length of an array, or restructuring it.387 // We might also be changing the length of an array, or restructuring it.
349 // e.g. [1][2][3]T -> [3][2]T.388 // e.g. [1][2][3]T -> [3][2]T.
350 // This case is important because it's permitted for types with ill-defined layouts.389 // This case is important because it's permitted for types with ill-defined layouts.
...@@ -402,7 +441,7 @@ fn loadComptimePtrInner(...@@ -402,7 +441,7 @@ fn loadComptimePtrInner(
402 cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;441 cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;
403 }442 }
404443
405 const need_bytes = if (host_bits > 0) (host_bits + 7) / 8 else load_ty.abiSize(zcu);444 const need_bytes = load_ty.abiSize(zcu);
406445
407 if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {446 if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {
408 return .{ .out_of_bounds = cur_val.typeOf(zcu) };447 return .{ .out_of_bounds = cur_val.typeOf(zcu) };
...@@ -453,7 +492,7 @@ fn loadComptimePtrInner(...@@ -453,7 +492,7 @@ fn loadComptimePtrInner(
453 },492 },
454 .@"struct" => switch (cur_ty.containerLayout(zcu)) {493 .@"struct" => switch (cur_ty.containerLayout(zcu)) {
455 .auto => unreachable, // ill-defined layout494 .auto => unreachable, // ill-defined layout
456 .@"packed" => break, // let the bitcast logic handle this495 .@"packed" => break, // let the memory reinterpret logic handle this
457 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {496 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
458 const start_off = cur_ty.structFieldOffset(field_idx, zcu);497 const start_off = cur_ty.structFieldOffset(field_idx, zcu);
459 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);498 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...@@ -466,9 +505,9 @@ fn loadComptimePtrInner(...@@ -466,9 +505,9 @@ fn loadComptimePtrInner(
466 },505 },
467 .@"union" => switch (cur_ty.containerLayout(zcu)) {506 .@"union" => switch (cur_ty.containerLayout(zcu)) {
468 .auto => unreachable, // ill-defined layout507 .auto => unreachable, // ill-defined layout
469 .@"packed" => break, // let the bitcast logic handle this508 .@"packed" => break, // let the memory reinterpret logic handle this
470 .@"extern" => {509 .@"extern" => {
471 // TODO: we have to let bitcast logic handle this for now.510 // TODO: we have to let the memory reinterpret logic handle this for now.
472 // Otherwise, we might traverse into a union field which doesn't allow pointers.511 // Otherwise, we might traverse into a union field which doesn't allow pointers.
473 // Figure out a solution!512 // Figure out a solution!
474 if (true) break;513 if (true) break;
...@@ -495,27 +534,13 @@ fn loadComptimePtrInner(...@@ -495,27 +534,13 @@ fn loadComptimePtrInner(
495534
496 // Fast path: check again if we're now at the type we want to load.535 // Fast path: check again if we're now at the type we want to load.
497 // If so, just return the loaded value.536 // If so, just return the loaded value.
498 if (cur_offset == 0 and host_bits == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {537 if (cur_offset == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
499 return .{ .success = cur_val };538 return .{ .success = cur_val };
500 }539 }
501540
502 var bitcast_src_val = try cur_val.intern(sema.pt, sema.arena);541 // Otherwise, use the memory reinterpretation logic to pull out the bytes we need.
503542 const reinterpret_val = try cur_val.intern(pt, sema.arena);
504 if (host_bits != 0) {543 const result_val = try sema.castMemory(reinterpret_val, load_ty, cur_offset) orelse return .runtime_load;
505 const src_bit_size = bitcast_src_val.typeOf(zcu).bitSize(zcu);
506 if (src_bit_size > host_bits) {
507 const truncate_ty = try pt.intType(.unsigned, @intCast(host_bits));
508 bitcast_src_val = try pt.getCoerced(bitcast_src_val, truncate_ty);
509 }
510 }
511
512 const result_val = try sema.bitCastVal(
513 bitcast_src_val,
514 load_ty,
515 cur_offset,
516 host_bits,
517 bit_offset,
518 ) orelse return .runtime_load;
519 return .{ .success = .{ .interned = result_val.toIntern() } };544 return .{ .success = .{ .interned = result_val.toIntern() } };
520}545}
521546
...@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {...@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {
546 val: *MutableValue,571 val: *MutableValue,
547 flat_elem_index: u64,572 flat_elem_index: u64,
548 },573 },
549 /// This value should be reinterpreted using bitcast logic to perform the574 /// This value should be reinterpreted using `Sema.spliceMemory` to perform
550 /// store. Only returned if `store_ty` and the type of `val` both have575 /// store. Only returned if `store_ty` and the type of `val` both have
551 /// well-defined layouts.576 /// well-defined layouts.
552 reinterpret: struct {577 reinterpret: struct {
...@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(...@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(
886 },911 },
887 .@"struct" => switch (cur_ty.containerLayout(zcu)) {912 .@"struct" => switch (cur_ty.containerLayout(zcu)) {
888 .auto => unreachable, // ill-defined layout913 .auto => unreachable, // ill-defined layout
889 .@"packed" => break, // let the bitcast logic handle this914 .@"packed" => break, // let the memory reinterp logic handle this
890 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {915 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
891 const start_off = cur_ty.structFieldOffset(field_idx, zcu);916 const start_off = cur_ty.structFieldOffset(field_idx, zcu);
892 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);917 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(...@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(
899 },924 },
900 .@"union" => switch (cur_ty.containerLayout(zcu)) {925 .@"union" => switch (cur_ty.containerLayout(zcu)) {
901 .auto => unreachable, // ill-defined layout926 .auto => unreachable, // ill-defined layout
902 .@"packed" => break, // let the bitcast logic handle this927 .@"packed" => break, // let the memory reinterp logic handle this
903 .@"extern" => {928 .@"extern" => {
904 // TODO: we have to let bitcast logic handle this for now.929 // TODO: we have to let the memory reinterp logic handle this for now.
905 // Otherwise, we might traverse into a union field which doesn't allow pointers.930 // Otherwise, we might traverse into a union field which doesn't allow pointers.
906 // Figure out a solution!931 // Figure out a solution!
907 if (true) break;932 if (true) break;
src/Sema/reinterpret.zig created+576
...@@ -0,0 +1,576 @@
1//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3//! a sequence of values in *packed* memory, and then unflatten through a combination of special
4//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5//!
6//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7//! have their fields reversed when represented as packed memory on such targets.
8
9/// If `host_bits` is `0`, attempts to convert the memory at offset
10/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11/// ignoring `bit_offset`.
12///
13/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14/// non-packed value consisting of `host_bits` bits. A value of type
15/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16/// into this value.
17///
18/// Returns `null` if the operation must be performed at runtime.
19pub fn castMemory(
20 sema: *Sema,
21 val: Value,
22 dest_ty: Type,
23 byte_offset: u64,
24) CompileError!?Value {
25 const pt = sema.pt;
26 const zcu = pt.zcu;
27
28 const val_ty = val.typeOf(zcu);
29
30 if (dest_ty.toIntern() == val_ty.toIntern()) {
31 assert(byte_offset == 0);
32 return val;
33 }
34
35 val_ty.assertHasLayout(zcu);
36 dest_ty.assertHasLayout(zcu);
37
38 var unpack: UnpackValueBytes = .{
39 .pt = pt,
40 .arena = sema.arena,
41 .skip_bytes = byte_offset,
42 .remaining_bytes = dest_ty.abiSize(zcu),
43 .unpacked = .init(sema.arena),
44 };
45 unpack.add(val) catch |err| switch (err) {
46 error.ReinterpretDeclRef => return null,
47 error.OutOfMemory => |e| return e,
48 };
49
50 var pack: PackValueBytes = .{
51 .pt = pt,
52 .arena = sema.arena,
53 .unpacked = unpack.unpacked.items,
54 };
55 return pack.get(dest_ty) catch |err| switch (err) {
56 error.ReinterpretDeclRef => return null,
57 error.OutOfMemory => |e| return e,
58 };
59}
60
61/// Splice the value `splice_val` into `val` at the given `byte_offset`, replacing overlapping bits
62/// and returning the modified value.
63pub fn spliceMemory(
64 sema: *Sema,
65 val: Value,
66 splice_val: Value,
67 byte_offset: u64,
68) CompileError!?Value {
69 const pt = sema.pt;
70 const zcu = pt.zcu;
71 const val_ty = val.typeOf(zcu);
72 const splice_val_ty = splice_val.typeOf(zcu);
73
74 val_ty.assertHasLayout(zcu);
75 splice_val_ty.assertHasLayout(zcu);
76
77 var unpack: UnpackValueBytes = .{
78 .pt = pt,
79 .arena = sema.arena,
80 .skip_bytes = 0,
81 .remaining_bytes = byte_offset,
82 .unpacked = .init(sema.arena),
83 };
84 unpack.add(val) catch |err| switch (err) {
85 error.ReinterpretDeclRef => return null,
86 error.OutOfMemory => |e| return e,
87 };
88
89 const splice_len = splice_val_ty.abiSize(zcu);
90
91 unpack.remaining_bytes = splice_len;
92 unpack.add(splice_val) catch |err| switch (err) {
93 error.ReinterpretDeclRef => return null,
94 error.OutOfMemory => |e| return e,
95 };
96
97 unpack.skip_bytes = byte_offset + splice_len;
98 unpack.remaining_bytes = val_ty.abiSize(zcu) * 8 - byte_offset - splice_len;
99 unpack.add(val) catch |err| switch (err) {
100 error.ReinterpretDeclRef => return null,
101 error.OutOfMemory => |e| return e,
102 };
103
104 var pack: PackValueBytes = .{
105 .pt = pt,
106 .arena = sema.arena,
107 .unpacked = unpack.unpacked.items,
108 };
109 return pack.get(val_ty) catch |err| switch (err) {
110 error.ReinterpretDeclRef => return null,
111 error.OutOfMemory => |e| return e,
112 };
113}
114
115/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
116/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
117/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
118/// list contains bits from the *final* byte of the value.
119const UnpackValueBytes = struct {
120 pt: Zcu.PerThread,
121 arena: Allocator,
122 skip_bytes: u64,
123 remaining_bytes: u64,
124 unpacked: std.array_list.Managed(InternPool.Index),
125
126 fn add(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
127 const pt = unpack.pt;
128 const zcu = pt.zcu;
129 const ip = &zcu.intern_pool;
130
131 if (unpack.remaining_bytes == 0) {
132 return;
133 }
134
135 const ty = val.typeOf(zcu);
136 const size = ty.abiSize(zcu);
137
138 if (unpack.skip_bytes >= size) {
139 unpack.skip_bytes -= size;
140 return;
141 }
142
143 switch (ip.indexToKey(val.toIntern())) {
144 .int_type,
145 .ptr_type,
146 .array_type,
147 .vector_type,
148 .opt_type,
149 .anyframe_type,
150 .error_union_type,
151 .simple_type,
152 .struct_type,
153 .tuple_type,
154 .union_type,
155 .opaque_type,
156 .spirv_type,
157 .enum_type,
158 .func_type,
159 .error_set_type,
160 .inferred_error_set_type,
161 .@"extern",
162 .func,
163 .err,
164 .error_union,
165 .enum_literal,
166 .slice,
167 .memoized_call,
168 => unreachable, // ill-defined layout or not real values
169
170 .undef,
171 .int,
172 .enum_tag,
173 .simple_value,
174 .float,
175 .ptr,
176 .opt,
177 => try unpack.primitive(val),
178
179 .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
180
181 .aggregate => switch (ty.zigTypeTag(zcu)) {
182 .vector => unreachable, // ill-defined layout
183 .array => {
184 for (0..@intCast(ty.arrayLen(zcu))) |elem_index| {
185 const elem_val = try val.elemValue(pt, @intCast(elem_index));
186 try unpack.add(elem_val);
187 }
188 if (ty.sentinel(zcu)) |s| {
189 try unpack.add(s);
190 }
191 },
192 .@"struct" => switch (ty.containerLayout(zcu)) {
193 .auto => unreachable, // ill-defined layout
194 .@"packed" => unreachable, // uses `.bitpack`, not `.aggregate`
195 .@"extern" => {
196 var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
197 var offset: u64 = 0;
198 while (it.next()) |field_index| {
199 const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
200 const field_val = try val.fieldValue(pt, field_index);
201 try unpack.padding(pad_bytes);
202 try unpack.add(field_val);
203 offset += pad_bytes + field_val.typeOf(zcu).abiSize(zcu);
204 }
205 try unpack.padding(size - offset);
206 },
207 },
208 else => unreachable,
209 },
210
211 .un => |un| {
212 const payload_val = Value.fromInterned(un.val);
213 const pad_bytes = size - payload_val.typeOf(zcu).abiSize(zcu);
214 try unpack.add(payload_val);
215 try unpack.padding(pad_bytes);
216 },
217 }
218 }
219
220 fn padding(unpack: *UnpackValueBytes, num_bytes: u64) Allocator.Error!void {
221 if (num_bytes == 0) return;
222 const undef_u8 = try unpack.pt.undefValue(Type.u8);
223 for (0..@intCast(num_bytes)) |_| {
224 unpack.primitive(undef_u8) catch |err| switch (err) {
225 error.OutOfMemory => |e| return e,
226 error.ReinterpretDeclRef => unreachable,
227 };
228 }
229 }
230
231 fn primitive(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
232 const pt = unpack.pt;
233 const zcu = pt.zcu;
234
235 if (unpack.remaining_bytes == 0) {
236 return;
237 }
238
239 const ty = val.typeOf(pt.zcu);
240 const size = ty.abiSize(zcu);
241
242 if (unpack.skip_bytes >= size) {
243 unpack.skip_bytes -= size;
244 return;
245 }
246
247 if (unpack.skip_bytes > 0) {
248 const offset = unpack.skip_bytes;
249 unpack.skip_bytes = 0;
250 return unpack.splitPrimitive(val, offset, @min(size - offset, unpack.remaining_bytes));
251 }
252
253 if (unpack.remaining_bytes < size) {
254 return unpack.splitPrimitive(val, 0, unpack.remaining_bytes);
255 }
256
257 unpack.remaining_bytes -= size;
258 try unpack.unpacked.append(val.toIntern());
259 }
260
261 fn splitPrimitive(unpack: *UnpackValueBytes, val: Value, offset: u64, len: u64) (error{ReinterpretDeclRef} || Allocator.Error)!void {
262 const pt = unpack.pt;
263 const zcu = pt.zcu;
264 const ty = val.typeOf(pt.zcu);
265
266 assert(offset + len <= ty.abiSize(zcu));
267
268 try unpack.unpacked.ensureUnusedCapacity(@intCast(len));
269 unpack.remaining_bytes -= len;
270
271 switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
272 // In the `ptr` case, this will return `error.ReinterpretDeclRef`
273 // if we're trying to split a non-integer pointer value.
274 .int, .float, .enum_tag, .ptr, .opt => {
275 const buf = try unpack.arena.alloc(u8, @intCast(ty.abiSize(zcu)));
276 val.writeToMemory(zcu, buf) catch |err| switch (err) {
277 error.IllDefinedMemoryLayout => unreachable,
278 else => |e| return e,
279 };
280 for (buf[@intCast(offset)..][0..@intCast(len)]) |byte_raw| {
281 const byte_val = try pt.intValue(.u8, byte_raw);
282 unpack.unpacked.appendAssumeCapacity(byte_val.toIntern());
283 }
284 },
285 .undef => {
286 const undef_u8 = try pt.undefValue(.u8);
287 for (0..@intCast(len)) |_| {
288 unpack.unpacked.appendAssumeCapacity(undef_u8.toIntern());
289 }
290 },
291 // The only values here with runtime bits are `true` and `false`.
292 // These are both 1 byte, so will never need splitting.
293 .simple_value => unreachable,
294 else => unreachable, // zero-bit or not primitives
295 }
296 }
297};
298
299/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBytes`),
300/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
301/// values and of pointers which align in virtual memory.
302const PackValueBytes = struct {
303 pt: Zcu.PerThread,
304 arena: Allocator,
305 byte_offset: u64 = 0,
306 unpacked: []const InternPool.Index,
307
308 fn get(pack: *PackValueBytes, ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
309 const pt = pack.pt;
310 const zcu = pt.zcu;
311 const ip = &zcu.intern_pool;
312 const arena = pack.arena;
313 switch (ty.zigTypeTag(zcu)) {
314 .vector => unreachable, // ill-defined layout
315 .array => {
316 // Each element is padded up to its ABI size. The final element does not have trailing padding.
317 const elem_ty = ty.childType(zcu);
318 const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
319
320 for (elems) |*elem| {
321 elem.* = (try pack.get(elem_ty)).toIntern();
322 }
323
324 if (ty.sentinel(zcu)) |s| {
325 _ = s; // TODO: validate sentinel was preserved!
326 pack.padding(elem_ty.abiSize(zcu));
327 }
328
329 return pt.aggregateValue(ty, elems);
330 },
331 .@"struct" => switch (ty.containerLayout(zcu)) {
332 .auto => unreachable, // ill-defined layout
333 .@"extern" => {
334 const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
335 @memset(elems, .none);
336 var offset: u64 = 0;
337 var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
338 while (it.next()) |field_index| {
339 const field_ty = ty.fieldType(field_index, zcu);
340 const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
341 pack.padding(pad_bytes);
342 elems[field_index] = (try pack.get(field_ty)).toIntern();
343 offset += pad_bytes + field_ty.abiSize(zcu);
344 }
345 pack.padding(ty.abiSize(zcu) - offset);
346 // Any fields which do not have runtime bits should be OPV or comptime fields.
347 // Fill those values now.
348 for (elems, 0..) |*elem, field_index| {
349 if (elem.* != .none) continue;
350 const val = (try ty.structFieldValueComptime(pt, field_index)).?;
351 elem.* = val.toIntern();
352 }
353 return pt.aggregateValue(ty, elems);
354 },
355 .@"packed" => {
356 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
357 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
358 return pt.bitpackValue(ty, backing_int_val);
359 },
360 },
361 .@"union" => switch (ty.containerLayout(zcu)) {
362 .auto => unreachable, // ill-defined layout
363 .@"extern" => {
364 // We will attempt to read as the backing representation. If this emits
365 // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
366 // We will also attempt smaller fields when we get `undefined`, as if some bits are
367 // defined we want to include them.
368 // TODO: this is very very bad. We need a more sophisticated union representation.
369
370 const prev_unpacked = pack.unpacked;
371 const prev_byte_offset = pack.byte_offset;
372
373 const backing_ty = try ty.externUnionBackingType(pt);
374
375 const backing_result: enum { undef, reinterpret_decl_ref } = backing: {
376 const backing_val = pack.get(backing_ty) catch |err| switch (err) {
377 error.ReinterpretDeclRef => break :backing .reinterpret_decl_ref,
378 else => |e| return e,
379 };
380 if (backing_val.isUndef(zcu)) break :backing .undef;
381 return .fromInterned(try pt.internUnion(.{
382 .ty = ty.toIntern(),
383 .tag = .none,
384 .val = backing_val.toIntern(),
385 }));
386 };
387
388 const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
389 for (field_order, 0..) |*f, i| f.* = @intCast(i);
390 // Sort `field_order` to put the fields with the largest ABI sizes first.
391 const SizeSortCtx = struct {
392 zcu: *const Zcu,
393 field_types: []const InternPool.Index,
394 fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
395 const a_ty: Type = .fromInterned(ctx.field_types[a_idx]);
396 const b_ty: Type = .fromInterned(ctx.field_types[b_idx]);
397 return a_ty.abiSize(ctx.zcu) > b_ty.abiSize(ctx.zcu);
398 }
399 };
400 std.mem.sortUnstable(u32, field_order, SizeSortCtx{
401 .zcu = zcu,
402 .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
403 }, SizeSortCtx.lessThan);
404
405 for (field_order) |field_index| {
406 pack.unpacked = prev_unpacked;
407 pack.byte_offset = prev_byte_offset;
408 const field_ty = ty.fieldType(field_index, zcu);
409 const field_val = pack.get(field_ty) catch |err| switch (err) {
410 error.ReinterpretDeclRef => continue,
411 else => |e| return e,
412 };
413 if (field_val.isUndef(zcu)) continue;
414 pack.padding(ty.abiSize(zcu) - field_ty.abiSize(zcu));
415 const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_index);
416 return pt.unionValue(ty, tag_val, field_val);
417 }
418
419 // No field could represent the value. Just do whatever happens when we try to read
420 // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
421 switch (backing_result) {
422 .undef => return pt.undefValue(ty),
423 .reinterpret_decl_ref => return error.ReinterpretDeclRef,
424 }
425 },
426 .@"packed" => {
427 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
428 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
429 return pt.bitpackValue(ty, backing_int_val);
430 },
431 },
432 .@"enum" => {
433 const tag_int_val = try pack.primitive(ty.intTagType(zcu));
434 if (tag_int_val.isUndef(zcu)) return pt.undefValue(ty);
435 return pt.enumValue(ty, tag_int_val.toIntern());
436 },
437 else => return pack.primitive(ty),
438 }
439 }
440
441 fn padding(pack: *PackValueBytes, num_bytes: u64) void {
442 _ = pack.prepareBytes(num_bytes);
443 }
444
445 fn primitive(pack: *PackValueBytes, want_ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
446 const pt = pack.pt;
447 const zcu = pt.zcu;
448
449 if (try want_ty.onePossibleValue(pt)) |opv| return opv;
450
451 const vals, const byte_offset = pack.prepareBytes(want_ty.abiSize(zcu));
452
453 for (vals) |val| {
454 if (!Value.fromInterned(val).isUndef(zcu)) break;
455 } else {
456 // All bits of the value are `undefined`.
457 return pt.undefValue(want_ty);
458 }
459
460 // TODO: we need to decide how to handle partially-undef values here.
461 // Currently, a value with some undefined bits becomes `0xAA` so that we
462 // preserve the well-defined bits, because we can't currently represent
463 // a partially-undefined primitive (e.g. an int with some undef bits).
464 // In future, we probably want to take one of these two routes:
465 // * Define that if any bits are `undefined`, the entire value is `undefined`.
466 // This is a major breaking change, and probably a footgun.
467 // * Introduce tracking for partially-undef values at comptime.
468 // This would complicate a lot of operations in Sema, such as basic
469 // arithmetic.
470 // This design complexity is tracked by #19634.
471
472 if (vals.len == 1 and
473 want_ty.isPtrAtRuntime(zcu) and
474 Value.fromInterned(vals[0]).typeOf(zcu).isPtrAtRuntime(zcu))
475 {
476 return pt.getCoerced(.fromInterned(vals[0]), want_ty);
477 }
478
479 // Reinterpret via an in-memory buffer.
480
481 var buf_len: u64 = 0;
482 for (vals) |ip_val| {
483 const val: Value = .fromInterned(ip_val);
484 buf_len += val.typeOf(zcu).abiSize(zcu);
485 }
486
487 const buf = try pack.arena.alloc(u8, @intCast(buf_len));
488 {
489 var offset: usize = 0;
490 for (vals) |ip_val| {
491 const val: Value = .fromInterned(ip_val);
492 const ty = val.typeOf(zcu);
493 const size = ty.abiSize(zcu);
494 if (val.isUndef(zcu)) {
495 @memset(buf[offset..][0..@intCast(size)], 0xAA);
496 } else {
497 val.writeToMemory(zcu, buf[offset..][0..@intCast(size)]) catch |err| switch (err) {
498 error.IllDefinedMemoryLayout => unreachable,
499 else => |e| return e,
500 };
501 }
502 offset += @intCast(size);
503 }
504 }
505 const bytes = buf[@intCast(byte_offset)..];
506
507 const target = zcu.getTarget();
508 const endian = target.cpu.arch.endian();
509 switch (want_ty.zigTypeTag(zcu)) {
510 .bool => return .makeBool(bytes[0] != 0),
511 .int => return .readIntFromMemory(want_ty, pt, bytes, pack.arena),
512 .float => switch (want_ty.floatBits(target)) {
513 16 => return pt.floatValue(want_ty, @as(f16, @bitCast(std.mem.readInt(u16, bytes[0..2], endian)))),
514 32 => return pt.floatValue(want_ty, @as(f32, @bitCast(std.mem.readInt(u32, bytes[0..4], endian)))),
515 64 => return pt.floatValue(want_ty, @as(f64, @bitCast(std.mem.readInt(u64, bytes[0..8], endian)))),
516 80 => return pt.floatValue(want_ty, @as(f80, @bitCast(std.mem.readInt(u80, bytes[0..10], endian)))),
517 128 => return pt.floatValue(want_ty, @as(f128, @bitCast(std.mem.readInt(u128, bytes[0..16], endian)))),
518 else => unreachable,
519 },
520 .pointer => {
521 assert(!want_ty.isSlice(zcu));
522 const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
523 return pt.ptrIntValue(want_ty, ptr_addr);
524 },
525 .optional => {
526 assert(want_ty.isPtrLikeOptional(zcu));
527 const ptr_ty = want_ty.optionalChild(zcu);
528 const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
529 return .fromInterned(try pt.intern(.{ .opt = .{
530 .ty = want_ty.toIntern(),
531 .val = if (ptr_addr == 0) .none else (try pt.ptrIntValue(ptr_ty, ptr_addr)).toIntern(),
532 } }));
533 },
534 else => unreachable,
535 }
536 }
537
538 fn prepareBytes(pack: *PackValueBytes, need_bytes: u64) struct { []const InternPool.Index, u64 } {
539 if (need_bytes == 0) return .{ &.{}, 0 };
540
541 const pt = pack.pt;
542 const zcu = pt.zcu;
543
544 var bytes: u64 = 0;
545 var len: usize = 0;
546 while (bytes < pack.byte_offset + need_bytes) {
547 bytes += Value.fromInterned(pack.unpacked[len]).typeOf(zcu).abiSize(zcu);
548 len += 1;
549 }
550
551 const result_vals = pack.unpacked[0..len];
552 const result_offset = pack.byte_offset;
553
554 const extra_bytes = bytes - pack.byte_offset - need_bytes;
555 if (extra_bytes == 0) {
556 pack.unpacked = pack.unpacked[len..];
557 pack.byte_offset = 0;
558 } else {
559 pack.unpacked = pack.unpacked[len - 1 ..];
560 pack.byte_offset = Value.fromInterned(pack.unpacked[0]).typeOf(zcu).abiSize(zcu) - extra_bytes;
561 }
562
563 return .{ result_vals, result_offset };
564 }
565};
566
567const std = @import("std");
568const Allocator = std.mem.Allocator;
569const assert = std.debug.assert;
570
571const Sema = @import("../Sema.zig");
572const Zcu = @import("../Zcu.zig");
573const InternPool = @import("../InternPool.zig");
574const Type = @import("../Type.zig");
575const Value = @import("../Value.zig");
576const CompileError = Zcu.CompileError;
src/Type.zig+47-106
...@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {...@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {
757 const ip = &zcu.intern_pool;757 const ip = &zcu.intern_pool;
758 return switch (ip.indexToKey(ty.toIntern())) {758 return switch (ip.indexToKey(ty.toIntern())) {
759 .int_type,759 .int_type,
760 .vector_type,
761 => true,760 => true,
762761
762 .vector_type,
763 .error_union_type,763 .error_union_type,
764 .error_set_type,764 .error_set_type,
765 .inferred_error_set_type,765 .inferred_error_set_type,
...@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {...@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {
1241}1241}
12421242
1243/// Asserts that `ty` is not an opaque or comptime-only type.1243/// Asserts that `ty` is not an opaque or comptime-only type.
1244/// Once #19755 is implemented, this query will only work on types with a defined bit-level representation.
1245pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {1244pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {
1246 const target = zcu.getTarget();1245 return switch (ty.zigTypeTag(zcu)) {
1247 const ip = &zcu.intern_pool;1246 .void => 0,
1248 assertHasLayout(ty, zcu);1247 .bool => 1,
1249 return switch (ip.indexToKey(ty.toIntern())) {1248 .float => ty.floatBits(zcu.getTarget()),
1250 .int_type => |int_type| int_type.bits,1249 .pointer, .optional => {
1251 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {1250 assert(ty.isPtrAtRuntime(zcu));
1252 .slice => target.ptrBitWidth() * 2,1251 return zcu.getTarget().ptrBitWidth();
1253 else => target.ptrBitWidth(),
1254 },
1255 .anyframe_type => target.ptrBitWidth(),
1256 .array_type => |array_type| {
1257 const elem_ty: Type = .fromInterned(array_type.child);
1258 const len = array_type.lenIncludingSentinel();
1259 return switch (zcu.comp.getZigBackend()) {
1260 .stage2_x86_64 => len * elem_ty.bitSize(zcu),
1261 // this case will be removed under #19755
1262 else => switch (len) {
1263 0 => 0,
1264 else => (len - 1) * 8 * elem_ty.abiSize(zcu) + elem_ty.bitSize(zcu),
1265 },
1266 };
1267 },
1268 .vector_type => |vec| vec.len * Type.fromInterned(vec.child).bitSize(zcu),
1269 .error_set_type, .inferred_error_set_type => zcu.errorSetBits(),
1270 .func_type => unreachable,
1271
1272 .simple_type => |t| switch (t) {
1273 .void => 0,
1274 .bool => 1,
1275 .anyerror, .adhoc_inferred_error_set => zcu.errorSetBits(),
1276 .usize, .isize => target.ptrBitWidth(),
1277
1278 .c_char => target.cTypeBitSize(.char),
1279 .c_short => target.cTypeBitSize(.short),
1280 .c_ushort => target.cTypeBitSize(.ushort),
1281 .c_int => target.cTypeBitSize(.int),
1282 .c_uint => target.cTypeBitSize(.uint),
1283 .c_long => target.cTypeBitSize(.long),
1284 .c_ulong => target.cTypeBitSize(.ulong),
1285 .c_longlong => target.cTypeBitSize(.longlong),
1286 .c_ulonglong => target.cTypeBitSize(.ulonglong),
1287 .c_longdouble => target.cTypeBitSize(.longdouble),
1288
1289 .f16 => 16,
1290 .f32 => 32,
1291 .f64 => 64,
1292 .f80 => 80,
1293 .f128 => 128,
1294
1295 .anyopaque => unreachable,
1296 .type => unreachable,
1297 .comptime_int => unreachable,
1298 .comptime_float => unreachable,
1299 .noreturn => unreachable,
1300 .null => unreachable,
1301 .undefined => unreachable,
1302 .enum_literal => unreachable,
1303 .generic_poison => unreachable,
1304 },
1305
1306 .struct_type => {
1307 const struct_obj = ip.loadStructType(ty.toIntern());
1308 switch (struct_obj.layout) {
1309 .@"packed" => return Type.fromInterned(struct_obj.packed_backing_int_type).bitSize(zcu),
1310 .auto, .@"extern" => return struct_obj.size * 8, // will be `unreachable` under #19755
1311 }
1312 },
1313 .union_type => {
1314 const union_obj = ip.loadUnionType(ty.toIntern());
1315 switch (union_obj.layout) {
1316 .@"packed" => return Type.fromInterned(union_obj.packed_backing_int_type).bitSize(zcu),
1317 .auto, .@"extern" => return union_obj.size * 8, // will be `unreachable` under #19755
1318 }
1319 },1252 },
1320 .enum_type => Type.fromInterned(ip.loadEnumType(ty.toIntern()).int_tag_type).bitSize(zcu),1253 .array, .vector => ty.arrayLenIncludingSentinel(zcu) * ty.childType(zcu).bitSize(zcu),
13211254 else => ty.intInfo(zcu).bits,
1322 // will be `unreachable` under #19755
1323 .opt_type,
1324 .error_union_type,
1325 .tuple_type,
1326 => ty.abiSize(zcu) * 8,
1327
1328 .opaque_type, .spirv_type => unreachable,
1329
1330 // values, not types
1331 .undef,
1332 .simple_value,
1333 .@"extern",
1334 .func,
1335 .int,
1336 .err,
1337 .error_union,
1338 .enum_literal,
1339 .enum_tag,
1340 .float,
1341 .ptr,
1342 .slice,
1343 .opt,
1344 .aggregate,
1345 .un,
1346 .bitpack,
1347 // memoization, not types
1348 .memoized_call,
1349 => unreachable,
1350 };1255 };
1351}1256}
13521257
...@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {...@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {
1528/// * `[*]T`1433/// * `[*]T`
1529/// * `[*c]T`1434/// * `[*c]T`
1530/// * `@SpirvType(.{ .runtime_array = T })`1435/// * `@SpirvType(.{ .runtime_array = T })`
1436/// * `*@SpirvType(.{ .runtime_array = T })`
1531pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {1437pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {
1532 const ip = &zcu.intern_pool;1438 const ip = &zcu.intern_pool;
1533 return switch (ip.indexToKey(ty.toIntern())) {1439 return switch (ip.indexToKey(ty.toIntern())) {
...@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool...@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3181 .frame,3087 .frame,
3182 => false,3088 => false,
31833089
3090 .vector => position == .param_ty or position == .ret_ty,
3091
3184 .void => switch (position) {3092 .void => switch (position) {
3185 .ret_ty,3093 .ret_ty,
3186 .union_field,3094 .union_field,
...@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool...@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3259 .other,3167 .other,
3260 => ty.childType(zcu).validateExtern(.element, zcu),3168 => ty.childType(zcu).validateExtern(.element, zcu),
3261 },3169 },
3262 .vector => ty.childType(zcu).validateExtern(.element, zcu),
3263 .optional => ty.isPtrLikeOptional(zcu),3170 .optional => ty.isPtrLikeOptional(zcu),
3264 };3171 };
3265}3172}
...@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {...@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {
3272 };3179 };
3273}3180}
32743181
3182/// Returns whether `ty` is considered by Zig to have a bit-level representation, meaning it is
3183/// allowed as the operand to `@bitSizeOf`. This is a superset of packable types.
3184pub fn hasBitRepresentation(ty: Type, zcu: *const Zcu) bool {
3185 return switch (ty.zigTypeTag(zcu)) {
3186 .@"fn",
3187 .noreturn,
3188 .undefined,
3189 .null,
3190 .@"opaque",
3191 .spirv,
3192 .type,
3193 .enum_literal,
3194 .comptime_float,
3195 .comptime_int,
3196 .error_set,
3197 .error_union,
3198 .frame,
3199 .@"anyframe",
3200 => false,
3201
3202 .void,
3203 .bool,
3204 .int,
3205 .float,
3206 => true,
3207
3208 .@"enum" => zcu.intern_pool.loadEnumType(ty.toIntern()).int_tag_mode == .explicit,
3209 .pointer, .optional => ty.isPtrAtRuntime(zcu),
3210 .@"struct", .@"union" => ty.containerLayout(zcu) == .@"packed",
3211
3212 .array, .vector => ty.childType(zcu).hasBitRepresentation(zcu),
3213 };
3214}
3215
3275/// Asserts that `ty` has resolved layout.3216/// Asserts that `ty` has resolved layout.
3276pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {3217pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {
3277 if (!std.debug.runtime_safety) {3218 if (!std.debug.runtime_safety) {
src/Value.zig+135-134
...@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {...@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {
248pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{248pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
249 ReinterpretDeclRef,249 ReinterpretDeclRef,
250 IllDefinedMemoryLayout,250 IllDefinedMemoryLayout,
251 Unimplemented,
252 OutOfMemory,251 OutOfMemory,
253}!void {252}!void {
254 const target = zcu.getTarget();253 const target = zcu.getTarget();
...@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
257 const ty = val.typeOf(zcu);256 const ty = val.typeOf(zcu);
258 if (val.isUndef(zcu)) {257 if (val.isUndef(zcu)) {
259 const size: usize = @intCast(ty.abiSize(zcu));258 const size: usize = @intCast(ty.abiSize(zcu));
260 @memset(buffer[0..size], 0xaa);259 @memset(buffer[0..size], 0xAA);
261 return;260 return;
262 }261 }
263 switch (ty.zigTypeTag(zcu)) {262 tag: switch (ty.zigTypeTag(zcu)) {
263 .type => return error.IllDefinedMemoryLayout,
264 .comptime_float => return error.IllDefinedMemoryLayout,
265 .comptime_int => return error.IllDefinedMemoryLayout,
266 .undefined => return error.IllDefinedMemoryLayout,
267 .null => return error.IllDefinedMemoryLayout,
268 .error_union => return error.IllDefinedMemoryLayout,
269 .enum_literal => return error.IllDefinedMemoryLayout,
270 .@"fn" => return error.IllDefinedMemoryLayout,
271 .spirv => return error.IllDefinedMemoryLayout,
272 .@"opaque" => unreachable,
273 .frame => unreachable,
274 .@"anyframe" => unreachable,
275 .noreturn => unreachable,
264 .void => {},276 .void => {},
265 .bool => {277 .bool => {
266 buffer[0] = @intFromBool(val.toBool());278 buffer[0] = @intFromBool(val.toBool());
267 },279 },
268 .int, .@"enum", .error_set, .pointer => |tag| {280 .pointer => {
269 const int_ty = if (tag == .pointer) int_ty: {281 if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
270 if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;282 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
271 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;283 continue :tag .int;
272 break :int_ty Type.usize;284 },
273 } else ty;285 .int, .@"enum", .error_set => {
274 const int_info = int_ty.intInfo(zcu);
275 const bits = int_info.bits;
276 const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
277
278 var bigint_buffer: BigIntSpace = undefined;286 var bigint_buffer: BigIntSpace = undefined;
279 const bigint = val.toBigInt(&bigint_buffer, zcu);287 const bigint = val.toBigInt(&bigint_buffer, zcu);
280 bigint.writeTwosComplement(buffer[0..byte_count], endian);288 bigint.writeTwosComplement(buffer[0..@intCast(ty.abiSize(zcu))], endian);
281 },289 },
282 .float => switch (ty.floatBits(target)) {290 .float => {
283 16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),291 const float_bits = ty.floatBits(target);
284 32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),292 switch (float_bits) {
285 64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),293 16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
286 80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),294 32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
287 128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),295 64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
288 else => unreachable,296 80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
297 128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
298 else => unreachable,
299 }
300 const float_bytes = @divExact(float_bits, 8);
301 const total_bytes: usize = @intCast(ty.abiSize(zcu));
302 @memset(buffer[float_bytes..total_bytes], 0); // padding
289 },303 },
290 .array => {304 .array => {
291 const aggregate = ip.indexToKey(val.toIntern()).aggregate;305 const aggregate = ip.indexToKey(val.toIntern()).aggregate;
...@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
302 }316 }
303 buf_off += elem_size;317 buf_off += elem_size;
304 }318 }
319 if (ty.sentinel(zcu)) |sentinel_val| {
320 try sentinel_val.writeToMemory(zcu, buffer[buf_off..]);
321 }
305 },322 },
306 .vector => {323 .vector => return error.IllDefinedMemoryLayout,
307 // We use byte_count instead of abi_size here, so that any padding bytes
308 // follow the data bytes, on both big- and little-endian systems.
309 const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
310 return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
311 },
312 .@"struct" => {324 .@"struct" => {
313 const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;325 const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
314 switch (struct_type.layout) {326 switch (struct_type.layout) {
315 .auto => return error.IllDefinedMemoryLayout,327 .auto => return error.IllDefinedMemoryLayout,
316 .@"extern" => for (0..struct_type.field_types.len) |field_index| {328 .@"extern" => {
317 const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));329 var last_off: usize = 0;
318 const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {330 for (struct_type.field_types.get(ip), 0..) |field_ty_ip, field_index| {
319 .bytes => |bytes| {331 const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
320 buffer[off] = bytes.at(field_index, ip);332 @memset(buffer[last_off..off], 0xAA);
321 continue;333 const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
322 },334 .bytes => |bytes| {
323 .elems => |elems| elems[field_index],335 buffer[off] = bytes.at(field_index, ip);
324 .repeated_elem => |elem| elem,336 continue;
325 });337 },
326 try writeToMemory(field_val, zcu, buffer[off..]);338 .elems => |elems| elems[field_index],
339 .repeated_elem => |elem| elem,
340 });
341 try writeToMemory(field_val, zcu, buffer[off..]);
342 last_off = @intCast(off + Type.fromInterned(field_ty_ip).abiSize(zcu));
343 }
344 const struct_size: usize = @intCast(ty.abiSize(zcu));
345 @memset(buffer[last_off..struct_size], 0xAA);
327 },346 },
328 .@"packed" => {347 .@"packed" => {
329 const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;348 const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
...@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
335 .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already354 .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
336 .@"extern" => {355 .@"extern" => {
337 const payload_val = val.unionPayload(zcu);356 const payload_val = val.unionPayload(zcu);
357 const payload_size: usize = @intCast(payload_val.typeOf(zcu).abiSize(zcu));
358 const union_size: usize = @intCast(ty.abiSize(zcu));
359 @memset(buffer[payload_size..union_size], 0xAA);
338 return writeToMemory(payload_val, zcu, buffer);360 return writeToMemory(payload_val, zcu, buffer);
339 },361 },
340 .@"packed" => {362 .@"packed" => {
...@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
352 @memset(buffer[0..@intCast(byte_count)], 0); // null pointer374 @memset(buffer[0..@intCast(byte_count)], 0); // null pointer
353 }375 }
354 },376 },
355 else => return error.Unimplemented,
356 }377 }
357}378}
358379
...@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
360///381///
361/// Both the start and the end of the provided buffer must be tight, since382/// Both the start and the end of the provided buffer must be tight, since
362/// big-endian packed memory layouts start at the end of the buffer.383/// big-endian packed memory layouts start at the end of the buffer.
384///
385/// Supports arrays and vectors, for which the value is written in logical bit
386/// order, i.e. with the first element at bit offset 0.
363pub fn writeToPackedMemory(387pub fn writeToPackedMemory(
364 val: Value,388 val: Value,
365 zcu: *const Zcu,389 zcu: *const Zcu,
366 buffer: []u8,390 buffer: []u8,
367 bit_offset: usize,391 bit_offset: usize,
368) error{ ReinterpretDeclRef, OutOfMemory }!void {392) void {
369 const ip = &zcu.intern_pool;393 const ip = &zcu.intern_pool;
370 const target = zcu.getTarget();394 const target = zcu.getTarget();
371 const endian = target.cpu.arch.endian();395 const endian = target.cpu.arch.endian();
...@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(...@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(
392 },416 },
393 .@"enum" => {417 .@"enum" => {
394 const int_val = val.intFromEnum(zcu);418 const int_val = val.intFromEnum(zcu);
395 return int_val.writeToPackedMemory(zcu, buffer, bit_offset);419 int_val.writeToPackedMemory(zcu, buffer, bit_offset);
396 },
397 .pointer => {
398 assert(!ty.isSlice(zcu)); // No well defined layout.
399 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
400 const addr = val.toUnsignedInt(zcu);
401 std.mem.writeVarPackedInt(buffer, bit_offset, zcu.getTarget().ptrBitWidth(), addr, endian);
402 },420 },
403 .int => {421 .int => {
404 const bits = ty.intInfo(zcu).bits;422 const bits = ty.intInfo(zcu).bits;
...@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(...@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(
416 128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),434 128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),
417 else => unreachable,435 else => unreachable,
418 },436 },
419 .vector => {
420 const elem_ty = ty.childType(zcu);
421 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
422 const len: usize = @intCast(ty.arrayLen(zcu));
423
424 var bits: u16 = 0;
425 var elem_i: usize = 0;
426 const aggregate = ip.indexToKey(val.toIntern()).aggregate;
427 while (elem_i < len) : (elem_i += 1) {
428 // On big-endian systems, LLVM reverses the element order of vectors by default
429 const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
430 switch (aggregate.storage) {
431 .bytes => |bytes| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
432 .elems => |elems| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
433 .repeated_elem => |elem| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
434 }
435 bits += elem_bit_size;
436 }
437 },
438 .@"struct", .@"union" => {437 .@"struct", .@"union" => {
439 assert(ty.containerLayout(zcu) == .@"packed");438 assert(ty.containerLayout(zcu) == .@"packed");
440 const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);439 const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
441 return int_val.writeToPackedMemory(zcu, buffer, bit_offset);440 int_val.writeToPackedMemory(zcu, buffer, bit_offset);
442 },441 },
443 .optional => {442 .array, .vector => {
444 assert(ty.isPtrLikeOptional(zcu));443 const elem_bits: usize = @intCast(ty.childType(zcu).bitSize(zcu));
445 if (val.optionalValue(zcu)) |ptr_val| {444 const len: usize = @intCast(ty.arrayLen(zcu));
446 return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);445 var elem_bit_off: usize = bit_offset;
447 } else {446 switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
448 return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);447 .repeated_elem => |elem_val_ip| {
448 const elem_val: Value = .fromInterned(elem_val_ip);
449 for (0..len) |_| {
450 elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
451 elem_bit_off += elem_bits;
452 }
453 },
454 .elems => |elems| for (elems[0..len]) |elem_val_ip| {
455 const elem_val: Value = .fromInterned(elem_val_ip);
456 elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
457 elem_bit_off += elem_bits;
458 },
459 .bytes => |bytes| for (bytes.toSlice(len, ip)) |raw_byte| {
460 std.mem.writeVarPackedInt(buffer, elem_bit_off, elem_bits, raw_byte, endian);
461 elem_bit_off += elem_bits;
462 },
463 }
464 if (ty.sentinel(zcu)) |sentinel_val| {
465 sentinel_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
449 }466 }
450 },467 },
451 else => @panic("TODO implement writeToPackedMemory for more types"),468 else => unreachable,
452 }469 }
453}470}
454471
455/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.472/// Load a Value from the contents of `buffer`, where `ty` is any integer type.
456///473///
457/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past474/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
458/// the end of the value in memory.475/// the end of the value in memory.
459pub fn readUintFromMemory(476pub fn readIntFromMemory(
460 ty: Type,477 ty: Type,
461 pt: Zcu.PerThread,478 pt: Zcu.PerThread,
462 buffer: []const u8,479 buffer: []const u8,
...@@ -465,23 +482,28 @@ pub fn readUintFromMemory(...@@ -465,23 +482,28 @@ pub fn readUintFromMemory(
465 const zcu = pt.zcu;482 const zcu = pt.zcu;
466 const endian = zcu.getTarget().cpu.arch.endian();483 const endian = zcu.getTarget().cpu.arch.endian();
467484
468 assert(ty.isUnsignedInt(zcu));485 const int = ty.intInfo(zcu);
469 const bits = ty.intInfo(zcu).bits;486 const abi_size: usize = @intCast(ty.abiSize(zcu));
470 const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);487 const exact_buf = buffer[0..abi_size];
471488
472 assert(buffer.len >= byte_count);489 if (abi_size <= 8) {
473490 const shift: u6 = @intCast(64 - int.bits);
474 if (bits <= 64) {491 switch (int.signedness) {
475 const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);492 .unsigned => {
476 const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));493 const x = std.mem.readVarInt(u64, exact_buf, endian);
477 return pt.intValue(ty, result);494 return pt.intValue(ty, (x << shift) >> shift);
495 },
496 .signed => {
497 const x = std.mem.readVarInt(i64, exact_buf, endian);
498 return pt.intValue(ty, (x << shift) >> shift);
499 },
500 }
478 } else {501 } else {
479 const Limb = std.math.big.Limb;502 const limb_count = std.math.big.int.calcTwosCompLimbCount(int.bits);
480 const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);503 const limbs_buffer = try arena.alloc(std.math.big.Limb, limb_count);
481 const limbs_buffer = try arena.alloc(Limb, limb_count);
482504
483 var bigint: BigIntMutable = .init(limbs_buffer, 0);505 var bigint: BigIntMutable = .init(limbs_buffer, 0);
484 bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);506 bigint.readTwosComplement(exact_buf, int.bits, endian, int.signedness);
485 return pt.intValue_big(ty, bigint.toConst());507 return pt.intValue_big(ty, bigint.toConst());
486 }508 }
487}509}
...@@ -490,17 +512,17 @@ pub fn readUintFromMemory(...@@ -490,17 +512,17 @@ pub fn readUintFromMemory(
490///512///
491/// Both the start and the end of the provided buffer must be tight, since513/// Both the start and the end of the provided buffer must be tight, since
492/// big-endian packed memory layouts start at the end of the buffer.514/// big-endian packed memory layouts start at the end of the buffer.
515///
516/// Supports arrays and vectors, for which the value is read in logical bit
517/// order, i.e. with the first element at bit offset 0.
493pub fn readFromPackedMemory(518pub fn readFromPackedMemory(
494 ty: Type,519 ty: Type,
495 pt: Zcu.PerThread,520 pt: Zcu.PerThread,
496 buffer: []const u8,521 buffer: []const u8,
497 bit_offset: usize,522 bit_offset: usize,
498 gpa: Allocator,523) Allocator.Error!Value {
499) error{
500 IllDefinedMemoryLayout,
501 OutOfMemory,
502}!Value {
503 const zcu = pt.zcu;524 const zcu = pt.zcu;
525 const gpa = zcu.comp.gpa;
504 const target = zcu.getTarget();526 const target = zcu.getTarget();
505 const endian = target.cpu.arch.endian();527 const endian = target.cpu.arch.endian();
506 switch (ty.zigTypeTag(zcu)) {528 switch (ty.zigTypeTag(zcu)) {
...@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(...@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(
543 },565 },
544 .@"enum" => {566 .@"enum" => {
545 const int_ty = ty.intTagType(zcu);567 const int_ty = ty.intTagType(zcu);
546 const int_val = try Value.readFromPackedMemory(int_ty, pt, buffer, bit_offset, gpa);568 const int_val: Value = try .readFromPackedMemory(int_ty, pt, buffer, bit_offset);
547 return pt.getCoerced(int_val, ty);569 return pt.getCoerced(int_val, ty);
548 },570 },
549 .float => return Value.fromInterned(try pt.intern(.{ .float = .{571 .float => return Value.fromInterned(try pt.intern(.{ .float = .{
...@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(...@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(
557 else => unreachable,579 else => unreachable,
558 },580 },
559 } })),581 } })),
560 .vector => {
561 const elem_ty = ty.childType(zcu);
562 const elems = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
563 defer gpa.free(elems);
564
565 var bits: u16 = 0;
566 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
567 for (elems, 0..) |_, i| {
568 // On big-endian systems, LLVM reverses the element order of vectors by default
569 const tgt_elem_i = if (endian == .big) elems.len - i - 1 else i;
570 elems[tgt_elem_i] = (try readFromPackedMemory(elem_ty, pt, buffer, bit_offset + bits, gpa)).toIntern();
571 bits += elem_bit_size;
572 }
573 return pt.aggregateValue(ty, elems);
574 },
575 .@"struct", .@"union" => {582 .@"struct", .@"union" => {
576 assert(ty.containerLayout(zcu) == .@"packed");583 assert(ty.containerLayout(zcu) == .@"packed");
577 const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset, gpa);584 const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset);
578 return pt.bitpackValue(ty, int_val);585 return pt.bitpackValue(ty, int_val);
579 },586 },
580 .pointer => {587 .array, .vector => {
581 assert(!ty.isSlice(zcu)); // No well defined layout.588 const elem_ty = ty.childType(zcu);
582 const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);589 const elem_bits: usize = @intCast(elem_ty.bitSize(zcu));
583 return pt.ptrIntValue(ty, addr);590 const elems_buf = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
584 },591 defer gpa.free(elems_buf);
585 .optional => {592 var elem_bit_off: usize = bit_offset;
586 assert(ty.isPtrLikeOptional(zcu));593 for (elems_buf) |*elem| {
587 const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);594 const elem_val = try readFromPackedMemory(elem_ty, pt, buffer, elem_bit_off);
588 return .fromInterned(try pt.intern(.{ .opt = .{595 elem.* = elem_val.toIntern();
589 .ty = ty.toIntern(),596 elem_bit_off += elem_bits;
590 .val = if (addr == 0) .none else (try pt.ptrIntValue(ty.childType(zcu), addr)).toIntern(),597 }
591 } }));598 return pt.aggregateValue(ty, elems_buf);
592 },599 },
593 else => @panic("TODO implement readFromPackedMemory for more types"),600 else => unreachable,
594 }601 }
595}602}
596603
...@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {...@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
887 const bfa = bfa_state.allocator();894 const bfa = bfa_state.allocator();
888 const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));895 const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
889 defer bfa.free(buf);896 defer bfa.free(buf);
890 int_val.writeToPackedMemory(zcu, buf, 0) catch |err| switch (err) {897 @memset(buf, 0);
891 error.ReinterpretDeclRef => unreachable, // it's an integer898 int_val.writeToPackedMemory(zcu, buf, 0);
892 error.OutOfMemory => |e| return e,899 return .readFromPackedMemory(field_ty, pt, buf, field_bit_offset);
893 };
894 return Value.readFromPackedMemory(field_ty, pt, buf, field_bit_offset, bfa) catch |err| switch (err) {
895 error.IllDefinedMemoryLayout => unreachable, // it's a bitpack
896 error.OutOfMemory => |e| return e,
897 };
898 },900 },
899 else => unreachable,901 else => unreachable,
900 };902 };
...@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {...@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
1619 // code late in compilation. So, this error handling is too aggressive and1621 // code late in compilation. So, this error handling is too aggressive and
1620 // causes some false negatives, causing less-than-ideal code generation.1622 // causes some false negatives, causing less-than-ideal code generation.
1621 error.IllDefinedMemoryLayout => return null,1623 error.IllDefinedMemoryLayout => return null,
1622 error.Unimplemented => return null,
1623 };1624 };
1624 const first_byte = byte_buffer[0];1625 const first_byte = byte_buffer[0];
1625 for (byte_buffer[1..]) |byte| {1626 for (byte_buffer[1..]) |byte| {
src/Zcu/PerThread.zig+4
...@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e...@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e
4544 tracy_trace.addText(fqn.toSlice(ip));4544 tracy_trace.addText(fqn.toSlice(ip));
4545 tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});4545 tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});
45464546
4547 Air.Verify.run(pt, func_index, air);
4548
4547 if (codegen.legalizeFeatures(pt, nav)) |features| {4549 if (codegen.legalizeFeatures(pt, nav)) |features| {
4548 try air.legalize(pt, features);4550 try air.legalize(pt, features);
4551 // Verify the AIR again post-legalization.
4552 Air.Verify.run(pt, func_index, air);
4549 }4553 }
45504554
4551 var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))4555 var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))
src/codegen/aarch64/Select.zig+23-7
...@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
292 .load,292 .load,
293 .fptrunc,293 .fptrunc,
294 .fpext,294 .fpext,
295 .intcast,295 .int_cast,
296 .intcast_safe,296 .int_cast_safe,
297 .trunc,297 .trunc,
298 .optional_payload,298 .optional_payload,
299 .optional_payload_ptr,299 .optional_payload_ptr,
...@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
334 air_inst_index = air_body[air_body_index];334 air_inst_index = air_body[air_body_index];
335 continue :air_tag air_tags[@intFromEnum(air_inst_index)];335 continue :air_tag air_tags[@intFromEnum(air_inst_index)];
336 },336 },
337 .bitcast => {337 .bit_cast,
338 .ptr_cast,
339 .ptr_from_int,
340 .int_from_ptr,
341 .error_cast,
342 .error_from_int,
343 .int_from_error,
344 .union_from_enum,
345 => {
338 const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;346 const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
339 maybe_noop: {347 maybe_noop: {
340 if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;348 if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
...@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3190 }3198 }
3191 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;3199 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3192 },3200 },
3193 .bitcast => |air_tag| {3201 .bit_cast,
3202 .ptr_cast,
3203 .ptr_from_int,
3204 .int_from_ptr,
3205 .error_cast,
3206 .error_from_int,
3207 .int_from_error,
3208 .union_from_enum,
3209 => |air_tag| {
3194 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {3210 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
3195 defer dst_vi.value.deref(isel);3211 defer dst_vi.value.deref(isel);
3196 const ty_op = air.data(air.inst_index).ty_op;3212 const ty_op = air.data(air.inst_index).ty_op;
...@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5221 }5237 }
5222 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;5238 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5223 },5239 },
5224 .intcast => |air_tag| {5240 .int_cast => |air_tag| {
5225 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {5241 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5226 defer dst_vi.value.deref(isel);5242 defer dst_vi.value.deref(isel);
52275243
...@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5312 }5328 }
5313 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;5329 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5314 },5330 },
5315 .intcast_safe => |air_tag| {5331 .int_cast_safe => |air_tag| {
5316 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {5332 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5317 defer dst_vi.value.deref(isel);5333 defer dst_vi.value.deref(isel);
53185334
...@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem...@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
11355 if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;11371 if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11356 constant.writeToMemory(zcu, buffer) catch |err| switch (err) {11372 constant.writeToMemory(zcu, buffer) catch |err| switch (err) {
11357 error.OutOfMemory => |e| return e,11373 error.OutOfMemory => |e| return e,
11358 error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,11374 error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
11359 };11375 };
11360 return true;11376 return true;
11361}11377}
src/codegen/aarch64/abi.zig+2-2
...@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
21 if (ty.containerLayout(zcu) == .@"packed") return .byval;21 if (ty.containerLayout(zcu) == .@"packed") return .byval;
22 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };22 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
2323
24 const bit_size = ty.bitSize(zcu);24 const bit_size = ty.abiSize(zcu) * 8;
25 if (bit_size > 128) return .memory;25 if (bit_size > 128) return .memory;
26 if (bit_size > 64) return .double_integer;26 if (bit_size > 64) return .double_integer;
27 return .integer;27 return .integer;
...@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
30 if (ty.containerLayout(zcu) == .@"packed") return .byval;30 if (ty.containerLayout(zcu) == .@"packed") return .byval;
31 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };31 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
3232
33 const bit_size = ty.bitSize(zcu);33 const bit_size = ty.abiSize(zcu) * 8;
34 if (bit_size > 128) return .memory;34 if (bit_size > 128) return .memory;
35 if (bit_size > 64) return .double_integer;35 if (bit_size > 64) return .double_integer;
36 return .integer;36 return .integer;
src/codegen/arm/abi.zig+6-6
...@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
30 const ip = &zcu.intern_pool;30 const ip = &zcu.intern_pool;
31 switch (ty.zigTypeTag(zcu)) {31 switch (ty.zigTypeTag(zcu)) {
32 .@"struct" => {32 .@"struct" => {
33 const bit_size = ty.bitSize(zcu);
34 if (ty.containerLayout(zcu) == .@"packed") {33 if (ty.containerLayout(zcu) == .@"packed") {
35 if (bit_size > 64) return .memory;34 if (ty.bitSize(zcu) > 64) return .memory;
36 return .byval;35 return .byval;
37 }36 }
37 const bit_size = ty.abiSize(zcu) * 8;
38 if (bit_size > max_byval_size) return .memory;38 if (bit_size > max_byval_size) return .memory;
39 const float_count = countFloats(ty, zcu, &maybe_float_bits);39 const float_count = countFloats(ty, zcu, &maybe_float_bits);
40 if (float_count <= byval_float_count) return .byval;40 if (float_count <= byval_float_count) return .byval;
...@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
47 var i: u32 = 0;47 var i: u32 = 0;
48 while (i < fields) : (i += 1) {48 while (i < fields) : (i += 1) {
49 const field_ty = ty.fieldType(i, zcu);49 const field_ty = ty.fieldType(i, zcu);
50 if (field_ty.bitSize(zcu) > 32) return Class.arrSize(bit_size, 64);50 if (field_ty.abiSize(zcu) > 4) return Class.arrSize(bit_size, 64);
51 }51 }
52 return Class.arrSize(bit_size, 32);52 return Class.arrSize(bit_size, 32);
53 },53 },
54 .@"union" => {54 .@"union" => {
55 const bit_size = ty.bitSize(zcu);
56 const union_obj = zcu.typeToUnion(ty).?;55 const union_obj = zcu.typeToUnion(ty).?;
57 if (union_obj.layout == .@"packed") {56 if (union_obj.layout == .@"packed") {
58 if (bit_size > 64) return .memory;57 if (ty.bitSize(zcu) > 64) return .memory;
59 return .byval;58 return .byval;
60 }59 }
60 const bit_size = ty.abiSize(zcu) * 8;
61 if (bit_size > max_byval_size) return .memory;61 if (bit_size > max_byval_size) return .memory;
62 const float_count = countFloats(ty, zcu, &maybe_float_bits);62 const float_count = countFloats(ty, zcu, &maybe_float_bits);
63 if (float_count <= byval_float_count) return .byval;63 if (float_count <= byval_float_count) return .byval;
...@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
67 }67 }
6868
69 for (union_obj.field_types.get(ip)) |field_ty| {69 for (union_obj.field_types.get(ip)) |field_ty| {
70 if (Type.fromInterned(field_ty).bitSize(zcu) > 32) {70 if (Type.fromInterned(field_ty).abiSize(zcu) > 4) {
71 return Class.arrSize(bit_size, 64);71 return Class.arrSize(bit_size, 64);
72 }72 }
73 }73 }
src/codegen/c.zig+270-104
...@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
27 return comptime switch (dev.env.supports(.legalize)) {27 return comptime switch (dev.env.supports(.legalize)) {
28 inline false, true => |supports_legalize| &.init(.{28 inline false, true => |supports_legalize| &.init(.{
29 // we don't currently ask zig1 to use safe optimization modes29 // we don't currently ask zig1 to use safe optimization modes
30 .expand_intcast_safe = supports_legalize,30 .expand_int_cast_safe = supports_legalize,
31 .expand_int_from_float_safe = supports_legalize,31 .expand_int_from_float_safe = supports_legalize,
32 .expand_int_from_float_optimized_safe = supports_legalize,32 .expand_int_from_float_optimized_safe = supports_legalize,
33 .expand_add_safe = supports_legalize,33 .expand_add_safe = supports_legalize,
...@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
38 .expand_packed_store = true,38 .expand_packed_store = true,
39 .expand_packed_struct_field_val = true,39 .expand_packed_struct_field_val = true,
40 .expand_packed_aggregate_init = true,40 .expand_packed_aggregate_init = true,
41
42 .scalarize_bit_cast_array = true,
43 .scalarize_bit_cast_vector_non_elementwise = true,
41 }),44 }),
42 };45 };
43}46}
...@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2636 // zig fmt: off2639 // zig fmt: off
2637 .inferred_alloc, .inferred_alloc_comptime => unreachable,2640 .inferred_alloc, .inferred_alloc_comptime => unreachable,
26382641
2639 // No "scalarize" legalizations are enabled, so these instructions never appear.2642 // Possible because `Air.Legalize.scalarize_bit_cast_vector_non_elementwise` is enabled.
2640 .legalize_vec_elem_val => unreachable,2643 .legalize_vec_elem_val => try airArrayElemVal(f, inst),
2641 .legalize_vec_store_elem => unreachable,2644 .legalize_vec_store_elem => try airLegalizeVecStoreElem(f, inst),
2642 // No soft float legalizations are enabled.2645 // No soft float legalizations are enabled.
2643 .legalize_compiler_rt_call => unreachable,2646 .legalize_compiler_rt_call => unreachable,
26442647
...@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2751 .alloc => try airAlloc(f, inst),2754 .alloc => try airAlloc(f, inst),
2752 .ret_ptr => try airRetPtr(f, inst),2755 .ret_ptr => try airRetPtr(f, inst),
2753 .assembly => try airAsm(f, inst),2756 .assembly => try airAsm(f, inst),
2754 .bitcast => try airBitcast(f, inst),2757 .ptr_cast => try airPtrCast(f, inst),
2755 .intcast => try airIntCast(f, inst),2758 .ptr_from_int => try airSimpleCast(f, inst),
2759 .int_from_ptr => try airSimpleCast(f, inst),
2760 .error_cast => try airNopCast(f, inst),
2761 .error_from_int => try airNopCast(f, inst),
2762 .int_from_error => try airNopCast(f, inst),
2763 .union_from_enum => try airUnionFromEnum(f, inst),
2764 .bit_cast => try airBitCast(f, inst),
2765 .int_cast => try airIntCast(f, inst),
2756 .trunc => try airTrunc(f, inst),2766 .trunc => try airTrunc(f, inst),
2757 .load => try airLoad(f, inst),2767 .load => try airLoad(f, inst),
2758 .store => try airStore(f, inst, false),2768 .store => try airStore(f, inst, false),
...@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2864 .add_safe,2874 .add_safe,
2865 .sub_safe,2875 .sub_safe,
2866 .mul_safe,2876 .mul_safe,
2867 .intcast_safe,2877 .int_cast_safe,
2868 .int_from_float_safe,2878 .int_from_float_safe,
2869 .int_from_float_optimized_safe,2879 .int_from_float_optimized_safe,
2870 => return f.fail("TODO implement safety_checked_instructions", .{}),2880 => return f.fail("TODO implement safety_checked_instructions", .{}),
...@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
3083 return local;3093 return local;
3084}3094}
30853095
3096fn airLegalizeVecStoreElem(f: *Function, inst: Air.Inst.Index) !CValue {
3097 const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3098 const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
3099
3100 const vec_ptr = try f.resolveInst(pl_op.operand);
3101 const index = try f.resolveInst(extra.lhs);
3102 const elem = try f.resolveInst(extra.rhs);
3103 try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs });
3104
3105 const w = &f.code.writer;
3106
3107 try f.writeCValueDerefMember(w, vec_ptr, .{ .identifier = "array" });
3108 try w.writeByte('[');
3109 try f.writeCValue(w, index, .other);
3110 try w.writeAll("] = ");
3111 try f.writeCValue(w, elem, .other);
3112 try w.writeByte(';');
3113 try f.newline();
3114
3115 return .none;
3116}
3117
3086fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {3118fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3087 const pt = f.dg.pt;3119 const pt = f.dg.pt;
3088 const zcu = pt.zcu;3120 const zcu = pt.zcu;
...@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
31903222
3191 try reap(f, inst, &.{ty_op.operand});3223 try reap(f, inst, &.{ty_op.operand});
31923224
3193 const is_aligned = if (ptr_info.flags.alignment != .none)3225 const is_aligned = switch (ptr_info.flags.alignment) {
3194 ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte)3226 .none => true,
3195 else3227 else => |ptr_align| ptr_align.compare(.gte, src_ty.abiAlignment(zcu)),
3196 true;3228 };
31973229
3198 const w = &f.code.writer;3230 const w = &f.code.writer;
3199 const local = try f.allocLocal(inst, src_ty);3231 const local = try f.allocLocal(inst, src_ty);
3200 const v = try Vectorize.start(f, inst, w, ptr_ty);
32013232
3202 if (!is_aligned) {3233 if (!is_aligned) {
3203 try w.writeAll("memcpy(&");3234 try w.writeAll("memcpy(&");
3204 try f.writeCValue(w, local, .other);3235 try f.writeCValue(w, local, .other);
3205 try v.elem(f, w);
3206 try w.writeAll(", (const char *)");3236 try w.writeAll(", (const char *)");
3207 try f.writeCValue(w, operand, .other);3237 switch (ptr_info.flags.vector_index) {
3208 try v.elem(f, w);3238 .none => try f.writeCValue(w, operand, .other),
3239 else => |index| {
3240 try w.writeByte('&');
3241 try f.writeCValue(w, operand, .other);
3242 try w.print("[{d}]", .{@intFromEnum(index)});
3243 },
3244 }
3209 try w.writeAll(", sizeof(");3245 try w.writeAll(", sizeof(");
3210 try f.renderType(w, src_ty);3246 try f.renderType(w, src_ty);
3211 try w.writeAll("))");3247 try w.writeAll("))");
3212 } else {3248 } else {
3213 try f.writeCValue(w, local, .other);3249 try f.writeCValue(w, local, .other);
3214 try v.elem(f, w);
3215 try w.writeAll(" = ");3250 try w.writeAll(" = ");
3216 try f.writeCValueDeref(w, operand);3251 switch (ptr_info.flags.vector_index) {
3217 try v.elem(f, w);3252 .none => try f.writeCValueDeref(w, operand),
3253 else => |index| {
3254 try f.writeCValue(w, operand, .other);
3255 try w.print("[{d}]", .{@intFromEnum(index)});
3256 },
3257 }
3218 }3258 }
3219 try w.writeByte(';');3259 try w.writeByte(';');
3220 try f.newline();3260 try f.newline();
3221 try v.end(f, inst, w);
32223261
3223 return local;3262 return local;
3224}3263}
...@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3433 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).3472 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
3434 assert(src_ty.eql(.fromInterned(ptr_info.child)));3473 assert(src_ty.eql(.fromInterned(ptr_info.child)));
34353474
3436 const v = try Vectorize.start(f, inst, w, ptr_ty);
3437 try w.writeAll("memcpy((char *)");3475 try w.writeAll("memcpy((char *)");
3438 try f.writeCValue(w, ptr_val, .other);3476 switch (ptr_info.flags.vector_index) {
3439 try v.elem(f, w);3477 .none => try f.writeCValue(w, ptr_val, .other),
3478 else => |index| {
3479 try w.writeByte('&');
3480 try f.writeCValue(w, ptr_val, .other);
3481 try w.print("[{d}]", .{@intFromEnum(index)});
3482 },
3483 }
3440 try w.writeAll(", &");3484 try w.writeAll(", &");
3441 switch (src_val) {3485 switch (src_val) {
3442 .constant => |val| try f.dg.renderValueAsLvalue(w, val),3486 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
3443 else => try f.writeCValue(w, src_val, .other),3487 else => try f.writeCValue(w, src_val, .other),
3444 }3488 }
3445 try v.elem(f, w);
3446 try w.writeAll(", sizeof(");3489 try w.writeAll(", sizeof(");
3447 try f.renderType(w, src_ty);3490 try f.renderType(w, src_ty);
3448 try w.writeAll("));");3491 try w.writeAll("));");
3449 try f.newline();3492 try f.newline();
3450 try v.end(f, inst, w);
3451 } else {3493 } else {
3452 switch (ptr_val) {3494 switch (ptr_val) {
3453 .local_ref => |ptr_local_index| switch (src_val) {3495 .local_ref => |ptr_local_index| switch (src_val) {
...@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3457 },3499 },
3458 else => {},3500 else => {},
3459 }3501 }
3460 const v = try Vectorize.start(f, inst, w, ptr_ty);3502
3461 try f.writeCValueDeref(w, ptr_val);3503 switch (ptr_info.flags.vector_index) {
3462 try v.elem(f, w);3504 .none => try f.writeCValueDeref(w, ptr_val),
3505 else => |index| {
3506 try f.writeCValue(w, ptr_val, .other);
3507 try w.print("[{d}]", .{@intFromEnum(index)});
3508 },
3509 }
3463 try w.writeAll(" = ");3510 try w.writeAll(" = ");
3464 try f.writeCValue(w, src_val, .other);3511 try f.writeCValue(w, src_val, .other);
3465 try v.elem(f, w);
3466 try w.writeByte(';');3512 try w.writeByte(';');
3467 try f.newline();3513 try f.newline();
3468 try v.end(f, inst, w);
3469 }3514 }
3470 return .none;3515 return .none;
3471}3516}
...@@ -3613,9 +3658,9 @@ fn airCmpOp(...@@ -3613,9 +3658,9 @@ fn airCmpOp(
3613 const lhs_ty = f.typeOf(data.lhs);3658 const lhs_ty = f.typeOf(data.lhs);
3614 const scalar_ty = lhs_ty.scalarType(zcu);3659 const scalar_ty = lhs_ty.scalarType(zcu);
36153660
3616 const scalar_bits = scalar_ty.bitSize(zcu);3661 if (scalar_ty.isInt(zcu)) {
3617 if (scalar_ty.isInt(zcu) and scalar_bits > 64)3662 const scalar_bits = scalar_ty.bitSize(zcu);
3618 return airCmpBuiltinCall(3663 if (scalar_bits > 64) return airCmpBuiltinCall(
3619 f,3664 f,
3620 inst,3665 inst,
3621 data,3666 data,
...@@ -3623,6 +3668,7 @@ fn airCmpOp(...@@ -3623,6 +3668,7 @@ fn airCmpOp(
3623 .cmp,3668 .cmp,
3624 if (scalar_bits > 128) .bits else .none,3669 if (scalar_bits > 128) .bits else .none,
3625 );3670 );
3671 }
3626 if (scalar_ty.isRuntimeFloat())3672 if (scalar_ty.isRuntimeFloat())
3627 return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);3673 return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);
36283674
...@@ -3668,9 +3714,9 @@ fn airEquality(...@@ -3668,9 +3714,9 @@ fn airEquality(
3668 const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;3714 const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
36693715
3670 const operand_ty = f.typeOf(bin_op.lhs);3716 const operand_ty = f.typeOf(bin_op.lhs);
3671 const operand_bits = operand_ty.bitSize(zcu);3717 if (operand_ty.isAbiInt(zcu)) {
3672 if (operand_ty.isAbiInt(zcu) and operand_bits > 64)3718 const operand_bits = operand_ty.bitSize(zcu);
3673 return airCmpBuiltinCall(3719 if (operand_bits > 64) return airCmpBuiltinCall(
3674 f,3720 f,
3675 inst,3721 inst,
3676 bin_op,3722 bin_op,
...@@ -3678,6 +3724,7 @@ fn airEquality(...@@ -3678,6 +3724,7 @@ fn airEquality(
3678 .cmp,3724 .cmp,
3679 if (operand_bits > 128) .bits else .none,3725 if (operand_bits > 128) .bits else .none,
3680 );3726 );
3727 }
3681 if (operand_ty.isRuntimeFloat())3728 if (operand_ty.isRuntimeFloat())
3682 return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);3729 return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);
36833730
...@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {...@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
4258 try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});4305 try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});
4259}4306}
42604307
4261fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {4308fn airPtrCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4309 const zcu = f.dg.pt.zcu;
4310
4311 const dest_ty = f.typeOfIndex(inst);
4312 const ptr_ty = switch (dest_ty.zigTypeTag(zcu)) {
4313 .optional => dest_ty.childType(zcu),
4314 .pointer => dest_ty,
4315 else => unreachable,
4316 };
4317
4318 if (!ptr_ty.isSlice(zcu)) {
4319 return airSimpleCast(f, inst);
4320 }
4321
4322 // For slice casts we need to assign both fields.
4323
4262 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4324 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4263 const inst_ty = f.typeOfIndex(inst);4325 const operand = try f.resolveInst(ty_op.operand);
4326
4327 const w = &f.code.writer;
4328 const dest_local = try f.allocLocal(inst, dest_ty);
4329
4330 try f.writeCValueMember(w, dest_local, .{ .identifier = "ptr" });
4331 try w.writeAll(" = (");
4332 try f.renderType(w, ptr_ty.slicePtrFieldType(zcu));
4333 try w.writeByte(')');
4334 try f.writeCValueMember(w, operand, .{ .identifier = "ptr" });
4335 try w.writeByte(';');
4336 try f.newline();
4337
4338 try f.writeCValueMember(w, dest_local, .{ .identifier = "len" });
4339 try w.writeAll(" = ");
4340 try f.writeCValueMember(w, operand, .{ .identifier = "len" });
4341 try w.writeByte(';');
4342 try f.newline();
4343
4344 try reap(f, inst, &.{ty_op.operand});
4345 return dest_local;
4346}
4347
4348fn airSimpleCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4349 const zcu = f.dg.pt.zcu;
4350
4351 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4352 const dest_ty = f.typeOfIndex(inst);
4353 const operand_ty = f.typeOf(ty_op.operand);
4354 const operand = try f.resolveInst(ty_op.operand);
4355
4356 const w = &f.code.writer;
4357 const dest_local = try f.allocLocal(inst, dest_ty);
4358 const v: Vectorize = try .start(f, inst, w, operand_ty);
4359 try f.writeCValue(w, dest_local, .other);
4360 try v.elem(f, w);
4361 try w.writeAll(" = (");
4362 try f.renderType(w, dest_ty.scalarType(zcu));
4363 try w.writeByte(')');
4364 try f.writeCValue(w, operand, .other);
4365 try v.elem(f, w);
4366 try w.writeByte(';');
4367 try f.newline();
4368 try v.end(f, inst, w);
4369
4370 try reap(f, inst, &.{ty_op.operand});
4371 return dest_local;
4372}
42644373
4374fn airNopCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4375 const zcu = f.dg.pt.zcu;
4376
4377 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4378 const dest_ty = f.typeOfIndex(inst);
4379 const operand_ty = f.typeOf(ty_op.operand);
4265 const operand = try f.resolveInst(ty_op.operand);4380 const operand = try f.resolveInst(ty_op.operand);
4381
4382 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
4383 assert(operand_ty.isAbiInt(zcu) == dest_ty.isAbiInt(zcu));
4384
4385 try reap(f, inst, &.{ty_op.operand});
4386 return f.moveCValue(inst, dest_ty, operand);
4387}
4388
4389fn airUnionFromEnum(f: *Function, inst: Air.Inst.Index) Error!CValue {
4390 const zcu = f.dg.pt.zcu;
4391
4392 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4393 const dest_ty = f.typeOfIndex(inst);
4266 const operand_ty = f.typeOf(ty_op.operand);4394 const operand_ty = f.typeOf(ty_op.operand);
4395 const operand = try f.resolveInst(ty_op.operand);
4396
4397 assert(dest_ty.zigTypeTag(zcu) == .@"union");
4398 assert(operand_ty.zigTypeTag(zcu) == .@"enum");
4399
4400 const w = &f.code.writer;
4401 const dest_local = try f.allocLocal(inst, dest_ty);
4402 try f.writeCValueMember(w, dest_local, .{ .identifier = "tag" });
4403 try w.writeAll(" = ");
4404 try f.writeCValue(w, operand, .other);
4405 try w.writeByte(';');
4406 try f.newline();
42674407
4268 const bitcasted = try bitcast(f, inst_ty, operand, operand_ty);
4269 try reap(f, inst, &.{ty_op.operand});4408 try reap(f, inst, &.{ty_op.operand});
4270 return f.moveCValue(inst, inst_ty, bitcasted);4409 return dest_local;
4271}4410}
42724411
4273fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CValue {4412fn airBitCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4274 const pt = f.dg.pt;4413 const pt = f.dg.pt;
4275 const zcu = pt.zcu;4414 const zcu = pt.zcu;
4276 const target = &f.dg.mod.resolved_target.result;
4277 const w = &f.code.writer;4415 const w = &f.code.writer;
42784416
4279 if (operand_ty.isAbiInt(zcu) and dest_ty.isAbiInt(zcu)) {4417 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4280 const src_info = dest_ty.intInfo(zcu);4418 const dest_ty = f.typeOfIndex(inst);
4281 const dest_info = operand_ty.intInfo(zcu);
4282 if (src_info.signedness == dest_info.signedness and
4283 src_info.bits == dest_info.bits) return operand;
4284 }
42854419
4286 if (dest_ty.isPtrAtRuntime(zcu) or operand_ty.isPtrAtRuntime(zcu)) {4420 const operand = try f.resolveInst(ty_op.operand);
4287 const local = try f.allocLocal(null, dest_ty);4421 const operand_ty = f.typeOf(ty_op.operand);
4288 try f.writeCValue(w, local, .other);4422
4423 const dest_local = try f.allocLocal(inst, dest_ty);
4424
4425 // Because we have `scalarize_bit_cast_array` and `scalarize_bit_cast_vector_non_elementwise`
4426 // enabled, we usually only see scalars here. The only case in which we may see vectors is when
4427 // the operation happens elementwise, which we can handle with `Vectorize`.
4428 var v: Vectorize = try .start(f, inst, w, operand_ty);
4429 const operand_scalar_ty = operand_ty.scalarType(zcu);
4430 const dest_scalar_ty = dest_ty.scalarType(zcu);
4431
4432 // Some cases are handled with a simple cast:
4433 // * float -> float
4434 // * bool -> int
4435 if ((operand_scalar_ty.isRuntimeFloat() and dest_scalar_ty.isRuntimeFloat()) or
4436 (operand_scalar_ty.toIntern() == .bool_type and dest_scalar_ty.isAbiInt(zcu)))
4437 {
4438 try f.writeCValue(w, dest_local, .other);
4439 try v.elem(f, w);
4289 try w.writeAll(" = (");4440 try w.writeAll(" = (");
4290 try f.renderType(w, dest_ty);4441 try f.renderType(w, dest_scalar_ty);
4291 try w.writeByte(')');4442 try w.writeByte(')');
4292 try f.writeCValue(w, operand, .other);4443 try f.writeCValue(w, operand, .other);
4444 try v.elem(f, w);
4293 try w.writeByte(';');4445 try w.writeByte(';');
4294 try f.newline();4446 try f.newline();
4295 return local;4447 } else if (dest_scalar_ty.toIntern() == .bool_type) {
4296 }4448 // If the result is a boolean type, just check if the operand is non-zero.
42974449 assert(operand_scalar_ty.isAbiInt(zcu));
4298 const local = try f.allocLocal(null, dest_ty);4450 try f.writeCValue(w, dest_local, .other);
4299 // On big-endian targets, copying ABI integers with padding bits is awkward, because the padding bits are at the low bytes of the value.4451 try v.elem(f, w);
4300 // We need to offset the source or destination pointer appropriately and copy the right number of bytes.4452 try w.writeAll(" = ");
4301 if (target.cpu.arch.endian() == .big and dest_ty.isAbiInt(zcu) and !operand_ty.isAbiInt(zcu)) {4453 try f.writeCValue(w, operand, .other);
4302 // e.g. [10]u8 -> u80. We need to offset the destination so that we copy to the least significant bits of the integer.4454 try v.elem(f, w);
4303 const offset = dest_ty.abiSize(zcu) - operand_ty.abiSize(zcu);4455 try w.writeAll(" != 0;");
4304 try w.writeAll("memcpy((char *)&");4456 try f.newline();
4305 try f.writeCValue(w, local, .other);4457 } else if (dest_scalar_ty.isRuntimeFloat()) {
4306 try w.print(" + {d}, &", .{offset});4458 // For int->float, just do a memcpy.
4307 switch (operand) {4459 assert(operand_scalar_ty.isAbiInt(zcu));
4308 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4309 else => try f.writeCValue(w, operand, .other),
4310 }
4311 try w.print(", {d});", .{operand_ty.abiSize(zcu)});
4312 } else if (target.cpu.arch.endian() == .big and operand_ty.isAbiInt(zcu) and !dest_ty.isAbiInt(zcu)) {
4313 // e.g. u80 -> [10]u8. We need to offset the source so that we copy from the least significant bits of the integer.
4314 const offset = operand_ty.abiSize(zcu) - dest_ty.abiSize(zcu);
4315 try w.writeAll("memcpy(&");4460 try w.writeAll("memcpy(&");
4316 try f.writeCValue(w, local, .other);4461 try f.writeCValue(w, dest_local, .other);
4317 try w.writeAll(", (const char *)&");4462 try v.elem(f, w);
4463 try w.writeAll(", &");
4318 switch (operand) {4464 switch (operand) {
4319 .constant => |val| try f.dg.renderValueAsLvalue(w, val),4465 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4320 else => try f.writeCValue(w, operand, .other),4466 else => try f.writeCValue(w, operand, .other),
4321 }4467 }
4322 try w.print(" + {d}, {d});", .{ offset, dest_ty.abiSize(zcu) });4468 try v.elem(f, w);
4469 try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
4470 try f.newline();
4323 } else {4471 } else {
4472 // The only remaining possibility is that the result is an integer. We will need to use
4473 // `zig_wrap_*` to correct the "padding" bits after we populate the value bits.
4474 assert(dest_scalar_ty.isAbiInt(zcu));
4475 assert(operand_scalar_ty.isRuntimeFloat() or operand_scalar_ty.isAbiInt(zcu));
4476
4477 // memcpy the value...
4324 try w.writeAll("memcpy(&");4478 try w.writeAll("memcpy(&");
4325 try f.writeCValue(w, local, .other);4479 try f.writeCValue(w, dest_local, .other);
4480 try v.elem(f, w);
4326 try w.writeAll(", &");4481 try w.writeAll(", &");
4327 switch (operand) {4482 switch (operand) {
4328 .constant => |val| try f.dg.renderValueAsLvalue(w, val),4483 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4329 else => try f.writeCValue(w, operand, .other),4484 else => try f.writeCValue(w, operand, .other),
4330 }4485 }
4331 try w.print(", {d});", .{@min(dest_ty.abiSize(zcu), operand_ty.abiSize(zcu))});4486 try v.elem(f, w);
4332 }4487 try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
43334488 try f.newline();
4334 try f.newline();
43354489
4336 // Ensure padding bits have the expected value.4490 // ...and ensure padding bits have the correct value.
4337 if (dest_ty.isAbiInt(zcu)) {4491 switch (CType.classifyInt(dest_scalar_ty, zcu)) {
4338 switch (CType.classifyInt(dest_ty, zcu)) {
4339 .void => unreachable, // opv4492 .void => unreachable, // opv
4340 .small => {4493 .small => {
4341 try f.writeCValue(w, local, .other);4494 try f.writeCValue(w, dest_local, .other);
4495 try v.elem(f, w);
4342 try w.writeAll(" = zig_wrap_");4496 try w.writeAll(" = zig_wrap_");
4343 try f.dg.renderTypeForBuiltinFnName(w, dest_ty);4497 try f.dg.renderTypeForBuiltinFnName(w, dest_scalar_ty);
4344 try w.writeByte('(');4498 try w.writeByte('(');
4345 try f.writeCValue(w, local, .other);4499 try f.writeCValue(w, dest_local, .other);
4346 try f.dg.renderBuiltinInfo(w, dest_ty, .bits);4500 try v.elem(f, w);
4501 try f.dg.renderBuiltinInfo(w, dest_scalar_ty, .bits);
4347 try w.writeAll(");");4502 try w.writeAll(");");
4348 try f.newline();4503 try f.newline();
4349 },4504 },
4350 .big => |big| {4505 .big => |big| {
4351 const dest_info = dest_ty.intInfo(zcu);4506 const dest_info = dest_scalar_ty.intInfo(zcu);
4352 const padding_index: u16 = switch (target.cpu.arch.endian()) {4507 const padding_index: u16 = switch (f.dg.mod.resolved_target.result.cpu.arch.endian()) {
4353 .little => big.limbs_len - 1,4508 .little => big.limbs_len - 1,
4354 .big => 0,4509 .big => 0,
4355 };4510 };
4356 const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;4511 const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;
4357 if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {4512 if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {
4358 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4513 try f.writeCValue(w, dest_local, .other);
4359 try w.print("[{d}] = zig_wrap_{c}{d}(", .{4514 try v.elem(f, w);
4515 try w.print(".limbs[{d}] = zig_wrap_{c}{d}(", .{
4360 padding_index,4516 padding_index,
4361 signAbbrev(dest_info.signedness),4517 signAbbrev(dest_info.signedness),
4362 big.limb_size.bits(),4518 big.limb_size.bits(),
4363 });4519 });
4364 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4520 try f.writeCValue(w, dest_local, .other);
4365 try w.print("[{d}], {d});", .{ padding_index, wrap_bits });4521 try v.elem(f, w);
4522 try w.print(".limbs[{d}], {d});", .{ padding_index, wrap_bits });
4366 } else {4523 } else {
4367 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4524 try f.writeCValue(w, dest_local, .other);
4368 try w.print("[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{4525 try v.elem(f, w);
4526 try w.print(".limbs[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
4369 padding_index,4527 padding_index,
4370 });4528 });
4371 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4529 try f.writeCValue(w, dest_local, .other);
4372 try w.print("[{d}]), {d}));", .{ padding_index, wrap_bits });4530 try v.elem(f, w);
4531 try w.print(".limbs[{d}]), {d}));", .{ padding_index, wrap_bits });
4373 try f.newline();4532 try f.newline();
4374 }4533 }
4375 },4534 },
4376 }4535 }
4377 }4536 }
43784537
4379 return local;4538 try v.end(f, inst, w);
4539
4540 try reap(f, inst, &.{ty_op.operand});
4541 return dest_local;
4380}4542}
43814543
4382fn airTrap(f: *Function) !void {4544fn airTrap(f: *Function) !void {
...@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
6151 return .none;6313 return .none;
6152 }6314 }
61536315
6154 if (elem_abi_size == 1 and !dest_ty.isVolatilePtr(zcu)) {6316 if (elem_abi_size == 1 and elem_ty.isAbiInt(zcu) and !dest_ty.isVolatilePtr(zcu)) {
6155 const bitcasted = try bitcast(f, .u8, value, elem_ty);
6156 try w.writeAll("memset(");6317 try w.writeAll("memset(");
6157 switch (dest_ty.ptrSize(zcu)) {6318 switch (dest_ty.ptrSize(zcu)) {
6158 .slice => {6319 .slice => {
6159 try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });6320 try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });
6160 try w.writeAll(", ");6321 try w.writeAll(", *(const char *)&");
6161 try f.writeCValue(w, bitcasted, .other);6322 switch (value) {
6323 .constant => |v| try f.dg.renderValueAsLvalue(w, v),
6324 else => try f.writeCValue(w, value, .other),
6325 }
6162 try w.writeAll(", ");6326 try w.writeAll(", ");
6163 try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });6327 try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });
6164 },6328 },
6165 .one => {6329 .one => {
6166 try f.writeCValue(w, dest_slice, .other);6330 try f.writeCValue(w, dest_slice, .other);
6167 try w.writeAll(", ");6331 try w.writeAll(", *(const char *)&");
6168 try f.writeCValue(w, bitcasted, .other);6332 switch (value) {
6333 .constant => |v| try f.dg.renderValueAsLvalue(w, v),
6334 else => try f.writeCValue(w, value, .other),
6335 }
6169 try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});6336 try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});
6170 },6337 },
6171 .many, .c => unreachable,6338 .many, .c => unreachable,
6172 }6339 }
6173 try w.writeAll(");");6340 try w.writeAll(");");
6174 try f.newline();6341 try f.newline();
6175 try f.freeCValue(inst, bitcasted);
6176 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });6342 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
6177 return .none;6343 return .none;
6178 }6344 }
src/codegen/c/type.zig+15-19
...@@ -514,40 +514,39 @@ pub const CType = union(enum) {...@@ -514,40 +514,39 @@ pub const CType = union(enum) {
514 }514 }
515 }515 }
516 fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {516 fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {
517 const is_ez80 = zcu.getTarget().cpu.arch == .ez80;517 const target = zcu.getTarget();
518 return switch (bits) {518 return switch (std.zig.target.intByteSize(target, bits)) {
519 0 => .void,519 0 => .void,
520 1...8 => switch (signedness) {520 1 => switch (signedness) {
521 .unsigned => .{ .small = .uint8_t },521 .unsigned => .{ .small = .uint8_t },
522 .signed => .{ .small = .int8_t },522 .signed => .{ .small = .int8_t },
523 },523 },
524 9...16 => switch (signedness) {524 2 => switch (signedness) {
525 .unsigned => .{ .small = .uint16_t },525 .unsigned => .{ .small = .uint16_t },
526 .signed => .{ .small = .int16_t },526 .signed => .{ .small = .int16_t },
527 },527 },
528 17...24 => switch (signedness) {528 3 => switch (signedness) {
529 .unsigned => .{ .small = if (is_ez80) .uint24_t else .uint32_t },529 .unsigned => .{ .small = .uint24_t },
530 .signed => .{ .small = if (is_ez80) .int24_t else .int32_t },530 .signed => .{ .small = .int24_t },
531 },531 },
532 25...32 => switch (signedness) {532 4 => switch (signedness) {
533 .unsigned => .{ .small = .uint32_t },533 .unsigned => .{ .small = .uint32_t },
534 .signed => .{ .small = .int32_t },534 .signed => .{ .small = .int32_t },
535 },535 },
536 33...48 => switch (signedness) {536 6 => switch (signedness) {
537 .unsigned => .{ .small = if (is_ez80) .uint48_t else .uint64_t },537 .unsigned => .{ .small = .uint48_t },
538 .signed => .{ .small = if (is_ez80) .int48_t else .int64_t },538 .signed => .{ .small = .int48_t },
539 },539 },
540 49...64 => switch (signedness) {540 8 => switch (signedness) {
541 .unsigned => .{ .small = .uint64_t },541 .unsigned => .{ .small = .uint64_t },
542 .signed => .{ .small = .int64_t },542 .signed => .{ .small = .int64_t },
543 },543 },
544 65...128 => switch (signedness) {544 16 => switch (signedness) {
545 .unsigned => .{ .small = .zig_u128 },545 .unsigned => .{ .small = .zig_u128 },
546 .signed => .{ .small = .zig_i128 },546 .signed => .{ .small = .zig_i128 },
547 },547 },
548 else => {548 else => |n| {
549 @branchHint(.unlikely);549 @branchHint(.unlikely);
550 const target = zcu.getTarget();
551 const limb_bytes = std.zig.target.intAlignment(target, bits);550 const limb_bytes = std.zig.target.intAlignment(target, bits);
552 return .{ .big = .{551 return .{ .big = .{
553 .limb_size = switch (limb_bytes) {552 .limb_size = switch (limb_bytes) {
...@@ -558,10 +557,7 @@ pub const CType = union(enum) {...@@ -558,10 +557,7 @@ pub const CType = union(enum) {
558 16 => .@"128",557 16 => .@"128",
559 else => unreachable,558 else => unreachable,
560 },559 },
561 .limbs_len = @divExact(560 .limbs_len = @divExact(n, limb_bytes),
562 std.zig.target.intByteSize(target, bits),
563 limb_bytes,
564 ),
565 } };561 } };
566 },562 },
567 };563 };
src/codegen/llvm.zig+156-288
...@@ -20,10 +20,8 @@ const Value = @import("../Value.zig");...@@ -20,10 +20,8 @@ const Value = @import("../Value.zig");
20const Zcu = @import("../Zcu.zig");20const Zcu = @import("../Zcu.zig");
21const aarch64_c_abi = @import("aarch64/abi.zig");21const aarch64_c_abi = @import("aarch64/abi.zig");
22const FuncGen = @import("llvm/FuncGen.zig");22const FuncGen = @import("llvm/FuncGen.zig");
23const buildAllocaInner = FuncGen.buildAllocaInner;
24const isByRef = FuncGen.isByRef;23const isByRef = FuncGen.isByRef;
25const firstParamSRet = FuncGen.firstParamSRet;24const fnReturnStrat = FuncGen.fnReturnStrat;
26const lowerFnRetTy = FuncGen.lowerFnRetTy;
27const iterateParamTypes = FuncGen.iterateParamTypes;25const iterateParamTypes = FuncGen.iterateParamTypes;
28const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;26const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
2927
...@@ -37,6 +35,11 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -37,6 +35,11 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
37 return comptime &.initMany(&.{35 return comptime &.initMany(&.{
38 .expand_int_from_float_safe,36 .expand_int_from_float_safe,
39 .expand_int_from_float_optimized_safe,37 .expand_int_from_float_optimized_safe,
38
39 .scalarize_bit_cast_array,
40 // Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest
41 // types are vectors with equal length (hence also with equal bits-per-element).
42 .scalarize_bit_cast_vector_non_elementwise,
40 });43 });
41}44}
4245
...@@ -728,8 +731,8 @@ pub const Object = struct {...@@ -728,8 +731,8 @@ pub const Object = struct {
728731
729 // TODO: Address space732 // TODO: Address space
730 const slice_ty = Type.slice_const_u8_sentinel_0;733 const slice_ty = Type.slice_const_u8_sentinel_0;
731 const llvm_usize_ty = try o.lowerType(.usize);734 const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
732 const llvm_slice_ty = try o.lowerType(slice_ty);735 const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
733 const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);736 const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
734737
735 llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);738 llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
...@@ -795,7 +798,7 @@ pub const Object = struct {...@@ -795,7 +798,7 @@ pub const Object = struct {
795 {798 {
796 if (o.errors_len_variable != .none) {799 if (o.errors_len_variable != .none) {
797 const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;800 const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
798 const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);801 const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
799 try o.errors_len_variable.setInitializer(init_val, &o.builder);802 try o.errors_len_variable.setInitializer(init_val, &o.builder);
800 }803 }
801 try o.genErrorNameTable();804 try o.genErrorNameTable();
...@@ -1187,7 +1190,7 @@ pub const Object = struct {...@@ -1187,7 +1190,7 @@ pub const Object = struct {
1187 };1190 };
1188 {1191 {
1189 const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);1192 const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
1190 global.type = try o.lowerType(fn_ty);1193 global.type = try o.lowerType(fn_ty, .in_memory);
1191 global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);1194 global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
1192 global.linkage = if (o.builder.strip) .private else .internal;1195 global.linkage = if (o.builder.strip) .private else .internal;
1193 global.visibility = .default;1196 global.visibility = .default;
...@@ -1274,165 +1277,7 @@ pub const Object = struct {...@@ -1274,165 +1277,7 @@ pub const Object = struct {
1274 } }, &o.builder);1277 } }, &o.builder);
1275 }1278 }
12761279
1277 var deinit_wip = true;1280 const file, const subprogram = if (!owner_mod.strip) debug_info: {
1278 var wip = try Builder.WipFunction.init(&o.builder, .{
1279 .function = llvm_function,
1280 .strip = owner_mod.strip,
1281 });
1282 defer if (deinit_wip) wip.deinit();
1283 wip.cursor = .{ .block = try wip.block(0, "Entry") };
1284
1285 // This is the list of args we will use that correspond directly to the AIR arg
1286 // instructions. Depending on the calling convention, this list is not necessarily
1287 // a bijection with the actual LLVM parameters of the function.
1288 var args: std.ArrayList(Builder.Value) = .empty;
1289 defer args.deinit(gpa);
1290
1291 const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
1292 var it = iterateParamTypes(o, fn_info);
1293
1294 const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
1295 const param = wip.arg(it.llvm_index);
1296 it.llvm_index += 1;
1297 break :param param;
1298 } else .none;
1299
1300 const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1301 const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1302 const param = wip.arg(it.llvm_index);
1303 it.llvm_index += 1;
1304 break :param param;
1305 } else .none;
1306
1307 while (try it.next()) |lowering| {
1308 try args.ensureUnusedCapacity(gpa, 1);
1309
1310 switch (lowering) {
1311 .no_bits => continue,
1312 .byval => {
1313 assert(!it.byval_attr);
1314 const param_index = it.zig_index - 1;
1315 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1316 const param = wip.arg(it.llvm_index - 1);
1317
1318 if (isByRef(param_ty, zcu)) {
1319 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1320 const param_llvm_ty = param.typeOfWip(&wip);
1321 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1322 _ = try wip.store(.normal, param, arg_ptr, alignment);
1323 args.appendAssumeCapacity(arg_ptr);
1324 } else {
1325 args.appendAssumeCapacity(param);
1326 }
1327 },
1328 .byref => {
1329 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1330 const param = wip.arg(it.llvm_index - 1);
1331
1332 if (isByRef(param_ty, zcu)) {
1333 args.appendAssumeCapacity(param);
1334 } else {
1335 const param_llvm_ty = try o.lowerType(param_ty);
1336 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1337 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1338 }
1339 },
1340 .byref_mut => {
1341 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1342 const param = wip.arg(it.llvm_index - 1);
1343
1344 if (isByRef(param_ty, zcu)) {
1345 args.appendAssumeCapacity(param);
1346 } else {
1347 const param_llvm_ty = try o.lowerType(param_ty);
1348 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1349 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1350 }
1351 },
1352 .abi_sized_int => {
1353 assert(!it.byval_attr);
1354 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1355 const param = wip.arg(it.llvm_index - 1);
1356
1357 const param_llvm_ty = try o.lowerType(param_ty);
1358 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1359 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1360 _ = try wip.store(.normal, param, arg_ptr, alignment);
1361
1362 if (isByRef(param_ty, zcu)) {
1363 args.appendAssumeCapacity(arg_ptr);
1364 } else {
1365 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1366 }
1367 },
1368 .slice => {
1369 assert(!it.byval_attr);
1370 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1371 assert(!isByRef(param_ty, zcu));
1372 const slice_val = try wip.buildAggregate(
1373 try o.lowerType(param_ty),
1374 &.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
1375 "",
1376 );
1377 args.appendAssumeCapacity(slice_val);
1378 },
1379 .multiple_llvm_types => {
1380 assert(!it.byval_attr);
1381 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1382 const param_llvm_ty = try o.lowerType(param_ty);
1383 const param_alignment = param_ty.abiAlignment(zcu);
1384 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
1385 const arg_ptr = try buildAllocaInner(&wip, llvm_ty, param_alignment.toLlvm(), target);
1386 const llvm_args_start = it.llvm_index - it.types_len;
1387 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
1388 const param = wip.arg(@intCast(llvm_arg_index));
1389 const part_ptr = try o.ptraddConst(&wip, arg_ptr, offset);
1390 _ = try wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
1391 }
1392
1393 if (isByRef(param_ty, zcu)) {
1394 args.appendAssumeCapacity(arg_ptr);
1395 } else {
1396 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment.toLlvm(), ""));
1397 }
1398 },
1399 .float_array => {
1400 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1401 const param_llvm_ty = try o.lowerType(param_ty);
1402 const param = wip.arg(it.llvm_index - 1);
1403
1404 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1405 const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1406 _ = try wip.store(.normal, param, arg_ptr, alignment);
1407
1408 if (isByRef(param_ty, zcu)) {
1409 args.appendAssumeCapacity(arg_ptr);
1410 } else {
1411 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1412 }
1413 },
1414 .i32_array, .i64_array => {
1415 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1416 const param_llvm_ty = try o.lowerType(param_ty);
1417 const param = wip.arg(it.llvm_index - 1);
1418
1419 const alignment = param_ty.abiAlignment(zcu).toLlvm();
1420 const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1421 _ = try wip.store(.normal, param, arg_ptr, alignment);
1422
1423 if (isByRef(param_ty, zcu)) {
1424 args.appendAssumeCapacity(arg_ptr);
1425 } else {
1426 args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
1427 }
1428 },
1429 }
1430 }
1431
1432 break :implicit_args .{ ret_ptr, err_ret_trace };
1433 };
1434
1435 const file, const subprogram = if (!wip.strip) debug_info: {
1436 const file = try o.getDebugFile(file_scope);1281 const file = try o.getDebugFile(file_scope);
14371282
1438 const line_number = zcu.navSrcLine(func.owner_nav) + 1;1283 const line_number = zcu.navSrcLine(func.owner_nav) + 1;
...@@ -1498,11 +1343,12 @@ pub const Object = struct {...@@ -1498,11 +1343,12 @@ pub const Object = struct {
1498 .gpa = gpa,1343 .gpa = gpa,
1499 .air = air.*,1344 .air = air.*,
1500 .liveness = liveness.*.?,1345 .liveness = liveness.*.?,
1501 .wip = wip,1346 .wip = try .init(&o.builder, .{
1347 .function = llvm_function,
1348 .strip = owner_mod.strip,
1349 }),
1502 .is_naked = fn_info.cc == .naked,1350 .is_naked = fn_info.cc == .naked,
1503 .fuzz = fuzz,1351 .fuzz = fuzz,
1504 .ret_ptr = ret_ptr,
1505 .args = args.items,
1506 .arg_index = 0,1352 .arg_index = 0,
1507 .arg_inline_index = 0,1353 .arg_inline_index = 0,
1508 .func_inst_table = .empty,1354 .func_inst_table = .empty,
...@@ -1516,14 +1362,18 @@ pub const Object = struct {...@@ -1516,14 +1362,18 @@ pub const Object = struct {
1516 .base_line = zcu.navSrcLine(func.owner_nav),1362 .base_line = zcu.navSrcLine(func.owner_nav),
1517 .prev_dbg_line = 0,1363 .prev_dbg_line = 0,
1518 .prev_dbg_column = 0,1364 .prev_dbg_column = 0,
1519 .err_ret_trace = err_ret_trace,
1520 .disable_intrinsics = disable_intrinsics,1365 .disable_intrinsics = disable_intrinsics,
1521 .allowzero_access = false,1366 .allowzero_access = false,
1367
1368 .ret_ptr = undefined, // populated by `genMainBody`
1369 .err_ret_trace = undefined, // populated by `genMainBody`
1370 .args = undefined, // populated by `genMainBody`
1522 };1371 };
1523 defer fg.deinit();1372 defer fg.deinit();
1524 deinit_wip = false;
15251373
1526 try fg.genBody(air.getMainBody(), .poi);1374 fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
1375
1376 try fg.genMainBody();
15271377
1528 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole1378 // If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
1529 // function as considering null pointers valid so that LLVM's optimizers don't remove these1379 // function as considering null pointers valid so that LLVM's optimizers don't remove these
...@@ -1589,10 +1439,10 @@ pub const Object = struct {...@@ -1589,10 +1439,10 @@ pub const Object = struct {
1589 // represent (because it doesn't have runtime bits), we instead lower as the zero-size1439 // represent (because it doesn't have runtime bits), we instead lower as the zero-size
1590 // type `[0 x i8]`. I don't think the type on an extern declaration actually does much1440 // type `[0 x i8]`. I don't think the type on an extern declaration actually does much
1591 // anyway.1441 // anyway.
1592 if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);1442 if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
1593 break :ty try o.builder.arrayType(0, .i8);1443 break :ty try o.builder.arrayType(0, .i8);
1594 } else if (nav_ty.hasRuntimeBits(zcu)) ty: {1444 } else if (nav_ty.hasRuntimeBits(zcu)) ty: {
1595 break :ty try o.lowerType(nav_ty);1445 break :ty try o.lowerType(nav_ty, .in_memory);
1596 } else {1446 } else {
1597 // This is a non-extern zero-bit `Nav`---we're not interested in it.1447 // This is a non-extern zero-bit `Nav`---we're not interested in it.
1598 // TODO: we might need to rethink this a little under incremental compilation. If a1448 // TODO: we might need to rethink this a little under incremental compilation. If a
...@@ -1685,7 +1535,7 @@ pub const Object = struct {...@@ -1685,7 +1535,7 @@ pub const Object = struct {
1685 llvm_variable.setAlignment(llvm_align, &o.builder);1535 llvm_variable.setAlignment(llvm_align, &o.builder);
1686 llvm_variable.setSection(llvm_section, &o.builder);1536 llvm_variable.setSection(llvm_section, &o.builder);
1687 llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);1537 llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
1688 try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);1538 try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
1689 llvm_variable.setThreadLocal(tl: {1539 llvm_variable.setThreadLocal(tl: {
1690 if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;1540 if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
1691 break :tl .default;1541 break :tl .default;
...@@ -2283,7 +2133,7 @@ pub const Object = struct {...@@ -2283,7 +2133,7 @@ pub const Object = struct {
2283 defer debug_param_types.deinit(gpa);2133 defer debug_param_types.deinit(gpa);
22842134
2285 // Return type goes first.2135 // Return type goes first.
2286 if (firstParamSRet(fn_info, zcu, target)) {2136 if (try fnReturnStrat(o, fn_info) == .sret) {
2287 // Actual return type is void, then first arg is the sret pointer.2137 // Actual return type is void, then first arg is the sret pointer.
2288 const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));2138 const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
2289 debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));2139 debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
...@@ -2831,12 +2681,12 @@ pub const Object = struct {...@@ -2831,12 +2681,12 @@ pub const Object = struct {
2831 if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);2681 if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
28322682
2833 var it = iterateParamTypes(o, fn_info);2683 var it = iterateParamTypes(o, fn_info);
2834 if (firstParamSRet(fn_info, zcu, target)) {2684 if (try fnReturnStrat(o, fn_info) == .sret) {
2835 // Sret pointers must not be address 02685 // Sret pointers must not be address 0
2836 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);2686 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2837 try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);2687 try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
28382688
2839 const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));2689 const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type), .in_memory);
2840 try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);2690 try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2841 it.llvm_index += 1;2691 it.llvm_index += 1;
2842 } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {2692 } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {
...@@ -2878,9 +2728,7 @@ pub const Object = struct {...@@ -2878,9 +2728,7 @@ pub const Object = struct {
2878 },2728 },
2879 .byref => {2729 .byref => {
2880 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);2730 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
2881 const param_llvm_ty = try o.lowerType(param_ty);2731 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
2882 const alignment = param_ty.abiAlignment(zcu);
2883 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
2884 },2732 },
2885 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),2733 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2886 .slice => {2734 .slice => {
...@@ -2990,14 +2838,31 @@ pub const Object = struct {...@@ -2990,14 +2838,31 @@ pub const Object = struct {
2990 }2838 }
2991 }2839 }
29922840
2993 pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {2841 pub const TypeRepr = enum {
2994 return o.builder.intType(o.zcu.errorSetBits());2842 /// The representation of the type when it is being manipulated as a value in a function.
2843 /// e.g. Zig `u5` -> LLVM `i5`
2844 by_value,
2845 /// The representation of the type when it is stored in memory.
2846 /// e.g. Zig `u5` -> LLVM `i8`
2847 in_memory,
2848 };
2849
2850 pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
2851 return o.builder.intType(switch (repr) {
2852 .by_value => o.zcu.errorSetBits(),
2853 .in_memory => @intCast(Type.anyerror.abiSize(o.zcu) * 8),
2854 });
2995 }2855 }
29962856
2997 pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {2857 pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
2998 const zcu = o.zcu;2858 const zcu = o.zcu;
2999 const target = zcu.getTarget();2859 const target = zcu.getTarget();
3000 const ip = &zcu.intern_pool;2860 const ip = &zcu.intern_pool;
2861
2862 if (repr == .by_value) {
2863 assert(!isByRef(t, zcu)); // by-ref types must only be manipulated in memory
2864 }
2865
3001 return switch (t.toIntern()) {2866 return switch (t.toIntern()) {
3002 .u0_type => unreachable, // no runtime bits2867 .u0_type => unreachable, // no runtime bits
3003 inline .u1_type,2868 inline .u1_type,
...@@ -3013,7 +2878,10 @@ pub const Object = struct {...@@ -3013,7 +2878,10 @@ pub const Object = struct {
3013 .u80_type,2878 .u80_type,
3014 .u128_type,2879 .u128_type,
3015 .i128_type,2880 .i128_type,
3016 => |tag| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),2881 => |tag| switch (repr) {
2882 .by_value => @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
2883 .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
2884 },
3017 .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),2885 .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
3018 inline .c_char_type,2886 inline .c_char_type,
3019 .c_short_type,2887 .c_short_type,
...@@ -3048,7 +2916,7 @@ pub const Object = struct {...@@ -3048,7 +2916,7 @@ pub const Object = struct {
3048 return .i8;2916 return .i8;
3049 },2917 },
3050 .bool_type => .i1,2918 .bool_type => .i1,
3051 .anyerror_type => try o.errorIntType(),2919 .anyerror_type => try o.errorIntType(repr),
3052 .void_type => unreachable, // no runtime bits2920 .void_type => unreachable, // no runtime bits
3053 .type_type => unreachable, // no runtime bits2921 .type_type => unreachable, // no runtime bits
3054 .comptime_int_type => unreachable, // no runtime bits2922 .comptime_int_type => unreachable, // no runtime bits
...@@ -3068,10 +2936,10 @@ pub const Object = struct {...@@ -3068,10 +2936,10 @@ pub const Object = struct {
3068 => .ptr,2936 => .ptr,
3069 .slice_const_u8_type,2937 .slice_const_u8_type,
3070 .slice_const_u8_sentinel_0_type,2938 .slice_const_u8_sentinel_0_type,
3071 => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),2939 => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
3072 .anyerror_void_error_union_type,2940 .anyerror_void_error_union_type,
3073 .adhoc_inferred_error_set_type,2941 .adhoc_inferred_error_set_type,
3074 => try o.errorIntType(),2942 => try o.errorIntType(repr),
3075 .generic_poison_type => unreachable,2943 .generic_poison_type => unreachable,
3076 // values, not types2944 // values, not types
3077 .undef,2945 .undef,
...@@ -3097,7 +2965,10 @@ pub const Object = struct {...@@ -3097,7 +2965,10 @@ pub const Object = struct {
3097 .none,2965 .none,
3098 => unreachable,2966 => unreachable,
3099 else => switch (ip.indexToKey(t.toIntern())) {2967 else => switch (ip.indexToKey(t.toIntern())) {
3100 .int_type => |int_type| try o.builder.intType(int_type.bits),2968 .int_type => |int_type| switch (repr) {
2969 .by_value => try o.builder.intType(int_type.bits),
2970 .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
2971 },
3101 .ptr_type => |ptr_type| type: {2972 .ptr_type => |ptr_type| type: {
3102 const ptr_ty = try o.builder.ptrType(2973 const ptr_ty = try o.builder.ptrType(
3103 toLlvmAddressSpace(ptr_type.flags.address_space, target),2974 toLlvmAddressSpace(ptr_type.flags.address_space, target),
...@@ -3106,18 +2977,18 @@ pub const Object = struct {...@@ -3106,18 +2977,18 @@ pub const Object = struct {
3106 .one, .many, .c => ptr_ty,2977 .one, .many, .c => ptr_ty,
3107 .slice => try o.builder.structType(.normal, &.{2978 .slice => try o.builder.structType(.normal, &.{
3108 ptr_ty,2979 ptr_ty,
3109 try o.lowerType(.usize),2980 try o.lowerType(.usize, repr),
3110 }),2981 }),
3111 };2982 };
3112 },2983 },
3113 .array_type => |array_type| o.builder.arrayType(2984 .array_type => |array_type| o.builder.arrayType(
3114 array_type.lenIncludingSentinel(),2985 array_type.lenIncludingSentinel(),
3115 try o.lowerType(.fromInterned(array_type.child)),2986 try o.lowerType(.fromInterned(array_type.child), repr),
3116 ),2987 ),
3117 .vector_type => |vector_type| o.builder.vectorType(2988 .vector_type => |vector_type| o.builder.vectorType(
3118 .normal,2989 .normal,
3119 vector_type.len,2990 vector_type.len,
3120 try o.lowerType(.fromInterned(vector_type.child)),2991 try o.lowerType(.fromInterned(vector_type.child), .by_value),
3121 ),2992 ),
3122 .opt_type => |child_ty| {2993 .opt_type => |child_ty| {
3123 // Must stay in sync with `opt_payload` logic in `lowerPtr`.2994 // Must stay in sync with `opt_payload` logic in `lowerPtr`.
...@@ -3127,8 +2998,11 @@ pub const Object = struct {...@@ -3127,8 +2998,11 @@ pub const Object = struct {
3127 .runtime, .partially_comptime => {},2998 .runtime, .partially_comptime => {},
3128 }2999 }
31293000
3130 const payload_ty = try o.lowerType(.fromInterned(child_ty));3001 if (t.optionalReprIsPayload(zcu)) {
3131 if (t.optionalReprIsPayload(zcu)) return payload_ty;3002 return o.lowerType(.fromInterned(child_ty), repr);
3003 }
3004
3005 const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
31323006
3133 comptime assert(optional_layout_version == 3);3007 comptime assert(optional_layout_version == 3);
3134 var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };3008 var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
...@@ -3146,7 +3020,7 @@ pub const Object = struct {...@@ -3146,7 +3020,7 @@ pub const Object = struct {
3146 .error_union_type => |error_union_type| {3020 .error_union_type => |error_union_type| {
3147 // Must stay in sync with `codegen.errUnionPayloadOffset`.3021 // Must stay in sync with `codegen.errUnionPayloadOffset`.
3148 // See logic in `lowerPtr`.3022 // See logic in `lowerPtr`.
3149 const error_type = try o.errorIntType();3023 const error_type = try o.errorIntType(repr);
31503024
3151 switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {3025 switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
3152 .fully_comptime => unreachable,3026 .fully_comptime => unreachable,
...@@ -3154,7 +3028,7 @@ pub const Object = struct {...@@ -3154,7 +3028,7 @@ pub const Object = struct {
3154 .runtime, .partially_comptime => {},3028 .runtime, .partially_comptime => {},
3155 }3029 }
31563030
3157 const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));3031 const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
31583032
3159 const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);3033 const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3160 const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));3034 const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
...@@ -3189,16 +3063,14 @@ pub const Object = struct {...@@ -3189,16 +3063,14 @@ pub const Object = struct {
3189 },3063 },
3190 .simple_type => unreachable,3064 .simple_type => unreachable,
3191 .struct_type => {3065 .struct_type => {
3192 if (o.type_map.get(t.toIntern())) |value| return value;
3193
3194 const struct_type = ip.loadStructType(t.toIntern());3066 const struct_type = ip.loadStructType(t.toIntern());
31953067
3196 if (struct_type.layout == .@"packed") {3068 if (struct_type.layout == .@"packed") {
3197 const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));3069 return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
3198 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3199 return int_ty;
3200 }3070 }
32013071
3072 if (o.type_map.get(t.toIntern())) |value| return value;
3073
3202 assert(struct_type.size > 0);3074 assert(struct_type.size > 0);
32033075
3204 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;3076 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
...@@ -3232,7 +3104,7 @@ pub const Object = struct {...@@ -3232,7 +3104,7 @@ pub const Object = struct {
32323104
3233 if (!field_ty.hasRuntimeBits(zcu)) continue;3105 if (!field_ty.hasRuntimeBits(zcu)) continue;
32343106
3235 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));3107 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
32363108
3237 offset += field_ty.abiSize(zcu);3109 offset += field_ty.abiSize(zcu);
3238 }3110 }
...@@ -3288,7 +3160,7 @@ pub const Object = struct {...@@ -3288,7 +3160,7 @@ pub const Object = struct {
3288 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {3160 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3289 continue;3161 continue;
3290 }3162 }
3291 try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));3163 try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
32923164
3293 offset += Type.fromInterned(field_ty).abiSize(zcu);3165 offset += Type.fromInterned(field_ty).abiSize(zcu);
3294 }3166 }
...@@ -3305,28 +3177,24 @@ pub const Object = struct {...@@ -3305,28 +3177,24 @@ pub const Object = struct {
3305 return o.builder.structType(.normal, llvm_field_types.items);3177 return o.builder.structType(.normal, llvm_field_types.items);
3306 },3178 },
3307 .union_type => {3179 .union_type => {
3308 if (o.type_map.get(t.toIntern())) |value| return value;
3309
3310 const union_obj = ip.loadUnionType(t.toIntern());3180 const union_obj = ip.loadUnionType(t.toIntern());
33113181
3312 if (union_obj.layout == .@"packed") {3182 if (union_obj.layout == .@"packed") {
3313 const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));3183 return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
3314 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3315 return int_ty;
3316 }3184 }
33173185
3318 assert(union_obj.size > 0);
3319
3320 const layout = Type.getUnionLayout(union_obj, zcu);3186 const layout = Type.getUnionLayout(union_obj, zcu);
33213187
3322 if (layout.payload_size == 0) {3188 if (layout.payload_size == 0) {
3323 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));3189 return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3324 try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3325 return enum_tag_ty;
3326 }3190 }
33273191
3192 if (o.type_map.get(t.toIntern())) |value| return value;
3193
3194 assert(union_obj.size > 0);
3195
3328 const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);3196 const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3329 const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);3197 const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
33303198
3331 const payload_ty = ty: {3199 const payload_ty = ty: {
3332 if (layout.most_aligned_field_size == layout.payload_size) {3200 if (layout.most_aligned_field_size == layout.payload_size) {
...@@ -3352,7 +3220,7 @@ pub const Object = struct {...@@ -3352,7 +3220,7 @@ pub const Object = struct {
3352 );3220 );
3353 return ty;3221 return ty;
3354 }3222 }
3355 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));3223 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
33563224
3357 // Put the tag before or after the payload depending on which one's3225 // Put the tag before or after the payload depending on which one's
3358 // alignment is greater.3226 // alignment is greater.
...@@ -3381,9 +3249,9 @@ pub const Object = struct {...@@ -3381,9 +3249,9 @@ pub const Object = struct {
3381 return ty;3249 return ty;
3382 },3250 },
3383 .opaque_type, .spirv_type => unreachable, // no runtime bits3251 .opaque_type, .spirv_type => unreachable, // no runtime bits
3384 .enum_type => try o.lowerType(t.intTagType(zcu)),3252 .enum_type => try o.lowerType(t.intTagType(zcu), repr),
3385 .func_type => |func_type| try o.lowerFnType(t, func_type),3253 .func_type => |func_type| try o.lowerFnType(t, func_type),
3386 .error_set_type, .inferred_error_set_type => try o.errorIntType(),3254 .error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
3387 // values, not types3255 // values, not types
3388 .undef,3256 .undef,
3389 .simple_value,3257 .simple_value,
...@@ -3415,12 +3283,12 @@ pub const Object = struct {...@@ -3415,12 +3283,12 @@ pub const Object = struct {
34153283
3416 assert(fn_ty.fnHasRuntimeBits(zcu));3284 assert(fn_ty.fnHasRuntimeBits(zcu));
34173285
3418 const ret_ty = try lowerFnRetTy(o, fn_info);3286 const ret_strat = try fnReturnStrat(o, fn_info);
34193287
3420 var llvm_params: std.ArrayList(Builder.Type) = .empty;3288 var llvm_params: std.ArrayList(Builder.Type) = .empty;
3421 defer llvm_params.deinit(o.gpa);3289 defer llvm_params.deinit(o.gpa);
34223290
3423 if (firstParamSRet(fn_info, zcu, target)) {3291 if (ret_strat == .sret) {
3424 try llvm_params.append(o.gpa, .ptr);3292 try llvm_params.append(o.gpa, .ptr);
3425 }3293 }
34263294
...@@ -3435,7 +3303,7 @@ pub const Object = struct {...@@ -3435,7 +3303,7 @@ pub const Object = struct {
3435 .no_bits => continue,3303 .no_bits => continue,
3436 .byval => {3304 .byval => {
3437 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3305 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3438 try llvm_params.append(o.gpa, try o.lowerType(param_ty));3306 try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .in_memory else .by_value));
3439 },3307 },
3440 .byref, .byref_mut => {3308 .byref, .byref_mut => {
3441 try llvm_params.append(o.gpa, .ptr);3309 try llvm_params.append(o.gpa, .ptr);
...@@ -3450,7 +3318,7 @@ pub const Object = struct {...@@ -3450,7 +3318,7 @@ pub const Object = struct {
3450 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3318 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3451 try llvm_params.appendSlice(o.gpa, &.{3319 try llvm_params.appendSlice(o.gpa, &.{
3452 try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),3320 try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3453 try o.lowerType(.usize),3321 try o.lowerType(.usize, .by_value),
3454 });3322 });
3455 },3323 },
3456 .multiple_llvm_types => {3324 .multiple_llvm_types => {
...@@ -3458,7 +3326,7 @@ pub const Object = struct {...@@ -3458,7 +3326,7 @@ pub const Object = struct {
3458 },3326 },
3459 .float_array => |count| {3327 .float_array => |count| {
3460 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3328 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3461 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);3329 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .in_memory);
3462 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));3330 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3463 },3331 },
3464 .i32_array, .i64_array => |arr_len| {3332 .i32_array, .i64_array => |arr_len| {
...@@ -3470,14 +3338,19 @@ pub const Object = struct {...@@ -3470,14 +3338,19 @@ pub const Object = struct {
3470 },3338 },
3471 };3339 };
34723340
3473 return o.builder.fnType(3341 const llvm_ret_ty: Builder.Type = switch (ret_strat) {
3474 ret_ty,3342 .void, .sret => .void,
3475 llvm_params.items,3343 .by_val => try o.lowerType(.fromInterned(fn_info.return_type), .by_value),
3476 if (fn_info.is_var_args) .vararg else .normal,3344 .mem_cast => |llvm_ret_ty| llvm_ret_ty,
3477 );3345 };
3346 const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
3347 true => .vararg,
3348 false => .normal,
3349 };
3350 return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
3478 }3351 }
34793352
3480 pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {3353 pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
3481 const zcu = o.zcu;3354 const zcu = o.zcu;
3482 const ip = &zcu.intern_pool;3355 const ip = &zcu.intern_pool;
3483 const target = zcu.getTarget();3356 const target = zcu.getTarget();
...@@ -3509,7 +3382,7 @@ pub const Object = struct {...@@ -3509,7 +3382,7 @@ pub const Object = struct {
3509 .inferred_error_set_type,3382 .inferred_error_set_type,
3510 => unreachable, // types, not values3383 => unreachable, // types, not values
35113384
3512 .undef => return o.builder.undefConst(try o.lowerType(ty)),3385 .undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
3513 .simple_value => |simple_value| switch (simple_value) {3386 .simple_value => |simple_value| switch (simple_value) {
3514 .void => unreachable, // non-runtime value3387 .void => unreachable, // non-runtime value
3515 .null => unreachable, // non-runtime value3388 .null => unreachable, // non-runtime value
...@@ -3524,15 +3397,15 @@ pub const Object = struct {...@@ -3524,15 +3397,15 @@ pub const Object = struct {
3524 .int => {3397 .int => {
3525 var bigint_space: Value.BigIntSpace = undefined;3398 var bigint_space: Value.BigIntSpace = undefined;
3526 const bigint = val.toBigInt(&bigint_space, zcu);3399 const bigint = val.toBigInt(&bigint_space, zcu);
3527 const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);3400 const llvm_int_ty = try o.lowerType(ty, repr);
3528 return o.builder.bigIntConst(llvm_int_ty, bigint);3401 return o.builder.bigIntConst(llvm_int_ty, bigint);
3529 },3402 },
3530 .err => |err| {3403 .err => |err| {
3531 const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;3404 const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
3532 return o.builder.intConst(try o.errorIntType(), int);3405 return o.builder.intConst(try o.errorIntType(repr), int);
3533 },3406 },
3534 .error_union => |error_union| {3407 .error_union => |error_union| {
3535 const llvm_error_ty = try o.errorIntType();3408 const llvm_error_ty = try o.errorIntType(repr);
3536 const llvm_error_value = switch (error_union.val) {3409 const llvm_error_value = switch (error_union.val) {
3537 .err_name => |name| try o.builder.intConst(3410 .err_name => |name| try o.builder.intConst(
3538 llvm_error_ty,3411 llvm_error_ty,
...@@ -3550,8 +3423,8 @@ pub const Object = struct {...@@ -3550,8 +3423,8 @@ pub const Object = struct {
3550 const payload_align = payload_type.abiAlignment(zcu);3423 const payload_align = payload_type.abiAlignment(zcu);
3551 const error_align = Type.errorAbiAlignment(zcu);3424 const error_align = Type.errorAbiAlignment(zcu);
3552 const llvm_payload_value = switch (error_union.val) {3425 const llvm_payload_value = switch (error_union.val) {
3553 .err_name => try o.builder.undefConst(try o.lowerType(payload_type)),3426 .err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
3554 .payload => |payload| try o.lowerValue(payload),3427 .payload => |payload| try o.lowerValue(payload, repr),
3555 };3428 };
35563429
3557 var fields: [3]Builder.Type = undefined;3430 var fields: [3]Builder.Type = undefined;
...@@ -3566,7 +3439,7 @@ pub const Object = struct {...@@ -3566,7 +3439,7 @@ pub const Object = struct {
3566 fields[0] = vals[0].typeOf(&o.builder);3439 fields[0] = vals[0].typeOf(&o.builder);
3567 fields[1] = vals[1].typeOf(&o.builder);3440 fields[1] = vals[1].typeOf(&o.builder);
35683441
3569 const llvm_ty = try o.lowerType(ty);3442 const llvm_ty = try o.lowerType(ty, repr);
3570 const llvm_ty_fields = llvm_ty.structFields(&o.builder);3443 const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3571 if (llvm_ty_fields.len > 2) {3444 if (llvm_ty_fields.len > 2) {
3572 assert(llvm_ty_fields.len == 3);3445 assert(llvm_ty_fields.len == 3);
...@@ -3578,7 +3451,7 @@ pub const Object = struct {...@@ -3578,7 +3451,7 @@ pub const Object = struct {
3578 fields[0..llvm_ty_fields.len],3451 fields[0..llvm_ty_fields.len],
3579 ), vals[0..llvm_ty_fields.len]);3452 ), vals[0..llvm_ty_fields.len]);
3580 },3453 },
3581 .enum_tag => |enum_tag| o.lowerValue(enum_tag.int),3454 .enum_tag => |enum_tag| o.lowerValue(enum_tag.int, repr),
3582 .float => switch (ty.floatBits(target)) {3455 .float => switch (ty.floatBits(target)) {
3583 16 => if (backendSupportsF16(target))3456 16 => if (backendSupportsF16(target))
3584 try o.builder.halfConst(val.toFloat(f16, zcu))3457 try o.builder.halfConst(val.toFloat(f16, zcu))
...@@ -3594,9 +3467,9 @@ pub const Object = struct {...@@ -3594,9 +3467,9 @@ pub const Object = struct {
3594 else => unreachable,3467 else => unreachable,
3595 },3468 },
3596 .ptr => try o.lowerPtr(arg_val, 0),3469 .ptr => try o.lowerPtr(arg_val, 0),
3597 .slice => |slice| return o.builder.structConst(try o.lowerType(ty), &.{3470 .slice => |slice| return o.builder.structConst(try o.lowerType(ty, repr), &.{
3598 try o.lowerValue(slice.ptr),3471 try o.lowerValue(slice.ptr, repr),
3599 try o.lowerValue(slice.len),3472 try o.lowerValue(slice.len, repr),
3600 }),3473 }),
3601 .opt => |opt| {3474 .opt => |opt| {
3602 comptime assert(optional_layout_version == 3);3475 comptime assert(optional_layout_version == 3);
...@@ -3606,7 +3479,7 @@ pub const Object = struct {...@@ -3606,7 +3479,7 @@ pub const Object = struct {
3606 if (!payload_ty.hasRuntimeBits(zcu)) {3479 if (!payload_ty.hasRuntimeBits(zcu)) {
3607 return non_null_bit;3480 return non_null_bit;
3608 }3481 }
3609 const llvm_ty = try o.lowerType(ty);3482 const llvm_ty = try o.lowerType(ty, repr);
3610 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {3483 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3611 .none => switch (llvm_ty.tag(&o.builder)) {3484 .none => switch (llvm_ty.tag(&o.builder)) {
3612 .integer => try o.builder.intConst(llvm_ty, 0),3485 .integer => try o.builder.intConst(llvm_ty, 0),
...@@ -3614,15 +3487,15 @@ pub const Object = struct {...@@ -3614,15 +3487,15 @@ pub const Object = struct {
3614 .structure => try o.builder.zeroInitConst(llvm_ty),3487 .structure => try o.builder.zeroInitConst(llvm_ty),
3615 else => unreachable,3488 else => unreachable,
3616 },3489 },
3617 else => |payload| try o.lowerValue(payload),3490 else => |payload| try o.lowerValue(payload, repr),
3618 };3491 };
3619 assert(payload_ty.zigTypeTag(zcu) != .@"fn");3492 assert(payload_ty.zigTypeTag(zcu) != .@"fn");
36203493
3621 var fields: [3]Builder.Type = undefined;3494 var fields: [3]Builder.Type = undefined;
3622 var vals: [3]Builder.Constant = undefined;3495 var vals: [3]Builder.Constant = undefined;
3623 vals[0] = switch (opt.val) {3496 vals[0] = switch (opt.val) {
3624 .none => try o.builder.undefConst(try o.lowerType(payload_ty)),3497 .none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
3625 else => |payload| try o.lowerValue(payload),3498 else => |payload| try o.lowerValue(payload, repr),
3626 };3499 };
3627 vals[1] = non_null_bit;3500 vals[1] = non_null_bit;
3628 fields[0] = vals[0].typeOf(&o.builder);3501 fields[0] = vals[0].typeOf(&o.builder);
...@@ -3639,14 +3512,14 @@ pub const Object = struct {...@@ -3639,14 +3512,14 @@ pub const Object = struct {
3639 fields[0..llvm_ty_fields.len],3512 fields[0..llvm_ty_fields.len],
3640 ), vals[0..llvm_ty_fields.len]);3513 ), vals[0..llvm_ty_fields.len]);
3641 },3514 },
3642 .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val),3515 .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val, repr),
3643 .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {3516 .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {
3644 .array_type => |array_type| switch (aggregate.storage) {3517 .array_type => |array_type| switch (aggregate.storage) {
3645 .bytes => |bytes| try o.builder.stringConst(try o.builder.string(3518 .bytes => |bytes| try o.builder.stringConst(try o.builder.string(
3646 bytes.toSlice(array_type.lenIncludingSentinel(), ip),3519 bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3647 )),3520 )),
3648 .elems => |elems| {3521 .elems => |elems| {
3649 const array_ty = try o.lowerType(ty);3522 const array_ty = try o.lowerType(ty, repr);
3650 const elem_ty = array_ty.childType(&o.builder);3523 const elem_ty = array_ty.childType(&o.builder);
3651 assert(elems.len == array_ty.aggregateLen(&o.builder));3524 assert(elems.len == array_ty.aggregateLen(&o.builder));
36523525
...@@ -3664,7 +3537,7 @@ pub const Object = struct {...@@ -3664,7 +3537,7 @@ pub const Object = struct {
36643537
3665 var need_unnamed = false;3538 var need_unnamed = false;
3666 for (vals, fields, elems) |*result_val, *result_field, elem| {3539 for (vals, fields, elems) |*result_val, *result_field, elem| {
3667 result_val.* = try o.lowerValue(elem);3540 result_val.* = try o.lowerValue(elem, repr);
3668 result_field.* = result_val.typeOf(&o.builder);3541 result_field.* = result_val.typeOf(&o.builder);
3669 if (result_field.* != elem_ty) need_unnamed = true;3542 if (result_field.* != elem_ty) need_unnamed = true;
3670 }3543 }
...@@ -3676,7 +3549,7 @@ pub const Object = struct {...@@ -3676,7 +3549,7 @@ pub const Object = struct {
3676 .repeated_elem => |elem| {3549 .repeated_elem => |elem| {
3677 const len: usize = @intCast(array_type.len);3550 const len: usize = @intCast(array_type.len);
3678 const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());3551 const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3679 const array_ty = try o.lowerType(ty);3552 const array_ty = try o.lowerType(ty, repr);
3680 const elem_ty = array_ty.childType(&o.builder);3553 const elem_ty = array_ty.childType(&o.builder);
36813554
3682 const ExpectedContents = extern struct {3555 const ExpectedContents = extern struct {
...@@ -3692,12 +3565,12 @@ pub const Object = struct {...@@ -3692,12 +3565,12 @@ pub const Object = struct {
3692 defer allocator.free(fields);3565 defer allocator.free(fields);
36933566
3694 var need_unnamed = false;3567 var need_unnamed = false;
3695 @memset(vals[0..len], try o.lowerValue(elem));3568 @memset(vals[0..len], try o.lowerValue(elem, repr));
3696 @memset(fields[0..len], vals[0].typeOf(&o.builder));3569 @memset(fields[0..len], vals[0].typeOf(&o.builder));
3697 if (fields[0] != elem_ty) need_unnamed = true;3570 if (fields[0] != elem_ty) need_unnamed = true;
36983571
3699 if (array_type.sentinel != .none) {3572 if (array_type.sentinel != .none) {
3700 vals[len] = try o.lowerValue(array_type.sentinel);3573 vals[len] = try o.lowerValue(array_type.sentinel, repr);
3701 fields[len] = vals[len].typeOf(&o.builder);3574 fields[len] = vals[len].typeOf(&o.builder);
3702 if (fields[len] != elem_ty) need_unnamed = true;3575 if (fields[len] != elem_ty) need_unnamed = true;
3703 }3576 }
...@@ -3709,7 +3582,7 @@ pub const Object = struct {...@@ -3709,7 +3582,7 @@ pub const Object = struct {
3709 },3582 },
3710 },3583 },
3711 .vector_type => |vector_type| {3584 .vector_type => |vector_type| {
3712 const vector_ty = try o.lowerType(ty);3585 const vector_ty = try o.lowerType(ty, repr);
3713 switch (aggregate.storage) {3586 switch (aggregate.storage) {
3714 .bytes, .elems => {3587 .bytes, .elems => {
3715 const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;3588 const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
...@@ -3724,7 +3597,7 @@ pub const Object = struct {...@@ -3724,7 +3597,7 @@ pub const Object = struct {
3724 result_val.* = try o.builder.intConst(.i8, byte);3597 result_val.* = try o.builder.intConst(.i8, byte);
3725 },3598 },
3726 .elems => |elems| for (vals, elems) |*result_val, elem| {3599 .elems => |elems| for (vals, elems) |*result_val, elem| {
3727 result_val.* = try o.lowerValue(elem);3600 result_val.* = try o.lowerValue(elem, .by_value);
3728 },3601 },
3729 .repeated_elem => unreachable,3602 .repeated_elem => unreachable,
3730 }3603 }
...@@ -3732,12 +3605,12 @@ pub const Object = struct {...@@ -3732,12 +3605,12 @@ pub const Object = struct {
3732 },3605 },
3733 .repeated_elem => |elem| return o.builder.splatConst(3606 .repeated_elem => |elem| return o.builder.splatConst(
3734 vector_ty,3607 vector_ty,
3735 try o.lowerValue(elem),3608 try o.lowerValue(elem, .by_value),
3736 ),3609 ),
3737 }3610 }
3738 },3611 },
3739 .tuple_type => |tuple| {3612 .tuple_type => |tuple| {
3740 const struct_ty = try o.lowerType(ty);3613 const struct_ty = try o.lowerType(ty, repr);
3741 const llvm_len = struct_ty.aggregateLen(&o.builder);3614 const llvm_len = struct_ty.aggregateLen(&o.builder);
37423615
3743 const ExpectedContents = extern struct {3616 const ExpectedContents = extern struct {
...@@ -3782,8 +3655,8 @@ pub const Object = struct {...@@ -3782,8 +3655,8 @@ pub const Object = struct {
37823655
3783 vals[llvm_index] = switch (aggregate.storage) {3656 vals[llvm_index] = switch (aggregate.storage) {
3784 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),3657 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3785 .elems => |elems| try o.lowerValue(elems[field_index]),3658 .elems => |elems| try o.lowerValue(elems[field_index], repr),
3786 .repeated_elem => |elem| try o.lowerValue(elem),3659 .repeated_elem => |elem| try o.lowerValue(elem, repr),
3787 };3660 };
3788 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);3661 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3789 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])3662 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...@@ -3812,7 +3685,7 @@ pub const Object = struct {...@@ -3812,7 +3685,7 @@ pub const Object = struct {
3812 },3685 },
3813 .struct_type => {3686 .struct_type => {
3814 const struct_type = ip.loadStructType(ty.toIntern());3687 const struct_type = ip.loadStructType(ty.toIntern());
3815 const struct_ty = try o.lowerType(ty);3688 const struct_ty = try o.lowerType(ty, repr);
3816 assert(struct_type.layout != .@"packed");3689 assert(struct_type.layout != .@"packed");
3817 const llvm_len = struct_ty.aggregateLen(&o.builder);3690 const llvm_len = struct_ty.aggregateLen(&o.builder);
38183691
...@@ -3856,8 +3729,8 @@ pub const Object = struct {...@@ -3856,8 +3729,8 @@ pub const Object = struct {
38563729
3857 vals[llvm_index] = switch (aggregate.storage) {3730 vals[llvm_index] = switch (aggregate.storage) {
3858 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),3731 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3859 .elems => |elems| try o.lowerValue(elems[field_index]),3732 .elems => |elems| try o.lowerValue(elems[field_index], repr),
3860 .repeated_elem => |elem| try o.lowerValue(elem),3733 .repeated_elem => |elem| try o.lowerValue(elem, repr),
3861 };3734 };
3862 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);3735 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3863 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])3736 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...@@ -3887,9 +3760,9 @@ pub const Object = struct {...@@ -3887,9 +3760,9 @@ pub const Object = struct {
3887 else => unreachable,3760 else => unreachable,
3888 },3761 },
3889 .un => |un| {3762 .un => |un| {
3890 const union_ty = try o.lowerType(ty);3763 const union_ty = try o.lowerType(ty, repr);
3891 const layout = ty.unionGetLayout(zcu);3764 const layout = ty.unionGetLayout(zcu);
3892 if (layout.payload_size == 0) return o.lowerValue(un.tag);3765 if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
38933766
3894 const union_obj = zcu.typeToUnion(ty).?;3767 const union_obj = zcu.typeToUnion(ty).?;
3895 const container_layout = union_obj.layout;3768 const container_layout = union_obj.layout;
...@@ -3910,7 +3783,7 @@ pub const Object = struct {...@@ -3910,7 +3783,7 @@ pub const Object = struct {
3910 const padding_len = layout.payload_size;3783 const padding_len = layout.payload_size;
3911 break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));3784 break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3912 }3785 }
3913 const payload = try o.lowerValue(un.val);3786 const payload = try o.lowerValue(un.val, repr);
3914 const payload_ty = payload.typeOf(&o.builder);3787 const payload_ty = payload.typeOf(&o.builder);
3915 if (payload_ty != union_ty.structFields(&o.builder)[3788 if (payload_ty != union_ty.structFields(&o.builder)[
3916 @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))3789 @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
...@@ -3925,7 +3798,7 @@ pub const Object = struct {...@@ -3925,7 +3798,7 @@ pub const Object = struct {
3925 );3798 );
3926 } else p: {3799 } else p: {
3927 assert(layout.tag_size == 0);3800 assert(layout.tag_size == 0);
3928 const union_val = try o.lowerValue(un.val);3801 const union_val = try o.lowerValue(un.val, repr);
3929 need_unnamed = true;3802 need_unnamed = true;
3930 break :p union_val;3803 break :p union_val;
3931 };3804 };
...@@ -3935,7 +3808,7 @@ pub const Object = struct {...@@ -3935,7 +3808,7 @@ pub const Object = struct {
3935 try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})3808 try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3936 else3809 else
3937 union_ty, &.{payload});3810 union_ty, &.{payload});
3938 const tag = try o.lowerValue(un.tag);3811 const tag = try o.lowerValue(un.tag, repr);
3939 const tag_ty = tag.typeOf(&o.builder);3812 const tag_ty = tag.typeOf(&o.builder);
3940 var fields: [3]Builder.Type = undefined;3813 var fields: [3]Builder.Type = undefined;
3941 var vals: [3]Builder.Constant = undefined;3814 var vals: [3]Builder.Constant = undefined;
...@@ -3989,8 +3862,8 @@ pub const Object = struct {...@@ -3989,8 +3862,8 @@ pub const Object = struct {
3989 },3862 },
3990 .int => try o.builder.castConst(3863 .int => try o.builder.castConst(
3991 .inttoptr,3864 .inttoptr,
3992 try o.builder.intConst(try o.lowerType(.usize), offset),3865 try o.builder.intConst(try o.lowerType(.usize, .by_value), offset),
3993 try o.lowerType(.fromInterned(ptr.ty)),3866 try o.lowerType(.fromInterned(ptr.ty), .by_value),
3994 ),3867 ),
3995 .eu_payload => |eu_ptr| try o.lowerPtr(3868 .eu_payload => |eu_ptr| try o.lowerPtr(
3996 eu_ptr,3869 eu_ptr,
...@@ -4037,7 +3910,7 @@ pub const Object = struct {...@@ -4037,7 +3910,7 @@ pub const Object = struct {
4037 @"addrspace": std.lang.AddressSpace,3910 @"addrspace": std.lang.AddressSpace,
4038 ) Allocator.Error!Builder.Constant {3911 ) Allocator.Error!Builder.Constant {
4039 const addr: u64 = @"align".toByteUnits().?;3912 const addr: u64 = @"align".toByteUnits().?;
4040 const llvm_usize = try o.lowerType(.usize);3913 const llvm_usize = try o.lowerType(.usize, .by_value);
4041 const llvm_addr = try o.builder.intConst(llvm_usize, addr);3914 const llvm_addr = try o.builder.intConst(llvm_usize, addr);
4042 const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));3915 const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
4043 return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);3916 return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
...@@ -4074,17 +3947,18 @@ pub const Object = struct {...@@ -4074,17 +3947,18 @@ pub const Object = struct {
4074 if (gop.found_existing) {3947 if (gop.found_existing) {
4075 // Keep the greater of the two alignments.3948 // Keep the greater of the two alignments.
4076 const llvm_variable = gop.value_ptr.*;3949 const llvm_variable = gop.value_ptr.*;
4077 const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));3950 const llvm_old_align = llvm_variable.getAlignment(&o.builder);
4078 llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);3951 const llvm_new_align = llvm_old_align.max(@"align".toLlvm());
3952 llvm_variable.setAlignment(llvm_new_align, &o.builder);
4079 return llvm_variable.ptrConst(&o.builder).global.toConst();3953 return llvm_variable.ptrConst(&o.builder).global.toConst();
4080 }3954 }
4081 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));3955 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
40823956
4083 const llvm_ty = try o.lowerType(uav_ty);3957 const llvm_ty = try o.lowerType(uav_ty, .in_memory);
4084 const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});3958 const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
4085 const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);3959 const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
4086 gop.value_ptr.* = llvm_variable;3960 gop.value_ptr.* = llvm_variable;
4087 try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);3961 try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
4088 llvm_variable.setMutability(.constant, &o.builder);3962 llvm_variable.setMutability(.constant, &o.builder);
4089 llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);3963 llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
4090 const llvm_global = llvm_variable.ptrConst(&o.builder).global;3964 const llvm_global = llvm_variable.ptrConst(&o.builder).global;
...@@ -4156,7 +4030,7 @@ pub const Object = struct {...@@ -4156,7 +4030,7 @@ pub const Object = struct {
4156 .x86_64_interrupt,4030 .x86_64_interrupt,
4157 .x86_interrupt,4031 .x86_interrupt,
4158 => {4032 => {
4159 const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));4033 const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
4160 try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);4034 try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4161 },4035 },
4162 }4036 }
...@@ -4178,14 +4052,15 @@ pub const Object = struct {...@@ -4178,14 +4052,15 @@ pub const Object = struct {
4178 o: *Object,4052 o: *Object,
4179 attributes: *Builder.FunctionAttributes.Wip,4053 attributes: *Builder.FunctionAttributes.Wip,
4180 llvm_arg_i: u32,4054 llvm_arg_i: u32,
4181 alignment: Builder.Alignment,
4182 byval: bool,4055 byval: bool,
4183 param_llvm_ty: Builder.Type,4056 param_ty: Type,
4184 ) Allocator.Error!void {4057 ) Allocator.Error!void {
4058 const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
4059 const alignment = param_ty.abiAlignment(o.zcu).toLlvm();
4185 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);4060 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4186 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);4061 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4187 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);4062 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);
4188 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);4063 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
4189 }4064 }
41904065
4191 pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {4066 pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
...@@ -4210,7 +4085,7 @@ pub const Object = struct {...@@ -4210,7 +4085,7 @@ pub const Object = struct {
4210 pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {4085 pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
4211 const builder = &o.builder;4086 const builder = &o.builder;
4212 if (o.errors_len_variable == .none) {4087 if (o.errors_len_variable == .none) {
4213 const llvm_err_int_ty = try o.errorIntType();4088 const llvm_err_int_ty = try o.errorIntType(.in_memory);
4214 const name = try builder.strtabString("__zig_errors_len");4089 const name = try builder.strtabString("__zig_errors_len");
4215 const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);4090 const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
4216 variable_index.setMutability(.constant, builder);4091 variable_index.setMutability(.constant, builder);
...@@ -4250,9 +4125,9 @@ pub const Object = struct {...@@ -4250,9 +4125,9 @@ pub const Object = struct {
4250 const ip = &zcu.intern_pool;4125 const ip = &zcu.intern_pool;
4251 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());4126 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
42524127
4253 const llvm_usize_ty = try o.lowerType(.usize);4128 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
4254 const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);4129 const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .by_value);
4255 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));4130 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
42564131
4257 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =4132 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4258 try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);4133 try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
...@@ -4301,7 +4176,7 @@ pub const Object = struct {...@@ -4301,7 +4176,7 @@ pub const Object = struct {
4301 const return_block = try wip.block(1, "Name");4176 const return_block = try wip.block(1, "Name");
4302 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {4177 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
4303 .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered4178 .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
4304 else => |tag_val_ip| try o.lowerValue(tag_val_ip),4179 else => |tag_val_ip| try o.lowerValue(tag_val_ip, .by_value),
4305 };4180 };
4306 try wip_switch.addCase(llvm_tag_val, return_block, &wip);4181 try wip_switch.addCase(llvm_tag_val, return_block, &wip);
43074182
...@@ -4347,7 +4222,7 @@ pub const Object = struct {...@@ -4347,7 +4222,7 @@ pub const Object = struct {
4347 const ip = &zcu.intern_pool;4222 const ip = &zcu.intern_pool;
4348 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());4223 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
43494224
4350 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));4225 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4351 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =4226 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4352 try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);4227 try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
43534228
...@@ -4374,7 +4249,7 @@ pub const Object = struct {...@@ -4374,7 +4249,7 @@ pub const Object = struct {
43744249
4375 if (loaded_enum.field_values.len > 0) {4250 if (loaded_enum.field_values.len > 0) {
4376 for (loaded_enum.field_values.get(ip)) |tag_val_ip| {4251 for (loaded_enum.field_values.get(ip)) |tag_val_ip| {
4377 const llvm_tag_val = try o.lowerValue(tag_val_ip);4252 const llvm_tag_val = try o.lowerValue(tag_val_ip, .by_value);
4378 try wip_switch.addCase(llvm_tag_val, named_block, &wip);4253 try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4379 }4254 }
4380 } else {4255 } else {
...@@ -4411,13 +4286,6 @@ pub const Object = struct {...@@ -4411,13 +4286,6 @@ pub const Object = struct {
4411 toLlvmAddressSpace(.generic, o.zcu.getTarget()),4286 toLlvmAddressSpace(.generic, o.zcu.getTarget()),
4412 );4287 );
4413 }4288 }
4414
4415 pub fn ptraddConst(o: *Object, wip: *Builder.WipFunction, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
4416 if (offset == 0) return ptr;
4417 const llvm_usize_ty = try o.lowerType(.usize);
4418 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
4419 return wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
4420 }
4421};4289};
44224290
4423const CallingConventionInfo = struct {4291const CallingConventionInfo = struct {
src/codegen/llvm/FuncGen.zig+1097-1176
...@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {...@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
164 const zcu = o.zcu;164 const zcu = o.zcu;
165 const ty = val.typeOf(zcu);165 const ty = val.typeOf(zcu);
166 if (!isByRef(ty, zcu)) {166 if (!isByRef(ty, zcu)) {
167 return o.lowerValue(val.toIntern());167 return o.lowerValue(val.toIntern(), .by_value);
168 } else {168 } else {
169 // We need a pointer to a global constant, i.e. a UAV.169 // We need a pointer to a global constant, i.e. a UAV.
170 return o.lowerUavRef(170 return o.lowerUavRef(
...@@ -175,7 +175,157 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {...@@ -175,7 +175,157 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
175 }175 }
176}176}
177177
178pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {178/// Populates `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` based on the parameters of the
179/// function type, then generates the entire function body.
180///
181/// The caller may initialize `fg.ret_ptr`, `fg.err_ret_trace`, and `fg.args` to undefined.
182pub fn genMainBody(fg: *FuncGen) TodoError!void {
183 const o = fg.object;
184 const zcu = o.zcu;
185 const ip = &zcu.intern_pool;
186 const comp = zcu.comp;
187 const gpa = comp.gpa;
188
189 const fn_ty: Type = .fromInterned(ip.getNav(fg.nav_index).resolved.?.type);
190 const fn_info = zcu.typeToFunc(fn_ty).?;
191 const param_types = fn_info.param_types.get(ip);
192
193 var it = iterateParamTypes(o, fn_info);
194
195 // Populate `fg.ret_ptr`...
196 fg.ret_ptr = switch (try fnReturnStrat(o, fn_info)) {
197 .sret => rp: {
198 defer it.llvm_index += 1;
199 break :rp fg.wip.arg(it.llvm_index);
200 },
201 else => .none,
202 };
203 // ...and `fg.err_ret_trace`...
204 if (fn_info.cc == .auto and comp.config.any_error_tracing) {
205 fg.err_ret_trace = fg.wip.arg(it.llvm_index);
206 it.llvm_index += 1;
207 } else {
208 fg.err_ret_trace = .none;
209 }
210 // ...and as for `fg.args`, we'll put all of the arguments into this ArrayList, and once that's
211 // done we'll use its buffer as `fg.args`.
212 var args: std.ArrayList(Builder.Value) = .empty;
213 defer args.deinit(gpa);
214
215 while (try it.next()) |lowering| {
216 try args.ensureUnusedCapacity(gpa, 1);
217
218 switch (lowering) {
219 .no_bits => continue,
220 .byval => {
221 assert(!it.byval_attr);
222 const param_index = it.zig_index - 1;
223 const param_ty: Type = .fromInterned(param_types[param_index]);
224 const param = fg.wip.arg(it.llvm_index - 1);
225
226 if (isByRef(param_ty, zcu)) {
227 const alignment = param_ty.abiAlignment(zcu).toLlvm();
228 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
229 // We don't need to handle non-ABI-sized integer types in memory here since they
230 // are never by-ref.
231 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
232 args.appendAssumeCapacity(arg_ptr);
233 } else {
234 args.appendAssumeCapacity(param);
235 }
236 },
237 .byref, .byref_mut => {
238 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
239 const param = fg.wip.arg(it.llvm_index - 1);
240
241 if (isByRef(param_ty, zcu)) {
242 args.appendAssumeCapacity(param);
243 } else {
244 args.appendAssumeCapacity(try fg.load(param, .none, param_ty, .normal));
245 }
246 },
247 .abi_sized_int => {
248 assert(!it.byval_attr);
249 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
250 const param = fg.wip.arg(it.llvm_index - 1);
251
252 const alignment = param_ty.abiAlignment(zcu).toLlvm();
253 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
254 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
255
256 if (isByRef(param_ty, zcu)) {
257 args.appendAssumeCapacity(arg_ptr);
258 } else {
259 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
260 }
261 },
262 .slice => {
263 assert(!it.byval_attr);
264 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
265 assert(!isByRef(param_ty, zcu));
266 const slice_val = try fg.wip.buildAggregate(
267 try o.lowerType(param_ty, .by_value),
268 &.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
269 "",
270 );
271 args.appendAssumeCapacity(slice_val);
272 },
273 .multiple_llvm_types => {
274 assert(!it.byval_attr);
275 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
276 const param_alignment = param_ty.abiAlignment(zcu);
277 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
278 const arg_ptr = try fg.buildAlloca(llvm_ty, param_alignment.toLlvm());
279 const llvm_args_start = it.llvm_index - it.types_len;
280 for (llvm_args_start.., it.offsets_buffer[0..it.types_len]) |llvm_arg_index, offset| {
281 const param = fg.wip.arg(@intCast(llvm_arg_index));
282 const part_ptr = try fg.ptraddConst(arg_ptr, offset);
283 _ = try fg.wip.store(.normal, param, part_ptr, param_alignment.offset(offset).toLlvm());
284 }
285
286 if (isByRef(param_ty, zcu)) {
287 args.appendAssumeCapacity(arg_ptr);
288 } else {
289 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
290 }
291 },
292 .float_array => {
293 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
294 const param = fg.wip.arg(it.llvm_index - 1);
295
296 const alignment = param_ty.abiAlignment(zcu).toLlvm();
297 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
298 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
299
300 if (isByRef(param_ty, zcu)) {
301 args.appendAssumeCapacity(arg_ptr);
302 } else {
303 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
304 }
305 },
306 .i32_array, .i64_array => {
307 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
308 const param = fg.wip.arg(it.llvm_index - 1);
309
310 const alignment = param_ty.abiAlignment(zcu).toLlvm();
311 const arg_ptr = try fg.buildAlloca(param.typeOfWip(&fg.wip), alignment);
312 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
313
314 if (isByRef(param_ty, zcu)) {
315 args.appendAssumeCapacity(arg_ptr);
316 } else {
317 args.appendAssumeCapacity(try fg.load(arg_ptr, .none, param_ty, .normal));
318 }
319 },
320 }
321 }
322
323 fg.args = args.items;
324
325 try fg.genBody(fg.air.getMainBody(), .poi);
326}
327
328fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
179 const o = self.object;329 const o = self.object;
180 const zcu = self.object.zcu;330 const zcu = self.object.zcu;
181 const ip = &zcu.intern_pool;331 const ip = &zcu.intern_pool;
...@@ -198,15 +348,16 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -198,15 +348,16 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
198 try fuzz.pcs.append(gpa, pc);348 try fuzz.pcs.append(gpa, pc);
199 },349 },
200 }350 }
201 for (body, 0..) |inst, i| {351 for (body) |inst| {
202 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;352 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
203353
204 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {354 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
205 // zig fmt: off355 // zig fmt: off
206356
207 // No "scalarize" legalizations are enabled, so these instructions never appear.357 // Required due to `.scalarize_bit_cast_vector_non_elementwise` being enabled.
208 .legalize_vec_elem_val => unreachable,358 .legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),
209 .legalize_vec_store_elem => unreachable,359 .legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),
360
210 // No soft float legalizations are enabled.361 // No soft float legalizations are enabled.
211 .legalize_compiler_rt_call => unreachable,362 .legalize_compiler_rt_call => unreachable,
212363
...@@ -309,29 +460,36 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -309,29 +460,36 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
309 .is_err => try self.airIsErr(inst, .ne, false),460 .is_err => try self.airIsErr(inst, .ne, false),
310 .is_err_ptr => try self.airIsErr(inst, .ne, true),461 .is_err_ptr => try self.airIsErr(inst, .ne, true),
311462
312 .alloc => try self.airAlloc(inst),463 .alloc => try self.airAlloc(inst),
313 .ret_ptr => try self.airRetPtr(inst),464 .ret_ptr => try self.airRetPtr(inst),
314 .arg => try self.airArg(inst),465 .arg => try self.airArg(inst),
315 .bitcast => try self.airBitCast(inst),466 .bit_cast => try self.airBitCast(inst),
316 .breakpoint => try self.airBreakpoint(inst),467 .ptr_cast => try self.airNopCast(inst),
317 .ret_addr => try self.airRetAddr(inst),468 .ptr_from_int => try self.airPtrFromInt(inst),
318 .frame_addr => try self.airFrameAddress(inst),469 .int_from_ptr => try self.airIntFromPtr(inst),
319 .@"try" => try self.airTry(inst, false),470 .error_cast => try self.airNopCast(inst),
320 .try_cold => try self.airTry(inst, true),471 .error_from_int => try self.airNopCast(inst),
321 .try_ptr => try self.airTryPtr(inst, false),472 .int_from_error => try self.airNopCast(inst),
322 .try_ptr_cold => try self.airTryPtr(inst, true),473 .union_from_enum => try self.airUnionFromEnum(inst),
323 .intcast => try self.airIntCast(inst, false),474 .breakpoint => try self.airBreakpoint(inst),
324 .intcast_safe => try self.airIntCast(inst, true),475 .ret_addr => try self.airRetAddr(inst),
325 .trunc => try self.airTrunc(inst),476 .frame_addr => try self.airFrameAddress(inst),
326 .fptrunc => try self.airFptrunc(inst),477 .@"try" => try self.airTry(inst, false),
327 .fpext => try self.airFpext(inst),478 .try_cold => try self.airTry(inst, true),
328 .load => try self.airLoad(inst),479 .try_ptr => try self.airTryPtr(inst, false),
329 .not => try self.airNot(inst),480 .try_ptr_cold => try self.airTryPtr(inst, true),
330 .store => try self.airStore(inst, false),481 .int_cast => try self.airIntCast(inst, false),
331 .store_safe => try self.airStore(inst, true),482 .int_cast_safe => try self.airIntCast(inst, true),
332 .assembly => try self.airAssembly(inst),483 .trunc => try self.airTrunc(inst),
333 .slice_ptr => try self.airSliceField(inst, 0),484 .fptrunc => try self.airFptrunc(inst),
334 .slice_len => try self.airSliceField(inst, 1),485 .fpext => try self.airFpext(inst),
486 .load => try self.airLoad(inst),
487 .not => try self.airNot(inst),
488 .store => try self.airStore(inst, false),
489 .store_safe => try self.airStore(inst, true),
490 .assembly => try self.airAssembly(inst),
491 .slice_ptr => try self.airSliceField(inst, 0),
492 .slice_len => try self.airSliceField(inst, 1),
335493
336 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),494 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
337 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),495 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
...@@ -400,8 +558,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -400,8 +558,8 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
400 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),558 .optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
401 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),559 .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
402560
403 .unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),561 .unwrap_errunion_payload => try self.airErrUnionPayload(inst),
404 .unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),562 .unwrap_errunion_payload_ptr => try self.airErrUnionPayloadPtr(inst),
405 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),563 .unwrap_errunion_err => try self.airErrUnionErr(inst, false),
406 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),564 .unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
407 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),565 .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
...@@ -409,9 +567,9 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air...@@ -409,9 +567,9 @@ pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air
409 .set_err_return_trace => try self.airSetErrReturnTrace(inst),567 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
410 .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),568 .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
411569
412 .wrap_optional => try self.airWrapOptional(body[i..]),570 .wrap_optional => try self.airWrapOptional(inst),
413 .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),571 .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
414 .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),572 .wrap_errunion_err => try self.airWrapErrUnionErr(inst),
415573
416 .wasm_memory_size => try self.airWasmMemorySize(inst),574 .wasm_memory_size => try self.airWasmMemorySize(inst),
417 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),575 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),
...@@ -594,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -594,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
594 break :llvm_fn try self.resolveInst(air_call.callee);752 break :llvm_fn try self.resolveInst(air_call.callee);
595 };753 };
596 const target = zcu.getTarget();754 const target = zcu.getTarget();
597 const sret = firstParamSRet(fn_info, zcu, target);755 const ret_strat = try fnReturnStrat(o, fn_info);
598756
599 var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);757 var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
600 defer llvm_args.deinit();758 defer llvm_args.deinit();
...@@ -612,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -612,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
612 .no_suspend, .always_inline, .compile_time => unreachable,770 .no_suspend, .always_inline, .compile_time => unreachable,
613 }771 }
614772
615 const ret_ptr = if (sret) ret_ptr: {773 const sret_alloc: ?Builder.Value = switch (ret_strat) {
616 const llvm_ret_ty = try o.lowerType(return_type);774 .sret => sret_alloc: {
617 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);775 try attributes.addParamAttr(0, .{ .sret = try o.lowerType(return_type, .in_memory) }, &o.builder);
618776
619 const alignment = return_type.abiAlignment(zcu).toLlvm();777 const ptr = try self.buildZigAlloca(return_type, .none);
620 const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);778 try llvm_args.append(ptr);
621 try llvm_args.append(ret_ptr);779 break :sret_alloc ptr;
622 break :ret_ptr ret_ptr;780 },
623 } else ret_ptr: {781 else => sret_alloc: {
624 if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {782 if (ccAbiPromoteInt(fn_info.cc, zcu, .fromInterned(fn_info.return_type))) |s| switch (s) {
625 .signed => try attributes.addRetAttr(.signext, &o.builder),783 .signed => try attributes.addRetAttr(.signext, &o.builder),
626 .unsigned => try attributes.addRetAttr(.zeroext, &o.builder),784 .unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
627 };785 };
628 break :ret_ptr null;786 break :sret_alloc null;
787 },
629 };788 };
630789
631 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;790 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
...@@ -641,9 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -641,9 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
641 const arg = args[it.zig_index - 1];800 const arg = args[it.zig_index - 1];
642 const param_ty = self.typeOf(arg);801 const param_ty = self.typeOf(arg);
643 const llvm_arg = try self.resolveInst(arg);802 const llvm_arg = try self.resolveInst(arg);
644 const llvm_param_ty = try o.lowerType(param_ty);
645 if (isByRef(param_ty, zcu)) {803 if (isByRef(param_ty, zcu)) {
646 const alignment = param_ty.abiAlignment(zcu).toLlvm();804 const alignment = param_ty.abiAlignment(zcu).toLlvm();
805 // We don't need to handle non-ABI-sized integer types in memory here since they are
806 // never by-ref.
807 const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
647 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");808 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
648 try llvm_args.append(loaded);809 try llvm_args.append(loaded);
649 } else {810 } else {
...@@ -657,10 +818,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -657,10 +818,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
657 if (isByRef(param_ty, zcu)) {818 if (isByRef(param_ty, zcu)) {
658 try llvm_args.append(llvm_arg);819 try llvm_args.append(llvm_arg);
659 } else {820 } else {
660 const alignment = param_ty.abiAlignment(zcu).toLlvm();821 const arg_ptr = try self.buildZigAlloca(param_ty, .none);
661 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);822 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
662 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
663 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
664 try llvm_args.append(arg_ptr);823 try llvm_args.append(arg_ptr);
665 }824 }
666 },825 },
...@@ -669,15 +828,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -669,15 +828,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
669 const param_ty = self.typeOf(arg);828 const param_ty = self.typeOf(arg);
670 const llvm_arg = try self.resolveInst(arg);829 const llvm_arg = try self.resolveInst(arg);
671830
672 const alignment = param_ty.abiAlignment(zcu).toLlvm();831 const arg_ptr = try self.buildZigAlloca(param_ty, .none);
673 const param_llvm_ty = try o.lowerType(param_ty);832 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
674 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
675 if (isByRef(param_ty, zcu)) {
676 const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
677 _ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
678 } else {
679 _ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
680 }
681 try llvm_args.append(arg_ptr);833 try llvm_args.append(arg_ptr);
682 },834 },
683 .abi_sized_int => {835 .abi_sized_int => {
...@@ -694,9 +846,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -694,9 +846,9 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
694 // LLVM does not allow bitcasting structs so we must allocate846 // LLVM does not allow bitcasting structs so we must allocate
695 // a local, store as one type, and then load as another type.847 // a local, store as one type, and then load as another type.
696 const alignment = param_ty.abiAlignment(zcu).toLlvm();848 const alignment = param_ty.abiAlignment(zcu).toLlvm();
697 const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);849 const ptr = try self.buildAlloca(int_llvm_ty, alignment);
698 _ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);850 try self.store(ptr, .none, llvm_arg, param_ty, .normal);
699 const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");851 const loaded = try self.wip.load(.normal, int_llvm_ty, ptr, alignment, "");
700 try llvm_args.append(loaded);852 try llvm_args.append(loaded);
701 }853 }
702 },854 },
...@@ -711,19 +863,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -711,19 +863,10 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
711 const arg = args[it.zig_index - 1];863 const arg = args[it.zig_index - 1];
712 const param_ty = self.typeOf(arg);864 const param_ty = self.typeOf(arg);
713 const llvm_arg = try self.resolveInst(arg);865 const llvm_arg = try self.resolveInst(arg);
714 const is_by_ref = isByRef(param_ty, zcu);
715 const param_alignment = param_ty.abiAlignment(zcu);866 const param_alignment = param_ty.abiAlignment(zcu);
716 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);867 const llvm_ty = try o.builder.arrayType(it.offsets_buffer[it.types_len], .i8);
717 const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());868 const arg_ptr = try self.buildAlloca(llvm_ty, param_alignment.toLlvm());
718 if (is_by_ref) _ = try self.wip.callMemCpy(869 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
719 arg_ptr,
720 param_alignment.toLlvm(),
721 llvm_arg,
722 param_alignment.toLlvm(),
723 try o.builder.intValue(try o.lowerType(.usize), param_ty.abiSize(zcu)),
724 .normal,
725 self.disable_intrinsics,
726 ) else _ = try self.wip.store(.normal, llvm_arg, arg_ptr, param_alignment.toLlvm());
727870
728 try llvm_args.ensureUnusedCapacity(it.types_len);871 try llvm_args.ensureUnusedCapacity(it.types_len);
729 for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {872 for (it.types_buffer[0..it.types_len], it.offsets_buffer[0..it.types_len]) |field_ty, offset| {
...@@ -735,35 +878,34 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -735,35 +878,34 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
735 .float_array => |count| {878 .float_array => |count| {
736 const arg = args[it.zig_index - 1];879 const arg = args[it.zig_index - 1];
737 const arg_ty = self.typeOf(arg);880 const arg_ty = self.typeOf(arg);
738 var llvm_arg = try self.resolveInst(arg);881 const arg_val = try self.resolveInst(arg);
739 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
740 if (!isByRef(arg_ty, zcu)) {
741 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
742 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
743 llvm_arg = ptr;
744 }
745882
746 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);883 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
884 const ptr = try self.buildZigAlloca(arg_ty, .none);
885 try self.store(ptr, .none, arg_val, arg_ty, .normal);
886 break :ptr ptr;
887 } else arg_val;
888
889 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?, .in_memory);
747 const array_ty = try o.builder.arrayType(count, float_ty);890 const array_ty = try o.builder.arrayType(count, float_ty);
748891
749 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");892 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
750 try llvm_args.append(loaded);893 try llvm_args.append(loaded);
751 },894 },
752 .i32_array, .i64_array => |arr_len| {895 .i32_array, .i64_array => |arr_len| {
753 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;896 const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
754 const arg = args[it.zig_index - 1];897 const arg = args[it.zig_index - 1];
755 const arg_ty = self.typeOf(arg);898 const arg_ty = self.typeOf(arg);
756 var llvm_arg = try self.resolveInst(arg);899 const arg_val = try self.resolveInst(arg);
757 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
758 if (!isByRef(arg_ty, zcu)) {
759 const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
760 _ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
761 llvm_arg = ptr;
762 }
763900
764 const array_ty =901 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
765 try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));902 const ptr = try self.buildZigAlloca(arg_ty, .none);
766 const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");903 try self.store(ptr, .none, arg_val, arg_ty, .normal);
904 break :ptr ptr;
905 } else arg_val;
906
907 const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
908 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
767 try llvm_args.append(loaded);909 try llvm_args.append(loaded);
768 },910 },
769 };911 };
...@@ -773,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -773,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
773 {915 {
774 // Add argument attributes.916 // Add argument attributes.
775 it = iterateParamTypes(o, fn_info);917 it = iterateParamTypes(o, fn_info);
776 it.llvm_index += @intFromBool(sret);918 it.llvm_index += @intFromBool(ret_strat == .sret);
777 it.llvm_index += @intFromBool(err_return_tracing);919 it.llvm_index += @intFromBool(err_return_tracing);
778 var remaining_inreg_int = cc_info.inreg_int_params;920 var remaining_inreg_int = cc_info.inreg_int_params;
779 var remaining_inreg_float = cc_info.inreg_float_params;921 var remaining_inreg_float = cc_info.inreg_float_params;
...@@ -802,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -802,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
802 },944 },
803 .byref => {945 .byref => {
804 const param_index = it.zig_index - 1;946 const param_index = it.zig_index - 1;
805 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);947 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]);
806 const param_llvm_ty = try o.lowerType(param_ty);948 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
807 const alignment = param_ty.abiAlignment(zcu).toLlvm();
808 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
809 },949 },
810 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),950 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
811 // No attributes needed for these.951 // No attributes needed for these.
...@@ -855,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -855,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
855 },995 },
856 cc_info.llvm_cc,996 cc_info.llvm_cc,
857 try attributes.finish(&o.builder),997 try attributes.finish(&o.builder),
858 try o.lowerType(zig_fn_ty),998 try o.lowerType(zig_fn_ty, .by_value),
859 llvm_fn,999 llvm_fn,
860 llvm_args.items,1000 llvm_args.items,
861 "",1001 "",
...@@ -865,45 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -865,45 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
865 return .none;1005 return .none;
866 }1006 }
8671007
868 if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {1008 if (self.liveness.isUnused(inst)) {
869 return .none;1009 return .none;
870 }1010 }
8711011
872 const llvm_ret_ty = try o.lowerType(return_type);1012 // We exit this `switch` if we have a pointer to the return value.
873 if (ret_ptr) |rp| {1013 const ret_val_ptr: Builder.Value = switch (ret_strat) {
874 if (isByRef(return_type, zcu)) {1014 .void => return .none,
875 return rp;1015 .by_val => return call,
876 } else {
877 // our by-ref status disagrees with sret so we must load.
878 const return_alignment = return_type.abiAlignment(zcu).toLlvm();
879 return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
880 }
881 }
882
883 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
884
885 if (abi_ret_ty != llvm_ret_ty) {
886 // In this case the function return type is honoring the calling convention by having
887 // a different LLVM type than the usual one. We solve this here at the callsite
888 // by using our canonical type, then loading it if necessary.
889 const alignment = return_type.abiAlignment(zcu).toLlvm();
890 const rp = try self.buildAlloca(abi_ret_ty, alignment);
891 _ = try self.wip.store(.normal, call, rp, alignment);
892 return if (isByRef(return_type, zcu))
893 rp
894 else
895 try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
896 }
8971016
1017 .sret => sret_alloc.?,
1018 .mem_cast => |llvm_ret_ty| ret_val_ptr: {
1019 const alignment = return_type.abiAlignment(zcu).toLlvm();
1020 const ptr = try self.buildAlloca(llvm_ret_ty, alignment);
1021 _ = try self.wip.store(.normal, call, ptr, alignment);
1022 break :ret_val_ptr ptr;
1023 },
1024 };
898 if (isByRef(return_type, zcu)) {1025 if (isByRef(return_type, zcu)) {
899 // our by-ref status disagrees with sret so we must allocate, store,1026 return ret_val_ptr;
900 // and return the allocation pointer.
901 const alignment = return_type.abiAlignment(zcu).toLlvm();
902 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
903 _ = try self.wip.store(.normal, call, rp, alignment);
904 return rp;
905 } else {1027 } else {
906 return call;1028 return self.load(ret_val_ptr, .none, return_type, .normal);
907 }1029 }
908}1030}
9091031
...@@ -913,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v...@@ -913,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v
913 const target = zcu.getTarget();1035 const target = zcu.getTarget();
914 const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));1036 const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));
915 const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;1037 const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
916 const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty));1038 const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty), .by_value);
9171039
918 const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);1040 const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
9191041
...@@ -937,72 +1059,24 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -937,72 +1059,24 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
937 const zcu = o.zcu;1059 const zcu = o.zcu;
938 const ip = &zcu.intern_pool;1060 const ip = &zcu.intern_pool;
939 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;1061 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
940 const ret_ty = self.typeOf(un_op);
941
942 if (self.ret_ptr != .none) {
943 const operand = try self.resolveInst(un_op);
944 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
945 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
946 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
947 _ = try self.wip.callMemSet(
948 self.ret_ptr,
949 ret_ty.abiAlignment(zcu).toLlvm(),
950 try o.builder.intValue(.i8, 0xaa),
951 len,
952 .normal,
953 self.disable_intrinsics,
954 );
955 const owner_mod = self.ownerModule();
956 if (owner_mod.valgrind) {
957 try self.valgrindMarkUndef(self.ret_ptr, len);
958 }
959 _ = try self.wip.retVoid();
960 return;
961 }
9621062
963 const unwrapped_operand = operand.unwrap();1063 const ret_ty = self.typeOf(un_op);
964 const unwrapped_ret = self.ret_ptr.unwrap();
965
966 // Return value was stored previously
967 if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
968 _ = try self.wip.retVoid();
969 return;
970 }
9711064
972 try self.store(
973 self.ret_ptr,
974 .none,
975 operand,
976 ret_ty,
977 );
978 _ = try self.wip.retVoid();
979 return;
980 }
981 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;1065 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
982 if (!ret_ty.hasRuntimeBits(zcu)) {
983 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
984 // Functions with an empty error set are emitted with an error code
985 // return type and return zero so they can be function pointers coerced
986 // to functions that return anyerror.
987 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
988 } else {
989 _ = try self.wip.retVoid();
990 }
991 return;
992 }
9931066
994 const abi_ret_ty = try lowerFnRetTy(o, fn_info);1067 const ret_strat = try fnReturnStrat(o, fn_info);
995 const operand = try self.resolveInst(un_op);
996 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;1068 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
997 const alignment = ret_ty.abiAlignment(zcu).toLlvm();1069 const ret_ty_align = ret_ty.abiAlignment(zcu);
9981070
999 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {1071 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1000 const llvm_ret_ty = operand.typeOfWip(&self.wip);1072 const rp = switch (self.ret_ptr) {
1001 const rp = try self.buildAlloca(llvm_ret_ty, alignment);1073 .none => try self.buildZigAlloca(ret_ty, .none),
1002 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));1074 else => |rp| rp,
1075 };
1076 const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), ret_ty.abiSize(zcu));
1003 _ = try self.wip.callMemSet(1077 _ = try self.wip.callMemSet(
1004 rp,1078 rp,
1005 alignment,1079 ret_ty_align.toLlvm(),
1006 try o.builder.intValue(.i8, 0xaa),1080 try o.builder.intValue(.i8, 0xaa),
1007 len,1081 len,
1008 .normal,1082 .normal,
...@@ -1012,27 +1086,47 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -1012,27 +1086,47 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1012 if (owner_mod.valgrind) {1086 if (owner_mod.valgrind) {
1013 try self.valgrindMarkUndef(rp, len);1087 try self.valgrindMarkUndef(rp, len);
1014 }1088 }
1015 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));1089 switch (ret_strat) {
1016 return;1090 .void => unreachable, // value is undef so return type cannot be OPV
1017 }1091 .sret => {
10181092 // We just stored directly to `self.ret_ptr`.
1019 if (isByRef(ret_ty, zcu)) {1093 _ = try self.wip.retVoid();
1020 // operand is a pointer however self.ret_ptr is null so that means1094 },
1021 // we need to return a value.1095 .by_val => {
1022 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));1096 const loaded = try self.load(rp, .none, ret_ty, .normal);
1097 _ = try self.wip.ret(loaded);
1098 },
1099 .mem_cast => |llvm_abi_ret_ty| {
1100 const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1101 _ = try self.wip.ret(loaded);
1102 },
1103 }
1023 return;1104 return;
1024 }1105 }
10251106
1026 const llvm_ret_ty = operand.typeOfWip(&self.wip);1107 switch (ret_strat) {
1027 if (abi_ret_ty == llvm_ret_ty) {1108 .void => _ = try self.wip.retVoid(),
1028 _ = try self.wip.ret(operand);1109 .sret => {
1029 return;1110 const operand = try self.resolveInst(un_op);
1111 try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
1112 _ = try self.wip.retVoid();
1113 },
1114 .by_val => {
1115 assert(!isByRef(ret_ty, zcu));
1116 const operand = try self.resolveInst(un_op);
1117 _ = try self.wip.ret(operand);
1118 },
1119 .mem_cast => |llvm_ret_ty| {
1120 const operand = try self.resolveInst(un_op);
1121 const ptr: Builder.Value = if (!isByRef(ret_ty, zcu)) ptr: {
1122 const ptr = try self.buildZigAlloca(ret_ty, .none);
1123 try self.store(ptr, .none, operand, ret_ty, .normal);
1124 break :ptr ptr;
1125 } else operand;
1126 const ret_val = try self.wip.load(.normal, llvm_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1127 _ = try self.wip.ret(ret_val);
1128 },
1030 }1129 }
1031
1032 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1033 _ = try self.wip.store(.normal, operand, rp, alignment);
1034 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
1035 return;
1036}1130}
10371131
1038fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {1132fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
...@@ -1043,33 +1137,32 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {...@@ -1043,33 +1137,32 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1043 const ptr_ty = self.typeOf(un_op);1137 const ptr_ty = self.typeOf(un_op);
1044 const ret_ty = ptr_ty.childType(zcu);1138 const ret_ty = ptr_ty.childType(zcu);
1045 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;1139 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1046 if (!ret_ty.hasRuntimeBits(zcu)) {1140 const ptr = try self.resolveInst(un_op);
1047 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {1141 switch (try fnReturnStrat(o, fn_info)) {
1048 // Functions with an empty error set are emitted with an error code1142 .void => _ = try self.wip.retVoid(),
1049 // return type and return zero so they can be function pointers coerced1143 .sret => {
1050 // to functions that return anyerror.1144 assert(self.ret_ptr != .none);
1051 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1052 } else {
1053 _ = try self.wip.retVoid();1145 _ = try self.wip.retVoid();
1054 }1146 },
1055 return;1147 .by_val => {
1056 }1148 assert(self.ret_ptr == .none);
1057 if (self.ret_ptr != .none) {1149 const loaded = try self.load(ptr, .none, ret_ty, .normal);
1058 _ = try self.wip.retVoid();1150 _ = try self.wip.ret(loaded);
1059 return;1151 },
1152 .mem_cast => |llvm_abi_ret_ty| {
1153 assert(self.ret_ptr == .none);
1154 const ret_ty_align = ret_ty.abiAlignment(zcu);
1155 const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1156 _ = try self.wip.ret(loaded);
1157 },
1060 }1158 }
1061 const ptr = try self.resolveInst(un_op);
1062 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1063 const alignment = ret_ty.abiAlignment(zcu).toLlvm();
1064 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
1065 return;
1066}1159}
10671160
1068fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {1161fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
1069 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1162 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1070 const list = try self.resolveInst(ty_op.operand);1163 const list = try self.resolveInst(ty_op.operand);
1071 const arg_ty = ty_op.ty.toType();1164 const arg_ty = ty_op.ty.toType();
1072 const llvm_arg_ty = try self.object.lowerType(arg_ty);1165 const llvm_arg_ty = try self.object.lowerType(arg_ty, .by_value);
10731166
1074 return self.wip.vaArg(list, llvm_arg_ty, "");1167 return self.wip.vaArg(list, llvm_arg_ty, "");
1075}1168}
...@@ -1080,16 +1173,14 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -1080,16 +1173,14 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
1080 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1173 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1081 const src_list = try self.resolveInst(ty_op.operand);1174 const src_list = try self.resolveInst(ty_op.operand);
1082 const va_list_ty = ty_op.ty.toType();1175 const va_list_ty = ty_op.ty.toType();
1083 const llvm_va_list_ty = try o.lowerType(va_list_ty);
10841176
1085 const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();1177 const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1086 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
10871178
1088 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");1179 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1089 return if (isByRef(va_list_ty, zcu))1180 return if (isByRef(va_list_ty, zcu))
1090 dest_list1181 dest_list
1091 else1182 else
1092 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");1183 try self.load(dest_list, .none, va_list_ty, .normal);
1093}1184}
10941185
1095fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {1186fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -1104,16 +1195,14 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -1104,16 +1195,14 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
1104 const o = self.object;1195 const o = self.object;
1105 const zcu = o.zcu;1196 const zcu = o.zcu;
1106 const va_list_ty = self.typeOfIndex(inst);1197 const va_list_ty = self.typeOfIndex(inst);
1107 const llvm_va_list_ty = try o.lowerType(va_list_ty);
11081198
1109 const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();1199 const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1110 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
11111200
1112 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");1201 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1113 return if (isByRef(va_list_ty, zcu))1202 return if (isByRef(va_list_ty, zcu))
1114 dest_list1203 dest_list
1115 else1204 else
1116 try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");1205 try self.load(dest_list, .none, va_list_ty, .normal);
1117}1206}
11181207
1119fn airCmp(1208fn airCmp(
...@@ -1147,13 +1236,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -1147,13 +1236,7 @@ fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
1147 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;1236 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1148 const operand = try self.resolveInst(un_op);1237 const operand = try self.resolveInst(un_op);
1149 const errors_len_ptr = try o.getErrorsLen();1238 const errors_len_ptr = try o.getErrorsLen();
1150 const errors_len_val = try self.wip.load(1239 const errors_len_val = try self.load(errors_len_ptr.toValue(&o.builder), .none, .anyerror, .normal);
1151 .normal,
1152 try o.errorIntType(),
1153 errors_len_ptr.toValue(&o.builder),
1154 Type.errorAbiAlignment(o.zcu).toLlvm(),
1155 "",
1156 );
1157 return self.wip.icmp(.ule, operand, errors_len_val, "");1240 return self.wip.icmp(.ule, operand, errors_len_val, "");
1158}1241}
11591242
...@@ -1291,18 +1374,10 @@ fn lowerBlock(...@@ -1291,18 +1374,10 @@ fn lowerBlock(
12911374
1292 // Create a phi node only if the block returns a value.1375 // Create a phi node only if the block returns a value.
1293 if (have_block_result) {1376 if (have_block_result) {
1294 const raw_llvm_ty = try o.lowerType(inst_ty);1377 const llvm_ty: Builder.Type = switch (isByRef(inst_ty, zcu)) {
1295 const llvm_ty: Builder.Type = ty: {1378 true => .ptr,
1296 // If the zig tag type is a function, this represents an actual function body; not1379 false => try o.lowerType(inst_ty, .by_value),
1297 // a pointer to it. LLVM IR allows the call instruction to use function bodies instead
1298 // of function pointers, however the phi makes it a runtime value and therefore
1299 // the LLVM type has to be wrapped in a pointer.
1300 if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
1301 break :ty .ptr;
1302 }
1303 break :ty raw_llvm_ty;
1304 };1380 };
1305
1306 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);1381 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1307 const phi = try self.wip.phi(llvm_ty, "");1382 const phi = try self.wip.phi(llvm_ty, "");
1308 phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);1383 phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
...@@ -1408,7 +1483,7 @@ fn lowerSwitchDispatch(...@@ -1408,7 +1483,7 @@ fn lowerSwitchDispatch(
1408 const table_index = try self.wip.conv(1483 const table_index = try self.wip.conv(
1409 .unsigned,1484 .unsigned,
1410 try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),1485 try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1411 try o.lowerType(.usize),1486 try o.lowerType(.usize, .by_value),
1412 "",1487 "",
1413 );1488 );
1414 const target_ptr_ptr = try self.ptraddScaled(1489 const target_ptr_ptr = try self.ptraddScaled(
...@@ -1433,7 +1508,7 @@ fn lowerSwitchDispatch(...@@ -1433,7 +1508,7 @@ fn lowerSwitchDispatch(
1433 // The switch prongs will correspond to our scalar cases. Ranges will1508 // The switch prongs will correspond to our scalar cases. Ranges will
1434 // be handled by conditional branches in the `else` prong.1509 // be handled by conditional branches in the `else` prong.
14351510
1436 const llvm_usize = try o.lowerType(.usize);1511 const llvm_usize = try o.lowerType(.usize, .by_value);
1437 const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)1512 const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
1438 try self.wip.cast(.ptrtoint, cond, llvm_usize, "")1513 try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1439 else1514 else
...@@ -1628,37 +1703,27 @@ fn lowerTry(...@@ -1628,37 +1703,27 @@ fn lowerTry(
1628 const zcu = o.zcu;1703 const zcu = o.zcu;
1629 const payload_ty = err_union_ty.errorUnionPayload(zcu);1704 const payload_ty = err_union_ty.errorUnionPayload(zcu);
1630 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);1705 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1631 const error_type = try o.errorIntType();
16321706
1633 const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{1707 const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
1634 operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
1635 operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
1636 } else .{ .none, .none };
16371708
1638 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {1709 if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
1639 const loaded = loaded: {1710 const loaded = loaded: {
1640 const access_kind: Builder.MemoryAccessKind =1711 if (payload_has_bits) {
1641 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;1712 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
16421713 } else if (!operand_is_ptr) {
1643 if (!payload_has_bits) {1714 break :loaded err_union;
1644 break :loaded if (operand_is_ptr)
1645 try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
1646 else
1647 err_union;
1648 }1715 }
16491716
1650 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1651 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);1717 const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
1652 const err_field_ptr = try fg.ptraddConst(err_union, offset);1718 const err_field_ptr = try fg.ptraddConst(err_union, offset);
1653 break :loaded try fg.wip.load(1719 break :loaded try fg.load(
1654 if (operand_is_ptr) access_kind else .normal,
1655 error_type,
1656 err_field_ptr,1720 err_field_ptr,
1657 err_set_align.toLlvm(),1721 operand_align.offset(offset),
1658 "",1722 .anyerror,
1723 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1659 );1724 );
1660 };1725 };
1661 const zero = try o.builder.intValue(error_type, 0);1726 const zero = try o.builder.intValue(try o.errorIntType(.by_value), 0);
1662 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");1727 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
16631728
1664 const return_block = try fg.wip.block(1, "TryRet");1729 const return_block = try fg.wip.block(1, "TryRet");
...@@ -1672,15 +1737,18 @@ fn lowerTry(...@@ -1672,15 +1737,18 @@ fn lowerTry(
1672 fg.wip.cursor = .{ .block = continue_block };1737 fg.wip.cursor = .{ .block = continue_block };
1673 }1738 }
1674 if (is_unused) return .none;1739 if (is_unused) return .none;
1675 if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;1740
1676 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits1741 if (!operand_is_ptr) {
1677 const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));1742 assert(payload_has_bits); // otherwise the result should be comptime-known
1743 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
1744 }
1745
1746 const offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
1747 const payload_ptr = try fg.ptraddConst(err_union, offset);
1678 if (operand_is_ptr) {1748 if (operand_is_ptr) {
1679 return payload_ptr;1749 return payload_ptr;
1680 } else if (isByRef(payload_ty, zcu)) {
1681 return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
1682 } else {1750 } else {
1683 return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), "");1751 return fg.load(payload_ptr, operand_align.offset(offset), payload_ty, .normal);
1684 }1752 }
1685}1753}
16861754
...@@ -1792,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod...@@ -1792,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod
1792 const table_includes_else = item_count != table_len;1860 const table_includes_else = item_count != table_len;
17931861
1794 break :jmp_table .{1862 break :jmp_table .{
1795 .min = try o.lowerValue(min.toIntern()),1863 .min = try o.lowerValue(min.toIntern(), .by_value),
1796 .max = try o.lowerValue(max.toIntern()),1864 .max = try o.lowerValue(max.toIntern(), .by_value),
1797 .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {1865 .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1798 .none, .cold => .none,1866 .none, .cold => .none,
1799 .unpredictable => .unpredictable,1867 .unpredictable => .unpredictable,
...@@ -1951,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -1951,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
1951 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2019 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1952 const operand_ty = self.typeOf(ty_op.operand);2020 const operand_ty = self.typeOf(ty_op.operand);
1953 const array_ty = operand_ty.childType(zcu);2021 const array_ty = operand_ty.childType(zcu);
1954 const llvm_usize = try o.lowerType(.usize);2022 const llvm_usize = try o.lowerType(.usize, .by_value);
1955 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));2023 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
1956 const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst));2024 const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst), .by_value);
1957 const operand = try self.resolveInst(ty_op.operand);2025 const operand = try self.resolveInst(ty_op.operand);
1958 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");2026 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
1959}2027}
...@@ -1970,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value...@@ -1970,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value
19702038
1971 const dest_ty = self.typeOfIndex(inst);2039 const dest_ty = self.typeOfIndex(inst);
1972 const dest_scalar_ty = dest_ty.scalarType(zcu);2040 const dest_scalar_ty = dest_ty.scalarType(zcu);
1973 const dest_llvm_ty = try o.lowerType(dest_ty);2041 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
1974 const target = zcu.getTarget();2042 const target = zcu.getTarget();
19752043
1976 if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(2044 if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
...@@ -2038,7 +2106,7 @@ fn airIntFromFloat(...@@ -2038,7 +2106,7 @@ fn airIntFromFloat(
20382106
2039 const dest_ty = self.typeOfIndex(inst);2107 const dest_ty = self.typeOfIndex(inst);
2040 const dest_scalar_ty = dest_ty.scalarType(zcu);2108 const dest_scalar_ty = dest_ty.scalarType(zcu);
2041 const dest_llvm_ty = try o.lowerType(dest_ty);2109 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
20422110
2043 if (intrinsicsAllowed(operand_scalar_ty, target)) {2111 if (intrinsicsAllowed(operand_scalar_ty, target)) {
2044 // TODO set fast math flag2112 // TODO set fast math flag
...@@ -2072,7 +2140,7 @@ fn airIntFromFloat(...@@ -2072,7 +2140,7 @@ fn airIntFromFloat(
2072 compiler_rt_dest_abbrev,2140 compiler_rt_dest_abbrev,
2073 });2141 });
20742142
2075 const operand_llvm_ty = try o.lowerType(operand_ty);2143 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
2076 const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);2144 const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
2077 var result = try self.wip.call(2145 var result = try self.wip.call(
2078 .normal,2146 .normal,
...@@ -2097,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B...@@ -2097,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B
2097fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {2165fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2098 const o = fg.object;2166 const o = fg.object;
2099 const zcu = o.zcu;2167 const zcu = o.zcu;
2100 const llvm_usize = try o.lowerType(.usize);2168 const llvm_usize = try o.lowerType(.usize, .by_value);
2101 switch (ty.ptrSize(zcu)) {2169 switch (ty.ptrSize(zcu)) {
2102 .slice => {2170 .slice => {
2103 const len = try fg.wip.extractValue(ptr, &.{1}, "");2171 const len = try fg.wip.extractValue(ptr, &.{1}, "");
...@@ -2144,11 +2212,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -2144,11 +2212,7 @@ fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2144 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));2212 const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
2145 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;2213 const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
2146 self.maybeMarkAllowZeroAccess(slice_info);2214 self.maybeMarkAllowZeroAccess(slice_info);
2147 if (isByRef(elem_ty, zcu)) {2215 return self.load(ptr, elem_align, elem_ty, access_kind);
2148 return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
2149 } else {
2150 return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
2151 }
2152}2216}
21532217
2154fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {2218fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -2173,18 +2237,41 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -2173,18 +2237,41 @@ fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2173 const elem_ty = array_ty.childType(zcu);2237 const elem_ty = array_ty.childType(zcu);
2174 if (isByRef(array_ty, zcu)) {2238 if (isByRef(array_ty, zcu)) {
2175 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));2239 const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
2176 if (isByRef(elem_ty, zcu)) {2240 return self.load(elem_ptr, .none, elem_ty, .normal);
2177 const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
2178 return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
2179 } else {
2180 return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
2181 }
2182 }2241 }
21832242
2184 // This branch can be reached for vectors, which are always by-value.2243 // This branch can be reached for vectors, which are always by-value.
2185 return self.wip.extractElement(array_llvm_val, rhs, "");2244 return self.wip.extractElement(array_llvm_val, rhs, "");
2186}2245}
21872246
2247fn airLegalizeVecElemVal(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2248 const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
2249 const vec = try fg.resolveInst(bin_op.lhs);
2250 const index = try fg.resolveInst(bin_op.rhs);
2251 return fg.wip.extractElement(vec, index, "");
2252}
2253fn airLegalizeVecStoreElem(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2254 const zcu = fg.object.zcu;
2255
2256 const pl_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
2257 const extra = fg.air.extraData(Air.Bin, pl_op.payload).data;
2258
2259 const ptr_ty = fg.typeOf(pl_op.operand);
2260 const vec_ty = ptr_ty.childType(zcu);
2261
2262 const ptr_align = ptr_ty.ptrAlignment(zcu);
2263
2264 const vec_ptr = try fg.resolveInst(pl_op.operand);
2265 const index = try fg.resolveInst(extra.lhs);
2266 const elem = try fg.resolveInst(extra.rhs);
2267
2268 const old_vec = try fg.load(vec_ptr, ptr_align, vec_ty, .normal);
2269 const new_vec = try fg.wip.insertElement(old_vec, elem, index, "");
2270 try fg.store(vec_ptr, ptr_align, new_vec, vec_ty, .normal);
2271
2272 return .none;
2273}
2274
2188fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {2275fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
2189 const zcu = self.object.zcu;2276 const zcu = self.object.zcu;
2190 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;2277 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
...@@ -2197,8 +2284,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -2197,8 +2284,8 @@ fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
21972284
2198 return self.load(2285 return self.load(
2199 try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),2286 try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
2287 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)),
2200 elem_ty,2288 elem_ty,
2201 ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(),
2202 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,2289 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2203 );2290 );
2204}2291}
...@@ -2214,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -2214,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
2214 const base_ptr = try self.resolveInst(bin_op.lhs);2301 const base_ptr = try self.resolveInst(bin_op.lhs);
2215 const rhs = try self.resolveInst(bin_op.rhs);2302 const rhs = try self.resolveInst(bin_op.rhs);
22162303
2217 const elem_ptr = ty_pl.ty.toType();
2218 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
2219
2220 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));2304 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2221}2305}
22222306
...@@ -2279,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2279,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2279 },2363 },
2280 .float => {2364 .float => {
2281 // bitcast int->float2365 // bitcast int->float
2282 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), "");2366 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty, .by_value), "");
2283 },2367 },
2284 }2368 }
2285 }2369 }
...@@ -2294,11 +2378,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2294,11 +2378,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2294 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);2378 const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
2295 const field_ptr_align = struct_ptr_align.offset(offset);2379 const field_ptr_align = struct_ptr_align.offset(offset);
22962380
2297 if (isByRef(field_ty, zcu)) {2381 return self.load(field_ptr, field_ptr_align, field_ty, .normal);
2298 return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
2299 } else {
2300 return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
2301 }
2302}2382}
23032383
2304fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {2384fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -2313,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2313,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2313 const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);2393 const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2314 if (field_offset == 0) return field_ptr;2394 if (field_offset == 0) return field_ptr;
23152395
2316 const res_ty = try o.lowerType(ty_pl.ty.toType());2396 const res_ty = try o.lowerType(ty_pl.ty.toType(), .by_value);
2317 const llvm_usize = try o.lowerType(.usize);2397 const llvm_usize = try o.lowerType(.usize, .by_value);
23182398
2319 const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");2399 const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2320 const base_ptr_int = try self.wip.bin(2400 const base_ptr_int = try self.wip.bin(
...@@ -2438,9 +2518,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er...@@ -2438,9 +2518,8 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
2438 // We avoid taking this path for naked functions because there's no guarantee that such2518 // We avoid taking this path for naked functions because there's no guarantee that such
2439 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.2519 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
24402520
2441 const alignment = operand_ty.abiAlignment(zcu).toLlvm();2521 const alloca = try self.buildZigAlloca(operand_ty, .none);
2442 const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);2522 try self.store(alloca, .none, operand, operand_ty, .normal);
2443 _ = try self.wip.store(.normal, operand, alloca, alignment);
2444 _ = try self.wip.callIntrinsic(2523 _ = try self.wip.callIntrinsic(
2445 .normal,2524 .normal,
2446 .none,2525 .none,
...@@ -2531,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2531,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2531 const output_inst = try self.resolveInst(output.operand);2610 const output_inst = try self.resolveInst(output.operand);
2532 const output_ty = self.typeOf(output.operand);2611 const output_ty = self.typeOf(output.operand);
2533 assert(output_ty.zigTypeTag(zcu) == .pointer);2612 assert(output_ty.zigTypeTag(zcu) == .pointer);
2534 const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu));2613 const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu), .by_value);
25352614
2536 switch (constraint[0]) {2615 switch (constraint[0]) {
2537 '=' => {},2616 '=' => {},
...@@ -2569,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2569,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2569 llvm_ret_indirect[output.index] = false;2648 llvm_ret_indirect[output.index] = false;
25702649
2571 const ret_ty = self.typeOfIndex(inst);2650 const ret_ty = self.typeOfIndex(inst);
2572 llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty);2651 llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty, .by_value);
2573 llvm_ret_i += 1;2652 llvm_ret_i += 1;
2574 }2653 }
25752654
...@@ -2608,9 +2687,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2608,9 +2687,8 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2608 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);2687 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2609 } else {2688 } else {
2610 const alignment = arg_ty.abiAlignment(zcu).toLlvm();2689 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2611 const arg_llvm_ty = try o.lowerType(arg_ty);2690 const arg_llvm_ty = try o.lowerType(arg_ty, .by_value);
2612 const load_inst =2691 const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2613 try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2614 llvm_param_values[llvm_param_i] = load_inst;2692 llvm_param_values[llvm_param_i] = load_inst;
2615 llvm_param_types[llvm_param_i] = arg_llvm_ty;2693 llvm_param_types[llvm_param_i] = arg_llvm_ty;
2616 }2694 }
...@@ -2621,7 +2699,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2621,7 +2699,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2621 } else {2699 } else {
2622 const alignment = arg_ty.abiAlignment(zcu).toLlvm();2700 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2623 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);2701 const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
2624 _ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);2702 try self.store(arg_ptr, .none, arg_llvm_value, arg_ty, .normal);
2625 llvm_param_values[llvm_param_i] = arg_ptr;2703 llvm_param_values[llvm_param_i] = arg_ptr;
2626 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);2704 llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
2627 }2705 }
...@@ -2649,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2649,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2649 llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {2727 llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2650 if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));2728 if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
26512729
2652 break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu));2730 break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu), .by_value);
2653 } else .none;2731 } else .none;
26542732
2655 llvm_param_i += 1;2733 llvm_param_i += 1;
...@@ -2663,19 +2741,13 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2663,19 +2741,13 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2663 if (constraint[0] != '+') continue;2741 if (constraint[0] != '+') continue;
26642742
2665 const rw_ty = self.typeOf(output.operand);2743 const rw_ty = self.typeOf(output.operand);
2666 const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu));2744 const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu), .by_value);
2667 if (llvm_ret_indirect[output.index]) {2745 if (llvm_ret_indirect[output.index]) {
2668 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];2746 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2669 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);2747 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
2670 } else {2748 } else {
2671 const alignment = rw_ty.abiAlignment(zcu).toLlvm();2749 const access_kind: Builder.MemoryAccessKind = if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
2672 const loaded = try self.wip.load(2750 const loaded = try self.load(llvm_rw_vals[output.index], .none, rw_ty.childType(zcu), access_kind);
2673 if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2674 llvm_elem_ty,
2675 llvm_rw_vals[output.index],
2676 alignment,
2677 "",
2678 );
2679 llvm_param_values[llvm_param_i] = loaded;2751 llvm_param_values[llvm_param_i] = loaded;
2680 llvm_param_types[llvm_param_i] = llvm_elem_ty;2752 llvm_param_types[llvm_param_i] = llvm_elem_ty;
2681 }2753 }
...@@ -2835,12 +2907,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2835,12 +2907,12 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2835 if (output != .none) {2907 if (output != .none) {
2836 const output_ptr = try self.resolveInst(output);2908 const output_ptr = try self.resolveInst(output);
2837 const output_ptr_ty = self.typeOf(output);2909 const output_ptr_ty = self.typeOf(output);
2838 const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();2910 try self.store(
2839 _ = try self.wip.store(
2840 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2841 output_value,
2842 output_ptr,2911 output_ptr,
2843 alignment,2912 output_ptr_ty.ptrAlignment(zcu),
2913 output_value,
2914 output_ptr_ty.childType(zcu),
2915 if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
2844 );2916 );
2845 } else {2917 } else {
2846 ret_val = output_value;2918 ret_val = output_value;
...@@ -2863,7 +2935,6 @@ fn airIsNonNull(...@@ -2863,7 +2935,6 @@ fn airIsNonNull(
2863 const operand = try self.resolveInst(un_op);2935 const operand = try self.resolveInst(un_op);
2864 const operand_ty = self.typeOf(un_op);2936 const operand_ty = self.typeOf(un_op);
2865 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;2937 const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2866 const optional_llvm_ty = try o.lowerType(optional_ty);
2867 const payload_ty = optional_ty.optionalChild(zcu);2938 const payload_ty = optional_ty.optionalChild(zcu);
28682939
2869 const access_kind: Builder.MemoryAccessKind =2940 const access_kind: Builder.MemoryAccessKind =
...@@ -2873,7 +2944,7 @@ fn airIsNonNull(...@@ -2873,7 +2944,7 @@ fn airIsNonNull(
28732944
2874 if (optional_ty.optionalReprIsPayload(zcu)) {2945 if (optional_ty.optionalReprIsPayload(zcu)) {
2875 const loaded = if (operand_is_ptr)2946 const loaded = if (operand_is_ptr)
2876 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")2947 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2877 else2948 else
2878 operand;2949 operand;
2879 if (payload_ty.isSlice(zcu)) {2950 if (payload_ty.isSlice(zcu)) {
...@@ -2884,14 +2955,14 @@ fn airIsNonNull(...@@ -2884,14 +2955,14 @@ fn airIsNonNull(
2884 ));2955 ));
2885 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");2956 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
2886 }2957 }
2887 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");2958 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty, .by_value)), "");
2888 }2959 }
28892960
2890 comptime assert(optional_layout_version == 3);2961 comptime assert(optional_layout_version == 3);
28912962
2892 if (!payload_ty.hasRuntimeBits(zcu)) {2963 if (!payload_ty.hasRuntimeBits(zcu)) {
2893 const loaded = if (operand_is_ptr)2964 const loaded = if (operand_is_ptr)
2894 try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")2965 try self.load(operand, operand_ty.ptrAlignment(zcu), optional_ty, access_kind)
2895 else2966 else
2896 operand;2967 operand;
2897 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");2968 return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
...@@ -2913,8 +2984,7 @@ fn airIsErr(...@@ -2913,8 +2984,7 @@ fn airIsErr(
2913 const operand_ty = self.typeOf(un_op);2984 const operand_ty = self.typeOf(un_op);
2914 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;2985 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
2915 const payload_ty = err_union_ty.errorUnionPayload(zcu);2986 const payload_ty = err_union_ty.errorUnionPayload(zcu);
2916 const error_type = try o.errorIntType();2987 const zero_err = try o.builder.intValue(try o.errorIntType(.by_value), 0);
2917 const zero = try o.builder.intValue(error_type, 0);
29182988
2919 const access_kind: Builder.MemoryAccessKind =2989 const access_kind: Builder.MemoryAccessKind =
2920 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;2990 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -2932,10 +3002,10 @@ fn airIsErr(...@@ -2932,10 +3002,10 @@ fn airIsErr(
29323002
2933 if (!payload_ty.hasRuntimeBits(zcu)) {3003 if (!payload_ty.hasRuntimeBits(zcu)) {
2934 const loaded = if (operand_is_ptr)3004 const loaded = if (operand_is_ptr)
2935 try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")3005 try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
2936 else3006 else
2937 operand;3007 operand;
2938 return self.wip.icmp(cond, loaded, zero, "");3008 return self.wip.icmp(cond, loaded, zero_err, "");
2939 }3009 }
2940 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref3010 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
29413011
...@@ -2944,8 +3014,8 @@ fn airIsErr(...@@ -2944,8 +3014,8 @@ fn airIsErr(
2944 else3014 else
2945 .none;3015 .none;
2946 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3016 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
2947 const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");3017 const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
2948 return self.wip.icmp(cond, loaded, zero, "");3018 return self.wip.icmp(cond, loaded, zero_err, "");
2949}3019}
29503020
2951fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {3021fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -2966,7 +3036,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2966,7 +3036,6 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2966 const optional_ptr_ty = self.typeOf(ty_op.operand);3036 const optional_ptr_ty = self.typeOf(ty_op.operand);
2967 const optional_ty = optional_ptr_ty.childType(zcu);3037 const optional_ty = optional_ptr_ty.childType(zcu);
2968 const payload_ty = optional_ty.optionalChild(zcu);3038 const payload_ty = optional_ty.optionalChild(zcu);
2969 const non_null_bit = try o.builder.intValue(.i8, 1);
29703039
2971 const access_kind: Builder.MemoryAccessKind =3040 const access_kind: Builder.MemoryAccessKind =
2972 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3041 if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -2976,7 +3045,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2976,7 +3045,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
29763045
2977 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.3046 // We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
2978 // Default alignment store because align of the non null bit is 1 anyway.3047 // Default alignment store because align of the non null bit is 1 anyway.
2979 _ = try self.wip.store(access_kind, non_null_bit, operand, .default);3048 try self.store(operand, .@"1", .true, .bool, access_kind);
2980 return operand;3049 return operand;
2981 }3050 }
2982 if (optional_ty.optionalReprIsPayload(zcu)) {3051 if (optional_ty.optionalReprIsPayload(zcu)) {
...@@ -2992,7 +3061,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -2992,7 +3061,7 @@ fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
2992 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));3061 self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
29933062
2994 // Default alignment store because align of the non null bit is 1 anyway.3063 // Default alignment store because align of the non null bit is 1 anyway.
2995 _ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);3064 try self.store(non_null_ptr, .@"1", .true, .bool, access_kind);
29963065
2997 // Then return the payload pointer (only if it's used).3066 // Then return the payload pointer (only if it's used).
2998 if (self.liveness.isUnused(inst)) return .none;3067 if (self.liveness.isUnused(inst)) return .none;
...@@ -3016,31 +3085,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -3016,31 +3085,29 @@ fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
3016 return self.optPayloadHandle(operand, optional_ty, false);3085 return self.optPayloadHandle(operand, optional_ty, false);
3017}3086}
30183087
3019fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value {3088fn airErrUnionPayload(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3020 const o = self.object;3089 const o = fg.object;
3021 const zcu = o.zcu;3090 const zcu = o.zcu;
3022 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3091 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3023 const operand = try self.resolveInst(ty_op.operand);3092 const operand = try fg.resolveInst(ty_op.operand);
3024 const operand_ty = self.typeOf(ty_op.operand);3093 const err_union_ty = fg.typeOf(ty_op.operand);
3025 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;3094 const payload_ty = fg.typeOfIndex(inst);
3026 const result_ty = self.typeOfIndex(inst);
3027 const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
30283095
3029 if (!payload_ty.hasRuntimeBits(zcu)) {3096 assert(payload_ty.hasRuntimeBits(zcu));
3030 return if (operand_is_ptr) operand else .none;
3031 }
3032 const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3033 if (operand_is_ptr) {
3034 return payload_ptr;
3035 }
3036 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits3097 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3037 const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();3098
3038 if (isByRef(payload_ty, zcu)) {3099 const payload_offset = codegen.errUnionPayloadOffset(payload_ty, zcu);
3039 return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);3100 const payload_ptr = try fg.ptraddConst(operand, payload_offset);
3040 } else {3101 return fg.load(payload_ptr, err_union_ty.abiAlignment(zcu).offset(payload_offset), payload_ty, .normal);
3041 const payload_llvm_ty = try o.lowerType(payload_ty);3102}
3042 return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");3103
3043 }3104fn airErrUnionPayloadPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3105 const o = fg.object;
3106 const zcu = o.zcu;
3107 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3108 const operand = try fg.resolveInst(ty_op.operand);
3109 const payload_ty = fg.typeOfIndex(inst).childType(zcu);
3110 return fg.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
3044}3111}
30453112
3046fn airErrUnionErr(3113fn airErrUnionErr(
...@@ -3053,40 +3120,28 @@ fn airErrUnionErr(...@@ -3053,40 +3120,28 @@ fn airErrUnionErr(
3053 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3120 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3054 const operand = try self.resolveInst(ty_op.operand);3121 const operand = try self.resolveInst(ty_op.operand);
3055 const operand_ty = self.typeOf(ty_op.operand);3122 const operand_ty = self.typeOf(ty_op.operand);
3056 const error_type = try o.errorIntType();
3057 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;3123 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3058 if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
3059 if (operand_is_ptr) {
3060 return operand;
3061 } else {
3062 return o.builder.intValue(error_type, 0);
3063 }
3064 }
30653124
3066 const access_kind: Builder.MemoryAccessKind =3125 const access_kind: Builder.MemoryAccessKind =
3067 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3126 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
30683127
3069 const payload_ty = err_union_ty.errorUnionPayload(zcu);3128 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3070 if (!payload_ty.hasRuntimeBits(zcu)) {
3071 if (!operand_is_ptr) return operand;
30723129
3073 self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));3130 if (payload_ty.hasRuntimeBits(zcu)) {
30743131 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3075 return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");3132 } else if (!operand_is_ptr) {
3133 return operand;
3076 }3134 }
30773135
3078 assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
3079
3080 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));3136 if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
30813137
3082 const err_align: InternPool.Alignment = a: {3138 const ptr_align = if (operand_is_ptr) operand_ty.ptrAlignment(zcu) else err_union_ty.abiAlignment(zcu);
3083 const err_abi_align = Type.anyerror.abiAlignment(zcu);
3084 if (!operand_is_ptr) break :a err_abi_align;
3085 break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
3086 };
30873139
3088 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3140 const err_offset = codegen.errUnionErrorOffset(payload_ty, zcu);
3089 return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");3141 const err_align = ptr_align.offset(err_offset);
3142 const err_ptr = try self.ptraddConst(operand, err_offset);
3143
3144 return self.load(err_ptr, err_align, .anyerror, access_kind);
3090}3145}
30913146
3092fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {3147fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -3099,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -3099,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3099 const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);3154 const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
31003155
3101 const payload_ty = err_union_ty.errorUnionPayload(zcu);3156 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3102 const non_error_val = try o.builder.intValue(try o.errorIntType(), 0);3157 const non_error_val = try o.builder.intValue(try o.errorIntType(.by_value), 0);
31033158
3104 const access_kind: Builder.MemoryAccessKind =3159 const access_kind: Builder.MemoryAccessKind =
3105 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3160 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -3107,10 +3162,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -3107,10 +3162,10 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3107 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));3162 self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
31083163
3109 {3164 {
3110 const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
3111 // First set the non-error value.3165 // First set the non-error value.
3112 const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3166 const error_off = codegen.errUnionErrorOffset(payload_ty, zcu);
3113 _ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);3167 const error_ptr = try self.ptraddConst(operand, error_off);
3168 try self.store(error_ptr, err_union_ptr_align.offset(error_off), non_error_val, .anyerror, access_kind);
3114 }3169 }
31153170
3116 // Then return the payload pointer (only if it is used).3171 // Then return the payload pointer (only if it is used).
...@@ -3142,127 +3197,73 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er...@@ -3142,127 +3197,73 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
3142 const field_offset = struct_ty.structFieldOffset(field_index, zcu);3197 const field_offset = struct_ty.structFieldOffset(field_index, zcu);
3143 const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);3198 const field_align = struct_ty.abiAlignment(zcu).offset(field_offset);
3144 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);3199 const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
3145 return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal);3200 return self.load(field_ptr, field_align, field_ty, .normal);
3146}
3147
3148/// As an optimization, we want to avoid unnecessary copies of
3149/// error union/optional types when returning from a function.
3150/// Here, we scan forward in the current block, looking to see
3151/// if the next instruction is a return (ignoring debug instructions).
3152///
3153/// The first instruction of `body_tail` is a wrap instruction.
3154fn isNextRet(
3155 self: *FuncGen,
3156 body_tail: []const Air.Inst.Index,
3157) bool {
3158 const air_tags = self.air.instructions.items(.tag);
3159 for (body_tail[1..]) |body_inst| {
3160 switch (air_tags[@intFromEnum(body_inst)]) {
3161 .ret => return true,
3162 .dbg_stmt => continue,
3163 else => return false,
3164 }
3165 }
3166 // The only way to get here is to hit the end of a loop instruction
3167 // (implicit repeat).
3168 return false;
3169}3201}
31703202
3171fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {3203fn airWrapOptional(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3172 const o = self.object;3204 const o = self.object;
3173 const zcu = o.zcu;3205 const zcu = o.zcu;
3174 const inst = body_tail[0];
3175 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3206 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3176 const payload_ty = self.typeOf(ty_op.operand);3207 const payload_ty = self.typeOf(ty_op.operand);
3177 const non_null_bit = try o.builder.intValue(.i8, 1);
3178 comptime assert(optional_layout_version == 3);3208 comptime assert(optional_layout_version == 3);
3179 assert(payload_ty.hasRuntimeBits(zcu));3209 assert(payload_ty.hasRuntimeBits(zcu));
3180 const operand = try self.resolveInst(ty_op.operand);3210 const operand = try self.resolveInst(ty_op.operand);
3181 const optional_ty = self.typeOfIndex(inst);3211 const optional_ty = self.typeOfIndex(inst);
3182 if (optional_ty.optionalReprIsPayload(zcu)) return operand;3212 if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3183 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`3213 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3184 const llvm_optional_ty = try o.lowerType(optional_ty);3214 const optional_ptr = try self.buildZigAlloca(optional_ty, .none);
3185 const optional_ptr = if (self.isNextRet(body_tail))
3186 self.ret_ptr
3187 else brk: {
3188 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3189 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3190 break :brk optional_ptr;
3191 };
31923215
3193 const payload_ptr = optional_ptr; // payload always at offset 03216 const payload_ptr = optional_ptr; // payload always at offset 0
3194 try self.store(3217 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3195 payload_ptr,3218
3196 .none,
3197 operand,
3198 payload_ty,
3199 );
3200 // Non-null bit immediately after payload (no padding because the bit has alignment 1).3219 // Non-null bit immediately after payload (no padding because the bit has alignment 1).
3201 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));3220 const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
3202 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);3221 try self.store(non_null_ptr, .none, .true, .bool, .normal);
3222
3203 return optional_ptr;3223 return optional_ptr;
3204}3224}
32053225
3206fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {3226fn airWrapErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3207 const o = self.object;3227 const o = self.object;
3208 const zcu = o.zcu;3228 const zcu = o.zcu;
3209 const inst = body_tail[0];
3210 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3229 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3211 const err_un_ty = self.typeOfIndex(inst);3230 const err_un_ty = self.typeOfIndex(inst);
3212 const operand = try self.resolveInst(ty_op.operand);3231 const operand = try self.resolveInst(ty_op.operand);
3213 const payload_ty = self.typeOf(ty_op.operand);3232 const payload_ty = self.typeOf(ty_op.operand);
3214 assert(payload_ty.hasRuntimeBits(zcu));3233 assert(payload_ty.hasRuntimeBits(zcu));
3215 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref3234 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3216 const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0);3235 const ok_err_code = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3217 const err_un_llvm_ty = try o.lowerType(err_un_ty);3236
32183237 const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3219 const result_ptr = if (self.isNextRet(body_tail))
3220 self.ret_ptr
3221 else brk: {
3222 const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm();
3223 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3224 break :brk result_ptr;
3225 };
32263238
3227 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3239 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3228 const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm();3240 try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
3229 _ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);3241
3230 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));3242 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3231 try self.store(3243 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
3232 payload_ptr,3244
3233 .none,
3234 operand,
3235 payload_ty,
3236 );
3237 return result_ptr;3245 return result_ptr;
3238}3246}
32393247
3240fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {3248fn airWrapErrUnionErr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3241 const o = self.object;3249 const o = self.object;
3242 const zcu = o.zcu;3250 const zcu = o.zcu;
3243 const inst = body_tail[0];
3244 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3251 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3245 const err_un_ty = self.typeOfIndex(inst);3252 const err_un_ty = self.typeOfIndex(inst);
3246 const payload_ty = err_un_ty.errorUnionPayload(zcu);3253 const payload_ty = err_un_ty.errorUnionPayload(zcu);
3247 const operand = try self.resolveInst(ty_op.operand);3254 const operand = try self.resolveInst(ty_op.operand);
3248 if (!payload_ty.hasRuntimeBits(zcu)) return operand;3255 if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3249 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref3256 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3250 const err_un_llvm_ty = try o.lowerType(err_un_ty);3257
32513258 const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3252 const result_ptr = if (self.isNextRet(body_tail))
3253 self.ret_ptr
3254 else brk: {
3255 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3256 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3257 break :brk result_ptr;
3258 };
32593259
3260 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3260 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3261 const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();3261 try self.store(err_ptr, .none, operand, .anyerror, .normal);
3262 _ = try self.wip.store(.normal, operand, err_ptr, error_alignment);3262
3263 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));3263 const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
3264 // TODO store undef to payload_ptr3264 // TODO store undef to payload_ptr
3265 _ = payload_ptr;3265 _ = payload_ptr;
3266
3266 return result_ptr;3267 return result_ptr;
3267}3268}
32683269
...@@ -3270,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -3270,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3270 const o = self.object;3271 const o = self.object;
3271 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;3272 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3272 const index = pl_op.payload;3273 const index = pl_op.payload;
3273 const llvm_usize = try o.lowerType(.usize);3274 const llvm_usize = try o.lowerType(.usize, .by_value);
3274 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{3275 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3275 try o.builder.intValue(.i32, index),3276 try o.builder.intValue(.i32, index),
3276 }, "");3277 }, "");
...@@ -3280,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -3280,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3280 const o = self.object;3281 const o = self.object;
3281 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;3282 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3282 const index = pl_op.payload;3283 const index = pl_op.payload;
3283 const llvm_isize = try o.lowerType(.isize);3284 const llvm_isize = try o.lowerType(.isize, .by_value);
3284 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{3285 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3285 try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),3286 try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3286 }, "");3287 }, "");
...@@ -3307,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -3307,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3307 .normal,3308 .normal,
3308 .none,3309 .none,
3309 if (scalar_ty.isSignedInt(zcu)) .smin else .umin,3310 if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3310 &.{try o.lowerType(inst_ty)},3311 &.{try o.lowerType(inst_ty, .by_value)},
3311 &.{ lhs, rhs },3312 &.{ lhs, rhs },
3312 "",3313 "",
3313 );3314 );
...@@ -3327,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -3327,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3327 .normal,3328 .normal,
3328 .none,3329 .none,
3329 if (scalar_ty.isSignedInt(zcu)) .smax else .umax,3330 if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3330 &.{try o.lowerType(inst_ty)},3331 &.{try o.lowerType(inst_ty, .by_value)},
3331 &.{ lhs, rhs },3332 &.{ lhs, rhs },
3332 "",3333 "",
3333 );3334 );
...@@ -3339,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3339,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3339 const ptr = try self.resolveInst(bin_op.lhs);3340 const ptr = try self.resolveInst(bin_op.lhs);
3340 const len = try self.resolveInst(bin_op.rhs);3341 const len = try self.resolveInst(bin_op.rhs);
3341 const inst_ty = self.typeOfIndex(inst);3342 const inst_ty = self.typeOfIndex(inst);
3342 return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, "");3343 return self.wip.buildAggregate(try self.object.lowerType(inst_ty, .by_value), &.{ ptr, len }, "");
3343}3344}
33443345
3345fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {3346fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...@@ -3370,7 +3371,7 @@ fn airSafeArithmetic(...@@ -3370,7 +3371,7 @@ fn airSafeArithmetic(
3370 const scalar_ty = inst_ty.scalarType(zcu);3371 const scalar_ty = inst_ty.scalarType(zcu);
33713372
3372 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;3373 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3373 const llvm_inst_ty = try o.lowerType(inst_ty);3374 const llvm_inst_ty = try o.lowerType(inst_ty, .by_value);
3374 const results =3375 const results =
3375 try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");3376 try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
33763377
...@@ -3420,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3420,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3420 .normal,3421 .normal,
3421 .none,3422 .none,
3422 if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",3423 if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3423 &.{try o.lowerType(inst_ty)},3424 &.{try o.lowerType(inst_ty, .by_value)},
3424 &.{ lhs, rhs },3425 &.{ lhs, rhs },
3425 "",3426 "",
3426 );3427 );
...@@ -3459,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3459,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3459 .normal,3460 .normal,
3460 .none,3461 .none,
3461 if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",3462 if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3462 &.{try o.lowerType(inst_ty)},3463 &.{try o.lowerType(inst_ty, .by_value)},
3463 &.{ lhs, rhs },3464 &.{ lhs, rhs },
3464 "",3465 "",
3465 );3466 );
...@@ -3498,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3498,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3498 .normal,3499 .normal,
3499 .none,3500 .none,
3500 if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",3501 if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3501 &.{try o.lowerType(inst_ty)},3502 &.{try o.lowerType(inst_ty, .by_value)},
3502 &.{ lhs, rhs, .@"0" },3503 &.{ lhs, rhs, .@"0" },
3503 "",3504 "",
3504 );3505 );
...@@ -3542,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)...@@ -3542,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)
3542 return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});3543 return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3543 }3544 }
3544 if (scalar_ty.isSignedInt(zcu)) {3545 if (scalar_ty.isSignedInt(zcu)) {
3545 const scalar_llvm_ty = try o.lowerType(scalar_ty);3546 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
3546 const inst_llvm_ty = try o.lowerType(inst_ty);3547 const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
35473548
3548 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;3549 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3549 var bfa_buf: ExpectedContents = undefined;3550 var bfa_buf: ExpectedContents = undefined;
...@@ -3617,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo...@@ -3617,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3617 const lhs = try self.resolveInst(bin_op.lhs);3618 const lhs = try self.resolveInst(bin_op.lhs);
3618 const rhs = try self.resolveInst(bin_op.rhs);3619 const rhs = try self.resolveInst(bin_op.rhs);
3619 const inst_ty = self.typeOfIndex(inst);3620 const inst_ty = self.typeOfIndex(inst);
3620 const inst_llvm_ty = try o.lowerType(inst_ty);3621 const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3621 const scalar_ty = inst_ty.scalarType(zcu);3622 const scalar_ty = inst_ty.scalarType(zcu);
36223623
3623 if (scalar_ty.isRuntimeFloat()) {3624 if (scalar_ty.isRuntimeFloat()) {
...@@ -3646,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo...@@ -3646,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3646 defer allocator.free(smin_big_int.limbs);3647 defer allocator.free(smin_big_int.limbs);
3647 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);3648 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3648 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(3649 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3649 try o.lowerType(scalar_ty),3650 try o.lowerType(scalar_ty, .by_value),
3650 smin_big_int.toConst(),3651 smin_big_int.toConst(),
3651 ));3652 ));
36523653
...@@ -3682,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3682,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3682 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3683 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3683 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;3684 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3684 const ptr_or_slice = try self.resolveInst(bin_op.lhs);3685 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3685 const llvm_usize_ty = try o.lowerType(.usize);3686 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
3686 const ptr_ty = self.typeOf(bin_op.lhs);3687 const ptr_ty = self.typeOf(bin_op.lhs);
3687 const elem_ty = ptr_ty.indexableElem(zcu);3688 const elem_ty = ptr_ty.indexableElem(zcu);
3688 const ptr = switch (ptr_ty.ptrSize(zcu)) {3689 const ptr = switch (ptr_ty.ptrSize(zcu)) {
...@@ -3715,27 +3716,28 @@ fn airOverflow(...@@ -3715,27 +3716,28 @@ fn airOverflow(
3715 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref3716 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
37163717
3717 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;3718 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3718 const llvm_inst_ty = try o.lowerType(inst_ty);3719 const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
3719 const llvm_lhs_ty = try o.lowerType(lhs_ty);
3720 const results =3720 const results =
3721 try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");3721 try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
37223722
3723 const result_val = try self.wip.extractValue(results, &.{0}, "");3723 const result_val = try self.wip.extractValue(results, &.{0}, "");
3724 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");3724 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
37253725
3726 const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();3726 const result_alignment = inst_ty.abiAlignment(zcu);
3727 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);3727 const alloca_inst = try self.buildZigAlloca(inst_ty, .none);
37283728
3729 {3729 {
3730 // Store to 'result: IntType' field3730 // Store to 'result: IntType' field
3731 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));3731 const field_off = inst_ty.structFieldOffset(0, zcu);
3732 _ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());3732 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3733 try self.store(field_ptr, result_alignment.offset(field_off), result_val, lhs_ty, .normal);
3733 }3734 }
37343735
3735 {3736 {
3736 // Store to 'overflow: u1' field3737 // Store to 'overflow: u1' field
3737 const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));3738 const field_off = inst_ty.structFieldOffset(1, zcu);
3738 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));3739 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
3740 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, inst_ty.fieldType(1, zcu), .normal);
3739 }3741 }
37403742
3741 return alloca_inst;3743 return alloca_inst;
...@@ -3787,7 +3789,7 @@ fn buildFloatCmp(...@@ -3787,7 +3789,7 @@ fn buildFloatCmp(
3787 const zcu = o.zcu;3789 const zcu = o.zcu;
3788 const target = zcu.getTarget();3790 const target = zcu.getTarget();
3789 const scalar_ty = ty.scalarType(zcu);3791 const scalar_ty = ty.scalarType(zcu);
3790 const scalar_llvm_ty = try o.lowerType(scalar_ty);3792 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
37913793
3792 if (intrinsicsAllowed(scalar_ty, target)) {3794 if (intrinsicsAllowed(scalar_ty, target)) {
3793 const cond: Builder.FloatCondition = switch (pred) {3795 const cond: Builder.FloatCondition = switch (pred) {
...@@ -3893,7 +3895,7 @@ fn buildFloatOp(...@@ -3893,7 +3895,7 @@ fn buildFloatOp(
3893 const zcu = o.zcu;3895 const zcu = o.zcu;
3894 const target = zcu.getTarget();3896 const target = zcu.getTarget();
3895 const scalar_ty = ty.scalarType(zcu);3897 const scalar_ty = ty.scalarType(zcu);
3896 const llvm_ty = try o.lowerType(ty);3898 const llvm_ty = try o.lowerType(ty, .by_value);
38973899
3898 if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {3900 if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
3899 // Some operations are dedicated LLVM instructions, not available as intrinsics3901 // Some operations are dedicated LLVM instructions, not available as intrinsics
...@@ -3998,7 +4000,7 @@ fn buildFloatOp(...@@ -3998,7 +4000,7 @@ fn buildFloatOp(
3998 }),4000 }),
3999 };4001 };
40004002
4001 const scalar_llvm_ty = try o.lowerType(scalar_ty);4003 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
4002 const libc_fn = try o.getLibcFunction(4004 const libc_fn = try o.getLibcFunction(
4003 fn_name,4005 fn_name,
4004 @as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],4006 @as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],
...@@ -4052,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -4052,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
40524054
4053 const dest_ty = self.typeOfIndex(inst);4055 const dest_ty = self.typeOfIndex(inst);
4054 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref4056 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4055 const llvm_dest_ty = try o.lowerType(dest_ty);
40564057
4057 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4058 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
40584059
4059 const result = try self.wip.bin(.shl, lhs, casted_rhs, "");4060 const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4060 const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))4061 const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
...@@ -4064,19 +4065,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -4064,19 +4065,21 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
40644065
4065 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");4066 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
40664067
4067 const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();4068 const result_alignment = dest_ty.abiAlignment(zcu);
4068 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);4069 const alloca_inst = try self.buildZigAlloca(dest_ty, .none);
40694070
4070 {4071 {
4071 // Store to 'result: IntType' field4072 // Store to 'result: IntType' field
4072 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));4073 const field_off = dest_ty.structFieldOffset(0, zcu);
4073 _ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());4074 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4075 try self.store(field_ptr, result_alignment.offset(field_off), result, lhs_ty, .normal);
4074 }4076 }
40754077
4076 {4078 {
4077 // Store to 'overflow: u1' field4079 // Store to 'overflow: u1' field
4078 const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));4080 const field_off = dest_ty.structFieldOffset(1, zcu);
4079 _ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));4081 const field_ptr = try self.ptraddConst(alloca_inst, field_off);
4082 try self.store(field_ptr, result_alignment.offset(field_off), overflow_bit, dest_ty.fieldType(1, zcu), .normal);
4080 }4083 }
40814084
4082 return alloca_inst;4085 return alloca_inst;
...@@ -4119,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -4119,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
4119 }4122 }
4120 const lhs_scalar_ty = lhs_ty.scalarType(zcu);4123 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
41214124
4122 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4125 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4123 return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))4126 return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4124 .@"shl nsw"4127 .@"shl nsw"
4125 else4128 else
...@@ -4140,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4140,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4140 // features which we do not use. Therefore this branch is currently impossible.4143 // features which we do not use. Therefore this branch is currently impossible.
4141 unreachable;4144 unreachable;
4142 }4145 }
4143 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4146 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4144 return self.wip.bin(.shl, lhs, casted_rhs, "");4147 return self.wip.bin(.shl, lhs, casted_rhs, "");
4145}4148}
41464149
...@@ -4154,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4154,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
41544157
4155 const lhs_ty = self.typeOf(bin_op.lhs);4158 const lhs_ty = self.typeOf(bin_op.lhs);
4156 const lhs_info = lhs_ty.intInfo(zcu);4159 const lhs_info = lhs_ty.intInfo(zcu);
4157 const llvm_lhs_ty = try o.lowerType(lhs_ty);4160 const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
4158 const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu));4161 const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu), .by_value);
41594162
4160 const rhs_ty = self.typeOf(bin_op.rhs);4163 const rhs_ty = self.typeOf(bin_op.rhs);
4161 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {4164 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
...@@ -4165,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4165,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4165 }4168 }
4166 const rhs_info = rhs_ty.intInfo(zcu);4169 const rhs_info = rhs_ty.intInfo(zcu);
4167 assert(rhs_info.signedness == .unsigned);4170 assert(rhs_info.signedness == .unsigned);
4168 const llvm_rhs_ty = try o.lowerType(rhs_ty);4171 const llvm_rhs_ty = try o.lowerType(rhs_ty, .by_value);
4169 const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu));4172 const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu), .by_value);
41704173
4171 const result = try self.wip.callIntrinsic(4174 const result = try self.wip.callIntrinsic(
4172 .normal,4175 .normal,
...@@ -4242,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!...@@ -4242,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!
4242 }4245 }
4243 const lhs_scalar_ty = lhs_ty.scalarType(zcu);4246 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
42444247
4245 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4248 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4246 const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);4249 const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
42474250
4248 return self.wip.bin(if (is_exact)4251 return self.wip.bin(if (is_exact)
...@@ -4263,8 +4266,8 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4263,8 +4266,8 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4263 .normal,4266 .normal,
4264 .none,4267 .none,
4265 .abs,4268 .abs,
4266 &.{try o.lowerType(operand_ty)},4269 &.{try o.lowerType(operand_ty, .by_value)},
4267 &.{ operand, try o.builder.intValue(.i1, 0) },4270 &.{ operand, .false },
4268 "",4271 "",
4269 ),4272 ),
4270 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),4273 .float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
...@@ -4277,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4277,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4277 const zcu = o.zcu;4280 const zcu = o.zcu;
4278 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4281 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4279 const dest_ty = fg.typeOfIndex(inst);4282 const dest_ty = fg.typeOfIndex(inst);
4280 const dest_llvm_ty = try o.lowerType(dest_ty);4283 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4281 const operand = try fg.resolveInst(ty_op.operand);4284 const operand = try fg.resolveInst(ty_op.operand);
4282 const operand_ty = fg.typeOf(ty_op.operand);4285 const operand_ty = fg.typeOf(ty_op.operand);
4283 const operand_info = operand_ty.intInfo(zcu);4286 const operand_info = operand_ty.intInfo(zcu);
...@@ -4305,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4305,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
43054308
4306 if (!have_min_check and !have_max_check) break :bounds_check;4309 if (!have_min_check and !have_max_check) break :bounds_check;
43074310
4308 const operand_llvm_ty = try o.lowerType(operand_ty);4311 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4309 const operand_scalar_llvm_ty = try o.lowerType(operand_scalar);4312 const operand_scalar_llvm_ty = try o.lowerType(operand_scalar, .by_value);
43104313
4311 const is_vector = operand_ty.zigTypeTag(zcu) == .vector;4314 const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4312 assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));4315 assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
...@@ -4384,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4384,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4384fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4387fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4385 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4388 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4386 const operand = try self.resolveInst(ty_op.operand);4389 const operand = try self.resolveInst(ty_op.operand);
4387 const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst));4390 const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst), .by_value);
4388 return self.wip.cast(.trunc, operand, dest_llvm_ty, "");4391 return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4389}4392}
43904393
...@@ -4398,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -4398,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4398 const target = zcu.getTarget();4401 const target = zcu.getTarget();
43994402
4400 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {4403 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4401 return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), "");4404 return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty, .by_value), "");
4402 } else {4405 } else {
4403 const operand_llvm_ty = try o.lowerType(operand_ty);4406 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4404 const dest_llvm_ty = try o.lowerType(dest_ty);4407 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
44054408
4406 const dest_bits = dest_ty.floatBits(target);4409 const dest_bits = dest_ty.floatBits(target);
4407 const src_bits = operand_ty.floatBits(target);4410 const src_bits = operand_ty.floatBits(target);
...@@ -4432,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4432,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4432 const target = zcu.getTarget();4435 const target = zcu.getTarget();
44334436
4434 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {4437 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4435 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");4438 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty, .by_value), "");
4436 } else {4439 } else {
4437 const operand_llvm_ty = try o.lowerType(operand_ty);4440 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4438 const dest_llvm_ty = try o.lowerType(dest_ty);4441 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
44394442
4440 const dest_bits = dest_ty.scalarType(zcu).floatBits(target);4443 const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
4441 const src_bits = operand_ty.scalarType(zcu).floatBits(target);4444 const src_bits = operand_ty.scalarType(zcu).floatBits(target);
...@@ -4462,114 +4465,80 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4462,114 +4465,80 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4462 }4465 }
4463}4466}
44644467
4465fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4468fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4466 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4469 const o = fg.object;
4467 const operand_ty = self.typeOf(ty_op.operand);
4468 const inst_ty = self.typeOfIndex(inst);
4469 const operand = try self.resolveInst(ty_op.operand);
4470 return self.bitCast(operand, operand_ty, inst_ty);
4471}
4472
4473fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value {
4474 const o = self.object;
4475 const zcu = o.zcu;4470 const zcu = o.zcu;
4476 const operand_is_ref = isByRef(operand_ty, zcu);
4477 const result_is_ref = isByRef(inst_ty, zcu);
4478 const llvm_dest_ty = try o.lowerType(inst_ty);
44794471
4480 if (operand_is_ref and result_is_ref) {4472 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4481 // They are both pointers, so just return the same opaque pointer :)4473 const operand_ty = fg.typeOf(ty_op.operand);
4482 return operand;4474 const dest_ty = fg.typeOfIndex(inst);
4483 }4475 const operand = try fg.resolveInst(ty_op.operand);
44844476
4485 if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {4477 // We have the following `Air.Legalize` features enabled:
4486 return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");4478 //
4487 }4479 // * `.scalarize_bit_cast_array`
4480 // * `.scalarize_bit_cast_vector_non_elementwise`
4481 //
4482 // That means the `bit_cast` instructions we might see are limited to the following:
4483 //
4484 // * bool/int/float <-> bool/int/float
4485 // * `@Vector(n, A)` <-> `@Vector(n, B)`
4486 //
4487 // All of these cases can be handled by LLVM's `bitcast` instruction.
44884488
4489 const operand_scalar_ty = operand_ty.scalarType(zcu);4489 assert(!isByRef(operand_ty, zcu));
4490 const inst_scalar_ty = inst_ty.scalarType(zcu);4490 assert(!isByRef(dest_ty, zcu));
4491 if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
4492 return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4493 }
4494 if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
4495 return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4496 }
4497
4498 if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
4499 const elem_ty = operand_scalar_ty;
4500 assert(result_is_ref); // arrays are always by-ref provided they have runtime bits
4501 const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4502 const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4503 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4504 if (bitcast_ok) {
4505 _ = try self.wip.store(.normal, operand, array_ptr, alignment);
4506 } else {
4507 // If the ABI size of the element type is not evenly divisible by size in bits;
4508 // a simple bitcast will not work, and we fall back to extractelement.
4509 const elem_size = elem_ty.abiSize(zcu);
4510 const vector_len = operand_ty.arrayLen(zcu);
4511 var i: u64 = 0;
4512 while (i < vector_len) : (i += 1) {
4513 const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
4514 const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
4515 _ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
4516 }
4517 }
4518 return array_ptr;
4519 } else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
4520 const elem_ty = operand_ty.childType(zcu);
4521 assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits
4522 const llvm_vector_ty = try o.lowerType(inst_ty);
4523
4524 const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
4525 if (bitcast_ok) {
4526 // The array is aligned to the element's alignment, while the vector might have a completely
4527 // different alignment. This means we need to enforce the alignment of this load.
4528 const alignment = elem_ty.abiAlignment(zcu).toLlvm();
4529 return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
4530 } else {
4531 // If the ABI size of the element type is not evenly divisible by size in bits;
4532 // a simple bitcast will not work, and we fall back to extractelement.
4533 const elem_llvm_ty = try o.lowerType(elem_ty);
4534 const elem_size = elem_ty.abiSize(zcu);
4535 const vector_len = operand_ty.arrayLen(zcu);
4536 var vector = try o.builder.poisonValue(llvm_vector_ty);
4537 var i: u64 = 0;
4538 while (i < vector_len) : (i += 1) {
4539 const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
4540 const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
4541 vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
4542 }
4543 return vector;
4544 }
4545 }
45464491
4547 if (operand_is_ref) {4492 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4548 const alignment = operand_ty.abiAlignment(zcu).toLlvm();4493 return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
4549 return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");4494}
4550 }
45514495
4552 if (result_is_ref) {4496fn airNopCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4553 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();4497 const zcu = fg.object.zcu;
4554 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);4498 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4555 _ = try self.wip.store(.normal, operand, result_ptr, alignment);4499 const operand_ty = fg.typeOf(ty_op.operand);
4556 return result_ptr;4500 const dest_ty = fg.typeOfIndex(inst);
4557 }4501 assert(isByRef(operand_ty, zcu) == isByRef(dest_ty, zcu));
4502 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
4503 return fg.resolveInst(ty_op.operand);
4504}
45584505
4559 if (inst_ty.isSliceAtRuntime(zcu) or4506fn airPtrFromInt(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4560 ((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and4507 const o = fg.object;
4561 operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))4508 const zcu = o.zcu;
4562 {4509 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4563 // Both our operand and our result are values, not pointers,4510 const operand_ty = fg.typeOf(ty_op.operand);
4564 // but LLVM won't let us bitcast struct values or vectors with padding bits.4511 const dest_ty = fg.typeOfIndex(inst);
4565 // Therefore, we store operand to alloca, then load for result.4512 assert(operand_ty.scalarType(zcu).toIntern() == .usize_type);
4566 const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();4513 assert(dest_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4567 const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
4568 _ = try self.wip.store(.normal, operand, result_ptr, alignment);
4569 return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
4570 }
45714514
4572 return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");4515 const operand = try fg.resolveInst(ty_op.operand);
4516 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4517 return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4518}
4519
4520fn airIntFromPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4521 const o = fg.object;
4522 const zcu = o.zcu;
4523 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4524 const operand_ty = fg.typeOf(ty_op.operand);
4525 const dest_ty = fg.typeOfIndex(inst);
4526 assert(operand_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4527 assert(dest_ty.scalarType(zcu).toIntern() == .usize_type);
4528
4529 const operand = try fg.resolveInst(ty_op.operand);
4530 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4531 return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4532}
4533
4534fn airUnionFromEnum(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4535 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4536 const enum_ty = fg.typeOf(ty_op.operand);
4537 const union_ty = fg.typeOfIndex(inst);
4538 const enum_val = try fg.resolveInst(ty_op.operand);
4539 const union_ptr = try fg.buildZigAlloca(union_ty, .none);
4540 try fg.store(union_ptr, .none, enum_val, enum_ty, .normal);
4541 return union_ptr;
4573}4542}
45744543
4575fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4544fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -4628,9 +4597,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4628,9 +4597,8 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4628 "",4597 "",
4629 );4598 );
4630 } else if (mod.optimize_mode == .Debug) {4599 } else if (mod.optimize_mode == .Debug) {
4631 const alignment = inst_ty.abiAlignment(zcu).toLlvm();4600 const alloca = try self.buildZigAlloca(inst_ty, .none);
4632 const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);4601 try self.store(alloca, .none, arg_val, inst_ty, .normal);
4633 _ = try self.wip.store(.normal, arg_val, alloca, alignment);
4634 _ = try self.wip.callIntrinsic(4602 _ = try self.wip.callIntrinsic(
4635 .normal,4603 .normal,
4636 .none,4604 .none,
...@@ -4671,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4671,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4671 if (!elem_ty.hasRuntimeBits(zcu)) {4639 if (!elem_ty.hasRuntimeBits(zcu)) {
4672 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();4640 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4673 }4641 }
4674 const llvm_elem_ty = try o.lowerType(elem_ty);4642 return self.buildZigAlloca(elem_ty, ptr_align);
4675 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
4676}4643}
46774644
4678fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4645fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -4685,32 +4652,71 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4685,32 +4652,71 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4685 if (!elem_ty.hasRuntimeBits(zcu)) {4652 if (!elem_ty.hasRuntimeBits(zcu)) {
4686 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();4653 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4687 }4654 }
4688 const llvm_elem_ty = try o.lowerType(elem_ty);4655 return self.buildZigAlloca(elem_ty, ptr_align);
4689 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
4690}4656}
46914657
4692/// Use this instead of builder.buildAlloca, because this function makes sure to4658fn buildZigAlloca(fg: *FuncGen, ty: Type, @"align": InternPool.Alignment) Allocator.Error!Builder.Value {
4693/// put the alloca instruction at the top of the function!4659 const o = fg.object;
4660 const resolved_align: InternPool.Alignment = switch (@"align") {
4661 .none => ty.abiAlignment(o.zcu),
4662 else => |a| a,
4663 };
4664 return fg.buildAlloca(
4665 try o.lowerType(ty, .in_memory),
4666 resolved_align.toLlvm(),
4667 );
4668}
4669
4670/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
4694fn buildAlloca(4671fn buildAlloca(
4695 self: *FuncGen,4672 fg: *FuncGen,
4696 llvm_ty: Builder.Type,4673 llvm_ty: Builder.Type,
4697 alignment: Builder.Alignment,4674 alignment: Builder.Alignment,
4698) Allocator.Error!Builder.Value {4675) Allocator.Error!Builder.Value {
4699 const target = self.object.zcu.getTarget();4676 const wip = &fg.wip;
4700 return buildAllocaInner(&self.wip, llvm_ty, alignment, target);4677
4678 const alloca = blk: {
4679 const prev_cursor = wip.cursor;
4680 const prev_debug_location = wip.debug_location;
4681 defer {
4682 wip.cursor = prev_cursor;
4683 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
4684 wip.debug_location = prev_debug_location;
4685 }
4686
4687 wip.cursor = .{ .block = .entry };
4688 wip.debug_location = .no_location;
4689 const address_space = llvmAllocaAddressSpace(fg.object.zcu.getTarget());
4690 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
4691 };
4692
4693 // The pointer returned from this function should have the generic address space,
4694 // if this isn't the case then cast it to the generic address space.
4695 return fg.wip.conv(.unneeded, alloca, .ptr, "");
4701}4696}
47024697
4703fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {4698fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
4704 const o = self.object;4699 const o = fg.object;
4705 const zcu = o.zcu;4700 const zcu = o.zcu;
4706 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;4701 const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
4707 const dest_ptr = try self.resolveInst(bin_op.lhs);4702 const ptr = try fg.resolveInst(bin_op.lhs);
4708 const ptr_ty = self.typeOf(bin_op.lhs);4703 const ptr_ty = fg.typeOf(bin_op.lhs);
4709 const operand_ty = ptr_ty.childType(zcu);4704 const ptr_info = ptr_ty.ptrInfo(zcu);
4705 const ptr_alignment = ptr_ty.ptrAlignment(zcu);
4706
4707 const elem_ty = fg.typeOf(bin_op.rhs);
4708 assert(elem_ty.hasRuntimeBits(zcu));
4709
4710 fg.maybeMarkAllowZeroAccess(ptr_info);
4711
4712 const access_kind: Builder.MemoryAccessKind = switch (ptr_info.flags.is_volatile) {
4713 true => .@"volatile",
4714 false => .normal,
4715 };
47104716
4711 const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;4717 const val_is_undef = if (bin_op.rhs.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
4712 if (val_is_undef and !self.needMemsetWorkaround(operand_ty.abiSize(zcu))) {4718 if (val_is_undef and !fg.needMemsetWorkaround(elem_ty.abiSize(zcu))) {
4713 const owner_mod = self.ownerModule();4719 const owner_mod = fg.ownerModule();
47144720
4715 // Even if safety is disabled, we still emit a memset to undefined since it conveys4721 // Even if safety is disabled, we still emit a memset to undefined since it conveys
4716 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference4722 // extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
...@@ -4725,7 +4731,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4725,7 +4731,6 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4725 return .none;4731 return .none;
4726 }4732 }
47274733
4728 const ptr_info = ptr_ty.ptrInfo(zcu);
4729 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);4734 const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
4730 if (needs_bitmask) {4735 if (needs_bitmask) {
4731 // TODO: only some bits are to be undef, we cannot write with a simple memset.4736 // TODO: only some bits are to be undef, we cannot write with a simple memset.
...@@ -4734,27 +4739,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4734,27 +4739,82 @@ fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4734 return .none;4739 return .none;
4735 }4740 }
47364741
4737 self.maybeMarkAllowZeroAccess(ptr_info);4742 const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), elem_ty.abiSize(zcu));
47384743 _ = try fg.wip.callMemSet(
4739 const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu));4744 ptr,
4740 _ = try self.wip.callMemSet(4745 ptr_alignment.toLlvm(),
4741 dest_ptr,
4742 ptr_ty.ptrAlignment(zcu).toLlvm(),
4743 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),4746 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
4744 len,4747 len,
4745 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,4748 access_kind,
4746 self.disable_intrinsics,4749 fg.disable_intrinsics,
4747 );4750 );
4748 if (safety and owner_mod.valgrind) {4751 if (safety and owner_mod.valgrind) {
4749 try self.valgrindMarkUndef(dest_ptr, len);4752 try fg.valgrindMarkUndef(ptr, len);
4750 }4753 }
4751 return .none;4754 return .none;
4752 }4755 }
47534756
4754 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));4757 const elem = try fg.resolveInst(bin_op.rhs);
4758
4759 if (ptr_info.flags.vector_index != .none) {
4760 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4761 const vec_ty = try fg.pt.vectorType(.{
4762 .len = ptr_info.packed_offset.host_size,
4763 .child = elem_ty.toIntern(),
4764 });
4765
4766 const loaded_vector = try fg.load(ptr, ptr_alignment, vec_ty, access_kind);
4767 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4768 const modified_vector = try fg.wip.insertElement(loaded_vector, elem, index_val, "");
4769
4770 try fg.store(ptr, ptr_alignment, modified_vector, vec_ty, access_kind);
4771 return .none;
4772 }
47554773
4756 const src_operand = try self.resolveInst(bin_op.rhs);4774 if (ptr_info.packed_offset.host_size != 0) {
4757 try self.storeFull(dest_ptr, ptr_ty, src_operand, .none);4775 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4776 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4777 const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
4778
4779 const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
4780
4781 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4782 const shift_amt = try o.builder.intConst(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4783
4784 // Convert to equally-sized integer type in order to perform the bit
4785 // operations on the value to store
4786 const new_val_bits_type = try o.builder.intType(@intCast(elem_bits));
4787 const new_val_bits = if (elem_ty.isPtrAtRuntime(zcu))
4788 try fg.wip.cast(.ptrtoint, elem, new_val_bits_type, "")
4789 else
4790 try fg.wip.cast(.bitcast, elem, new_val_bits_type, "");
4791
4792 const mask_val = blk: {
4793 const zext = try fg.wip.cast(
4794 .zext,
4795 try o.builder.intValue(new_val_bits_type, -1),
4796 llvm_backing_int_ty,
4797 "",
4798 );
4799 const shl = try fg.wip.bin(.shl, zext, shift_amt.toValue(), "");
4800 break :blk try fg.wip.bin(
4801 .xor,
4802 shl,
4803 try o.builder.intValue(llvm_backing_int_ty, -1),
4804 "",
4805 );
4806 };
4807
4808 const masked_backing_int_val = try fg.wip.bin(.@"and", backing_int_val, mask_val, "");
4809 const extended_new_val = try fg.wip.cast(.zext, new_val_bits, llvm_backing_int_ty, "");
4810 const shifted_new_val = try fg.wip.bin(.shl, extended_new_val, shift_amt.toValue(), "");
4811 const new_backing_int_val = try fg.wip.bin(.@"or", shifted_new_val, masked_backing_int_val, "");
4812
4813 try fg.store(ptr, ptr_alignment, new_backing_int_val, backing_int_ty, access_kind);
4814 return .none;
4815 }
4816
4817 try fg.store(ptr, ptr_alignment, elem, elem_ty, access_kind);
4758 return .none;4818 return .none;
4759}4819}
47604820
...@@ -4766,7 +4826,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4766,7 +4826,7 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4766 const ptr_info = ptr_ty.ptrInfo(zcu);4826 const ptr_info = ptr_ty.ptrInfo(zcu);
4767 const ptr = try fg.resolveInst(ty_op.operand);4827 const ptr = try fg.resolveInst(ty_op.operand);
4768 const elem_ty = ptr_ty.childType(zcu);4828 const elem_ty = ptr_ty.childType(zcu);
4769 const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();4829 const ptr_align = ptr_ty.ptrAlignment(zcu);
47704830
4771 fg.maybeMarkAllowZeroAccess(ptr_info);4831 fg.maybeMarkAllowZeroAccess(ptr_info);
47724832
...@@ -4774,36 +4834,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4774,36 +4834,32 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4774 if (ptr_info.flags.is_volatile) .@"volatile" else .normal;4834 if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
47754835
4776 if (ptr_info.flags.vector_index != .none) {4836 if (ptr_info.flags.vector_index != .none) {
4777 const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index);4837 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4778 const vec_elem_ty = try o.lowerType(elem_ty);4838 const vec_ty = try fg.pt.vectorType(.{
4779 const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty);4839 .len = ptr_info.packed_offset.host_size,
47804840 .child = elem_ty.toIntern(),
4781 const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, "");4841 });
4782 return fg.wip.extractElement(loaded_vector, index_u32, "");4842 const vector_val = try fg.load(ptr, ptr_align, vec_ty, access_kind);
4843 const index_val = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
4844 return fg.wip.extractElement(vector_val, index_val, "");
4783 }4845 }
47844846
4785 if (ptr_info.packed_offset.host_size == 0) {4847 if (ptr_info.packed_offset.host_size == 0) {
4786 return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind);4848 return fg.load(ptr, ptr_align, elem_ty, access_kind);
4787 }4849 }
47884850
4789 const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8));4851 assert(!isByRef(elem_ty, zcu)); // all packable types are by-val
4790 const containing_int =
4791 try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, "");
47924852
4793 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);4853 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4794 const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset);4854 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4795 const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, "");4855 const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
4796 const elem_llvm_ty = try o.lowerType(elem_ty);
47974856
4798 if (isByRef(elem_ty, zcu)) {4857 const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
4799 const result_align = elem_ty.abiAlignment(zcu).toLlvm();
4800 const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align);
48014858
4802 const same_size_int = try o.builder.intType(@intCast(elem_bits));4859 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4803 const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");4860 const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4804 _ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align);4861 const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
4805 return result_ptr;4862 const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
4806 }
48074863
4808 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {4864 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
4809 const same_size_int = try o.builder.intType(@intCast(elem_bits));4865 const same_size_int = try o.builder.intType(@intCast(elem_bits));
...@@ -4853,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -4853,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
4853fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4909fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4854 _ = inst;4910 _ = inst;
4855 const o = self.object;4911 const o = self.object;
4856 const llvm_usize = try o.lowerType(.usize);4912 const llvm_usize = try o.lowerType(.usize, .by_value);
4857 if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {4913 if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
4858 // https://github.com/ziglang/zig/issues/119464914 // https://github.com/ziglang/zig/issues/11946
4859 return o.builder.intValue(llvm_usize, 0);4915 return o.builder.intValue(llvm_usize, 0);
...@@ -4865,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -4865,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4865fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4921fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4866 _ = inst;4922 _ = inst;
4867 const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");4923 const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
4868 return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), "");4924 return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize, .by_value), "");
4869}4925}
48704926
4871fn airCmpxchg(4927fn airCmpxchg(
...@@ -4882,7 +4938,7 @@ fn airCmpxchg(...@@ -4882,7 +4938,7 @@ fn airCmpxchg(
4882 var expected_value = try self.resolveInst(extra.expected_value);4938 var expected_value = try self.resolveInst(extra.expected_value);
4883 var new_value = try self.resolveInst(extra.new_value);4939 var new_value = try self.resolveInst(extra.new_value);
4884 const operand_ty = ptr_ty.childType(zcu);4940 const operand_ty = ptr_ty.childType(zcu);
4885 const llvm_operand_ty = try o.lowerType(operand_ty);4941 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
4886 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);4942 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
4887 if (llvm_abi_ty != .none) {4943 if (llvm_abi_ty != .none) {
4888 // operand needs widening and truncating4944 // operand needs widening and truncating
...@@ -4918,21 +4974,22 @@ fn airCmpxchg(...@@ -4918,21 +4974,22 @@ fn airCmpxchg(
4918 return self.wip.select(.normal, success_bit, zero, payload, "");4974 return self.wip.select(.normal, success_bit, zero, payload, "");
4919 }4975 }
49204976
4921 assert(isByRef(optional_ty, zcu));4977 assert(!isByRef(operand_ty, zcu)); // can only cmpxchg non-by-ref types
4978 assert(isByRef(optional_ty, zcu)); // all optionals are by-ref
49224979
4923 comptime assert(optional_layout_version == 3);4980 comptime assert(optional_layout_version == 3);
49244981
4925 const non_null_bit = try self.wip.not(success_bit, "");4982 const non_null_bit = try self.wip.not(success_bit, "");
49264983
4927 const payload_align = operand_ty.abiAlignment(zcu).toLlvm();4984 const payload_align = operand_ty.abiAlignment(zcu);
4928 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align);4985 const alloca_inst = try self.buildZigAlloca(optional_ty, .none);
49294986
4930 // Payload is always the first field at offset 0, so address is `alloca_inst`4987 // Payload is always the first field at offset 0, so address is `alloca_inst`
4931 _ = try self.wip.store(.normal, payload, alloca_inst, payload_align);4988 try self.store(alloca_inst, .none, payload, operand_ty, .normal);
49324989
4933 // Non-null bit is after payload with no padding because it has alignment 14990 // Non-null bit is after payload with no padding because it has alignment 1
4934 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));4991 const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
4935 _ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));4992 try self.store(non_null_ptr, payload_align, non_null_bit, .bool, .normal);
49364993
4937 return alloca_inst;4994 return alloca_inst;
4938}4995}
...@@ -4951,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -4951,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
4951 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);5008 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
4952 const ordering = toLlvmAtomicOrdering(extra.ordering());5009 const ordering = toLlvmAtomicOrdering(extra.ordering());
4953 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);5010 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
4954 const llvm_operand_ty = try o.lowerType(operand_ty);5011 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
49555012
4956 const access_kind: Builder.MemoryAccessKind =5013 const access_kind: Builder.MemoryAccessKind =
4957 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;5014 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -4980,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -4980,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
49805037
4981 // If we are storing a pointer we need to convert to and from a plain old integer.5038 // If we are storing a pointer we need to convert to and from a plain old integer.
4982 const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {5039 const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
4983 .pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""),5040 .pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize, .by_value), ""),
4984 else => operand,5041 else => operand,
4985 };5042 };
49865043
...@@ -5019,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -5019,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
5019 Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();5076 Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5020 const access_kind: Builder.MemoryAccessKind =5077 const access_kind: Builder.MemoryAccessKind =
5021 if (info.flags.is_volatile) .@"volatile" else .normal;5078 if (info.flags.is_volatile) .@"volatile" else .normal;
5022 const elem_llvm_ty = try o.lowerType(elem_ty);5079 const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
50235080
5024 self.maybeMarkAllowZeroAccess(info);5081 self.maybeMarkAllowZeroAccess(info);
50255082
...@@ -5073,7 +5130,17 @@ fn airAtomicStore(...@@ -5073,7 +5130,17 @@ fn airAtomicStore(
50735130
5074 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));5131 self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
50755132
5076 try self.storeFull(ptr, ptr_ty, element, ordering);5133 assert(!isByRef(operand_ty, zcu));
5134
5135 _ = try self.wip.storeAtomic(
5136 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
5137 element,
5138 ptr,
5139 self.sync_scope,
5140 ordering,
5141 ptr_ty.ptrAlignment(zcu).toLlvm(),
5142 );
5143
5077 return .none;5144 return .none;
5078}5145}
50795146
...@@ -5084,7 +5151,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5084,7 +5151,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5084 const dest_slice = try self.resolveInst(bin_op.lhs);5151 const dest_slice = try self.resolveInst(bin_op.lhs);
5085 const ptr_ty = self.typeOf(bin_op.lhs);5152 const ptr_ty = self.typeOf(bin_op.lhs);
5086 const elem_ty = self.typeOf(bin_op.rhs);5153 const elem_ty = self.typeOf(bin_op.rhs);
5087 const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();5154 const dest_ptr_align = ptr_ty.ptrAlignment(zcu);
5088 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);5155 const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
5089 const access_kind: Builder.MemoryAccessKind =5156 const access_kind: Builder.MemoryAccessKind =
5090 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;5157 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -5110,7 +5177,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5110,7 +5177,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5110 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);5177 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5111 _ = try self.wip.callMemSet(5178 _ = try self.wip.callMemSet(
5112 dest_ptr,5179 dest_ptr,
5113 dest_ptr_align,5180 dest_ptr_align.toLlvm(),
5114 fill_byte,5181 fill_byte,
5115 len,5182 len,
5116 access_kind,5183 access_kind,
...@@ -5132,7 +5199,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5132,7 +5199,7 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5132 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);5199 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5133 _ = try self.wip.callMemSet(5200 _ = try self.wip.callMemSet(
5134 dest_ptr,5201 dest_ptr,
5135 dest_ptr_align,5202 dest_ptr_align.toLlvm(),
5136 fill_byte,5203 fill_byte,
5137 len,5204 len,
5138 access_kind,5205 access_kind,
...@@ -5145,14 +5212,31 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5145,14 +5212,31 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5145 const value = try self.resolveInst(bin_op.rhs);5212 const value = try self.resolveInst(bin_op.rhs);
5146 const elem_abi_size = elem_ty.abiSize(zcu);5213 const elem_abi_size = elem_ty.abiSize(zcu);
51475214
5148 if (allow_byte_memset and elem_abi_size == 1 and elem_ty.bitSize(zcu) == 8) {5215 intrinsic: {
5149 // In this case we can take advantage of LLVM's intrinsic.5216 if (!allow_byte_memset) break :intrinsic;
5150 const fill_byte = try self.bitCast(value, elem_ty, Type.u8);5217 if (elem_abi_size != 1) break :intrinsic;
5218 // To use LLVM's intrinsic, we need to convert the operand to a raw 8-bit integer value.
5219 const fill_byte: Builder.Value = byte: {
5220 if (isByRef(elem_ty, zcu)) {
5221 break :byte try self.load(value, elem_ty.abiAlignment(zcu), .u8, .normal);
5222 }
5223 if (elem_ty.isAbiInt(zcu)) {
5224 const info = elem_ty.intInfo(zcu);
5225 break :byte try self.wip.conv(switch (info.signedness) {
5226 .unsigned => .unsigned,
5227 .signed => .signed,
5228 }, value, .i8, "");
5229 }
5230 if (elem_ty.toIntern() == .bool_type) {
5231 break :byte try self.wip.cast(.zext, value, .i8, "");
5232 }
5233 break :intrinsic;
5234 };
5235 // Great, we can use the intrinsic!
5151 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);5236 const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
5152
5153 _ = try self.wip.callMemSet(5237 _ = try self.wip.callMemSet(
5154 dest_ptr,5238 dest_ptr,
5155 dest_ptr_align,5239 dest_ptr_align.toLlvm(),
5156 fill_byte,5240 fill_byte,
5157 len,5241 len,
5158 access_kind,5242 access_kind,
...@@ -5182,7 +5266,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5182,7 +5266,6 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5182 const body_block = try self.wip.block(1, "InlineMemsetBody");5266 const body_block = try self.wip.block(1, "InlineMemsetBody");
5183 const end_block = try self.wip.block(1, "InlineMemsetEnd");5267 const end_block = try self.wip.block(1, "InlineMemsetEnd");
51845268
5185 const llvm_usize_ty = try o.lowerType(.usize);
5186 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {5269 const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
5187 .slice => try self.ptraddScaled(5270 .slice => try self.ptraddScaled(
5188 dest_ptr,5271 dest_ptr,
...@@ -5201,18 +5284,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5201,18 +5284,8 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
52015284
5202 self.wip.cursor = .{ .block = body_block };5285 self.wip.cursor = .{ .block = body_block };
5203 const elem_abi_align = elem_ty.abiAlignment(zcu);5286 const elem_abi_align = elem_ty.abiAlignment(zcu);
5204 const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();5287 const it_ptr_align: InternPool.Alignment = dest_ptr_align.min(elem_abi_align);
5205 if (isByRef(elem_ty, zcu)) {5288 try self.store(it_ptr.toValue(), it_ptr_align, value, elem_ty, access_kind);
5206 _ = try self.wip.callMemCpy(
5207 it_ptr.toValue(),
5208 it_ptr_align,
5209 value,
5210 elem_abi_align.toLlvm(),
5211 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
5212 access_kind,
5213 self.disable_intrinsics,
5214 );
5215 } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
5216 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);5289 const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
5217 _ = try self.wip.br(loop_block);5290 _ = try self.wip.br(loop_block);
52185291
...@@ -5289,14 +5362,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder....@@ -5289,14 +5362,11 @@ fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
52895362
5290 const union_ptr = try self.resolveInst(bin_op.lhs);5363 const union_ptr = try self.resolveInst(bin_op.lhs);
5291 const new_tag = try self.resolveInst(bin_op.rhs);5364 const new_tag = try self.resolveInst(bin_op.rhs);
5365 const tag_ty = self.typeOf(bin_op.rhs);
5292 const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);5366 const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
5293 if (layout.payload_size == 0) {
5294 _ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
5295 return .none;
5296 }
5297 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());5367 const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
5298 const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());5368 const tag_ptr_align = union_ptr_align.offset(layout.tagOffset());
5299 _ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());5369 try self.store(tag_field_ptr, tag_ptr_align, new_tag, tag_ty, access_kind);
5300 return .none;5370 return .none;
5301}5371}
53025372
...@@ -5308,16 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder....@@ -5308,16 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5308 const layout = un_ty.unionGetLayout(zcu);5378 const layout = un_ty.unionGetLayout(zcu);
5309 assert(layout.tag_size != 0);5379 assert(layout.tag_size != 0);
5310 const operand = try self.resolveInst(ty_op.operand);5380 const operand = try self.resolveInst(ty_op.operand);
5311 if (isByRef(un_ty, zcu)) {5381 assert(isByRef(un_ty, zcu));
5312 const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?);5382 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5313 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());5383 return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5314 return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
5315 } else {
5316 // This is only possible if all fields are zero-bit, in which case `operand` is already an
5317 // integer value (the union is lowered as its enum tag).
5318 assert(layout.payload_size == 0);
5319 return operand;
5320 }
5321}5384}
53225385
5323fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {5386fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
...@@ -5347,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)...@@ -5347,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5347 .normal,5410 .normal,
5348 .none,5411 .none,
5349 intrinsic,5412 intrinsic,
5350 &.{try o.lowerType(operand_ty)},5413 &.{try o.lowerType(operand_ty, .by_value)},
5351 &.{ operand, .false },5414 &.{ operand, .false },
5352 "",5415 "",
5353 );5416 );
5354 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5417 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5355}5418}
53565419
5357fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {5420fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
...@@ -5365,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)...@@ -5365,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5365 .normal,5428 .normal,
5366 .none,5429 .none,
5367 intrinsic,5430 intrinsic,
5368 &.{try o.lowerType(operand_ty)},5431 &.{try o.lowerType(operand_ty, .by_value)},
5369 &.{operand},5432 &.{operand},
5370 "",5433 "",
5371 );5434 );
5372 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5435 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5373}5436}
53745437
5375fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5438fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5382,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5382,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
53825445
5383 const inst_ty = self.typeOfIndex(inst);5446 const inst_ty = self.typeOfIndex(inst);
5384 var operand = try self.resolveInst(ty_op.operand);5447 var operand = try self.resolveInst(ty_op.operand);
5385 var llvm_operand_ty = try o.lowerType(operand_ty);5448 var llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
53865449
5387 if (bits % 16 == 8) {5450 if (bits % 16 == 8) {
5388 // If not an even byte-multiple, we need zero-extend + shift-left 1 byte5451 // If not an even byte-multiple, we need zero-extend + shift-left 1 byte
...@@ -5403,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5403,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
54035466
5404 const result =5467 const result =
5405 try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");5468 try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5406 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5469 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5407}5470}
54085471
5409fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5472fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5423,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui...@@ -5423,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui
54235486
5424 for (0..names.len) |name_index| {5487 for (0..names.len) |name_index| {
5425 const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;5488 const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5426 const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int);5489 const this_tag_int_value = try o.builder.intConst(try o.errorIntType(.by_value), err_int);
5427 try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);5490 try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5428 }5491 }
5429 self.wip.cursor = .{ .block = valid_block };5492 self.wip.cursor = .{ .block = valid_block };
...@@ -5480,21 +5543,20 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5480,21 +5543,20 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5480 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;5543 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5481 const operand = try self.resolveInst(un_op);5544 const operand = try self.resolveInst(un_op);
5482 const slice_ty = self.typeOfIndex(inst);5545 const slice_ty = self.typeOfIndex(inst);
5483 const slice_llvm_ty = try o.lowerType(slice_ty);
54845546
5485 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.5547 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5486 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");5548 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize, .by_value), "");
54875549
5488 const error_name_table_ptr = try o.getErrorNameTable();5550 const error_name_table_ptr = try o.getErrorNameTable();
5489 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));5551 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
5490 return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");5552 return self.load(error_name_ptr, .none, slice_ty, .normal);
5491}5553}
54925554
5493fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5555fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5494 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5556 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5495 const scalar = try self.resolveInst(ty_op.operand);5557 const scalar = try self.resolveInst(ty_op.operand);
5496 const vector_ty = self.typeOfIndex(inst);5558 const vector_ty = self.typeOfIndex(inst);
5497 return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, "");5559 return self.wip.splatVector(try self.object.lowerType(vector_ty, .by_value), scalar, "");
5498}5560}
54995561
5500fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5562fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5517,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5517,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5517 const operand = try fg.resolveInst(unwrapped.operand);5579 const operand = try fg.resolveInst(unwrapped.operand);
5518 const mask = unwrapped.mask;5580 const mask = unwrapped.mask;
5519 const operand_ty = fg.typeOf(unwrapped.operand);5581 const operand_ty = fg.typeOf(unwrapped.operand);
5520 const llvm_operand_ty = try o.lowerType(operand_ty);5582 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5521 const llvm_result_ty = try o.lowerType(unwrapped.result_ty);5583 const llvm_result_ty = try o.lowerType(unwrapped.result_ty, .by_value);
5522 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));5584 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5523 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);5585 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
5524 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);5586 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5525 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);5587 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
...@@ -5549,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5549,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5549 .elem => llvm_poison_elem,5611 .elem => llvm_poison_elem,
5550 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {5612 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
5551 any_defined_comptime_value = true;5613 any_defined_comptime_value = true;
5552 break :elem try o.lowerValue(val);5614 break :elem try o.lowerValue(val, .by_value);
5553 } else llvm_poison_elem,5615 } else llvm_poison_elem,
5554 };5616 };
5555 }5617 }
...@@ -5621,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5621,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5621 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);5683 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
56225684
5623 const mask = unwrapped.mask;5685 const mask = unwrapped.mask;
5624 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));5686 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5625 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);5687 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5626 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);5688 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
56275689
...@@ -5696,15 +5758,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5696,15 +5758,13 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5696/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.5758/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
5697///5759///
5698/// Equivalent to:5760/// Equivalent to:
5699/// reduce: {5761/// ```
5700/// var i: usize = 0;5762/// var accum: T = init;
5701/// var accum: T = init;5763/// for (0..i) |i| {
5702/// while (i < vec.len) : (i += 1) {5764/// accum = llvm_fn(accum, vec[i]);
5703/// accum = llvm_fn(accum, vec[i]);5765/// }
5704/// }5766/// // result is 'accum'
5705/// break :reduce accum;5767/// ```
5706/// }
5707///
5708fn buildReducedCall(5768fn buildReducedCall(
5709 self: *FuncGen,5769 self: *FuncGen,
5710 llvm_fn: Builder.Function.Index,5770 llvm_fn: Builder.Function.Index,
...@@ -5713,56 +5773,54 @@ fn buildReducedCall(...@@ -5713,56 +5773,54 @@ fn buildReducedCall(
5713 accum_init: Builder.Value,5773 accum_init: Builder.Value,
5714) Allocator.Error!Builder.Value {5774) Allocator.Error!Builder.Value {
5715 const o = self.object;5775 const o = self.object;
5716 const usize_ty = try o.lowerType(.usize);5776 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
5717 const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);5777 const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
5718 const llvm_result_ty = accum_init.typeOfWip(&self.wip);5778 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
57195779
5720 // Allocate and initialize our mutable variables5780 const entry_block = self.wip.cursor.block;
5721 const i_ptr = try self.buildAlloca(usize_ty, .default);
5722 _ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
5723 const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
5724 _ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
57255781
5726 // Setup the loop5782 const cond_block = try self.wip.block(2, "ReduceLoopCond");
5727 const loop = try self.wip.block(2, "ReduceLoop");5783 const body_block = try self.wip.block(1, "ReduceLoopBody");
5728 const loop_exit = try self.wip.block(1, "AfterReduce");5784 const exit_block = try self.wip.block(1, "ReduceLoopExit");
5729 _ = try self.wip.br(loop);5785
5730 {5786 _ = try self.wip.br(cond_block);
5731 self.wip.cursor = .{ .block = loop };5787
57325788 // ReduceLoopCond:
5733 // while (i < vec.len)5789 // %index = phi iN [0, %Entry], [%new_index, %ReduceLoopBody]
5734 const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");5790 // %accum = phi T [%accum_init, %Entry], [%new_accum, %ReduceLoopBody]
5735 const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");5791 // %cond = icmp ult iN %index, %vector_len
5736 const loop_then = try self.wip.block(1, "ReduceLoopThen");5792 // br i1 %cond, label %ReduceLoopBody, label %ReduceLoopExit
57375793 self.wip.cursor = .{ .block = cond_block };
5738 _ = try self.wip.brCond(cond, loop_then, loop_exit, .none);5794 const index = try self.wip.phi(llvm_usize_ty, "");
57395795 const accum = try self.wip.phi(llvm_result_ty, "");
5740 {5796 const cond = try self.wip.icmp(.ult, index.toValue(), llvm_vector_len, "");
5741 self.wip.cursor = .{ .block = loop_then };5797 _ = try self.wip.brCond(cond, body_block, exit_block, .none);
57425798
5743 // accum = f(accum, vec[i]);5799 // ReduceLoopBody:
5744 const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");5800 // %elem = extractelement <n x T> %operand_vec, iN %index
5745 const element = try self.wip.extractElement(operand_vector, i, "");5801 // %new_accum = call T @llvm_fn(T %accum, T %elem)
5746 const new_accum = try self.wip.call(5802 // %new_index = add nuw iN %index, 1
5747 .normal,5803 // br label %ReduceLoopCond
5748 .ccc,5804 self.wip.cursor = .{ .block = body_block };
5749 .none,5805 const elem = try self.wip.extractElement(operand_vector, index.toValue(), "");
5750 llvm_fn.typeOf(&o.builder),5806 const new_accum = try self.wip.call(
5751 llvm_fn.toValue(&o.builder),5807 .normal,
5752 &.{ accum, element },5808 .ccc,
5753 "",5809 .none,
5754 );5810 llvm_fn.typeOf(&o.builder),
5755 _ = try self.wip.store(.normal, new_accum, accum_ptr, .default);5811 llvm_fn.toValue(&o.builder),
5812 &.{ accum.toValue(), elem },
5813 "",
5814 );
5815 const new_index = try self.wip.bin(.@"add nuw", index.toValue(), try o.builder.intValue(llvm_usize_ty, 1), "");
5816 _ = try self.wip.br(cond_block);
57565817
5757 // i += 15818 const index_init = try o.builder.intValue(llvm_usize_ty, 0);
5758 const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");5819 index.finish(&.{ index_init, new_index }, &.{ entry_block, body_block }, &self.wip);
5759 _ = try self.wip.store(.normal, new_i, i_ptr, .default);5820 accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
5760 _ = try self.wip.br(loop);
5761 }
5762 }
57635821
5764 self.wip.cursor = .{ .block = loop_exit };5822 self.wip.cursor = .{ .block = exit_block };
5765 return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");5823 return accum.toValue();
5766}5824}
57675825
5768fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {5826fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...@@ -5773,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A...@@ -5773,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A
5773 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;5831 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
5774 const operand = try self.resolveInst(reduce.operand);5832 const operand = try self.resolveInst(reduce.operand);
5775 const operand_ty = self.typeOf(reduce.operand);5833 const operand_ty = self.typeOf(reduce.operand);
5776 const llvm_operand_ty = try o.lowerType(operand_ty);5834 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5777 const scalar_ty = self.typeOfIndex(inst);5835 const scalar_ty = self.typeOfIndex(inst);
5778 const llvm_scalar_ty = try o.lowerType(scalar_ty);5836 const llvm_scalar_ty = try o.lowerType(scalar_ty, .by_value);
57795837
5780 switch (reduce.operation) {5838 switch (reduce.operation) {
5781 .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {5839 .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
...@@ -5882,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5882,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5882 const result_ty = self.typeOfIndex(inst);5940 const result_ty = self.typeOfIndex(inst);
5883 const len: usize = @intCast(result_ty.arrayLen(zcu));5941 const len: usize = @intCast(result_ty.arrayLen(zcu));
5884 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);5942 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
5885 const llvm_result_ty = try o.lowerType(result_ty);
58865943
5887 switch (result_ty.zigTypeTag(zcu)) {5944 switch (result_ty.zigTypeTag(zcu)) {
5888 .vector => {5945 .vector => {
5946 const llvm_result_ty = try o.lowerType(result_ty, .by_value);
5889 var vector = try o.builder.poisonValue(llvm_result_ty);5947 var vector = try o.builder.poisonValue(llvm_result_ty);
5890 for (elements, 0..) |elem, i| {5948 for (elements, 0..) |elem, i| {
5891 const index_u32 = try o.builder.intValue(.i32, i);5949 const index_u32 = try o.builder.intValue(.i32, i);
...@@ -5927,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5927,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5927 // TODO in debug builds init to undef so that the padding will be 0xaa5985 // TODO in debug builds init to undef so that the padding will be 0xaa
5928 // even if we fully populate the fields.5986 // even if we fully populate the fields.
5929 const struct_align = result_ty.abiAlignment(zcu);5987 const struct_align = result_ty.abiAlignment(zcu);
5930 const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());5988 const alloca_inst = try self.buildZigAlloca(result_ty, .none);
59315989
5932 for (elements, 0..) |elem, field_index| {5990 for (elements, 0..) |elem, field_index| {
5933 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;5991 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
...@@ -5939,24 +5997,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5939,24 +5997,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
59395997
5940 const llvm_field_val = try self.resolveInst(elem);5998 const llvm_field_val = try self.resolveInst(elem);
59415999
5942 if (isByRef(field_ty, zcu)) {6000 try self.store(field_ptr, field_ptr_align, llvm_field_val, field_ty, .normal);
5943 _ = try self.wip.callMemCpy(
5944 field_ptr,
5945 field_ptr_align.toLlvm(),
5946 llvm_field_val,
5947 field_ty.abiAlignment(zcu).toLlvm(),
5948 try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)),
5949 .normal,
5950 self.disable_intrinsics,
5951 );
5952 } else {
5953 _ = try self.wip.store(
5954 .normal,
5955 llvm_field_val,
5956 field_ptr,
5957 field_ptr_align.toLlvm(),
5958 );
5959 }
5960 }6001 }
59616002
5962 return alloca_inst;6003 return alloca_inst;
...@@ -5965,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5965,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5965 .array => {6006 .array => {
5966 assert(isByRef(result_ty, zcu));6007 assert(isByRef(result_ty, zcu));
59676008
5968 const alignment = result_ty.abiAlignment(zcu).toLlvm();6009 const alloca_inst = try self.buildZigAlloca(result_ty, .none);
5969 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
59706010
5971 const array_info = result_ty.arrayInfo(zcu);6011 const array_info = result_ty.arrayInfo(zcu);
59726012
...@@ -5975,12 +6015,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -5975,12 +6015,12 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
5975 for (elements, 0..) |elem, i| {6015 for (elements, 0..) |elem, i| {
5976 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);6016 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
5977 const llvm_elem = try self.resolveInst(elem);6017 const llvm_elem = try self.resolveInst(elem);
5978 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type);6018 try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type, .normal);
5979 }6019 }
5980 if (array_info.sentinel) |sent_val| {6020 if (array_info.sentinel) |sent_val| {
5981 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);6021 const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
5982 const llvm_elem = try self.resolveValue(sent_val);6022 const llvm_elem = try self.resolveValue(sent_val);
5983 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type);6023 try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type, .normal);
5984 }6024 }
59856025
5986 return alloca_inst;6026 return alloca_inst;
...@@ -5996,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5996,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5996 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6036 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5997 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;6037 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
5998 const union_ty = self.typeOfIndex(inst);6038 const union_ty = self.typeOfIndex(inst);
5999 const union_llvm_ty = try o.lowerType(union_ty);
6000 const union_obj = zcu.typeToUnion(union_ty).?;6039 const union_obj = zcu.typeToUnion(union_ty).?;
60016040
6002 assert(union_obj.layout != .@"packed");6041 assert(union_obj.layout != .@"packed");
...@@ -6006,28 +6045,28 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6006,28 +6045,28 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6006 assert(layout.payload_size != 0); // otherwise the value would be comptime-known6045 assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6007 assert(isByRef(union_ty, zcu));6046 assert(isByRef(union_ty, zcu));
60086047
6009 const alignment = layout.abi_align.toLlvm();6048 const result_ptr = try self.buildZigAlloca(union_ty, layout.abi_align);
6010 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
6011 const llvm_payload = try self.resolveInst(extra.init);6049 const llvm_payload = try self.resolveInst(extra.init);
6012 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);6050 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6013 assert(field_ty.hasRuntimeBits(zcu));6051 assert(field_ty.hasRuntimeBits(zcu));
60146052
6015 {6053 {
6016 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());6054 const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
6017 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty);6055 try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty, .normal);
6018 }6056 }
60196057
6020 if (layout.tag_size != 0) {6058 if (layout.tag_size != 0) {
6021 const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type);6059 const tag_ty: Type = .fromInterned(union_obj.enum_tag_type);
6060 const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6022 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {6061 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6023 .none => try o.builder.intConst(6062 .none => try o.builder.intConst(
6024 try o.lowerType(.fromInterned(union_obj.enum_tag_type)),6063 try o.lowerType(.fromInterned(union_obj.enum_tag_type), .by_value),
6025 extra.field_index, // auto-numbered6064 extra.field_index, // auto-numbered
6026 ),6065 ),
6027 else => |tag_val_ip| try o.lowerValue(tag_val_ip),6066 else => |tag_val_ip| try o.lowerValue(tag_val_ip, .by_value),
6028 };6067 };
6029 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());6068 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6030 _ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());6069 try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
6031 }6070 }
60326071
6033 return result_ptr;6072 return result_ptr;
...@@ -6086,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6086,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6086 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6125 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6087 const inst_ty = self.typeOfIndex(inst);6126 const inst_ty = self.typeOfIndex(inst);
6088 const operand = try self.resolveInst(ty_op.operand);6127 const operand = try self.resolveInst(ty_op.operand);
6089 return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), "");6128 return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty, .by_value), "");
6090}6129}
60916130
6092fn workIntrinsic(6131fn workIntrinsic(
...@@ -6134,7 +6173,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6134,7 +6173,7 @@ fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6134 // Load the work_group_* member from the struct as u16.6173 // Load the work_group_* member from the struct as u16.
6135 // Just treat the dispatch pointer as an array of u16 to keep things simple.6174 // Just treat the dispatch pointer as an array of u16 to keep things simple.
6136 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);6175 const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
6137 return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");6176 return self.load(workgroup_size_ptr, .@"2", .u16, .normal);
6138 },6177 },
6139 .nvptx, .nvptx64 => {6178 .nvptx, .nvptx64 => {
6140 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");6179 return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
...@@ -6169,8 +6208,8 @@ fn optCmpNull(...@@ -6169,8 +6208,8 @@ fn optCmpNull(
6169 comptime assert(optional_layout_version == 3);6208 comptime assert(optional_layout_version == 3);
6170 // Non-null bit is always after the payload, with no padding because it has alignment 1.6209 // Non-null bit is always after the payload, with no padding because it has alignment 1.
6171 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));6210 const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
6172 const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");6211 const non_null = try self.load(non_null_ptr, .@"1", .bool, access_kind);
6173 return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");6212 return self.wip.icmp(cond, non_null, .false, "");
6174}6213}
61756214
6176/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.6215/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
...@@ -6187,13 +6226,9 @@ fn optPayloadHandle(...@@ -6187,13 +6226,9 @@ fn optPayloadHandle(
6187 // Payload is first field so always at the same address as the optional itself.6226 // Payload is first field so always at the same address as the optional itself.
6188 const payload_ptr = opt_ptr;6227 const payload_ptr = opt_ptr;
61896228
6190 const payload_align = payload_ty.abiAlignment(zcu).toLlvm();6229 if (can_elide_load and isByRef(payload_ty, zcu)) return payload_ptr;
6191 if (isByRef(payload_ty, zcu)) {6230
6192 if (can_elide_load) return payload_ptr;6231 return fg.load(payload_ptr, .none, payload_ty, .normal);
6193 return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
6194 } else {
6195 return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
6196 }
6197}6232}
61986233
6199fn fieldPtr(6234fn fieldPtr(
...@@ -6217,220 +6252,156 @@ fn fieldPtr(...@@ -6217,220 +6252,156 @@ fn fieldPtr(
6217 return self.ptraddConst(aggregate_ptr, offset);6252 return self.ptraddConst(aggregate_ptr, offset);
6218}6253}
62196254
6220/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.6255/// Non-atomic, non-bitpacked load of type `load_ty` from pointer `ptr`.
6221fn loadTruncate(6256///
6222 fg: *FuncGen,6257/// `ptr` has alignment `ptr_align`, or `load_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6223 access_kind: Builder.MemoryAccessKind,6258///
6224 payload_ty: Type,6259/// If `load_ty` is a by-ref type, then the value is copied to a new alloca with a memcpy, and a
6225 payload_ptr: Builder.Value,6260/// pointer to that alloca is returned.
6226 payload_alignment: Builder.Alignment,
6227) Allocator.Error!Builder.Value {
6228 // from https://llvm.org/docs/LangRef.html#load-instruction :
6229 // "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
6230 // => so load the byte aligned value and trunc the unwanted bits.
6231
6232 const o = fg.object;
6233 const zcu = o.zcu;
6234 const payload_llvm_ty = try o.lowerType(payload_ty);
6235 const abi_size = payload_ty.abiSize(zcu);
6236
6237 const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
6238 try o.builder.intType(@intCast(abi_size * 8))
6239 else
6240 payload_llvm_ty;
6241 const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
6242 const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big)
6243 try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
6244 load_llvm_ty,
6245 (payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
6246 ), "")
6247 else
6248 loaded;
6249
6250 return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
6251}
6252
6253/// Load a by-ref type by constructing a new alloca and performing a memcpy.
6254fn loadByRef(
6255 fg: *FuncGen,
6256 ptr: Builder.Value,
6257 pointee_type: Type,
6258 ptr_alignment: Builder.Alignment,
6259 access_kind: Builder.MemoryAccessKind,
6260) Allocator.Error!Builder.Value {
6261 const o = fg.object;
6262 const pointee_llvm_ty = try o.lowerType(pointee_type);
6263 const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
6264 .max(pointee_type.abiAlignment(o.zcu)).toLlvm();
6265 const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
6266 const size_bytes = pointee_type.abiSize(o.zcu);
6267 _ = try fg.wip.callMemCpy(
6268 result_ptr,
6269 result_align,
6270 ptr,
6271 ptr_alignment,
6272 try o.builder.intValue(try o.lowerType(.usize), size_bytes),
6273 access_kind,
6274 fg.disable_intrinsics,
6275 );
6276 return result_ptr;
6277}
6278
6279/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value
6280/// into a new alloca.
6281fn load(6261fn load(
6282 fg: *FuncGen,6262 fg: *FuncGen,
6283 ptr: Builder.Value,6263 ptr: Builder.Value,
6284 elem_ty: Type,6264 ptr_align: InternPool.Alignment,
6285 ptr_alignment: Builder.Alignment,6265 load_ty: Type,
6286 access_kind: Builder.MemoryAccessKind,6266 access_kind: Builder.MemoryAccessKind,
6287) Allocator.Error!Builder.Value {6267) Allocator.Error!Builder.Value {
6288 const zcu = fg.object.zcu;6268 const o = fg.object;
6289 if (isByRef(elem_ty, zcu)) {
6290 return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
6291 } else {
6292 return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
6293 }
6294}
6295
6296fn storeFull(
6297 self: *FuncGen,
6298 ptr: Builder.Value,
6299 ptr_ty: Type,
6300 elem: Builder.Value,
6301 ordering: Builder.AtomicOrdering,
6302) Allocator.Error!void {
6303 const o = self.object;
6304 const zcu = o.zcu;6269 const zcu = o.zcu;
6305 const info = ptr_ty.ptrInfo(zcu);
6306 const elem_ty = Type.fromInterned(info.child);
6307 if (!elem_ty.hasRuntimeBits(zcu)) {
6308 return;
6309 }
6310 const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
6311 const access_kind: Builder.MemoryAccessKind =
6312 if (info.flags.is_volatile) .@"volatile" else .normal;
6313
6314 if (info.flags.vector_index != .none) {
6315 const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
6316 const vec_elem_ty = try o.lowerType(elem_ty);
6317 const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
63186270
6319 const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");6271 const abi_align = load_ty.abiAlignment(zcu);
6272 const abi_size = load_ty.abiSize(zcu);
63206273
6321 const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");6274 const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6275 .none => abi_align.toLlvm(),
6276 else => |a| a.toLlvm(),
6277 };
63226278
6323 assert(ordering == .none);6279 if (isByRef(load_ty, zcu)) {
6324 _ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);6280 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6325 return;6281 const result_ptr = try fg.buildZigAlloca(load_ty, .none);
6282 _ = try fg.wip.callMemCpy(
6283 result_ptr,
6284 abi_align.toLlvm(),
6285 ptr,
6286 llvm_ptr_align,
6287 try o.builder.intValue(llvm_usize_ty, abi_size),
6288 access_kind,
6289 fg.disable_intrinsics,
6290 );
6291 return result_ptr;
6326 }6292 }
63276293
6328 if (info.packed_offset.host_size != 0) {6294 const llvm_memory_ty = try o.lowerType(load_ty, .in_memory);
6329 const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));6295 const llvm_value_ty = try o.lowerType(load_ty, .by_value);
6330 assert(ordering == .none);
6331 const containing_int =
6332 try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
6333 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
6334 const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
6335 // Convert to equally-sized integer type in order to perform the bit
6336 // operations on the value to store
6337 const value_bits_type = try o.builder.intType(@intCast(elem_bits));
6338 const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
6339 try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
6340 else
6341 try self.wip.cast(.bitcast, elem, value_bits_type, "");
6342
6343 const mask_val = blk: {
6344 const zext = try self.wip.cast(
6345 .zext,
6346 try o.builder.intValue(value_bits_type, -1),
6347 containing_int_ty,
6348 "",
6349 );
6350 const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
6351 break :blk try self.wip.bin(
6352 .xor,
6353 shl,
6354 try o.builder.intValue(containing_int_ty, -1),
6355 "",
6356 );
6357 };
6358
6359 const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
6360 const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
6361 const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
6362 const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
63636296
6364 assert(ordering == .none);6297 if (llvm_memory_ty != llvm_value_ty) {
6365 _ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);6298 assert(load_ty.isAbiInt(zcu));
6366 return;6299 // `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6367 }6300 // handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6368 if (!isByRef(elem_ty, zcu)) {6301 // semantics. However:
6369 _ = try self.wip.storeAtomic(6302 //
6370 access_kind,6303 // * LLVM's lowering for these integer types generally leads to poor codegen, as integers
6371 elem,6304 // are only extended to the next byte, instead of to the next "natural" integer type.
6372 ptr,6305 //
6373 self.sync_scope,6306 // * Clang never emits loads or stores of these types, so LLVM's support for them is rather
6374 ordering,6307 // flaky---we have encountered several LLVM bugs caused by incorrect handling of them.
6375 ptr_alignment,6308 //
6376 );6309 // Therefore, we handle these memory accesses specially: in this case we will actually load
6377 return;6310 // the next-largest "natural" integer type and then truncate to `load_ty`.
6311 const loaded = try fg.wip.load(access_kind, llvm_memory_ty, ptr, llvm_ptr_align, "");
6312 // For packed structs, current Zig semantics don't really allow us to make the padding bits
6313 // well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6314 // implemented, but until then, do a normal trunc for packed types.
6315 return fg.wip.cast(switch (load_ty.zigTypeTag(zcu)) {
6316 .@"struct", .@"union" => .trunc,
6317 else => switch (load_ty.intInfo(zcu).signedness) {
6318 .unsigned => .@"trunc nuw",
6319 .signed => .@"trunc nsw",
6320 },
6321 }, loaded, llvm_value_ty, "");
6378 }6322 }
6379 assert(ordering == .none);6323
6380 _ = try self.wip.callMemCpy(6324 // `load_ty` is a simple by-val type which requires no special handling.
6381 ptr,6325 return fg.wip.load(access_kind, llvm_value_ty, ptr, llvm_ptr_align, "");
6382 ptr_alignment,
6383 elem,
6384 elem_ty.abiAlignment(zcu).toLlvm(),
6385 try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)),
6386 access_kind,
6387 self.disable_intrinsics,
6388 );
6389}6326}
63906327
6391/// Non-atomic, non-volatile, non-packed store.6328/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
6329///
6330/// `ptr` has alignment `ptr_align`, or `elem_ty.abiAlignment(zcu)` if `ptr_align` is `.none`.
6331///
6332/// If `elem_ty` is a by-ref type, then `elem` is itself a pointer, and a memcpy is emitted.
6392fn store(6333fn store(
6393 fg: *FuncGen,6334 fg: *FuncGen,
6394 ptr: Builder.Value,6335 ptr: Builder.Value,
6395 ptr_align: InternPool.Alignment,6336 ptr_align: InternPool.Alignment,
6396 elem: Builder.Value,6337 elem: Builder.Value,
6397 elem_ty: Type,6338 elem_ty: Type,
6339 access_kind: Builder.MemoryAccessKind,
6398) Allocator.Error!void {6340) Allocator.Error!void {
6399 const o = fg.object;6341 const o = fg.object;
6400 const zcu = o.zcu;6342 const zcu = o.zcu;
6343
6344 const abi_align = elem_ty.abiAlignment(zcu);
6345 const abi_size = elem_ty.abiSize(zcu);
6346
6401 const llvm_ptr_align = switch (ptr_align) {6347 const llvm_ptr_align = switch (ptr_align) {
6402 .none => elem_ty.abiAlignment(zcu).toLlvm(),6348 .none => abi_align.toLlvm(),
6403 else => ptr_align.toLlvm(),6349 else => ptr_align.toLlvm(),
6404 };6350 };
6351
6405 if (isByRef(elem_ty, zcu)) {6352 if (isByRef(elem_ty, zcu)) {
6353 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6406 _ = try fg.wip.callMemCpy(6354 _ = try fg.wip.callMemCpy(
6407 ptr,6355 ptr,
6408 llvm_ptr_align,6356 llvm_ptr_align,
6409 elem,6357 elem,
6410 elem_ty.abiAlignment(zcu).toLlvm(),6358 abi_align.toLlvm(),
6411 try o.builder.intValue(6359 try o.builder.intValue(llvm_usize_ty, abi_size),
6412 try o.lowerType(.usize),6360 access_kind,
6413 elem_ty.abiSize(zcu),
6414 ),
6415 .normal,
6416 fg.disable_intrinsics,6361 fg.disable_intrinsics,
6417 );6362 );
6418 } else {6363 return;
6364 }
6365
6366 assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty, .by_value));
6367
6368 const llvm_memory_ty = try o.lowerType(elem_ty, .in_memory);
6369 const llvm_value_ty = try o.lowerType(elem_ty, .by_value);
6370
6371 if (llvm_memory_ty != llvm_value_ty) {
6372 assert(elem_ty.isAbiInt(zcu));
6373 // `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6374 // the corresponding comment in `FuncGen.load` for more details.
6375 const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
6376 .unsigned => .zext,
6377 .signed => .sext,
6378 }, elem, llvm_memory_ty, "");
6419 _ = try fg.wip.storeAtomic(6379 _ = try fg.wip.storeAtomic(
6420 .normal,6380 access_kind,
6421 elem,6381 extended,
6422 ptr,6382 ptr,
6423 fg.sync_scope,6383 fg.sync_scope,
6424 .none,6384 .none,
6425 llvm_ptr_align,6385 llvm_ptr_align,
6426 );6386 );
6387 return;
6427 }6388 }
6389
6390 // `elem_ty` is a simple by-val type which requires no special handling.
6391 _ = try fg.wip.storeAtomic(
6392 access_kind,
6393 elem,
6394 ptr,
6395 fg.sync_scope,
6396 .none,
6397 llvm_ptr_align,
6398 );
6428}6399}
64296400
6430fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {6401fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6431 const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;6402 const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6432 const o = fg.object;6403 const o = fg.object;
6433 const usize_ty = try o.lowerType(.usize);6404 const usize_ty = try o.lowerType(.usize, .by_value);
6434 const zero = try o.builder.intValue(usize_ty, 0);6405 const zero = try o.builder.intValue(usize_ty, 0);
6435 const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);6406 const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6436 const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");6407 const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
...@@ -6452,19 +6423,19 @@ fn valgrindClientRequest(...@@ -6452,19 +6423,19 @@ fn valgrindClientRequest(
6452 const target = zcu.getTarget();6423 const target = zcu.getTarget();
6453 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;6424 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
64546425
6455 const llvm_usize = try o.lowerType(.usize);6426 const llvm_usize = try o.lowerType(.usize, .by_value);
6456 const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();6427 const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
64576428
6458 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);6429 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
6459 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {6430 const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
6460 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);6431 const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_align);
6461 fg.valgrind_client_request_array = array_ptr;6432 fg.valgrind_client_request_array = array_ptr;
6462 break :a array_ptr;6433 break :a array_ptr;
6463 } else fg.valgrind_client_request_array;6434 } else fg.valgrind_client_request_array;
6464 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };6435 const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
6465 for (array_elements, 0..) |elem, i| {6436 for (array_elements, 0..) |elem, i| {
6466 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));6437 const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
6467 _ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);6438 try fg.store(elem_ptr, .none, elem, .usize, .normal);
6468 }6439 }
64696440
6470 const arch_specific: struct {6441 const arch_specific: struct {
...@@ -6734,7 +6705,7 @@ const ParamTypeIterator = struct {...@@ -6734,7 +6705,7 @@ const ParamTypeIterator = struct {
6734 while (field_it.next()) |field_index| {6705 while (field_it.next()) |field_index| {
6735 const field_ty = ty.fieldType(field_index, zcu);6706 const field_ty = ty.fieldType(field_index, zcu);
6736 if (!field_ty.hasRuntimeBits(zcu)) continue;6707 if (!field_ty.hasRuntimeBits(zcu)) continue;
6737 it.types_buffer[it.types_len] = try it.object.lowerType(field_ty);6708 it.types_buffer[it.types_len] = try it.object.lowerType(field_ty, .by_value);
6738 it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);6709 it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);
6739 it.types_len += 1;6710 it.types_len += 1;
6740 }6711 }
...@@ -6751,7 +6722,7 @@ const ParamTypeIterator = struct {...@@ -6751,7 +6722,7 @@ const ParamTypeIterator = struct {
6751 it.llvm_index += 1;6722 it.llvm_index += 1;
6752 return .byval;6723 return .byval;
6753 } else {6724 } else {
6754 it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty)};6725 it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty, .by_value)};
6755 it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };6726 it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };
6756 it.types_len = 1;6727 it.types_len = 1;
6757 it.llvm_index += 1;6728 it.llvm_index += 1;
...@@ -6932,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para...@@ -6932,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para
6932 };6903 };
6933}6904}
69346905
6935fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool {6906pub const FnReturnStrat = union(enum) {
6936 if (isByRef(ty, zcu)) {6907 /// The function return type is OPV (zero-bit), so the LLVM function return type is `void`.
6937 return true;6908 void,
6938 } else if (target.cpu.arch.isX86() and6909 /// An sret parameter is used. The LLVM function return type is `void`.
6939 !target.cpu.has(.x86, .avx512f) and6910 sret,
6940 ty.totalVectorBits(zcu) >= 512)6911 /// The function's return type directly corresponds to the LLVM function return type.
6941 {6912 ///
6942 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns6913 /// The return type is by-val, i.e. `isByRef` returns `false`.
6943 // "512-bit vector arguments require 'avx512f' for AVX512"6914 by_val,
6944 return true;6915 /// The LLVM function returns the given `Builder.Type` by reinterpreting memory containing the
6945 } else {6916 /// actual return value. The actual return type may be by-val or by-ref.
6946 return false;6917 mem_cast: Builder.Type,
6947 }
6948}
69496918
6950pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool {6919 fn forceByVal(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
6951 const return_type = Type.fromInterned(fn_info.return_type);6920 if (!isByRef(ret_ty, o.zcu)) return .by_val;
6952 if (!return_type.hasRuntimeBits(zcu)) return false;6921 return .{ .mem_cast = try o.lowerType(ret_ty, .in_memory) };
6953
6954 return switch (fn_info.cc) {
6955 .auto => returnTypeByRef(zcu, target, return_type),
6956 .x86_64_sysv, .x86_64_x32 => firstParamSRetSystemV(return_type, zcu, target),
6957 .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
6958 .x86_sysv, .x86_win => isByRef(return_type, zcu),
6959 .x86_stdcall => !isScalar(zcu, return_type),
6960 .x86_fastcall => firstParamSRetX86Fastcall(zcu, return_type),
6961 .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
6962 .aarch64_aapcs,
6963 .aarch64_aapcs_darwin,
6964 .aarch64_aapcs_win,
6965 => aarch64_c_abi.classifyType(return_type, zcu) == .memory,
6966 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
6967 .memory, .i64_array => true,
6968 .i32_array => |size| size != 1,
6969 .byval => false,
6970 },
6971 .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
6972 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
6973 .memory, .i32_array => true,
6974 .byval => false,
6975 },
6976 else => false, // TODO: investigate other targets/callconvs
6977 };
6978}
6979
6980fn firstParamSRetX86Fastcall(zcu: *Zcu, ty: Type) bool {
6981 if (isScalar(zcu, ty)) {
6982 return false;
6983 }
6984 const tag = ty.zigTypeTag(zcu);
6985 if (tag == .@"struct" or tag == .@"union") {
6986 const size = ty.abiSize(zcu);
6987 if (size == 1 or size == 2 or size == 4 or size == 8) {
6988 return false;
6989 }
6990 }6922 }
6991 return true;6923};
6992}
6993
6994fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool {
6995 if (isScalar(zcu, ty)) return false;
6996 const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
6997 if (class[0] == .memory) return true;
6998 if (class[0] == .x87 and class[2] != .none) return true;
6999 return false;
7000}
7001
7002/// In order to support the C calling convention, some return types need to be lowered6924/// In order to support the C calling convention, some return types need to be lowered
7003/// completely differently in the function prototype to honor the C ABI, and then6925/// completely differently in the function prototype to honor the C ABI, and then
7004/// be effectively bitcasted to the actual return type.6926/// be effectively bitcasted to the actual return type.
7005pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {6927pub fn fnReturnStrat(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7006 const zcu = o.zcu;6928 const zcu = o.zcu;
7007 const return_type = Type.fromInterned(fn_info.return_type);6929 const ret_ty: Type = .fromInterned(fn_info.return_type);
7008 if (!return_type.hasRuntimeBits(zcu)) {6930 ret_ty.assertHasLayout(zcu);
7009 assert(!return_type.isError(zcu));6931 if (!ret_ty.hasRuntimeBits(zcu)) return .void;
7010 return .void;
7011 }
7012 const target = zcu.getTarget();
7013 switch (fn_info.cc) {6932 switch (fn_info.cc) {
7014 .@"inline" => unreachable,6933 .@"inline" => unreachable,
7015 .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type),6934 .auto => {
6935 if (isByRef(ret_ty, zcu)) return .sret;
6936
6937 const target = zcu.getTarget();
6938 if (target.cpu.arch.isX86() and
6939 !target.cpu.has(.x86, .avx512f) and
6940 ret_ty.totalVectorBits(zcu) >= 512)
6941 {
6942 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6943 // "512-bit vector arguments require 'avx512f' for AVX512"
6944 return .sret;
6945 }
6946
6947 return .by_val;
6948 },
7016 .x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),6949 .x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),
7017 .x86_64_win => return lowerWin64FnRetTy(o, fn_info),6950 .x86_64_win => return lowerWin64FnRetTy(o, fn_info),
7018 .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void,6951 .x86_stdcall => if (isScalar(zcu, ret_ty)) {
7019 .x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, return_type),6952 assert(!isByRef(ret_ty, zcu));
7020 .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type),6953 return .by_val;
7021 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {6954 } else return .sret,
7022 .memory => return .void,6955 .x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, ret_ty),
7023 .float_array => return o.lowerType(return_type),6956 .x86_sysv, .x86_win => return if (isByRef(ret_ty, zcu)) .sret else .by_val,
7024 .byval => return o.lowerType(return_type),6957 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(ret_ty, zcu)) {
7025 .integer => return .i64,6958 .memory => return .sret,
7026 .double_integer => return o.builder.arrayType(2, .i64),6959 .float_array, .byval => return .forceByVal(o, ret_ty),
6960 .integer => return .{ .mem_cast = .i64 },
6961 .double_integer => return .{ .mem_cast = try o.builder.arrayType(2, .i64) },
7027 },6962 },
7028 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {6963 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(ret_ty, zcu, .ret)) {
7029 .memory, .i64_array => return .void,6964 .memory, .i64_array => return .sret,
7030 .i32_array => |len| return if (len == 1) .i32 else .void,6965 .i32_array => |len| return if (len == 1) .{ .mem_cast = .i32 } else .sret,
7031 .byval => return o.lowerType(return_type),6966 .byval => return .forceByVal(o, ret_ty),
7032 },6967 },
7033 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {6968 .mips_o32 => switch (mips_c_abi.classifyType(ret_ty, zcu, .ret)) {
7034 .memory, .i32_array => return .void,6969 .memory, .i32_array => return .sret,
7035 .byval => return o.lowerType(return_type),6970 .byval => return .forceByVal(o, ret_ty),
7036 },6971 },
7037 .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {6972 .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(ret_ty, zcu)) {
7038 .memory => return .void,6973 .memory => return .sret,
7039 .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),6974 .integer => return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7040 .double_integer => {6975 .double_integer => {
7041 const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {6976 const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7042 .riscv64, .riscv64be => .i64,6977 .riscv64, .riscv64be => .i64,
7043 .riscv32, .riscv32be => .i32,6978 .riscv32, .riscv32be => .i32,
7044 else => unreachable,6979 else => unreachable,
7045 };6980 };
7046 return o.builder.structType(.normal, &.{ integer, integer });6981 return .{ .mem_cast = try o.builder.structType(.normal, &.{ integer, integer }) };
7047 },6982 },
7048 .byval => return o.lowerType(return_type),6983 .byval => return .forceByVal(o, ret_ty),
7049 .fields => {6984 .fields => {
7050 var types_len: usize = 0;6985 var types_len: usize = 0;
7051 var types: [8]Builder.Type = undefined;6986 var types: [8]Builder.Type = undefined;
7052 for (0..return_type.structFieldCount(zcu)) |field_index| {6987 for (0..ret_ty.structFieldCount(zcu)) |field_index| {
7053 const field_ty = return_type.fieldType(field_index, zcu);6988 const field_ty = ret_ty.fieldType(field_index, zcu);
7054 if (!field_ty.hasRuntimeBits(zcu)) continue;6989 if (!field_ty.hasRuntimeBits(zcu)) continue;
7055 types[types_len] = try o.lowerType(field_ty);6990 types[types_len] = try o.lowerType(field_ty, .by_value);
7056 types_len += 1;6991 types_len += 1;
7057 }6992 }
7058 return o.builder.structType(.normal, types[0..types_len]);6993 return .{ .mem_cast = try o.builder.structType(.normal, types[0..types_len]) };
7059 },6994 },
7060 },6995 },
7061 .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {6996 .wasm_mvp => switch (wasm_c_abi.classifyType(ret_ty, zcu)) {
7062 .direct => |scalar_ty| return o.lowerType(scalar_ty),6997 .direct => |scalar_ty| if (scalar_ty.toIntern() == ret_ty.toIntern()) {
7063 .indirect => return .void,6998 assert(!isByRef(ret_ty, zcu));
6999 return .by_val;
7000 } else {
7001 return .{ .mem_cast = try o.lowerType(scalar_ty, .by_value) };
7002 },
7003 .indirect => return .sret,
7064 },7004 },
7065 // TODO investigate other callconvs7005 // TODO investigate other callconvs
7066 else => return o.lowerType(return_type),7006 else => return .forceByVal(o, ret_ty),
7067 }7007 }
7068}7008}
70697009
7070fn lowerX86FastcallFnRetTy(o: *Object, zcu: *Zcu, ty: Type) Allocator.Error!Builder.Type {7010fn lowerX86FastcallFnRetTy(o: *Object, zcu: *Zcu, ty: Type) Allocator.Error!FnReturnStrat {
7071 if (isScalar(zcu, ty)) {7011 if (isScalar(zcu, ty)) {
7072 return o.lowerType(ty);7012 assert(!isByRef(ty, zcu));
7013 return .by_val;
7073 }7014 }
7074 const tag = ty.zigTypeTag(zcu);7015 const tag = ty.zigTypeTag(zcu);
7075 if (tag == .@"struct" or tag == .@"union") {7016 if (tag == .@"struct" or tag == .@"union") {
7076 const size = ty.abiSize(zcu);7017 const size = ty.abiSize(zcu);
7077 if (size == 1 or size == 2 or size == 4 or size == 8) {7018 if (size == 1 or size == 2 or size == 4 or size == 8) {
7078 return o.builder.intType(@intCast(size * 8));7019 return .{ .mem_cast = try o.builder.intType(@intCast(size * 8)) };
7079 }7020 }
7080 }7021 }
7081 return .void;7022 return .sret;
7082}7023}
70837024
7084fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {7025fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7085 const zcu = o.zcu;7026 const zcu = o.zcu;
7086 const return_type = Type.fromInterned(fn_info.return_type);7027 const ret_ty = Type.fromInterned(fn_info.return_type);
7087 switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {7028 switch (x86_64_abi.classifyWindows(ret_ty, zcu, zcu.getTarget(), .ret)) {
7088 .integer => {7029 .integer => if (isScalar(zcu, ret_ty)) {
7089 if (isScalar(zcu, return_type)) {7030 assert(!isByRef(ret_ty, zcu));
7090 return o.lowerType(return_type);7031 return .by_val;
7091 } else {7032 } else {
7092 return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));7033 return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) };
7093 }
7094 },7034 },
7035 .win_i128 => return .{ .mem_cast = try o.builder.vectorType(.normal, 2, .i64) },
7036 .memory => return .sret,
7037
7095 .sse,7038 .sse,
7096 .bool_vector_mask,7039 .bool_vector_mask,
7097 .integer_per_element,7040 .integer_per_element,
...@@ -7100,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err...@@ -7100,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7100 .sse_per_xword,7043 .sse_per_xword,
7101 .sse_per_yword,7044 .sse_per_yword,
7102 .sse_per_zword,7045 .sse_per_zword,
7103 => return o.lowerType(return_type),7046 => {
7047 assert(!isByRef(ret_ty, zcu));
7048 return .by_val;
7049 },
7104 .sseup,7050 .sseup,
7105 .x87,7051 .x87,
7106 .x87up,7052 .x87up,
...@@ -7108,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err...@@ -7108,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7108 .float,7054 .float,
7109 .float_combine,7055 .float_combine,
7110 => unreachable,7056 => unreachable,
7111 .win_i128 => return o.builder.vectorType(.normal, 2, .i64),
7112 .memory => return .void,
7113 }7057 }
7114}7058}
71157059
7116fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {7060fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7117 const zcu = o.zcu;7061 const zcu = o.zcu;
7118 const ip = &zcu.intern_pool;7062 const ip = &zcu.intern_pool;
7119 const return_type = Type.fromInterned(fn_info.return_type);7063 const ret_ty = Type.fromInterned(fn_info.return_type);
7120 return_type.assertHasLayout(zcu);7064 if (isScalar(zcu, ret_ty)) {
7121 if (isScalar(zcu, return_type)) {7065 assert(!isByRef(ret_ty, zcu));
7122 return o.lowerType(return_type);7066 return .by_val;
7123 }7067 }
7124 const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);7068 const classes = x86_64_abi.classifySystemV(ret_ty, zcu, zcu.getTarget(), .ret);
7125 var types_index: u32 = 0;7069 var types_index: u32 = 0;
7126 var types_buffer: [8]Builder.Type = undefined;7070 var types_buffer: [8]Builder.Type = undefined;
7127 for (classes) |class| {7071 for (classes) |class| {
...@@ -7151,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7151,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7151 types_index += 1;7095 types_index += 1;
7152 },7096 },
7153 .x87 => {7097 .x87 => {
7154 if (types_index != 0 or classes[2] != .none) return .void;7098 if (types_index != 0 or classes[2] != .none) return .sret;
7155 types_buffer[types_index] = .x86_fp80;7099 types_buffer[types_index] = .x86_fp80;
7156 types_index += 1;7100 types_index += 1;
7157 },7101 },
7158 .x87up => continue,7102 .x87up => continue,
7159 .none => break,7103 .none => break,
7160 .memory => return .void,7104 .memory => return .sret,
7161 .win_i128 => unreachable, // windows only7105 .win_i128 => unreachable, // windows only
7162 .bool_vector_mask,7106 .bool_vector_mask,
7163 .integer_per_element,7107 .integer_per_element,
...@@ -7172,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7172,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7172 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});7116 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7173 if (first_non_integer == null or classes[first_non_integer.?] == .none) {7117 if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7174 assert(first_non_integer orelse classes.len == types_index);7118 assert(first_non_integer orelse classes.len == types_index);
7175 switch (ip.indexToKey(return_type.toIntern())) {7119 switch (ip.indexToKey(ret_ty.toIntern())) {
7176 .struct_type => {7120 .struct_type => {
7177 const size = return_type.abiSize(zcu);7121 const size = ret_ty.abiSize(zcu);
7178 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);7122 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7179 if (size % 8 > 0) {7123 if (size % 8 > 0) {
7180 types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));7124 types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
...@@ -7182,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7182,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7182 },7126 },
7183 else => {},7127 else => {},
7184 }7128 }
7185 if (types_index == 1) return types_buffer[0];7129 if (types_index == 1) return .{ .mem_cast = types_buffer[0] };
7186 }7130 }
7187 return o.builder.structType(.normal, types_buffer[0..types_index]);7131 return .{ .mem_cast = try o.builder.structType(.normal, types_buffer[0..types_index]) };
7188}7132}
71897133
7190/// This function deliberately does not handle `_BitInt` because it typically7134/// This function deliberately does not handle `_BitInt` because it typically
...@@ -7283,33 +7227,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {...@@ -7283,33 +7227,6 @@ fn isScalar(zcu: *Zcu, ty: Type) bool {
7283 };7227 };
7284}7228}
72857229
7286pub fn buildAllocaInner(
7287 wip: *Builder.WipFunction,
7288 llvm_ty: Builder.Type,
7289 alignment: Builder.Alignment,
7290 target: *const std.Target,
7291) Allocator.Error!Builder.Value {
7292 const address_space = llvmAllocaAddressSpace(target);
7293
7294 const alloca = blk: {
7295 const prev_cursor = wip.cursor;
7296 const prev_debug_location = wip.debug_location;
7297 defer {
7298 wip.cursor = prev_cursor;
7299 if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
7300 wip.debug_location = prev_debug_location;
7301 }
7302
7303 wip.cursor = .{ .block = .entry };
7304 wip.debug_location = .no_location;
7305 break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
7306 };
7307
7308 // The pointer returned from this function should have the generic address space,
7309 // if this isn't the case then cast it to the generic address space.
7310 return wip.conv(.unneeded, alloca, .ptr, "");
7311}
7312
7313/// This is the one source of truth for whether a type is passed around as an LLVM pointer,7230/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
7314/// or as an LLVM value.7231/// or as an LLVM value.
7315pub fn isByRef(ty: Type, zcu: *const Zcu) bool {7232pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
...@@ -7351,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {...@@ -7351,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7351 },7268 },
7352 .@"union" => switch (ty.containerLayout(zcu)) {7269 .@"union" => switch (ty.containerLayout(zcu)) {
7353 .@"packed" => false,7270 .@"packed" => false,
7354 else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),7271 else => ty.hasRuntimeBits(zcu),
7355 },7272 },
7356 };7273 };
7357}7274}
...@@ -7380,7 +7297,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E...@@ -7380,7 +7297,11 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7380}7297}
73817298
7382fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {7299fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7383 return fg.object.ptraddConst(&fg.wip, ptr, offset);7300 if (offset == 0) return ptr;
7301 const o = fg.object;
7302 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
7303 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7304 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7384}7305}
7385fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {7306fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
7386 if (scale == 0) return ptr;7307 if (scale == 0) return ptr;
src/codegen/mips/abi.zig+4-5
...@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
18 const max_direct_size = target.ptrBitWidth() * 2;18 const max_direct_size = target.ptrBitWidth() * 2;
19 switch (ty.zigTypeTag(zcu)) {19 switch (ty.zigTypeTag(zcu)) {
20 .@"struct" => {20 .@"struct" => {
21 const bit_size = ty.bitSize(zcu);
22 if (ty.containerLayout(zcu) == .@"packed") {21 if (ty.containerLayout(zcu) == .@"packed") {
23 if (bit_size > max_direct_size) return .memory;22 if (ty.bitSize(zcu) > max_direct_size) return .memory;
24 return .byval;23 return .byval;
25 }24 }
25 const bit_size = ty.abiSize(zcu) * 8;
26 if (bit_size > max_direct_size) return .memory;26 if (bit_size > max_direct_size) return .memory;
27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29 return .{ .i32_array = count };29 return .{ .i32_array = count };
30 },30 },
31 .@"union" => {31 .@"union" => {
32 const bit_size = ty.bitSize(zcu);
33 if (ty.containerLayout(zcu) == .@"packed") {32 if (ty.containerLayout(zcu) == .@"packed") {
34 if (bit_size > max_direct_size) return .memory;33 if (ty.bitSize(zcu) > max_direct_size) return .memory;
35 return .byval;34 return .byval;
36 }35 }
36 const bit_size = ty.abiSize(zcu) * 8;
37 if (bit_size > max_direct_size) return .memory;37 if (bit_size > max_direct_size) return .memory;
38
39 return .byval;38 return .byval;
40 },39 },
41 .bool => return .byval,40 .bool => return .byval,
src/codegen/riscv64/CodeGen.zig+12-7
...@@ -51,7 +51,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};...@@ -51,7 +51,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};
5151
52pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {52pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
53 return comptime &.initMany(&.{53 return comptime &.initMany(&.{
54 .expand_intcast_safe,54 .expand_int_cast_safe,
55 .expand_int_from_float_safe,55 .expand_int_from_float_safe,
56 .expand_int_from_float_optimized_safe,56 .expand_int_from_float_optimized_safe,
57 .expand_add_safe,57 .expand_add_safe,
...@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1453 .add_safe,1453 .add_safe,
1454 .sub_safe,1454 .sub_safe,
1455 .mul_safe,1455 .mul_safe,
1456 .intcast_safe,1456 .int_cast_safe,
1457 .int_from_float_safe,1457 .int_from_float_safe,
1458 .int_from_float_optimized_safe,1458 .int_from_float_optimized_safe,
1459 => return func.fail("TODO implement safety_checked_instructions", .{}),1459 => return func.fail("TODO implement safety_checked_instructions", .{}),
...@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1479 .ret_ptr => try func.airRetPtr(inst),1479 .ret_ptr => try func.airRetPtr(inst),
1480 .arg => try func.airArg(inst),1480 .arg => try func.airArg(inst),
1481 .assembly => try func.airAsm(inst),1481 .assembly => try func.airAsm(inst),
1482 .bitcast => try func.airBitCast(inst),1482 .bit_cast => try func.airBitCast(inst),
1483 .ptr_cast => try func.airBitCast(inst),
1484 .ptr_from_int => try func.airBitCast(inst),
1485 .int_from_ptr => try func.airBitCast(inst),
1486 .error_cast => try func.airBitCast(inst),
1487 .error_from_int => try func.airBitCast(inst),
1488 .int_from_error => try func.airBitCast(inst),
1489 .union_from_enum => try func.airBitCast(inst),
1483 .block => try func.airBlock(inst),1490 .block => try func.airBlock(inst),
1484 .br => try func.airBr(inst),1491 .br => try func.airBr(inst),
1485 .repeat => try func.airRepeat(inst),1492 .repeat => try func.airRepeat(inst),
...@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1493 .dbg_empty_stmt => func.finishAirBookkeeping(),1500 .dbg_empty_stmt => func.finishAirBookkeeping(),
1494 .fptrunc => try func.airFptrunc(inst),1501 .fptrunc => try func.airFptrunc(inst),
1495 .fpext => try func.airFpext(inst),1502 .fpext => try func.airFpext(inst),
1496 .intcast => try func.airIntCast(inst),1503 .int_cast => try func.airIntCast(inst),
1497 .trunc => try func.airTrunc(inst),1504 .trunc => try func.airTrunc(inst),
1498 .is_non_null => try func.airIsNonNull(inst),1505 .is_non_null => try func.airIsNonNull(inst),
1499 .is_non_null_ptr => try func.airIsNonNullPtr(inst),1506 .is_non_null_ptr => try func.airIsNonNullPtr(inst),
...@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {...@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3953 const elem_ptr_ty = func.typeOfIndex(inst);3960 const elem_ptr_ty = func.typeOfIndex(inst);
3954 const base_ptr_ty = func.typeOf(extra.lhs);3961 const base_ptr_ty = func.typeOf(extra.lhs);
39553962
3956 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {3963 assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
3957 @panic("audit");
3958 }
39593964
3960 const base_ptr_mcv = try func.resolveInst(extra.lhs);3965 const base_ptr_mcv = try func.resolveInst(extra.lhs);
3961 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {3966 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
src/codegen/riscv64/abi.zig+5-6
...@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
16 const max_byval_size = target.ptrBitWidth() * 2;16 const max_byval_size = target.ptrBitWidth() * 2;
17 switch (ty.zigTypeTag(zcu)) {17 switch (ty.zigTypeTag(zcu)) {
18 .@"struct" => {18 .@"struct" => {
19 const bit_size = ty.bitSize(zcu);
20 if (ty.containerLayout(zcu) == .@"packed") {19 if (ty.containerLayout(zcu) == .@"packed") {
21 if (bit_size > max_byval_size) return .memory;20 if (ty.bitSize(zcu) > max_byval_size) return .memory;
22 return .byval;21 return .byval;
23 }22 }
2423
...@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
40 }39 }
4140
42 // TODO this doesn't exactly match what clang produces but its better than nothing41 // TODO this doesn't exactly match what clang produces but its better than nothing
42 const bit_size = ty.abiSize(zcu) * 8;
43 if (bit_size > max_byval_size) return .memory;43 if (bit_size > max_byval_size) return .memory;
44 if (bit_size > max_byval_size / 2) return .double_integer;44 if (bit_size > max_byval_size / 2) return .double_integer;
45 return .integer;45 return .integer;
46 },46 },
47 .@"union" => {47 .@"union" => {
48 const bit_size = ty.bitSize(zcu);
49 if (ty.containerLayout(zcu) == .@"packed") {48 if (ty.containerLayout(zcu) == .@"packed") {
50 if (bit_size > max_byval_size) return .memory;49 if (ty.bitSize(zcu) > max_byval_size) return .memory;
51 return .byval;50 return .byval;
52 }51 }
53 // TODO this doesn't exactly match what clang produces but its better than nothing52 // TODO this doesn't exactly match what clang produces but its better than nothing
53 const bit_size = ty.abiSize(zcu) * 8;
54 if (bit_size > max_byval_size) return .memory;54 if (bit_size > max_byval_size) return .memory;
55 if (bit_size > max_byval_size / 2) return .double_integer;55 if (bit_size > max_byval_size / 2) return .double_integer;
56 return .integer;56 return .integer;
...@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {...@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {
153 },153 },
154 .error_union => {154 .error_union => {
155 const payload_ty = ty.errorUnionPayload(zcu);155 const payload_ty = ty.errorUnionPayload(zcu);
156 const payload_bits = payload_ty.bitSize(zcu);
157156
158 // the error union itself157 // the error union itself
159 result[0] = .integer;158 result[0] = .integer;
160159
161 // anyerror!void can fit into one register160 // anyerror!void can fit into one register
162 if (payload_bits == 0) return result;161 if (!payload_ty.hasRuntimeBits(zcu)) return result;
163162
164 return memory_class;163 return memory_class;
165 },164 },
src/codegen/sparc64/CodeGen.zig+11-4
...@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
538 .ret_ptr => try self.airRetPtr(inst),538 .ret_ptr => try self.airRetPtr(inst),
539 .arg => try self.airArg(inst),539 .arg => try self.airArg(inst),
540 .assembly => try self.airAsm(inst),540 .assembly => try self.airAsm(inst),
541 .bitcast => try self.airBitCast(inst),541 .bit_cast => try self.airBitCast(inst),
542 .ptr_cast => try self.airBitCast(inst),
543 .ptr_from_int => try self.airBitCast(inst),
544 .int_from_ptr => try self.airBitCast(inst),
545 .error_cast => try self.airBitCast(inst),
546 .error_from_int => try self.airBitCast(inst),
547 .int_from_error => try self.airBitCast(inst),
548 .union_from_enum => try self.airBitCast(inst),
542 .block => try self.airBlock(inst),549 .block => try self.airBlock(inst),
543 .br => try self.airBr(inst),550 .br => try self.airBr(inst),
544 .repeat => return self.fail("TODO implement `repeat`", .{}),551 .repeat => return self.fail("TODO implement `repeat`", .{}),
...@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
550 .cond_br => try self.airCondBr(inst),557 .cond_br => try self.airCondBr(inst),
551 .fptrunc => @panic("TODO try self.airFptrunc(inst)"),558 .fptrunc => @panic("TODO try self.airFptrunc(inst)"),
552 .fpext => @panic("TODO try self.airFpext(inst)"),559 .fpext => @panic("TODO try self.airFpext(inst)"),
553 .intcast => try self.airIntCast(inst),560 .int_cast => try self.airIntCast(inst),
554 .trunc => try self.airTrunc(inst),561 .trunc => try self.airTrunc(inst),
555 .is_non_null => try self.airIsNonNull(inst),562 .is_non_null => try self.airIsNonNull(inst),
556 .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),563 .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
...@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
689 .add_safe,696 .add_safe,
690 .sub_safe,697 .sub_safe,
691 .mul_safe,698 .mul_safe,
692 .intcast_safe,699 .int_cast_safe,
693 .int_from_float_safe,700 .int_from_float_safe,
694 .int_from_float_optimized_safe,701 .int_from_float_optimized_safe,
695 => @panic("TODO implement safety_checked_instructions"),702 => @panic("TODO implement safety_checked_instructions"),
...@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1659 const info_a = operand_ty.intInfo(zcu);1666 const info_a = operand_ty.intInfo(zcu);
1660 const info_b = self.typeOfIndex(inst).intInfo(zcu);1667 const info_b = self.typeOfIndex(inst).intInfo(zcu);
1661 if (info_a.signedness != info_b.signedness)1668 if (info_a.signedness != info_b.signedness)
1662 return self.fail("TODO gen intcast sign safety in semantic analysis", .{});1669 return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
16631670
1664 if (info_a.bits == info_b.bits)1671 if (info_a.bits == info_b.bits)
1665 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });1672 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
src/codegen/spirv/CodeGen.zig+67-28
...@@ -34,7 +34,7 @@ const CodeGen = @This();...@@ -34,7 +34,7 @@ const CodeGen = @This();
3434
35pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {35pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
36 return comptime &.initMany(&.{36 return comptime &.initMany(&.{
37 .expand_intcast_safe,37 .expand_int_cast_safe,
38 .expand_int_from_float_safe,38 .expand_int_from_float_safe,
39 .expand_int_from_float_optimized_safe,39 .expand_int_from_float_optimized_safe,
40 .expand_add_safe,40 .expand_add_safe,
...@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {...@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1848 .pointer => {1848 .pointer => {
1849 const ptr_info = ty.ptrInfo(zcu);1849 const ptr_info = ty.ptrInfo(zcu);
18501850
1851 const child_ty: Type = .fromInterned(ptr_info.child);1851 const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
1852 0 => .fromInterned(ptr_info.child),
1853 else => switch (ptr_info.flags.vector_index) {
1854 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
1855 .none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
1856 else => try pt.vectorType(.{
1857 .child = ptr_info.child,
1858 .len = ptr_info.packed_offset.host_size,
1859 }),
1860 },
1861 };
1852 const child_ty_id = try cg.resolveType(child_ty, .indirect);1862 const child_ty_id = try cg.resolveType(child_ty, .indirect);
1853 const storage_class = cg.module.storageClass(ptr_info.flags.address_space);1863 const storage_class = cg.module.storageClass(ptr_info.flags.address_space);
1854 const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);1864 const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);
...@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {...@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
3847 .min => try cg.airMinMax(inst, .min),3857 .min => try cg.airMinMax(inst, .min),
3848 .max => try cg.airMinMax(inst, .max),3858 .max => try cg.airMinMax(inst, .max),
38493859
3850 .bitcast => try cg.airBitCast(inst),3860 .bit_cast => try cg.airBitCast(inst),
3851 .intcast, .trunc => try cg.airIntCast(inst),3861 .ptr_cast => try cg.airBitCast(inst),
3852 .float_from_int => try cg.airFloatFromInt(inst),3862 .ptr_from_int => try cg.airBitCast(inst),
3853 .int_from_float => try cg.airIntFromFloat(inst),3863 .int_from_ptr => try cg.airBitCast(inst),
3854 .fpext, .fptrunc => try cg.airFloatCast(inst),3864 .error_cast => try cg.airBitCast(inst),
3855 .not => try cg.airNot(inst),3865 .error_from_int => try cg.airBitCast(inst),
3866 .int_from_error => try cg.airBitCast(inst),
3867 .union_from_enum => try cg.airBitCast(inst),
3868 .int_cast, .trunc => try cg.airIntCast(inst),
3869 .float_from_int => try cg.airFloatFromInt(inst),
3870 .int_from_float => try cg.airIntFromFloat(inst),
3871 .fpext, .fptrunc => try cg.airFloatCast(inst),
3872 .not => try cg.airNot(inst),
38563873
3857 .array_to_slice => try cg.airArrayToSlice(inst),3874 .array_to_slice => try cg.airArrayToSlice(inst),
3858 .slice => try cg.airSlice(inst),3875 .slice => try cg.airSlice(inst),
...@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6913 const zcu = cg.module.zcu;6930 const zcu = cg.module.zcu;
6914 const pt = cg.pt;6931 const pt = cg.pt;
6915 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6932 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6916 const ptr_ty = cg.typeOf(ty_op.operand);6933
6917 const ptr_info = ptr_ty.ptrInfo(zcu);6934 const ptr_info = cg.typeOf(ty_op.operand).ptrInfo(zcu);
6935
6918 const elem_ty = cg.typeOfIndex(inst);6936 const elem_ty = cg.typeOfIndex(inst);
6919 const operand = try cg.resolve(ty_op.operand);6937 const operand_ptr_id = try cg.resolve(ty_op.operand);
6920 if (!ptr_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
69216938
6922 if (cg.virtual_allocas.get(operand)) |stored| return stored.?;6939 assert(ptr_info.child == elem_ty.toIntern());
6940
6941 if (cg.virtual_allocas.get(operand_ptr_id)) |stored| return stored.?;
69236942
6924 if (ptr_info.packed_offset.host_size != 0 and6943 if (ptr_info.packed_offset.host_size != 0 and
6925 ptr_info.flags.vector_index == .none)6944 ptr_info.flags.vector_index == .none)
...@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6927 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;6946 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6928 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));6947 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6929 const host_int_ty = try pt.intType(.unsigned, host_bits);6948 const host_int_ty = try pt.intType(.unsigned, host_bits);
6930 const host_val = try cg.load(host_int_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });6949 const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6931 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;6950 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6932 const field_int_ty = try pt.intType(signedness, elem_bit_size);6951 const field_int_ty = try pt.intType(signedness, elem_bit_size);
6933 const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {6952 const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
...@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6946 return try cg.bitCast(elem_ty, field_int_ty, result_id);6965 return try cg.bitCast(elem_ty, field_int_ty, result_id);
6947 }6966 }
69486967
6949 return try cg.load(elem_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });6968 const ptr_id = switch (ptr_info.flags.vector_index) {
6969 .none => operand_ptr_id,
6970 else => |index| ptr_id: {
6971 const elem_ptr_ty_id = try cg.module.ptrType(
6972 try cg.resolveType(elem_ty, .indirect),
6973 cg.module.storageClass(ptr_info.flags.address_space),
6974 );
6975 break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
6976 },
6977 };
6978 return try cg.load(elem_ty, ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6950}6979}
69516980
6952fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {6981fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6953 const zcu = cg.module.zcu;6982 const zcu = cg.module.zcu;
6954 const pt = cg.pt;6983 const pt = cg.pt;
6955 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6984 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6956 const ptr_ty = cg.typeOf(bin_op.lhs);6985 const ptr_info = cg.typeOf(bin_op.lhs).ptrInfo(zcu);
6957 const ptr_info = ptr_ty.ptrInfo(zcu);6986 const elem_ty: Type = .fromInterned(ptr_info.child);
6958 const elem_ty = ptr_ty.childType(zcu);6987 const operand_ptr_id = try cg.resolve(bin_op.lhs);
6959 const ptr = try cg.resolve(bin_op.lhs);6988 const value_id = try cg.resolve(bin_op.rhs);
6960 const value = try cg.resolve(bin_op.rhs);
69616989
6962 if (cg.virtual_allocas.getPtr(ptr)) |slot| {6990 if (cg.virtual_allocas.getPtr(operand_ptr_id)) |slot| {
6963 slot.* = value;6991 slot.* = value_id;
6964 return;6992 return;
6965 }6993 }
69666994
...@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6969 {6997 {
6970 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;6998 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6971 const host_int_ty = try pt.intType(.unsigned, host_bits);6999 const host_int_ty = try pt.intType(.unsigned, host_bits);
6972 const host_val = try cg.load(host_int_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7000 const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6973 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));7001 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6974 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;7002 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6975 const field_int_ty = try pt.intType(signedness, elem_bit_size);7003 const field_int_ty = try pt.intType(signedness, elem_bit_size);
69767004
6977 var value_as_int: Id = undefined;7005 var value_as_int: Id = undefined;
6978 if (elem_ty.ip_index == .bool_type) {7006 if (elem_ty.ip_index == .bool_type) {
6979 value_as_int = try cg.convertToIndirect(.bool, value);7007 value_as_int = try cg.convertToIndirect(.bool, value_id);
6980 value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);7008 value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
6981 } else if (elem_ty.isInt(zcu)) {7009 } else if (elem_ty.isInt(zcu)) {
6982 value_as_int = value;7010 value_as_int = value_id;
6983 } else {7011 } else {
6984 value_as_int = try cg.bitCast(field_int_ty, elem_ty, value);7012 value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
6985 }7013 }
69867014
6987 const extended = blk: {7015 const extended = blk: {
...@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7002 const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);7030 const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7003 const combined_id = try combined.materialize(cg);7031 const combined_id = try combined.materialize(cg);
70047032
7005 try cg.store(host_int_ty, ptr, combined_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7033 try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7006 return;7034 return;
7007 }7035 }
70087036
7009 try cg.store(elem_ty, ptr, value, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7037 const ptr_id = switch (ptr_info.flags.vector_index) {
7038 .none => operand_ptr_id,
7039 else => |index| ptr_id: {
7040 const elem_ptr_ty_id = try cg.module.ptrType(
7041 try cg.resolveType(elem_ty, .indirect),
7042 cg.module.storageClass(ptr_info.flags.address_space),
7043 );
7044 break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
7045 },
7046 };
7047
7048 try cg.store(elem_ty, ptr_id, value_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7010}7049}
70117050
7012fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {7051fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {
src/codegen/wasm/CodeGen.zig+97-25
...@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;...@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
3232
33pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {33pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
34 return comptime &.initMany(&.{34 return comptime &.initMany(&.{
35 .expand_intcast_safe,35 .expand_int_cast_safe,
36 .expand_int_from_float_safe,36 .expand_int_from_float_safe,
37 .expand_int_from_float_optimized_safe,37 .expand_int_from_float_optimized_safe,
38 .expand_add_safe,38 .expand_add_safe,
...@@ -83,7 +83,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -83,7 +83,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
83 .scalarize_shl_sat,83 .scalarize_shl_sat,
84 .scalarize_xor,84 .scalarize_xor,
85 .scalarize_not,85 .scalarize_not,
86 .scalarize_bitcast,
87 .scalarize_clz,86 .scalarize_clz,
88 .scalarize_ctz,87 .scalarize_ctz,
89 .scalarize_popcount,88 .scalarize_popcount,
...@@ -109,7 +108,10 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -109,7 +108,10 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
109 .scalarize_cmp_vector_optimized,108 .scalarize_cmp_vector_optimized,
110 .scalarize_fptrunc,109 .scalarize_fptrunc,
111 .scalarize_fpext,110 .scalarize_fpext,
112 .scalarize_intcast,111 .scalarize_int_cast,
112 .scalarize_ptr_cast,
113 .scalarize_ptr_from_int,
114 .scalarize_int_from_ptr,
113 .scalarize_trunc,115 .scalarize_trunc,
114 .scalarize_int_from_float,116 .scalarize_int_from_float,
115 .scalarize_int_from_float_optimized,117 .scalarize_int_from_float_optimized,
...@@ -120,6 +122,8 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -120,6 +122,8 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
120 .scalarize_shuffle_two,122 .scalarize_shuffle_two,
121 .scalarize_select,123 .scalarize_select,
122 .scalarize_mul_add,124 .scalarize_mul_add,
125
126 .scalarize_bit_cast_padded_elems,
123 });127 });
124}128}
125129
...@@ -1550,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1550,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1550 try cg.finishAir(inst, result, &.{ty_op.operand});1554 try cg.finishAir(inst, result, &.{ty_op.operand});
1551 },1555 },
15521556
1553 .bitcast => cg.airBitcast(inst),1557 .ptr_cast => cg.airNopCast(inst),
1558 .error_cast => cg.airNopCast(inst),
1559 .error_from_int => cg.airNopCast(inst),
1560 .int_from_error => cg.airNopCast(inst),
1561 .ptr_from_int => cg.airNopCast(inst),
1562 .int_from_ptr => cg.airIntFromPtr(inst),
1563
1564 .bit_cast => cg.airBitcast(inst),
1565 .union_from_enum => cg.airBitcast(inst),
15541566
1555 .intcast => {1567 .int_cast => {
1556 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1568 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
15571569
1558 const dest_ty = ty_op.ty.toType();1570 const dest_ty = ty_op.ty.toType();
...@@ -1560,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1560,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1560 const src_ty = cg.typeOf(ty_op.operand);1572 const src_ty = cg.typeOf(ty_op.operand);
15611573
1562 if (dest_ty.zigTypeTag(zcu) == .vector) {1574 if (dest_ty.zigTypeTag(zcu) == .vector) {
1563 return cg.fail("TODO: implement AIR op: intcast for vectors", .{});1575 return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1564 }1576 }
15651577
1566 const src_int_ty: IntType = .fromType(cg, src_ty);1578 const src_int_ty: IntType = .fromType(cg, src_ty);
...@@ -1875,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1875,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1875 .add_safe,1887 .add_safe,
1876 .sub_safe,1888 .sub_safe,
1877 .mul_safe,1889 .mul_safe,
1878 .intcast_safe,1890 .int_cast_safe,
1879 .int_from_float_safe,1891 .int_from_float_safe,
1880 .int_from_float_optimized_safe,1892 .int_from_float_optimized_safe,
1881 => return cg.fail("TODO implement safety_checked_instructions", .{}),1893 => return cg.fail("TODO implement safety_checked_instructions", .{}),
...@@ -2099,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2099,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2099 const rhs = try cg.resolveInst(bin_op.rhs);2111 const rhs = try cg.resolveInst(bin_op.rhs);
2100 const ptr_ty = cg.typeOf(bin_op.lhs);2112 const ptr_ty = cg.typeOf(bin_op.lhs);
2101 const ptr_info = ptr_ty.ptrInfo(zcu);2113 const ptr_info = ptr_ty.ptrInfo(zcu);
2102 const ty = ptr_ty.childType(zcu);2114 const elem_ty = ptr_ty.childType(zcu);
21032115
2104 if (!safety and bin_op.rhs == .undef) {2116 if (!safety and bin_op.rhs == .undef) {
2105 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2117 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2106 }2118 }
21072119
2108 assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_store2120 const offset: u32 = switch (ptr_info.flags.vector_index) {
21092121 .none => offset: {
2110 try cg.store(lhs, rhs, ty, 0);2122 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
21112123 break :offset 0;
2124 },
2125 else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
2126 };
2127 try cg.store(lhs, rhs, elem_ty, offset);
2112 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2128 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2113}2129}
21142130
...@@ -2121,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2121,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2121 if (!ty.hasRuntimeBits(zcu)) return;2137 if (!ty.hasRuntimeBits(zcu)) return;
21222138
2123 if (isByRef(ty, zcu, cg.target)) {2139 if (isByRef(ty, zcu, cg.target)) {
2124 return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });2140 const offset_ptr: WValue = switch (offset + lhs.offset()) {
2141 0 => lhs,
2142 else => |total_offset| ptr: {
2143 try cg.emitWValue(lhs);
2144 try cg.addImm32(total_offset);
2145 try cg.addTag(.i32_add);
2146 break :ptr .stack;
2147 },
2148 };
2149 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2125 }2150 }
21262151
2127 if (ty.zigTypeTag(zcu) == .vector) {2152 if (ty.zigTypeTag(zcu) == .vector) {
...@@ -2133,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2133,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2133 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2158 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2134 @intFromEnum(std.wasm.SimdOpcode.v128_store),2159 @intFromEnum(std.wasm.SimdOpcode.v128_store),
2135 offset + lhs.offset(),2160 offset + lhs.offset(),
2136 @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),2161 @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2137 });2162 });
2138 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2163 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2139 }2164 }
...@@ -2174,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2174,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2174 const zcu = pt.zcu;2199 const zcu = pt.zcu;
2175 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2200 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2176 const operand = try cg.resolveInst(ty_op.operand);2201 const operand = try cg.resolveInst(ty_op.operand);
2177 const ty = ty_op.ty.toType();2202 const elem_ty = ty_op.ty.toType();
2178 const ptr_ty = cg.typeOf(ty_op.operand);2203 const ptr_ty = cg.typeOf(ty_op.operand);
2179 const ptr_info = ptr_ty.ptrInfo(zcu);2204 const ptr_info = ptr_ty.ptrInfo(zcu);
21802205
2181 if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});2206 assert(elem_ty.hasRuntimeBits(zcu));
2182
2183 assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_load
21842207
2185 const result = try cg.load(operand, ty, 0);2208 const offset: u32 = switch (ptr_info.flags.vector_index) {
2209 .none => offset: {
2210 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2211 break :offset 0;
2212 },
2213 else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
2214 };
2215 const result = try cg.load(operand, elem_ty, offset);
2186 return cg.finishAir(inst, result, &.{ty_op.operand});2216 return cg.finishAir(inst, result, &.{ty_op.operand});
2187}2217}
21882218
...@@ -2191,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2191,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2191fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {2221fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2192 const zcu = cg.pt.zcu;2222 const zcu = cg.pt.zcu;
2193 if (isByRef(ty, zcu, cg.target)) {2223 if (isByRef(ty, zcu, cg.target)) {
2194 const val = try cg.allocStack(ty);2224 const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
2195 try cg.store(val, try operand.toLocal(cg, .usize), ty, 0);2225 0 => operand,
2196 return val;2226 else => |total_offset| ptr: {
2227 try cg.emitWValue(operand);
2228 try cg.addImm32(total_offset);
2229 try cg.addTag(.i32_add);
2230 break :ptr .stack;
2231 },
2232 };
2233 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2234 const new_ptr = try cg.allocStack(ty);
2235 try cg.store(new_ptr, src_ptr, ty, 0);
2236 return new_ptr;
2197 }2237 }
21982238
2199 // load local's value from memory by its stack position2239 // load local's value from memory by its stack position
...@@ -5235,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5235,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5235 return cg.finishAir(inst, .none, &.{});5275 return cg.finishAir(inst, .none, &.{});
5236}5276}
52375277
5278fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5279 const zcu = cg.pt.zcu;
5280 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5281
5282 const operand_ty = cg.typeOf(ty_op.operand);
5283 const dest_ty = cg.typeOfIndex(inst);
5284 assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5285 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5286 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5287
5288 const operand = try cg.resolveInst(ty_op.operand);
5289 const result = cg.reuseOperand(ty_op.operand, operand);
5290 return cg.finishAir(inst, result, &.{ty_op.operand});
5291}
5292
5293fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5294 const zcu = cg.pt.zcu;
5295 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5296
5297 const operand_ty = cg.typeOf(ty_op.operand);
5298 const dest_ty = cg.typeOfIndex(inst);
5299 assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5300 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5301 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5302
5303 const operand = try cg.resolveInst(ty_op.operand);
5304 const result = switch (operand) {
5305 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5306 else => cg.reuseOperand(ty_op.operand, operand),
5307 };
5308 return cg.finishAir(inst, result, &.{ty_op.operand});
5309}
5310
5238fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5311fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5239 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5312 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5240 const operand = try cg.resolveInst(ty_op.operand);5313 const operand = try cg.resolveInst(ty_op.operand);
...@@ -6340,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6340,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6340 }6413 }
6341 }6414 }
63426415
6343 const elem_result = if (isByRef(elem_ty, zcu, cg.target))6416 const result = if (isByRef(elem_ty, zcu, cg.target))
6344 .stack6417 .stack
6345 else6418 else
6346 try cg.load(.stack, elem_ty, 0);6419 try cg.load(.stack, elem_ty, 0);
63476420 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6348 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6349}6421}
63506422
6351fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6423fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
src/codegen/x86_64/CodeGen.zig+33-19
...@@ -47,7 +47,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -47,7 +47,6 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
47 .scalarize_shl,47 .scalarize_shl,
48 .scalarize_shl_exact,48 .scalarize_shl_exact,
49 .scalarize_shl_sat,49 .scalarize_shl_sat,
50 .scalarize_bitcast,
51 .scalarize_ctz,50 .scalarize_ctz,
52 .scalarize_popcount,51 .scalarize_popcount,
53 .scalarize_byte_swap,52 .scalarize_byte_swap,
...@@ -58,11 +57,13 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -58,11 +57,13 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
58 .scalarize_shuffle_two,57 .scalarize_shuffle_two,
59 .scalarize_select,58 .scalarize_select,
6059
60 .scalarize_bit_cast_padded_elems,
61
61 //.unsplat_shift_rhs,62 //.unsplat_shift_rhs,
62 .reduce_one_elem_to_bitcast,63 .reduce_one_elem_to_bit_cast,
63 .splat_one_elem_to_bitcast,64 .splat_one_elem_to_bit_cast,
6465
65 .expand_intcast_safe,66 .expand_int_cast_safe,
66 .expand_int_from_float_safe,67 .expand_int_from_float_safe,
67 .expand_int_from_float_optimized_safe,68 .expand_int_from_float_optimized_safe,
68 .expand_add_safe,69 .expand_add_safe,
...@@ -67433,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -67433,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67433 };67434 };
67434 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);67435 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
67435 },67436 },
67436 .bitcast => try cg.airBitCast(inst),67437 .bit_cast,
67438 .ptr_cast,
67439 .ptr_from_int,
67440 .int_from_ptr,
67441 .error_cast,
67442 .error_from_int,
67443 .int_from_error,
67444 .union_from_enum,
67445 => try cg.airBitCast(inst),
67437 .block => {67446 .block => {
67438 const block = cg.air.unwrapBlock(inst);67447 const block = cg.air.unwrapBlock(inst);
67439 if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);67448 if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
...@@ -93374,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -93374,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
93374 };93383 };
93375 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);93384 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
93376 },93385 },
93377 .intcast => |air_tag| {93386 .int_cast => |air_tag| {
93378 const ty_op = air_datas[@intFromEnum(inst)].ty_op;93387 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
93379 const dst_ty = ty_op.ty.toType();93388 const dst_ty = ty_op.ty.toType();
93380 const src_ty = cg.typeOf(ty_op.operand);93389 const src_ty = cg.typeOf(ty_op.operand);
...@@ -98132,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -98132,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
98132 };98141 };
98133 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);98142 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
98134 },98143 },
98135 .intcast_safe => unreachable,98144 .int_cast_safe => unreachable,
98136 .trunc => |air_tag| {98145 .trunc => |air_tag| {
98137 const ty_op = air_datas[@intFromEnum(inst)].ty_op;98146 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
98138 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});98147 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
...@@ -104350,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -104350,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104350 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });104359 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
104351 try ops[0].toSlicePtr(cg);104360 try ops[0].toSlicePtr(cg);
104352 const dst_ty = ty_pl.ty.toType();104361 const dst_ty = ty_pl.ty.toType();
104353 if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {104362 zero_offset: {
104354 const elem_size = dst_ty.childType(zcu).abiSize(zcu);104363 const elem_size = dst_ty.childType(zcu).abiSize(zcu);
104355 if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;104364 if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;
104356 while (true) for (&ops) |*op| {104365 while (true) for (&ops) |*op| {
...@@ -179053,8 +179062,11 @@ fn genSetReg(...@@ -179053,8 +179062,11 @@ fn genSetReg(
179053 const zcu = pt.zcu;179062 const zcu = pt.zcu;
179054 const abi_size: u32 = @intCast(ty.abiSize(zcu));179063 const abi_size: u32 = @intCast(ty.abiSize(zcu));
179055 const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));179064 const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));
179056 if (ty.bitSize(zcu) > dst_alias.size().bitSize(cg.target))179065 {
179057 return cg.fail("genSetReg called with a value larger than dst_reg", .{});179066 const ty_bit_size = if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else 8 * abi_size;
179067 if (ty_bit_size > dst_alias.size().bitSize(cg.target))
179068 return cg.fail("genSetReg called with a value larger than dst_reg", .{});
179069 }
179058 switch (src_mcv) {179070 switch (src_mcv) {
179059 .none,179071 .none,
179060 .unreach,179072 .unreach,
...@@ -180127,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {...@@ -180127,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
180127 break :dst dst_mcv;180139 break :dst dst_mcv;
180128 };180140 };
180129180141
180130 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;180142 switch (dst_ty.zigTypeTag(zcu)) {
180143 .float, .error_union, .error_set, .vector => break :result dst_mcv,
180144 .@"struct", .@"union" => if (dst_ty.containerLayout(zcu) != .@"packed") break :result dst_mcv,
180145 .optional, .pointer => if (!dst_ty.isPtrAtRuntime(zcu)) break :result dst_mcv,
180146 else => {},
180147 }
180131180148
180132 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and180149 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
180133 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;180150 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
180134180151
180135 const abi_size = dst_ty.abiSize(zcu);180152 const abi_size = dst_ty.abiSize(zcu);
180136 const bit_size = dst_ty.bitSize(zcu);180153 const bit_size = dst_ty.bitSize(zcu);
180137 if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;180154 if (abi_size * 8 <= bit_size) break :result dst_mcv;
180138180155
180139 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;180156 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
180140 const high_mcv: MCValue = switch (dst_mcv) {180157 const high_mcv: MCValue = switch (dst_mcv) {
...@@ -182410,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {...@@ -182410,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
182410 const zcu = pt.zcu;182427 const zcu = pt.zcu;
182411 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{182428 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
182412 .signedness = .unsigned,182429 .signedness = .unsigned,
182413 .bits = @intCast(ty.bitSize(zcu)),182430 .bits = @intCast(if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else ty.abiSize(zcu) * 8),
182414 };182431 };
182415 const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;182432 const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
182416 try self.spillEflagsIfOccupied();182433 try self.spillEflagsIfOccupied();
...@@ -182450,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {...@@ -182450,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {
182450 };182467 };
182451}182468}
182452182469
182453fn regExtraBits(self: *CodeGen, ty: Type) u64 {
182454 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
182455}
182456
182457fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {182470fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
182458 return switch (feature) {182471 return switch (feature) {
182459 .@"64bit" => switch (cg.target.cpu.arch) {182472 .@"64bit" => switch (cg.target.cpu.arch) {
...@@ -182569,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {...@@ -182569,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {
182569 .bool_type => vector_type.len,182582 .bool_type => vector_type.len,
182570 else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),182583 else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),
182571 },182584 },
182572 else => @intCast(ty.bitSize(zcu)),182585 else => if (ty.hasBitRepresentation(zcu) or ty.isAbiInt(zcu)) @intCast(ty.bitSize(zcu)) else @intCast(ty.abiSize(zcu) * 8),
182573 };182586 };
182574}182587}
182575182588
...@@ -192242,7 +192255,8 @@ const Select = struct {...@@ -192242,7 +192255,8 @@ const Select = struct {
192242 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),192255 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
192243 .@"8_size_sub_bit_size" => {192256 .@"8_size_sub_bit_size" => {
192244 const ty = op.flags.base.ref.typeOf(s);192257 const ty = op.flags.base.ref.typeOf(s);
192245 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));192258 const bit_size = s.cg.intInfo(ty).?.bits;
192259 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - bit_size);
192246 },192260 },
192247 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),192261 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
192248 .elem_limbs => @intCast(@divExact(192262 .elem_limbs => @intCast(@divExact(
stage1/zig.h+55-19
...@@ -11,8 +11,6 @@...@@ -11,8 +11,6 @@
11#elif defined(__GNUC__)11#elif defined(__GNUC__)
12#define zig_gcc12#define zig_gcc
13#define zig_gnuc13#define zig_gnuc
14#elif defined(__IBMC__)
15#define zig_xlc
16#elif defined(__TINYC__)14#elif defined(__TINYC__)
17#define zig_tinyc15#define zig_tinyc
18#elif defined(__slimcc__)16#elif defined(__slimcc__)
...@@ -28,8 +26,18 @@...@@ -28,8 +26,18 @@
28#define zig_arm26#define zig_arm
29#elif defined(__arm__)27#elif defined(__arm__)
30#define zig_arm28#define zig_arm
29#elif defined(__arc__)
30#define zig_arc
31#elif defined(__csky__)
32#define zig_csky
31#elif defined(__hexagon__)33#elif defined(__hexagon__)
32#define zig_hexagon34#define zig_hexagon
35#elif defined(__hppa__) && defined(_LP64)
36#define zig_hppa64
37#define zig_hppa
38#elif defined(__hppa__)
39#define zig_hppa32
40#define zig_hppa
33#elif defined(__kvx__)41#elif defined(__kvx__)
34#define zig_kvx42#define zig_kvx
35#elif defined(__loongarch32)43#elif defined(__loongarch32)
...@@ -42,6 +50,8 @@...@@ -42,6 +50,8 @@
42#define zig_m68k50#define zig_m68k
43#elif defined(__m88k__)51#elif defined(__m88k__)
44#define zig_m88k52#define zig_m88k
53#elif defined(__microblaze__)
54#define zig_microblaze
45#elif defined(__mips64)55#elif defined(__mips64)
46#define zig_mips6456#define zig_mips64
47#define zig_mips57#define zig_mips
...@@ -64,6 +74,8 @@...@@ -64,6 +74,8 @@
64#define zig_riscv74#define zig_riscv
65#elif defined(__s390x__)75#elif defined(__s390x__)
66#define zig_s390x76#define zig_s390x
77#elif defined(__sh__)
78#define zig_sh
67#elif defined(__sparc__) && defined(__arch64__)79#elif defined(__sparc__) && defined(__arch64__)
68#define zig_sparc6480#define zig_sparc64
69#define zig_sparc81#define zig_sparc
...@@ -100,32 +112,30 @@...@@ -100,32 +112,30 @@
100#define zig_big_endian 1112#define zig_big_endian 1
101#endif113#endif
102114
103#if defined(__MACH__)115#if defined(__APPLE__)
104#define zig_darwin116#define zig_darwin
105#elif defined(__DragonFly__)117#elif defined(__DragonFly__)
106#define zig_dragonfly118#define zig_dragonfly
107#define zig_bsd
108#elif defined(__EMSCRIPTEN__)119#elif defined(__EMSCRIPTEN__)
109#define zig_emscripten120#define zig_emscripten
110#elif defined(__FreeBSD__)121#elif defined(__FreeBSD__)
111#define zig_freebsd122#define zig_freebsd
112#define zig_bsd
113#elif defined(__Fuchsia__)123#elif defined(__Fuchsia__)
114#define zig_fuchsia124#define zig_fuchsia
115#elif defined(__HAIKU__)125#elif defined(__HAIKU__)
116#define zig_haiku126#define zig_haiku
117#elif defined(__gnu_hurd__)127#elif defined(__gnu_hurd__)
118#define zig_hurd128#define zig_hurd
129#elif defined(__illumos__)
130#define zig_illumos
119#elif defined(__linux__)131#elif defined(__linux__)
120#define zig_linux132#define zig_linux
121#elif defined(__NetBSD__)133#elif defined(__NetBSD__)
122#define zig_netbsd134#define zig_netbsd
123#define zig_bsd
124#elif defined(__OpenBSD__)135#elif defined(__OpenBSD__)
125#define zig_openbsd136#define zig_openbsd
126#define zig_bsd137#elif defined(__serenity__)
127#elif defined(__SVR4)138#define zig_serenity
128#define zig_solaris
129#elif defined(__wasi__)139#elif defined(__wasi__)
130#define zig_wasi140#define zig_wasi
131#elif defined(_WIN32)141#elif defined(_WIN32)
...@@ -404,14 +414,22 @@...@@ -404,14 +414,22 @@
404#define zig_trap() __asm__ volatile("udf #0xfe")414#define zig_trap() __asm__ volatile("udf #0xfe")
405#elif defined(zig_arm) || defined(zig_aarch64)415#elif defined(zig_arm) || defined(zig_aarch64)
406#define zig_trap() __asm__ volatile("udf #0xfdee")416#define zig_trap() __asm__ volatile("udf #0xfdee")
417#elif defined(zig_arc)
418#define zig_trap() __asm__ volatile("unimp_s")
419#elif defined(zig_csky)
420#define zig_trap() __asm__ volatile(".word 0x3fff")
407#elif defined(zig_hexagon)421#elif defined(zig_hexagon)
408#define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)")422#define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)")
423#elif defined(zig_hppa)
424#define zig_trap() __asm__ volatile("iitlbp %r0, (%sr0, %r0)")
409#elif defined(zig_kvx) || defined(zig_loongarch) || defined(zig_powerpc)425#elif defined(zig_kvx) || defined(zig_loongarch) || defined(zig_powerpc)
410#define zig_trap() __asm__ volatile(".word 0x0")426#define zig_trap() __asm__ volatile(".word 0x0")
411#elif defined(zig_m68k)427#elif defined(zig_m68k)
412#define zig_trap() __asm__ volatile("illegal")428#define zig_trap() __asm__ volatile("illegal")
413#elif defined(zig_m88k)429#elif defined(zig_m88k)
414#define zig_trap() __asm__ volatile("tb0 0, %%r0, 511")430#define zig_trap() __asm__ volatile("tb0 0, %%r0, 511")
431#elif defined(zig_microblaze)
432#define zig_trap() __asm__ volatile("getd r0, r0")
415#elif defined(zig_mips)433#elif defined(zig_mips)
416#define zig_trap() __asm__ volatile(".word 0x3d")434#define zig_trap() __asm__ volatile(".word 0x3d")
417#elif defined(zig_or1k)435#elif defined(zig_or1k)
...@@ -420,6 +438,8 @@...@@ -420,6 +438,8 @@
420#define zig_trap() __asm__ volatile("unimp")438#define zig_trap() __asm__ volatile("unimp")
421#elif defined(zig_s390x)439#elif defined(zig_s390x)
422#define zig_trap() __asm__ volatile("j 0x2")440#define zig_trap() __asm__ volatile("j 0x2")
441#elif defined(zig_sh)
442#define zig_trap() __asm__ volatile(".word 0x0001")
423#elif defined(zig_sparc)443#elif defined(zig_sparc)
424#define zig_trap() __asm__ volatile("illtrap")444#define zig_trap() __asm__ volatile("illtrap")
425#elif defined(zig_x86_16)445#elif defined(zig_x86_16)
...@@ -446,16 +466,22 @@...@@ -446,16 +466,22 @@
446466
447#if defined(zig_alpha)467#if defined(zig_alpha)
448#define zig_breakpoint() __asm__ volatile("call_pal 0x000080")468#define zig_breakpoint() __asm__ volatile("call_pal 0x000080")
449#elif defined(zig_arm)469#elif defined(zig_arm) || defined(zig_csky)
450#define zig_breakpoint() __asm__ volatile("bkpt #0x0")470#define zig_breakpoint() __asm__ volatile("bkpt #0x0")
451#elif defined(zig_aarch64)471#elif defined(zig_aarch64)
452#define zig_breakpoint() __asm__ volatile("brk #0xf000")472#define zig_breakpoint() __asm__ volatile("brk #0xf000")
473#elif defined(zig_arc)
474#define zig_breakpoint() __asm__ volatile("brk_s")
453#elif defined(zig_hexagon)475#elif defined(zig_hexagon)
454#define zig_breakpoint() __asm__ volatile("brkpt")476#define zig_breakpoint() __asm__ volatile("brkpt")
477#elif defined(zig_hppa)
478#define zig_breakpoint() __asm__ volatile("break 0x04, 0x0008")
455#elif defined(zig_kvx) || defined(zig_loongarch)479#elif defined(zig_kvx) || defined(zig_loongarch)
456#define zig_breakpoint() __asm__ volatile("break 0x0")480#define zig_breakpoint() __asm__ volatile("break 0x0")
457#elif defined(zig_m88k)481#elif defined(zig_m88k)
458#define zig_breakpoint() __asm__ volatile("illop1")482#define zig_breakpoint() __asm__ volatile("illop1")
483#elif defined(zig_microblaze)
484#define zig_breakpoint() __asm__ volatile("brki r16, 0x0018")
459#elif defined(zig_mips)485#elif defined(zig_mips)
460#define zig_breakpoint() __asm__ volatile("break")486#define zig_breakpoint() __asm__ volatile("break")
461#elif defined(zig_or1k)487#elif defined(zig_or1k)
...@@ -466,6 +492,8 @@...@@ -466,6 +492,8 @@
466#define zig_breakpoint() __asm__ volatile("ebreak")492#define zig_breakpoint() __asm__ volatile("ebreak")
467#elif defined(zig_s390x)493#elif defined(zig_s390x)
468#define zig_breakpoint() __asm__ volatile("j 0x6")494#define zig_breakpoint() __asm__ volatile("j 0x6")
495#elif defined(zig_sh)
496#define zig_breakpoint() __asm__ volatile("trapa #0xc3")
469#elif defined(zig_sparc)497#elif defined(zig_sparc)
470#define zig_breakpoint() __asm__ volatile("ta 0x1")498#define zig_breakpoint() __asm__ volatile("ta 0x1")
471#elif defined(zig_x86)499#elif defined(zig_x86)
...@@ -4529,9 +4557,12 @@ static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 a...@@ -4529,9 +4557,12 @@ static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 a
4529#include <intrin.h>4557#include <intrin.h>
4530#endif4558#endif
45314559
4560static inline void* zig_e_zig_windows_teb(void) zig_mangled(zig_e_zig_windows_teb, "zig_windows_teb");
4561static inline void* zig_e_zig_windows_peb(void) zig_mangled(zig_e_zig_windows_peb, "zig_windows_peb");
4562
4532#if defined(zig_thumb)4563#if defined(zig_thumb)
45334564
4534static inline void* zig_windows_teb(void) {4565static inline void* zig_e_zig_windows_teb(void) {
4535 void* teb = 0;4566 void* teb = 0;
4536#if defined(zig_msvc)4567#if defined(zig_msvc)
4537 teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2);4568 teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2);
...@@ -4543,7 +4574,7 @@ static inline void* zig_windows_teb(void) {...@@ -4543,7 +4574,7 @@ static inline void* zig_windows_teb(void) {
45434574
4544#elif defined(zig_aarch64)4575#elif defined(zig_aarch64)
45454576
4546static inline void* zig_windows_teb(void) {4577static inline void* zig_e_zig_windows_teb(void) {
4547 void* teb = 0;4578 void* teb = 0;
4548#if defined(zig_msvc)4579#if defined(zig_msvc)
4549 teb = (void*)__readx18qword(0x0);4580 teb = (void*)__readx18qword(0x0);
...@@ -4555,7 +4586,7 @@ static inline void* zig_windows_teb(void) {...@@ -4555,7 +4586,7 @@ static inline void* zig_windows_teb(void) {
45554586
4556#elif defined(zig_x86_32)4587#elif defined(zig_x86_32)
45574588
4558static inline void* zig_windows_teb(void) {4589static inline void* zig_e_zig_windows_teb(void) {
4559 void* teb = 0;4590 void* teb = 0;
4560#if defined(zig_msvc)4591#if defined(zig_msvc)
4561 teb = (void*)__readfsdword(0x18);4592 teb = (void*)__readfsdword(0x18);
...@@ -4565,7 +4596,7 @@ static inline void* zig_windows_teb(void) {...@@ -4565,7 +4596,7 @@ static inline void* zig_windows_teb(void) {
4565 return teb;4596 return teb;
4566}4597}
45674598
4568static inline void* zig_windows_peb(void) {4599static inline void* zig_e_zig_windows_peb(void) {
4569 void* peb = 0;4600 void* peb = 0;
4570#if defined(zig_msvc)4601#if defined(zig_msvc)
4571 peb = (void*)__readfsdword(0x30);4602 peb = (void*)__readfsdword(0x30);
...@@ -4577,7 +4608,7 @@ static inline void* zig_windows_peb(void) {...@@ -4577,7 +4608,7 @@ static inline void* zig_windows_peb(void) {
45774608
4578#elif defined(zig_x86_64)4609#elif defined(zig_x86_64)
45794610
4580static inline void* zig_windows_teb(void) {4611static inline void* zig_e_zig_windows_teb(void) {
4581 void* teb = 0;4612 void* teb = 0;
4582#if defined(zig_msvc)4613#if defined(zig_msvc)
4583 teb = (void*)__readgsqword(0x30);4614 teb = (void*)__readgsqword(0x30);
...@@ -4587,7 +4618,7 @@ static inline void* zig_windows_teb(void) {...@@ -4587,7 +4618,7 @@ static inline void* zig_windows_teb(void) {
4587 return teb;4618 return teb;
4588}4619}
45894620
4590static inline void* zig_windows_peb(void) {4621static inline void* zig_e_zig_windows_peb(void) {
4591 void* peb = 0;4622 void* peb = 0;
4592#if defined(zig_msvc)4623#if defined(zig_msvc)
4593 peb = (void*)__readgsqword(0x60);4624 peb = (void*)__readgsqword(0x60);
...@@ -4601,7 +4632,9 @@ static inline void* zig_windows_peb(void) {...@@ -4601,7 +4632,9 @@ static inline void* zig_windows_peb(void) {
46014632
4602#if defined(zig_loongarch)4633#if defined(zig_loongarch)
46034634
4604static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {4635static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) zig_mangled(zig_e_zig_loongarch_cpucfg, "zig_loongarch_cpucfg");
4636
4637static inline void zig_e_zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
4605#if defined(zig_gnuc_asm)4638#if defined(zig_gnuc_asm)
4606 __asm__("cpucfg %[result], %[word]" : [result] "=r" (result) : [word] "r" (word));4639 __asm__("cpucfg %[result], %[word]" : [result] "=r" (result) : [word] "r" (word));
4607#else4640#else
...@@ -4611,7 +4644,10 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {...@@ -4611,7 +4644,10 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
46114644
4612#elif defined(zig_x86) && !defined(zig_x86_16)4645#elif defined(zig_x86) && !defined(zig_x86_16)
46134646
4614static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {4647static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) zig_mangled(zig_e_zig_x86_cpuid, "zig_x86_cpuid");
4648static inline uint32_t zig_e_zig_x86_get_xcr0(void) zig_mangled(zig_e_zig_x86_get_xcr0, "zig_x86_get_xcr0");
4649
4650static inline void zig_e_zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
4615#if defined(zig_msvc)4651#if defined(zig_msvc)
4616 int cpu_info[4];4652 int cpu_info[4];
4617 __cpuidex(cpu_info, leaf_id, subid);4653 __cpuidex(cpu_info, leaf_id, subid);
...@@ -4629,7 +4665,7 @@ static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax...@@ -4629,7 +4665,7 @@ static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax
4629#endif4665#endif
4630}4666}
46314667
4632static inline uint32_t zig_x86_get_xcr0(void) {4668static inline uint32_t zig_e_zig_x86_get_xcr0(void) {
4633#if defined(zig_msvc)4669#if defined(zig_msvc)
4634 return (uint32_t)_xgetbv(0);4670 return (uint32_t)_xgetbv(0);
4635#elif defined(zig_gnuc_asm)4671#elif defined(zig_gnuc_asm)
stage1/zig1.wasm
Binary files a/stage1/zig1.wasm and b/stage1/zig1.wasm differ
test/behavior/asm.zig-36
...@@ -209,42 +209,6 @@ test "packed output types (x86_64)" {...@@ -209,42 +209,6 @@ test "packed output types (x86_64)" {
209 }209 }
210}210}
211211
212test "extern output types (x86_64)" {
213 if (builtin.target.cpu.arch != .x86_64) return error.SkipZigTest;
214 if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows) return error.SkipZigTest; // MSVC doesn't support inline assembly
215 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/31531
216
217 const S = extern struct { x: u32 };
218 {
219 const s: S = asm volatile ("mov $123, %[ret]"
220 : [ret] "=r" (-> S),
221 );
222 try expect(s.x == 123);
223 }
224 {
225 var s: S = undefined;
226 asm volatile ("mov $123, %[ret]"
227 : [ret] "=r" (s),
228 );
229 try expect(s.x == 123);
230 }
231
232 const U = extern union { x: u32 };
233 {
234 const u: U = asm volatile ("mov $123, %[ret]"
235 : [ret] "=r" (-> U),
236 );
237 try expect(u.x == 123);
238 }
239 {
240 var u: U = undefined;
241 asm volatile ("mov $123, %[ret]"
242 : [ret] "=r" (u),
243 );
244 try expect(u.x == 123);
245 }
246}
247
248test "abi register aliases as clobbers (RISC-V)" {212test "abi register aliases as clobbers (RISC-V)" {
249 if (!builtin.target.cpu.arch.isRISCV()) return error.SkipZigTest;213 if (!builtin.target.cpu.arch.isRISCV()) return error.SkipZigTest;
250 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;214 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
test/behavior/atomics.zig-1
...@@ -358,7 +358,6 @@ test "atomics with different types" {...@@ -358,7 +358,6 @@ test "atomics with different types" {
358 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO358 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
359 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;359 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
360 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;360 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
361 if (builtin.target.cpu.arch.endian() == .big) return error.SkipZigTest; // #24282
362361
363 try testAtomicsWithType(bool, true, false);362 try testAtomicsWithType(bool, true, false);
364363
test/behavior/bitcast.zig+141-167
...@@ -74,55 +74,6 @@ fn conv_uN(comptime N: usize, x: @Int(.unsigned, N)) @Int(.signed, N) {...@@ -74,55 +74,6 @@ fn conv_uN(comptime N: usize, x: @Int(.unsigned, N)) @Int(.signed, N) {
74 return @as(@Int(.signed, N), @bitCast(x));74 return @as(@Int(.signed, N), @bitCast(x));
75}75}
7676
77test "bitcast uX to bytes" {
78 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
79 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
80 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
81 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
82 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
83
84 const bit_values = [_]usize{ 1, 48, 27, 512, 493, 293, 125, 204, 112 };
85 inline for (bit_values) |bits| {
86 try testBitCast(bits);
87 try comptime testBitCast(bits);
88 }
89}
90
91fn testBitCastuXToBytes(comptime N: usize) !void {
92
93 // The location of padding bits in these layouts are technically not defined
94 // by LLVM, but we currently allow exotic integers to be cast (at comptime)
95 // to types that expose their padding bits anyway.
96 //
97 // This test at least makes sure those bits are matched by the runtime behavior
98 // on the platforms we target. If the above behavior is restricted after all,
99 // this test should be deleted.
100
101 const T = @Int(.unsigned, N);
102 for ([_]T{ 0, ~@as(T, 0) }) |init_value| {
103 var x: T = init_value;
104 const bytes = std.mem.asBytes(&x);
105
106 const byte_count = (N + 7) / 8;
107 switch (native_endian) {
108 .little => {
109 var byte_i = 0;
110 while (byte_i < (byte_count - 1)) : (byte_i += 1) {
111 try expect(bytes[byte_i] == 0xff);
112 }
113 try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
114 },
115 .big => {
116 var byte_i = byte_count - 1;
117 while (byte_i > 0) : (byte_i -= 1) {
118 try expect(bytes[byte_i] == 0xff);
119 }
120 try expect(((bytes[byte_i] ^ 0xff) << -%@as(u3, @truncate(N))) == 0);
121 },
122 }
123 }
124}
125
126test "nested bitcast" {77test "nested bitcast" {
127 const S = struct {78 const S = struct {
128 fn moo(x: isize) !void {79 fn moo(x: isize) !void {
...@@ -183,39 +134,6 @@ test "@bitCast packed structs at runtime and comptime" {...@@ -183,39 +134,6 @@ test "@bitCast packed structs at runtime and comptime" {
183 try comptime S.doTheTest();134 try comptime S.doTheTest();
184}135}
185136
186test "@bitCast extern structs at runtime and comptime" {
187 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
188 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
189 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
190
191 const Full = extern struct {
192 number: u16,
193 };
194 const TwoHalves = extern struct {
195 half1: u8,
196 half2: u8,
197 };
198 const S = struct {
199 fn doTheTest() !void {
200 var full = Full{ .number = 0x1234 };
201 _ = &full;
202 const two_halves: TwoHalves = @bitCast(full);
203 switch (native_endian) {
204 .big => {
205 try expect(two_halves.half1 == 0x12);
206 try expect(two_halves.half2 == 0x34);
207 },
208 .little => {
209 try expect(two_halves.half1 == 0x34);
210 try expect(two_halves.half2 == 0x12);
211 },
212 }
213 }
214 };
215 try S.doTheTest();
216 try comptime S.doTheTest();
217}
218
219test "bitcast packed struct to integer and back" {137test "bitcast packed struct to integer and back" {
220 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;138 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
221 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;139 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
...@@ -363,32 +281,12 @@ test "comptime @bitCast packed struct to int and back" {...@@ -363,32 +281,12 @@ test "comptime @bitCast packed struct to int and back" {
363 }281 }
364}282}
365283
366test "comptime bitcast with fields following f80" {
367 if (true) {
368 // https://github.com/ziglang/zig/issues/19387
369 return error.SkipZigTest;
370 }
371
372 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
373 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
374 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
375
376 const FloatT = extern struct { f: f80, x: u128 align(16) };
377 const x: FloatT = .{ .f = 0.5, .x = 123 };
378 var x_as_uint: u256 = comptime @as(u256, @bitCast(x));
379 _ = &x_as_uint;
380
381 try expect(x.f == @as(FloatT, @bitCast(x_as_uint)).f);
382 try expect(x.x == @as(FloatT, @bitCast(x_as_uint)).x);
383}
384
385test "bitcast vector to integer and back" {284test "bitcast vector to integer and back" {
386 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;285 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
387 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO286 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
388 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO287 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
389 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;288 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
390 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;289 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
391 if (builtin.cpu.arch.endian() == .big and builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
392290
393 var vec: @Vector(16, bool) = @splat(true);291 var vec: @Vector(16, bool) = @splat(true);
394 vec[1] = false;292 vec[1] = false;
...@@ -524,87 +422,163 @@ test "@bitCast of packed struct of bools all false" {...@@ -524,87 +422,163 @@ test "@bitCast of packed struct of bools all false" {
524 try expect(@as(u8, @as(u4, @bitCast(p))) == 0);422 try expect(@as(u8, @as(u4, @bitCast(p))) == 0);
525}423}
526424
527test "@bitCast of extern struct containing pointer" {425test "@bitCast of packed struct with void field to integer" {
528 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;426 const S = packed struct(u8) {
529 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO427 v: void,
530 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO428 x: u8,
531 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; // TODO
532
533 const S = struct {
534 const A = extern struct {
535 ptr: *const u32,
536 };
537
538 const B = extern struct {
539 ptr: *const i32,
540 };
541429
542 fn doTheTest() !void {430 fn doTheTest(x: u8) !void {
543 const x: u32 = 123;431 // Intentionally using `@as` to avoid RLS which masks the bug
544 var a: A = undefined;432 const foo = @as(@This(), .{ .v = {}, .x = x });
545 a = .{ .ptr = &x };433 const as_int: u8 = @bitCast(foo);
546 const b: B = @bitCast(a);434 try expect(as_int == x);
547 try expect(b.ptr.* == 123);
548 }435 }
549 };436 };
550437 try S.doTheTest(123);
551 try S.doTheTest();438 try comptime S.doTheTest(123);
552 try comptime S.doTheTest();
553}439}
554440
555test "@bitCast of extern struct to float" {441test "@bitCast vector to array with different element size" {
556 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;442 const static = struct {
557 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;443 fn doTheTest(v: @Vector(4, u5)) !void {
558444 const result: [5]u4 = @bitCast(v);
559 const S = struct {445 // See the definition of `v` in the test proper for these values.
560 const S = extern struct {446 try expect(result[0] == 0b0010);
561 x: u16,447 try expect(result[1] == 0b1110);
562 y: u16,448 try expect(result[2] == 0b0101);
563 };449 try expect(result[3] == 0b0110);
564 fn doTheTest() !void {450 try expect(result[4] == 0b0000);
565 var s: S = .{ .x = 0, .y = 0 };451 }
566 _ = &s;452 };
567 const a: f32 = @bitCast(s);453 // The strange digit groupings here are to indicate how this maps to `expected` above.
568 try expect(a == 0);454 const v: @Vector(4, u5) = .{
455 0b0_0010,
456 0b01_111,
457 0b110_01,
458 0b0000_0,
459 };
460 try static.doTheTest(v);
461 try comptime static.doTheTest(v);
462}
463
464test "@bitCast packed struct to array of bits" {
465 const S = packed struct(u16) {
466 foo: u5,
467 bar: i7,
468 baz: u3,
469 qux: bool,
470 fn doTheTest(val: @This(), comptime Bits: type) !void {
471 const bits: Bits = @bitCast(val);
472
473 // foo
474 try expect(bits[0] == 1);
475 try expect(bits[1] == 0);
476 try expect(bits[2] == 0);
477 try expect(bits[3] == 1);
478 try expect(bits[4] == 0);
479 // bar
480 try expect(bits[5] == 0);
481 try expect(bits[6] == 1);
482 try expect(bits[7] == 1);
483 try expect(bits[8] == 1);
484 try expect(bits[9] == 1);
485 try expect(bits[10] == 1);
486 try expect(bits[11] == 1);
487 // baz
488 try expect(bits[12] == 0);
489 try expect(bits[13] == 1);
490 try expect(bits[14] == 0);
491 // qux
492 try expect(bits[15] == 1);
569 }493 }
570 };494 };
571495
572 try S.doTheTest();496 const val: S = .{
573 try comptime S.doTheTest();497 .foo = 0b01001,
498 .bar = -2,
499 .baz = 0b010,
500 .qux = true,
501 };
502
503 try val.doTheTest(@Vector(16, u1));
504 try val.doTheTest([16]u1);
505
506 try comptime val.doTheTest(@Vector(16, u1));
507 try comptime val.doTheTest([16]u1);
574}508}
575509
576test "@bitCast of float to extern struct" {510test "@bitCast nested arrays of vectors" {
577 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;511 const Src = [2][2]@Vector(4, u5);
578 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;512 const Dest = [5]@Vector(2, u8);
579513
580 const S = struct {514 // The strange digit groupings here are to indicate how this maps to the output.
581 const S = extern struct {515 const src: Src = .{ .{
582 x: u32,516 .{ 0b00011, 0b00_100, 0b11100, 0b0010_1 },
583 };517 .{ 0b1_0110, 0b11011, 0b101_10, 0b10101 },
584 fn doTheTest() !void {518 }, .{
585 var a: f32 = -0.0;519 .{ 0b10101, 0b00_001, 0b01011, 0b0001_0 },
586 _ = &a;520 .{ 0b0_0001, 0b01111, 0b111_10, 0b00001 },
587 const s: S = @bitCast(a);521 } };
588 try expect(s.x == 0x80000000);522
523 const expected: Dest = .{
524 .{ 0b10000011, 0b11110000 },
525 .{ 0b01100010, 0b10110111 },
526 .{ 0b10101101, 0b00110101 },
527 .{ 0b00101100, 0b00010001 },
528 .{ 0b10011110, 0b00001111 },
529 };
530
531 const static = struct {
532 fn doTheTest(src_arg: Src) !void {
533 const actual: Dest = @bitCast(src_arg);
534 for (actual, expected) |actual_vec, expected_vec| {
535 try expect(actual_vec[0] == expected_vec[0]);
536 try expect(actual_vec[1] == expected_vec[1]);
537 }
589 }538 }
590 };539 };
591540
592 try S.doTheTest();541 try static.doTheTest(src);
593 try comptime S.doTheTest();542 try comptime static.doTheTest(src);
594}543}
595544
596test "@bitCast of packed struct with void field to integer" {545test "@bitCast nested arrays of bool to scalar" {
597 const S = packed struct(u8) {546 const static = struct {
598 v: void,547 fn doTheTest(src: [4][4]bool) !void {
599 x: u8,548 const result: u16 = @bitCast(src);
600549 try expect(result == 0b1100_0101_1010_0011);
601 fn doTheTest(x: u8) !void {
602 // Intentionally using `@as` to avoid RLS which masks the bug
603 const foo = @as(@This(), .{ .v = {}, .x = x });
604 const as_int: u8 = @bitCast(foo);
605 try expect(as_int == x);
606 }550 }
607 };551 };
608 try S.doTheTest(123);552 const src: [4][4]bool = .{
609 try comptime S.doTheTest(123);553 .{ true, true, false, false }, // 0b0011
554 .{ false, true, false, true }, // 0b1010
555 .{ true, false, true, false }, // 0b0101
556 .{ false, false, true, true }, // 0b1100
557 };
558 try static.doTheTest(src);
559 try comptime static.doTheTest(src);
560}
561
562test "@bitCast deeply nested arrays to scalar" {
563 const static = struct {
564 fn doTheTest(src: [2][1][3][5]u4) !void {
565 const signed: i120 = @bitCast(src);
566 try expect(signed < 0); // top nibble is 0x8 so sign bit is 1
567 const unsigned: u120 = @bitCast(src);
568 try expect(unsigned == 0x8873B_5BF6F_F4020_0E7AC_1EFED_40F51);
569 try expect(@as(i120, @bitCast(unsigned)) == signed);
570 try expect(@as(u120, @bitCast(signed)) == unsigned);
571 }
572 };
573 const src: [2][1][3][5]u4 = .{ .{.{
574 .{ 0x1, 0x5, 0xF, 0x0, 0x4 },
575 .{ 0xD, 0xE, 0xF, 0xE, 0x1 },
576 .{ 0xC, 0xA, 0x7, 0xE, 0x0 },
577 }}, .{.{
578 .{ 0x0, 0x2, 0x0, 0x4, 0xF },
579 .{ 0xF, 0x6, 0xF, 0xB, 0x5 },
580 .{ 0xB, 0x3, 0x7, 0x8, 0x8 },
581 }} };
582 try static.doTheTest(src);
583 try comptime static.doTheTest(src);
610}584}
test/behavior/comptime_memory.zig+16
...@@ -583,3 +583,19 @@ test "comptime store to extern struct reinterpreted as byte array" {...@@ -583,3 +583,19 @@ test "comptime store to extern struct reinterpreted as byte array" {
583583
584 comptime std.debug.assert(val.x == 0);584 comptime std.debug.assert(val.x == 0);
585}585}
586
587test "reinterpret sentinel-terminated array as packed struct" {
588 const S = packed struct(u16) { lo: u8, hi: u8 };
589 const data: [2:0]u8 = .{ 0x12, 0x34 };
590 const ptr: *align(1) const S = @ptrCast(&data);
591 switch (endian) {
592 .little => {
593 try testing.expect(ptr.lo == 0x12);
594 try testing.expect(ptr.hi == 0x34);
595 },
596 .big => {
597 try testing.expect(ptr.lo == 0x34);
598 try testing.expect(ptr.hi == 0x12);
599 },
600 }
601}
test/behavior/sizeof_and_typeof.zig-11
...@@ -151,9 +151,6 @@ test "branching logic inside @TypeOf" {...@@ -151,9 +151,6 @@ test "branching logic inside @TypeOf" {
151test "@bitSizeOf" {151test "@bitSizeOf" {
152 try expect(@bitSizeOf(u2) == 2);152 try expect(@bitSizeOf(u2) == 2);
153 try expect(@bitSizeOf(u8) == @sizeOf(u8) * 8);153 try expect(@bitSizeOf(u8) == @sizeOf(u8) * 8);
154 try expect(@bitSizeOf(struct {
155 a: u2,
156 }) == 8);
157 try expect(@bitSizeOf(packed struct {154 try expect(@bitSizeOf(packed struct {
158 a: u2,155 a: u2,
159 }) == 2);156 }) == 2);
...@@ -281,14 +278,6 @@ test "@offsetOf zero-bit field" {...@@ -281,14 +278,6 @@ test "@offsetOf zero-bit field" {
281 try expect(@offsetOf(S, "b") == @offsetOf(S, "c"));278 try expect(@offsetOf(S, "b") == @offsetOf(S, "c"));
282}279}
283280
284test "@bitSizeOf on array of structs" {
285 const S = struct {
286 foo: u64,
287 };
288
289 try expectEqual(128, @bitSizeOf([2]S));
290}
291
292test "lazy abi size used in comparison" {281test "lazy abi size used in comparison" {
293 const S = struct { a: usize };282 const S = struct { a: usize };
294 var rhs: i32 = 100;283 var rhs: i32 = 100;
test/behavior/vector.zig+3-2
...@@ -1536,12 +1536,12 @@ test "vector pointer is indexable" {...@@ -1536,12 +1536,12 @@ test "vector pointer is indexable" {
1536 const V = @Vector(2, u32);1536 const V = @Vector(2, u32);
15371537
1538 const x: V = .{ 123, 456 };1538 const x: V = .{ 123, 456 };
1539 comptime assert(@TypeOf(&(&x)[0]) == *const u32); // validate constness1539 comptime assert(@typeInfo(@TypeOf(&(&x)[0])).pointer.attrs.@"const");
1540 try expectEqual(@as(u32, 123), (&x)[0]);1540 try expectEqual(@as(u32, 123), (&x)[0]);
1541 try expectEqual(@as(u32, 456), (&x)[1]);1541 try expectEqual(@as(u32, 456), (&x)[1]);
15421542
1543 var y: V = .{ 123, 456 };1543 var y: V = .{ 123, 456 };
1544 comptime assert(@TypeOf(&(&y)[0]) == *u32); // validate constness1544 comptime assert(!@typeInfo(@TypeOf(&(&y)[0])).pointer.attrs.@"const");
1545 try expectEqual(@as(u32, 123), (&y)[0]);1545 try expectEqual(@as(u32, 123), (&y)[0]);
1546 try expectEqual(@as(u32, 456), (&y)[1]);1546 try expectEqual(@as(u32, 456), (&y)[1]);
15471547
...@@ -1620,6 +1620,7 @@ test "bitcast vector to array of smaller vectors" {...@@ -1620,6 +1620,7 @@ test "bitcast vector to array of smaller vectors" {
1620 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;1620 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
1621 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO1621 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1622 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;1622 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1623 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
16231624
1624 const u8x32 = @Vector(32, u8);1625 const u8x32 = @Vector(32, u8);
1625 const u8x64 = @Vector(64, u8);1626 const u8x64 = @Vector(64, u8);
test/c_abi/main.zig+4-1
...@@ -14501,6 +14501,7 @@ test "@Vector(4, f64)" {...@@ -14501,6 +14501,7 @@ test "@Vector(4, f64)" {
14501 if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;14501 if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
14502 if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;14502 if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
14503 if (builtin.cpu.arch == .s390x) return error.SkipZigTest;14503 if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
14504 if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
1450414505
14505 const v = c_ret_vector_4_f64();14506 const v = c_ret_vector_4_f64();
14506 try expect(v[0] == 33);14507 try expect(v[0] == 33);
...@@ -14570,6 +14571,7 @@ test "@Vector(8, f64)" {...@@ -14570,6 +14571,7 @@ test "@Vector(8, f64)" {
14570 if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;14571 if (builtin.cpu.arch.isMIPS32()) return error.SkipZigTest;
14571 if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;14572 if (builtin.cpu.arch.isPowerPC64()) return error.SkipZigTest;
14572 if (builtin.cpu.arch == .s390x) return error.SkipZigTest;14573 if (builtin.cpu.arch == .s390x) return error.SkipZigTest;
14574 if (builtin.cpu.arch.isArm()) return error.SkipZigTest; // https://codeberg.org/ziglang/zig/issues/35899
1457314575
14574 const v = c_ret_vector_8_f64();14576 const v = c_ret_vector_8_f64();
14575 try expect(v[0] == 81);14577 try expect(v[0] == 81);
...@@ -17109,7 +17111,8 @@ const byval_tail_callsite_attr = struct {...@@ -17109,7 +17111,8 @@ const byval_tail_callsite_attr = struct {
17109 }17111 }
1711017112
17111 fn cast(self: MyRect) struct_Rect {17113 fn cast(self: MyRect) struct_Rect {
17112 return @bitCast(self);17114 const ptr: *const struct_Rect = @ptrCast(&self);
17115 return ptr.*;
17113 }17116 }
1711417117
17115 extern fn c_byval_tail_callsite_attr(struct_Rect) f64;17118 extern fn c_byval_tail_callsite_attr(struct_Rect) f64;
test/cases/compile_errors/asm_output_type_no_guaranteed_in_memory_layout.zig+28-4
...@@ -25,14 +25,38 @@ export fn entry4() void {...@@ -25,14 +25,38 @@ export fn entry4() void {
25 : [_] "=r" (u),25 : [_] "=r" (u),
26 );26 );
27}27}
28const ES = extern struct { x: u32 };
29export fn entry5() void {
30 var es: ES = undefined;
31 asm volatile (""
32 : [_] "=r" (es),
33 );
34}
35const EU = extern union { x: u32 };
36export fn entry6() void {
37 var eu: EU = undefined;
38 asm volatile (""
39 : [_] "=r" (eu),
40 );
41}
2842
29// error43// error
30//44//
31// :4:24: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout45// :4:24: error: invalid inline assembly output type 'tmp.S'
46// :4:24: note: struct types cannot be passed to inline assembly
32// :1:11: note: struct declared here47// :1:11: note: struct declared here
33// :11:21: error: invalid inline assembly output type; 'tmp.S' does not have a guaranteed in-memory layout48// :11:21: error: invalid inline assembly output type 'tmp.S'
49// :11:21: note: struct types cannot be passed to inline assembly
34// :1:11: note: struct declared here50// :1:11: note: struct declared here
35// :18:24: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout51// :18:24: error: invalid inline assembly output type 'tmp.U'
52// :18:24: note: union types cannot be passed to inline assembly
36// :15:11: note: union declared here53// :15:11: note: union declared here
37// :25:21: error: invalid inline assembly output type; 'tmp.U' does not have a guaranteed in-memory layout54// :25:21: error: invalid inline assembly output type 'tmp.U'
55// :25:21: note: union types cannot be passed to inline assembly
38// :15:11: note: union declared here56// :15:11: note: union declared here
57// :32:21: error: invalid inline assembly output type 'tmp.ES'
58// :32:21: note: struct types cannot be passed to inline assembly
59// :28:19: note: struct declared here
60// :39:21: error: invalid inline assembly output type 'tmp.EU'
61// :39:21: note: union types cannot be passed to inline assembly
62// :35:19: note: union declared here
test/cases/compile_errors/bitCast_extern_struct.zig created+10
...@@ -0,0 +1,10 @@
1const S = extern struct { x: u32 };
2export fn foo(s: S) void {
3 const as_int: u32 = @bitCast(s);
4 _ = as_int;
5}
6
7// error
8//
9// :3:34: error: cannot @bitCast from 'tmp.S'
10// :1:18: note: struct declared here
test/cases/compile_errors/bitCast_to_enum_type.zig deleted-10
...@@ -1,10 +0,0 @@
1export fn entry() void {
2 const E = enum(u32) { a, b };
3 const y: E = @bitCast(@as(u32, 3));
4 _ = y;
5}
6
7// error
8//
9// :3:18: error: cannot @bitCast to 'tmp.entry.E'
10// :3:18: note: use @enumFromInt to cast from 'u32'
test/cases/compile_errors/bitCast_vector_of_pointer.zig created+9
...@@ -0,0 +1,9 @@
1export fn foo(p: *u32) void {
2 const vec: @Vector(2, *u32) = .{ p, p };
3 const raw: [2]usize = @bitCast(vec);
4 _ = raw;
5}
6
7// error
8//
9// :3:36: error: cannot @bitCast from '@Vector(2, *u32)'
test/cases/compile_errors/bitCast_with_invalid_array_element_type.zig+2-5
...@@ -18,9 +18,6 @@ export fn baz() void {...@@ -18,9 +18,6 @@ export fn baz() void {
1818
19// error19// error
20//20//
21// :5:29: error: cannot @bitCast from '[1]tmp.foo.S'21// :5:42: error: cannot @bitCast from '[1]tmp.foo.S'
22// :5:29: note: array element type 'tmp.foo.S' does not have a guaranteed in-memory layout
23// :12:19: error: cannot @bitCast to '[1]tmp.bar.S'22// :12:19: error: cannot @bitCast to '[1]tmp.bar.S'
24// :12:19: note: array element type 'tmp.bar.S' does not have a guaranteed in-memory layout23// :16:45: error: cannot @bitCast from '[1]comptime_int'
25// :16:21: error: cannot @bitCast from '[1]comptime_int'
26// :16:21: note: array element type 'comptime_int' does not have a guaranteed in-memory layout
test/cases/compile_errors/error_in_nested_declaration.zig created+29
...@@ -0,0 +1,29 @@
1const S = struct {
2 b: u32,
3 c: i32,
4 a: struct {
5 pub fn str(_: @This(), extra: []u32) []i32 {
6 return @bitCast(extra);
7 }
8 },
9};
10
11pub export fn entry() void {
12 var s: S = undefined;
13 _ = s.a.str(undefined);
14}
15
16const S2 = struct {
17 a: [*c]anyopaque,
18};
19
20pub export fn entry2() void {
21 var s: S2 = undefined;
22 _ = &s;
23}
24
25// error
26//
27// :6:20: error: cannot @bitCast to '[]i32'
28// :6:20: note: use @ptrCast to cast from '[]u32'
29// :17:12: error: indexable pointer to opaque type 'anyopaque' not allowed
test/cases/error_in_nested_declaration.zig deleted-30
...@@ -1,30 +0,0 @@
1const S = struct {
2 b: u32,
3 c: i32,
4 a: struct {
5 pub fn str(_: @This(), extra: []u32) []i32 {
6 return @bitCast(extra);
7 }
8 },
9};
10
11pub export fn entry() void {
12 var s: S = undefined;
13 _ = s.a.str(undefined);
14}
15
16const S2 = struct {
17 a: [*c]anyopaque,
18};
19
20pub export fn entry2() void {
21 var s: S2 = undefined;
22 _ = &s;
23}
24
25// error
26// backend=selfhosted,llvm
27//
28// :6:20: error: cannot @bitCast to '[]i32'
29// :6:20: note: use @ptrCast to cast from '[]u32'
30// :17:12: error: indexable pointer to opaque type 'anyopaque' not allowed