Commit 181b25ce4fcebc32f6fdc7498148c0f5e131dda9 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 23:14:02+00:00
committer	GitHub <noreply@github.com> 2025-11-12 23:14:02+00:00
log	181b25ce4fcebc32f6fdc7498148c0f5e131dda9
tree	f5cfe981b1b158e8bac17cc3a3bc909ffb7b1aa1
parent	dfd7b7f2337d84ce660253a37079489f7780b055
parent	532aa3c5758f110eb7cf0992eb394088ab563899
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #25772 from mlugg/kill-dead-code

compiler: rewrite some legalizations, and remove a bunch of dead code

39 files changed, 1451 insertions(+), 3853 deletions(-)

CMakeLists.txt
lib
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - print.zig
  - types_resolved.zig
- InternPool.zig
- Sema.zig
- Sema
  - comptime_ptr_access.zig
- Type.zig
- Value.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
  - c.zig
  - c
    Type.zig
  - llvm.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
- link
  - Dwarf.zig
stage1
- zig.h
test
- behavior
  - union.zig

CMakeLists.txt+1-2

...	...	@@ -211,10 +211,10 @@ set(ZIG_STAGE2_SOURCES
211	211	lib/compiler_rt/absvti2.zig
212	212	lib/compiler_rt/adddf3.zig
213	213	lib/compiler_rt/addf3.zig
214		lib/compiler_rt/addo.zig
215	214	lib/compiler_rt/addsf3.zig
216	215	lib/compiler_rt/addtf3.zig
217	216	lib/compiler_rt/addvsi3.zig
	217	lib/compiler_rt/addvdi3.zig
218	218	lib/compiler_rt/addxf3.zig
219	219	lib/compiler_rt/arm.zig
220	220	lib/compiler_rt/atomics.zig
...	...	@@ -354,7 +354,6 @@ set(ZIG_STAGE2_SOURCES
354	354	lib/compiler_rt/sqrt.zig
355	355	lib/compiler_rt/stack_probe.zig
356	356	lib/compiler_rt/subdf3.zig
357		lib/compiler_rt/subo.zig
358	357	lib/compiler_rt/subsf3.zig
359	358	lib/compiler_rt/subtf3.zig
360	359	lib/compiler_rt/subvdi3.zig

lib/compiler_rt.zig+3-2

...	...	@@ -28,12 +28,13 @@ comptime {
28	28	_ = @import("compiler_rt/negv.zig");
29	29
30	30	_ = @import("compiler_rt/addvsi3.zig");
	31	_ = @import("compiler_rt/addvdi3.zig");
	32
31	33	_ = @import("compiler_rt/subvsi3.zig");
32	34	_ = @import("compiler_rt/subvdi3.zig");
	35
33	36	_ = @import("compiler_rt/mulvsi3.zig");
34	37
35		_ = @import("compiler_rt/addo.zig");
36		_ = @import("compiler_rt/subo.zig");
37	38	_ = @import("compiler_rt/mulo.zig");
38	39
39	40	// Float routines

lib/compiler_rt/addo.zig deleted-46

...	...	@@ -1,46 +0,0 @@
1		const std = @import("std");
2		const common = @import("./common.zig");
3		pub const panic = @import("common.zig").panic;
4
5		comptime {
6		@export(&__addosi4, .{ .name = "__addosi4", .linkage = common.linkage, .visibility = common.visibility });
7		@export(&__addodi4, .{ .name = "__addodi4", .linkage = common.linkage, .visibility = common.visibility });
8		@export(&__addoti4, .{ .name = "__addoti4", .linkage = common.linkage, .visibility = common.visibility });
9		}
10
11		// addo - add overflow
12		// * return a+%b.
13		// * return if a+b overflows => 1 else => 0
14		// - addoXi4_generic as default
15
16		inline fn addoXi4_generic(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST {
17		@setRuntimeSafety(common.test_safety);
18		overflow.* = 0;
19		const sum: ST = a +% b;
20		// Hackers Delight: section Overflow Detection, subsection Signed Add/Subtract
21		// Let sum = a +% b == a + b + carry == wraparound addition.
22		// Overflow in a+b+carry occurs, iff a and b have opposite signs
23		// and the sign of a+b+carry is the same as a (or equivalently b).
24		// Slower routine: res = ~(a ^ b) & ((sum ^ a)
25		// Faster routine: res = (sum ^ a) & (sum ^ b)
26		// Overflow occurred, iff (res < 0)
27		if (((sum ^ a) & (sum ^ b)) < 0)
28		overflow.* = 1;
29		return sum;
30		}
31
32		pub fn __addosi4(a: i32, b: i32, overflow: *c_int) callconv(.c) i32 {
33		return addoXi4_generic(i32, a, b, overflow);
34		}
35		pub fn __addodi4(a: i64, b: i64, overflow: *c_int) callconv(.c) i64 {
36		return addoXi4_generic(i64, a, b, overflow);
37		}
38		pub fn __addoti4(a: i128, b: i128, overflow: *c_int) callconv(.c) i128 {
39		return addoXi4_generic(i128, a, b, overflow);
40		}
41
42		test {
43		_ = @import("addosi4_test.zig");
44		_ = @import("addodi4_test.zig");
45		_ = @import("addoti4_test.zig");
46		}

lib/compiler_rt/addodi4_test.zig deleted-77

...	...	@@ -1,77 +0,0 @@
1		const addv = @import("addo.zig");
2		const std = @import("std");
3		const testing = std.testing;
4		const math = std.math;
5
6		fn test__addodi4(a: i64, b: i64) !void {
7		var result_ov: c_int = undefined;
8		var expected_ov: c_int = undefined;
9		const result = addv.__addodi4(a, b, &result_ov);
10		const expected: i64 = simple_addodi4(a, b, &expected_ov);
11		try testing.expectEqual(expected, result);
12		try testing.expectEqual(expected_ov, result_ov);
13		}
14
15		fn simple_addodi4(a: i64, b: i64, overflow: *c_int) i64 {
16		overflow.* = 0;
17		const min: i64 = math.minInt(i64);
18		const max: i64 = math.maxInt(i64);
19		if (((a > 0) and (b > max - a)) or
20		((a < 0) and (b < min - a)))
21		overflow.* = 1;
22		return a +% b;
23		}
24
25		test "addodi4" {
26		const min: i64 = math.minInt(i64);
27		const max: i64 = math.maxInt(i64);
28		var i: i64 = 1;
29		while (i < max) : (i *\|= 2) {
30		try test__addodi4(i, i);
31		try test__addodi4(-i, -i);
32		try test__addodi4(i, -i);
33		try test__addodi4(-i, i);
34		}
35
36		// edge cases
37		// 0 + 0 = 0
38		// MIN + MIN overflow
39		// MAX + MAX overflow
40		// 0 + MIN MIN
41		// 0 + MAX MAX
42		// MIN + 0 MIN
43		// MAX + 0 MAX
44		// MIN + MAX -1
45		// MAX + MIN -1
46		try test__addodi4(0, 0);
47		try test__addodi4(min, min);
48		try test__addodi4(max, max);
49		try test__addodi4(0, min);
50		try test__addodi4(0, max);
51		try test__addodi4(min, 0);
52		try test__addodi4(max, 0);
53		try test__addodi4(min, max);
54		try test__addodi4(max, min);
55
56		// derived edge cases
57		// MIN+1 + MIN overflow
58		// MAX-1 + MAX overflow
59		// 1 + MIN = MIN+1
60		// -1 + MIN overflow
61		// -1 + MAX = MAX-1
62		// +1 + MAX overflow
63		// MIN + 1 = MIN+1
64		// MIN + -1 overflow
65		// MAX + 1 overflow
66		// MAX + -1 = MAX-1
67		try test__addodi4(min + 1, min);
68		try test__addodi4(max - 1, max);
69		try test__addodi4(1, min);
70		try test__addodi4(-1, min);
71		try test__addodi4(-1, max);
72		try test__addodi4(1, max);
73		try test__addodi4(min, 1);
74		try test__addodi4(min, -1);
75		try test__addodi4(max, -1);
76		try test__addodi4(max, 1);
77		}

lib/compiler_rt/addosi4_test.zig deleted-78

...	...	@@ -1,78 +0,0 @@
1		const addv = @import("addo.zig");
2		const testing = @import("std").testing;
3
4		fn test__addosi4(a: i32, b: i32) !void {
5		var result_ov: c_int = undefined;
6		var expected_ov: c_int = undefined;
7		const result = addv.__addosi4(a, b, &result_ov);
8		const expected: i32 = simple_addosi4(a, b, &expected_ov);
9		try testing.expectEqual(expected, result);
10		try testing.expectEqual(expected_ov, result_ov);
11		}
12
13		fn simple_addosi4(a: i32, b: i32, overflow: *c_int) i32 {
14		overflow.* = 0;
15		const min: i32 = -2147483648;
16		const max: i32 = 2147483647;
17		if (((a > 0) and (b > max - a)) or
18		((a < 0) and (b < min - a)))
19		overflow.* = 1;
20		return a +% b;
21		}
22
23		test "addosi4" {
24		// -2^31 <= i32 <= 2^31-1
25		// 2^31 = 2147483648
26		// 2^31-1 = 2147483647
27		const min: i32 = -2147483648;
28		const max: i32 = 2147483647;
29		var i: i32 = 1;
30		while (i < max) : (i *\|= 2) {
31		try test__addosi4(i, i);
32		try test__addosi4(-i, -i);
33		try test__addosi4(i, -i);
34		try test__addosi4(-i, i);
35		}
36
37		// edge cases
38		// 0 + 0 = 0
39		// MIN + MIN overflow
40		// MAX + MAX overflow
41		// 0 + MIN MIN
42		// 0 + MAX MAX
43		// MIN + 0 MIN
44		// MAX + 0 MAX
45		// MIN + MAX -1
46		// MAX + MIN -1
47		try test__addosi4(0, 0);
48		try test__addosi4(min, min);
49		try test__addosi4(max, max);
50		try test__addosi4(0, min);
51		try test__addosi4(0, max);
52		try test__addosi4(min, 0);
53		try test__addosi4(max, 0);
54		try test__addosi4(min, max);
55		try test__addosi4(max, min);
56
57		// derived edge cases
58		// MIN+1 + MIN overflow
59		// MAX-1 + MAX overflow
60		// 1 + MIN = MIN+1
61		// -1 + MIN overflow
62		// -1 + MAX = MAX-1
63		// +1 + MAX overflow
64		// MIN + 1 = MIN+1
65		// MIN + -1 overflow
66		// MAX + 1 overflow
67		// MAX + -1 = MAX-1
68		try test__addosi4(min + 1, min);
69		try test__addosi4(max - 1, max);
70		try test__addosi4(1, min);
71		try test__addosi4(-1, min);
72		try test__addosi4(-1, max);
73		try test__addosi4(1, max);
74		try test__addosi4(min, 1);
75		try test__addosi4(min, -1);
76		try test__addosi4(max, -1);
77		try test__addosi4(max, 1);
78		}

lib/compiler_rt/addoti4_test.zig deleted-80

...	...	@@ -1,80 +0,0 @@
1		const addv = @import("addo.zig");
2		const builtin = @import("builtin");
3		const std = @import("std");
4		const testing = std.testing;
5		const math = std.math;
6
7		fn test__addoti4(a: i128, b: i128) !void {
8		var result_ov: c_int = undefined;
9		var expected_ov: c_int = undefined;
10		const result = addv.__addoti4(a, b, &result_ov);
11		const expected: i128 = simple_addoti4(a, b, &expected_ov);
12		try testing.expectEqual(expected, result);
13		try testing.expectEqual(expected_ov, result_ov);
14		}
15
16		fn simple_addoti4(a: i128, b: i128, overflow: *c_int) i128 {
17		overflow.* = 0;
18		const min: i128 = math.minInt(i128);
19		const max: i128 = math.maxInt(i128);
20		if (((a > 0) and (b > max - a)) or
21		((a < 0) and (b < min - a)))
22		overflow.* = 1;
23		return a +% b;
24		}
25
26		test "addoti4" {
27		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
28
29		const min: i128 = math.minInt(i128);
30		const max: i128 = math.maxInt(i128);
31		var i: i128 = 1;
32		while (i < max) : (i *\|= 2) {
33		try test__addoti4(i, i);
34		try test__addoti4(-i, -i);
35		try test__addoti4(i, -i);
36		try test__addoti4(-i, i);
37		}
38
39		// edge cases
40		// 0 + 0 = 0
41		// MIN + MIN overflow
42		// MAX + MAX overflow
43		// 0 + MIN MIN
44		// 0 + MAX MAX
45		// MIN + 0 MIN
46		// MAX + 0 MAX
47		// MIN + MAX -1
48		// MAX + MIN -1
49		try test__addoti4(0, 0);
50		try test__addoti4(min, min);
51		try test__addoti4(max, max);
52		try test__addoti4(0, min);
53		try test__addoti4(0, max);
54		try test__addoti4(min, 0);
55		try test__addoti4(max, 0);
56		try test__addoti4(min, max);
57		try test__addoti4(max, min);
58
59		// derived edge cases
60		// MIN+1 + MIN overflow
61		// MAX-1 + MAX overflow
62		// 1 + MIN = MIN+1
63		// -1 + MIN overflow
64		// -1 + MAX = MAX-1
65		// +1 + MAX overflow
66		// MIN + 1 = MIN+1
67		// MIN + -1 overflow
68		// MAX + 1 overflow
69		// MAX + -1 = MAX-1
70		try test__addoti4(min + 1, min);
71		try test__addoti4(max - 1, max);
72		try test__addoti4(1, min);
73		try test__addoti4(-1, min);
74		try test__addoti4(-1, max);
75		try test__addoti4(1, max);
76		try test__addoti4(min, 1);
77		try test__addoti4(min, -1);
78		try test__addoti4(max, -1);
79		try test__addoti4(max, 1);
80		}

lib/compiler_rt/addvdi3.zig created+26

...	...	@@ -0,0 +1,26 @@
	1	const common = @import("./common.zig");
	2	const testing = @import("std").testing;
	3
	4	pub const panic = common.panic;
	5
	6	comptime {
	7	@export(&__addvdi3, .{ .name = "__addvdi3", .linkage = common.linkage, .visibility = common.visibility });
	8	}
	9
	10	pub fn __addvdi3(a: i64, b: i64) callconv(.c) i64 {
	11	const sum = a +% b;
	12	// Overflow occurred iff both operands have the same sign, and the sign of the sum does
	13	// not match it. In other words, iff the sum sign is not the sign of either operand.
	14	if (((sum ^ a) & (sum ^ b)) < 0) @panic("compiler-rt: integer overflow");
	15	return sum;
	16	}
	17
	18	test "addvdi3" {
	19	// const min: i64 = -9223372036854775808
	20	// const max: i64 = 9223372036854775807
	21	// TODO write panic handler for testing panics
	22	// try test__addvdi3(-9223372036854775808, -1, -1); // panic
	23	// try test__addvdi3(9223372036854775807, 1, 1); // panic
	24	try testing.expectEqual(-9223372036854775808, __addvdi3(-9223372036854775807, -1));
	25	try testing.expectEqual(9223372036854775807, __addvdi3(9223372036854775806, 1));
	26	}

lib/compiler_rt/addvsi3.zig+4-4

...	...	@@ -1,4 +1,3 @@
1		const addv = @import("addo.zig");
2	1	const common = @import("./common.zig");
3	2	const testing = @import("std").testing;
4	3
...	...	@@ -9,9 +8,10 @@ comptime {
9	8	}
10	9
11	10	pub fn __addvsi3(a: i32, b: i32) callconv(.c) i32 {
12		var overflow: c_int = 0;
13		const sum = addv.__addosi4(a, b, &overflow);
14		if (overflow != 0) @panic("compiler-rt: integer overflow");
	11	const sum = a +% b;
	12	// Overflow occurred iff both operands have the same sign, and the sign of the sum does
	13	// not match it. In other words, iff the sum sign is not the sign of either operand.
	14	if (((sum ^ a) & (sum ^ b)) < 0) @panic("compiler-rt: integer overflow");
15	15	return sum;
16	16	}
17	17

lib/compiler_rt/subo.zig deleted-47

...	...	@@ -1,47 +0,0 @@
1		//! subo - subtract overflow
2		//! * return a-%b.
3		//! * return if a-b overflows => 1 else => 0
4		//! - suboXi4_generic as default
5
6		const std = @import("std");
7		const builtin = @import("builtin");
8		const common = @import("common.zig");
9
10		pub const panic = common.panic;
11
12		comptime {
13		@export(&__subosi4, .{ .name = "__subosi4", .linkage = common.linkage, .visibility = common.visibility });
14		@export(&__subodi4, .{ .name = "__subodi4", .linkage = common.linkage, .visibility = common.visibility });
15		@export(&__suboti4, .{ .name = "__suboti4", .linkage = common.linkage, .visibility = common.visibility });
16		}
17
18		pub fn __subosi4(a: i32, b: i32, overflow: *c_int) callconv(.c) i32 {
19		return suboXi4_generic(i32, a, b, overflow);
20		}
21		pub fn __subodi4(a: i64, b: i64, overflow: *c_int) callconv(.c) i64 {
22		return suboXi4_generic(i64, a, b, overflow);
23		}
24		pub fn __suboti4(a: i128, b: i128, overflow: *c_int) callconv(.c) i128 {
25		return suboXi4_generic(i128, a, b, overflow);
26		}
27
28		inline fn suboXi4_generic(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST {
29		overflow.* = 0;
30		const sum: ST = a -% b;
31		// Hackers Delight: section Overflow Detection, subsection Signed Add/Subtract
32		// Let sum = a -% b == a - b - carry == wraparound subtraction.
33		// Overflow in a-b-carry occurs, iff a and b have opposite signs
34		// and the sign of a-b-carry is opposite of a (or equivalently same as b).
35		// Faster routine: res = (a ^ b) & (sum ^ a)
36		// Slower routine: res = (sum^a) & ~(sum^b)
37		// Overflow occurred, iff (res < 0)
38		if (((a ^ b) & (sum ^ a)) < 0)
39		overflow.* = 1;
40		return sum;
41		}
42
43		test {
44		_ = @import("subosi4_test.zig");
45		_ = @import("subodi4_test.zig");
46		_ = @import("suboti4_test.zig");
47		}

lib/compiler_rt/subodi4_test.zig deleted-81

...	...	@@ -1,81 +0,0 @@
1		const subo = @import("subo.zig");
2		const std = @import("std");
3		const testing = std.testing;
4		const math = std.math;
5
6		fn test__subodi4(a: i64, b: i64) !void {
7		var result_ov: c_int = undefined;
8		var expected_ov: c_int = undefined;
9		const result = subo.__subodi4(a, b, &result_ov);
10		const expected: i64 = simple_subodi4(a, b, &expected_ov);
11		try testing.expectEqual(expected, result);
12		try testing.expectEqual(expected_ov, result_ov);
13		}
14
15		// 2 cases on evaluating `a-b`:
16		// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
17		// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
18		// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
19		pub fn simple_subodi4(a: i64, b: i64, overflow: *c_int) i64 {
20		overflow.* = 0;
21		const min: i64 = math.minInt(i64);
22		const max: i64 = math.maxInt(i64);
23		if (((b > 0) and (a < min + b)) or
24		((b < 0) and (a > max + b)))
25		overflow.* = 1;
26		return a -% b;
27		}
28
29		test "subodi3" {
30		const min: i64 = math.minInt(i64);
31		const max: i64 = math.maxInt(i64);
32		var i: i64 = 1;
33		while (i < max) : (i *\|= 2) {
34		try test__subodi4(i, i);
35		try test__subodi4(-i, -i);
36		try test__subodi4(i, -i);
37		try test__subodi4(-i, i);
38		}
39
40		// edge cases
41		// 0 - 0 = 0
42		// MIN - MIN = 0
43		// MAX - MAX = 0
44		// 0 - MIN overflow
45		// 0 - MAX = MIN+1
46		// MIN - 0 = MIN
47		// MAX - 0 = MAX
48		// MIN - MAX overflow
49		// MAX - MIN overflow
50		try test__subodi4(0, 0);
51		try test__subodi4(min, min);
52		try test__subodi4(max, max);
53		try test__subodi4(0, min);
54		try test__subodi4(0, max);
55		try test__subodi4(min, 0);
56		try test__subodi4(max, 0);
57		try test__subodi4(min, max);
58		try test__subodi4(max, min);
59
60		// derived edge cases
61		// MIN+1 - MIN = 1
62		// MAX-1 - MAX = -1
63		// 1 - MIN overflow
64		// -1 - MIN = MAX
65		// -1 - MAX = MIN
66		// +1 - MAX = MIN+2
67		// MIN - 1 overflow
68		// MIN - -1 = MIN+1
69		// MAX - 1 = MAX-1
70		// MAX - -1 overflow
71		try test__subodi4(min + 1, min);
72		try test__subodi4(max - 1, max);
73		try test__subodi4(1, min);
74		try test__subodi4(-1, min);
75		try test__subodi4(-1, max);
76		try test__subodi4(1, max);
77		try test__subodi4(min, 1);
78		try test__subodi4(min, -1);
79		try test__subodi4(max, -1);
80		try test__subodi4(max, 1);
81		}

lib/compiler_rt/subosi4_test.zig deleted-82

...	...	@@ -1,82 +0,0 @@
1		const subo = @import("subo.zig");
2		const testing = @import("std").testing;
3
4		fn test__subosi4(a: i32, b: i32) !void {
5		var result_ov: c_int = undefined;
6		var expected_ov: c_int = undefined;
7		const result = subo.__subosi4(a, b, &result_ov);
8		const expected: i32 = simple_subosi4(a, b, &expected_ov);
9		try testing.expectEqual(expected, result);
10		try testing.expectEqual(expected_ov, result_ov);
11		}
12
13		// 2 cases on evaluating `a-b`:
14		// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
15		// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
16		// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
17		pub fn simple_subosi4(a: i32, b: i32, overflow: *c_int) i32 {
18		overflow.* = 0;
19		const min: i32 = -2147483648;
20		const max: i32 = 2147483647;
21		if (((b > 0) and (a < min + b)) or
22		((b < 0) and (a > max + b)))
23		overflow.* = 1;
24		return a -% b;
25		}
26
27		test "subosi3" {
28		// -2^31 <= i32 <= 2^31-1
29		// 2^31 = 2147483648
30		// 2^31-1 = 2147483647
31		const min: i32 = -2147483648;
32		const max: i32 = 2147483647;
33		var i: i32 = 1;
34		while (i < max) : (i *\|= 2) {
35		try test__subosi4(i, i);
36		try test__subosi4(-i, -i);
37		try test__subosi4(i, -i);
38		try test__subosi4(-i, i);
39		}
40
41		// edge cases
42		// 0 - 0 = 0
43		// MIN - MIN = 0
44		// MAX - MAX = 0
45		// 0 - MIN overflow
46		// 0 - MAX = MIN+1
47		// MIN - 0 = MIN
48		// MAX - 0 = MAX
49		// MIN - MAX overflow
50		// MAX - MIN overflow
51		try test__subosi4(0, 0);
52		try test__subosi4(min, min);
53		try test__subosi4(max, max);
54		try test__subosi4(0, min);
55		try test__subosi4(0, max);
56		try test__subosi4(min, 0);
57		try test__subosi4(max, 0);
58		try test__subosi4(min, max);
59		try test__subosi4(max, min);
60
61		// derived edge cases
62		// MIN+1 - MIN = 1
63		// MAX-1 - MAX = -1
64		// 1 - MIN overflow
65		// -1 - MIN = MAX
66		// -1 - MAX = MIN
67		// +1 - MAX = MIN+2
68		// MIN - 1 overflow
69		// MIN - -1 = MIN+1
70		// MAX - 1 = MAX-1
71		// MAX - -1 overflow
72		try test__subosi4(min + 1, min);
73		try test__subosi4(max - 1, max);
74		try test__subosi4(1, min);
75		try test__subosi4(-1, min);
76		try test__subosi4(-1, max);
77		try test__subosi4(1, max);
78		try test__subosi4(min, 1);
79		try test__subosi4(min, -1);
80		try test__subosi4(max, -1);
81		try test__subosi4(max, 1);
82		}

lib/compiler_rt/suboti4_test.zig deleted-84

...	...	@@ -1,84 +0,0 @@
1		const subo = @import("subo.zig");
2		const builtin = @import("builtin");
3		const std = @import("std");
4		const testing = std.testing;
5		const math = std.math;
6
7		fn test__suboti4(a: i128, b: i128) !void {
8		var result_ov: c_int = undefined;
9		var expected_ov: c_int = undefined;
10		const result = subo.__suboti4(a, b, &result_ov);
11		const expected: i128 = simple_suboti4(a, b, &expected_ov);
12		try testing.expectEqual(expected, result);
13		try testing.expectEqual(expected_ov, result_ov);
14		}
15
16		// 2 cases on evaluating `a-b`:
17		// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
18		// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
19		// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
20		pub fn simple_suboti4(a: i128, b: i128, overflow: *c_int) i128 {
21		overflow.* = 0;
22		const min: i128 = math.minInt(i128);
23		const max: i128 = math.maxInt(i128);
24		if (((b > 0) and (a < min + b)) or
25		((b < 0) and (a > max + b)))
26		overflow.* = 1;
27		return a -% b;
28		}
29
30		test "suboti3" {
31		if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
32
33		const min: i128 = math.minInt(i128);
34		const max: i128 = math.maxInt(i128);
35		var i: i128 = 1;
36		while (i < max) : (i *\|= 2) {
37		try test__suboti4(i, i);
38		try test__suboti4(-i, -i);
39		try test__suboti4(i, -i);
40		try test__suboti4(-i, i);
41		}
42
43		// edge cases
44		// 0 - 0 = 0
45		// MIN - MIN = 0
46		// MAX - MAX = 0
47		// 0 - MIN overflow
48		// 0 - MAX = MIN+1
49		// MIN - 0 = MIN
50		// MAX - 0 = MAX
51		// MIN - MAX overflow
52		// MAX - MIN overflow
53		try test__suboti4(0, 0);
54		try test__suboti4(min, min);
55		try test__suboti4(max, max);
56		try test__suboti4(0, min);
57		try test__suboti4(0, max);
58		try test__suboti4(min, 0);
59		try test__suboti4(max, 0);
60		try test__suboti4(min, max);
61		try test__suboti4(max, min);
62
63		// derived edge cases
64		// MIN+1 - MIN = 1
65		// MAX-1 - MAX = -1
66		// 1 - MIN overflow
67		// -1 - MIN = MAX
68		// -1 - MAX = MIN
69		// +1 - MAX = MIN+2
70		// MIN - 1 overflow
71		// MIN - -1 = MIN+1
72		// MAX - 1 = MAX-1
73		// MAX - -1 overflow
74		try test__suboti4(min + 1, min);
75		try test__suboti4(max - 1, max);
76		try test__suboti4(1, min);
77		try test__suboti4(-1, min);
78		try test__suboti4(-1, max);
79		try test__suboti4(1, max);
80		try test__suboti4(min, 1);
81		try test__suboti4(min, -1);
82		try test__suboti4(max, -1);
83		try test__suboti4(max, 1);
84		}

lib/compiler_rt/subvdi3.zig+4-4

...	...	@@ -1,4 +1,3 @@
1		const subv = @import("subo.zig");
2	1	const common = @import("./common.zig");
3	2	const testing = @import("std").testing;
4	3
...	...	@@ -9,9 +8,10 @@ comptime {
9	8	}
10	9
11	10	pub fn __subvdi3(a: i64, b: i64) callconv(.c) i64 {
12		var overflow: c_int = 0;
13		const sum = subv.__subodi4(a, b, &overflow);
14		if (overflow != 0) @panic("compiler-rt: integer overflow");
	11	const sum = a -% b;
	12	// Overflow occurred iff the operands have opposite signs, and the sign of the
	13	// sum is the opposite of the lhs sign.
	14	if (((a ^ b) & (sum ^ a)) < 0) @panic("compiler-rt: integer overflow");
15	15	return sum;
16	16	}
17	17

lib/compiler_rt/subvsi3.zig+4-4

...	...	@@ -1,4 +1,3 @@
1		const subv = @import("subo.zig");
2	1	const common = @import("./common.zig");
3	2	const testing = @import("std").testing;
4	3
...	...	@@ -9,9 +8,10 @@ comptime {
9	8	}
10	9
11	10	pub fn __subvsi3(a: i32, b: i32) callconv(.c) i32 {
12		var overflow: c_int = 0;
13		const sum = subv.__subosi4(a, b, &overflow);
14		if (overflow != 0) @panic("compiler-rt: integer overflow");
	11	const sum = a -% b;
	12	// Overflow occurred iff the operands have opposite signs, and the sign of the
	13	// sum is the opposite of the lhs sign.
	14	if (((a ^ b) & (sum ^ a)) < 0) @panic("compiler-rt: integer overflow");
15	15	return sum;
16	16	}
17	17

lib/zig.h+12-24

...	...	@@ -809,15 +809,13 @@ static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
809	809	#endif
810	810	}
811	811
812		zig_extern int32_t __addosi4(int32_t lhs, int32_t rhs, int *overflow);
813	812	static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
814	813	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
815	814	int32_t full_res;
816	815	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
817	816	#else
818		int overflow_int;
819		int32_t full_res = __addosi4(lhs, rhs, &overflow_int);
820		bool overflow = overflow_int != 0;
	817	int32_t full_res = (int32_t)((uint32_t)lhs + (uint32_t)rhs);
	818	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
821	819	#endif
822	820	*res = zig_wrap_i32(full_res, bits);
823	821	return overflow \|\| full_res < zig_minInt_i(32, bits) \|\| full_res > zig_maxInt_i(32, bits);
...	...	@@ -835,15 +833,13 @@ static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
835	833	#endif
836	834	}
837	835
838		zig_extern int64_t __addodi4(int64_t lhs, int64_t rhs, int *overflow);
839	836	static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
840	837	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
841	838	int64_t full_res;
842	839	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
843	840	#else
844		int overflow_int;
845		int64_t full_res = __addodi4(lhs, rhs, &overflow_int);
846		bool overflow = overflow_int != 0;
	841	int64_t full_res = (int64_t)((uint64_t)lhs + (uint64_t)rhs);
	842	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
847	843	#endif
848	844	*res = zig_wrap_i64(full_res, bits);
849	845	return overflow \|\| full_res < zig_minInt_i(64, bits) \|\| full_res > zig_maxInt_i(64, bits);
...	...	@@ -917,15 +913,13 @@ static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
917	913	#endif
918	914	}
919	915
920		zig_extern int32_t __subosi4(int32_t lhs, int32_t rhs, int *overflow);
921	916	static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
922	917	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
923	918	int32_t full_res;
924	919	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
925	920	#else
926		int overflow_int;
927		int32_t full_res = __subosi4(lhs, rhs, &overflow_int);
928		bool overflow = overflow_int != 0;
	921	int32_t full_res = (int32_t)((uint32_t)lhs - (uint32_t)rhs);
	922	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
929	923	#endif
930	924	*res = zig_wrap_i32(full_res, bits);
931	925	return overflow \|\| full_res < zig_minInt_i(32, bits) \|\| full_res > zig_maxInt_i(32, bits);
...	...	@@ -943,15 +937,13 @@ static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
943	937	#endif
944	938	}
945	939
946		zig_extern int64_t __subodi4(int64_t lhs, int64_t rhs, int *overflow);
947	940	static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
948	941	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
949	942	int64_t full_res;
950	943	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
951	944	#else
952		int overflow_int;
953		int64_t full_res = __subodi4(lhs, rhs, &overflow_int);
954		bool overflow = overflow_int != 0;
	945	int64_t full_res = (int64_t)((uint64_t)lhs - (uint64_t)rhs);
	946	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
955	947	#endif
956	948	*res = zig_wrap_i64(full_res, bits);
957	949	return overflow \|\| full_res < zig_minInt_i(64, bits) \|\| full_res > zig_maxInt_i(64, bits);
...	...	@@ -1755,15 +1747,13 @@ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
1755	1747	#endif
1756	1748	}
1757	1749
1758		zig_extern zig_i128 __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
1759	1750	static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
1760	1751	#if zig_has_builtin(add_overflow)
1761	1752	zig_i128 full_res;
1762	1753	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
1763	1754	#else
1764		int overflow_int;
1765		zig_i128 full_res = __addoti4(lhs, rhs, &overflow_int);
1766		bool overflow = overflow_int != 0;
	1755	zig_i128 full_res = (zig_i128)((zig_u128)lhs + (zig_u128)rhs);
	1756	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
1767	1757	#endif
1768	1758	*res = zig_wrap_i128(full_res, bits);
1769	1759	return overflow \|\| full_res < zig_minInt_i(128, bits) \|\| full_res > zig_maxInt_i(128, bits);
...	...	@@ -1781,15 +1771,13 @@ static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
1781	1771	#endif
1782	1772	}
1783	1773
1784		zig_extern zig_i128 __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
1785	1774	static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
1786	1775	#if zig_has_builtin(sub_overflow)
1787	1776	zig_i128 full_res;
1788	1777	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
1789	1778	#else
1790		int overflow_int;
1791		zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int);
1792		bool overflow = overflow_int != 0;
	1779	zig_i128 full_res = (zig_i128)((zig_u128)lhs - (zig_u128)rhs);
	1780	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
1793	1781	#endif
1794	1782	*res = zig_wrap_i128(full_res, bits);
1795	1783	return overflow \|\| full_res < zig_minInt_i(128, bits) \|\| full_res > zig_maxInt_i(128, bits);

src/Air.zig+26-14

...	...	@@ -660,8 +660,8 @@ pub const Inst = struct {
660	660	/// Given a pointer to a slice, return a pointer to the pointer of the slice.
661	661	/// Uses the `ty_op` field.
662	662	ptr_slice_ptr_ptr,
663		/// Given an (array value or vector value) and element index,
664		/// return the element value at that index.
	663	/// Given an (array value or vector value) and element index, return the element value at
	664	/// that index. If the lhs is a vector value, the index is guaranteed to be comptime-known.
665	665	/// Result type is the element type of the array operand.
666	666	/// Uses the `bin_op` field.
667	667	array_elem_val,
...	...	@@ -874,10 +874,6 @@ pub const Inst = struct {
874	874	/// Uses the `ty_pl` field.
875	875	save_err_return_trace_index,
876	876
877		/// Store an element to a vector pointer at an index.
878		/// Uses the `vector_store_elem` field.
879		vector_store_elem,
880
881	877	/// Compute a pointer to a `Nav` at runtime, always one of:
882	878	///
883	879	/// * `threadlocal var`
...	...	@@ -919,6 +915,26 @@ pub const Inst = struct {
919	915	/// Operand is unused and set to Ref.none
920	916	work_group_id,
921	917
	918	// The remaining instructions are not emitted by Sema. They are only emitted by `Legalize`,
	919	// depending on the enabled features. As such, backends can consider them `unreachable` if
	920	// they do not enable the relevant legalizations.
	921
	922	/// Given a pointer to a vector, a runtime-known index, and a scalar value, store the value
	923	/// into the vector at the given index. Zig does not support this operation, but `Legalize`
	924	/// may emit it when scalarizing vector operations.
	925	///
	926	/// Uses the `pl_op` field with payload `Bin`. `operand` is the vector pointer. `lhs` is the
	927	/// element index of type `usize`. `rhs` is the element value. Result is always void.
	928	legalize_vec_store_elem,
	929	/// Given a vector value and a runtime-known index, return the element value at that index.
	930	/// This instruction is similar to `array_elem_val`; the only difference is that the index
	931	/// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
	932	/// this instruction when scalarizing vector operations.
	933	///
	934	/// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
	935	/// type is the vector element type.
	936	legalize_vec_elem_val,
	937
922	938	pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
923	939	switch (op) {
924	940	.lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
...	...	@@ -1220,11 +1236,6 @@ pub const Inst = struct {
1220	1236	operand: Ref,
1221	1237	operation: std.builtin.ReduceOp,
1222	1238	},
1223		vector_store_elem: struct {
1224		vector_ptr: Ref,
1225		// Index into a different array.
1226		payload: u32,
1227		},
1228	1239	ty_nav: struct {
1229	1240	ty: InternPool.Index,
1230	1241	nav: InternPool.Nav.Index,
...	...	@@ -1689,8 +1700,8 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1689	1700	.set_union_tag,
1690	1701	.prefetch,
1691	1702	.set_err_return_trace,
1692		.vector_store_elem,
1693	1703	.c_va_end,
	1704	.legalize_vec_store_elem,
1694	1705	=> return .void,
1695	1706
1696	1707	.slice_len,
...	...	@@ -1709,7 +1720,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1709	1720	return .fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));
1710	1721	},
1711	1722
1712		.slice_elem_val, .ptr_elem_val, .array_elem_val => {
	1723	.slice_elem_val, .ptr_elem_val, .array_elem_val, .legalize_vec_elem_val => {
1713	1724	const ptr_ty = air.typeOf(datas[@intFromEnum(inst)].bin_op.lhs, ip);
1714	1725	return ptr_ty.childTypeIp(ip);
1715	1726	},
...	...	@@ -1857,7 +1868,6 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1857	1868	.prefetch,
1858	1869	.wasm_memory_grow,
1859	1870	.set_err_return_trace,
1860		.vector_store_elem,
1861	1871	.c_va_arg,
1862	1872	.c_va_copy,
1863	1873	.c_va_end,
...	...	@@ -1868,6 +1878,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1868	1878	.intcast_safe,
1869	1879	.int_from_float_safe,
1870	1880	.int_from_float_optimized_safe,
	1881	.legalize_vec_store_elem,
1871	1882	=> true,
1872	1883
1873	1884	.add,
...	...	@@ -2013,6 +2024,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2013	2024	.work_item_id,
2014	2025	.work_group_size,
2015	2026	.work_group_id,
	2027	.legalize_vec_elem_val,
2016	2028	=> false,
2017	2029
2018	2030	.is_non_null_ptr, .is_null_ptr, .is_non_err_ptr, .is_err_ptr => air.typeOf(data.un_op, ip).isVolatilePtrIp(ip),

src/Air/Legalize.zig+989-1362

...	...	@@ -14,7 +14,7 @@ features: if (switch (dev.env) {
14	14	return comptime bootstrap_features.contains(feature);
15	15	}
16	16	/// `inline` to propagate comptime-known result.
17		fn hasAny(_: @This(), comptime features: []const Feature) bool {
	17	inline fn hasAny(_: @This(), comptime features: []const Feature) bool {
18	18	return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());
19	19	}
20	20	} else struct {
...	...	@@ -154,9 +154,9 @@ pub const Feature = enum {
154	154	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
155	155	/// first byte of memory until bit pointers know their backing type.
156	156	expand_packed_store,
157		/// Replace `struct_field_val` of a packed field with a `store` and packed `load`.
	157	/// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
158	158	expand_packed_struct_field_val,
159		/// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`.
	159	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
160	160	expand_packed_aggregate_init,
161	161
162	162	fn scalarize(tag: Air.Inst.Tag) Feature {
...	...	@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
320	320	.xor,
321	321	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
322	322	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
323		if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
	323	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
	324	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
	325	}
324	326	},
325	327	.add_safe => if (l.features.has(.expand_add_safe)) {
326	328	assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
327	329	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
328	330	} else if (l.features.has(.scalarize_add_safe)) {
329	331	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
330		if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
	332	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
	333	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
	334	}
331	335	},
332	336	.sub_safe => if (l.features.has(.expand_sub_safe)) {
333	337	assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
334	338	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
335	339	} else if (l.features.has(.scalarize_sub_safe)) {
336	340	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
337		if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
	341	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
	342	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
	343	}
338	344	},
339	345	.mul_safe => if (l.features.has(.expand_mul_safe)) {
340	346	assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
341	347	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
342	348	} else if (l.features.has(.scalarize_mul_safe)) {
343	349	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
344		if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);
	350	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
	351	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
	352	}
345	353	},
346	354	.ptr_add, .ptr_sub => {},
347	355	inline .add_with_overflow,
...	...	@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
350	358	.shl_with_overflow,
351	359	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
352	360	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
353		if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
	361	if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) {
	362	continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
	363	}
354	364	},
355	365	.alloc => {},
356	366	.inferred_alloc, .inferred_alloc_comptime => unreachable,
...	...	@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
387	397	}
388	398	}
389	399	}
390		if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);
	400	if (l.features.has(comptime .scalarize(air_tag))) {
	401	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
	402	}
391	403	}
392	404	},
393	405	inline .not,
...	...	@@ -406,64 +418,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
406	418	.float_from_int,
407	419	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
408	420	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
409		if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
	421	if (ty_op.ty.toType().isVector(zcu)) {
	422	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
	423	}
410	424	},
411	425	.bitcast => if (l.features.has(.scalarize_bitcast)) {
412		const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
413
414		const to_ty = ty_op.ty.toType();
415		const to_ty_tag = to_ty.zigTypeTag(zcu);
416		const to_ty_legal = legal: switch (to_ty_tag) {
417		else => true,
418		.array, .vector => {
419		if (to_ty.arrayLen(zcu) == 1) break :legal true;
420		const to_elem_ty = to_ty.childType(zcu);
421		break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu);
422		},
423		};
424
425		const from_ty = l.typeOf(ty_op.operand);
426		const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) {
427		else => true,
428		.array, .vector => {
429		if (from_ty.arrayLen(zcu) == 1) break :legal true;
430		const from_elem_ty = from_ty.childType(zcu);
431		break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu);
432		},
433		};
434
435		if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) {
436		else => unreachable,
437		.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)),
438		.vector => continue :inst try l.scalarize(inst, .bitcast),
439		};
440		if (!to_ty_legal) switch (to_ty_tag) {
441		else => unreachable,
442		.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)),
443		.vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)),
444		};
445		if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst));
	426	if (try l.scalarizeBitcastBlockPayload(inst)) \|payload\| {
	427	continue :inst l.replaceInst(inst, .block, payload);
	428	}
446	429	},
447	430	.intcast_safe => if (l.features.has(.expand_intcast_safe)) {
448	431	assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
449	432	continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
450	433	} else if (l.features.has(.scalarize_intcast_safe)) {
451	434	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
452		if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
	435	if (ty_op.ty.toType().isVector(zcu)) {
	436	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
	437	}
453	438	},
454	439	.int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {
455	440	assert(!l.features.has(.scalarize_int_from_float_safe));
456	441	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));
457	442	} else if (l.features.has(.scalarize_int_from_float_safe)) {
458	443	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
459		if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
	444	if (ty_op.ty.toType().isVector(zcu)) {
	445	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
	446	}
460	447	},
461	448	.int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {
462	449	assert(!l.features.has(.scalarize_int_from_float_optimized_safe));
463	450	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));
464	451	} else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {
465	452	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
466		if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);
	453	if (ty_op.ty.toType().isVector(zcu)) {
	454	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
	455	}
467	456	},
468	457	.block, .loop => {
469	458	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	...	@@ -498,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
498	487	.neg_optimized,
499	488	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
500	489	const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
501		if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op);
	490	if (l.typeOf(un_op).isVector(zcu)) {
	491	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op));
	492	}
502	493	},
503	494	.cmp_lt,
504	495	.cmp_lt_optimized,
...	...	@@ -515,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
515	506	=> {},
516	507	inline .cmp_vector, .cmp_vector_optimized => \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
517	508	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
518		if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector);
	509	if (ty_pl.ty.toType().isVector(zcu)) {
	510	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector));
	511	}
519	512	},
520	513	.cond_br => {
521	514	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
...	...	@@ -570,13 +563,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
570	563	.load => if (l.features.has(.expand_packed_load)) {
571	564	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
572	565	const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
573		if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
	566	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
	567	continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
	568	}
574	569	},
575	570	.ret, .ret_safe, .ret_load => {},
576	571	.store, .store_safe => if (l.features.has(.expand_packed_store)) {
577	572	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
578	573	const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
579		if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
	574	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
	575	continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
	576	}
580	577	},
581	578	.unreach,
582	579	.optional_payload,
...	...	@@ -624,7 +621,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
624	621	switch (vector_ty.vectorLen(zcu)) {
625	622	0 => unreachable,
626	623	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
627		.ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()),
	624	.ty = .fromType(vector_ty.childType(zcu)),
628	625	.operand = reduce.operand,
629	626	} }),
630	627	else => {},
...	...	@@ -641,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
641	638	else => {},
642	639	}
643	640	},
644		.shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),
645		.shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),
646		.select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select),
	641	.shuffle_one => if (l.features.has(.scalarize_shuffle_one)) {
	642	continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst));
	643	},
	644	.shuffle_two => if (l.features.has(.scalarize_shuffle_two)) {
	645	continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst));
	646	},
	647	.select => if (l.features.has(.scalarize_select)) {
	648	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select));
	649	},
647	650	.memset,
648	651	.memset_safe,
649	652	.memcpy,
...	...	@@ -666,16 +669,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
666	669	const agg_ty = ty_pl.ty.toType();
667	670	switch (agg_ty.zigTypeTag(zcu)) {
668	671	else => {},
669		.@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {
	672	.@"union" => unreachable,
	673	.@"struct" => switch (agg_ty.containerLayout(zcu)) {
670	674	.auto, .@"extern" => {},
671		.@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
	675	.@"packed" => switch (agg_ty.structFieldCount(zcu)) {
	676	0 => unreachable,
	677	// An `aggregate_init` of a packed struct with 1 field is just a fancy bitcast.
	678	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	679	.ty = .fromType(agg_ty),
	680	.operand = @enumFromInt(l.air_extra.items[ty_pl.payload]),
	681	} }),
	682	else => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
	683	},
672	684	},
673	685	}
674	686	},
675	687	.union_init, .prefetch => {},
676	688	.mul_add => if (l.features.has(.scalarize_mul_add)) {
677	689	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
678		if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin);
	690	if (l.typeOf(pl_op.operand).isVector(zcu)) {
	691	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin));
	692	}
679	693	},
680	694	.field_parent_ptr,
681	695	.wasm_memory_size,
...	...	@@ -685,7 +699,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
685	699	.set_err_return_trace,
686	700	.addrspace_cast,
687	701	.save_err_return_trace_index,
688		.vector_store_elem,
689	702	.runtime_nav_ptr,
690	703	.c_va_arg,
691	704	.c_va_copy,
...	...	@@ -694,1003 +707,757 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
694	707	.work_item_id,
695	708	.work_group_size,
696	709	.work_group_id,
	710	.legalize_vec_elem_val,
	711	.legalize_vec_store_elem,
697	712	=> {},
698	713	}
699	714	}
700	715	}
701	716
702		const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select };
703		/// inline to propagate comptime-known `replaceInst` result.
704		inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
705		return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
706		}
707		fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
	717	const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, select };
	718	fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: ScalarizeForm) Error!Air.Inst.Data {
708	719	const pt = l.pt;
709	720	const zcu = pt.zcu;
710	721
711	722	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
712	723	const res_ty = l.typeOfIndex(orig_inst);
713		const res_len = res_ty.vectorLen(zcu);
714
715		const extra_insts = switch (form) {
716		.un_op, .ty_op, .bitcast => 1,
717		.bin_op, .cmp_vector => 2,
718		.pl_op_bin => 3,
719		.shuffle_one, .shuffle_two => 13,
720		.select => 6,
	724	const result_is_array = switch (res_ty.zigTypeTag(zcu)) {
	725	.vector => false,
	726	.array => true,
	727	else => unreachable,
721	728	};
722		var inst_buf: [5 + extra_insts + 9]Air.Inst.Index = undefined;
723		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	729	const res_len = res_ty.arrayLen(zcu);
	730	const res_elem_ty = res_ty.childType(zcu);
724	731
725		var res_block: Block = .init(&inst_buf);
726		{
727		const res_alloc_inst = res_block.add(l, .{
728		.tag = .alloc,
729		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
730		});
731		const index_alloc_inst = res_block.add(l, .{
732		.tag = .alloc,
733		.data = .{ .ty = .ptr_usize },
734		});
735		_ = res_block.add(l, .{
736		.tag = .store,
737		.data = .{ .bin_op = .{
738		.lhs = index_alloc_inst.toRef(),
739		.rhs = .zero_usize,
740		} },
741		});
	732	if (result_is_array) {
	733	// This is only allowed when legalizing an elementwise bitcast.
	734	assert(orig.tag == .bitcast);
	735	assert(form == .ty_op);
	736	}
742	737
743		var loop: Loop = .init(l, &res_block);
744		loop.block = .init(res_block.stealRemainingCapacity());
745		{
746		const cur_index_inst = loop.block.add(l, .{
747		.tag = .load,
748		.data = .{ .ty_op = .{
749		.ty = .usize_type,
750		.operand = index_alloc_inst.toRef(),
751		} },
752		});
753		_ = loop.block.add(l, .{
754		.tag = .vector_store_elem,
755		.data = .{ .vector_store_elem = .{
756		.vector_ptr = res_alloc_inst.toRef(),
757		.payload = try l.addExtra(Air.Bin, .{
758		.lhs = cur_index_inst.toRef(),
759		.rhs = res_elem: switch (form) {
760		.un_op => loop.block.add(l, .{
761		.tag = orig.tag,
762		.data = .{ .un_op = loop.block.add(l, .{
763		.tag = .array_elem_val,
764		.data = .{ .bin_op = .{
765		.lhs = orig.data.un_op,
766		.rhs = cur_index_inst.toRef(),
767		} },
768		}).toRef() },
769		}).toRef(),
770		.ty_op => loop.block.add(l, .{
771		.tag = orig.tag,
772		.data = .{ .ty_op = .{
773		.ty = Air.internedToRef(res_ty.childType(zcu).toIntern()),
774		.operand = loop.block.add(l, .{
775		.tag = .array_elem_val,
776		.data = .{ .bin_op = .{
777		.lhs = orig.data.ty_op.operand,
778		.rhs = cur_index_inst.toRef(),
779		} },
780		}).toRef(),
781		} },
782		}).toRef(),
783		.bin_op => loop.block.add(l, .{
784		.tag = orig.tag,
785		.data = .{ .bin_op = .{
786		.lhs = loop.block.add(l, .{
787		.tag = .array_elem_val,
788		.data = .{ .bin_op = .{
789		.lhs = orig.data.bin_op.lhs,
790		.rhs = cur_index_inst.toRef(),
791		} },
792		}).toRef(),
793		.rhs = loop.block.add(l, .{
794		.tag = .array_elem_val,
795		.data = .{ .bin_op = .{
796		.lhs = orig.data.bin_op.rhs,
797		.rhs = cur_index_inst.toRef(),
798		} },
799		}).toRef(),
800		} },
801		}).toRef(),
802		.pl_op_bin => {
803		const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
804		break :res_elem loop.block.add(l, .{
805		.tag = orig.tag,
806		.data = .{ .pl_op = .{
807		.payload = try l.addExtra(Air.Bin, .{
808		.lhs = loop.block.add(l, .{
809		.tag = .array_elem_val,
810		.data = .{ .bin_op = .{
811		.lhs = extra.lhs,
812		.rhs = cur_index_inst.toRef(),
813		} },
814		}).toRef(),
815		.rhs = loop.block.add(l, .{
816		.tag = .array_elem_val,
817		.data = .{ .bin_op = .{
818		.lhs = extra.rhs,
819		.rhs = cur_index_inst.toRef(),
820		} },
821		}).toRef(),
822		}),
823		.operand = loop.block.add(l, .{
824		.tag = .array_elem_val,
825		.data = .{ .bin_op = .{
826		.lhs = orig.data.pl_op.operand,
827		.rhs = cur_index_inst.toRef(),
828		} },
829		}).toRef(),
830		} },
831		}).toRef();
832		},
833		.bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{
834		.tag = .array_elem_val,
835		.data = .{ .bin_op = .{
836		.lhs = orig.data.ty_op.operand,
837		.rhs = cur_index_inst.toRef(),
838		} },
839		}).toRef()),
840		.cmp_vector => {
841		const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
842		break :res_elem (try loop.block.addCmp(
843		l,
844		extra.compareOperator(),
845		loop.block.add(l, .{
846		.tag = .array_elem_val,
847		.data = .{ .bin_op = .{
848		.lhs = extra.lhs,
849		.rhs = cur_index_inst.toRef(),
850		} },
851		}).toRef(),
852		loop.block.add(l, .{
853		.tag = .array_elem_val,
854		.data = .{ .bin_op = .{
855		.lhs = extra.rhs,
856		.rhs = cur_index_inst.toRef(),
857		} },
858		}).toRef(),
859		.{ .optimized = switch (orig.tag) {
860		else => unreachable,
861		.cmp_vector => false,
862		.cmp_vector_optimized => true,
863		} },
864		)).toRef();
865		},
866		.shuffle_one, .shuffle_two => {
867		const ip = &zcu.intern_pool;
868		const unwrapped = switch (form) {
869		else => comptime unreachable,
870		.shuffle_one => l.getTmpAir().unwrapShuffleOne(zcu, orig_inst),
871		.shuffle_two => l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst),
872		};
873		const operand_a = switch (form) {
874		else => comptime unreachable,
875		.shuffle_one => unwrapped.operand,
876		.shuffle_two => unwrapped.operand_a,
877		};
878		const operand_a_len = l.typeOf(operand_a).vectorLen(zcu);
879		const elem_ty = res_ty.childType(zcu);
880		var res_elem: Result = .init(l, elem_ty, &loop.block);
881		res_elem.block = .init(loop.block.stealCapacity(extra_insts));
882		{
883		const ExpectedContents = extern struct {
884		mask_elems: [128]InternPool.Index,
885		ct_elems: switch (form) {
886		else => unreachable,
887		.shuffle_one => extern struct {
888		keys: [152]InternPool.Index,
889		header: u8 align(@alignOf(u32)),
890		index: [256][2]u8,
891		},
892		.shuffle_two => void,
893		},
894		};
895		var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
896		std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
897		const gpa = stack.get();
	738	// Our output will be a loop doing elementwise stores:
	739	//
	740	// %1 = block(@Vector(N, Scalar), {
	741	// %2 = alloc(*usize)
	742	// %3 = alloc(*@Vector(N, Scalar))
	743	// %4 = store(%2, @zero_usize)
	744	// %5 = loop({
	745	// %6 = load(%2)
	746	// %7 = <scalar result of operation at index %5>
	747	// %8 = legalize_vec_store_elem(%3, %5, %6)
	748	// %9 = cmp_eq(%6, <usize, N-1>)
	749	// %10 = cond_br(%9, {
	750	// %11 = load(%3)
	751	// %12 = br(%1, %11)
	752	// }, {
	753	// %13 = add(%6, @one_usize)
	754	// %14 = store(%2, %13)
	755	// %15 = repeat(%5)
	756	// })
	757	// })
	758	// })
	759	//
	760	// If scalarizing an elementwise bitcast, the result might be an array, in which case
	761	// `legalize_vec_store_elem` becomes two instructions (`ptr_elem_ptr` and `store`).
	762	// Therefore, there are 13 or 14 instructions in the block, plus however many are
	763	// needed to compute each result element for `form`.
	764	const inst_per_form: usize = switch (form) {
	765	.un_op, .ty_op => 2,
	766	.bin_op, .cmp_vector => 3,
	767	.pl_op_bin => 4,
	768	.select => 7,
	769	};
	770	const max_inst_per_form = 7; // maximum value in the above switch
	771	var inst_buf: [14 + max_inst_per_form]Air.Inst.Index = undefined;
898	772
899		const mask_elems = try gpa.alloc(InternPool.Index, res_len);
900		defer gpa.free(mask_elems);
901
902		var ct_elems: switch (form) {
903		else => unreachable,
904		.shuffle_one => std.AutoArrayHashMapUnmanaged(InternPool.Index, void),
905		.shuffle_two => struct {
906		const empty: @This() = .{};
907		inline fn deinit(_: @This(), _: std.mem.Allocator) void {}
908		inline fn ensureTotalCapacity(_: @This(), _: std.mem.Allocator, _: usize) error{}!void {}
909		},
910		} = .empty;
911		defer ct_elems.deinit(gpa);
912		try ct_elems.ensureTotalCapacity(gpa, res_len);
913
914		const mask_elem_ty = try pt.intType(.signed, 1 + Type.smallestUnsignedBits(@max(operand_a_len, switch (form) {
915		else => comptime unreachable,
916		.shuffle_one => res_len,
917		.shuffle_two => l.typeOf(unwrapped.operand_b).vectorLen(zcu),
918		})));
919		for (mask_elems, unwrapped.mask) \|mask_elem_val, mask_elem\| mask_elem_val. = (try pt.intValue(mask_elem_ty, switch (form) {
920		else => comptime unreachable,
921		.shuffle_one => switch (mask_elem.unwrap()) {
922		.elem => \|index\| index,
923		.value => \|elem_val\| if (ip.isUndef(elem_val))
924		operand_a_len
925		else
926		~@as(i33, @intCast((ct_elems.getOrPutAssumeCapacity(elem_val)).index)),
927		},
928		.shuffle_two => switch (mask_elem.unwrap()) {
929		.a_elem => \|a_index\| a_index,
930		.b_elem => \|b_index\| ~@as(i33, b_index),
931		.undef => operand_a_len,
932		},
933		})).toIntern();
934		const mask_ty = try pt.arrayType(.{
935		.len = res_len,
936		.child = mask_elem_ty.toIntern(),
937		});
938		const mask_elem_inst = res_elem.block.add(l, .{
939		.tag = .ptr_elem_val,
940		.data = .{ .bin_op = .{
941		.lhs = Air.internedToRef(try pt.intern(.{ .ptr = .{
942		.ty = (try pt.manyConstPtrType(mask_elem_ty)).toIntern(),
943		.base_addr = .{ .uav = .{
944		.val = (try pt.aggregateValue(mask_ty, mask_elems)).toIntern(),
945		.orig_ty = (try pt.singleConstPtrType(mask_ty)).toIntern(),
946		} },
947		.byte_offset = 0,
948		} })),
949		.rhs = cur_index_inst.toRef(),
950		} },
951		});
952		var def_cond_br: CondBr = .init(l, (try res_elem.block.addCmp(
953		l,
954		.lt,
955		mask_elem_inst.toRef(),
956		try pt.intRef(mask_elem_ty, operand_a_len),
957		.{},
958		)).toRef(), &res_elem.block, .{});
959		def_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
960		{
961		const operand_b_used = switch (form) {
962		else => comptime unreachable,
963		.shuffle_one => ct_elems.count() > 0,
964		.shuffle_two => true,
965		};
966		var operand_cond_br: CondBr = undefined;
967		operand_cond_br.then_block = if (operand_b_used) then_block: {
968		operand_cond_br = .init(l, (try def_cond_br.then_block.addCmp(
969		l,
970		.gte,
971		mask_elem_inst.toRef(),
972		try pt.intRef(mask_elem_ty, 0),
973		.{},
974		)).toRef(), &def_cond_br.then_block, .{});
975		break :then_block .init(def_cond_br.then_block.stealRemainingCapacity());
976		} else def_cond_br.then_block;
977		_ = operand_cond_br.then_block.add(l, .{
978		.tag = .br,
979		.data = .{ .br = .{
980		.block_inst = res_elem.inst,
981		.operand = operand_cond_br.then_block.add(l, .{
982		.tag = .array_elem_val,
983		.data = .{ .bin_op = .{
984		.lhs = operand_a,
985		.rhs = operand_cond_br.then_block.add(l, .{
986		.tag = .intcast,
987		.data = .{ .ty_op = .{
988		.ty = .usize_type,
989		.operand = mask_elem_inst.toRef(),
990		} },
991		}).toRef(),
992		} },
993		}).toRef(),
994		} },
995		});
996		if (operand_b_used) {
997		operand_cond_br.else_block = .init(operand_cond_br.then_block.stealRemainingCapacity());
998		_ = operand_cond_br.else_block.add(l, .{
999		.tag = .br,
1000		.data = .{ .br = .{
1001		.block_inst = res_elem.inst,
1002		.operand = if (switch (form) {
1003		else => comptime unreachable,
1004		.shuffle_one => ct_elems.count() > 1,
1005		.shuffle_two => true,
1006		}) operand_cond_br.else_block.add(l, .{
1007		.tag = switch (form) {
1008		else => comptime unreachable,
1009		.shuffle_one => .ptr_elem_val,
1010		.shuffle_two => .array_elem_val,
1011		},
1012		.data = .{ .bin_op = .{
1013		.lhs = operand_b: switch (form) {
1014		else => comptime unreachable,
1015		.shuffle_one => {
1016		const ct_elems_ty = try pt.arrayType(.{
1017		.len = ct_elems.count(),
1018		.child = elem_ty.toIntern(),
1019		});
1020		break :operand_b Air.internedToRef(try pt.intern(.{ .ptr = .{
1021		.ty = (try pt.manyConstPtrType(elem_ty)).toIntern(),
1022		.base_addr = .{ .uav = .{
1023		.val = (try pt.aggregateValue(ct_elems_ty, ct_elems.keys())).toIntern(),
1024		.orig_ty = (try pt.singleConstPtrType(ct_elems_ty)).toIntern(),
1025		} },
1026		.byte_offset = 0,
1027		} }));
1028		},
1029		.shuffle_two => unwrapped.operand_b,
1030		},
1031		.rhs = operand_cond_br.else_block.add(l, .{
1032		.tag = .intcast,
1033		.data = .{ .ty_op = .{
1034		.ty = .usize_type,
1035		.operand = operand_cond_br.else_block.add(l, .{
1036		.tag = .not,
1037		.data = .{ .ty_op = .{
1038		.ty = Air.internedToRef(mask_elem_ty.toIntern()),
1039		.operand = mask_elem_inst.toRef(),
1040		} },
1041		}).toRef(),
1042		} },
1043		}).toRef(),
1044		} },
1045		}).toRef() else res_elem_br: {
1046		_ = operand_cond_br.else_block.stealCapacity(3);
1047		break :res_elem_br Air.internedToRef(ct_elems.keys()[0]);
1048		},
1049		} },
1050		});
1051		def_cond_br.else_block = .init(operand_cond_br.else_block.stealRemainingCapacity());
1052		try operand_cond_br.finish(l);
1053		} else {
1054		def_cond_br.then_block = operand_cond_br.then_block;
1055		_ = def_cond_br.then_block.stealCapacity(6);
1056		def_cond_br.else_block = .init(def_cond_br.then_block.stealRemainingCapacity());
1057		}
1058		}
1059		_ = def_cond_br.else_block.add(l, .{
1060		.tag = .br,
1061		.data = .{ .br = .{
1062		.block_inst = res_elem.inst,
1063		.operand = try pt.undefRef(elem_ty),
1064		} },
1065		});
1066		try def_cond_br.finish(l);
1067		}
1068		try res_elem.finish(l);
1069		break :res_elem res_elem.inst.toRef();
1070		},
1071		.select => {
1072		const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1073		var res_elem: Result = .init(l, l.typeOf(extra.lhs).childType(zcu), &loop.block);
1074		res_elem.block = .init(loop.block.stealCapacity(extra_insts));
1075		{
1076		var select_cond_br: CondBr = .init(l, res_elem.block.add(l, .{
1077		.tag = .array_elem_val,
1078		.data = .{ .bin_op = .{
1079		.lhs = orig.data.pl_op.operand,
1080		.rhs = cur_index_inst.toRef(),
1081		} },
1082		}).toRef(), &res_elem.block, .{});
1083		select_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
1084		_ = select_cond_br.then_block.add(l, .{
1085		.tag = .br,
1086		.data = .{ .br = .{
1087		.block_inst = res_elem.inst,
1088		.operand = select_cond_br.then_block.add(l, .{
1089		.tag = .array_elem_val,
1090		.data = .{ .bin_op = .{
1091		.lhs = extra.lhs,
1092		.rhs = cur_index_inst.toRef(),
1093		} },
1094		}).toRef(),
1095		} },
1096		});
1097		select_cond_br.else_block = .init(select_cond_br.then_block.stealRemainingCapacity());
1098		_ = select_cond_br.else_block.add(l, .{
1099		.tag = .br,
1100		.data = .{ .br = .{
1101		.block_inst = res_elem.inst,
1102		.operand = select_cond_br.else_block.add(l, .{
1103		.tag = .array_elem_val,
1104		.data = .{ .bin_op = .{
1105		.lhs = extra.rhs,
1106		.rhs = cur_index_inst.toRef(),
1107		} },
1108		}).toRef(),
1109		} },
1110		});
1111		try select_cond_br.finish(l);
1112		}
1113		try res_elem.finish(l);
1114		break :res_elem res_elem.inst.toRef();
1115		},
1116		},
1117		}),
1118		} },
1119		});
	773	var main_block: Block = .init(&inst_buf);
	774	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1120	775
1121		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
	776	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
	777	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(res_ty)).toRef();
	778
	779	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
	780
	781	var loop: Loop = .init(l, &main_block);
	782	loop.block = .init(main_block.stealRemainingCapacity());
	783
	784	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
	785	const elem_val: Air.Inst.Ref = switch (form) {
	786	.un_op => elem: {
	787	const orig_operand = orig.data.un_op;
	788	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
	789	break :elem loop.block.addUnOp(l, orig.tag, operand).toRef();
	790	},
	791	.ty_op => elem: {
	792	const orig_operand = orig.data.ty_op.operand;
	793	const operand_is_array = switch (l.typeOf(orig_operand).zigTypeTag(zcu)) {
	794	.vector => false,
	795	.array => true,
	796	else => unreachable,
	797	};
	798	const operand = loop.block.addBinOp(
1122	799	l,
1123		.lt,
1124		cur_index_inst.toRef(),
1125		try pt.intRef(.usize, res_len - 1),
1126		.{},
1127		)).toRef(), &loop.block, .{});
1128		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1129		{
1130		_ = loop_cond_br.then_block.add(l, .{
1131		.tag = .store,
1132		.data = .{ .bin_op = .{
1133		.lhs = index_alloc_inst.toRef(),
1134		.rhs = loop_cond_br.then_block.add(l, .{
1135		.tag = .add,
1136		.data = .{ .bin_op = .{
1137		.lhs = cur_index_inst.toRef(),
1138		.rhs = .one_usize,
1139		} },
1140		}).toRef(),
1141		} },
1142		});
1143		_ = loop_cond_br.then_block.add(l, .{
1144		.tag = .repeat,
1145		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1146		});
1147		}
1148		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1149		_ = loop_cond_br.else_block.add(l, .{
1150		.tag = .br,
1151		.data = .{ .br = .{
1152		.block_inst = orig_inst,
1153		.operand = loop_cond_br.else_block.add(l, .{
1154		.tag = .load,
1155		.data = .{ .ty_op = .{
1156		.ty = Air.internedToRef(res_ty.toIntern()),
1157		.operand = res_alloc_inst.toRef(),
1158		} },
1159		}).toRef(),
	800	if (operand_is_array) .array_elem_val else .legalize_vec_elem_val,
	801	orig_operand,
	802	index_val,
	803	).toRef();
	804	break :elem loop.block.addTyOp(l, orig.tag, res_elem_ty, operand).toRef();
	805	},
	806	.bin_op => elem: {
	807	const orig_bin = orig.data.bin_op;
	808	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
	809	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
	810	break :elem loop.block.addBinOp(l, orig.tag, lhs, rhs).toRef();
	811	},
	812	.pl_op_bin => elem: {
	813	const orig_operand = orig.data.pl_op.operand;
	814	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
	815	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
	816	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
	817	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
	818	break :elem loop.block.add(l, .{
	819	.tag = orig.tag,
	820	.data = .{ .pl_op = .{
	821	.operand = operand,
	822	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
	823	} },
	824	}).toRef();
	825	},
	826	.cmp_vector => elem: {
	827	const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
	828	const cmp_op = orig_payload.compareOperator();
	829	const optimized = switch (orig.tag) {
	830	.cmp_vector => false,
	831	.cmp_vector_optimized => true,
	832	else => unreachable,
	833	};
	834	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.lhs, index_val).toRef();
	835	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.rhs, index_val).toRef();
	836	break :elem loop.block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
	837	},
	838	.select => elem: {
	839	const orig_cond = orig.data.pl_op.operand;
	840	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
	841
	842	const elem_block_inst = loop.block.add(l, .{
	843	.tag = .block,
	844	.data = .{ .ty_pl = .{
	845	.ty = .fromType(res_elem_ty),
	846	.payload = undefined,
1160	847	} },
1161	848	});
1162		try loop_cond_br.finish(l);
1163		}
1164		try loop.finish(l);
	849	var elem_block: Block = .init(loop.block.stealCapacity(2));
	850	const cond = elem_block.addBinOp(l, .legalize_vec_elem_val, orig_cond, index_val).toRef();
	851
	852	var condbr: CondBr = .init(l, cond, &elem_block, .{});
	853
	854	condbr.then_block = .init(loop.block.stealCapacity(2));
	855	const lhs = condbr.then_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
	856	condbr.then_block.addBr(l, elem_block_inst, lhs);
	857
	858	condbr.else_block = .init(loop.block.stealCapacity(2));
	859	const rhs = condbr.else_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
	860	condbr.else_block.addBr(l, elem_block_inst, rhs);
	861
	862	try condbr.finish(l);
	863
	864	const inst_data = l.air_instructions.items(.data);
	865	inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
	866
	867	break :elem elem_block_inst.toRef();
	868	},
	869	};
	870	_ = loop.block.stealCapacity(max_inst_per_form - inst_per_form);
	871	if (result_is_array) {
	872	const elem_ptr = loop.block.add(l, .{
	873	.tag = .ptr_elem_ptr,
	874	.data = .{ .ty_pl = .{
	875	.ty = .fromType(try pt.singleMutPtrType(res_elem_ty)),
	876	.payload = try l.addExtra(Air.Bin, .{
	877	.lhs = result_ptr,
	878	.rhs = index_val,
	879	}),
	880	} },
	881	}).toRef();
	882	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
	883	} else {
	884	_ = loop.block.add(l, .{
	885	.tag = .legalize_vec_store_elem,
	886	.data = .{ .pl_op = .{
	887	.operand = result_ptr,
	888	.payload = try l.addExtra(Air.Bin, .{
	889	.lhs = index_val,
	890	.rhs = elem_val,
	891	}),
	892	} },
	893	});
	894	_ = loop.block.stealCapacity(1);
1165	895	}
	896	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, res_len - 1))).toRef();
	897
	898	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
	899	condbr.then_block = .init(loop.block.stealRemainingCapacity());
	900	const result_val = condbr.then_block.addTyOp(l, .load, res_ty, result_ptr).toRef();
	901	condbr.then_block.addBr(l, orig_inst, result_val);
	902
	903	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
	904	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
	905	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
	906	_ = condbr.else_block.add(l, .{
	907	.tag = .repeat,
	908	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
	909	});
	910
	911	try condbr.finish(l);
	912
	913	try loop.finish(l);
	914
1166	915	return .{ .ty_pl = .{
1167		.ty = Air.internedToRef(res_ty.toIntern()),
1168		.payload = try l.addBlockBody(res_block.body()),
	916	.ty = .fromType(res_ty),
	917	.payload = try l.addBlockBody(main_block.body()),
1169	918	} };
1170	919	}
1171		fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
	920	fn scalarizeShuffleOneBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1172	921	const pt = l.pt;
1173	922	const zcu = pt.zcu;
	923	const gpa = zcu.gpa;
1174	924
1175		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1176		const res_ty = orig_ty_op.ty.toType();
1177		const res_elem_ty = res_ty.childType(zcu);
1178		const res_len = res_ty.arrayLen(zcu);
	925	const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
1179	926
1180		var inst_buf: [16]Air.Inst.Index = undefined;
1181		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	927	// We're going to emit something like this:
	928	//
	929	// var x: @Vector(N, T) = all_comptime_known_elems;
	930	// for (out_idxs, in_idxs) \|i, j\| x[i] = operand[j];
	931	//
	932	// So we must first compute `out_idxs` and `in_idxs`.
1182	933
1183		var res_block: Block = .init(&inst_buf);
1184		{
1185		const res_alloc_inst = res_block.add(l, .{
1186		.tag = .alloc,
1187		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1188		});
1189		const index_alloc_inst = res_block.add(l, .{
1190		.tag = .alloc,
1191		.data = .{ .ty = .ptr_usize },
1192		});
1193		_ = res_block.add(l, .{
1194		.tag = .store,
1195		.data = .{ .bin_op = .{
1196		.lhs = index_alloc_inst.toRef(),
1197		.rhs = .zero_usize,
1198		} },
1199		});
	934	var sfba_state = std.heap.stackFallback(512, gpa);
	935	const sfba = sfba_state.get();
1200	936
1201		var loop: Loop = .init(l, &res_block);
1202		loop.block = .init(res_block.stealRemainingCapacity());
1203		{
1204		const cur_index_inst = loop.block.add(l, .{
1205		.tag = .load,
1206		.data = .{ .ty_op = .{
1207		.ty = .usize_type,
1208		.operand = index_alloc_inst.toRef(),
1209		} },
1210		});
1211		_ = loop.block.add(l, .{
1212		.tag = .store,
1213		.data = .{ .bin_op = .{
1214		.lhs = loop.block.add(l, .{
1215		.tag = .ptr_elem_ptr,
1216		.data = .{ .ty_pl = .{
1217		.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1218		.payload = try l.addExtra(Air.Bin, .{
1219		.lhs = res_alloc_inst.toRef(),
1220		.rhs = cur_index_inst.toRef(),
1221		}),
1222		} },
1223		}).toRef(),
1224		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1225		.tag = .array_elem_val,
1226		.data = .{ .bin_op = .{
1227		.lhs = orig_ty_op.operand,
1228		.rhs = cur_index_inst.toRef(),
1229		} },
1230		}).toRef()),
1231		} },
1232		});
	937	const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
	938	defer sfba.free(out_idxs_buf);
	939
	940	const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
	941	defer sfba.free(in_idxs_buf);
	942
	943	var n: usize = 0;
	944	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
	945	.value => {},
	946	.elem => \|in_idx\| {
	947	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
	948	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
	949	n += 1;
	950	},
	951	};
	952
	953	const init_val: Value = init: {
	954	const undef_val = try pt.undefValue(shuffle.result_ty.childType(zcu));
	955	const elems = try sfba.alloc(InternPool.Index, shuffle.mask.len);
	956	defer sfba.free(elems);
	957	for (shuffle.mask, elems) \|mask, elem\| elem. = switch (mask.unwrap()) {
	958	.value => \|ip_index\| ip_index,
	959	.elem => undef_val.toIntern(),
	960	};
	961	break :init try pt.aggregateValue(shuffle.result_ty, elems);
	962	};
	963
	964	// %1 = block(@Vector(N, T), {
	965	// %2 = alloc(*@Vector(N, T))
	966	// %3 = alloc(*usize)
	967	// %4 = store(%2, <init_val>)
	968	// %5 = [addScalarizedShuffle]
	969	// %6 = load(%2)
	970	// %7 = br(%1, %6)
	971	// })
	972
	973	var inst_buf: [6]Air.Inst.Index = undefined;
	974	var main_block: Block = .init(&inst_buf);
	975	try l.air_instructions.ensureUnusedCapacity(gpa, 19);
	976
	977	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
	978	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
	979
	980	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(init_val));
	981
	982	try l.addScalarizedShuffle(
	983	&main_block,
	984	shuffle.operand,
	985	result_ptr,
	986	index_ptr,
	987	out_idxs_buf[0..n],
	988	in_idxs_buf[0..n],
	989	);
	990
	991	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
	992	main_block.addBr(l, orig_inst, result_val);
1233	993
1234		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1235		l,
1236		.lt,
1237		cur_index_inst.toRef(),
1238		try pt.intRef(.usize, res_len - 1),
1239		.{},
1240		)).toRef(), &loop.block, .{});
1241		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1242		{
1243		_ = loop_cond_br.then_block.add(l, .{
1244		.tag = .store,
1245		.data = .{ .bin_op = .{
1246		.lhs = index_alloc_inst.toRef(),
1247		.rhs = loop_cond_br.then_block.add(l, .{
1248		.tag = .add,
1249		.data = .{ .bin_op = .{
1250		.lhs = cur_index_inst.toRef(),
1251		.rhs = .one_usize,
1252		} },
1253		}).toRef(),
1254		} },
1255		});
1256		_ = loop_cond_br.then_block.add(l, .{
1257		.tag = .repeat,
1258		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1259		});
1260		}
1261		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1262		_ = loop_cond_br.else_block.add(l, .{
1263		.tag = .br,
1264		.data = .{ .br = .{
1265		.block_inst = orig_inst,
1266		.operand = loop_cond_br.else_block.add(l, .{
1267		.tag = .load,
1268		.data = .{ .ty_op = .{
1269		.ty = Air.internedToRef(res_ty.toIntern()),
1270		.operand = res_alloc_inst.toRef(),
1271		} },
1272		}).toRef(),
1273		} },
1274		});
1275		try loop_cond_br.finish(l);
1276		}
1277		try loop.finish(l);
1278		}
1279	994	return .{ .ty_pl = .{
1280		.ty = Air.internedToRef(res_ty.toIntern()),
1281		.payload = try l.addBlockBody(res_block.body()),
	995	.ty = .fromType(shuffle.result_ty),
	996	.payload = try l.addBlockBody(main_block.body()),
1282	997	} };
1283	998	}
1284		fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
	999	fn scalarizeShuffleTwoBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1285	1000	const pt = l.pt;
1286	1001	const zcu = pt.zcu;
	1002	const gpa = zcu.gpa;
1287	1003
1288		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1289		const res_ty = orig_ty_op.ty.toType();
1290		const operand_ty = l.typeOf(orig_ty_op.operand);
1291		const int_bits: u16 = @intCast(operand_ty.bitSize(zcu));
1292		const int_ty = try pt.intType(.unsigned, int_bits);
1293		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1294		const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
1295		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1296
1297		var inst_buf: [22]Air.Inst.Index = undefined;
1298		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	1004	const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
1299	1005
1300		var res_block: Block = .init(&inst_buf);
1301		{
1302		const int_alloc_inst = res_block.add(l, .{
1303		.tag = .alloc,
1304		.data = .{ .ty = try pt.singleMutPtrType(int_ty) },
1305		});
1306		_ = res_block.add(l, .{
1307		.tag = .store,
1308		.data = .{ .bin_op = .{
1309		.lhs = int_alloc_inst.toRef(),
1310		.rhs = try pt.intRef(int_ty, 0),
1311		} },
1312		});
1313		const index_alloc_inst = res_block.add(l, .{
1314		.tag = .alloc,
1315		.data = .{ .ty = .ptr_usize },
1316		});
1317		_ = res_block.add(l, .{
1318		.tag = .store,
1319		.data = .{ .bin_op = .{
1320		.lhs = index_alloc_inst.toRef(),
1321		.rhs = .zero_usize,
1322		} },
1323		});
	1006	// We're going to emit something like this:
	1007	//
	1008	// var x: @Vector(N, T) = undefined;
	1009	// for (out_idxs_a, in_idxs_a) \|i, j\| x[i] = operand_a[j];
	1010	// for (out_idxs_b, in_idxs_b) \|i, j\| x[i] = operand_b[j];
	1011	//
	1012	// The AIR will look like this:
	1013	//
	1014	// %1 = block(@Vector(N, T), {
	1015	// %2 = alloc(*@Vector(N, T))
	1016	// %3 = alloc(*usize)
	1017	// %4 = store(%2, <@Vector(N, T), undefined>)
	1018	// %5 = [addScalarizedShuffle]
	1019	// %6 = [addScalarizedShuffle]
	1020	// %7 = load(%2)
	1021	// %8 = br(%1, %7)
	1022	// })
1324	1023
1325		var loop: Loop = .init(l, &res_block);
1326		loop.block = .init(res_block.stealRemainingCapacity());
1327		{
1328		const cur_index_inst = loop.block.add(l, .{
1329		.tag = .load,
1330		.data = .{ .ty_op = .{
1331		.ty = .usize_type,
1332		.operand = index_alloc_inst.toRef(),
1333		} },
1334		});
1335		const cur_int_inst = loop.block.add(l, .{
1336		.tag = .bit_or,
1337		.data = .{ .bin_op = .{
1338		.lhs = loop.block.add(l, .{
1339		.tag = .shl_exact,
1340		.data = .{ .bin_op = .{
1341		.lhs = loop.block.add(l, .{
1342		.tag = .intcast,
1343		.data = .{ .ty_op = .{
1344		.ty = Air.internedToRef(int_ty.toIntern()),
1345		.operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{
1346		.tag = .array_elem_val,
1347		.data = .{ .bin_op = .{
1348		.lhs = orig_ty_op.operand,
1349		.rhs = cur_index_inst.toRef(),
1350		} },
1351		}).toRef()),
1352		} },
1353		}).toRef(),
1354		.rhs = loop.block.add(l, .{
1355		.tag = .mul,
1356		.data = .{ .bin_op = .{
1357		.lhs = loop.block.add(l, .{
1358		.tag = .intcast,
1359		.data = .{ .ty_op = .{
1360		.ty = Air.internedToRef(shift_ty.toIntern()),
1361		.operand = cur_index_inst.toRef(),
1362		} },
1363		}).toRef(),
1364		.rhs = try pt.intRef(shift_ty, elem_bits),
1365		} },
1366		}).toRef(),
1367		} },
1368		}).toRef(),
1369		.rhs = loop.block.add(l, .{
1370		.tag = .load,
1371		.data = .{ .ty_op = .{
1372		.ty = Air.internedToRef(int_ty.toIntern()),
1373		.operand = int_alloc_inst.toRef(),
1374		} },
1375		}).toRef(),
1376		} },
1377		});
	1024	var sfba_state = std.heap.stackFallback(512, gpa);
	1025	const sfba = sfba_state.get();
1378	1026
1379		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1380		l,
1381		.lt,
1382		cur_index_inst.toRef(),
1383		try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1),
1384		.{},
1385		)).toRef(), &loop.block, .{});
1386		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1387		{
1388		_ = loop_cond_br.then_block.add(l, .{
1389		.tag = .store,
1390		.data = .{ .bin_op = .{
1391		.lhs = int_alloc_inst.toRef(),
1392		.rhs = cur_int_inst.toRef(),
1393		} },
1394		});
1395		_ = loop_cond_br.then_block.add(l, .{
1396		.tag = .store,
1397		.data = .{ .bin_op = .{
1398		.lhs = index_alloc_inst.toRef(),
1399		.rhs = loop_cond_br.then_block.add(l, .{
1400		.tag = .add,
1401		.data = .{ .bin_op = .{
1402		.lhs = cur_index_inst.toRef(),
1403		.rhs = .one_usize,
1404		} },
1405		}).toRef(),
1406		} },
1407		});
1408		_ = loop_cond_br.then_block.add(l, .{
1409		.tag = .repeat,
1410		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1411		});
1412		}
1413		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1414		_ = loop_cond_br.else_block.add(l, .{
1415		.tag = .br,
1416		.data = .{ .br = .{
1417		.block_inst = orig_inst,
1418		.operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()),
1419		} },
1420		});
1421		try loop_cond_br.finish(l);
1422		}
1423		try loop.finish(l);
	1027	const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
	1028	defer sfba.free(out_idxs_buf);
	1029
	1030	const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
	1031	defer sfba.free(in_idxs_buf);
	1032
	1033	// Iterate `shuffle.mask` before doing anything, because modifying AIR invalidates it.
	1034	const out_idxs_a, const in_idxs_a, const out_idxs_b, const in_idxs_b = idxs: {
	1035	var n: usize = 0;
	1036	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
	1037	.undef, .b_elem => {},
	1038	.a_elem => \|in_idx\| {
	1039	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
	1040	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
	1041	n += 1;
	1042	},
	1043	};
	1044	const a_len = n;
	1045	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
	1046	.undef, .a_elem => {},
	1047	.b_elem => \|in_idx\| {
	1048	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
	1049	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
	1050	n += 1;
	1051	},
	1052	};
	1053	break :idxs .{
	1054	out_idxs_buf[0..a_len],
	1055	in_idxs_buf[0..a_len],
	1056	out_idxs_buf[a_len..n],
	1057	in_idxs_buf[a_len..n],
	1058	};
	1059	};
	1060
	1061	var inst_buf: [7]Air.Inst.Index = undefined;
	1062	var main_block: Block = .init(&inst_buf);
	1063	try l.air_instructions.ensureUnusedCapacity(gpa, 33);
	1064
	1065	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
	1066	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
	1067
	1068	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.undefValue(shuffle.result_ty)));
	1069
	1070	if (out_idxs_a.len == 0) {
	1071	_ = main_block.stealCapacity(1);
	1072	} else {
	1073	try l.addScalarizedShuffle(
	1074	&main_block,
	1075	shuffle.operand_a,
	1076	result_ptr,
	1077	index_ptr,
	1078	out_idxs_a,
	1079	in_idxs_a,
	1080	);
	1081	}
	1082
	1083	if (out_idxs_b.len == 0) {
	1084	_ = main_block.stealCapacity(1);
	1085	} else {
	1086	try l.addScalarizedShuffle(
	1087	&main_block,
	1088	shuffle.operand_b,
	1089	result_ptr,
	1090	index_ptr,
	1091	out_idxs_b,
	1092	in_idxs_b,
	1093	);
1424	1094	}
	1095
	1096	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
	1097	main_block.addBr(l, orig_inst, result_val);
	1098
1425	1099	return .{ .ty_pl = .{
1426		.ty = Air.internedToRef(res_ty.toIntern()),
1427		.payload = try l.addBlockBody(res_block.body()),
	1100	.ty = .fromType(shuffle.result_ty),
	1101	.payload = try l.addBlockBody(main_block.body()),
1428	1102	} };
1429	1103	}
1430		fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
	1104	/// Adds code to `parent_block` which behaves like this loop:
	1105	///
	1106	/// for (out_idxs, in_idxs) \|i, j\| result_vec_ptr[i] = operand_vec[j];
	1107	///
	1108	/// The actual AIR adds exactly one instruction to `parent_block` itself and 14 instructions
	1109	/// overall, and is as follows:
	1110	///
	1111	/// %1 = block(void, {
	1112	/// %2 = store(index_ptr, @zero_usize)
	1113	/// %3 = loop({
	1114	/// %4 = load(index_ptr)
	1115	/// %5 = ptr_elem_val(out_idxs_ptr, %4)
	1116	/// %6 = ptr_elem_val(in_idxs_ptr, %4)
	1117	/// %7 = legalize_vec_elem_val(operand_vec, %6)
	1118	/// %8 = legalize_vec_store_elem(result_vec_ptr, %4, %7)
	1119	/// %9 = cmp_eq(%4, <usize, out_idxs.len-1>)
	1120	/// %10 = cond_br(%9, {
	1121	/// %11 = br(%1, @void_value)
	1122	/// }, {
	1123	/// %12 = add(%4, @one_usize)
	1124	/// %13 = store(index_ptr, %12)
	1125	/// %14 = repeat(%3)
	1126	/// })
	1127	/// })
	1128	/// })
	1129	///
	1130	/// The caller is responsible for reserving space in `l.air_instructions`.
	1131	fn addScalarizedShuffle(
	1132	l: *Legalize,
	1133	parent_block: *Block,
	1134	operand_vec: Air.Inst.Ref,
	1135	result_vec_ptr: Air.Inst.Ref,
	1136	index_ptr: Air.Inst.Ref,
	1137	out_idxs: []const InternPool.Index,
	1138	in_idxs: []const InternPool.Index,
	1139	) Error!void {
1431	1140	const pt = l.pt;
1432		const zcu = pt.zcu;
1433	1141
1434		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1435		const res_ty = orig_ty_op.ty.toType();
1436		const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1437		const int_ty = try pt.intType(.unsigned, int_bits);
1438		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1439		const res_elem_ty = res_ty.childType(zcu);
1440		const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1441		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
	1142	assert(out_idxs.len == in_idxs.len);
	1143	const n = out_idxs.len;
1442	1144
1443		var inst_buf: [20]Air.Inst.Index = undefined;
1444		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	1145	const idxs_ty = try pt.arrayType(.{ .len = n, .child = .usize_type });
	1146	const idxs_ptr_ty = try pt.singleConstPtrType(idxs_ty);
	1147	const manyptr_usize_ty = try pt.manyConstPtrType(.usize);
1445	1148
1446		var res_block: Block = .init(&inst_buf);
1447		{
1448		const res_alloc_inst = res_block.add(l, .{
1449		.tag = .alloc,
1450		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1451		});
1452		const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1453		const index_alloc_inst = res_block.add(l, .{
1454		.tag = .alloc,
1455		.data = .{ .ty = .ptr_usize },
1456		});
1457		_ = res_block.add(l, .{
1458		.tag = .store,
1459		.data = .{ .bin_op = .{
1460		.lhs = index_alloc_inst.toRef(),
1461		.rhs = .zero_usize,
1462		} },
1463		});
	1149	const out_idxs_ptr = try pt.intern(.{ .ptr = .{
	1150	.ty = manyptr_usize_ty.toIntern(),
	1151	.base_addr = .{ .uav = .{
	1152	.val = (try pt.aggregateValue(idxs_ty, out_idxs)).toIntern(),
	1153	.orig_ty = idxs_ptr_ty.toIntern(),
	1154	} },
	1155	.byte_offset = 0,
	1156	} });
	1157	const in_idxs_ptr = try pt.intern(.{ .ptr = .{
	1158	.ty = manyptr_usize_ty.toIntern(),
	1159	.base_addr = .{ .uav = .{
	1160	.val = (try pt.aggregateValue(idxs_ty, in_idxs)).toIntern(),
	1161	.orig_ty = idxs_ptr_ty.toIntern(),
	1162	} },
	1163	.byte_offset = 0,
	1164	} });
1464	1165
1465		var loop: Loop = .init(l, &res_block);
1466		loop.block = .init(res_block.stealRemainingCapacity());
1467		{
1468		const cur_index_inst = loop.block.add(l, .{
1469		.tag = .load,
1470		.data = .{ .ty_op = .{
1471		.ty = .usize_type,
1472		.operand = index_alloc_inst.toRef(),
1473		} },
1474		});
1475		_ = loop.block.add(l, .{
1476		.tag = .store,
1477		.data = .{ .bin_op = .{
1478		.lhs = loop.block.add(l, .{
1479		.tag = .ptr_elem_ptr,
1480		.data = .{ .ty_pl = .{
1481		.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1482		.payload = try l.addExtra(Air.Bin, .{
1483		.lhs = res_alloc_inst.toRef(),
1484		.rhs = cur_index_inst.toRef(),
1485		}),
1486		} },
1487		}).toRef(),
1488		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1489		.tag = .trunc,
1490		.data = .{ .ty_op = .{
1491		.ty = Air.internedToRef(elem_int_ty.toIntern()),
1492		.operand = loop.block.add(l, .{
1493		.tag = .shr,
1494		.data = .{ .bin_op = .{
1495		.lhs = int_ref,
1496		.rhs = loop.block.add(l, .{
1497		.tag = .mul,
1498		.data = .{ .bin_op = .{
1499		.lhs = loop.block.add(l, .{
1500		.tag = .intcast,
1501		.data = .{ .ty_op = .{
1502		.ty = Air.internedToRef(shift_ty.toIntern()),
1503		.operand = cur_index_inst.toRef(),
1504		} },
1505		}).toRef(),
1506		.rhs = try pt.intRef(shift_ty, elem_bits),
1507		} },
1508		}).toRef(),
1509		} },
1510		}).toRef(),
1511		} },
1512		}).toRef()),
1513		} },
1514		});
	1166	const main_block_inst = parent_block.add(l, .{
	1167	.tag = .block,
	1168	.data = .{ .ty_pl = .{
	1169	.ty = .void_type,
	1170	.payload = undefined,
	1171	} },
	1172	});
1515	1173
1516		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1517		l,
1518		.lt,
1519		cur_index_inst.toRef(),
1520		try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1),
1521		.{},
1522		)).toRef(), &loop.block, .{});
1523		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1524		{
1525		_ = loop_cond_br.then_block.add(l, .{
1526		.tag = .store,
1527		.data = .{ .bin_op = .{
1528		.lhs = index_alloc_inst.toRef(),
1529		.rhs = loop_cond_br.then_block.add(l, .{
1530		.tag = .add,
1531		.data = .{ .bin_op = .{
1532		.lhs = cur_index_inst.toRef(),
1533		.rhs = .one_usize,
1534		} },
1535		}).toRef(),
1536		} },
1537		});
1538		_ = loop_cond_br.then_block.add(l, .{
1539		.tag = .repeat,
1540		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1541		});
1542		}
1543		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1544		_ = loop_cond_br.else_block.add(l, .{
1545		.tag = .br,
1546		.data = .{ .br = .{
1547		.block_inst = orig_inst,
1548		.operand = loop_cond_br.else_block.add(l, .{
1549		.tag = .load,
1550		.data = .{ .ty_op = .{
1551		.ty = Air.internedToRef(res_ty.toIntern()),
1552		.operand = res_alloc_inst.toRef(),
1553		} },
1554		}).toRef(),
1555		} },
1556		});
1557		try loop_cond_br.finish(l);
1558		}
1559		try loop.finish(l);
1560		}
1561		return .{ .ty_pl = .{
1562		.ty = Air.internedToRef(res_ty.toIntern()),
1563		.payload = try l.addBlockBody(res_block.body()),
1564		} };
	1174	var inst_buf: [13]Air.Inst.Index = undefined;
	1175	var main_block: Block = .init(&inst_buf);
	1176
	1177	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
	1178
	1179	var loop: Loop = .init(l, &main_block);
	1180	loop.block = .init(main_block.stealRemainingCapacity());
	1181
	1182	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
	1183	const in_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(in_idxs_ptr), index_val).toRef();
	1184	const out_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(out_idxs_ptr), index_val).toRef();
	1185
	1186	const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, operand_vec, in_idx_val).toRef();
	1187	_ = loop.block.add(l, .{
	1188	.tag = .legalize_vec_store_elem,
	1189	.data = .{ .pl_op = .{
	1190	.operand = result_vec_ptr,
	1191	.payload = try l.addExtra(Air.Bin, .{
	1192	.lhs = out_idx_val,
	1193	.rhs = elem_val,
	1194	}),
	1195	} },
	1196	});
	1197
	1198	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, n - 1))).toRef();
	1199	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
	1200	condbr.then_block = .init(loop.block.stealRemainingCapacity());
	1201	condbr.then_block.addBr(l, main_block_inst, .void_value);
	1202
	1203	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
	1204	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
	1205	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
	1206	_ = condbr.else_block.add(l, .{
	1207	.tag = .repeat,
	1208	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
	1209	});
	1210
	1211	try condbr.finish(l);
	1212	try loop.finish(l);
	1213
	1214	const inst_data = l.air_instructions.items(.data);
	1215	inst_data[@intFromEnum(main_block_inst)].ty_pl.payload = try l.addBlockBody(main_block.body());
1565	1216	}
1566		fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
	1217	fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1567	1218	const pt = l.pt;
1568	1219	const zcu = pt.zcu;
1569	1220
1570		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1571		const res_ty = orig_ty_op.ty.toType();
1572		const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1573		const int_ty = try pt.intType(.unsigned, int_bits);
1574		const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1575		const res_elem_ty = res_ty.childType(zcu);
1576		const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1577		const elem_int_ty = try pt.intType(.unsigned, elem_bits);
	1221	const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1578	1222
1579		var inst_buf: [19]Air.Inst.Index = undefined;
	1223	const dest_ty = ty_op.ty.toType();
	1224	const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
	1225	else => true,
	1226	.array, .vector => legal: {
	1227	if (dest_ty.arrayLen(zcu) == 1) break :legal true;
	1228	const dest_elem_ty = dest_ty.childType(zcu);
	1229	break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
	1230	},
	1231	};
	1232
	1233	const operand_ty = l.typeOf(ty_op.operand);
	1234	const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
	1235	else => true,
	1236	.array, .vector => legal: {
	1237	if (operand_ty.arrayLen(zcu) == 1) break :legal true;
	1238	const operand_elem_ty = operand_ty.childType(zcu);
	1239	break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
	1240	},
	1241	};
	1242
	1243	if (dest_legal and operand_legal) return null;
	1244
	1245	if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
	1246	// from_ty and to_ty are both arrays or vectors of types with the same bit size,
	1247	// so we can do an elementwise bitcast.
	1248	return try l.scalarizeBlockPayload(orig_inst, .ty_op);
	1249	}
	1250
	1251	// Fallback path. Our strategy is to use an unsigned integer type as an intermediate
	1252	// "bag of bits" representation which can be manipulated by bitwise operations.
	1253
	1254	const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
	1255	assert(operand_ty.bitSize(zcu) == num_bits);
	1256	const uint_ty = try pt.intType(.unsigned, num_bits);
	1257	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
	1258
	1259	var inst_buf: [39]Air.Inst.Index = undefined;
	1260	var main_block: Block = .init(&inst_buf);
1580	1261	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1581	1262
1582		var res_block: Block = .init(&inst_buf);
1583		{
1584		const res_alloc_inst = res_block.add(l, .{
1585		.tag = .alloc,
1586		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1587		});
1588		const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);
1589		const index_alloc_inst = res_block.add(l, .{
1590		.tag = .alloc,
1591		.data = .{ .ty = .ptr_usize },
1592		});
1593		_ = res_block.add(l, .{
1594		.tag = .store,
1595		.data = .{ .bin_op = .{
1596		.lhs = index_alloc_inst.toRef(),
1597		.rhs = .zero_usize,
	1263	// First, convert `operand_ty` to `uint_ty` (`uN`).
	1264
	1265	const uint_val: Air.Inst.Ref = uint_val: {
	1266	if (operand_legal) {
	1267	_ = main_block.stealCapacity(19);
	1268	break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
	1269	}
	1270
	1271	// %1 = block({
	1272	// %2 = alloc(*usize)
	1273	// %3 = alloc(*uN)
	1274	// %4 = store(%2, <usize, operand_len>)
	1275	// %5 = store(%3, <uN, 0>)
	1276	// %6 = loop({
	1277	// %7 = load(%2)
	1278	// %8 = array_elem_val(orig_operand, %7)
	1279	// %9 = bitcast(uE, %8)
	1280	// %10 = intcast(uN, %9)
	1281	// %11 = load(%3)
	1282	// %12 = shl_exact(%11, <uS, E>)
	1283	// %13 = bit_or(%12, %10)
	1284	// %14 = cmp_eq(%4, @zero_usize)
	1285	// %15 = cond_br(%14, {
	1286	// %16 = br(%1, %13)
	1287	// }, {
	1288	// %17 = store(%3, %13)
	1289	// %18 = sub(%7, @one_usize)
	1290	// %19 = store(%2, %18)
	1291	// %20 = repeat(%6)
	1292	// })
	1293	// })
	1294	// })
	1295
	1296	const elem_bits = operand_ty.childType(zcu).bitSize(zcu);
	1297	const elem_bits_val = try pt.intValue(shift_ty, elem_bits);
	1298	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
	1299
	1300	const uint_block_inst = main_block.add(l, .{
	1301	.tag = .block,
	1302	.data = .{ .ty_pl = .{
	1303	.ty = .fromType(uint_ty),
	1304	.payload = undefined,
1598	1305	} },
1599	1306	});
	1307	var uint_block: Block = .init(main_block.stealCapacity(19));
	1308
	1309	const index_ptr = uint_block.addTy(l, .alloc, .ptr_usize).toRef();
	1310	const result_ptr = uint_block.addTy(l, .alloc, try pt.singleMutPtrType(uint_ty)).toRef();
	1311	_ = uint_block.addBinOp(
	1312	l,
	1313	.store,
	1314	index_ptr,
	1315	.fromValue(try pt.intValue(.usize, operand_ty.arrayLen(zcu))),
	1316	);
	1317	_ = uint_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.intValue(uint_ty, 0)));
	1318
	1319	var loop: Loop = .init(l, &uint_block);
	1320	loop.block = .init(uint_block.stealRemainingCapacity());
	1321
	1322	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
	1323	const raw_elem = loop.block.addBinOp(
	1324	l,
	1325	if (operand_ty.zigTypeTag(zcu) == .vector) .legalize_vec_elem_val else .array_elem_val,
	1326	ty_op.operand,
	1327	index_val,
	1328	).toRef();
	1329	const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
	1330	const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
	1331	const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
	1332	const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
	1333	const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
	1334
	1335	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .zero_usize).toRef();
	1336	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
	1337
	1338	condbr.then_block = .init(loop.block.stealRemainingCapacity());
	1339	condbr.then_block.addBr(l, uint_block_inst, new_result);
1600	1340
1601		var loop: Loop = .init(l, &res_block);
1602		loop.block = .init(res_block.stealRemainingCapacity());
1603		{
1604		const cur_index_inst = loop.block.add(l, .{
1605		.tag = .load,
1606		.data = .{ .ty_op = .{
1607		.ty = .usize_type,
1608		.operand = index_alloc_inst.toRef(),
1609		} },
1610		});
1611		_ = loop.block.add(l, .{
1612		.tag = .vector_store_elem,
1613		.data = .{ .vector_store_elem = .{
1614		.vector_ptr = res_alloc_inst.toRef(),
1615		.payload = try l.addExtra(Air.Bin, .{
1616		.lhs = cur_index_inst.toRef(),
1617		.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1618		.tag = .trunc,
1619		.data = .{ .ty_op = .{
1620		.ty = Air.internedToRef(elem_int_ty.toIntern()),
1621		.operand = loop.block.add(l, .{
1622		.tag = .shr,
1623		.data = .{ .bin_op = .{
1624		.lhs = int_ref,
1625		.rhs = loop.block.add(l, .{
1626		.tag = .mul,
1627		.data = .{ .bin_op = .{
1628		.lhs = loop.block.add(l, .{
1629		.tag = .intcast,
1630		.data = .{ .ty_op = .{
1631		.ty = Air.internedToRef(shift_ty.toIntern()),
1632		.operand = cur_index_inst.toRef(),
1633		} },
1634		}).toRef(),
1635		.rhs = try pt.intRef(shift_ty, elem_bits),
1636		} },
1637		}).toRef(),
1638		} },
1639		}).toRef(),
1640		} },
1641		}).toRef()),
1642		}),
1643		} },
1644		});
	1341	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
	1342	_ = condbr.else_block.addBinOp(l, .store, result_ptr, new_result);
	1343	const new_index_val = condbr.else_block.addBinOp(l, .sub, index_val, .one_usize).toRef();
	1344	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
	1345	_ = condbr.else_block.add(l, .{
	1346	.tag = .repeat,
	1347	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
	1348	});
1645	1349
1646		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1647		l,
1648		.lt,
1649		cur_index_inst.toRef(),
1650		try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),
1651		.{},
1652		)).toRef(), &loop.block, .{});
1653		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1654		{
1655		_ = loop_cond_br.then_block.add(l, .{
1656		.tag = .store,
1657		.data = .{ .bin_op = .{
1658		.lhs = index_alloc_inst.toRef(),
1659		.rhs = loop_cond_br.then_block.add(l, .{
1660		.tag = .add,
1661		.data = .{ .bin_op = .{
1662		.lhs = cur_index_inst.toRef(),
1663		.rhs = .one_usize,
1664		} },
1665		}).toRef(),
	1350	try condbr.finish(l);
	1351	try loop.finish(l);
	1352
	1353	const inst_data = l.air_instructions.items(.data);
	1354	inst_data[@intFromEnum(uint_block_inst)].ty_pl.payload = try l.addBlockBody(uint_block.body());
	1355
	1356	break :uint_val uint_block_inst.toRef();
	1357	};
	1358
	1359	// Now convert `uint_ty` (`uN`) to `dest_ty`.
	1360
	1361	if (dest_legal) {
	1362	_ = main_block.stealCapacity(17);
	1363	const result = main_block.addBitCast(l, dest_ty, uint_val);
	1364	main_block.addBr(l, orig_inst, result);
	1365	} else {
	1366	// %1 = alloc(*usize)
	1367	// %2 = alloc(*@Vector(N, Result))
	1368	// %3 = store(%1, @zero_usize)
	1369	// %4 = loop({
	1370	// %5 = load(%1)
	1371	// %6 = mul(%5, <usize, E>)
	1372	// %7 = intcast(uS, %6)
	1373	// %8 = shr(uint_val, %7)
	1374	// %9 = trunc(uE, %8)
	1375	// %10 = bitcast(Result, %9)
	1376	// %11 = legalize_vec_store_elem(%2, %5, %10)
	1377	// %12 = cmp_eq(%5, <usize, vec_len>)
	1378	// %13 = cond_br(%12, {
	1379	// %14 = load(%2)
	1380	// %15 = br(%0, %14)
	1381	// }, {
	1382	// %16 = add(%5, @one_usize)
	1383	// %17 = store(%1, %16)
	1384	// %18 = repeat(%4)
	1385	// })
	1386	// })
	1387	//
	1388	// The result might be an array, in which case `legalize_vec_store_elem`
	1389	// becomes `ptr_elem_ptr` followed by `store`.
	1390
	1391	const elem_ty = dest_ty.childType(zcu);
	1392	const elem_bits = elem_ty.bitSize(zcu);
	1393	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
	1394
	1395	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
	1396	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(dest_ty)).toRef();
	1397	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
	1398
	1399	var loop: Loop = .init(l, &main_block);
	1400	loop.block = .init(main_block.stealRemainingCapacity());
	1401
	1402	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
	1403	const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
	1404	const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
	1405	const shifted_uint = loop.block.addBinOp(l, .shr, index_val, casted_bit_offset).toRef();
	1406	const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
	1407	const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
	1408	switch (dest_ty.zigTypeTag(zcu)) {
	1409	.array => {
	1410	const elem_ptr = loop.block.add(l, .{
	1411	.tag = .ptr_elem_ptr,
	1412	.data = .{ .ty_pl = .{
	1413	.ty = .fromType(try pt.singleMutPtrType(elem_ty)),
	1414	.payload = try l.addExtra(Air.Bin, .{
	1415	.lhs = result_ptr,
	1416	.rhs = index_val,
	1417	}),
	1418	} },
	1419	}).toRef();
	1420	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
	1421	},
	1422	.vector => {
	1423	_ = loop.block.add(l, .{
	1424	.tag = .legalize_vec_store_elem,
	1425	.data = .{ .pl_op = .{
	1426	.operand = result_ptr,
	1427	.payload = try l.addExtra(Air.Bin, .{
	1428	.lhs = index_val,
	1429	.rhs = elem_val,
	1430	}),
1666	1431	} },
1667	1432	});
1668		_ = loop_cond_br.then_block.add(l, .{
1669		.tag = .repeat,
1670		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1671		});
1672		}
1673		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1674		_ = loop_cond_br.else_block.add(l, .{
1675		.tag = .br,
1676		.data = .{ .br = .{
1677		.block_inst = orig_inst,
1678		.operand = loop_cond_br.else_block.add(l, .{
1679		.tag = .load,
1680		.data = .{ .ty_op = .{
1681		.ty = Air.internedToRef(res_ty.toIntern()),
1682		.operand = res_alloc_inst.toRef(),
1683		} },
1684		}).toRef(),
1685		} },
1686		});
1687		try loop_cond_br.finish(l);
	1433	_ = loop.block.stealCapacity(1);
	1434	},
	1435	else => unreachable,
1688	1436	}
	1437
	1438	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, dest_ty.arrayLen(zcu) - 1))).toRef();
	1439
	1440	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
	1441
	1442	condbr.then_block = .init(loop.block.stealRemainingCapacity());
	1443	const result_val = condbr.then_block.addTyOp(l, .load, dest_ty, result_ptr).toRef();
	1444	condbr.then_block.addBr(l, orig_inst, result_val);
	1445
	1446	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
	1447	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
	1448	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
	1449	_ = condbr.else_block.add(l, .{
	1450	.tag = .repeat,
	1451	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
	1452	});
	1453
	1454	try condbr.finish(l);
1689	1455	try loop.finish(l);
1690	1456	}
	1457
1691	1458	return .{ .ty_pl = .{
1692		.ty = Air.internedToRef(res_ty.toIntern()),
1693		.payload = try l.addBlockBody(res_block.body()),
	1459	.ty = .fromType(dest_ty),
	1460	.payload = try l.addBlockBody(main_block.body()),
1694	1461	} };
1695	1462	}
1696	1463	fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
...	...	@@ -1698,169 +1465,145 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!
1698	1465	const zcu = pt.zcu;
1699	1466
1700	1467	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
1701		const res_ty = l.typeOfIndex(orig_inst);
1702		const wrapped_res_ty = res_ty.fieldType(0, zcu);
1703		const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu);
1704		const res_len = wrapped_res_ty.vectorLen(zcu);
	1468	const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
	1469
	1470	const vec_tuple_ty = l.typeOfIndex(orig_inst);
	1471	const vec_int_ty = vec_tuple_ty.fieldType(0, zcu);
	1472	const vec_overflow_ty = vec_tuple_ty.fieldType(1, zcu);
	1473
	1474	assert(l.typeOf(orig_operands.lhs).toIntern() == vec_int_ty.toIntern());
	1475	if (orig.tag != .shl_with_overflow) {
	1476	assert(l.typeOf(orig_operands.rhs).toIntern() == vec_int_ty.toIntern());
	1477	}
	1478
	1479	const scalar_int_ty = vec_int_ty.childType(zcu);
	1480	const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
	1481
	1482	// %1 = block(struct { @Vector(N, Int), @Vector(N, u1) }, {
	1483	// %2 = alloc(*usize)
	1484	// %3 = alloc(*struct { @Vector(N, Int), @Vector(N, u1) })
	1485	// %4 = struct_field_ptr_index_0(*@Vector(N, Int), %3)
	1486	// %5 = struct_field_ptr_index_1(*@Vector(N, u1), %3)
	1487	// %6 = store(%2, @zero_usize)
	1488	// %7 = loop({
	1489	// %8 = load(%2)
	1490	// %9 = legalize_vec_elem_val(orig_lhs, %8)
	1491	// %10 = legalize_vec_elem_val(orig_rhs, %8)
	1492	// %11 = ???_with_overflow(struct { Int, u1 }, %9, %10)
	1493	// %12 = struct_field_val(%11, 0)
	1494	// %13 = struct_field_val(%11, 1)
	1495	// %14 = legalize_vec_store_elem(%4, %8, %12)
	1496	// %15 = legalize_vec_store_elem(%4, %8, %13)
	1497	// %16 = cmp_eq(%8, <usize, N-1>)
	1498	// %17 = cond_br(%16, {
	1499	// %18 = load(%3)
	1500	// %19 = br(%1, %18)
	1501	// }, {
	1502	// %20 = add(%8, @one_usize)
	1503	// %21 = store(%2, %20)
	1504	// %22 = repeat(%7)
	1505	// })
	1506	// })
	1507	// })
	1508
	1509	const elems_len = vec_int_ty.vectorLen(zcu);
1705	1510
1706	1511	var inst_buf: [21]Air.Inst.Index = undefined;
	1512	var main_block: Block = .init(&inst_buf);
1707	1513	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1708	1514
1709		var res_block: Block = .init(&inst_buf);
1710		{
1711		const res_alloc_inst = res_block.add(l, .{
1712		.tag = .alloc,
1713		.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1714		});
1715		const ptr_wrapped_res_inst = res_block.add(l, .{
1716		.tag = .struct_field_ptr_index_0,
1717		.data = .{ .ty_op = .{
1718		.ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()),
1719		.operand = res_alloc_inst.toRef(),
1720		} },
1721		});
1722		const ptr_overflow_res_inst = res_block.add(l, .{
1723		.tag = .struct_field_ptr_index_1,
1724		.data = .{ .ty_op = .{
1725		.ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()),
1726		.operand = res_alloc_inst.toRef(),
1727		} },
1728		});
1729		const index_alloc_inst = res_block.add(l, .{
1730		.tag = .alloc,
1731		.data = .{ .ty = .ptr_usize },
1732		});
1733		_ = res_block.add(l, .{
1734		.tag = .store,
1735		.data = .{ .bin_op = .{
1736		.lhs = index_alloc_inst.toRef(),
1737		.rhs = .zero_usize,
1738		} },
1739		});
	1515	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
	1516	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(vec_tuple_ty)).toRef();
	1517	const result_int_ptr = main_block.addTyOp(
	1518	l,
	1519	.struct_field_ptr_index_0,
	1520	try pt.singleMutPtrType(vec_int_ty),
	1521	result_ptr,
	1522	).toRef();
	1523	const result_overflow_ptr = main_block.addTyOp(
	1524	l,
	1525	.struct_field_ptr_index_1,
	1526	try pt.singleMutPtrType(vec_overflow_ty),
	1527	result_ptr,
	1528	).toRef();
	1529
	1530	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
	1531
	1532	var loop: Loop = .init(l, &main_block);
	1533	loop.block = .init(main_block.stealRemainingCapacity());
	1534
	1535	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
	1536	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.lhs, index_val).toRef();
	1537	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.rhs, index_val).toRef();
	1538	const elem_result = loop.block.add(l, .{
	1539	.tag = orig.tag,
	1540	.data = .{ .ty_pl = .{
	1541	.ty = .fromType(scalar_tuple_ty),
	1542	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
	1543	} },
	1544	}).toRef();
	1545	const int_elem = loop.block.add(l, .{
	1546	.tag = .struct_field_val,
	1547	.data = .{ .ty_pl = .{
	1548	.ty = .fromType(scalar_int_ty),
	1549	.payload = try l.addExtra(Air.StructField, .{
	1550	.struct_operand = elem_result,
	1551	.field_index = 0,
	1552	}),
	1553	} },
	1554	}).toRef();
	1555	const overflow_elem = loop.block.add(l, .{
	1556	.tag = .struct_field_val,
	1557	.data = .{ .ty_pl = .{
	1558	.ty = .u1_type,
	1559	.payload = try l.addExtra(Air.StructField, .{
	1560	.struct_operand = elem_result,
	1561	.field_index = 1,
	1562	}),
	1563	} },
	1564	}).toRef();
	1565	_ = loop.block.add(l, .{
	1566	.tag = .legalize_vec_store_elem,
	1567	.data = .{ .pl_op = .{
	1568	.operand = result_int_ptr,
	1569	.payload = try l.addExtra(Air.Bin, .{
	1570	.lhs = index_val,
	1571	.rhs = int_elem,
	1572	}),
	1573	} },
	1574	});
	1575	_ = loop.block.add(l, .{
	1576	.tag = .legalize_vec_store_elem,
	1577	.data = .{ .pl_op = .{
	1578	.operand = result_overflow_ptr,
	1579	.payload = try l.addExtra(Air.Bin, .{
	1580	.lhs = index_val,
	1581	.rhs = overflow_elem,
	1582	}),
	1583	} },
	1584	});
1740	1585
1741		var loop: Loop = .init(l, &res_block);
1742		loop.block = .init(res_block.stealRemainingCapacity());
1743		{
1744		const cur_index_inst = loop.block.add(l, .{
1745		.tag = .load,
1746		.data = .{ .ty_op = .{
1747		.ty = .usize_type,
1748		.operand = index_alloc_inst.toRef(),
1749		} },
1750		});
1751		const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1752		const res_elem = loop.block.add(l, .{
1753		.tag = orig.tag,
1754		.data = .{ .ty_pl = .{
1755		.ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{
1756		.types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type },
1757		.values = &(.{.none} ** 2),
1758		})),
1759		.payload = try l.addExtra(Air.Bin, .{
1760		.lhs = loop.block.add(l, .{
1761		.tag = .array_elem_val,
1762		.data = .{ .bin_op = .{
1763		.lhs = extra.lhs,
1764		.rhs = cur_index_inst.toRef(),
1765		} },
1766		}).toRef(),
1767		.rhs = loop.block.add(l, .{
1768		.tag = .array_elem_val,
1769		.data = .{ .bin_op = .{
1770		.lhs = extra.rhs,
1771		.rhs = cur_index_inst.toRef(),
1772		} },
1773		}).toRef(),
1774		}),
1775		} },
1776		});
1777		_ = loop.block.add(l, .{
1778		.tag = .vector_store_elem,
1779		.data = .{ .vector_store_elem = .{
1780		.vector_ptr = ptr_overflow_res_inst.toRef(),
1781		.payload = try l.addExtra(Air.Bin, .{
1782		.lhs = cur_index_inst.toRef(),
1783		.rhs = loop.block.add(l, .{
1784		.tag = .struct_field_val,
1785		.data = .{ .ty_pl = .{
1786		.ty = .u1_type,
1787		.payload = try l.addExtra(Air.StructField, .{
1788		.struct_operand = res_elem.toRef(),
1789		.field_index = 1,
1790		}),
1791		} },
1792		}).toRef(),
1793		}),
1794		} },
1795		});
1796		_ = loop.block.add(l, .{
1797		.tag = .vector_store_elem,
1798		.data = .{ .vector_store_elem = .{
1799		.vector_ptr = ptr_wrapped_res_inst.toRef(),
1800		.payload = try l.addExtra(Air.Bin, .{
1801		.lhs = cur_index_inst.toRef(),
1802		.rhs = loop.block.add(l, .{
1803		.tag = .struct_field_val,
1804		.data = .{ .ty_pl = .{
1805		.ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()),
1806		.payload = try l.addExtra(Air.StructField, .{
1807		.struct_operand = res_elem.toRef(),
1808		.field_index = 0,
1809		}),
1810		} },
1811		}).toRef(),
1812		}),
1813		} },
1814		});
	1586	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, elems_len - 1))).toRef();
	1587	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
	1588
	1589	condbr.then_block = .init(loop.block.stealRemainingCapacity());
	1590	const result_val = condbr.then_block.addTyOp(l, .load, vec_tuple_ty, result_ptr).toRef();
	1591	condbr.then_block.addBr(l, orig_inst, result_val);
	1592
	1593	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
	1594	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
	1595	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
	1596	_ = condbr.else_block.add(l, .{
	1597	.tag = .repeat,
	1598	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
	1599	});
	1600
	1601	try condbr.finish(l);
	1602	try loop.finish(l);
1815	1603
1816		var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1817		l,
1818		.lt,
1819		cur_index_inst.toRef(),
1820		try pt.intRef(.usize, res_len - 1),
1821		.{},
1822		)).toRef(), &loop.block, .{});
1823		loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1824		{
1825		_ = loop_cond_br.then_block.add(l, .{
1826		.tag = .store,
1827		.data = .{ .bin_op = .{
1828		.lhs = index_alloc_inst.toRef(),
1829		.rhs = loop_cond_br.then_block.add(l, .{
1830		.tag = .add,
1831		.data = .{ .bin_op = .{
1832		.lhs = cur_index_inst.toRef(),
1833		.rhs = .one_usize,
1834		} },
1835		}).toRef(),
1836		} },
1837		});
1838		_ = loop_cond_br.then_block.add(l, .{
1839		.tag = .repeat,
1840		.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1841		});
1842		}
1843		loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1844		_ = loop_cond_br.else_block.add(l, .{
1845		.tag = .br,
1846		.data = .{ .br = .{
1847		.block_inst = orig_inst,
1848		.operand = loop_cond_br.else_block.add(l, .{
1849		.tag = .load,
1850		.data = .{ .ty_op = .{
1851		.ty = Air.internedToRef(res_ty.toIntern()),
1852		.operand = res_alloc_inst.toRef(),
1853		} },
1854		}).toRef(),
1855		} },
1856		});
1857		try loop_cond_br.finish(l);
1858		}
1859		try loop.finish(l);
1860		}
1861	1604	return .{ .ty_pl = .{
1862		.ty = Air.internedToRef(res_ty.toIntern()),
1863		.payload = try l.addBlockBody(res_block.body()),
	1605	.ty = .fromType(vec_tuple_ty),
	1606	.payload = try l.addBlockBody(main_block.body()),
1864	1607	} };
1865	1608	}
1866	1609
...	...	@@ -2047,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz
2047	1790
2048	1791	// We emit 9 instructions in the worst case.
2049	1792	var inst_buf: [9]Air.Inst.Index = undefined;
2050		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	1793	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2051	1794	var main_block: Block = .init(&inst_buf);
2052	1795
2053	1796	// This check is a bit annoying because of floating-point rounding and the fact that this
...	...	@@ -2231,37 +1974,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2231	1974	} };
2232	1975	}
2233	1976
2234		fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2235		const pt = l.pt;
2236		const zcu = pt.zcu;
2237		const ip = &zcu.intern_pool;
2238
2239		const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
2240		const res_ty = orig_ty_op.ty.toType();
2241		const res_ty_key = ip.indexToKey(res_ty.toIntern());
2242		const operand_ty = l.typeOf(orig_ty_op.operand);
2243		const operand_ty_key = ip.indexToKey(operand_ty.toIntern());
2244		_ = res_ty_key;
2245		_ = operand_ty_key;
2246
2247		var inst_buf: [1]Air.Inst.Index = undefined;
2248		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2249
2250		var res_block: Block = .init(&inst_buf);
2251		{
2252		_ = res_block.add(l, .{
2253		.tag = .br,
2254		.data = .{ .br = .{
2255		.block_inst = orig_inst,
2256		.operand = try pt.undefRef(res_ty),
2257		} },
2258		});
2259		}
2260		return .{ .ty_pl = .{
2261		.ty = Air.internedToRef(res_ty.toIntern()),
2262		.payload = try l.addBlockBody(res_block.body()),
2263		} };
2264		}
2265	1977	fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2266	1978	const pt = l.pt;
2267	1979	const zcu = pt.zcu;
...	...	@@ -2431,89 +2143,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err
2431	2143	const field_ty = orig_ty_pl.ty.toType();
2432	2144	const agg_ty = l.typeOf(orig_extra.struct_operand);
2433	2145
	2146	const agg_bits: u16 = @intCast(agg_ty.bitSize(zcu));
	2147	const bit_offset = zcu.structPackedFieldBitOffset(zcu.typeToStruct(agg_ty).?, orig_extra.field_index);
	2148
	2149	const agg_int_ty = try pt.intType(.unsigned, agg_bits);
	2150	const field_int_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
	2151
	2152	const agg_shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, agg_bits));
	2153	const bit_offset_ref: Air.Inst.Ref = .fromValue(try pt.intValue(agg_shift_ty, bit_offset));
	2154
2434	2155	var inst_buf: [5]Air.Inst.Index = undefined;
	2156	var main_block: Block = .init(&inst_buf);
2435	2157	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2436	2158
2437		var res_block: Block = .init(&inst_buf);
2438		{
2439		const agg_alloc_inst = res_block.add(l, .{
2440		.tag = .alloc,
2441		.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },
2442		});
2443		_ = res_block.add(l, .{
2444		.tag = .store,
2445		.data = .{ .bin_op = .{
2446		.lhs = agg_alloc_inst.toRef(),
2447		.rhs = orig_extra.struct_operand,
2448		} },
2449		});
2450		_ = res_block.add(l, .{
2451		.tag = .br,
2452		.data = .{ .br = .{
2453		.block_inst = orig_inst,
2454		.operand = res_block.add(l, .{
2455		.tag = .load,
2456		.data = .{ .ty_op = .{
2457		.ty = Air.internedToRef(field_ty.toIntern()),
2458		.operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(),
2459		} },
2460		}).toRef(),
2461		} },
2462		});
2463		}
	2159	const agg_int = main_block.addBitCast(l, agg_int_ty, orig_extra.struct_operand);
	2160	const shifted_agg_int = main_block.addBinOp(l, .shr, agg_int, bit_offset_ref).toRef();
	2161	const field_int = main_block.addTyOp(l, .trunc, field_int_ty, shifted_agg_int).toRef();
	2162	const field_val = main_block.addBitCast(l, field_ty, field_int);
	2163	main_block.addBr(l, orig_inst, field_val);
	2164
2464	2165	return .{ .ty_pl = .{
2465		.ty = Air.internedToRef(field_ty.toIntern()),
2466		.payload = try l.addBlockBody(res_block.body()),
	2166	.ty = .fromType(field_ty),
	2167	.payload = try l.addBlockBody(main_block.body()),
2467	2168	} };
2468	2169	}
2469	2170	fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2470	2171	const pt = l.pt;
2471	2172	const zcu = pt.zcu;
	2173	const gpa = zcu.gpa;
2472	2174
2473	2175	const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;
2474		const field_ty = orig_ty_pl.ty.toType();
2475	2176	const agg_ty = orig_ty_pl.ty.toType();
2476	2177	const agg_field_count = agg_ty.structFieldCount(zcu);
2477	2178
2478		const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index;
2479		var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
2480		std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
2481		const gpa = stack.get();
	2179	var sfba_state = std.heap.stackFallback(@sizeOf([4 * 32 + 2]Air.Inst.Index), gpa);
	2180	const sfba = sfba_state.get();
2482	2181
2483		const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2);
2484		defer gpa.free(inst_buf);
2485		try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
	2182	const inst_buf = try sfba.alloc(Air.Inst.Index, 4 * agg_field_count + 2);
	2183	defer sfba.free(inst_buf);
2486	2184
2487		var res_block: Block = .init(inst_buf);
2488		{
2489		const agg_alloc_inst = res_block.add(l, .{
2490		.tag = .alloc,
2491		.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },
2492		});
2493		for (0..agg_field_count, orig_ty_pl.payload..) \|field_index, extra_index\| _ = res_block.add(l, .{
2494		.tag = .store,
2495		.data = .{ .bin_op = .{
2496		.lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(),
2497		.rhs = @enumFromInt(l.air_extra.items[extra_index]),
2498		} },
2499		});
2500		_ = res_block.add(l, .{
2501		.tag = .br,
2502		.data = .{ .br = .{
2503		.block_inst = orig_inst,
2504		.operand = res_block.add(l, .{
2505		.tag = .load,
2506		.data = .{ .ty_op = .{
2507		.ty = Air.internedToRef(field_ty.toIntern()),
2508		.operand = agg_alloc_inst.toRef(),
2509		} },
2510		}).toRef(),
2511		} },
2512		});
	2185	var main_block: Block = .init(inst_buf);
	2186	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
	2187
	2188	const num_bits: u16 = @intCast(agg_ty.bitSize(zcu));
	2189	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
	2190	const uint_ty = try pt.intType(.unsigned, num_bits);
	2191	var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
	2192
	2193	var field_idx = agg_field_count;
	2194	while (field_idx > 0) {
	2195	field_idx -= 1;
	2196	const field_ty = agg_ty.fieldType(field_idx, zcu);
	2197	const field_uint_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
	2198	const field_bit_size_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, field_ty.bitSize(zcu)));
	2199	const field_val: Air.Inst.Ref = @enumFromInt(l.air_extra.items[orig_ty_pl.payload + field_idx]);
	2200
	2201	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
	2202	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
	2203	const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
	2204	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2513	2205	}
	2206
	2207	const result = main_block.addBitCast(l, agg_ty, cur_uint);
	2208	main_block.addBr(l, orig_inst, result);
	2209
2514	2210	return .{ .ty_pl = .{
2515		.ty = Air.internedToRef(field_ty.toIntern()),
2516		.payload = try l.addBlockBody(res_block.body()),
	2211	.ty = .fromType(agg_ty),
	2212	.payload = try l.addBlockBody(main_block.body()),
2517	2213	} };
2518	2214	}
2519	2215
...	...	@@ -2571,6 +2267,36 @@ const Block = struct {
2571	2267	b.len += 1;
2572	2268	return inst;
2573	2269	}
	2270	fn addBr(b: Block, l: Legalize, target: Air.Inst.Index, operand: Air.Inst.Ref) void {
	2271	_ = b.add(l, .{
	2272	.tag = .br,
	2273	.data = .{ .br = .{ .block_inst = target, .operand = operand } },
	2274	});
	2275	}
	2276	fn addTy(b: Block, l: Legalize, tag: Air.Inst.Tag, ty: Type) Air.Inst.Index {
	2277	return b.add(l, .{ .tag = tag, .data = .{ .ty = ty } });
	2278	}
	2279	fn addBinOp(b: Block, l: Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
	2280	return b.add(l, .{
	2281	.tag = tag,
	2282	.data = .{ .bin_op = .{ .lhs = lhs, .rhs = rhs } },
	2283	});
	2284	}
	2285	fn addUnOp(b: Block, l: Legalize, tag: Air.Inst.Tag, operand: Air.Inst.Ref) Air.Inst.Index {
	2286	return b.add(l, .{
	2287	.tag = tag,
	2288	.data = .{ .un_op = operand },
	2289	});
	2290	}
	2291	fn addTyOp(b: Block, l: Legalize, tag: Air.Inst.Tag, ty: Type, operand: Air.Inst.Ref) Air.Inst.Index {
	2292	return b.add(l, .{
	2293	.tag = tag,
	2294	.data = .{ .ty_op = .{
	2295	.ty = .fromType(ty),
	2296	.operand = operand,
	2297	} },
	2298	});
	2299	}
2574	2300
2575	2301	/// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
2576	2302	/// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
...	...	@@ -2625,14 +2351,27 @@ const Block = struct {
2625	2351	} },
2626	2352	});
2627	2353	}
	2354	return addCmpScalar(b, l, op, lhs, rhs, opts.optimized);
	2355	}
	2356
	2357	/// Similar to `addCmp`, but for scalars only. Unlike `addCmp`, this function is
	2358	/// infallible, because it doesn't need to add entries to `extra`.
	2359	fn addCmpScalar(
	2360	b: *Block,
	2361	l: *Legalize,
	2362	op: std.math.CompareOperator,
	2363	lhs: Air.Inst.Ref,
	2364	rhs: Air.Inst.Ref,
	2365	optimized: bool,
	2366	) Air.Inst.Index {
2628	2367	return b.add(l, .{
2629	2368	.tag = switch (op) {
2630		.lt => if (opts.optimized) .cmp_lt_optimized else .cmp_lt,
2631		.lte => if (opts.optimized) .cmp_lte_optimized else .cmp_lte,
2632		.eq => if (opts.optimized) .cmp_eq_optimized else .cmp_eq,
2633		.gte => if (opts.optimized) .cmp_gte_optimized else .cmp_gte,
2634		.gt => if (opts.optimized) .cmp_gt_optimized else .cmp_gt,
2635		.neq => if (opts.optimized) .cmp_neq_optimized else .cmp_neq,
	2369	.lt => if (optimized) .cmp_lt_optimized else .cmp_lt,
	2370	.lte => if (optimized) .cmp_lte_optimized else .cmp_lte,
	2371	.eq => if (optimized) .cmp_eq_optimized else .cmp_eq,
	2372	.gte => if (optimized) .cmp_gte_optimized else .cmp_gte,
	2373	.gt => if (optimized) .cmp_gt_optimized else .cmp_gt,
	2374	.neq => if (optimized) .cmp_neq_optimized else .cmp_neq,
2636	2375	},
2637	2376	.data = .{ .bin_op = .{
2638	2377	.lhs = lhs,
...	...	@@ -2641,93 +2380,6 @@ const Block = struct {
2641	2380	});
2642	2381	}
2643	2382
2644		/// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add`
2645		/// that selects the optimized instruction encoding to use, although it does compute the
2646		/// proper field pointer type.
2647		fn addStructFieldPtr(
2648		b: *Block,
2649		l: *Legalize,
2650		struct_operand: Air.Inst.Ref,
2651		field_index: usize,
2652		) Error!Air.Inst.Index {
2653		const pt = l.pt;
2654		const zcu = pt.zcu;
2655
2656		const agg_ptr_ty = l.typeOf(struct_operand);
2657		const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu);
2658		const agg_ty: Type = .fromInterned(agg_ptr_info.child);
2659		const agg_ptr_align = switch (agg_ptr_info.flags.alignment) {
2660		.none => agg_ty.abiAlignment(zcu),
2661		else => \|agg_ptr_align\| agg_ptr_align,
2662		};
2663		const agg_layout = agg_ty.containerLayout(zcu);
2664		const field_ty = agg_ty.fieldType(field_index, zcu);
2665		var field_ptr_info: InternPool.Key.PtrType = .{
2666		.child = field_ty.toIntern(),
2667		.flags = .{
2668		.is_const = agg_ptr_info.flags.is_const,
2669		.is_volatile = agg_ptr_info.flags.is_volatile,
2670		.address_space = agg_ptr_info.flags.address_space,
2671		},
2672		};
2673		field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) {
2674		.auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align),
2675		.@"extern" => switch (agg_ty.zigTypeTag(zcu)) {
2676		else => unreachable,
2677		.@"struct" => .fromLog2Units(@min(
2678		agg_ptr_align.toLog2Units(),
2679		@ctz(agg_ty.structFieldOffset(field_index, zcu)),
2680		)),
2681		.@"union" => agg_ptr_align,
2682		},
2683		.@"packed" => switch (agg_ty.zigTypeTag(zcu)) {
2684		else => unreachable,
2685		.@"struct" => {
2686		const packed_offset = agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt);
2687		field_ptr_info.packed_offset = packed_offset;
2688		break :field_ptr_align agg_ptr_align;
2689		},
2690		.@"union" => {
2691		field_ptr_info.packed_offset = .{
2692		.host_size = switch (agg_ptr_info.packed_offset.host_size) {
2693		0 => @intCast(agg_ty.abiSize(zcu)),
2694		else => \|host_size\| host_size,
2695		},
2696		.bit_offset = agg_ptr_info.packed_offset.bit_offset,
2697		};
2698		break :field_ptr_align agg_ptr_align;
2699		},
2700		},
2701		};
2702		const field_ptr_ty = try pt.ptrType(field_ptr_info);
2703		const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern());
2704		return switch (field_index) {
2705		inline 0...3 => \|ct_field_index\| b.add(l, .{
2706		.tag = switch (ct_field_index) {
2707		0 => .struct_field_ptr_index_0,
2708		1 => .struct_field_ptr_index_1,
2709		2 => .struct_field_ptr_index_2,
2710		3 => .struct_field_ptr_index_3,
2711		else => comptime unreachable,
2712		},
2713		.data = .{ .ty_op = .{
2714		.ty = field_ptr_ty_ref,
2715		.operand = struct_operand,
2716		} },
2717		}),
2718		else => b.add(l, .{
2719		.tag = .struct_field_ptr,
2720		.data = .{ .ty_pl = .{
2721		.ty = field_ptr_ty_ref,
2722		.payload = try l.addExtra(Air.StructField, .{
2723		.struct_operand = struct_operand,
2724		.field_index = @intCast(field_index),
2725		}),
2726		} },
2727		}),
2728		};
2729		}
2730
2731	2383	/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
2732	2384	/// no-op casts.
2733	2385	fn addBitCast(
...	...	@@ -2774,31 +2426,6 @@ const Block = struct {
2774	2426	}
2775	2427	};
2776	2428
2777		const Result = struct {
2778		inst: Air.Inst.Index,
2779		block: Block,
2780
2781		/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2782		/// to initialize it.
2783		fn init(l: Legalize, ty: Type, parent_block: Block) Result {
2784		return .{
2785		.inst = parent_block.add(l, .{
2786		.tag = .block,
2787		.data = .{ .ty_pl = .{
2788		.ty = Air.internedToRef(ty.toIntern()),
2789		.payload = undefined,
2790		} },
2791		}),
2792		.block = undefined,
2793		};
2794		}
2795
2796		fn finish(res: Result, l: *Legalize) Error!void {
2797		const data = &l.air_instructions.items(.data)[@intFromEnum(res.inst)];
2798		data.ty_pl.payload = try l.addBlockBody(res.block.body());
2799		}
2800		};
2801
2802	2429	const Loop = struct {
2803	2430	inst: Air.Inst.Index,
2804	2431	block: Block,

src/Air/Liveness.zig+7-6

...	...	@@ -458,17 +458,12 @@ fn analyzeInst(
458	458	.memset_safe,
459	459	.memcpy,
460	460	.memmove,
	461	.legalize_vec_elem_val,
461	462	=> {
462	463	const o = inst_datas[@intFromEnum(inst)].bin_op;
463	464	return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
464	465	},
465	466
466		.vector_store_elem => {
467		const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
468		const extra = a.air.extraData(Air.Bin, o.payload).data;
469		return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
470		},
471
472	467	.arg,
473	468	.alloc,
474	469	.ret_ptr,
...	...	@@ -775,6 +770,12 @@ fn analyzeInst(
775	770	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
776	771	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
777	772	},
	773
	774	.legalize_vec_store_elem => {
	775	const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
	776	const bin = a.air.extraData(Air.Bin, pl_op.payload).data;
	777	return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });
	778	},
778	779	}
779	780	}
780	781

src/Air/Liveness/Verify.zig+6-5

...	...	@@ -272,6 +272,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
272	272	.memset_safe,
273	273	.memcpy,
274	274	.memmove,
	275	.legalize_vec_elem_val,
275	276	=> {
276	277	const bin_op = data[@intFromEnum(inst)].bin_op;
277	278	try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
...	...	@@ -322,11 +323,6 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
322	323	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
323	324	try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
324	325	},
325		.vector_store_elem => {
326		const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
327		const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
328		try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
329		},
330	326	.cmpxchg_strong,
331	327	.cmpxchg_weak,
332	328	=> {
...	...	@@ -582,6 +578,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
582	578
583	579	try self.verifyInst(inst);
584	580	},
	581	.legalize_vec_store_elem => {
	582	const pl_op = data[@intFromEnum(inst)].pl_op;
	583	const bin = self.air.extraData(Air.Bin, pl_op.payload).data;
	584	try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
	585	},
585	586	}
586	587	}
587	588	}

src/Air/print.zig+14-12

...	...	@@ -171,6 +171,7 @@ const Writer = struct {
171	171	.memmove,
172	172	.memset,
173	173	.memset_safe,
	174	.legalize_vec_elem_val,
174	175	=> try w.writeBinOp(s, inst),
175	176
176	177	.is_null,
...	...	@@ -330,8 +331,8 @@ const Writer = struct {
330	331	.shuffle_two => try w.writeShuffleTwo(s, inst),
331	332	.reduce, .reduce_optimized => try w.writeReduce(s, inst),
332	333	.cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
333		.vector_store_elem => try w.writeVectorStoreElem(s, inst),
334	334	.runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),
	335	.legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),
335	336
336	337	.work_item_id,
337	338	.work_group_size,
...	...	@@ -509,6 +510,18 @@ const Writer = struct {
509	510	try w.writeOperand(s, inst, 2, pl_op.operand);
510	511	}
511	512
	513	fn writeLegalizeVecStoreElem(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
	514	const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	515	const bin = w.air.extraData(Air.Bin, pl_op.payload).data;
	516
	517	try w.writeOperand(s, inst, 0, pl_op.operand);
	518	try s.writeAll(", ");
	519	try w.writeOperand(s, inst, 1, bin.lhs);
	520	try s.writeAll(", ");
	521	try w.writeOperand(s, inst, 2, bin.rhs);
	522	try s.writeAll(", ");
	523	}
	524
512	525	fn writeShuffleOne(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
513	526	const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
514	527	try w.writeType(s, unwrapped.result_ty);
...	...	@@ -576,17 +589,6 @@ const Writer = struct {
576	589	try w.writeOperand(s, inst, 1, extra.rhs);
577	590	}
578	591
579		fn writeVectorStoreElem(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
580		const data = w.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
581		const extra = w.air.extraData(Air.VectorCmp, data.payload).data;
582
583		try w.writeOperand(s, inst, 0, data.vector_ptr);
584		try s.writeAll(", ");
585		try w.writeOperand(s, inst, 1, extra.lhs);
586		try s.writeAll(", ");
587		try w.writeOperand(s, inst, 2, extra.rhs);
588		}
589
590	592	fn writeRuntimeNavPtr(w: Writer, s: std.Io.Writer, inst: Air.Inst.Index) Error!void {
591	593	const ip = &w.pt.zcu.intern_pool;
592	594	const ty_nav = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;

src/Air/types_resolved.zig+2-7

...	...	@@ -88,6 +88,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
88	88	.atomic_store_monotonic,
89	89	.atomic_store_release,
90	90	.atomic_store_seq_cst,
	91	.legalize_vec_elem_val,
91	92	=> {
92	93	if (!checkRef(data.bin_op.lhs, zcu)) return false;
93	94	if (!checkRef(data.bin_op.rhs, zcu)) return false;
...	...	@@ -316,19 +317,13 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
316	317	if (!checkRef(data.prefetch.ptr, zcu)) return false;
317	318	},
318	319
319		.vector_store_elem => {
320		const bin = air.extraData(Air.Bin, data.vector_store_elem.payload).data;
321		if (!checkRef(data.vector_store_elem.vector_ptr, zcu)) return false;
322		if (!checkRef(bin.lhs, zcu)) return false;
323		if (!checkRef(bin.rhs, zcu)) return false;
324		},
325
326	320	.runtime_nav_ptr => {
327	321	if (!checkType(.fromInterned(data.ty_nav.ty), zcu)) return false;
328	322	},
329	323
330	324	.select,
331	325	.mul_add,
	326	.legalize_vec_store_elem,
332	327	=> {
333	328	const bin = air.extraData(Air.Bin, data.pl_op.payload).data;
334	329	if (!checkRef(data.pl_op.operand, zcu)) return false;

src/InternPool.zig+2-5

...	...	@@ -2104,7 +2104,6 @@ pub const Key = union(enum) {
2104	2104
2105	2105	pub const VectorIndex = enum(u16) {
2106	2106	none = std.math.maxInt(u16),
2107		runtime = std.math.maxInt(u16) - 1,
2108	2107	_,
2109	2108	};
2110	2109
...	...	@@ -3739,10 +3738,8 @@ pub const LoadedStructType = struct {
3739	3738	return s.field_inits.get(ip)[i];
3740	3739	}
3741	3740
3742		/// Returns `none` in the case the struct is a tuple.
3743		pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) OptionalNullTerminatedString {
3744		if (s.field_names.len == 0) return .none;
3745		return s.field_names.get(ip)[i].toOptional();
	3741	pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) NullTerminatedString {
	3742	return s.field_names.get(ip)[i];
3746	3743	}
3747	3744
3748	3745	pub fn fieldIsComptime(s: LoadedStructType, ip: *const InternPool, i: usize) bool {

src/Sema.zig+31-94

...	...	@@ -15919,24 +15919,30 @@ fn zirOverflowArithmetic(
15919	15919	},
15920	15920	.mul_with_overflow => {
15921	15921	// If either of the arguments is zero, the result is zero and no overflow occured.
15922		// If either of the arguments is one, the result is the other and no overflow occured.
15923		// Otherwise, if either of the arguments is undefined, both results are undefined.
15924		const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1);
15925	15922	if (maybe_lhs_val) \|lhs_val\| {
15926		if (!lhs_val.isUndef(zcu)) {
15927		if (try lhs_val.compareAllWithZeroSema(.eq, pt)) {
15928		break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15929		} else if (try sema.compareAll(lhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) {
15930		break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15931		}
	15923	if (!lhs_val.isUndef(zcu) and try lhs_val.compareAllWithZeroSema(.eq, pt)) {
	15924	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15932	15925	}
15933	15926	}
15934
15935	15927	if (maybe_rhs_val) \|rhs_val\| {
15936		if (!rhs_val.isUndef(zcu)) {
15937		if (try rhs_val.compareAllWithZeroSema(.eq, pt)) {
	15928	if (!rhs_val.isUndef(zcu) and try rhs_val.compareAllWithZeroSema(.eq, pt)) {
	15929	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
	15930	}
	15931	}
	15932	// If either of the arguments is one, the result is the other and no overflow occured.
	15933	const dest_scalar_ty = dest_ty.scalarType(zcu);
	15934	const dest_scalar_int = dest_scalar_ty.intInfo(zcu);
	15935	// We could still be working with i1, where '1' is not a legal value!
	15936	if (!(dest_scalar_int.bits == 1 and dest_scalar_int.signedness == .signed)) {
	15937	const scalar_one = try pt.intValue(dest_scalar_ty, 1);
	15938	const vec_one = try sema.splat(dest_ty, scalar_one);
	15939	if (maybe_lhs_val) \|lhs_val\| {
	15940	if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {
15938	15941	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15939		} else if (try sema.compareAll(rhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) {
	15942	}
	15943	}
	15944	if (maybe_rhs_val) \|rhs_val\| {
	15945	if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {
15940	15946	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15941	15947	}
15942	15948	}
...	...	@@ -15947,7 +15953,6 @@ fn zirOverflowArithmetic(
15947	15953	if (lhs_val.isUndef(zcu) or rhs_val.isUndef(zcu)) {
15948	15954	break :result .{ .overflow_bit = .undef, .wrapped = .undef };
15949	15955	}
15950
15951	15956	const result = try arith.mulWithOverflow(sema, dest_ty, lhs_val, rhs_val);
15952	15957	break :result .{ .overflow_bit = result.overflow_bit, .wrapped = result.wrapped_result };
15953	15958	}
...	...	@@ -17751,10 +17756,7 @@ fn zirTypeInfo(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
17751	17756	try ty.resolveStructFieldInits(pt);
17752	17757
17753	17758	for (struct_field_vals, 0..) \|*field_val, field_index\| {
17754		const field_name = if (struct_type.fieldName(ip, field_index).unwrap()) \|field_name\|
17755		field_name
17756		else
17757		try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
	17759	const field_name = struct_type.fieldName(ip, field_index);
17758	17760	const field_name_len = field_name.length(ip);
17759	17761	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
17760	17762	const field_init = struct_type.fieldInit(ip, field_index);
...	...	@@ -28345,6 +28347,10 @@ fn elemPtrArray(
28345	28347	break :o index;
28346	28348	} else null;
28347	28349
	28350	if (offset == null and array_ty.zigTypeTag(zcu) == .vector) {
	28351	return sema.fail(block, elem_index_src, "vector index not comptime known", .{});
	28352	}
	28353
28348	28354	const elem_ptr_ty = try array_ptr_ty.elemPtrType(offset, pt);
28349	28355
28350	28356	if (maybe_undef_array_ptr_val) \|array_ptr_val\| {
...	...	@@ -28362,10 +28368,6 @@ fn elemPtrArray(
28362	28368	try sema.validateRuntimeValue(block, array_ptr_src, array_ptr);
28363	28369	}
28364	28370
28365		if (offset == null and array_ty.zigTypeTag(zcu) == .vector) {
28366		return sema.fail(block, elem_index_src, "vector index not comptime known", .{});
28367		}
28368
28369	28371	// Runtime check is only needed if unable to comptime check.
28370	28372	if (oob_safety and block.wantSafety() and offset == null) {
28371	28373	const len_inst = try pt.intRef(.usize, array_len);
...	...	@@ -30397,22 +30399,6 @@ fn storePtr2(
30397	30399
30398	30400	const is_ret = air_tag == .ret_ptr;
30399	30401
30400		// Detect if we are storing an array operand to a bitcasted vector pointer.
30401		// If so, we instead reach through the bitcasted pointer to the vector pointer,
30402		// bitcast the array operand to a vector, and then lower this as a store of
30403		// a vector value to a vector pointer. This generally results in better code,
30404		// as well as working around an LLVM bug:
30405		// https://github.com/ziglang/zig/issues/11154
30406		if (sema.obtainBitCastedVectorPtr(ptr)) \|vector_ptr\| {
30407		const vector_ty = sema.typeOf(vector_ptr).childType(zcu);
30408		const vector = sema.coerceExtra(block, vector_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch \|err\| switch (err) {
30409		error.NotCoercible => unreachable,
30410		else => \|e\| return e,
30411		};
30412		try sema.storePtr2(block, src, vector_ptr, ptr_src, vector, operand_src, .store);
30413		return;
30414		}
30415
30416	30402	const operand = sema.coerceExtra(block, elem_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch \|err\| switch (err) {
30417	30403	error.NotCoercible => unreachable,
30418	30404	else => \|e\| return e,
...	...	@@ -30445,29 +30431,6 @@ fn storePtr2(
30445	30431
30446	30432	try sema.requireRuntimeBlock(block, src, runtime_src);
30447	30433
30448		if (ptr_ty.ptrInfo(zcu).flags.vector_index == .runtime) {
30449		const ptr_inst = ptr.toIndex().?;
30450		const air_tags = sema.air_instructions.items(.tag);
30451		if (air_tags[@intFromEnum(ptr_inst)] == .ptr_elem_ptr) {
30452		const ty_pl = sema.air_instructions.items(.data)[@intFromEnum(ptr_inst)].ty_pl;
30453		const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data;
30454		_ = try block.addInst(.{
30455		.tag = .vector_store_elem,
30456		.data = .{ .vector_store_elem = .{
30457		.vector_ptr = bin_op.lhs,
30458		.payload = try block.sema.addExtra(Air.Bin{
30459		.lhs = bin_op.rhs,
30460		.rhs = operand,
30461		}),
30462		} },
30463		});
30464		return;
30465		}
30466		return sema.fail(block, ptr_src, "unable to determine vector element index of type '{f}'", .{
30467		ptr_ty.fmt(pt),
30468		});
30469		}
30470
30471	30434	const store_inst = if (is_ret)
30472	30435	try block.addBinOp(.store, ptr, operand)
30473	30436	else
...	...	@@ -30567,37 +30530,6 @@ fn markMaybeComptimeAllocRuntime(sema: Sema, block: Block, alloc_inst: Air.Ins
30567	30530	}
30568	30531	}
30569	30532
30570		/// Traverse an arbitrary number of bitcasted pointers and return the underyling vector
30571		/// pointer. Only if the final element type matches the vector element type, and the
30572		/// lengths match.
30573		fn obtainBitCastedVectorPtr(sema: *Sema, ptr: Air.Inst.Ref) ?Air.Inst.Ref {
30574		const pt = sema.pt;
30575		const zcu = pt.zcu;
30576		const array_ty = sema.typeOf(ptr).childType(zcu);
30577		if (array_ty.zigTypeTag(zcu) != .array) return null;
30578		var ptr_ref = ptr;
30579		var ptr_inst = ptr_ref.toIndex() orelse return null;
30580		const air_datas = sema.air_instructions.items(.data);
30581		const air_tags = sema.air_instructions.items(.tag);
30582		const vector_ty = while (air_tags[@intFromEnum(ptr_inst)] == .bitcast) {
30583		ptr_ref = air_datas[@intFromEnum(ptr_inst)].ty_op.operand;
30584		if (!sema.isKnownZigType(ptr_ref, .pointer)) return null;
30585		const child_ty = sema.typeOf(ptr_ref).childType(zcu);
30586		if (child_ty.zigTypeTag(zcu) == .vector) break child_ty;
30587		ptr_inst = ptr_ref.toIndex() orelse return null;
30588		} else return null;
30589
30590		// We have a pointer-to-array and a pointer-to-vector. If the elements and
30591		// lengths match, return the result.
30592		if (array_ty.childType(zcu).eql(vector_ty.childType(zcu), zcu) and
30593		array_ty.arrayLen(zcu) == vector_ty.vectorLen(zcu))
30594		{
30595		return ptr_ref;
30596		} else {
30597		return null;
30598		}
30599		}
30600
30601	30533	/// Call when you have Value objects rather than Air instructions, and you want to
30602	30534	/// assert the store must be done at comptime.
30603	30535	fn storePtrVal(
...	...	@@ -35577,8 +35509,13 @@ fn structFieldInits(
35577	35509	const default_val = try sema.resolveConstValue(&block_scope, init_src, coerced, null);
35578	35510
35579	35511	if (default_val.canMutateComptimeVarState(zcu)) {
35580		const field_name = struct_type.fieldName(ip, field_i).unwrap().?;
35581		return sema.failWithContainsReferenceToComptimeVar(&block_scope, init_src, field_name, "field default value", default_val);
	35512	return sema.failWithContainsReferenceToComptimeVar(
	35513	&block_scope,
	35514	init_src,
	35515	struct_type.fieldName(ip, field_i),
	35516	"field default value",
	35517	default_val,
	35518	);
35582	35519	}
35583	35520	struct_type.field_inits.get(ip)[field_i] = default_val.toIntern();
35584	35521	}

src/Sema/comptime_ptr_access.zig-2

...	...	@@ -24,7 +24,6 @@ pub fn loadComptimePtr(sema: Sema, block: Block, src: LazySrcLoc, ptr: Value)
24	24	const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
25	25	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
26	26	.none => 0,
27		.runtime => return .runtime_load,
28	27	else => \|idx\| switch (pt.zcu.getTarget().cpu.arch.endian()) {
29	28	.little => child_bits * @intFromEnum(idx),
30	29	.big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
...	...	@@ -81,7 +80,6 @@ pub fn storeComptimePtr(
81	80	};
82	81	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
83	82	.none => 0,
84		.runtime => return .runtime_store,
85	83	else => \|idx\| switch (zcu.getTarget().cpu.arch.endian()) {
86	84	.little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
87	85	.big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian

src/Type.zig+4-6

...	...	@@ -198,9 +198,7 @@ pub fn print(ty: Type, writer: *std.Io.Writer, pt: Zcu.PerThread) std.Io.Writer.
198	198	info.packed_offset.bit_offset, info.packed_offset.host_size,
199	199	});
200	200	}
201		if (info.flags.vector_index == .runtime) {
202		try writer.writeAll(":?");
203		} else if (info.flags.vector_index != .none) {
	201	if (info.flags.vector_index != .none) {
204	202	try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)});
205	203	}
206	204	try writer.writeAll(") ");
...	...	@@ -3113,7 +3111,7 @@ pub fn enumTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {
3113	3111	pub fn structFieldName(ty: Type, index: usize, zcu: *const Zcu) InternPool.OptionalNullTerminatedString {
3114	3112	const ip = &zcu.intern_pool;
3115	3113	return switch (ip.indexToKey(ty.toIntern())) {
3116		.struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index),
	3114	.struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index).toOptional(),
3117	3115	.tuple_type => .none,
3118	3116	else => unreachable,
3119	3117	};
...	...	@@ -3558,7 +3556,7 @@ pub fn packedStructFieldPtrInfo(
3558	3556	} else .{
3559	3557	switch (zcu.comp.getZigBackend()) {
3560	3558	else => (running_bits + 7) / 8,
3561		.stage2_x86_64 => @intCast(struct_ty.abiSize(zcu)),
	3559	.stage2_x86_64, .stage2_c => @intCast(struct_ty.abiSize(zcu)),
3562	3560	},
3563	3561	bit_offset,
3564	3562	};
...	...	@@ -3985,7 +3983,7 @@ pub fn elemPtrType(ptr_ty: Type, offset: ?usize, pt: Zcu.PerThread) !Type {
3985	3983	break :blk .{
3986	3984	.host_size = @intCast(parent_ty.arrayLen(zcu)),
3987	3985	.alignment = parent_ty.abiAlignment(zcu),
3988		.vector_index = if (offset) \|some\| @enumFromInt(some) else .runtime,
	3986	.vector_index = @enumFromInt(offset.?),
3989	3987	};
3990	3988	} else .{};
3991	3989

src/Value.zig+21-150

...	...	@@ -574,166 +574,37 @@ pub fn writeToPackedMemory(
574	574	}
575	575	}
576	576
577		/// Load a Value from the contents of `buffer`.
	577	/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.
578	578	///
579	579	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
580	580	/// the end of the value in memory.
581		pub fn readFromMemory(
	581	pub fn readUintFromMemory(
582	582	ty: Type,
583	583	pt: Zcu.PerThread,
584	584	buffer: []const u8,
585	585	arena: Allocator,
586		) error{
587		IllDefinedMemoryLayout,
588		Unimplemented,
589		OutOfMemory,
590		}!Value {
	586	) Allocator.Error!Value {
591	587	const zcu = pt.zcu;
592		const ip = &zcu.intern_pool;
593		const target = zcu.getTarget();
594		const endian = target.cpu.arch.endian();
595		switch (ty.zigTypeTag(zcu)) {
596		.void => return Value.void,
597		.bool => {
598		if (buffer[0] == 0) {
599		return Value.false;
600		} else {
601		return Value.true;
602		}
603		},
604		.int, .@"enum" => \|ty_tag\| {
605		const int_ty = switch (ty_tag) {
606		.int => ty,
607		.@"enum" => ty.intTagType(zcu),
608		else => unreachable,
609		};
610		const int_info = int_ty.intInfo(zcu);
611		const bits = int_info.bits;
612		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
613		if (bits == 0 or buffer.len == 0) return zcu.getCoerced(try zcu.intValue(int_ty, 0), ty);
	588	const endian = zcu.getTarget().cpu.arch.endian();
614	589
615		if (bits <= 64) switch (int_info.signedness) { // Fast path for integers <= u64
616		.signed => {
617		const val = std.mem.readVarInt(i64, buffer[0..byte_count], endian);
618		const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
619		return zcu.getCoerced(try zcu.intValue(int_ty, result), ty);
620		},
621		.unsigned => {
622		const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
623		const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
624		return zcu.getCoerced(try zcu.intValue(int_ty, result), ty);
625		},
626		} else { // Slow path, we have to construct a big-int
627		const Limb = std.math.big.Limb;
628		const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
629		const limbs_buffer = try arena.alloc(Limb, limb_count);
630
631		var bigint = BigIntMutable.init(limbs_buffer, 0);
632		bigint.readTwosComplement(buffer[0..byte_count], bits, endian, int_info.signedness);
633		return zcu.getCoerced(try zcu.intValue_big(int_ty, bigint.toConst()), ty);
634		}
635		},
636		.float => return Value.fromInterned(try pt.intern(.{ .float = .{
637		.ty = ty.toIntern(),
638		.storage = switch (ty.floatBits(target)) {
639		16 => .{ .f16 = @bitCast(std.mem.readInt(u16, buffer[0..2], endian)) },
640		32 => .{ .f32 = @bitCast(std.mem.readInt(u32, buffer[0..4], endian)) },
641		64 => .{ .f64 = @bitCast(std.mem.readInt(u64, buffer[0..8], endian)) },
642		80 => .{ .f80 = @bitCast(std.mem.readInt(u80, buffer[0..10], endian)) },
643		128 => .{ .f128 = @bitCast(std.mem.readInt(u128, buffer[0..16], endian)) },
644		else => unreachable,
645		},
646		} })),
647		.array => {
648		const elem_ty = ty.childType(zcu);
649		const elem_size = elem_ty.abiSize(zcu);
650		const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
651		var offset: usize = 0;
652		for (elems) \|*elem\| {
653		elem.* = (try readFromMemory(elem_ty, zcu, buffer[offset..], arena)).toIntern();
654		offset += @intCast(elem_size);
655		}
656		return pt.aggregateValue(ty, elems);
657		},
658		.vector => {
659		// We use byte_count instead of abi_size here, so that any padding bytes
660		// follow the data bytes, on both big- and little-endian systems.
661		const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
662		return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
663		},
664		.@"struct" => {
665		const struct_type = zcu.typeToStruct(ty).?;
666		switch (struct_type.layout) {
667		.auto => unreachable, // Sema is supposed to have emitted a compile error already
668		.@"extern" => {
669		const field_types = struct_type.field_types;
670		const field_vals = try arena.alloc(InternPool.Index, field_types.len);
671		for (field_vals, 0..) \|*field_val, i\| {
672		const field_ty = Type.fromInterned(field_types.get(ip)[i]);
673		const off: usize = @intCast(ty.structFieldOffset(i, zcu));
674		const sz: usize = @intCast(field_ty.abiSize(zcu));
675		field_val.* = (try readFromMemory(field_ty, zcu, buffer[off..(off + sz)], arena)).toIntern();
676		}
677		return pt.aggregateValue(ty, field_vals);
678		},
679		.@"packed" => {
680		const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
681		return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
682		},
683		}
684		},
685		.error_set => {
686		const bits = zcu.errorSetBits();
687		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
688		const int = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
689		const index = (int << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
690		const name = zcu.global_error_set.keys()[@intCast(index)];
	590	assert(ty.isUnsignedInt(zcu));
	591	const bits = ty.intInfo(zcu).bits;
	592	const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
691	593
692		return Value.fromInterned(try pt.intern(.{ .err = .{
693		.ty = ty.toIntern(),
694		.name = name,
695		} }));
696		},
697		.@"union" => switch (ty.containerLayout(zcu)) {
698		.auto => return error.IllDefinedMemoryLayout,
699		.@"extern" => {
700		const union_size = ty.abiSize(zcu);
701		const array_ty = try zcu.arrayType(.{ .len = union_size, .child = .u8_type });
702		const val = (try readFromMemory(array_ty, zcu, buffer, arena)).toIntern();
703		return Value.fromInterned(try pt.internUnion(.{
704		.ty = ty.toIntern(),
705		.tag = .none,
706		.val = val,
707		}));
708		},
709		.@"packed" => {
710		const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
711		return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
712		},
713		},
714		.pointer => {
715		assert(!ty.isSlice(zcu)); // No well defined layout.
716		const int_val = try readFromMemory(Type.usize, zcu, buffer, arena);
717		return Value.fromInterned(try pt.intern(.{ .ptr = .{
718		.ty = ty.toIntern(),
719		.base_addr = .int,
720		.byte_offset = int_val.toUnsignedInt(zcu),
721		} }));
722		},
723		.optional => {
724		assert(ty.isPtrLikeOptional(zcu));
725		const child_ty = ty.optionalChild(zcu);
726		const child_val = try readFromMemory(child_ty, zcu, buffer, arena);
727		return Value.fromInterned(try pt.intern(.{ .opt = .{
728		.ty = ty.toIntern(),
729		.val = switch (child_val.orderAgainstZero(pt)) {
730		.lt => unreachable,
731		.eq => .none,
732		.gt => child_val.toIntern(),
733		},
734		} }));
735		},
736		else => return error.Unimplemented,
	594	assert(buffer.len >= byte_count);
	595
	596	if (bits <= 64) {
	597	const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
	598	const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
	599	return pt.intValue(ty, result);
	600	} else {
	601	const Limb = std.math.big.Limb;
	602	const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
	603	const limbs_buffer = try arena.alloc(Limb, limb_count);
	604
	605	var bigint: BigIntMutable = .init(limbs_buffer, 0);
	606	bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);
	607	return pt.intValue_big(ty, bigint.toConst());
737	608	}
738	609	}
739	610

src/Zcu/PerThread.zig+22-4

...	...	@@ -3512,7 +3512,6 @@ pub fn ptrType(pt: Zcu.PerThread, info: InternPool.Key.PtrType) Allocator.Error!
3512	3512	canon_info.packed_offset.host_size = 0;
3513	3513	}
3514	3514	},
3515		.runtime => {},
3516	3515	_ => assert(@intFromEnum(info.flags.vector_index) < info.packed_offset.host_size),
3517	3516	}
3518	3517
...	...	@@ -3663,21 +3662,40 @@ pub fn intRef(pt: Zcu.PerThread, ty: Type, x: anytype) Allocator.Error!Air.Inst.
3663	3662	}
3664	3663
3665	3664	pub fn intValue_big(pt: Zcu.PerThread, ty: Type, x: BigIntConst) Allocator.Error!Value {
3666		return Value.fromInterned(try pt.intern(.{ .int = .{
	3665	if (ty.toIntern() != .comptime_int_type) {
	3666	const int_info = ty.intInfo(pt.zcu);
	3667	assert(x.fitsInTwosComp(int_info.signedness, int_info.bits));
	3668	}
	3669	return .fromInterned(try pt.intern(.{ .int = .{
3667	3670	.ty = ty.toIntern(),
3668	3671	.storage = .{ .big_int = x },
3669	3672	} }));
3670	3673	}
3671	3674
3672	3675	pub fn intValue_u64(pt: Zcu.PerThread, ty: Type, x: u64) Allocator.Error!Value {
3673		return Value.fromInterned(try pt.intern(.{ .int = .{
	3676	if (ty.toIntern() != .comptime_int_type and x != 0) {
	3677	const int_info = ty.intInfo(pt.zcu);
	3678	const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed);
	3679	assert(unsigned_bits >= std.math.log2(x) + 1);
	3680	}
	3681	return .fromInterned(try pt.intern(.{ .int = .{
3674	3682	.ty = ty.toIntern(),
3675	3683	.storage = .{ .u64 = x },
3676	3684	} }));
3677	3685	}
3678	3686
3679	3687	pub fn intValue_i64(pt: Zcu.PerThread, ty: Type, x: i64) Allocator.Error!Value {
3680		return Value.fromInterned(try pt.intern(.{ .int = .{
	3688	if (ty.toIntern() != .comptime_int_type and x != 0) {
	3689	const int_info = ty.intInfo(pt.zcu);
	3690	const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed);
	3691	if (x > 0) {
	3692	assert(unsigned_bits >= std.math.log2(x) + 1);
	3693	} else {
	3694	assert(int_info.signedness == .signed);
	3695	assert(unsigned_bits >= std.math.log2_int_ceil(u64, @abs(x)));
	3696	}
	3697	}
	3698	return .fromInterned(try pt.intern(.{ .int = .{
3681	3699	.ty = ty.toIntern(),
3682	3700	.storage = .{ .i64 = x },
3683	3701	} }));

src/codegen/aarch64/Select.zig+9-12

...	...	@@ -134,6 +134,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
134	134	var air_inst_index = air_body[air_body_index];
135	135	const initial_def_order_len = isel.def_order.count();
136	136	air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {
	137	// No "scalarize" legalizations are enabled, so these instructions never appear.
	138	.legalize_vec_elem_val => unreachable,
	139	.legalize_vec_store_elem => unreachable,
	140
137	141	.arg,
138	142	.ret_addr,
139	143	.frame_addr,
...	...	@@ -826,18 +830,6 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
826	830
827	831	try isel.analyzeUse(un_op);
828	832
829		air_body_index += 1;
830		air_inst_index = air_body[air_body_index];
831		continue :air_tag air_tags[@intFromEnum(air_inst_index)];
832		},
833		.vector_store_elem => {
834		const vector_store_elem = air_data[@intFromEnum(air_inst_index)].vector_store_elem;
835		const bin_op = isel.air.extraData(Air.Bin, vector_store_elem.payload).data;
836
837		try isel.analyzeUse(vector_store_elem.vector_ptr);
838		try isel.analyzeUse(bin_op.lhs);
839		try isel.analyzeUse(bin_op.rhs);
840
841	833	air_body_index += 1;
842	834	air_inst_index = air_body[air_body_index];
843	835	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
...	...	@@ -962,6 +954,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
962	954	};
963	955	air_tag: switch (air.next().?) {
964	956	else => \|air_tag\| return isel.fail("unimplemented {t}", .{air_tag}),
	957
	958	// No "scalarize" legalizations are enabled, so these instructions never appear.
	959	.legalize_vec_elem_val => unreachable,
	960	.legalize_vec_store_elem => unreachable,
	961
965	962	.arg => {
966	963	const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
967	964	defer arg_vi.deref(isel);

src/codegen/c.zig+135-450

...	...	@@ -37,6 +37,7 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
37	37	.expand_packed_load = true,
38	38	.expand_packed_store = true,
39	39	.expand_packed_struct_field_val = true,
	40	.expand_packed_aggregate_init = true,
40	41	}),
41	42	};
42	43	}
...	...	@@ -1392,114 +1393,21 @@ pub const DeclGen = struct {
1392	1393	try w.writeByte('}');
1393	1394	},
1394	1395	.@"packed" => {
1395		const int_info = ty.intInfo(zcu);
1396
1397		const bits = Type.smallestUnsignedBits(int_info.bits - 1);
1398		const bit_offset_ty = try pt.intType(.unsigned, bits);
1399
1400		var bit_offset: u64 = 0;
1401		var eff_num_fields: usize = 0;
1402
1403		for (0..loaded_struct.field_types.len) \|field_index\| {
1404		const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1405		if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1406		eff_num_fields += 1;
1407		}
1408
1409		if (eff_num_fields == 0) {
1410		try w.writeByte('(');
1411		try dg.renderUndefValue(w, ty, location);
1412		try w.writeByte(')');
1413		} else if (ty.bitSize(zcu) > 64) {
1414		// zig_or_u128(zig_or_u128(zig_shl_u128(a, a_off), zig_shl_u128(b, b_off)), zig_shl_u128(c, c_off))
1415		var num_or = eff_num_fields - 1;
1416		while (num_or > 0) : (num_or -= 1) {
1417		try w.writeAll("zig_or_");
1418		try dg.renderTypeForBuiltinFnName(w, ty);
1419		try w.writeByte('(');
1420		}
1421
1422		var eff_index: usize = 0;
1423		var needs_closing_paren = false;
1424		for (0..loaded_struct.field_types.len) \|field_index\| {
1425		const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1426		if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1427
1428		const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
1429		.bytes => \|bytes\| try pt.intern(.{ .int = .{
1430		.ty = field_ty.toIntern(),
1431		.storage = .{ .u64 = bytes.at(field_index, ip) },
1432		} }),
1433		.elems => \|elems\| elems[field_index],
1434		.repeated_elem => \|elem\| elem,
1435		};
1436		const cast_context = IntCastContext{ .value = .{ .value = Value.fromInterned(field_val) } };
1437		if (bit_offset != 0) {
1438		try w.writeAll("zig_shl_");
1439		try dg.renderTypeForBuiltinFnName(w, ty);
1440		try w.writeByte('(');
1441		try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument);
1442		try w.writeAll(", ");
1443		try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument);
1444		try w.writeByte(')');
1445		} else {
1446		try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument);
1447		}
1448
1449		if (needs_closing_paren) try w.writeByte(')');
1450		if (eff_index != eff_num_fields - 1) try w.writeAll(", ");
1451
1452		bit_offset += field_ty.bitSize(zcu);
1453		needs_closing_paren = true;
1454		eff_index += 1;
1455		}
1456		} else {
1457		try w.writeByte('(');
1458		// a << a_off \| b << b_off \| c << c_off
1459		var empty = true;
1460		for (0..loaded_struct.field_types.len) \|field_index\| {
1461		const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1462		if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1463
1464		if (!empty) try w.writeAll(" \| ");
1465		try w.writeByte('(');
1466		try dg.renderCType(w, ctype);
1467		try w.writeByte(')');
1468
1469		const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
1470		.bytes => \|bytes\| try pt.intern(.{ .int = .{
1471		.ty = field_ty.toIntern(),
1472		.storage = .{ .u64 = bytes.at(field_index, ip) },
1473		} }),
1474		.elems => \|elems\| elems[field_index],
1475		.repeated_elem => \|elem\| elem,
1476		};
1477
1478		const field_int_info: std.builtin.Type.Int = if (field_ty.isAbiInt(zcu))
1479		field_ty.intInfo(zcu)
1480		else
1481		.{ .signedness = .unsigned, .bits = undefined };
1482		switch (field_int_info.signedness) {
1483		.signed => {
1484		try w.writeByte('(');
1485		try dg.renderValue(w, Value.fromInterned(field_val), .Other);
1486		try w.writeAll(" & ");
1487		const field_uint_ty = try pt.intType(.unsigned, field_int_info.bits);
1488		try dg.renderValue(w, try field_uint_ty.maxIntScalar(pt, field_uint_ty), .Other);
1489		try w.writeByte(')');
1490		},
1491		.unsigned => try dg.renderValue(w, Value.fromInterned(field_val), .Other),
1492		}
1493		if (bit_offset != 0) {
1494		try w.writeAll(" << ");
1495		try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument);
1496		}
1497
1498		bit_offset += field_ty.bitSize(zcu);
1499		empty = false;
1500		}
1501		try w.writeByte(')');
1502		}
	1396	// https://github.com/ziglang/zig/issues/24657 will eliminate most of the
	1397	// following logic, leaving only the recursive `renderValue` call. Once
	1398	// that proposal is implemented, a `packed struct` will literally be
	1399	// represented in the InternPool by its comptime-known backing integer.
	1400	var arena: std.heap.ArenaAllocator = .init(zcu.gpa);
	1401	defer arena.deinit();
	1402	const backing_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip));
	1403	const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu)));
	1404	val.writeToMemory(pt, buf) catch \|err\| switch (err) {
	1405	error.IllDefinedMemoryLayout => unreachable,
	1406	error.OutOfMemory => \|e\| return e,
	1407	error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed struct value", .{}),
	1408	};
	1409	const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator());
	1410	return dg.renderValue(w, backing_val, location);
1503	1411	},
1504	1412	}
1505	1413	},
...	...	@@ -1507,33 +1415,38 @@ pub const DeclGen = struct {
1507	1415	},
1508	1416	.un => \|un\| {
1509	1417	const loaded_union = ip.loadUnionType(ty.toIntern());
	1418	if (loaded_union.flagsUnordered(ip).layout == .@"packed") {
	1419	// https://github.com/ziglang/zig/issues/24657 will eliminate most of the
	1420	// following logic, leaving only the recursive `renderValue` call. Once
	1421	// that proposal is implemented, a `packed union` will literally be
	1422	// represented in the InternPool by its comptime-known backing integer.
	1423	var arena: std.heap.ArenaAllocator = .init(zcu.gpa);
	1424	defer arena.deinit();
	1425	const backing_ty = try ty.unionBackingType(pt);
	1426	const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu)));
	1427	val.writeToMemory(pt, buf) catch \|err\| switch (err) {
	1428	error.IllDefinedMemoryLayout => unreachable,
	1429	error.OutOfMemory => \|e\| return e,
	1430	error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed union value", .{}),
	1431	};
	1432	const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator());
	1433	return dg.renderValue(w, backing_val, location);
	1434	}
1510	1435	if (un.tag == .none) {
1511	1436	const backing_ty = try ty.unionBackingType(pt);
1512		switch (loaded_union.flagsUnordered(ip).layout) {
1513		.@"packed" => {
1514		if (!location.isInitializer()) {
1515		try w.writeByte('(');
1516		try dg.renderType(w, backing_ty);
1517		try w.writeByte(')');
1518		}
1519		try dg.renderValue(w, Value.fromInterned(un.val), location);
1520		},
1521		.@"extern" => {
1522		if (location == .StaticInitializer) {
1523		return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{});
1524		}
1525
1526		const ptr_ty = try pt.singleConstPtrType(ty);
1527		try w.writeAll("*((");
1528		try dg.renderType(w, ptr_ty);
1529		try w.writeAll(")(");
1530		try dg.renderType(w, backing_ty);
1531		try w.writeAll("){");
1532		try dg.renderValue(w, Value.fromInterned(un.val), location);
1533		try w.writeAll("})");
1534		},
1535		else => unreachable,
	1437	assert(loaded_union.flagsUnordered(ip).layout == .@"extern");
	1438	if (location == .StaticInitializer) {
	1439	return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{});
1536	1440	}
	1441
	1442	const ptr_ty = try pt.singleConstPtrType(ty);
	1443	try w.writeAll("*((");
	1444	try dg.renderType(w, ptr_ty);
	1445	try w.writeAll(")(");
	1446	try dg.renderType(w, backing_ty);
	1447	try w.writeAll("){");
	1448	try dg.renderValue(w, Value.fromInterned(un.val), location);
	1449	try w.writeAll("})");
1537	1450	} else {
1538	1451	if (!location.isInitializer()) {
1539	1452	try w.writeByte('(');
...	...	@@ -1544,21 +1457,6 @@ pub const DeclGen = struct {
1544	1457	const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?;
1545	1458	const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]);
1546	1459	const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index];
1547		if (loaded_union.flagsUnordered(ip).layout == .@"packed") {
1548		if (field_ty.hasRuntimeBits(zcu)) {
1549		if (field_ty.isPtrAtRuntime(zcu)) {
1550		try w.writeByte('(');
1551		try dg.renderCType(w, ctype);
1552		try w.writeByte(')');
1553		} else if (field_ty.zigTypeTag(zcu) == .float) {
1554		try w.writeByte('(');
1555		try dg.renderCType(w, ctype);
1556		try w.writeByte(')');
1557		}
1558		try dg.renderValue(w, Value.fromInterned(un.val), location);
1559		} else try w.writeByte('0');
1560		return;
1561		}
1562	1460
1563	1461	const has_tag = loaded_union.hasTag(ip);
1564	1462	if (has_tag) try w.writeByte('{');
...	...	@@ -1745,9 +1643,11 @@ pub const DeclGen = struct {
1745	1643	}
1746	1644	return w.writeByte('}');
1747	1645	},
1748		.@"packed" => return w.print("{f}", .{
1749		try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other),
1750		}),
	1646	.@"packed" => return dg.renderUndefValue(
	1647	w,
	1648	.fromInterned(loaded_struct.backingIntTypeUnordered(ip)),
	1649	location,
	1650	),
1751	1651	}
1752	1652	},
1753	1653	.tuple_type => \|tuple_info\| {
...	...	@@ -1815,9 +1715,11 @@ pub const DeclGen = struct {
1815	1715	}
1816	1716	if (has_tag) try w.writeByte('}');
1817	1717	},
1818		.@"packed" => return w.print("{f}", .{
1819		try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other),
1820		}),
	1718	.@"packed" => return dg.renderUndefValue(
	1719	w,
	1720	try ty.unionBackingType(pt),
	1721	location,
	1722	),
1821	1723	}
1822	1724	},
1823	1725	.error_union_type => \|error_union_type\| switch (ctype.info(ctype_pool)) {
...	...	@@ -2445,10 +2347,7 @@ pub const DeclGen = struct {
2445	2347	const ty = val.typeOf(zcu);
2446	2348	return .{ .data = .{
2447	2349	.dg = dg,
2448		.int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed")
2449		.{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) }
2450		else
2451		ty.intInfo(zcu),
	2350	.int_info = ty.intInfo(zcu),
2452	2351	.kind = kind,
2453	2352	.ctype = try dg.ctypeFromType(ty, kind),
2454	2353	.val = val,
...	...	@@ -3426,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3426	3325	// zig fmt: off
3427	3326	.inferred_alloc, .inferred_alloc_comptime => unreachable,
3428	3327
	3328	// No "scalarize" legalizations are enabled, so these instructions never appear.
	3329	.legalize_vec_elem_val => unreachable,
	3330	.legalize_vec_store_elem => unreachable,
	3331
3429	3332	.arg => try airArg(f, inst),
3430	3333
3431	3334	.breakpoint => try airBreakpoint(f),
...	...	@@ -3656,7 +3559,6 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3656	3559
3657	3560	.is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}),
3658	3561	.error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}),
3659		.vector_store_elem => return f.fail("TODO: C backend: implement vector_store_elem", .{}),
3660	3562
3661	3563	.runtime_nav_ptr => try airRuntimeNavPtr(f, inst),
3662	3564
...	...	@@ -3899,6 +3801,24 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3899	3801	});
3900	3802	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });
3901	3803	try f.allocs.put(zcu.gpa, local.new_local, true);
	3804
	3805	switch (elem_ty.zigTypeTag(zcu)) {
	3806	.@"struct", .@"union" => switch (elem_ty.containerLayout(zcu)) {
	3807	.@"packed" => {
	3808	// For packed aggregates, we zero-initialize to try and work around a design flaw
	3809	// related to how `packed`, `undefined`, and RLS interact. See comment in `airStore`
	3810	// for details.
	3811	const w = &f.object.code.writer;
	3812	try w.print("memset(&t{d}, 0x00, sizeof(", .{local.new_local});
	3813	try f.renderType(w, elem_ty);
	3814	try w.writeAll("));");
	3815	try f.object.newline();
	3816	},
	3817	.auto, .@"extern" => {},
	3818	},
	3819	else => {},
	3820	}
	3821
3902	3822	return .{ .local_ref = local.new_local };
3903	3823	}
3904	3824
...	...	@@ -3918,6 +3838,24 @@ fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue {
3918	3838	});
3919	3839	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });
3920	3840	try f.allocs.put(zcu.gpa, local.new_local, true);
	3841
	3842	switch (elem_ty.zigTypeTag(zcu)) {
	3843	.@"struct", .@"union" => switch (elem_ty.containerLayout(zcu)) {
	3844	.@"packed" => {
	3845	// For packed aggregates, we zero-initialize to try and work around a design flaw
	3846	// related to how `packed`, `undefined`, and RLS interact. See comment in `airStore`
	3847	// for details.
	3848	const w = &f.object.code.writer;
	3849	try w.print("memset(&t{d}, 0x00, sizeof(", .{local.new_local});
	3850	try f.renderType(w, elem_ty);
	3851	try w.writeAll("));");
	3852	try f.object.newline();
	3853	},
	3854	.auto, .@"extern" => {},
	3855	},
	3856	else => {},
	3857	}
	3858
3921	3859	return .{ .local_ref = local.new_local };
3922	3860	}
3923	3861
...	...	@@ -3956,6 +3894,10 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3956	3894	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);
3957	3895	const src_ty: Type = .fromInterned(ptr_info.child);
3958	3896
	3897	// `Air.Legalize.Feature.expand_packed_load` should ensure that the only
	3898	// bit-pointers we see here are vector element pointers.
	3899	assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none);
	3900
3959	3901	if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3960	3902	try reap(f, inst, &.{ty_op.operand});
3961	3903	return .none;
...	...	@@ -3987,40 +3929,6 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3987	3929	try w.writeAll(", sizeof(");
3988	3930	try f.renderType(w, src_ty);
3989	3931	try w.writeAll("))");
3990		} else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
3991		const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
3992		const host_ty = try pt.intType(.unsigned, host_bits);
3993
3994		const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1));
3995		const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset);
3996
3997		const field_ty = try pt.intType(.unsigned, @as(u16, @intCast(src_ty.bitSize(zcu))));
3998
3999		try f.writeCValue(w, local, .Other);
4000		try v.elem(f, w);
4001		try w.writeAll(" = (");
4002		try f.renderType(w, src_ty);
4003		try w.writeAll(")zig_wrap_");
4004		try f.object.dg.renderTypeForBuiltinFnName(w, field_ty);
4005		try w.writeAll("((");
4006		try f.renderType(w, field_ty);
4007		try w.writeByte(')');
4008		const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64;
4009		if (cant_cast) {
4010		if (field_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
4011		try w.writeAll("zig_lo_");
4012		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4013		try w.writeByte('(');
4014		}
4015		try w.writeAll("zig_shr_");
4016		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4017		try w.writeByte('(');
4018		try f.writeCValueDeref(w, operand);
4019		try v.elem(f, w);
4020		try w.print(", {f})", .{try f.fmtIntLiteralDec(bit_offset_val)});
4021		if (cant_cast) try w.writeByte(')');
4022		try f.object.dg.renderBuiltinInfo(w, field_ty, .bits);
4023		try w.writeByte(')');
4024	3932	} else {
4025	3933	try f.writeCValue(w, local, .Other);
4026	3934	try v.elem(f, w);
...	...	@@ -4213,6 +4121,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4213	4121	const ptr_scalar_ty = ptr_ty.scalarType(zcu);
4214	4122	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);
4215	4123
	4124	// `Air.Legalize.Feature.expand_packed_store` should ensure that the only
	4125	// bit-pointers we see here are vector element pointers.
	4126	assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none);
	4127
4216	4128	const ptr_val = try f.resolveInst(bin_op.lhs);
4217	4129	const src_ty = f.typeOf(bin_op.rhs);
4218	4130
...	...	@@ -4222,9 +4134,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4222	4134	if (val_is_undef) {
4223	4135	try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
4224	4136	if (safety and ptr_info.packed_offset.host_size == 0) {
	4137	// If the thing we're initializing is a packed struct/union, we set to 0 instead of
	4138	// 0xAA. This is a hack to work around a problem with partially-undefined packed
	4139	// aggregates. If we used 0xAA here, then a later initialization through RLS would
	4140	// not zero the high padding bits (for a packed type which is not 8/16/32/64/etc bits),
	4141	// so we would get a miscompilation. Using 0x00 here avoids this bug in some cases. It
	4142	// is not a correct fix; for instance it misses any case where packed structs are
	4143	// nested in other aggregates. A proper fix for this will involve changing the language,
	4144	// such as to remove RLS. This just prevents miscompilations in some common cases.
	4145	const byte_str: []const u8 = switch (src_ty.zigTypeTag(zcu)) {
	4146	else => "0xaa",
	4147	.@"struct", .@"union" => switch (src_ty.containerLayout(zcu)) {
	4148	.auto, .@"extern" => "0xaa",
	4149	.@"packed" => "0x00",
	4150	},
	4151	};
4225	4152	try w.writeAll("memset(");
4226	4153	try f.writeCValue(w, ptr_val, .FunctionArgument);
4227		try w.writeAll(", 0xaa, sizeof(");
	4154	try w.print(", {s}, sizeof(", .{byte_str});
4228	4155	try f.renderType(w, .fromInterned(ptr_info.child));
4229	4156	try w.writeAll("));");
4230	4157	try f.object.newline();
...	...	@@ -4277,66 +4204,6 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4277	4204	try w.writeByte(';');
4278	4205	try f.object.newline();
4279	4206	try v.end(f, inst, w);
4280		} else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
4281		const host_bits = ptr_info.packed_offset.host_size * 8;
4282		const host_ty = try pt.intType(.unsigned, host_bits);
4283
4284		const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1));
4285		const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset);
4286
4287		const src_bits = src_ty.bitSize(zcu);
4288
4289		const ExpectedContents = [BigInt.Managed.default_capacity]BigIntLimb;
4290		var stack align(@alignOf(ExpectedContents)) =
4291		std.heap.stackFallback(@sizeOf(ExpectedContents), f.object.dg.gpa);
4292
4293		var mask = try BigInt.Managed.initCapacity(stack.get(), BigInt.calcTwosCompLimbCount(host_bits));
4294		defer mask.deinit();
4295
4296		try mask.setTwosCompIntLimit(.max, .unsigned, @intCast(src_bits));
4297		try mask.shiftLeft(&mask, ptr_info.packed_offset.bit_offset);
4298		try mask.bitNotWrap(&mask, .unsigned, host_bits);
4299
4300		const mask_val = try pt.intValue_big(host_ty, mask.toConst());
4301
4302		const v = try Vectorize.start(f, inst, w, ptr_ty);
4303		const a = try Assignment.start(f, w, src_scalar_ctype);
4304		try f.writeCValueDeref(w, ptr_val);
4305		try v.elem(f, w);
4306		try a.assign(f, w);
4307		try w.writeAll("zig_or_");
4308		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4309		try w.writeAll("(zig_and_");
4310		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4311		try w.writeByte('(');
4312		try f.writeCValueDeref(w, ptr_val);
4313		try v.elem(f, w);
4314		try w.print(", {f}), zig_shl_", .{try f.fmtIntLiteralHex(mask_val)});
4315		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4316		try w.writeByte('(');
4317		const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64;
4318		if (cant_cast) {
4319		if (src_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
4320		try w.writeAll("zig_make_");
4321		try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4322		try w.writeAll("(0, ");
4323		} else {
4324		try w.writeByte('(');
4325		try f.renderType(w, host_ty);
4326		try w.writeByte(')');
4327		}
4328
4329		if (src_ty.isPtrAtRuntime(zcu)) {
4330		try w.writeByte('(');
4331		try f.renderType(w, .usize);
4332		try w.writeByte(')');
4333		}
4334		try f.writeCValue(w, src_val, .Other);
4335		try v.elem(f, w);
4336		if (cant_cast) try w.writeByte(')');
4337		try w.print(", {f}))", .{try f.fmtIntLiteralDec(bit_offset_val)});
4338		try a.end(f, w);
4339		try v.end(f, inst, w);
4340	4207	} else {
4341	4208	switch (ptr_val) {
4342	4209	.local_ref => \|ptr_local_index\| switch (src_val) {
...	...	@@ -6015,10 +5882,7 @@ fn fieldLocation(
6015	5882	else if (!field_ptr_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu))
6016	5883	.{ .byte_offset = loaded_struct.offsets.get(ip)[field_index] }
6017	5884	else
6018		.{ .field = if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\|
6019		.{ .identifier = field_name.toSlice(ip) }
6020		else
6021		.{ .field = field_index } },
	5885	.{ .field = .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) } },
6022	5886	.@"packed" => if (field_ptr_ty.ptrInfo(zcu).packed_offset.host_size == 0)
6023	5887	.{ .byte_offset = @divExact(zcu.structPackedFieldBitOffset(loaded_struct, field_index) +
6024	5888	container_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset, 8) }
...	...	@@ -6202,115 +6066,20 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue {
6202	6066	// Ensure complete type definition is visible before accessing fields.
6203	6067	_ = try f.ctypeFromType(struct_ty, .complete);
6204	6068
	6069	assert(struct_ty.containerLayout(zcu) != .@"packed"); // `Air.Legalize.Feature.expand_packed_struct_field_val` handles this case
6205	6070	const field_name: CValue = switch (ip.indexToKey(struct_ty.toIntern())) {
6206		.struct_type => field_name: {
6207		const loaded_struct = ip.loadStructType(struct_ty.toIntern());
6208		switch (loaded_struct.layout) {
6209		.auto, .@"extern" => break :field_name if (loaded_struct.fieldName(ip, extra.field_index).unwrap()) \|field_name\|
6210		.{ .identifier = field_name.toSlice(ip) }
6211		else
6212		.{ .field = extra.field_index },
6213		.@"packed" => {
6214		const int_info = struct_ty.intInfo(zcu);
6215
6216		const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1));
6217
6218		const bit_offset = zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index);
6219
6220		const field_int_signedness = if (inst_ty.isAbiInt(zcu))
6221		inst_ty.intInfo(zcu).signedness
6222		else
6223		.unsigned;
6224		const field_int_ty = try pt.intType(field_int_signedness, @as(u16, @intCast(inst_ty.bitSize(zcu))));
6225
6226		const temp_local = try f.allocLocal(inst, field_int_ty);
6227		try f.writeCValue(w, temp_local, .Other);
6228		try w.writeAll(" = zig_wrap_");
6229		try f.object.dg.renderTypeForBuiltinFnName(w, field_int_ty);
6230		try w.writeAll("((");
6231		try f.renderType(w, field_int_ty);
6232		try w.writeByte(')');
6233		const cant_cast = int_info.bits > 64;
6234		if (cant_cast) {
6235		if (field_int_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
6236		try w.writeAll("zig_lo_");
6237		try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty);
6238		try w.writeByte('(');
6239		}
6240		if (bit_offset > 0) {
6241		try w.writeAll("zig_shr_");
6242		try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty);
6243		try w.writeByte('(');
6244		}
6245		try f.writeCValue(w, struct_byval, .Other);
6246		if (bit_offset > 0) try w.print(", {f})", .{
6247		try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)),
6248		});
6249		if (cant_cast) try w.writeByte(')');
6250		try f.object.dg.renderBuiltinInfo(w, field_int_ty, .bits);
6251		try w.writeAll(");");
6252		try f.object.newline();
6253		if (inst_ty.eql(field_int_ty, zcu)) return temp_local;
6254
6255		const local = try f.allocLocal(inst, inst_ty);
6256		if (local.new_local != temp_local.new_local) {
6257		try w.writeAll("memcpy(");
6258		try f.writeCValue(w, .{ .local_ref = local.new_local }, .FunctionArgument);
6259		try w.writeAll(", ");
6260		try f.writeCValue(w, .{ .local_ref = temp_local.new_local }, .FunctionArgument);
6261		try w.writeAll(", sizeof(");
6262		try f.renderType(w, inst_ty);
6263		try w.writeAll("));");
6264		try f.object.newline();
6265		}
6266		try freeLocal(f, inst, temp_local.new_local, null);
6267		return local;
6268		},
	6071	.struct_type => .{ .identifier = struct_ty.structFieldName(extra.field_index, zcu).unwrap().?.toSlice(ip) },
	6072	.union_type => name: {
	6073	const union_type = ip.loadUnionType(struct_ty.toIntern());
	6074	const enum_tag_ty: Type = .fromInterned(union_type.enum_tag_ty);
	6075	const field_name_str = enum_tag_ty.enumFieldName(extra.field_index, zcu).toSlice(ip);
	6076	if (union_type.hasTag(ip)) {
	6077	break :name .{ .payload_identifier = field_name_str };
	6078	} else {
	6079	break :name .{ .identifier = field_name_str };
6269	6080	}
6270	6081	},
6271	6082	.tuple_type => .{ .field = extra.field_index },
6272		.union_type => field_name: {
6273		const loaded_union = ip.loadUnionType(struct_ty.toIntern());
6274		switch (loaded_union.flagsUnordered(ip).layout) {
6275		.auto, .@"extern" => {
6276		const name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index];
6277		break :field_name if (loaded_union.hasTag(ip))
6278		.{ .payload_identifier = name.toSlice(ip) }
6279		else
6280		.{ .identifier = name.toSlice(ip) };
6281		},
6282		.@"packed" => {
6283		const operand_lval = if (struct_byval == .constant) blk: {
6284		const operand_local = try f.allocLocal(inst, struct_ty);
6285		try f.writeCValue(w, operand_local, .Other);
6286		try w.writeAll(" = ");
6287		try f.writeCValue(w, struct_byval, .Other);
6288		try w.writeByte(';');
6289		try f.object.newline();
6290		break :blk operand_local;
6291		} else struct_byval;
6292		const local = try f.allocLocal(inst, inst_ty);
6293		if (switch (local) {
6294		.new_local, .local => \|local_index\| switch (operand_lval) {
6295		.new_local, .local => \|operand_local_index\| local_index != operand_local_index,
6296		else => true,
6297		},
6298		else => true,
6299		}) {
6300		try w.writeAll("memcpy(&");
6301		try f.writeCValue(w, local, .Other);
6302		try w.writeAll(", &");
6303		try f.writeCValue(w, operand_lval, .Other);
6304		try w.writeAll(", sizeof(");
6305		try f.renderType(w, inst_ty);
6306		try w.writeAll("));");
6307		try f.object.newline();
6308		}
6309		try f.freeCValue(inst, operand_lval);
6310		return local;
6311		},
6312		}
6313		},
6314	6083	else => unreachable,
6315	6084	};
6316	6085
...	...	@@ -7702,98 +7471,13 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue {
7702	7471	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7703	7472
7704	7473	const a = try Assignment.start(f, w, try f.ctypeFromType(field_ty, .complete));
7705		try f.writeCValueMember(w, local, if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\|
7706		.{ .identifier = field_name.toSlice(ip) }
7707		else
7708		.{ .field = field_index });
	7474	try f.writeCValueMember(w, local, .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) });
7709	7475	try a.assign(f, w);
7710	7476	try f.writeCValue(w, resolved_elements[field_index], .Other);
7711	7477	try a.end(f, w);
7712	7478	}
7713	7479	},
7714		.@"packed" => {
7715		try f.writeCValue(w, local, .Other);
7716		try w.writeAll(" = ");
7717
7718		const backing_int_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip));
7719		const int_info = backing_int_ty.intInfo(zcu);
7720
7721		const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1));
7722
7723		var bit_offset: u64 = 0;
7724
7725		var empty = true;
7726		for (0..elements.len) \|field_index\| {
7727		if (inst_ty.structFieldIsComptime(field_index, zcu)) continue;
7728		const field_ty = inst_ty.fieldType(field_index, zcu);
7729		if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7730
7731		if (!empty) {
7732		try w.writeAll("zig_or_");
7733		try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7734		try w.writeByte('(');
7735		}
7736		empty = false;
7737		}
7738		empty = true;
7739		for (resolved_elements, 0..) \|element, field_index\| {
7740		if (inst_ty.structFieldIsComptime(field_index, zcu)) continue;
7741		const field_ty = inst_ty.fieldType(field_index, zcu);
7742		if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7743
7744		if (!empty) try w.writeAll(", ");
7745		// TODO: Skip this entire shift if val is 0?
7746		try w.writeAll("zig_shlw_");
7747		try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7748		try w.writeByte('(');
7749
7750		if (field_ty.isAbiInt(zcu)) {
7751		try w.writeAll("zig_and_");
7752		try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7753		try w.writeByte('(');
7754		}
7755
7756		if (inst_ty.isAbiInt(zcu) and (field_ty.isAbiInt(zcu) or field_ty.isPtrAtRuntime(zcu))) {
7757		try f.renderIntCast(w, inst_ty, element, .{}, field_ty, .FunctionArgument);
7758		} else {
7759		try w.writeByte('(');
7760		try f.renderType(w, inst_ty);
7761		try w.writeByte(')');
7762		if (field_ty.isPtrAtRuntime(zcu)) {
7763		try w.writeByte('(');
7764		try f.renderType(w, switch (int_info.signedness) {
7765		.unsigned => .usize,
7766		.signed => .isize,
7767		});
7768		try w.writeByte(')');
7769		}
7770		try f.writeCValue(w, element, .Other);
7771		}
7772
7773		if (field_ty.isAbiInt(zcu)) {
7774		try w.writeAll(", ");
7775		const field_int_info = field_ty.intInfo(zcu);
7776		const field_mask = if (int_info.signedness == .signed and int_info.bits == field_int_info.bits)
7777		try pt.intValue(backing_int_ty, -1)
7778		else
7779		try (try pt.intType(.unsigned, field_int_info.bits)).maxIntScalar(pt, backing_int_ty);
7780		try f.object.dg.renderValue(w, field_mask, .FunctionArgument);
7781		try w.writeByte(')');
7782		}
7783
7784		try w.print(", {f}", .{
7785		try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)),
7786		});
7787		try f.object.dg.renderBuiltinInfo(w, inst_ty, .bits);
7788		try w.writeByte(')');
7789		if (!empty) try w.writeByte(')');
7790
7791		bit_offset += field_ty.bitSize(zcu);
7792		empty = false;
7793		}
7794		try w.writeByte(';');
7795		try f.object.newline();
7796		},
	7480	.@"packed" => unreachable, // `Air.Legalize.Feature.expand_packed_struct_init` handles this case
7797	7481	}
7798	7482	},
7799	7483	.tuple_type => \|tuple_info\| for (0..tuple_info.types.len) \|field_index\| {
...	...	@@ -7828,9 +7512,10 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue {
7828	7512	try reap(f, inst, &.{extra.init});
7829	7513
7830	7514	const w = &f.object.code.writer;
7831		const local = try f.allocLocal(inst, union_ty);
7832	7515	if (loaded_union.flagsUnordered(ip).layout == .@"packed") return f.moveCValue(inst, union_ty, payload);
7833	7516
	7517	const local = try f.allocLocal(inst, union_ty);
	7518
7834	7519	const field: CValue = if (union_ty.unionTagTypeSafety(zcu)) \|tag_ty\| field: {
7835	7520	const layout = union_ty.unionGetLayout(zcu);
7836	7521	if (layout.tag_size != 0) {

src/codegen/c/Type.zig+1-5

...	...	@@ -2514,11 +2514,7 @@ pub const Pool = struct {
2514	2514	kind.noParameter(),
2515	2515	);
2516	2516	if (field_ctype.index == .void) continue;
2517		const field_name = if (loaded_struct.fieldName(ip, field_index)
2518		.unwrap()) \|field_name\|
2519		try pool.string(allocator, field_name.toSlice(ip))
2520		else
2521		String.fromUnnamed(@intCast(field_index));
	2517	const field_name = try pool.string(allocator, loaded_struct.fieldName(ip, field_index).toSlice(ip));
2522	2518	const field_alignas = AlignAs.fromAlignment(.{
2523	2519	.@"align" = loaded_struct.fieldAlign(ip, field_index),
2524	2520	.abi = field_type.abiAlignment(zcu),

src/codegen/llvm.zig+13-45

...	...	@@ -2409,8 +2409,7 @@ pub const Object = struct {
2409	2409	const field_size = field_ty.abiSize(zcu);
2410	2410	const field_align = ty.fieldAlignment(field_index, zcu);
2411	2411	const field_offset = ty.structFieldOffset(field_index, zcu);
2412		const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse
2413		try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
	2412	const field_name = struct_type.fieldName(ip, field_index);
2414	2413	fields.appendAssumeCapacity(try o.builder.debugMemberType(
2415	2414	try o.builder.metadataString(field_name.toSlice(ip)),
2416	2415	null, // File
...	...	@@ -4885,6 +4884,11 @@ pub const FuncGen = struct {
4885	4884
4886	4885	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4887	4886	// zig fmt: off
	4887
	4888	// No "scalarize" legalizations are enabled, so these instructions never appear.
	4889	.legalize_vec_elem_val => unreachable,
	4890	.legalize_vec_store_elem => unreachable,
	4891
4888	4892	.add => try self.airAdd(inst, .normal),
4889	4893	.add_optimized => try self.airAdd(inst, .fast),
4890	4894	.add_wrap => try self.airAddWrap(inst),
...	...	@@ -5091,8 +5095,6 @@ pub const FuncGen = struct {
5091	5095	.wasm_memory_size => try self.airWasmMemorySize(inst),
5092	5096	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
5093	5097
5094		.vector_store_elem => try self.airVectorStoreElem(inst),
5095
5096	5098	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
5097	5099
5098	5100	.inferred_alloc, .inferred_alloc_comptime => unreachable,
...	...	@@ -6871,16 +6873,14 @@ pub const FuncGen = struct {
6871	6873	const array_llvm_ty = try o.lowerType(pt, array_ty);
6872	6874	const elem_ty = array_ty.childType(zcu);
6873	6875	if (isByRef(array_ty, zcu)) {
6874		const indices: [2]Builder.Value = .{
6875		try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs,
6876		};
	6876	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
	6877	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
	6878	rhs,
	6879	}, "");
6877	6880	if (isByRef(elem_ty, zcu)) {
6878		const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6879	6881	const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6880	6882	return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6881	6883	} else {
6882		const elem_ptr =
6883		try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6884	6884	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
6885	6885	}
6886	6886	}
...	...	@@ -8138,33 +8138,6 @@ pub const FuncGen = struct {
8138	8138	}, "");
8139	8139	}
8140	8140
8141		fn airVectorStoreElem(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8142		const o = self.ng.object;
8143		const pt = self.ng.pt;
8144		const zcu = pt.zcu;
8145		const data = self.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
8146		const extra = self.air.extraData(Air.Bin, data.payload).data;
8147
8148		const vector_ptr = try self.resolveInst(data.vector_ptr);
8149		const vector_ptr_ty = self.typeOf(data.vector_ptr);
8150		const index = try self.resolveInst(extra.lhs);
8151		const operand = try self.resolveInst(extra.rhs);
8152
8153		self.maybeMarkAllowZeroAccess(vector_ptr_ty.ptrInfo(zcu));
8154
8155		// TODO: Emitting a load here is a violation of volatile semantics. Not fixable in general.
8156		// https://github.com/ziglang/zig/issues/18652#issuecomment-2452844908
8157		const access_kind: Builder.MemoryAccessKind =
8158		if (vector_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
8159		const elem_llvm_ty = try o.lowerType(pt, vector_ptr_ty.childType(zcu));
8160		const alignment = vector_ptr_ty.ptrAlignment(zcu).toLlvm();
8161		const loaded = try self.wip.load(access_kind, elem_llvm_ty, vector_ptr, alignment, "");
8162
8163		const new_vector = try self.wip.insertElement(loaded, operand, index, "");
8164		_ = try self.store(vector_ptr, vector_ptr_ty, new_vector, .none);
8165		return .none;
8166		}
8167
8168	8141	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8169	8142	const o = fg.ng.object;
8170	8143	const pt = fg.ng.pt;
...	...	@@ -8301,8 +8274,7 @@ pub const FuncGen = struct {
8301	8274	const rhs = try self.resolveInst(bin_op.rhs);
8302	8275	const inst_ty = self.typeOfIndex(inst);
8303	8276	const scalar_ty = inst_ty.scalarType(zcu);
8304
8305		if (scalar_ty.isAnyFloat()) return self.todo("saturating float add", .{});
	8277	assert(scalar_ty.zigTypeTag(zcu) == .int);
8306	8278	return self.wip.callIntrinsic(
8307	8279	.normal,
8308	8280	.none,
...	...	@@ -8342,8 +8314,7 @@ pub const FuncGen = struct {
8342	8314	const rhs = try self.resolveInst(bin_op.rhs);
8343	8315	const inst_ty = self.typeOfIndex(inst);
8344	8316	const scalar_ty = inst_ty.scalarType(zcu);
8345
8346		if (scalar_ty.isAnyFloat()) return self.todo("saturating float sub", .{});
	8317	assert(scalar_ty.zigTypeTag(zcu) == .int);
8347	8318	return self.wip.callIntrinsic(
8348	8319	.normal,
8349	8320	.none,
...	...	@@ -8383,8 +8354,7 @@ pub const FuncGen = struct {
8383	8354	const rhs = try self.resolveInst(bin_op.rhs);
8384	8355	const inst_ty = self.typeOfIndex(inst);
8385	8356	const scalar_ty = inst_ty.scalarType(zcu);
8386
8387		if (scalar_ty.isAnyFloat()) return self.todo("saturating float mul", .{});
	8357	assert(scalar_ty.zigTypeTag(zcu) == .int);
8388	8358	return self.wip.callIntrinsic(
8389	8359	.normal,
8390	8360	.none,
...	...	@@ -11452,7 +11422,6 @@ pub const FuncGen = struct {
11452	11422	const access_kind: Builder.MemoryAccessKind =
11453	11423	if (info.flags.is_volatile) .@"volatile" else .normal;
11454	11424
11455		assert(info.flags.vector_index != .runtime);
11456	11425	if (info.flags.vector_index != .none) {
11457	11426	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11458	11427	const vec_elem_ty = try o.lowerType(pt, elem_ty);
...	...	@@ -11522,7 +11491,6 @@ pub const FuncGen = struct {
11522	11491	const access_kind: Builder.MemoryAccessKind =
11523	11492	if (info.flags.is_volatile) .@"volatile" else .normal;
11524	11493
11525		assert(info.flags.vector_index != .runtime);
11526	11494	if (info.flags.vector_index != .none) {
11527	11495	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11528	11496	const vec_elem_ty = try o.lowerType(pt, elem_ty);

src/codegen/riscv64/CodeGen.zig+5-1

...	...	@@ -1391,6 +1391,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1391	1391	const tag = air_tags[@intFromEnum(inst)];
1392	1392	switch (tag) {
1393	1393	// zig fmt: off
	1394
	1395	// No "scalarize" legalizations are enabled, so these instructions never appear.
	1396	.legalize_vec_elem_val => unreachable,
	1397	.legalize_vec_store_elem => unreachable,
	1398
1394	1399	.add,
1395	1400	.add_wrap,
1396	1401	.sub,
...	...	@@ -1633,7 +1638,6 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1633	1638
1634	1639	.is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}),
1635	1640	.error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}),
1636		.vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}),
1637	1641
1638	1642	.c_va_arg => return func.fail("TODO implement c_va_arg", .{}),
1639	1643	.c_va_copy => return func.fail("TODO implement c_va_copy", .{}),

src/codegen/sparc64/CodeGen.zig+5-1

...	...	@@ -479,6 +479,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
479	479	self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
480	480	switch (air_tags[@intFromEnum(inst)]) {
481	481	// zig fmt: off
	482
	483	// No "scalarize" legalizations are enabled, so these instructions never appear.
	484	.legalize_vec_elem_val => unreachable,
	485	.legalize_vec_store_elem => unreachable,
	486
482	487	.ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
483	488	.ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
484	489
...	...	@@ -702,7 +707,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
702	707
703	708	.is_named_enum_value => @panic("TODO implement is_named_enum_value"),
704	709	.error_set_has_value => @panic("TODO implement error_set_has_value"),
705		.vector_store_elem => @panic("TODO implement vector_store_elem"),
706	710	.runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"),
707	711
708	712	.c_va_arg => return self.fail("TODO implement c_va_arg", .{}),

src/codegen/spirv/CodeGen.zig+1-27

...	...	@@ -1520,8 +1520,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1520	1520	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1521	1521	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1522	1522
1523		const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse
1524		try ip.getOrPutStringFmt(zcu.gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
	1523	const field_name = struct_type.fieldName(ip, field_index);
1525	1524	try member_types.append(try cg.resolveType(field_ty, .indirect));
1526	1525	try member_names.append(field_name.toSlice(ip));
1527	1526	try member_offsets.append(@intCast(ty.structFieldOffset(field_index, zcu)));
...	...	@@ -2726,8 +2725,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
2726	2725	.ptr_elem_val => try cg.airPtrElemVal(inst),
2727	2726	.array_elem_val => try cg.airArrayElemVal(inst),
2728	2727
2729		.vector_store_elem => return cg.airVectorStoreElem(inst),
2730
2731	2728	.set_union_tag => return cg.airSetUnionTag(inst),
2732	2729	.get_union_tag => try cg.airGetUnionTag(inst),
2733	2730	.union_init => try cg.airUnionInit(inst),
...	...	@@ -4446,29 +4443,6 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4446	4443	return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
4447	4444	}
4448	4445
4449		fn airVectorStoreElem(cg: *CodeGen, inst: Air.Inst.Index) !void {
4450		const zcu = cg.module.zcu;
4451		const data = cg.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
4452		const extra = cg.air.extraData(Air.Bin, data.payload).data;
4453
4454		const vector_ptr_ty = cg.typeOf(data.vector_ptr);
4455		const vector_ty = vector_ptr_ty.childType(zcu);
4456		const scalar_ty = vector_ty.scalarType(zcu);
4457
4458		const scalar_ty_id = try cg.resolveType(scalar_ty, .indirect);
4459		const storage_class = cg.module.storageClass(vector_ptr_ty.ptrAddressSpace(zcu));
4460		const scalar_ptr_ty_id = try cg.module.ptrType(scalar_ty_id, storage_class);
4461
4462		const vector_ptr = try cg.resolve(data.vector_ptr);
4463		const index = try cg.resolve(extra.lhs);
4464		const operand = try cg.resolve(extra.rhs);
4465
4466		const elem_ptr_id = try cg.accessChainId(scalar_ptr_ty_id, vector_ptr, &.{index});
4467		try cg.store(scalar_ty, elem_ptr_id, operand, .{
4468		.is_volatile = vector_ptr_ty.isVolatilePtr(zcu),
4469		});
4470		}
4471
4472	4446	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void {
4473	4447	const zcu = cg.module.zcu;
4474	4448	const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

src/codegen/wasm/CodeGen.zig+4-1

...	...	@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum
1786	1786	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1787	1787	const air_tags = cg.air.instructions.items(.tag);
1788	1788	return switch (air_tags[@intFromEnum(inst)]) {
	1789	// No "scalarize" legalizations are enabled, so these instructions never appear.
	1790	.legalize_vec_elem_val => unreachable,
	1791	.legalize_vec_store_elem => unreachable,
	1792
1789	1793	.inferred_alloc, .inferred_alloc_comptime => unreachable,
1790	1794
1791	1795	.add => cg.airBinOp(inst, .add),
...	...	@@ -1978,7 +1982,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1978	1982	.save_err_return_trace_index,
1979	1983	.is_named_enum_value,
1980	1984	.addrspace_cast,
1981		.vector_store_elem,
1982	1985	.c_va_arg,
1983	1986	.c_va_copy,
1984	1987	.c_va_end,

src/codegen/x86_64/CodeGen.zig+65-972

...	...	@@ -854,12 +854,6 @@ const FrameAlloc = struct {
854	854	}
855	855	};
856	856
857		const StackAllocation = struct {
858		inst: ?Air.Inst.Index,
859		/// TODO do we need size? should be determined by inst.ty.abiSize(zcu)
860		size: u32,
861		};
862
863	857	const BlockData = struct {
864	858	relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
865	859	state: State,
...	...	@@ -89326,7 +89320,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89326	89320	error.SelectFailed => res[0] = try ops[0].load(val_ty, .{
89327	89321	.disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) {
89328	89322	.none => 0,
89329		.runtime => unreachable,
89330	89323	else => \|vector_index\| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
89331	89324	},
89332	89325	}, cg),
...	...	@@ -89569,7 +89562,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89569	89562	error.SelectFailed => try ops[0].store(&ops[1], .{
89570	89563	.disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) {
89571	89564	.none => 0,
89572		.runtime => unreachable,
89573	89565	else => \|vector_index\| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)),
89574	89566	},
89575	89567	.safe = switch (air_tag) {
...	...	@@ -103934,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103934	103926	try ops[0].toOffset(0, cg);
103935	103927	try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);
103936	103928	},
103937		.array_elem_val => {
	103929	.array_elem_val, .legalize_vec_elem_val => {
103938	103930	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
103939	103931	const array_ty = cg.typeOf(bin_op.lhs);
103940	103932	const res_ty = array_ty.elemType2(zcu);
...	...	@@ -171402,8 +171394,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
171402	171394	.aggregate_init => \|air_tag\| fallback: {
171403	171395	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
171404	171396	const agg_ty = ty_pl.ty.toType();
171405		if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or
171406		(agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst);
	171397	if (agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) {
	171398	break :fallback try cg.airAggregateInitBoolVec(inst);
	171399	}
171407	171400	var res = try cg.tempAllocMem(agg_ty);
171408	171401	const reset_index = cg.next_temp_index;
171409	171402	var bt = cg.liveness.iterateBigTomb(inst);
...	...	@@ -171441,10 +171434,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
171441	171434	}
171442	171435	}
171443	171436	},
171444		.@"packed" => return cg.fail("failed to select {s} {f}", .{
171445		@tagName(air_tag),
171446		agg_ty.fmt(pt),
171447		}),
	171437	.@"packed" => unreachable,
171448	171438	}
171449	171439	},
171450	171440	.tuple_type => \|tuple_type\| {
...	...	@@ -173054,10 +173044,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173054	173044	try ert.die(cg);
173055	173045	try res.finish(inst, &.{}, &.{}, cg);
173056	173046	},
173057		.vector_store_elem => {
173058		const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem;
173059		const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data;
173060		var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs });
	173047	.runtime_nav_ptr => {
	173048	const ty_nav = air_datas[@intFromEnum(inst)].ty_nav;
	173049	const nav = ip.getNav(ty_nav.nav);
	173050	const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip);
	173051	if (is_threadlocal) if (cg.target.ofmt == .coff or cg.mod.pic) {
	173052	try cg.spillRegisters(&.{ .rdi, .rax });
	173053	} else {
	173054	try cg.spillRegisters(&.{.rax});
	173055	};
	173056	var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav });
	173057	if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {};
	173058	try res.finish(inst, &.{}, &.{}, cg);
	173059	},
	173060	.c_va_arg => try cg.airVaArg(inst),
	173061	.c_va_copy => try cg.airVaCopy(inst),
	173062	.c_va_end => try cg.airVaEnd(inst),
	173063	.c_va_start => try cg.airVaStart(inst),
	173064	.legalize_vec_store_elem => {
	173065	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
	173066	const bin = cg.air.extraData(Air.Bin, pl_op.payload).data;
	173067	// vector_ptr, index, elem_val
	173068	var ops = try cg.tempsFromOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
173061	173069	cg.select(&.{}, &.{}, &ops, comptime &.{ .{
173062	173070	.src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173063	173071	.patterns = &.{
...	...	@@ -173639,7 +173647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173639	173647	} },
173640	173648	} }) catch \|err\| switch (err) {
173641	173649	error.SelectFailed => {
173642		const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu);
	173650	const elem_size = cg.typeOf(bin.rhs).abiSize(zcu);
173643	173651	while (try ops[0].toRegClass(true, .general_purpose, cg) or
173644	173652	try ops[1].toRegClass(true, .general_purpose, cg))
173645	173653	{}
...	...	@@ -173681,23 +173689,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173681	173689	};
173682	173690	for (ops) \|op\| try op.die(cg);
173683	173691	},
173684		.runtime_nav_ptr => {
173685		const ty_nav = air_datas[@intFromEnum(inst)].ty_nav;
173686		const nav = ip.getNav(ty_nav.nav);
173687		const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip);
173688		if (is_threadlocal) if (cg.target.ofmt == .coff or cg.mod.pic) {
173689		try cg.spillRegisters(&.{ .rdi, .rax });
173690		} else {
173691		try cg.spillRegisters(&.{.rax});
173692		};
173693		var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav });
173694		if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {};
173695		try res.finish(inst, &.{}, &.{}, cg);
173696		},
173697		.c_va_arg => try cg.airVaArg(inst),
173698		.c_va_copy => try cg.airVaCopy(inst),
173699		.c_va_end => try cg.airVaEnd(inst),
173700		.c_va_start => try cg.airVaStart(inst),
173701	173692	.work_item_id, .work_group_size, .work_group_id => unreachable,
173702	173693	}
173703	173694	try cg.resetTemps(@enumFromInt(0));
...	...	@@ -180646,944 +180637,57 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
180646	180637	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
180647	180638	}
180648	180639
180649		fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {
	180640	fn airAggregateInitBoolVec(self: *CodeGen, inst: Air.Inst.Index) !void {
180650	180641	const pt = self.pt;
180651	180642	const zcu = pt.zcu;
	180643	const result_ty = self.typeOfIndex(inst);
	180644	const len: usize = @intCast(result_ty.arrayLen(zcu));
180652	180645	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
180653		const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;
180654
180655		const dst_ty = self.typeOfIndex(inst);
180656		const elem_ty = dst_ty.childType(zcu);
180657		const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu));
180658		const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
180659		const lhs_ty = self.typeOf(extra.a);
180660		const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu));
180661		const rhs_ty = self.typeOf(extra.b);
180662		const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu));
180663		const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
180664
180665		const ExpectedContents = [32]?i32;
180666		var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
180667		std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
180668		const allocator = stack.get();
180669
180670		const mask_elems = try allocator.alloc(?i32, extra.mask_len);
180671		defer allocator.free(mask_elems);
180672		for (mask_elems, 0..) \|*mask_elem, elem_index\| {
180673		const mask_elem_val =
180674		Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable;
180675		mask_elem.* = if (mask_elem_val.isUndef(zcu))
180676		null
180677		else
180678		@intCast(mask_elem_val.toSignedInt(zcu));
180679		}
180680
180681		const has_avx = self.hasFeature(.avx);
180682		const result = @as(?MCValue, result: {
180683		for (mask_elems) \|mask_elem\| {
180684		if (mask_elem) \|_\| break;
180685		} else break :result try self.allocRegOrMem(inst, true);
180686
180687		for (mask_elems, 0..) \|mask_elem, elem_index\| {
180688		if (mask_elem orelse continue != elem_index) break;
180689		} else {
180690		const lhs_mcv = try self.resolveInst(extra.a);
180691		if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
180692		const dst_mcv = try self.allocRegOrMem(inst, true);
180693		try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
180694		break :result dst_mcv;
180695		}
180696
180697		for (mask_elems, 0..) \|mask_elem, elem_index\| {
180698		if (~(mask_elem orelse continue) != elem_index) break;
180699		} else {
180700		const rhs_mcv = try self.resolveInst(extra.b);
180701		if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
180702		const dst_mcv = try self.allocRegOrMem(inst, true);
180703		try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
180704		break :result dst_mcv;
180705		}
180706
180707		for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {
180708		if (elem_abi_size > 8) break :unpck;
180709		if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck;
180710
180711		var sources: [2]?u1 = @splat(null);
180712		for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180713		const mask_elem = maybe_mask_elem orelse continue;
180714		const mask_elem_index =
180715		std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
180716		const elem_byte = (elem_index >> 1) * elem_abi_size;
180717		if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
180718		.unpckl => 0b0000,
180719		.unpckh => 0b1000,
180720		else => unreachable,
180721		}) \| (elem_byte << 1 & 0b10000)) break :unpck;
180722
180723		const source = @intFromBool(mask_elem < 0);
180724		if (sources[elem_index & 0b00001]) \|prev_source\| {
180725		if (source != prev_source) break :unpck;
180726		} else sources[elem_index & 0b00001] = source;
180727		}
180728		if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
180729
180730		const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180731		const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180732		const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180733		const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180734
180735		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180736		self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180737		lhs_mcv
180738		else if (has_avx and lhs_mcv.isRegister())
180739		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180740		else
180741		try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180742		const dst_reg = dst_mcv.getReg().?;
180743		const dst_alias = registerAlias(dst_reg, max_abi_size);
180744
180745		const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
180746		(dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
180747		4 => if (has_avx) .v_ps else ._ps,
180748		8 => if (has_avx) .v_pd else ._pd,
180749		else => unreachable,
180750		}, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
180751		.unpckl => switch (elem_abi_size) {
180752		1 => .unpcklbw,
180753		2 => .unpcklwd,
180754		4 => .unpckldq,
180755		8 => .unpcklqdq,
180756		else => unreachable,
180757		},
180758		.unpckh => switch (elem_abi_size) {
180759		1 => .unpckhbw,
180760		2 => .unpckhwd,
180761		4 => .unpckhdq,
180762		8 => .unpckhqdq,
180763		else => unreachable,
180764		},
180765		else => unreachable,
180766		} };
180767		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory(
180768		mir_tag,
180769		dst_alias,
180770		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180771		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180772		) else try self.asmRegisterRegisterRegister(
180773		mir_tag,
180774		dst_alias,
180775		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180776		registerAlias(if (rhs_mcv.isRegister())
180777		rhs_mcv.getReg().?
180778		else
180779		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180780		) else if (rhs_mcv.isBase()) try self.asmRegisterMemory(
180781		mir_tag,
180782		dst_alias,
180783		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180784		) else try self.asmRegisterRegister(
180785		mir_tag,
180786		dst_alias,
180787		registerAlias(if (rhs_mcv.isRegister())
180788		rhs_mcv.getReg().?
180789		else
180790		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180791		);
180792		break :result dst_mcv;
180793		}
180794
180795		pshufd: {
180796		if (elem_abi_size != 4) break :pshufd;
180797		if (max_abi_size > self.vectorSize(.float)) break :pshufd;
180798
180799		var control: u8 = 0b00_00_00_00;
180800		var sources: [1]?u1 = @splat(null);
180801		for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180802		const mask_elem = maybe_mask_elem orelse continue;
180803		const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180804		if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
180805
180806		const source = @intFromBool(mask_elem < 0);
180807		if (sources[0]) \|prev_source\| {
180808		if (source != prev_source) break :pshufd;
180809		} else sources[(elem_index & 0b010) >> 1] = source;
180810
180811		const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
180812		const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
180813		if (elem_index & 0b100 == 0)
180814		control \|= select_mask
180815		else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
180816		}
180817
180818		const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180819		const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180820		const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
180821
180822		const dst_reg = if (src_mcv.isRegister() and
180823		self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
180824		src_mcv.getReg().?
180825		else
180826		try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
180827		const dst_alias = registerAlias(dst_reg, max_abi_size);
180828
180829		if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180830		.{ if (has_avx) .vp_d else .p_d, .shuf },
180831		dst_alias,
180832		try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180833		.u(control),
180834		) else try self.asmRegisterRegisterImmediate(
180835		.{ if (has_avx) .vp_d else .p_d, .shuf },
180836		dst_alias,
180837		registerAlias(if (src_mcv.isRegister())
180838		src_mcv.getReg().?
180839		else
180840		try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
180841		.u(control),
180842		);
180843		break :result .{ .register = dst_reg };
180844		}
180845
180846		shufps: {
180847		if (elem_abi_size != 4) break :shufps;
180848		if (max_abi_size > self.vectorSize(.float)) break :shufps;
180849
180850		var control: u8 = 0b00_00_00_00;
180851		var sources: [2]?u1 = @splat(null);
180852		for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180853		const mask_elem = maybe_mask_elem orelse continue;
180854		const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180855		if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
180856
180857		const source = @intFromBool(mask_elem < 0);
180858		if (sources[(elem_index & 0b010) >> 1]) \|prev_source\| {
180859		if (source != prev_source) break :shufps;
180860		} else sources[(elem_index & 0b010) >> 1] = source;
180861
180862		const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
180863		const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
180864		if (elem_index & 0b100 == 0)
180865		control \|= select_mask
180866		else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
180867		}
180868		if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
180869
180870		const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180871		const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180872		const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180873		const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180874
180875		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180876		self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180877		lhs_mcv
180878		else if (has_avx and lhs_mcv.isRegister())
180879		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180880		else
180881		try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180882		const dst_reg = dst_mcv.getReg().?;
180883		const dst_alias = registerAlias(dst_reg, max_abi_size);
180884
180885		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
180886		.{ .v_ps, .shuf },
180887		dst_alias,
180888		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180889		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180890		.u(control),
180891		) else try self.asmRegisterRegisterRegisterImmediate(
180892		.{ .v_ps, .shuf },
180893		dst_alias,
180894		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180895		registerAlias(if (rhs_mcv.isRegister())
180896		rhs_mcv.getReg().?
180897		else
180898		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180899		.u(control),
180900		) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180901		.{ ._ps, .shuf },
180902		dst_alias,
180903		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180904		.u(control),
180905		) else try self.asmRegisterRegisterImmediate(
180906		.{ ._ps, .shuf },
180907		dst_alias,
180908		registerAlias(if (rhs_mcv.isRegister())
180909		rhs_mcv.getReg().?
180910		else
180911		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180912		.u(control),
180913		);
180914		break :result dst_mcv;
180915		}
180916
180917		shufpd: {
180918		if (elem_abi_size != 8) break :shufpd;
180919		if (max_abi_size > self.vectorSize(.float)) break :shufpd;
180920
180921		var control: u4 = 0b0_0_0_0;
180922		var sources: [2]?u1 = @splat(null);
180923		for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180924		const mask_elem = maybe_mask_elem orelse continue;
180925		const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180926		if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
180927
180928		const source = @intFromBool(mask_elem < 0);
180929		if (sources[elem_index & 0b01]) \|prev_source\| {
180930		if (source != prev_source) break :shufpd;
180931		} else sources[elem_index & 0b01] = source;
180932
180933		control \|= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
180934		}
180935		if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
180936
180937		const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
180938		const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
180939		const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180940		const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180941
180942		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180943		self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180944		lhs_mcv
180945		else if (has_avx and lhs_mcv.isRegister())
180946		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180947		else
180948		try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180949		const dst_reg = dst_mcv.getReg().?;
180950		const dst_alias = registerAlias(dst_reg, max_abi_size);
180951
180952		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
180953		.{ .v_pd, .shuf },
180954		dst_alias,
180955		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180956		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180957		.u(control),
180958		) else try self.asmRegisterRegisterRegisterImmediate(
180959		.{ .v_pd, .shuf },
180960		dst_alias,
180961		registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180962		registerAlias(if (rhs_mcv.isRegister())
180963		rhs_mcv.getReg().?
180964		else
180965		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180966		.u(control),
180967		) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180968		.{ ._pd, .shuf },
180969		dst_alias,
180970		try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180971		.u(control),
180972		) else try self.asmRegisterRegisterImmediate(
180973		.{ ._pd, .shuf },
180974		dst_alias,
180975		registerAlias(if (rhs_mcv.isRegister())
180976		rhs_mcv.getReg().?
180977		else
180978		try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180979		.u(control),
180980		);
180981		break :result dst_mcv;
180982		}
180983
180984		blend: {
180985		if (elem_abi_size < 2) break :blend;
180986		if (dst_abi_size > self.vectorSize(.float)) break :blend;
180987		if (!self.hasFeature(.sse4_1)) break :blend;
180988
180989		var control: u8 = 0b0_0_0_0_0_0_0_0;
180990		for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180991		const mask_elem = maybe_mask_elem orelse continue;
180992		const mask_elem_index =
180993		std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
180994		if (mask_elem_index != elem_index) break :blend;
180995
180996		const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
180997		if (elem_index & 0b1000 == 0)
180998		control \|= select_mask
180999		else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
181000		}
181001
181002		if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
181003		const expanded_control = switch (elem_abi_size) {
181004		4 => control,
181005		8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
181006		@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
181007		@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
181008		@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
181009		else => break :vpblendd,
181010		};
181011
181012		const lhs_mcv = try self.resolveInst(extra.a);
181013		const lhs_reg = if (lhs_mcv.isRegister())
181014		lhs_mcv.getReg().?
181015		else
181016		try self.copyToTmpRegister(dst_ty, lhs_mcv);
181017		const lhs_lock = self.register_manager.lockReg(lhs_reg);
181018		defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
181019
181020		const rhs_mcv = try self.resolveInst(extra.b);
181021		const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
181022		if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181023		.{ .vp_d, .blend },
181024		registerAlias(dst_reg, dst_abi_size),
181025		registerAlias(lhs_reg, dst_abi_size),
181026		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181027		.u(expanded_control),
181028		) else try self.asmRegisterRegisterRegisterImmediate(
181029		.{ .vp_d, .blend },
181030		registerAlias(dst_reg, dst_abi_size),
181031		registerAlias(lhs_reg, dst_abi_size),
181032		registerAlias(if (rhs_mcv.isRegister())
181033		rhs_mcv.getReg().?
181034		else
181035		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181036		.u(expanded_control),
181037		);
181038		break :result .{ .register = dst_reg };
181039		}
181040
181041		if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
181042		const expanded_control = switch (elem_abi_size) {
181043		2 => control,
181044		4 => if (dst_abi_size <= 16 or
181045		@as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
181046		@as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
181047		@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
181048		@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
181049		@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
181050		else
181051		break :pblendw,
181052		8 => if (dst_abi_size <= 16 or
181053		@as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
181054		@as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) \|
181055		@as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
181056		else
181057		break :pblendw,
181058		16 => break :pblendw,
181059		else => unreachable,
181060		};
181061
181062		const lhs_mcv = try self.resolveInst(extra.a);
181063		const rhs_mcv = try self.resolveInst(extra.b);
181064
181065		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181066		self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181067		lhs_mcv
181068		else if (has_avx and lhs_mcv.isRegister())
181069		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181070		else
181071		try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181072		const dst_reg = dst_mcv.getReg().?;
181073
181074		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181075		.{ .vp_w, .blend },
181076		registerAlias(dst_reg, dst_abi_size),
181077		registerAlias(if (lhs_mcv.isRegister())
181078		lhs_mcv.getReg().?
181079		else
181080		dst_reg, dst_abi_size),
181081		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181082		.u(expanded_control),
181083		) else try self.asmRegisterRegisterRegisterImmediate(
181084		.{ .vp_w, .blend },
181085		registerAlias(dst_reg, dst_abi_size),
181086		registerAlias(if (lhs_mcv.isRegister())
181087		lhs_mcv.getReg().?
181088		else
181089		dst_reg, dst_abi_size),
181090		registerAlias(if (rhs_mcv.isRegister())
181091		rhs_mcv.getReg().?
181092		else
181093		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181094		.u(expanded_control),
181095		) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
181096		.{ .p_w, .blend },
181097		registerAlias(dst_reg, dst_abi_size),
181098		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181099		.u(expanded_control),
181100		) else try self.asmRegisterRegisterImmediate(
181101		.{ .p_w, .blend },
181102		registerAlias(dst_reg, dst_abi_size),
181103		registerAlias(if (rhs_mcv.isRegister())
181104		rhs_mcv.getReg().?
181105		else
181106		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181107		.u(expanded_control),
181108		);
181109		break :result .{ .register = dst_reg };
181110		}
181111
181112		const expanded_control = switch (elem_abi_size) {
181113		4, 8 => control,
181114		16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
181115		@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
181116		else => unreachable,
181117		};
181118
181119		const lhs_mcv = try self.resolveInst(extra.a);
181120		const rhs_mcv = try self.resolveInst(extra.b);
181121
181122		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181123		self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181124		lhs_mcv
181125		else if (has_avx and lhs_mcv.isRegister())
181126		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181127		else
181128		try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181129		const dst_reg = dst_mcv.getReg().?;
181130
181131		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181132		switch (elem_abi_size) {
181133		4 => .{ .v_ps, .blend },
181134		8, 16 => .{ .v_pd, .blend },
181135		else => unreachable,
181136		},
181137		registerAlias(dst_reg, dst_abi_size),
181138		registerAlias(if (lhs_mcv.isRegister())
181139		lhs_mcv.getReg().?
181140		else
181141		dst_reg, dst_abi_size),
181142		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181143		.u(expanded_control),
181144		) else try self.asmRegisterRegisterRegisterImmediate(
181145		switch (elem_abi_size) {
181146		4 => .{ .v_ps, .blend },
181147		8, 16 => .{ .v_pd, .blend },
181148		else => unreachable,
181149		},
181150		registerAlias(dst_reg, dst_abi_size),
181151		registerAlias(if (lhs_mcv.isRegister())
181152		lhs_mcv.getReg().?
181153		else
181154		dst_reg, dst_abi_size),
181155		registerAlias(if (rhs_mcv.isRegister())
181156		rhs_mcv.getReg().?
181157		else
181158		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181159		.u(expanded_control),
181160		) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
181161		switch (elem_abi_size) {
181162		4 => .{ ._ps, .blend },
181163		8, 16 => .{ ._pd, .blend },
181164		else => unreachable,
181165		},
181166		registerAlias(dst_reg, dst_abi_size),
181167		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181168		.u(expanded_control),
181169		) else try self.asmRegisterRegisterImmediate(
181170		switch (elem_abi_size) {
181171		4 => .{ ._ps, .blend },
181172		8, 16 => .{ ._pd, .blend },
181173		else => unreachable,
181174		},
181175		registerAlias(dst_reg, dst_abi_size),
181176		registerAlias(if (rhs_mcv.isRegister())
181177		rhs_mcv.getReg().?
181178		else
181179		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181180		.u(expanded_control),
181181		);
181182		break :result .{ .register = dst_reg };
181183		}
181184
181185		blendv: {
181186		if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv;
181187
181188		const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);
181189		const select_mask_ty = try pt.vectorType(.{
181190		.len = @intCast(mask_elems.len),
181191		.child = select_mask_elem_ty.toIntern(),
181192		});
181193		var select_mask_elems: [32]InternPool.Index = undefined;
181194		for (
181195		select_mask_elems[0..mask_elems.len],
181196		mask_elems,
181197		0..,
181198		) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
181199		const mask_elem = maybe_mask_elem orelse continue;
181200		const mask_elem_index =
181201		std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
181202		if (mask_elem_index != elem_index) break :blendv;
181203
181204		select_mask_elem.* = (if (mask_elem < 0)
181205		try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)
181206		else
181207		try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
181208		}
181209		const select_mask_mcv = try self.lowerValue(
181210		try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]),
181211		);
	180646	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
181212	180647
181213		if (self.hasFeature(.sse4_1)) {
181214		const mir_tag: Mir.Inst.FixedTag = .{
181215		if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
181216		(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
181217		4 => if (has_avx) .v_ps else ._ps,
181218		8 => if (has_avx) .v_pd else ._pd,
181219		else => unreachable,
181220		} else if (has_avx) .vp_b else .p_b,
181221		.blendv,
181222		};
	180648	assert(result_ty.zigTypeTag(zcu) == .vector);
	180649	assert(result_ty.childType(zcu).toIntern() == .bool_type);
181223	180650
181224		const select_mask_reg = if (!has_avx) reg: {
181225		try self.register_manager.getKnownReg(.xmm0, null);
181226		try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
181227		break :reg .xmm0;
181228		} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
181229		const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
181230		const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
181231		defer self.register_manager.unlockReg(select_mask_lock);
181232
181233		const lhs_mcv = try self.resolveInst(extra.a);
181234		const rhs_mcv = try self.resolveInst(extra.b);
181235
181236		const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181237		self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181238		lhs_mcv
181239		else if (has_avx and lhs_mcv.isRegister())
181240		.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181241		else
181242		try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181243		const dst_reg = dst_mcv.getReg().?;
181244		const dst_alias = registerAlias(dst_reg, dst_abi_size);
	180651	const result_size = result_ty.abiSize(zcu);
	180652	if (result_size > 8) return self.fail("TODO airAggregateInitBoolVec over 8 bytes", .{});
181245	180653
181246		if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(
181247		mir_tag,
181248		dst_alias,
181249		if (lhs_mcv.isRegister())
181250		registerAlias(lhs_mcv.getReg().?, dst_abi_size)
181251		else
181252		dst_alias,
181253		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181254		select_mask_alias,
181255		) else try self.asmRegisterRegisterRegisterRegister(
181256		mir_tag,
181257		dst_alias,
181258		if (lhs_mcv.isRegister())
181259		registerAlias(lhs_mcv.getReg().?, dst_abi_size)
181260		else
181261		dst_alias,
181262		registerAlias(if (rhs_mcv.isRegister())
181263		rhs_mcv.getReg().?
181264		else
181265		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181266		select_mask_alias,
181267		) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
181268		mir_tag,
181269		dst_alias,
181270		try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181271		select_mask_alias,
181272		) else try self.asmRegisterRegisterRegister(
181273		mir_tag,
181274		dst_alias,
181275		registerAlias(if (rhs_mcv.isRegister())
181276		rhs_mcv.getReg().?
181277		else
181278		try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181279		select_mask_alias,
181280		);
181281		break :result dst_mcv;
181282		}
	180654	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
181283	180655
181284		const lhs_mcv = try self.resolveInst(extra.a);
181285		const rhs_mcv = try self.resolveInst(extra.b);
	180656	{
	180657	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
	180658	defer self.register_manager.unlockReg(dst_lock);
181286	180659
181287		const dst_mcv: MCValue = if (rhs_mcv.isRegister() and
181288		self.reuseOperand(inst, extra.b, 1, rhs_mcv))
181289		rhs_mcv
181290		else
181291		try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);
181292		const dst_reg = dst_mcv.getReg().?;
181293		const dst_alias = registerAlias(dst_reg, dst_abi_size);
	180660	try self.spillEflagsIfOccupied();
	180661	try self.asmRegisterRegister(
	180662	.{ ._, .xor },
	180663	registerAlias(dst_reg, @min(result_size, 4)),
	180664	registerAlias(dst_reg, @min(result_size, 4)),
	180665	);
181294	180666
181295		const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
181296		const mask_alias = registerAlias(mask_reg, dst_abi_size);
181297		const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);
181298		defer self.register_manager.unlockReg(mask_lock);
	180667	for (elements, 0..) \|elem, elem_i\| {
	180668	const elem_reg = try self.copyToTmpRegister(.bool, .{ .air_ref = elem });
	180669	const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
	180670	defer self.register_manager.unlockReg(elem_lock);
181299	180671
181300		const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())
181301		switch (elem_ty.floatBits(self.target)) {
181302		16, 80, 128 => .p_,
181303		32 => ._ps,
181304		64 => ._pd,
181305		else => unreachable,
181306		}
181307		else
181308		.p_;
181309		try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
181310		if (lhs_mcv.isBase()) try self.asmRegisterMemory(
181311		.{ mir_fixes, .andn },
181312		mask_alias,
181313		try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181314		) else try self.asmRegisterRegister(
181315		.{ mir_fixes, .andn },
181316		mask_alias,
181317		if (lhs_mcv.isRegister())
181318		lhs_mcv.getReg().?
181319		else
181320		try self.copyToTmpRegister(dst_ty, lhs_mcv),
	180672	try self.asmRegisterImmediate(
	180673	.{ ._, .@"and" },
	180674	registerAlias(elem_reg, @min(result_size, 4)),
	180675	.u(1),
181321	180676	);
181322		try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);
181323		break :result dst_mcv;
181324		}
181325
181326		pshufb: {
181327		if (max_abi_size > 16) break :pshufb;
181328		if (!self.hasFeature(.ssse3)) break :pshufb;
181329
181330		const temp_regs =
181331		try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
181332		const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
181333		defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
181334
181335		const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
181336		try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
181337
181338		const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
181339		try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
181340
181341		var lhs_mask_elems: [16]InternPool.Index = undefined;
181342		for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {
181343		const elem_index = byte_index / elem_abi_size;
181344		lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
181345		const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
181346		if (mask_elem < 0) break :elem 0b1_00_00000;
181347		const mask_elem_index: u31 = @intCast(mask_elem);
181348		const byte_off: u32 = @intCast(byte_index % elem_abi_size);
181349		break :elem mask_elem_index * elem_abi_size + byte_off;
181350		})).toIntern();
181351		}
181352		const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
181353		const lhs_mask_mcv = try self.lowerValue(
181354		try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]),
	180677	if (elem_i > 0) try self.asmRegisterImmediate(
	180678	.{ ._l, .sh },
	180679	registerAlias(elem_reg, @intCast(result_size)),
	180680	.u(@intCast(elem_i)),
181355	180681	);
181356		const lhs_mask_mem: Memory = .{
181357		.base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) },
181358		.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
181359		};
181360		if (has_avx) try self.asmRegisterRegisterMemory(
181361		.{ .vp_b, .shuf },
181362		lhs_temp_alias,
181363		lhs_temp_alias,
181364		lhs_mask_mem,
181365		) else try self.asmRegisterMemory(
181366		.{ .p_b, .shuf },
181367		lhs_temp_alias,
181368		lhs_mask_mem,
	180682	try self.asmRegisterRegister(
	180683	.{ ._, .@"or" },
	180684	registerAlias(dst_reg, @intCast(result_size)),
	180685	registerAlias(elem_reg, @intCast(result_size)),
181369	180686	);
181370
181371		var rhs_mask_elems: [16]InternPool.Index = undefined;
181372		for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {
181373		const elem_index = byte_index / elem_abi_size;
181374		rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
181375		const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
181376		if (mask_elem >= 0) break :elem 0b1_00_00000;
181377		const mask_elem_index: u31 = @intCast(~mask_elem);
181378		const byte_off: u32 = @intCast(byte_index % elem_abi_size);
181379		break :elem mask_elem_index * elem_abi_size + byte_off;
181380		})).toIntern();
181381		}
181382		const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
181383		const rhs_mask_mcv = try self.lowerValue(
181384		try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]),
181385		);
181386		const rhs_mask_mem: Memory = .{
181387		.base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) },
181388		.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
181389		};
181390		if (has_avx) try self.asmRegisterRegisterMemory(
181391		.{ .vp_b, .shuf },
181392		rhs_temp_alias,
181393		rhs_temp_alias,
181394		rhs_mask_mem,
181395		) else try self.asmRegisterMemory(
181396		.{ .p_b, .shuf },
181397		rhs_temp_alias,
181398		rhs_mask_mem,
181399		);
181400
181401		if (has_avx) try self.asmRegisterRegisterRegister(
181402		.{ switch (elem_ty.zigTypeTag(zcu)) {
181403		else => break :result null,
181404		.int => .vp_,
181405		.float => switch (elem_ty.floatBits(self.target)) {
181406		32 => .v_ps,
181407		64 => .v_pd,
181408		16, 80, 128 => break :result null,
181409		else => unreachable,
181410		},
181411		}, .@"or" },
181412		lhs_temp_alias,
181413		lhs_temp_alias,
181414		rhs_temp_alias,
181415		) else try self.asmRegisterRegister(
181416		.{ switch (elem_ty.zigTypeTag(zcu)) {
181417		else => break :result null,
181418		.int => .p_,
181419		.float => switch (elem_ty.floatBits(self.target)) {
181420		32 => ._ps,
181421		64 => ._pd,
181422		16, 80, 128 => break :result null,
181423		else => unreachable,
181424		},
181425		}, .@"or" },
181426		lhs_temp_alias,
181427		rhs_temp_alias,
181428		);
181429		break :result .{ .register = temp_regs[0] };
181430	180687	}
	180688	}
181431	180689
181432		break :result null;
181433		}) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{
181434		lhs_ty.fmt(pt),
181435		rhs_ty.fmt(pt),
181436		dst_ty.fmt(pt),
181437		Value.fromInterned(extra.mask).fmtValue(pt),
181438		});
181439		return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
181440		}
181441
181442		fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
181443		const pt = self.pt;
181444		const zcu = pt.zcu;
181445		const result_ty = self.typeOfIndex(inst);
181446		const len: usize = @intCast(result_ty.arrayLen(zcu));
181447		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
181448		const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
181449		const result: MCValue = result: {
181450		switch (result_ty.zigTypeTag(zcu)) {
181451		.@"struct" => {
181452		if (result_ty.containerLayout(zcu) == .@"packed") return self.fail(
181453		"TODO implement airAggregateInit for {f}",
181454		.{result_ty.fmt(pt)},
181455		);
181456		const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
181457		const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
181458		try self.genInlineMemset(
181459		.{ .lea_frame = .{ .index = frame_index } },
181460		.{ .immediate = 0 },
181461		.{ .immediate = result_ty.abiSize(zcu) },
181462		.{},
181463		);
181464		for (elements, 0..) \|elem, elem_i_usize\| {
181465		const elem_i: u32 = @intCast(elem_i_usize);
181466		if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
181467
181468		const elem_ty = result_ty.fieldType(elem_i, zcu);
181469		const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
181470		if (elem_bit_size > 64) {
181471		return self.fail(
181472		"TODO airAggregateInit implement packed structs with large fields",
181473		.{},
181474		);
181475		}
181476		const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
181477		const elem_abi_bits = elem_abi_size * 8;
181478		const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i);
181479		const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
181480		const elem_bit_off = elem_off % elem_abi_bits;
181481		const elem_mcv = try self.resolveInst(elem);
181482		const elem_lock = switch (elem_mcv) {
181483		.register => \|reg\| self.register_manager.lockReg(reg),
181484		.immediate => \|imm\| lock: {
181485		if (imm == 0) continue;
181486		break :lock null;
181487		},
181488		else => null,
181489		};
181490		defer if (elem_lock) \|lock\| self.register_manager.unlockReg(lock);
181491
181492		const elem_extra_bits = self.regExtraBits(elem_ty);
181493		{
181494		const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181495		const temp_alias = registerAlias(temp_reg, elem_abi_size);
181496		const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181497		defer self.register_manager.unlockReg(temp_lock);
181498
181499		if (elem_bit_off < elem_extra_bits) {
181500		try self.truncateRegister(elem_ty, temp_alias);
181501		}
181502		if (elem_bit_off > 0) try self.genShiftBinOpMir(
181503		.{ ._l, .sh },
181504		elem_ty,
181505		.{ .register = temp_alias },
181506		.u8,
181507		.{ .immediate = elem_bit_off },
181508		);
181509		try self.genBinOpMir(
181510		.{ ._, .@"or" },
181511		elem_ty,
181512		.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
181513		.{ .register = temp_alias },
181514		);
181515		}
181516		if (elem_bit_off > elem_extra_bits) {
181517		const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181518		const temp_alias = registerAlias(temp_reg, elem_abi_size);
181519		const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181520		defer self.register_manager.unlockReg(temp_lock);
181521
181522		if (elem_extra_bits > 0) {
181523		try self.truncateRegister(elem_ty, temp_alias);
181524		}
181525		try self.genShiftBinOpMir(
181526		.{ ._r, .sh },
181527		elem_ty,
181528		.{ .register = temp_reg },
181529		.u8,
181530		.{ .immediate = elem_abi_bits - elem_bit_off },
181531		);
181532		try self.genBinOpMir(
181533		.{ ._, .@"or" },
181534		elem_ty,
181535		.{ .load_frame = .{
181536		.index = frame_index,
181537		.off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
181538		} },
181539		.{ .register = temp_alias },
181540		);
181541		}
181542		}
181543		break :result .{ .load_frame = .{ .index = frame_index } };
181544		},
181545		.vector => {
181546		const elem_ty = result_ty.childType(zcu);
181547		if (elem_ty.toIntern() != .bool_type) return self.fail(
181548		"TODO implement airAggregateInit for {f}",
181549		.{result_ty.fmt(pt)},
181550		);
181551		const result_size: u32 = @intCast(result_ty.abiSize(zcu));
181552		const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
181553		const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
181554		defer self.register_manager.unlockReg(dst_lock);
181555		try self.asmRegisterRegister(
181556		.{ ._, .xor },
181557		registerAlias(dst_reg, @min(result_size, 4)),
181558		registerAlias(dst_reg, @min(result_size, 4)),
181559		);
181560
181561		for (elements, 0..) \|elem, elem_i\| {
181562		const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
181563		const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
181564		defer self.register_manager.unlockReg(elem_lock);
181565
181566		try self.asmRegisterImmediate(
181567		.{ ._, .@"and" },
181568		registerAlias(elem_reg, @min(result_size, 4)),
181569		.u(1),
181570		);
181571		if (elem_i > 0) try self.asmRegisterImmediate(
181572		.{ ._l, .sh },
181573		registerAlias(elem_reg, result_size),
181574		.u(@intCast(elem_i)),
181575		);
181576		try self.asmRegisterRegister(
181577		.{ ._, .@"or" },
181578		registerAlias(dst_reg, result_size),
181579		registerAlias(elem_reg, result_size),
181580		);
181581		}
181582		break :result .{ .register = dst_reg };
181583		},
181584		else => unreachable,
181585		}
181586		};
	180690	const result: MCValue = .{ .register = dst_reg };
181587	180691
181588	180692	if (elements.len <= Air.Liveness.bpi - 1) {
181589	180693	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
...	...	@@ -182269,15 +181373,6 @@ fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMe
182269	181373	};
182270	181374	}
182271	181375
182272		fn failMsg(cg: CodeGen, msg: Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
182273		@branchHint(.cold);
182274		const zcu = cg.pt.zcu;
182275		return switch (cg.owner) {
182276		.nav_index => \|i\| zcu.codegenFailMsg(i, msg),
182277		.lazy_sym => \|s\| zcu.codegenFailTypeMsg(s.ty, msg),
182278		};
182279		}
182280
182281	181376	fn parseRegName(name: []const u8) ?Register {
182282	181377	if (std.mem.startsWith(u8, name, "db")) return @enumFromInt(
182283	181378	@intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null),
...	...	@@ -188819,7 +187914,6 @@ const Select = struct {
188819	187914	const ptr_info = ty.ptrInfo(zcu);
188820	187915	return switch (ptr_info.flags.vector_index) {
188821	187916	.none => false,
188822		.runtime => unreachable,
188823	187917	else => ptr_info.child == .bool_type,
188824	187918	};
188825	187919	},
...	...	@@ -188827,7 +187921,6 @@ const Select = struct {
188827	187921	const ptr_info = ty.ptrInfo(zcu);
188828	187922	return switch (ptr_info.flags.vector_index) {
188829	187923	.none => false,
188830		.runtime => unreachable,
188831	187924	else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size,
188832	187925	};
188833	187926	},
...	...	@@ -190814,7 +189907,7 @@ const Select = struct {
190814	189907	.src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *
190815	189908	Select.Operand.Ref.src1.valueOf(s).immediate),
190816	189909	.vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) {
190817		.none, .runtime => unreachable,
	189910	.none => unreachable,
190818	189911	else => \|vector_index\| @intFromEnum(vector_index),
190819	189912	},
190820	189913	.src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate),

src/link/Dwarf.zig+6-22

...	...	@@ -3158,11 +3158,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo
3158	3158	.struct_field
3159	3159	else
3160	3160	.struct_field);
3161		if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\| try wip_nav.strp(field_name.toSlice(ip)) else {
3162		var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
3163		const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
3164		try wip_nav.strp(field_name);
3165		}
	3161	try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
3166	3162	try wip_nav.refType(field_type);
3167	3163	if (!is_comptime) {
3168	3164	try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
...	...	@@ -3187,7 +3183,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo
3187	3183	var field_bit_offset: u16 = 0;
3188	3184	for (0..loaded_struct.field_types.len) \|field_index\| {
3189	3185	try wip_nav.abbrevCode(.packed_struct_field);
3190		try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip));
	3186	try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
3191	3187	const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
3192	3188	try wip_nav.refType(field_type);
3193	3189	try diw.writeUleb128(field_bit_offset);
...	...	@@ -4269,11 +4265,7 @@ fn updateLazyValue(
4269	4265	.comptime_value_field_runtime_bits
4270	4266	else
4271	4267	continue);
4272		if (loaded_struct_type.fieldName(ip, field_index).unwrap()) \|field_name\| try wip_nav.strp(field_name.toSlice(ip)) else {
4273		var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
4274		const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
4275		try wip_nav.strp(field_name);
4276		}
	4268	try wip_nav.strp(loaded_struct_type.fieldName(ip, field_index).toSlice(ip));
4277	4269	const field_value: Value = .fromInterned(switch (aggregate.storage) {
4278	4270	.bytes => unreachable,
4279	4271	.elems => \|elems\| elems[field_index],
...	...	@@ -4467,11 +4459,7 @@ fn updateContainerTypeWriterError(
4467	4459	.struct_field
4468	4460	else
4469	4461	.struct_field);
4470		if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\| try wip_nav.strp(field_name.toSlice(ip)) else {
4471		var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
4472		const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
4473		try wip_nav.strp(field_name);
4474		}
	4462	try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
4475	4463	try wip_nav.refType(field_type);
4476	4464	if (!is_comptime) {
4477	4465	try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
...	...	@@ -4573,11 +4561,7 @@ fn updateContainerTypeWriterError(
4573	4561	.struct_field
4574	4562	else
4575	4563	.struct_field);
4576		if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\| try wip_nav.strp(field_name.toSlice(ip)) else {
4577		var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
4578		const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
4579		try wip_nav.strp(field_name);
4580		}
	4564	try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
4581	4565	try wip_nav.refType(field_type);
4582	4566	if (!is_comptime) {
4583	4567	try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
...	...	@@ -4600,7 +4584,7 @@ fn updateContainerTypeWriterError(
4600	4584	var field_bit_offset: u16 = 0;
4601	4585	for (0..loaded_struct.field_types.len) \|field_index\| {
4602	4586	try wip_nav.abbrevCode(.packed_struct_field);
4603		try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip));
	4587	try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
4604	4588	const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
4605	4589	try wip_nav.refType(field_type);
4606	4590	try diw.writeUleb128(field_bit_offset);

stage1/zig.h+25-34

...	...	@@ -40,6 +40,8 @@
40	40	#elif defined(__mips__)
41	41	#define zig_mips32
42	42	#define zig_mips
	43	#elif defined(__or1k__)
	44	#define zig_or1k
43	45	#elif defined(__powerpc64__)
44	46	#define zig_powerpc64
45	47	#define zig_powerpc
...	...	@@ -72,6 +74,9 @@
72	74	#elif defined (__x86_64__) \|\| (defined(zig_msvc) && defined(_M_X64))
73	75	#define zig_x86_64
74	76	#define zig_x86
	77	#elif defined(__I86__)
	78	#define zig_x86_16
	79	#define zig_x86
75	80	#endif
76	81
77	82	#if defined(zig_msvc) \|\| __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
...	...	@@ -82,9 +87,7 @@
82	87	#define zig_big_endian 1
83	88	#endif
84	89
85		#if defined(_AIX)
86		#define zig_aix
87		#elif defined(__MACH__)
	90	#if defined(__MACH__)
88	91	#define zig_darwin
89	92	#elif defined(__DragonFly__)
90	93	#define zig_dragonfly
...	...	@@ -114,20 +117,14 @@
114	117	#define zig_wasi
115	118	#elif defined(_WIN32)
116	119	#define zig_windows
117		#elif defined(__MVS__)
118		#define zig_zos
119	120	#endif
120	121
121	122	#if defined(zig_windows)
122	123	#define zig_coff
123	124	#elif defined(__ELF__)
124	125	#define zig_elf
125		#elif defined(zig_zos)
126		#define zig_goff
127	126	#elif defined(zig_darwin)
128	127	#define zig_macho
129		#elif defined(zig_aix)
130		#define zig_xcoff
131	128	#endif
132	129
133	130	#define zig_concat(lhs, rhs) lhs##rhs
...	...	@@ -390,12 +387,16 @@
390	387	#define zig_trap() __asm__ volatile(".word 0x0")
391	388	#elif defined(zig_mips)
392	389	#define zig_trap() __asm__ volatile(".word 0x3d")
	390	#elif defined(zig_or1k)
	391	#define zig_trap() __asm__ volatile("l.cust8")
393	392	#elif defined(zig_riscv)
394	393	#define zig_trap() __asm__ volatile("unimp")
395	394	#elif defined(zig_s390x)
396	395	#define zig_trap() __asm__ volatile("j 0x2")
397	396	#elif defined(zig_sparc)
398	397	#define zig_trap() __asm__ volatile("illtrap")
	398	#elif defined(zig_x86_16)
	399	#define zig_trap() __asm__ volatile("int $0x3")
399	400	#elif defined(zig_x86)
400	401	#define zig_trap() __asm__ volatile("ud2")
401	402	#else
...	...	@@ -422,6 +423,8 @@
422	423	#define zig_breakpoint() __asm__ volatile("break 0x0")
423	424	#elif defined(zig_mips)
424	425	#define zig_breakpoint() __asm__ volatile("break")
	426	#elif defined(zig_or1k)
	427	#define zig_breakpoint() __asm__ volatile("l.trap 0x0")
425	428	#elif defined(zig_powerpc)
426	429	#define zig_breakpoint() __asm__ volatile("trap")
427	430	#elif defined(zig_riscv)
...	...	@@ -804,15 +807,13 @@ static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
804	807	#endif
805	808	}
806	809
807		zig_extern int32_t __addosi4(int32_t lhs, int32_t rhs, int *overflow);
808	810	static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
809	811	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
810	812	int32_t full_res;
811	813	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
812	814	#else
813		int overflow_int;
814		int32_t full_res = __addosi4(lhs, rhs, &overflow_int);
815		bool overflow = overflow_int != 0;
	815	int32_t full_res = (int32_t)((uint32_t)lhs + (uint32_t)rhs);
	816	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
816	817	#endif
817	818	*res = zig_wrap_i32(full_res, bits);
818	819	return overflow \|\| full_res < zig_minInt_i(32, bits) \|\| full_res > zig_maxInt_i(32, bits);
...	...	@@ -830,15 +831,13 @@ static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
830	831	#endif
831	832	}
832	833
833		zig_extern int64_t __addodi4(int64_t lhs, int64_t rhs, int *overflow);
834	834	static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
835	835	#if zig_has_builtin(add_overflow) \|\| defined(zig_gcc)
836	836	int64_t full_res;
837	837	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
838	838	#else
839		int overflow_int;
840		int64_t full_res = __addodi4(lhs, rhs, &overflow_int);
841		bool overflow = overflow_int != 0;
	839	int64_t full_res = (int64_t)((uint64_t)lhs + (uint64_t)rhs);
	840	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
842	841	#endif
843	842	*res = zig_wrap_i64(full_res, bits);
844	843	return overflow \|\| full_res < zig_minInt_i(64, bits) \|\| full_res > zig_maxInt_i(64, bits);
...	...	@@ -912,15 +911,13 @@ static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
912	911	#endif
913	912	}
914	913
915		zig_extern int32_t __subosi4(int32_t lhs, int32_t rhs, int *overflow);
916	914	static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
917	915	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
918	916	int32_t full_res;
919	917	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
920	918	#else
921		int overflow_int;
922		int32_t full_res = __subosi4(lhs, rhs, &overflow_int);
923		bool overflow = overflow_int != 0;
	919	int32_t full_res = (int32_t)((uint32_t)lhs - (uint32_t)rhs);
	920	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
924	921	#endif
925	922	*res = zig_wrap_i32(full_res, bits);
926	923	return overflow \|\| full_res < zig_minInt_i(32, bits) \|\| full_res > zig_maxInt_i(32, bits);
...	...	@@ -938,15 +935,13 @@ static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
938	935	#endif
939	936	}
940	937
941		zig_extern int64_t __subodi4(int64_t lhs, int64_t rhs, int *overflow);
942	938	static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
943	939	#if zig_has_builtin(sub_overflow) \|\| defined(zig_gcc)
944	940	int64_t full_res;
945	941	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
946	942	#else
947		int overflow_int;
948		int64_t full_res = __subodi4(lhs, rhs, &overflow_int);
949		bool overflow = overflow_int != 0;
	943	int64_t full_res = (int64_t)((uint64_t)lhs - (uint64_t)rhs);
	944	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
950	945	#endif
951	946	*res = zig_wrap_i64(full_res, bits);
952	947	return overflow \|\| full_res < zig_minInt_i(64, bits) \|\| full_res > zig_maxInt_i(64, bits);
...	...	@@ -1750,15 +1745,13 @@ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
1750	1745	#endif
1751	1746	}
1752	1747
1753		zig_extern zig_i128 __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
1754	1748	static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
1755	1749	#if zig_has_builtin(add_overflow)
1756	1750	zig_i128 full_res;
1757	1751	bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
1758	1752	#else
1759		int overflow_int;
1760		zig_i128 full_res = __addoti4(lhs, rhs, &overflow_int);
1761		bool overflow = overflow_int != 0;
	1753	zig_i128 full_res = (zig_i128)((zig_u128)lhs + (zig_u128)rhs);
	1754	bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
1762	1755	#endif
1763	1756	*res = zig_wrap_i128(full_res, bits);
1764	1757	return overflow \|\| full_res < zig_minInt_i(128, bits) \|\| full_res > zig_maxInt_i(128, bits);
...	...	@@ -1776,15 +1769,13 @@ static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
1776	1769	#endif
1777	1770	}
1778	1771
1779		zig_extern zig_i128 __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
1780	1772	static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
1781	1773	#if zig_has_builtin(sub_overflow)
1782	1774	zig_i128 full_res;
1783	1775	bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
1784	1776	#else
1785		int overflow_int;
1786		zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int);
1787		bool overflow = overflow_int != 0;
	1777	zig_i128 full_res = (zig_i128)((zig_u128)lhs - (zig_u128)rhs);
	1778	bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
1788	1779	#endif
1789	1780	*res = zig_wrap_i128(full_res, bits);
1790	1781	return overflow \|\| full_res < zig_minInt_i(128, bits) \|\| full_res > zig_maxInt_i(128, bits);
...	...	@@ -4213,7 +4204,7 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
4213	4204	#endif
4214	4205	}
4215	4206
4216		#elif defined(zig_x86)
	4207	#elif defined(zig_x86) && !defined(zig_x86_16)
4217	4208
4218	4209	static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
4219	4210	#if defined(zig_msvc)

test/behavior/union.zig+4-1

...	...	@@ -218,10 +218,13 @@ test "union with specified enum tag" {
218	218	}
219	219
220	220	test "packed union generates correctly aligned type" {
	221	// This test will be removed after the following accepted proposal is implemented:
	222	// https://github.com/ziglang/zig/issues/24657
221	223	if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
222	224	if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
223	225	if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
224	226	if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
	227	if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
225	228
226	229	const U = packed union {
227	230	f1: *const fn () error{TestUnexpectedResult}!void,
...	...	@@ -1544,7 +1547,7 @@ test "packed union field pointer has correct alignment" {
1544	1547
1545	1548	const host_size = switch (builtin.zig_backend) {
1546	1549	else => comptime std.math.divCeil(comptime_int, @bitSizeOf(S), 8) catch unreachable,
1547		.stage2_x86_64 => @sizeOf(S),
	1550	.stage2_x86_64, .stage2_c => @sizeOf(S),
1548	1551	};
1549	1552	comptime assert(@TypeOf(ap) == *align(4:2:host_size) u20);
1550	1553	comptime assert(@TypeOf(bp) == *align(1:2:host_size) u20);