Commit 181b25ce4fcebc32f6fdc7498148c0f5e131dda9 | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2025-11-12 23:14:02+00:00
committer	GitHub <noreply@github.com> 2025-11-12 23:14:02+00:00
log	181b25ce4fcebc32f6fdc7498148c0f5e131dda9
tree	f5cfe981b1b158e8bac17cc3a3bc909ffb7b1aa1
parent	dfd7b7f2337d84ce660253a37079489f7780b055
parent	532aa3c5758f110eb7cf0992eb394088ab563899
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #25772 from mlugg/kill-dead-code

compiler: rewrite some legalizations, and remove a bunch of dead code

39 files changed, 1451 insertions(+), 3853 deletions(-)

CMakeLists.txt
lib
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - print.zig
  - types_resolved.zig
- InternPool.zig
- Sema.zig
- Sema
  - comptime_ptr_access.zig
- Type.zig
- Value.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
  - c.zig
  - c
    Type.zig
  - llvm.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
- link
  - Dwarf.zig
stage1
- zig.h
test
- behavior
  - union.zig

CMakeLists.txt+1-2

 @@ -211,10 +211,10 @@ set(ZIG_STAGE2_SOURCES
     lib/compiler_rt/absvti2.zig
     lib/compiler_rt/adddf3.zig
     lib/compiler_rt/addf3.zig
-    lib/compiler_rt/addo.zig
     lib/compiler_rt/addsf3.zig
     lib/compiler_rt/addtf3.zig
     lib/compiler_rt/addvsi3.zig
+    lib/compiler_rt/addvdi3.zig
     lib/compiler_rt/addxf3.zig
     lib/compiler_rt/arm.zig
     lib/compiler_rt/atomics.zig
 @@ -354,7 +354,6 @@ set(ZIG_STAGE2_SOURCES
     lib/compiler_rt/sqrt.zig
     lib/compiler_rt/stack_probe.zig
     lib/compiler_rt/subdf3.zig
-    lib/compiler_rt/subo.zig
     lib/compiler_rt/subsf3.zig
     lib/compiler_rt/subtf3.zig
     lib/compiler_rt/subvdi3.zig

lib/compiler_rt.zig+3-2

 @@ -28,12 +28,13 @@ comptime {
     _ = @import("compiler_rt/negv.zig");
     _ = @import("compiler_rt/addvsi3.zig");
+    _ = @import("compiler_rt/addvdi3.zig");
     _ = @import("compiler_rt/subvsi3.zig");
     _ = @import("compiler_rt/subvdi3.zig");
     _ = @import("compiler_rt/mulvsi3.zig");
-    _ = @import("compiler_rt/addo.zig");
-    _ = @import("compiler_rt/subo.zig");
     _ = @import("compiler_rt/mulo.zig");
     // Float routines

lib/compiler_rt/addo.zig deleted-46

...	@@ -1,46 +0,0 @@
1	const std = @import("std");
2	const common = @import("./common.zig");
3	pub const panic = @import("common.zig").panic;
4
5	comptime {
6	@export(&__addosi4, .{ .name = "__addosi4", .linkage = common.linkage, .visibility = common.visibility });
7	@export(&__addodi4, .{ .name = "__addodi4", .linkage = common.linkage, .visibility = common.visibility });
8	@export(&__addoti4, .{ .name = "__addoti4", .linkage = common.linkage, .visibility = common.visibility });
9	}
10
11	// addo - add overflow
12	// * return a+%b.
13	// * return if a+b overflows => 1 else => 0
14	// - addoXi4_generic as default
15
16	inline fn addoXi4_generic(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST {
17	@setRuntimeSafety(common.test_safety);
18	overflow.* = 0;
19	const sum: ST = a +% b;
20	// Hackers Delight: section Overflow Detection, subsection Signed Add/Subtract
21	// Let sum = a +% b == a + b + carry == wraparound addition.
22	// Overflow in a+b+carry occurs, iff a and b have opposite signs
23	// and the sign of a+b+carry is the same as a (or equivalently b).
24	// Slower routine: res = ~(a ^ b) & ((sum ^ a)
25	// Faster routine: res = (sum ^ a) & (sum ^ b)
26	// Overflow occurred, iff (res < 0)
27	if (((sum ^ a) & (sum ^ b)) < 0)
28	overflow.* = 1;
29	return sum;
30	}
31
32	pub fn __addosi4(a: i32, b: i32, overflow: *c_int) callconv(.c) i32 {
33	return addoXi4_generic(i32, a, b, overflow);
34	}
35	pub fn __addodi4(a: i64, b: i64, overflow: *c_int) callconv(.c) i64 {
36	return addoXi4_generic(i64, a, b, overflow);
37	}
38	pub fn __addoti4(a: i128, b: i128, overflow: *c_int) callconv(.c) i128 {
39	return addoXi4_generic(i128, a, b, overflow);
40	}
41
42	test {
43	_ = @import("addosi4_test.zig");
44	_ = @import("addodi4_test.zig");
45	_ = @import("addoti4_test.zig");
46	}

lib/compiler_rt/addodi4_test.zig deleted-77

...	@@ -1,77 +0,0 @@
1	const addv = @import("addo.zig");
2	const std = @import("std");
3	const testing = std.testing;
4	const math = std.math;
5
6	fn test__addodi4(a: i64, b: i64) !void {
7	var result_ov: c_int = undefined;
8	var expected_ov: c_int = undefined;
9	const result = addv.__addodi4(a, b, &result_ov);
10	const expected: i64 = simple_addodi4(a, b, &expected_ov);
11	try testing.expectEqual(expected, result);
12	try testing.expectEqual(expected_ov, result_ov);
13	}
14
15	fn simple_addodi4(a: i64, b: i64, overflow: *c_int) i64 {
16	overflow.* = 0;
17	const min: i64 = math.minInt(i64);
18	const max: i64 = math.maxInt(i64);
19	if (((a > 0) and (b > max - a)) or
20	((a < 0) and (b < min - a)))
21	overflow.* = 1;
22	return a +% b;
23	}
24
25	test "addodi4" {
26	const min: i64 = math.minInt(i64);
27	const max: i64 = math.maxInt(i64);
28	var i: i64 = 1;
29	while (i < max) : (i *\|= 2) {
30	try test__addodi4(i, i);
31	try test__addodi4(-i, -i);
32	try test__addodi4(i, -i);
33	try test__addodi4(-i, i);
34	}
35
36	// edge cases
37	// 0 + 0 = 0
38	// MIN + MIN overflow
39	// MAX + MAX overflow
40	// 0 + MIN MIN
41	// 0 + MAX MAX
42	// MIN + 0 MIN
43	// MAX + 0 MAX
44	// MIN + MAX -1
45	// MAX + MIN -1
46	try test__addodi4(0, 0);
47	try test__addodi4(min, min);
48	try test__addodi4(max, max);
49	try test__addodi4(0, min);
50	try test__addodi4(0, max);
51	try test__addodi4(min, 0);
52	try test__addodi4(max, 0);
53	try test__addodi4(min, max);
54	try test__addodi4(max, min);
55
56	// derived edge cases
57	// MIN+1 + MIN overflow
58	// MAX-1 + MAX overflow
59	// 1 + MIN = MIN+1
60	// -1 + MIN overflow
61	// -1 + MAX = MAX-1
62	// +1 + MAX overflow
63	// MIN + 1 = MIN+1
64	// MIN + -1 overflow
65	// MAX + 1 overflow
66	// MAX + -1 = MAX-1
67	try test__addodi4(min + 1, min);
68	try test__addodi4(max - 1, max);
69	try test__addodi4(1, min);
70	try test__addodi4(-1, min);
71	try test__addodi4(-1, max);
72	try test__addodi4(1, max);
73	try test__addodi4(min, 1);
74	try test__addodi4(min, -1);
75	try test__addodi4(max, -1);
76	try test__addodi4(max, 1);
77	}

lib/compiler_rt/addosi4_test.zig deleted-78

...	@@ -1,78 +0,0 @@
1	const addv = @import("addo.zig");
2	const testing = @import("std").testing;
3
4	fn test__addosi4(a: i32, b: i32) !void {
5	var result_ov: c_int = undefined;
6	var expected_ov: c_int = undefined;
7	const result = addv.__addosi4(a, b, &result_ov);
8	const expected: i32 = simple_addosi4(a, b, &expected_ov);
9	try testing.expectEqual(expected, result);
10	try testing.expectEqual(expected_ov, result_ov);
11	}
12
13	fn simple_addosi4(a: i32, b: i32, overflow: *c_int) i32 {
14	overflow.* = 0;
15	const min: i32 = -2147483648;
16	const max: i32 = 2147483647;
17	if (((a > 0) and (b > max - a)) or
18	((a < 0) and (b < min - a)))
19	overflow.* = 1;
20	return a +% b;
21	}
22
23	test "addosi4" {
24	// -2^31 <= i32 <= 2^31-1
25	// 2^31 = 2147483648
26	// 2^31-1 = 2147483647
27	const min: i32 = -2147483648;
28	const max: i32 = 2147483647;
29	var i: i32 = 1;
30	while (i < max) : (i *\|= 2) {
31	try test__addosi4(i, i);
32	try test__addosi4(-i, -i);
33	try test__addosi4(i, -i);
34	try test__addosi4(-i, i);
35	}
36
37	// edge cases
38	// 0 + 0 = 0
39	// MIN + MIN overflow
40	// MAX + MAX overflow
41	// 0 + MIN MIN
42	// 0 + MAX MAX
43	// MIN + 0 MIN
44	// MAX + 0 MAX
45	// MIN + MAX -1
46	// MAX + MIN -1
47	try test__addosi4(0, 0);
48	try test__addosi4(min, min);
49	try test__addosi4(max, max);
50	try test__addosi4(0, min);
51	try test__addosi4(0, max);
52	try test__addosi4(min, 0);
53	try test__addosi4(max, 0);
54	try test__addosi4(min, max);
55	try test__addosi4(max, min);
56
57	// derived edge cases
58	// MIN+1 + MIN overflow
59	// MAX-1 + MAX overflow
60	// 1 + MIN = MIN+1
61	// -1 + MIN overflow
62	// -1 + MAX = MAX-1
63	// +1 + MAX overflow
64	// MIN + 1 = MIN+1
65	// MIN + -1 overflow
66	// MAX + 1 overflow
67	// MAX + -1 = MAX-1
68	try test__addosi4(min + 1, min);
69	try test__addosi4(max - 1, max);
70	try test__addosi4(1, min);
71	try test__addosi4(-1, min);
72	try test__addosi4(-1, max);
73	try test__addosi4(1, max);
74	try test__addosi4(min, 1);
75	try test__addosi4(min, -1);
76	try test__addosi4(max, -1);
77	try test__addosi4(max, 1);
78	}

lib/compiler_rt/addoti4_test.zig deleted-80

...	@@ -1,80 +0,0 @@
1	const addv = @import("addo.zig");
2	const builtin = @import("builtin");
3	const std = @import("std");
4	const testing = std.testing;
5	const math = std.math;
6
7	fn test__addoti4(a: i128, b: i128) !void {
8	var result_ov: c_int = undefined;
9	var expected_ov: c_int = undefined;
10	const result = addv.__addoti4(a, b, &result_ov);
11	const expected: i128 = simple_addoti4(a, b, &expected_ov);
12	try testing.expectEqual(expected, result);
13	try testing.expectEqual(expected_ov, result_ov);
14	}
15
16	fn simple_addoti4(a: i128, b: i128, overflow: *c_int) i128 {
17	overflow.* = 0;
18	const min: i128 = math.minInt(i128);
19	const max: i128 = math.maxInt(i128);
20	if (((a > 0) and (b > max - a)) or
21	((a < 0) and (b < min - a)))
22	overflow.* = 1;
23	return a +% b;
24	}
25
26	test "addoti4" {
27	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
28
29	const min: i128 = math.minInt(i128);
30	const max: i128 = math.maxInt(i128);
31	var i: i128 = 1;
32	while (i < max) : (i *\|= 2) {
33	try test__addoti4(i, i);
34	try test__addoti4(-i, -i);
35	try test__addoti4(i, -i);
36	try test__addoti4(-i, i);
37	}
38
39	// edge cases
40	// 0 + 0 = 0
41	// MIN + MIN overflow
42	// MAX + MAX overflow
43	// 0 + MIN MIN
44	// 0 + MAX MAX
45	// MIN + 0 MIN
46	// MAX + 0 MAX
47	// MIN + MAX -1
48	// MAX + MIN -1
49	try test__addoti4(0, 0);
50	try test__addoti4(min, min);
51	try test__addoti4(max, max);
52	try test__addoti4(0, min);
53	try test__addoti4(0, max);
54	try test__addoti4(min, 0);
55	try test__addoti4(max, 0);
56	try test__addoti4(min, max);
57	try test__addoti4(max, min);
58
59	// derived edge cases
60	// MIN+1 + MIN overflow
61	// MAX-1 + MAX overflow
62	// 1 + MIN = MIN+1
63	// -1 + MIN overflow
64	// -1 + MAX = MAX-1
65	// +1 + MAX overflow
66	// MIN + 1 = MIN+1
67	// MIN + -1 overflow
68	// MAX + 1 overflow
69	// MAX + -1 = MAX-1
70	try test__addoti4(min + 1, min);
71	try test__addoti4(max - 1, max);
72	try test__addoti4(1, min);
73	try test__addoti4(-1, min);
74	try test__addoti4(-1, max);
75	try test__addoti4(1, max);
76	try test__addoti4(min, 1);
77	try test__addoti4(min, -1);
78	try test__addoti4(max, -1);
79	try test__addoti4(max, 1);
80	}

lib/compiler_rt/addvdi3.zig created+26

...	@@ -0,0 +1,26 @@
1	const common = @import("./common.zig");
2	const testing = @import("std").testing;
3
4	pub const panic = common.panic;
5
6	comptime {
7	@export(&__addvdi3, .{ .name = "__addvdi3", .linkage = common.linkage, .visibility = common.visibility });
8	}
9
10	pub fn __addvdi3(a: i64, b: i64) callconv(.c) i64 {
11	const sum = a +% b;
12	// Overflow occurred iff both operands have the same sign, and the sign of the sum does
13	// not match it. In other words, iff the sum sign is not the sign of either operand.
14	if (((sum ^ a) & (sum ^ b)) < 0) @panic("compiler-rt: integer overflow");
15	return sum;
16	}
17
18	test "addvdi3" {
19	// const min: i64 = -9223372036854775808
20	// const max: i64 = 9223372036854775807
21	// TODO write panic handler for testing panics
22	// try test__addvdi3(-9223372036854775808, -1, -1); // panic
23	// try test__addvdi3(9223372036854775807, 1, 1); // panic
24	try testing.expectEqual(-9223372036854775808, __addvdi3(-9223372036854775807, -1));
25	try testing.expectEqual(9223372036854775807, __addvdi3(9223372036854775806, 1));
26	}

lib/compiler_rt/addvsi3.zig+4-4

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	const addv = @import("addo.zig");
2	const common = @import("./common.zig");	1	const common = @import("./common.zig");
3	const testing = @import("std").testing;	2	const testing = @import("std").testing;
4		3
...	@@ -9,9 +8,10 @@ comptime {	...	@@ -9,9 +8,10 @@ comptime {
9	}	8	}
10		9
11	pub fn __addvsi3(a: i32, b: i32) callconv(.c) i32 {	10	pub fn __addvsi3(a: i32, b: i32) callconv(.c) i32 {
12	var overflow: c_int = 0;	11	const sum = a +% b;
13	const sum = addv.__addosi4(a, b, &overflow);	12	// Overflow occurred iff both operands have the same sign, and the sign of the sum does
14	if (overflow != 0) @panic("compiler-rt: integer overflow");	13	// not match it. In other words, iff the sum sign is not the sign of either operand.
		14	if (((sum ^ a) & (sum ^ b)) < 0) @panic("compiler-rt: integer overflow");
15	return sum;	15	return sum;
16	}	16	}
17		17

lib/compiler_rt/subo.zig deleted-47

...	@@ -1,47 +0,0 @@
1	//! subo - subtract overflow
2	//! * return a-%b.
3	//! * return if a-b overflows => 1 else => 0
4	//! - suboXi4_generic as default
5
6	const std = @import("std");
7	const builtin = @import("builtin");
8	const common = @import("common.zig");
9
10	pub const panic = common.panic;
11
12	comptime {
13	@export(&__subosi4, .{ .name = "__subosi4", .linkage = common.linkage, .visibility = common.visibility });
14	@export(&__subodi4, .{ .name = "__subodi4", .linkage = common.linkage, .visibility = common.visibility });
15	@export(&__suboti4, .{ .name = "__suboti4", .linkage = common.linkage, .visibility = common.visibility });
16	}
17
18	pub fn __subosi4(a: i32, b: i32, overflow: *c_int) callconv(.c) i32 {
19	return suboXi4_generic(i32, a, b, overflow);
20	}
21	pub fn __subodi4(a: i64, b: i64, overflow: *c_int) callconv(.c) i64 {
22	return suboXi4_generic(i64, a, b, overflow);
23	}
24	pub fn __suboti4(a: i128, b: i128, overflow: *c_int) callconv(.c) i128 {
25	return suboXi4_generic(i128, a, b, overflow);
26	}
27
28	inline fn suboXi4_generic(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST {
29	overflow.* = 0;
30	const sum: ST = a -% b;
31	// Hackers Delight: section Overflow Detection, subsection Signed Add/Subtract
32	// Let sum = a -% b == a - b - carry == wraparound subtraction.
33	// Overflow in a-b-carry occurs, iff a and b have opposite signs
34	// and the sign of a-b-carry is opposite of a (or equivalently same as b).
35	// Faster routine: res = (a ^ b) & (sum ^ a)
36	// Slower routine: res = (sum^a) & ~(sum^b)
37	// Overflow occurred, iff (res < 0)
38	if (((a ^ b) & (sum ^ a)) < 0)
39	overflow.* = 1;
40	return sum;
41	}
42
43	test {
44	_ = @import("subosi4_test.zig");
45	_ = @import("subodi4_test.zig");
46	_ = @import("suboti4_test.zig");
47	}

lib/compiler_rt/subodi4_test.zig deleted-81

...	@@ -1,81 +0,0 @@
1	const subo = @import("subo.zig");
2	const std = @import("std");
3	const testing = std.testing;
4	const math = std.math;
5
6	fn test__subodi4(a: i64, b: i64) !void {
7	var result_ov: c_int = undefined;
8	var expected_ov: c_int = undefined;
9	const result = subo.__subodi4(a, b, &result_ov);
10	const expected: i64 = simple_subodi4(a, b, &expected_ov);
11	try testing.expectEqual(expected, result);
12	try testing.expectEqual(expected_ov, result_ov);
13	}
14
15	// 2 cases on evaluating `a-b`:
16	// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
17	// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
18	// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
19	pub fn simple_subodi4(a: i64, b: i64, overflow: *c_int) i64 {
20	overflow.* = 0;
21	const min: i64 = math.minInt(i64);
22	const max: i64 = math.maxInt(i64);
23	if (((b > 0) and (a < min + b)) or
24	((b < 0) and (a > max + b)))
25	overflow.* = 1;
26	return a -% b;
27	}
28
29	test "subodi3" {
30	const min: i64 = math.minInt(i64);
31	const max: i64 = math.maxInt(i64);
32	var i: i64 = 1;
33	while (i < max) : (i *\|= 2) {
34	try test__subodi4(i, i);
35	try test__subodi4(-i, -i);
36	try test__subodi4(i, -i);
37	try test__subodi4(-i, i);
38	}
39
40	// edge cases
41	// 0 - 0 = 0
42	// MIN - MIN = 0
43	// MAX - MAX = 0
44	// 0 - MIN overflow
45	// 0 - MAX = MIN+1
46	// MIN - 0 = MIN
47	// MAX - 0 = MAX
48	// MIN - MAX overflow
49	// MAX - MIN overflow
50	try test__subodi4(0, 0);
51	try test__subodi4(min, min);
52	try test__subodi4(max, max);
53	try test__subodi4(0, min);
54	try test__subodi4(0, max);
55	try test__subodi4(min, 0);
56	try test__subodi4(max, 0);
57	try test__subodi4(min, max);
58	try test__subodi4(max, min);
59
60	// derived edge cases
61	// MIN+1 - MIN = 1
62	// MAX-1 - MAX = -1
63	// 1 - MIN overflow
64	// -1 - MIN = MAX
65	// -1 - MAX = MIN
66	// +1 - MAX = MIN+2
67	// MIN - 1 overflow
68	// MIN - -1 = MIN+1
69	// MAX - 1 = MAX-1
70	// MAX - -1 overflow
71	try test__subodi4(min + 1, min);
72	try test__subodi4(max - 1, max);
73	try test__subodi4(1, min);
74	try test__subodi4(-1, min);
75	try test__subodi4(-1, max);
76	try test__subodi4(1, max);
77	try test__subodi4(min, 1);
78	try test__subodi4(min, -1);
79	try test__subodi4(max, -1);
80	try test__subodi4(max, 1);
81	}

lib/compiler_rt/subosi4_test.zig deleted-82

...	@@ -1,82 +0,0 @@
1	const subo = @import("subo.zig");
2	const testing = @import("std").testing;
3
4	fn test__subosi4(a: i32, b: i32) !void {
5	var result_ov: c_int = undefined;
6	var expected_ov: c_int = undefined;
7	const result = subo.__subosi4(a, b, &result_ov);
8	const expected: i32 = simple_subosi4(a, b, &expected_ov);
9	try testing.expectEqual(expected, result);
10	try testing.expectEqual(expected_ov, result_ov);
11	}
12
13	// 2 cases on evaluating `a-b`:
14	// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
15	// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
16	// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
17	pub fn simple_subosi4(a: i32, b: i32, overflow: *c_int) i32 {
18	overflow.* = 0;
19	const min: i32 = -2147483648;
20	const max: i32 = 2147483647;
21	if (((b > 0) and (a < min + b)) or
22	((b < 0) and (a > max + b)))
23	overflow.* = 1;
24	return a -% b;
25	}
26
27	test "subosi3" {
28	// -2^31 <= i32 <= 2^31-1
29	// 2^31 = 2147483648
30	// 2^31-1 = 2147483647
31	const min: i32 = -2147483648;
32	const max: i32 = 2147483647;
33	var i: i32 = 1;
34	while (i < max) : (i *\|= 2) {
35	try test__subosi4(i, i);
36	try test__subosi4(-i, -i);
37	try test__subosi4(i, -i);
38	try test__subosi4(-i, i);
39	}
40
41	// edge cases
42	// 0 - 0 = 0
43	// MIN - MIN = 0
44	// MAX - MAX = 0
45	// 0 - MIN overflow
46	// 0 - MAX = MIN+1
47	// MIN - 0 = MIN
48	// MAX - 0 = MAX
49	// MIN - MAX overflow
50	// MAX - MIN overflow
51	try test__subosi4(0, 0);
52	try test__subosi4(min, min);
53	try test__subosi4(max, max);
54	try test__subosi4(0, min);
55	try test__subosi4(0, max);
56	try test__subosi4(min, 0);
57	try test__subosi4(max, 0);
58	try test__subosi4(min, max);
59	try test__subosi4(max, min);
60
61	// derived edge cases
62	// MIN+1 - MIN = 1
63	// MAX-1 - MAX = -1
64	// 1 - MIN overflow
65	// -1 - MIN = MAX
66	// -1 - MAX = MIN
67	// +1 - MAX = MIN+2
68	// MIN - 1 overflow
69	// MIN - -1 = MIN+1
70	// MAX - 1 = MAX-1
71	// MAX - -1 overflow
72	try test__subosi4(min + 1, min);
73	try test__subosi4(max - 1, max);
74	try test__subosi4(1, min);
75	try test__subosi4(-1, min);
76	try test__subosi4(-1, max);
77	try test__subosi4(1, max);
78	try test__subosi4(min, 1);
79	try test__subosi4(min, -1);
80	try test__subosi4(max, -1);
81	try test__subosi4(max, 1);
82	}

lib/compiler_rt/suboti4_test.zig deleted-84

...	@@ -1,84 +0,0 @@
1	const subo = @import("subo.zig");
2	const builtin = @import("builtin");
3	const std = @import("std");
4	const testing = std.testing;
5	const math = std.math;
6
7	fn test__suboti4(a: i128, b: i128) !void {
8	var result_ov: c_int = undefined;
9	var expected_ov: c_int = undefined;
10	const result = subo.__suboti4(a, b, &result_ov);
11	const expected: i128 = simple_suboti4(a, b, &expected_ov);
12	try testing.expectEqual(expected, result);
13	try testing.expectEqual(expected_ov, result_ov);
14	}
15
16	// 2 cases on evaluating `a-b`:
17	// 1. `a-b` may underflow, iff b>0 && a<0 and a-b < min <=> a<min+b
18	// 2. `a-b` may overflow, iff b<0 && a>0 and a-b > max <=> a>max+b
19	// `-b` evaluation may overflow, iff b==min, but this is handled by the hardware
20	pub fn simple_suboti4(a: i128, b: i128, overflow: *c_int) i128 {
21	overflow.* = 0;
22	const min: i128 = math.minInt(i128);
23	const max: i128 = math.maxInt(i128);
24	if (((b > 0) and (a < min + b)) or
25	((b < 0) and (a > max + b)))
26	overflow.* = 1;
27	return a -% b;
28	}
29
30	test "suboti3" {
31	if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
32
33	const min: i128 = math.minInt(i128);
34	const max: i128 = math.maxInt(i128);
35	var i: i128 = 1;
36	while (i < max) : (i *\|= 2) {
37	try test__suboti4(i, i);
38	try test__suboti4(-i, -i);
39	try test__suboti4(i, -i);
40	try test__suboti4(-i, i);
41	}
42
43	// edge cases
44	// 0 - 0 = 0
45	// MIN - MIN = 0
46	// MAX - MAX = 0
47	// 0 - MIN overflow
48	// 0 - MAX = MIN+1
49	// MIN - 0 = MIN
50	// MAX - 0 = MAX
51	// MIN - MAX overflow
52	// MAX - MIN overflow
53	try test__suboti4(0, 0);
54	try test__suboti4(min, min);
55	try test__suboti4(max, max);
56	try test__suboti4(0, min);
57	try test__suboti4(0, max);
58	try test__suboti4(min, 0);
59	try test__suboti4(max, 0);
60	try test__suboti4(min, max);
61	try test__suboti4(max, min);
62
63	// derived edge cases
64	// MIN+1 - MIN = 1
65	// MAX-1 - MAX = -1
66	// 1 - MIN overflow
67	// -1 - MIN = MAX
68	// -1 - MAX = MIN
69	// +1 - MAX = MIN+2
70	// MIN - 1 overflow
71	// MIN - -1 = MIN+1
72	// MAX - 1 = MAX-1
73	// MAX - -1 overflow
74	try test__suboti4(min + 1, min);
75	try test__suboti4(max - 1, max);
76	try test__suboti4(1, min);
77	try test__suboti4(-1, min);
78	try test__suboti4(-1, max);
79	try test__suboti4(1, max);
80	try test__suboti4(min, 1);
81	try test__suboti4(min, -1);
82	try test__suboti4(max, -1);
83	try test__suboti4(max, 1);
84	}

lib/compiler_rt/subvdi3.zig+4-4

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	const subv = @import("subo.zig");
2	const common = @import("./common.zig");	1	const common = @import("./common.zig");
3	const testing = @import("std").testing;	2	const testing = @import("std").testing;
4		3
...	@@ -9,9 +8,10 @@ comptime {	...	@@ -9,9 +8,10 @@ comptime {
9	}	8	}
10		9
11	pub fn __subvdi3(a: i64, b: i64) callconv(.c) i64 {	10	pub fn __subvdi3(a: i64, b: i64) callconv(.c) i64 {
12	var overflow: c_int = 0;	11	const sum = a -% b;
13	const sum = subv.__subodi4(a, b, &overflow);	12	// Overflow occurred iff the operands have opposite signs, and the sign of the
14	if (overflow != 0) @panic("compiler-rt: integer overflow");	13	// sum is the opposite of the lhs sign.
		14	if (((a ^ b) & (sum ^ a)) < 0) @panic("compiler-rt: integer overflow");
15	return sum;	15	return sum;
16	}	16	}
17		17

lib/compiler_rt/subvsi3.zig+4-4

...	@@ -1,4 +1,3 @@	...	@@ -1,4 +1,3 @@
1	const subv = @import("subo.zig");
2	const common = @import("./common.zig");	1	const common = @import("./common.zig");
3	const testing = @import("std").testing;	2	const testing = @import("std").testing;
4		3
...	@@ -9,9 +8,10 @@ comptime {	...	@@ -9,9 +8,10 @@ comptime {
9	}	8	}
10		9
11	pub fn __subvsi3(a: i32, b: i32) callconv(.c) i32 {	10	pub fn __subvsi3(a: i32, b: i32) callconv(.c) i32 {
12	var overflow: c_int = 0;	11	const sum = a -% b;
13	const sum = subv.__subosi4(a, b, &overflow);	12	// Overflow occurred iff the operands have opposite signs, and the sign of the
14	if (overflow != 0) @panic("compiler-rt: integer overflow");	13	// sum is the opposite of the lhs sign.
		14	if (((a ^ b) & (sum ^ a)) < 0) @panic("compiler-rt: integer overflow");
15	return sum;	15	return sum;
16	}	16	}
17		17

lib/zig.h+12-24

 @@ -809,15 +809,13 @@ static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
 #endif
 }
-zig_extern int32_t  __addosi4(int32_t lhs, int32_t rhs, int *overflow);
 static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow) || defined(zig_gcc)
     int32_t full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int32_t full_res = (int32_t)((uint32_t)lhs + (uint32_t)rhs);
-    int32_t full_res = __addosi4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i32(full_res, bits);
     return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits);
 @@ -835,15 +833,13 @@ static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
 #endif
 }
-zig_extern int64_t  __addodi4(int64_t lhs, int64_t rhs, int *overflow);
 static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow) || defined(zig_gcc)
     int64_t full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int64_t full_res = (int64_t)((uint64_t)lhs + (uint64_t)rhs);
-    int64_t full_res = __addodi4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i64(full_res, bits);
     return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits);
 @@ -917,15 +913,13 @@ static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
 #endif
 }
-zig_extern int32_t  __subosi4(int32_t lhs, int32_t rhs, int *overflow);
 static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow) || defined(zig_gcc)
     int32_t full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int32_t full_res = (int32_t)((uint32_t)lhs - (uint32_t)rhs);
-    int32_t full_res = __subosi4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i32(full_res, bits);
     return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits);
 @@ -943,15 +937,13 @@ static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
 #endif
 }
-zig_extern int64_t  __subodi4(int64_t lhs, int64_t rhs, int *overflow);
 static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow) || defined(zig_gcc)
     int64_t full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int64_t full_res = (int64_t)((uint64_t)lhs - (uint64_t)rhs);
-    int64_t full_res = __subodi4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i64(full_res, bits);
     return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits);
 @@ -1755,15 +1747,13 @@ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
 #endif
 }
-zig_extern zig_i128  __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
 static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow)
     zig_i128 full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    zig_i128 full_res = (zig_i128)((zig_u128)lhs + (zig_u128)rhs);
-    zig_i128 full_res =  __addoti4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i128(full_res, bits);
     return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits);
 @@ -1781,15 +1771,13 @@ static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
 #endif
 }
-zig_extern zig_i128  __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
 static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow)
     zig_i128 full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    zig_i128 full_res = (zig_i128)((zig_u128)lhs - (zig_u128)rhs);
-    zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i128(full_res, bits);
     return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits);

src/Air.zig+26-14

 @@ -660,8 +660,8 @@ pub const Inst = struct {
         /// Given a pointer to a slice, return a pointer to the pointer of the slice.
         /// Uses the `ty_op` field.
         ptr_slice_ptr_ptr,
-        /// Given an (array value or vector value) and element index,
+        /// Given an (array value or vector value) and element index, return the element value at
-        /// return the element value at that index.
+        /// that index. If the lhs is a vector value, the index is guaranteed to be comptime-known.
         /// Result type is the element type of the array operand.
         /// Uses the `bin_op` field.
         array_elem_val,
 @@ -874,10 +874,6 @@ pub const Inst = struct {
         /// Uses the `ty_pl` field.
         save_err_return_trace_index,
-        /// Store an element to a vector pointer at an index.
-        /// Uses the `vector_store_elem` field.
-        vector_store_elem,
         /// Compute a pointer to a `Nav` at runtime, always one of:
         ///
         /// * `threadlocal var`
 @@ -919,6 +915,26 @@ pub const Inst = struct {
         /// Operand is unused and set to Ref.none
         work_group_id,
+        // The remaining instructions are not emitted by Sema. They are only emitted by `Legalize`,
+        // depending on the enabled features. As such, backends can consider them `unreachable` if
+        // they do not enable the relevant legalizations.
+        /// Given a pointer to a vector, a runtime-known index, and a scalar value, store the value
+        /// into the vector at the given index. Zig does not support this operation, but `Legalize`
+        /// may emit it when scalarizing vector operations.
+        ///
+        /// Uses the `pl_op` field with payload `Bin`. `operand` is the vector pointer. `lhs` is the
+        /// element index of type `usize`. `rhs` is the element value. Result is always void.
+        legalize_vec_store_elem,
+        /// Given a vector value and a runtime-known index, return the element value at that index.
+        /// This instruction is similar to `array_elem_val`; the only difference is that the index
+        /// here is runtime-known, which is usually not allowed for vectors. `Legalize` may emit
+        /// this instruction when scalarizing vector operations.
+        ///
+        /// Uses the `bin_op` field. `lhs` is the vector pointer. `rhs` is the element index. Result
+        /// type is the vector element type.
+        legalize_vec_elem_val,
         pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
             switch (op) {
                 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
 @@ -1220,11 +1236,6 @@ pub const Inst = struct {
             operand: Ref,
             operation: std.builtin.ReduceOp,
         },
-        vector_store_elem: struct {
-            vector_ptr: Ref,
-            // Index into a different array.
-            payload: u32,
-        },
         ty_nav: struct {
             ty: InternPool.Index,
             nav: InternPool.Nav.Index,
 @@ -1689,8 +1700,8 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
         .set_union_tag,
         .prefetch,
         .set_err_return_trace,
-        .vector_store_elem,
         .c_va_end,
+        .legalize_vec_store_elem,
         => return .void,
         .slice_len,
 @@ -1709,7 +1720,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
             return .fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));
         },
-        .slice_elem_val, .ptr_elem_val, .array_elem_val => {
+        .slice_elem_val, .ptr_elem_val, .array_elem_val, .legalize_vec_elem_val => {
             const ptr_ty = air.typeOf(datas[@intFromEnum(inst)].bin_op.lhs, ip);
             return ptr_ty.childTypeIp(ip);
         },
 @@ -1857,7 +1868,6 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .prefetch,
         .wasm_memory_grow,
         .set_err_return_trace,
-        .vector_store_elem,
         .c_va_arg,
         .c_va_copy,
         .c_va_end,
 @@ -1868,6 +1878,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .intcast_safe,
         .int_from_float_safe,
         .int_from_float_optimized_safe,
+        .legalize_vec_store_elem,
         => true,
         .add,
 @@ -2013,6 +2024,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
         .work_item_id,
         .work_group_size,
         .work_group_id,
+        .legalize_vec_elem_val,
         => false,
         .is_non_null_ptr, .is_null_ptr, .is_non_err_ptr, .is_err_ptr => air.typeOf(data.un_op, ip).isVolatilePtrIp(ip),

src/Air/Legalize.zig+989-1362

...	@@ -14,7 +14,7 @@ features: if (switch (dev.env) {	...	@@ -14,7 +14,7 @@ features: if (switch (dev.env) {
14	return comptime bootstrap_features.contains(feature);	14	return comptime bootstrap_features.contains(feature);
15	}	15	}
16	/// `inline` to propagate comptime-known result.	16	/// `inline` to propagate comptime-known result.
17	fn hasAny(_: @This(), comptime features: []const Feature) bool {	17	inline fn hasAny(_: @This(), comptime features: []const Feature) bool {
18	return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());	18	return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.initEmpty());
19	}	19	}
20	} else struct {	20	} else struct {
...	@@ -154,9 +154,9 @@ pub const Feature = enum {	...	@@ -154,9 +154,9 @@ pub const Feature = enum {
154	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the	154	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
155	/// first byte of memory until bit pointers know their backing type.	155	/// first byte of memory until bit pointers know their backing type.
156	expand_packed_store,	156	expand_packed_store,
157	/// Replace `struct_field_val` of a packed field with a `store` and packed `load`.	157	/// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
158	expand_packed_struct_field_val,	158	expand_packed_struct_field_val,
159	/// Replace `aggregate_init` of a packed aggregate with a series a packed `store`s followed by a `load`.	159	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
160	expand_packed_aggregate_init,	160	expand_packed_aggregate_init,
161		161
162	fn scalarize(tag: Air.Inst.Tag) Feature {	162	fn scalarize(tag: Air.Inst.Tag) Feature {
...	@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -320,28 +320,36 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
320	.xor,	320	.xor,
321	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {	321	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
322	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;	322	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
323	if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);	323	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
		324	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
		325	}
324	},	326	},
325	.add_safe => if (l.features.has(.expand_add_safe)) {	327	.add_safe => if (l.features.has(.expand_add_safe)) {
326	assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both	328	assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
327	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));	329	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
328	} else if (l.features.has(.scalarize_add_safe)) {	330	} else if (l.features.has(.scalarize_add_safe)) {
329	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;	331	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
330	if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);	332	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
		333	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
		334	}
331	},	335	},
332	.sub_safe => if (l.features.has(.expand_sub_safe)) {	336	.sub_safe => if (l.features.has(.expand_sub_safe)) {
333	assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both	337	assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
334	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));	338	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
335	} else if (l.features.has(.scalarize_sub_safe)) {	339	} else if (l.features.has(.scalarize_sub_safe)) {
336	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;	340	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
337	if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);	341	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
		342	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
		343	}
338	},	344	},
339	.mul_safe => if (l.features.has(.expand_mul_safe)) {	345	.mul_safe => if (l.features.has(.expand_mul_safe)) {
340	assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both	346	assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
341	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));	347	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
342	} else if (l.features.has(.scalarize_mul_safe)) {	348	} else if (l.features.has(.scalarize_mul_safe)) {
343	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;	349	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
344	if (l.typeOf(bin_op.lhs).isVector(zcu)) continue :inst try l.scalarize(inst, .bin_op);	350	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
		351	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
		352	}
345	},	353	},
346	.ptr_add, .ptr_sub => {},	354	.ptr_add, .ptr_sub => {},
347	inline .add_with_overflow,	355	inline .add_with_overflow,
...	@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -350,7 +358,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
350	.shl_with_overflow,	358	.shl_with_overflow,
351	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {	359	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
352	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;	360	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
353	if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));	361	if (ty_pl.ty.toType().fieldType(0, zcu).isVector(zcu)) {
		362	continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
		363	}
354	},	364	},
355	.alloc => {},	365	.alloc => {},
356	.inferred_alloc, .inferred_alloc_comptime => unreachable,	366	.inferred_alloc, .inferred_alloc_comptime => unreachable,
...	@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -387,7 +397,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
387	}	397	}
388	}	398	}
389	}	399	}
390	if (l.features.has(comptime .scalarize(air_tag))) continue :inst try l.scalarize(inst, .bin_op);	400	if (l.features.has(comptime .scalarize(air_tag))) {
		401	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
		402	}
391	}	403	}
392	},	404	},
393	inline .not,	405	inline .not,
...	@@ -406,64 +418,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -406,64 +418,41 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
406	.float_from_int,	418	.float_from_int,
407	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {	419	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
408	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	420	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
409	if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);	421	if (ty_op.ty.toType().isVector(zcu)) {
		422	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
		423	}
410	},	424	},
411	.bitcast => if (l.features.has(.scalarize_bitcast)) {	425	.bitcast => if (l.features.has(.scalarize_bitcast)) {
412	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	426	if (try l.scalarizeBitcastBlockPayload(inst)) \|payload\| {
413		427	continue :inst l.replaceInst(inst, .block, payload);
414	const to_ty = ty_op.ty.toType();	428	}
415	const to_ty_tag = to_ty.zigTypeTag(zcu);
416	const to_ty_legal = legal: switch (to_ty_tag) {
417	else => true,
418	.array, .vector => {
419	if (to_ty.arrayLen(zcu) == 1) break :legal true;
420	const to_elem_ty = to_ty.childType(zcu);
421	break :legal to_elem_ty.bitSize(zcu) == 8 * to_elem_ty.abiSize(zcu);
422	},
423	};
424
425	const from_ty = l.typeOf(ty_op.operand);
426	const from_ty_legal = legal: switch (from_ty.zigTypeTag(zcu)) {
427	else => true,
428	.array, .vector => {
429	if (from_ty.arrayLen(zcu) == 1) break :legal true;
430	const from_elem_ty = from_ty.childType(zcu);
431	break :legal from_elem_ty.bitSize(zcu) == 8 * from_elem_ty.abiSize(zcu);
432	},
433	};
434
435	if (!to_ty_legal and !from_ty_legal and to_ty.arrayLen(zcu) == from_ty.arrayLen(zcu)) switch (to_ty_tag) {
436	else => unreachable,
437	.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastToArrayBlockPayload(inst)),
438	.vector => continue :inst try l.scalarize(inst, .bitcast),
439	};
440	if (!to_ty_legal) switch (to_ty_tag) {
441	else => unreachable,
442	.array => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultArrayBlockPayload(inst)),
443	.vector => continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastResultVectorBlockPayload(inst)),
444	};
445	if (!from_ty_legal) continue :inst l.replaceInst(inst, .block, try l.scalarizeBitcastOperandBlockPayload(inst));
446	},	429	},
447	.intcast_safe => if (l.features.has(.expand_intcast_safe)) {	430	.intcast_safe => if (l.features.has(.expand_intcast_safe)) {
448	assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both	431	assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
449	continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));	432	continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
450	} else if (l.features.has(.scalarize_intcast_safe)) {	433	} else if (l.features.has(.scalarize_intcast_safe)) {
451	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	434	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
452	if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);	435	if (ty_op.ty.toType().isVector(zcu)) {
		436	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
		437	}
453	},	438	},
454	.int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {	439	.int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {
455	assert(!l.features.has(.scalarize_int_from_float_safe));	440	assert(!l.features.has(.scalarize_int_from_float_safe));
456	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));	441	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));
457	} else if (l.features.has(.scalarize_int_from_float_safe)) {	442	} else if (l.features.has(.scalarize_int_from_float_safe)) {
458	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	443	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
459	if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);	444	if (ty_op.ty.toType().isVector(zcu)) {
		445	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
		446	}
460	},	447	},
461	.int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {	448	.int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {
462	assert(!l.features.has(.scalarize_int_from_float_optimized_safe));	449	assert(!l.features.has(.scalarize_int_from_float_optimized_safe));
463	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));	450	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));
464	} else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {	451	} else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {
465	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	452	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
466	if (ty_op.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .ty_op);	453	if (ty_op.ty.toType().isVector(zcu)) {
		454	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
		455	}
467	},	456	},
468	.block, .loop => {	457	.block, .loop => {
469	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;	458	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
...	@@ -498,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -498,7 +487,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
498	.neg_optimized,	487	.neg_optimized,
499	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {	488	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
500	const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;	489	const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
501	if (l.typeOf(un_op).isVector(zcu)) continue :inst try l.scalarize(inst, .un_op);	490	if (l.typeOf(un_op).isVector(zcu)) {
		491	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op));
		492	}
502	},	493	},
503	.cmp_lt,	494	.cmp_lt,
504	.cmp_lt_optimized,	495	.cmp_lt_optimized,
...	@@ -515,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -515,7 +506,9 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
515	=> {},	506	=> {},
516	inline .cmp_vector, .cmp_vector_optimized => \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {	507	inline .cmp_vector, .cmp_vector_optimized => \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
517	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;	508	const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
518	if (ty_pl.ty.toType().isVector(zcu)) continue :inst try l.scalarize(inst, .cmp_vector);	509	if (ty_pl.ty.toType().isVector(zcu)) {
		510	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector));
		511	}
519	},	512	},
520	.cond_br => {	513	.cond_br => {
521	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;	514	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
...	@@ -570,13 +563,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -570,13 +563,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
570	.load => if (l.features.has(.expand_packed_load)) {	563	.load => if (l.features.has(.expand_packed_load)) {
571	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;	564	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
572	const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);	565	const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
573	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));	566	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
		567	continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
		568	}
574	},	569	},
575	.ret, .ret_safe, .ret_load => {},	570	.ret, .ret_safe, .ret_load => {},
576	.store, .store_safe => if (l.features.has(.expand_packed_store)) {	571	.store, .store_safe => if (l.features.has(.expand_packed_store)) {
577	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;	572	const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
578	const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);	573	const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
579	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));	574	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
		575	continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
		576	}
580	},	577	},
581	.unreach,	578	.unreach,
582	.optional_payload,	579	.optional_payload,
...	@@ -624,7 +621,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -624,7 +621,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
624	switch (vector_ty.vectorLen(zcu)) {	621	switch (vector_ty.vectorLen(zcu)) {
625	0 => unreachable,	622	0 => unreachable,
626	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{	623	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
627	.ty = Air.internedToRef(vector_ty.childType(zcu).toIntern()),	624	.ty = .fromType(vector_ty.childType(zcu)),
628	.operand = reduce.operand,	625	.operand = reduce.operand,
629	} }),	626	} }),
630	else => {},	627	else => {},
...	@@ -641,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -641,9 +638,15 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
641	else => {},	638	else => {},
642	}	639	}
643	},	640	},
644	.shuffle_one => if (l.features.has(.scalarize_shuffle_one)) continue :inst try l.scalarize(inst, .shuffle_one),	641	.shuffle_one => if (l.features.has(.scalarize_shuffle_one)) {
645	.shuffle_two => if (l.features.has(.scalarize_shuffle_two)) continue :inst try l.scalarize(inst, .shuffle_two),	642	continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst));
646	.select => if (l.features.has(.scalarize_select)) continue :inst try l.scalarize(inst, .select),	643	},
		644	.shuffle_two => if (l.features.has(.scalarize_shuffle_two)) {
		645	continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst));
		646	},
		647	.select => if (l.features.has(.scalarize_select)) {
		648	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select));
		649	},
647	.memset,	650	.memset,
648	.memset_safe,	651	.memset_safe,
649	.memcpy,	652	.memcpy,
...	@@ -666,16 +669,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -666,16 +669,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
666	const agg_ty = ty_pl.ty.toType();	669	const agg_ty = ty_pl.ty.toType();
667	switch (agg_ty.zigTypeTag(zcu)) {	670	switch (agg_ty.zigTypeTag(zcu)) {
668	else => {},	671	else => {},
669	.@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {	672	.@"union" => unreachable,
		673	.@"struct" => switch (agg_ty.containerLayout(zcu)) {
670	.auto, .@"extern" => {},	674	.auto, .@"extern" => {},
671	.@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),	675	.@"packed" => switch (agg_ty.structFieldCount(zcu)) {
		676	0 => unreachable,
		677	// An `aggregate_init` of a packed struct with 1 field is just a fancy bitcast.
		678	1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
		679	.ty = .fromType(agg_ty),
		680	.operand = @enumFromInt(l.air_extra.items[ty_pl.payload]),
		681	} }),
		682	else => continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst)),
		683	},
672	},	684	},
673	}	685	}
674	},	686	},
675	.union_init, .prefetch => {},	687	.union_init, .prefetch => {},
676	.mul_add => if (l.features.has(.scalarize_mul_add)) {	688	.mul_add => if (l.features.has(.scalarize_mul_add)) {
677	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;	689	const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
678	if (l.typeOf(pl_op.operand).isVector(zcu)) continue :inst try l.scalarize(inst, .pl_op_bin);	690	if (l.typeOf(pl_op.operand).isVector(zcu)) {
		691	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin));
		692	}
679	},	693	},
680	.field_parent_ptr,	694	.field_parent_ptr,
681	.wasm_memory_size,	695	.wasm_memory_size,
...	@@ -685,7 +699,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -685,7 +699,6 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
685	.set_err_return_trace,	699	.set_err_return_trace,
686	.addrspace_cast,	700	.addrspace_cast,
687	.save_err_return_trace_index,	701	.save_err_return_trace_index,
688	.vector_store_elem,
689	.runtime_nav_ptr,	702	.runtime_nav_ptr,
690	.c_va_arg,	703	.c_va_arg,
691	.c_va_copy,	704	.c_va_copy,
...	@@ -694,1003 +707,757 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {	...	@@ -694,1003 +707,757 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
694	.work_item_id,	707	.work_item_id,
695	.work_group_size,	708	.work_group_size,
696	.work_group_id,	709	.work_group_id,
		710	.legalize_vec_elem_val,
		711	.legalize_vec_store_elem,
697	=> {},	712	=> {},
698	}	713	}
699	}	714	}
700	}	715	}
701		716
702	const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, bitcast, cmp_vector, shuffle_one, shuffle_two, select };	717	const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, select };
703	/// inline to propagate comptime-known `replaceInst` result.	718	fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: ScalarizeForm) Error!Air.Inst.Data {
704	inline fn scalarize(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Tag {
705	return l.replaceInst(orig_inst, .block, try l.scalarizeBlockPayload(orig_inst, form));
706	}
707	fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, comptime form: ScalarizeForm) Error!Air.Inst.Data {
708	const pt = l.pt;	719	const pt = l.pt;
709	const zcu = pt.zcu;	720	const zcu = pt.zcu;
710		721
711	const orig = l.air_instructions.get(@intFromEnum(orig_inst));	722	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
712	const res_ty = l.typeOfIndex(orig_inst);	723	const res_ty = l.typeOfIndex(orig_inst);
713	const res_len = res_ty.vectorLen(zcu);	724	const result_is_array = switch (res_ty.zigTypeTag(zcu)) {
714		725	.vector => false,
715	const extra_insts = switch (form) {	726	.array => true,
716	.un_op, .ty_op, .bitcast => 1,	727	else => unreachable,
717	.bin_op, .cmp_vector => 2,
718	.pl_op_bin => 3,
719	.shuffle_one, .shuffle_two => 13,
720	.select => 6,
721	};	728	};
722	var inst_buf: [5 + extra_insts + 9]Air.Inst.Index = undefined;	729	const res_len = res_ty.arrayLen(zcu);
723	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	730	const res_elem_ty = res_ty.childType(zcu);
724		731
725	var res_block: Block = .init(&inst_buf);	732	if (result_is_array) {
726	{	733	// This is only allowed when legalizing an elementwise bitcast.
727	const res_alloc_inst = res_block.add(l, .{	734	assert(orig.tag == .bitcast);
728	.tag = .alloc,	735	assert(form == .ty_op);
729	.data = .{ .ty = try pt.singleMutPtrType(res_ty) },	736	}
730	});
731	const index_alloc_inst = res_block.add(l, .{
732	.tag = .alloc,
733	.data = .{ .ty = .ptr_usize },
734	});
735	_ = res_block.add(l, .{
736	.tag = .store,
737	.data = .{ .bin_op = .{
738	.lhs = index_alloc_inst.toRef(),
739	.rhs = .zero_usize,
740	} },
741	});
742		737
743	var loop: Loop = .init(l, &res_block);	738	// Our output will be a loop doing elementwise stores:
744	loop.block = .init(res_block.stealRemainingCapacity());	739	//
745	{	740	// %1 = block(@Vector(N, Scalar), {
746	const cur_index_inst = loop.block.add(l, .{	741	// %2 = alloc(*usize)
747	.tag = .load,	742	// %3 = alloc(*@Vector(N, Scalar))
748	.data = .{ .ty_op = .{	743	// %4 = store(%2, @zero_usize)
749	.ty = .usize_type,	744	// %5 = loop({
750	.operand = index_alloc_inst.toRef(),	745	// %6 = load(%2)
751	} },	746	// %7 = <scalar result of operation at index %5>
752	});	747	// %8 = legalize_vec_store_elem(%3, %5, %6)
753	_ = loop.block.add(l, .{	748	// %9 = cmp_eq(%6, <usize, N-1>)
754	.tag = .vector_store_elem,	749	// %10 = cond_br(%9, {
755	.data = .{ .vector_store_elem = .{	750	// %11 = load(%3)
756	.vector_ptr = res_alloc_inst.toRef(),	751	// %12 = br(%1, %11)
757	.payload = try l.addExtra(Air.Bin, .{	752	// }, {
758	.lhs = cur_index_inst.toRef(),	753	// %13 = add(%6, @one_usize)
759	.rhs = res_elem: switch (form) {	754	// %14 = store(%2, %13)
760	.un_op => loop.block.add(l, .{	755	// %15 = repeat(%5)
761	.tag = orig.tag,	756	// })
762	.data = .{ .un_op = loop.block.add(l, .{	757	// })
763	.tag = .array_elem_val,	758	// })
764	.data = .{ .bin_op = .{	759	//
765	.lhs = orig.data.un_op,	760	// If scalarizing an elementwise bitcast, the result might be an array, in which case
766	.rhs = cur_index_inst.toRef(),	761	// `legalize_vec_store_elem` becomes two instructions (`ptr_elem_ptr` and `store`).
767	} },	762	// Therefore, there are 13 or 14 instructions in the block, plus however many are
768	}).toRef() },	763	// needed to compute each result element for `form`.
769	}).toRef(),	764	const inst_per_form: usize = switch (form) {
770	.ty_op => loop.block.add(l, .{	765	.un_op, .ty_op => 2,
771	.tag = orig.tag,	766	.bin_op, .cmp_vector => 3,
772	.data = .{ .ty_op = .{	767	.pl_op_bin => 4,
773	.ty = Air.internedToRef(res_ty.childType(zcu).toIntern()),	768	.select => 7,
774	.operand = loop.block.add(l, .{	769	};
775	.tag = .array_elem_val,	770	const max_inst_per_form = 7; // maximum value in the above switch
776	.data = .{ .bin_op = .{	771	var inst_buf: [14 + max_inst_per_form]Air.Inst.Index = undefined;
777	.lhs = orig.data.ty_op.operand,
778	.rhs = cur_index_inst.toRef(),
779	} },
780	}).toRef(),
781	} },
782	}).toRef(),
783	.bin_op => loop.block.add(l, .{
784	.tag = orig.tag,
785	.data = .{ .bin_op = .{
786	.lhs = loop.block.add(l, .{
787	.tag = .array_elem_val,
788	.data = .{ .bin_op = .{
789	.lhs = orig.data.bin_op.lhs,
790	.rhs = cur_index_inst.toRef(),
791	} },
792	}).toRef(),
793	.rhs = loop.block.add(l, .{
794	.tag = .array_elem_val,
795	.data = .{ .bin_op = .{
796	.lhs = orig.data.bin_op.rhs,
797	.rhs = cur_index_inst.toRef(),
798	} },
799	}).toRef(),
800	} },
801	}).toRef(),
802	.pl_op_bin => {
803	const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
804	break :res_elem loop.block.add(l, .{
805	.tag = orig.tag,
806	.data = .{ .pl_op = .{
807	.payload = try l.addExtra(Air.Bin, .{
808	.lhs = loop.block.add(l, .{
809	.tag = .array_elem_val,
810	.data = .{ .bin_op = .{
811	.lhs = extra.lhs,
812	.rhs = cur_index_inst.toRef(),
813	} },
814	}).toRef(),
815	.rhs = loop.block.add(l, .{
816	.tag = .array_elem_val,
817	.data = .{ .bin_op = .{
818	.lhs = extra.rhs,
819	.rhs = cur_index_inst.toRef(),
820	} },
821	}).toRef(),
822	}),
823	.operand = loop.block.add(l, .{
824	.tag = .array_elem_val,
825	.data = .{ .bin_op = .{
826	.lhs = orig.data.pl_op.operand,
827	.rhs = cur_index_inst.toRef(),
828	} },
829	}).toRef(),
830	} },
831	}).toRef();
832	},
833	.bitcast => loop.block.addBitCast(l, res_ty.childType(zcu), loop.block.add(l, .{
834	.tag = .array_elem_val,
835	.data = .{ .bin_op = .{
836	.lhs = orig.data.ty_op.operand,
837	.rhs = cur_index_inst.toRef(),
838	} },
839	}).toRef()),
840	.cmp_vector => {
841	const extra = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
842	break :res_elem (try loop.block.addCmp(
843	l,
844	extra.compareOperator(),
845	loop.block.add(l, .{
846	.tag = .array_elem_val,
847	.data = .{ .bin_op = .{
848	.lhs = extra.lhs,
849	.rhs = cur_index_inst.toRef(),
850	} },
851	}).toRef(),
852	loop.block.add(l, .{
853	.tag = .array_elem_val,
854	.data = .{ .bin_op = .{
855	.lhs = extra.rhs,
856	.rhs = cur_index_inst.toRef(),
857	} },
858	}).toRef(),
859	.{ .optimized = switch (orig.tag) {
860	else => unreachable,
861	.cmp_vector => false,
862	.cmp_vector_optimized => true,
863	} },
864	)).toRef();
865	},
866	.shuffle_one, .shuffle_two => {
867	const ip = &zcu.intern_pool;
868	const unwrapped = switch (form) {
869	else => comptime unreachable,
870	.shuffle_one => l.getTmpAir().unwrapShuffleOne(zcu, orig_inst),
871	.shuffle_two => l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst),
872	};
873	const operand_a = switch (form) {
874	else => comptime unreachable,
875	.shuffle_one => unwrapped.operand,
876	.shuffle_two => unwrapped.operand_a,
877	};
878	const operand_a_len = l.typeOf(operand_a).vectorLen(zcu);
879	const elem_ty = res_ty.childType(zcu);
880	var res_elem: Result = .init(l, elem_ty, &loop.block);
881	res_elem.block = .init(loop.block.stealCapacity(extra_insts));
882	{
883	const ExpectedContents = extern struct {
884	mask_elems: [128]InternPool.Index,
885	ct_elems: switch (form) {
886	else => unreachable,
887	.shuffle_one => extern struct {
888	keys: [152]InternPool.Index,
889	header: u8 align(@alignOf(u32)),
890	index: [256][2]u8,
891	},
892	.shuffle_two => void,
893	},
894	};
895	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
896	std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
897	const gpa = stack.get();
898		772
899	const mask_elems = try gpa.alloc(InternPool.Index, res_len);	773	var main_block: Block = .init(&inst_buf);
900	defer gpa.free(mask_elems);	774	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
901
902	var ct_elems: switch (form) {
903	else => unreachable,
904	.shuffle_one => std.AutoArrayHashMapUnmanaged(InternPool.Index, void),
905	.shuffle_two => struct {
906	const empty: @This() = .{};
907	inline fn deinit(_: @This(), _: std.mem.Allocator) void {}
908	inline fn ensureTotalCapacity(_: @This(), _: std.mem.Allocator, _: usize) error{}!void {}
909	},
910	} = .empty;
911	defer ct_elems.deinit(gpa);
912	try ct_elems.ensureTotalCapacity(gpa, res_len);
913
914	const mask_elem_ty = try pt.intType(.signed, 1 + Type.smallestUnsignedBits(@max(operand_a_len, switch (form) {
915	else => comptime unreachable,
916	.shuffle_one => res_len,
917	.shuffle_two => l.typeOf(unwrapped.operand_b).vectorLen(zcu),
918	})));
919	for (mask_elems, unwrapped.mask) \|mask_elem_val, mask_elem\| mask_elem_val. = (try pt.intValue(mask_elem_ty, switch (form) {
920	else => comptime unreachable,
921	.shuffle_one => switch (mask_elem.unwrap()) {
922	.elem => \|index\| index,
923	.value => \|elem_val\| if (ip.isUndef(elem_val))
924	operand_a_len
925	else
926	~@as(i33, @intCast((ct_elems.getOrPutAssumeCapacity(elem_val)).index)),
927	},
928	.shuffle_two => switch (mask_elem.unwrap()) {
929	.a_elem => \|a_index\| a_index,
930	.b_elem => \|b_index\| ~@as(i33, b_index),
931	.undef => operand_a_len,
932	},
933	})).toIntern();
934	const mask_ty = try pt.arrayType(.{
935	.len = res_len,
936	.child = mask_elem_ty.toIntern(),
937	});
938	const mask_elem_inst = res_elem.block.add(l, .{
939	.tag = .ptr_elem_val,
940	.data = .{ .bin_op = .{
941	.lhs = Air.internedToRef(try pt.intern(.{ .ptr = .{
942	.ty = (try pt.manyConstPtrType(mask_elem_ty)).toIntern(),
943	.base_addr = .{ .uav = .{
944	.val = (try pt.aggregateValue(mask_ty, mask_elems)).toIntern(),
945	.orig_ty = (try pt.singleConstPtrType(mask_ty)).toIntern(),
946	} },
947	.byte_offset = 0,
948	} })),
949	.rhs = cur_index_inst.toRef(),
950	} },
951	});
952	var def_cond_br: CondBr = .init(l, (try res_elem.block.addCmp(
953	l,
954	.lt,
955	mask_elem_inst.toRef(),
956	try pt.intRef(mask_elem_ty, operand_a_len),
957	.{},
958	)).toRef(), &res_elem.block, .{});
959	def_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
960	{
961	const operand_b_used = switch (form) {
962	else => comptime unreachable,
963	.shuffle_one => ct_elems.count() > 0,
964	.shuffle_two => true,
965	};
966	var operand_cond_br: CondBr = undefined;
967	operand_cond_br.then_block = if (operand_b_used) then_block: {
968	operand_cond_br = .init(l, (try def_cond_br.then_block.addCmp(
969	l,
970	.gte,
971	mask_elem_inst.toRef(),
972	try pt.intRef(mask_elem_ty, 0),
973	.{},
974	)).toRef(), &def_cond_br.then_block, .{});
975	break :then_block .init(def_cond_br.then_block.stealRemainingCapacity());
976	} else def_cond_br.then_block;
977	_ = operand_cond_br.then_block.add(l, .{
978	.tag = .br,
979	.data = .{ .br = .{
980	.block_inst = res_elem.inst,
981	.operand = operand_cond_br.then_block.add(l, .{
982	.tag = .array_elem_val,
983	.data = .{ .bin_op = .{
984	.lhs = operand_a,
985	.rhs = operand_cond_br.then_block.add(l, .{
986	.tag = .intcast,
987	.data = .{ .ty_op = .{
988	.ty = .usize_type,
989	.operand = mask_elem_inst.toRef(),
990	} },
991	}).toRef(),
992	} },
993	}).toRef(),
994	} },
995	});
996	if (operand_b_used) {
997	operand_cond_br.else_block = .init(operand_cond_br.then_block.stealRemainingCapacity());
998	_ = operand_cond_br.else_block.add(l, .{
999	.tag = .br,
1000	.data = .{ .br = .{
1001	.block_inst = res_elem.inst,
1002	.operand = if (switch (form) {
1003	else => comptime unreachable,
1004	.shuffle_one => ct_elems.count() > 1,
1005	.shuffle_two => true,
1006	}) operand_cond_br.else_block.add(l, .{
1007	.tag = switch (form) {
1008	else => comptime unreachable,
1009	.shuffle_one => .ptr_elem_val,
1010	.shuffle_two => .array_elem_val,
1011	},
1012	.data = .{ .bin_op = .{
1013	.lhs = operand_b: switch (form) {
1014	else => comptime unreachable,
1015	.shuffle_one => {
1016	const ct_elems_ty = try pt.arrayType(.{
1017	.len = ct_elems.count(),
1018	.child = elem_ty.toIntern(),
1019	});
1020	break :operand_b Air.internedToRef(try pt.intern(.{ .ptr = .{
1021	.ty = (try pt.manyConstPtrType(elem_ty)).toIntern(),
1022	.base_addr = .{ .uav = .{
1023	.val = (try pt.aggregateValue(ct_elems_ty, ct_elems.keys())).toIntern(),
1024	.orig_ty = (try pt.singleConstPtrType(ct_elems_ty)).toIntern(),
1025	} },
1026	.byte_offset = 0,
1027	} }));
1028	},
1029	.shuffle_two => unwrapped.operand_b,
1030	},
1031	.rhs = operand_cond_br.else_block.add(l, .{
1032	.tag = .intcast,
1033	.data = .{ .ty_op = .{
1034	.ty = .usize_type,
1035	.operand = operand_cond_br.else_block.add(l, .{
1036	.tag = .not,
1037	.data = .{ .ty_op = .{
1038	.ty = Air.internedToRef(mask_elem_ty.toIntern()),
1039	.operand = mask_elem_inst.toRef(),
1040	} },
1041	}).toRef(),
1042	} },
1043	}).toRef(),
1044	} },
1045	}).toRef() else res_elem_br: {
1046	_ = operand_cond_br.else_block.stealCapacity(3);
1047	break :res_elem_br Air.internedToRef(ct_elems.keys()[0]);
1048	},
1049	} },
1050	});
1051	def_cond_br.else_block = .init(operand_cond_br.else_block.stealRemainingCapacity());
1052	try operand_cond_br.finish(l);
1053	} else {
1054	def_cond_br.then_block = operand_cond_br.then_block;
1055	_ = def_cond_br.then_block.stealCapacity(6);
1056	def_cond_br.else_block = .init(def_cond_br.then_block.stealRemainingCapacity());
1057	}
1058	}
1059	_ = def_cond_br.else_block.add(l, .{
1060	.tag = .br,
1061	.data = .{ .br = .{
1062	.block_inst = res_elem.inst,
1063	.operand = try pt.undefRef(elem_ty),
1064	} },
1065	});
1066	try def_cond_br.finish(l);
1067	}
1068	try res_elem.finish(l);
1069	break :res_elem res_elem.inst.toRef();
1070	},
1071	.select => {
1072	const extra = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1073	var res_elem: Result = .init(l, l.typeOf(extra.lhs).childType(zcu), &loop.block);
1074	res_elem.block = .init(loop.block.stealCapacity(extra_insts));
1075	{
1076	var select_cond_br: CondBr = .init(l, res_elem.block.add(l, .{
1077	.tag = .array_elem_val,
1078	.data = .{ .bin_op = .{
1079	.lhs = orig.data.pl_op.operand,
1080	.rhs = cur_index_inst.toRef(),
1081	} },
1082	}).toRef(), &res_elem.block, .{});
1083	select_cond_br.then_block = .init(res_elem.block.stealRemainingCapacity());
1084	_ = select_cond_br.then_block.add(l, .{
1085	.tag = .br,
1086	.data = .{ .br = .{
1087	.block_inst = res_elem.inst,
1088	.operand = select_cond_br.then_block.add(l, .{
1089	.tag = .array_elem_val,
1090	.data = .{ .bin_op = .{
1091	.lhs = extra.lhs,
1092	.rhs = cur_index_inst.toRef(),
1093	} },
1094	}).toRef(),
1095	} },
1096	});
1097	select_cond_br.else_block = .init(select_cond_br.then_block.stealRemainingCapacity());
1098	_ = select_cond_br.else_block.add(l, .{
1099	.tag = .br,
1100	.data = .{ .br = .{
1101	.block_inst = res_elem.inst,
1102	.operand = select_cond_br.else_block.add(l, .{
1103	.tag = .array_elem_val,
1104	.data = .{ .bin_op = .{
1105	.lhs = extra.rhs,
1106	.rhs = cur_index_inst.toRef(),
1107	} },
1108	}).toRef(),
1109	} },
1110	});
1111	try select_cond_br.finish(l);
1112	}
1113	try res_elem.finish(l);
1114	break :res_elem res_elem.inst.toRef();
1115	},
1116	},
1117	}),
1118	} },
1119	});
1120		775
1121	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(	776	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
		777	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(res_ty)).toRef();
		778
		779	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
		780
		781	var loop: Loop = .init(l, &main_block);
		782	loop.block = .init(main_block.stealRemainingCapacity());
		783
		784	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
		785	const elem_val: Air.Inst.Ref = switch (form) {
		786	.un_op => elem: {
		787	const orig_operand = orig.data.un_op;
		788	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
		789	break :elem loop.block.addUnOp(l, orig.tag, operand).toRef();
		790	},
		791	.ty_op => elem: {
		792	const orig_operand = orig.data.ty_op.operand;
		793	const operand_is_array = switch (l.typeOf(orig_operand).zigTypeTag(zcu)) {
		794	.vector => false,
		795	.array => true,
		796	else => unreachable,
		797	};
		798	const operand = loop.block.addBinOp(
1122	l,	799	l,
1123	.lt,	800	if (operand_is_array) .array_elem_val else .legalize_vec_elem_val,
1124	cur_index_inst.toRef(),	801	orig_operand,
1125	try pt.intRef(.usize, res_len - 1),	802	index_val,
1126	.{},	803	).toRef();
1127	)).toRef(), &loop.block, .{});	804	break :elem loop.block.addTyOp(l, orig.tag, res_elem_ty, operand).toRef();
1128	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());	805	},
1129	{	806	.bin_op => elem: {
1130	_ = loop_cond_br.then_block.add(l, .{	807	const orig_bin = orig.data.bin_op;
1131	.tag = .store,	808	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
1132	.data = .{ .bin_op = .{	809	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
1133	.lhs = index_alloc_inst.toRef(),	810	break :elem loop.block.addBinOp(l, orig.tag, lhs, rhs).toRef();
1134	.rhs = loop_cond_br.then_block.add(l, .{	811	},
1135	.tag = .add,	812	.pl_op_bin => elem: {
1136	.data = .{ .bin_op = .{	813	const orig_operand = orig.data.pl_op.operand;
1137	.lhs = cur_index_inst.toRef(),	814	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1138	.rhs = .one_usize,	815	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
1139	} },	816	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
1140	}).toRef(),	817	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
1141	} },	818	break :elem loop.block.add(l, .{
1142	});	819	.tag = orig.tag,
1143	_ = loop_cond_br.then_block.add(l, .{	820	.data = .{ .pl_op = .{
1144	.tag = .repeat,	821	.operand = operand,
1145	.data = .{ .repeat = .{ .loop_inst = loop.inst } },	822	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1146	});	823	} },
1147	}	824	}).toRef();
1148	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());	825	},
1149	_ = loop_cond_br.else_block.add(l, .{	826	.cmp_vector => elem: {
1150	.tag = .br,	827	const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
1151	.data = .{ .br = .{	828	const cmp_op = orig_payload.compareOperator();
1152	.block_inst = orig_inst,	829	const optimized = switch (orig.tag) {
1153	.operand = loop_cond_br.else_block.add(l, .{	830	.cmp_vector => false,
1154	.tag = .load,	831	.cmp_vector_optimized => true,
1155	.data = .{ .ty_op = .{	832	else => unreachable,
1156	.ty = Air.internedToRef(res_ty.toIntern()),	833	};
1157	.operand = res_alloc_inst.toRef(),	834	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.lhs, index_val).toRef();
1158	} },	835	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.rhs, index_val).toRef();
1159	}).toRef(),	836	break :elem loop.block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
		837	},
		838	.select => elem: {
		839	const orig_cond = orig.data.pl_op.operand;
		840	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
		841
		842	const elem_block_inst = loop.block.add(l, .{
		843	.tag = .block,
		844	.data = .{ .ty_pl = .{
		845	.ty = .fromType(res_elem_ty),
		846	.payload = undefined,
1160	} },	847	} },
1161	});	848	});
1162	try loop_cond_br.finish(l);	849	var elem_block: Block = .init(loop.block.stealCapacity(2));
1163	}	850	const cond = elem_block.addBinOp(l, .legalize_vec_elem_val, orig_cond, index_val).toRef();
1164	try loop.finish(l);	851
		852	var condbr: CondBr = .init(l, cond, &elem_block, .{});
		853
		854	condbr.then_block = .init(loop.block.stealCapacity(2));
		855	const lhs = condbr.then_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
		856	condbr.then_block.addBr(l, elem_block_inst, lhs);
		857
		858	condbr.else_block = .init(loop.block.stealCapacity(2));
		859	const rhs = condbr.else_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
		860	condbr.else_block.addBr(l, elem_block_inst, rhs);
		861
		862	try condbr.finish(l);
		863
		864	const inst_data = l.air_instructions.items(.data);
		865	inst_data[@intFromEnum(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
		866
		867	break :elem elem_block_inst.toRef();
		868	},
		869	};
		870	_ = loop.block.stealCapacity(max_inst_per_form - inst_per_form);
		871	if (result_is_array) {
		872	const elem_ptr = loop.block.add(l, .{
		873	.tag = .ptr_elem_ptr,
		874	.data = .{ .ty_pl = .{
		875	.ty = .fromType(try pt.singleMutPtrType(res_elem_ty)),
		876	.payload = try l.addExtra(Air.Bin, .{
		877	.lhs = result_ptr,
		878	.rhs = index_val,
		879	}),
		880	} },
		881	}).toRef();
		882	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
		883	} else {
		884	_ = loop.block.add(l, .{
		885	.tag = .legalize_vec_store_elem,
		886	.data = .{ .pl_op = .{
		887	.operand = result_ptr,
		888	.payload = try l.addExtra(Air.Bin, .{
		889	.lhs = index_val,
		890	.rhs = elem_val,
		891	}),
		892	} },
		893	});
		894	_ = loop.block.stealCapacity(1);
1165	}	895	}
		896	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, res_len - 1))).toRef();
		897
		898	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
		899	condbr.then_block = .init(loop.block.stealRemainingCapacity());
		900	const result_val = condbr.then_block.addTyOp(l, .load, res_ty, result_ptr).toRef();
		901	condbr.then_block.addBr(l, orig_inst, result_val);
		902
		903	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
		904	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
		905	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
		906	_ = condbr.else_block.add(l, .{
		907	.tag = .repeat,
		908	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
		909	});
		910
		911	try condbr.finish(l);
		912
		913	try loop.finish(l);
		914
1166	return .{ .ty_pl = .{	915	return .{ .ty_pl = .{
1167	.ty = Air.internedToRef(res_ty.toIntern()),	916	.ty = .fromType(res_ty),
1168	.payload = try l.addBlockBody(res_block.body()),	917	.payload = try l.addBlockBody(main_block.body()),
1169	} };	918	} };
1170	}	919	}
1171	fn scalarizeBitcastToArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	920	fn scalarizeShuffleOneBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1172	const pt = l.pt;	921	const pt = l.pt;
1173	const zcu = pt.zcu;	922	const zcu = pt.zcu;
		923	const gpa = zcu.gpa;
1174		924
1175	const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;	925	const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
1176	const res_ty = orig_ty_op.ty.toType();
1177	const res_elem_ty = res_ty.childType(zcu);
1178	const res_len = res_ty.arrayLen(zcu);
1179		926
1180	var inst_buf: [16]Air.Inst.Index = undefined;	927	// We're going to emit something like this:
1181	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	928	//
		929	// var x: @Vector(N, T) = all_comptime_known_elems;
		930	// for (out_idxs, in_idxs) \|i, j\| x[i] = operand[j];
		931	//
		932	// So we must first compute `out_idxs` and `in_idxs`.
1182		933
1183	var res_block: Block = .init(&inst_buf);	934	var sfba_state = std.heap.stackFallback(512, gpa);
1184	{	935	const sfba = sfba_state.get();
1185	const res_alloc_inst = res_block.add(l, .{
1186	.tag = .alloc,
1187	.data = .{ .ty = try pt.singleMutPtrType(res_ty) },
1188	});
1189	const index_alloc_inst = res_block.add(l, .{
1190	.tag = .alloc,
1191	.data = .{ .ty = .ptr_usize },
1192	});
1193	_ = res_block.add(l, .{
1194	.tag = .store,
1195	.data = .{ .bin_op = .{
1196	.lhs = index_alloc_inst.toRef(),
1197	.rhs = .zero_usize,
1198	} },
1199	});
1200		936
1201	var loop: Loop = .init(l, &res_block);	937	const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1202	loop.block = .init(res_block.stealRemainingCapacity());	938	defer sfba.free(out_idxs_buf);
1203	{	939
1204	const cur_index_inst = loop.block.add(l, .{	940	const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1205	.tag = .load,	941	defer sfba.free(in_idxs_buf);
1206	.data = .{ .ty_op = .{	942
1207	.ty = .usize_type,	943	var n: usize = 0;
1208	.operand = index_alloc_inst.toRef(),	944	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
1209	} },	945	.value => {},
1210	});	946	.elem => \|in_idx\| {
1211	_ = loop.block.add(l, .{	947	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1212	.tag = .store,	948	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1213	.data = .{ .bin_op = .{	949	n += 1;
1214	.lhs = loop.block.add(l, .{	950	},
1215	.tag = .ptr_elem_ptr,	951	};
1216	.data = .{ .ty_pl = .{	952
1217	.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),	953	const init_val: Value = init: {
1218	.payload = try l.addExtra(Air.Bin, .{	954	const undef_val = try pt.undefValue(shuffle.result_ty.childType(zcu));
1219	.lhs = res_alloc_inst.toRef(),	955	const elems = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1220	.rhs = cur_index_inst.toRef(),	956	defer sfba.free(elems);
1221	}),	957	for (shuffle.mask, elems) \|mask, elem\| elem. = switch (mask.unwrap()) {
1222	} },	958	.value => \|ip_index\| ip_index,
1223	}).toRef(),	959	.elem => undef_val.toIntern(),
1224	.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{	960	};
1225	.tag = .array_elem_val,	961	break :init try pt.aggregateValue(shuffle.result_ty, elems);
1226	.data = .{ .bin_op = .{	962	};
1227	.lhs = orig_ty_op.operand,	963
1228	.rhs = cur_index_inst.toRef(),	964	// %1 = block(@Vector(N, T), {
1229	} },	965	// %2 = alloc(*@Vector(N, T))
1230	}).toRef()),	966	// %3 = alloc(*usize)
1231	} },	967	// %4 = store(%2, <init_val>)
1232	});	968	// %5 = [addScalarizedShuffle]
		969	// %6 = load(%2)
		970	// %7 = br(%1, %6)
		971	// })
		972
		973	var inst_buf: [6]Air.Inst.Index = undefined;
		974	var main_block: Block = .init(&inst_buf);
		975	try l.air_instructions.ensureUnusedCapacity(gpa, 19);
		976
		977	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
		978	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
		979
		980	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(init_val));
		981
		982	try l.addScalarizedShuffle(
		983	&main_block,
		984	shuffle.operand,
		985	result_ptr,
		986	index_ptr,
		987	out_idxs_buf[0..n],
		988	in_idxs_buf[0..n],
		989	);
		990
		991	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
		992	main_block.addBr(l, orig_inst, result_val);
1233		993
1234	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1235	l,
1236	.lt,
1237	cur_index_inst.toRef(),
1238	try pt.intRef(.usize, res_len - 1),
1239	.{},
1240	)).toRef(), &loop.block, .{});
1241	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1242	{
1243	_ = loop_cond_br.then_block.add(l, .{
1244	.tag = .store,
1245	.data = .{ .bin_op = .{
1246	.lhs = index_alloc_inst.toRef(),
1247	.rhs = loop_cond_br.then_block.add(l, .{
1248	.tag = .add,
1249	.data = .{ .bin_op = .{
1250	.lhs = cur_index_inst.toRef(),
1251	.rhs = .one_usize,
1252	} },
1253	}).toRef(),
1254	} },
1255	});
1256	_ = loop_cond_br.then_block.add(l, .{
1257	.tag = .repeat,
1258	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1259	});
1260	}
1261	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1262	_ = loop_cond_br.else_block.add(l, .{
1263	.tag = .br,
1264	.data = .{ .br = .{
1265	.block_inst = orig_inst,
1266	.operand = loop_cond_br.else_block.add(l, .{
1267	.tag = .load,
1268	.data = .{ .ty_op = .{
1269	.ty = Air.internedToRef(res_ty.toIntern()),
1270	.operand = res_alloc_inst.toRef(),
1271	} },
1272	}).toRef(),
1273	} },
1274	});
1275	try loop_cond_br.finish(l);
1276	}
1277	try loop.finish(l);
1278	}
1279	return .{ .ty_pl = .{	994	return .{ .ty_pl = .{
1280	.ty = Air.internedToRef(res_ty.toIntern()),	995	.ty = .fromType(shuffle.result_ty),
1281	.payload = try l.addBlockBody(res_block.body()),	996	.payload = try l.addBlockBody(main_block.body()),
1282	} };	997	} };
1283	}	998	}
1284	fn scalarizeBitcastOperandBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	999	fn scalarizeShuffleTwoBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1285	const pt = l.pt;	1000	const pt = l.pt;
1286	const zcu = pt.zcu;	1001	const zcu = pt.zcu;
		1002	const gpa = zcu.gpa;
1287		1003
1288	const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;	1004	const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
1289	const res_ty = orig_ty_op.ty.toType();
1290	const operand_ty = l.typeOf(orig_ty_op.operand);
1291	const int_bits: u16 = @intCast(operand_ty.bitSize(zcu));
1292	const int_ty = try pt.intType(.unsigned, int_bits);
1293	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1294	const elem_bits: u16 = @intCast(operand_ty.childType(zcu).bitSize(zcu));
1295	const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1296
1297	var inst_buf: [22]Air.Inst.Index = undefined;
1298	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1299		1005
1300	var res_block: Block = .init(&inst_buf);	1006	// We're going to emit something like this:
1301	{	1007	//
1302	const int_alloc_inst = res_block.add(l, .{	1008	// var x: @Vector(N, T) = undefined;
1303	.tag = .alloc,	1009	// for (out_idxs_a, in_idxs_a) \|i, j\| x[i] = operand_a[j];
1304	.data = .{ .ty = try pt.singleMutPtrType(int_ty) },	1010	// for (out_idxs_b, in_idxs_b) \|i, j\| x[i] = operand_b[j];
1305	});	1011	//
1306	_ = res_block.add(l, .{	1012	// The AIR will look like this:
1307	.tag = .store,	1013	//
1308	.data = .{ .bin_op = .{	1014	// %1 = block(@Vector(N, T), {
1309	.lhs = int_alloc_inst.toRef(),	1015	// %2 = alloc(*@Vector(N, T))
1310	.rhs = try pt.intRef(int_ty, 0),	1016	// %3 = alloc(*usize)
1311	} },	1017	// %4 = store(%2, <@Vector(N, T), undefined>)
1312	});	1018	// %5 = [addScalarizedShuffle]
1313	const index_alloc_inst = res_block.add(l, .{	1019	// %6 = [addScalarizedShuffle]
1314	.tag = .alloc,	1020	// %7 = load(%2)
1315	.data = .{ .ty = .ptr_usize },	1021	// %8 = br(%1, %7)
1316	});	1022	// })
1317	_ = res_block.add(l, .{
1318	.tag = .store,
1319	.data = .{ .bin_op = .{
1320	.lhs = index_alloc_inst.toRef(),
1321	.rhs = .zero_usize,
1322	} },
1323	});
1324		1023
1325	var loop: Loop = .init(l, &res_block);	1024	var sfba_state = std.heap.stackFallback(512, gpa);
1326	loop.block = .init(res_block.stealRemainingCapacity());	1025	const sfba = sfba_state.get();
1327	{
1328	const cur_index_inst = loop.block.add(l, .{
1329	.tag = .load,
1330	.data = .{ .ty_op = .{
1331	.ty = .usize_type,
1332	.operand = index_alloc_inst.toRef(),
1333	} },
1334	});
1335	const cur_int_inst = loop.block.add(l, .{
1336	.tag = .bit_or,
1337	.data = .{ .bin_op = .{
1338	.lhs = loop.block.add(l, .{
1339	.tag = .shl_exact,
1340	.data = .{ .bin_op = .{
1341	.lhs = loop.block.add(l, .{
1342	.tag = .intcast,
1343	.data = .{ .ty_op = .{
1344	.ty = Air.internedToRef(int_ty.toIntern()),
1345	.operand = loop.block.addBitCast(l, elem_int_ty, loop.block.add(l, .{
1346	.tag = .array_elem_val,
1347	.data = .{ .bin_op = .{
1348	.lhs = orig_ty_op.operand,
1349	.rhs = cur_index_inst.toRef(),
1350	} },
1351	}).toRef()),
1352	} },
1353	}).toRef(),
1354	.rhs = loop.block.add(l, .{
1355	.tag = .mul,
1356	.data = .{ .bin_op = .{
1357	.lhs = loop.block.add(l, .{
1358	.tag = .intcast,
1359	.data = .{ .ty_op = .{
1360	.ty = Air.internedToRef(shift_ty.toIntern()),
1361	.operand = cur_index_inst.toRef(),
1362	} },
1363	}).toRef(),
1364	.rhs = try pt.intRef(shift_ty, elem_bits),
1365	} },
1366	}).toRef(),
1367	} },
1368	}).toRef(),
1369	.rhs = loop.block.add(l, .{
1370	.tag = .load,
1371	.data = .{ .ty_op = .{
1372	.ty = Air.internedToRef(int_ty.toIntern()),
1373	.operand = int_alloc_inst.toRef(),
1374	} },
1375	}).toRef(),
1376	} },
1377	});
1378		1026
1379	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(	1027	const out_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1380	l,	1028	defer sfba.free(out_idxs_buf);
1381	.lt,	1029
1382	cur_index_inst.toRef(),	1030	const in_idxs_buf = try sfba.alloc(InternPool.Index, shuffle.mask.len);
1383	try pt.intRef(.usize, operand_ty.arrayLen(zcu) - 1),	1031	defer sfba.free(in_idxs_buf);
1384	.{},	1032
1385	)).toRef(), &loop.block, .{});	1033	// Iterate `shuffle.mask` before doing anything, because modifying AIR invalidates it.
1386	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());	1034	const out_idxs_a, const in_idxs_a, const out_idxs_b, const in_idxs_b = idxs: {
1387	{	1035	var n: usize = 0;
1388	_ = loop_cond_br.then_block.add(l, .{	1036	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
1389	.tag = .store,	1037	.undef, .b_elem => {},
1390	.data = .{ .bin_op = .{	1038	.a_elem => \|in_idx\| {
1391	.lhs = int_alloc_inst.toRef(),	1039	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1392	.rhs = cur_int_inst.toRef(),	1040	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1393	} },	1041	n += 1;
1394	});	1042	},
1395	_ = loop_cond_br.then_block.add(l, .{	1043	};
1396	.tag = .store,	1044	const a_len = n;
1397	.data = .{ .bin_op = .{	1045	for (shuffle.mask, 0..) \|mask, out_idx\| switch (mask.unwrap()) {
1398	.lhs = index_alloc_inst.toRef(),	1046	.undef, .a_elem => {},
1399	.rhs = loop_cond_br.then_block.add(l, .{	1047	.b_elem => \|in_idx\| {
1400	.tag = .add,	1048	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1401	.data = .{ .bin_op = .{	1049	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1402	.lhs = cur_index_inst.toRef(),	1050	n += 1;
1403	.rhs = .one_usize,	1051	},
1404	} },	1052	};
1405	}).toRef(),	1053	break :idxs .{
1406	} },	1054	out_idxs_buf[0..a_len],
1407	});	1055	in_idxs_buf[0..a_len],
1408	_ = loop_cond_br.then_block.add(l, .{	1056	out_idxs_buf[a_len..n],
1409	.tag = .repeat,	1057	in_idxs_buf[a_len..n],
1410	.data = .{ .repeat = .{ .loop_inst = loop.inst } },	1058	};
1411	});	1059	};
1412	}	1060
1413	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());	1061	var inst_buf: [7]Air.Inst.Index = undefined;
1414	_ = loop_cond_br.else_block.add(l, .{	1062	var main_block: Block = .init(&inst_buf);
1415	.tag = .br,	1063	try l.air_instructions.ensureUnusedCapacity(gpa, 33);
1416	.data = .{ .br = .{	1064
1417	.block_inst = orig_inst,	1065	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
1418	.operand = loop_cond_br.else_block.addBitCast(l, res_ty, cur_int_inst.toRef()),	1066	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1419	} },	1067
1420	});	1068	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.undefValue(shuffle.result_ty)));
1421	try loop_cond_br.finish(l);	1069
1422	}	1070	if (out_idxs_a.len == 0) {
1423	try loop.finish(l);	1071	_ = main_block.stealCapacity(1);
		1072	} else {
		1073	try l.addScalarizedShuffle(
		1074	&main_block,
		1075	shuffle.operand_a,
		1076	result_ptr,
		1077	index_ptr,
		1078	out_idxs_a,
		1079	in_idxs_a,
		1080	);
		1081	}
		1082
		1083	if (out_idxs_b.len == 0) {
		1084	_ = main_block.stealCapacity(1);
		1085	} else {
		1086	try l.addScalarizedShuffle(
		1087	&main_block,
		1088	shuffle.operand_b,
		1089	result_ptr,
		1090	index_ptr,
		1091	out_idxs_b,
		1092	in_idxs_b,
		1093	);
1424	}	1094	}
		1095
		1096	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
		1097	main_block.addBr(l, orig_inst, result_val);
		1098
1425	return .{ .ty_pl = .{	1099	return .{ .ty_pl = .{
1426	.ty = Air.internedToRef(res_ty.toIntern()),	1100	.ty = .fromType(shuffle.result_ty),
1427	.payload = try l.addBlockBody(res_block.body()),	1101	.payload = try l.addBlockBody(main_block.body()),
1428	} };	1102	} };
1429	}	1103	}
1430	fn scalarizeBitcastResultArrayBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	1104	/// Adds code to `parent_block` which behaves like this loop:
		1105	///
		1106	/// for (out_idxs, in_idxs) \|i, j\| result_vec_ptr[i] = operand_vec[j];
		1107	///
		1108	/// The actual AIR adds exactly one instruction to `parent_block` itself and 14 instructions
		1109	/// overall, and is as follows:
		1110	///
		1111	/// %1 = block(void, {
		1112	/// %2 = store(index_ptr, @zero_usize)
		1113	/// %3 = loop({
		1114	/// %4 = load(index_ptr)
		1115	/// %5 = ptr_elem_val(out_idxs_ptr, %4)
		1116	/// %6 = ptr_elem_val(in_idxs_ptr, %4)
		1117	/// %7 = legalize_vec_elem_val(operand_vec, %6)
		1118	/// %8 = legalize_vec_store_elem(result_vec_ptr, %4, %7)
		1119	/// %9 = cmp_eq(%4, <usize, out_idxs.len-1>)
		1120	/// %10 = cond_br(%9, {
		1121	/// %11 = br(%1, @void_value)
		1122	/// }, {
		1123	/// %12 = add(%4, @one_usize)
		1124	/// %13 = store(index_ptr, %12)
		1125	/// %14 = repeat(%3)
		1126	/// })
		1127	/// })
		1128	/// })
		1129	///
		1130	/// The caller is responsible for reserving space in `l.air_instructions`.
		1131	fn addScalarizedShuffle(
		1132	l: *Legalize,
		1133	parent_block: *Block,
		1134	operand_vec: Air.Inst.Ref,
		1135	result_vec_ptr: Air.Inst.Ref,
		1136	index_ptr: Air.Inst.Ref,
		1137	out_idxs: []const InternPool.Index,
		1138	in_idxs: []const InternPool.Index,
		1139	) Error!void {
1431	const pt = l.pt;	1140	const pt = l.pt;
1432	const zcu = pt.zcu;
1433		1141
1434	const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;	1142	assert(out_idxs.len == in_idxs.len);
1435	const res_ty = orig_ty_op.ty.toType();	1143	const n = out_idxs.len;
1436	const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1437	const int_ty = try pt.intType(.unsigned, int_bits);
1438	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1439	const res_elem_ty = res_ty.childType(zcu);
1440	const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1441	const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1442		1144
1443	var inst_buf: [20]Air.Inst.Index = undefined;	1145	const idxs_ty = try pt.arrayType(.{ .len = n, .child = .usize_type });
1444	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	1146	const idxs_ptr_ty = try pt.singleConstPtrType(idxs_ty);
		1147	const manyptr_usize_ty = try pt.manyConstPtrType(.usize);
1445		1148
1446	var res_block: Block = .init(&inst_buf);	1149	const out_idxs_ptr = try pt.intern(.{ .ptr = .{
1447	{	1150	.ty = manyptr_usize_ty.toIntern(),
1448	const res_alloc_inst = res_block.add(l, .{	1151	.base_addr = .{ .uav = .{
1449	.tag = .alloc,	1152	.val = (try pt.aggregateValue(idxs_ty, out_idxs)).toIntern(),
1450	.data = .{ .ty = try pt.singleMutPtrType(res_ty) },	1153	.orig_ty = idxs_ptr_ty.toIntern(),
1451	});	1154	} },
1452	const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);	1155	.byte_offset = 0,
1453	const index_alloc_inst = res_block.add(l, .{	1156	} });
1454	.tag = .alloc,	1157	const in_idxs_ptr = try pt.intern(.{ .ptr = .{
1455	.data = .{ .ty = .ptr_usize },	1158	.ty = manyptr_usize_ty.toIntern(),
1456	});	1159	.base_addr = .{ .uav = .{
1457	_ = res_block.add(l, .{	1160	.val = (try pt.aggregateValue(idxs_ty, in_idxs)).toIntern(),
1458	.tag = .store,	1161	.orig_ty = idxs_ptr_ty.toIntern(),
1459	.data = .{ .bin_op = .{	1162	} },
1460	.lhs = index_alloc_inst.toRef(),	1163	.byte_offset = 0,
1461	.rhs = .zero_usize,	1164	} });
1462	} },
1463	});
1464		1165
1465	var loop: Loop = .init(l, &res_block);	1166	const main_block_inst = parent_block.add(l, .{
1466	loop.block = .init(res_block.stealRemainingCapacity());	1167	.tag = .block,
1467	{	1168	.data = .{ .ty_pl = .{
1468	const cur_index_inst = loop.block.add(l, .{	1169	.ty = .void_type,
1469	.tag = .load,	1170	.payload = undefined,
1470	.data = .{ .ty_op = .{	1171	} },
1471	.ty = .usize_type,	1172	});
1472	.operand = index_alloc_inst.toRef(),
1473	} },
1474	});
1475	_ = loop.block.add(l, .{
1476	.tag = .store,
1477	.data = .{ .bin_op = .{
1478	.lhs = loop.block.add(l, .{
1479	.tag = .ptr_elem_ptr,
1480	.data = .{ .ty_pl = .{
1481	.ty = Air.internedToRef((try pt.singleMutPtrType(res_elem_ty)).toIntern()),
1482	.payload = try l.addExtra(Air.Bin, .{
1483	.lhs = res_alloc_inst.toRef(),
1484	.rhs = cur_index_inst.toRef(),
1485	}),
1486	} },
1487	}).toRef(),
1488	.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1489	.tag = .trunc,
1490	.data = .{ .ty_op = .{
1491	.ty = Air.internedToRef(elem_int_ty.toIntern()),
1492	.operand = loop.block.add(l, .{
1493	.tag = .shr,
1494	.data = .{ .bin_op = .{
1495	.lhs = int_ref,
1496	.rhs = loop.block.add(l, .{
1497	.tag = .mul,
1498	.data = .{ .bin_op = .{
1499	.lhs = loop.block.add(l, .{
1500	.tag = .intcast,
1501	.data = .{ .ty_op = .{
1502	.ty = Air.internedToRef(shift_ty.toIntern()),
1503	.operand = cur_index_inst.toRef(),
1504	} },
1505	}).toRef(),
1506	.rhs = try pt.intRef(shift_ty, elem_bits),
1507	} },
1508	}).toRef(),
1509	} },
1510	}).toRef(),
1511	} },
1512	}).toRef()),
1513	} },
1514	});
1515		1173
1516	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(	1174	var inst_buf: [13]Air.Inst.Index = undefined;
1517	l,	1175	var main_block: Block = .init(&inst_buf);
1518	.lt,	1176
1519	cur_index_inst.toRef(),	1177	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1520	try pt.intRef(.usize, res_ty.arrayLen(zcu) - 1),	1178
1521	.{},	1179	var loop: Loop = .init(l, &main_block);
1522	)).toRef(), &loop.block, .{});	1180	loop.block = .init(main_block.stealRemainingCapacity());
1523	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());	1181
1524	{	1182	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1525	_ = loop_cond_br.then_block.add(l, .{	1183	const in_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(in_idxs_ptr), index_val).toRef();
1526	.tag = .store,	1184	const out_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(out_idxs_ptr), index_val).toRef();
1527	.data = .{ .bin_op = .{	1185
1528	.lhs = index_alloc_inst.toRef(),	1186	const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, operand_vec, in_idx_val).toRef();
1529	.rhs = loop_cond_br.then_block.add(l, .{	1187	_ = loop.block.add(l, .{
1530	.tag = .add,	1188	.tag = .legalize_vec_store_elem,
1531	.data = .{ .bin_op = .{	1189	.data = .{ .pl_op = .{
1532	.lhs = cur_index_inst.toRef(),	1190	.operand = result_vec_ptr,
1533	.rhs = .one_usize,	1191	.payload = try l.addExtra(Air.Bin, .{
1534	} },	1192	.lhs = out_idx_val,
1535	}).toRef(),	1193	.rhs = elem_val,
1536	} },	1194	}),
1537	});	1195	} },
1538	_ = loop_cond_br.then_block.add(l, .{	1196	});
1539	.tag = .repeat,	1197
1540	.data = .{ .repeat = .{ .loop_inst = loop.inst } },	1198	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, n - 1))).toRef();
1541	});	1199	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1542	}	1200	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1543	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());	1201	condbr.then_block.addBr(l, main_block_inst, .void_value);
1544	_ = loop_cond_br.else_block.add(l, .{	1202
1545	.tag = .br,	1203	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1546	.data = .{ .br = .{	1204	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1547	.block_inst = orig_inst,	1205	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1548	.operand = loop_cond_br.else_block.add(l, .{	1206	_ = condbr.else_block.add(l, .{
1549	.tag = .load,	1207	.tag = .repeat,
1550	.data = .{ .ty_op = .{	1208	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1551	.ty = Air.internedToRef(res_ty.toIntern()),	1209	});
1552	.operand = res_alloc_inst.toRef(),	1210
1553	} },	1211	try condbr.finish(l);
1554	}).toRef(),	1212	try loop.finish(l);
1555	} },	1213
1556	});	1214	const inst_data = l.air_instructions.items(.data);
1557	try loop_cond_br.finish(l);	1215	inst_data[@intFromEnum(main_block_inst)].ty_pl.payload = try l.addBlockBody(main_block.body());
1558	}
1559	try loop.finish(l);
1560	}
1561	return .{ .ty_pl = .{
1562	.ty = Air.internedToRef(res_ty.toIntern()),
1563	.payload = try l.addBlockBody(res_block.body()),
1564	} };
1565	}	1216	}
1566	fn scalarizeBitcastResultVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	1217	fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1567	const pt = l.pt;	1218	const pt = l.pt;
1568	const zcu = pt.zcu;	1219	const zcu = pt.zcu;
1569		1220
1570	const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;	1221	const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
1571	const res_ty = orig_ty_op.ty.toType();
1572	const int_bits: u16 = @intCast(res_ty.bitSize(zcu));
1573	const int_ty = try pt.intType(.unsigned, int_bits);
1574	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, int_bits));
1575	const res_elem_ty = res_ty.childType(zcu);
1576	const elem_bits: u16 = @intCast(res_elem_ty.bitSize(zcu));
1577	const elem_int_ty = try pt.intType(.unsigned, elem_bits);
1578		1222
1579	var inst_buf: [19]Air.Inst.Index = undefined;	1223	const dest_ty = ty_op.ty.toType();
		1224	const dest_legal = switch (dest_ty.zigTypeTag(zcu)) {
		1225	else => true,
		1226	.array, .vector => legal: {
		1227	if (dest_ty.arrayLen(zcu) == 1) break :legal true;
		1228	const dest_elem_ty = dest_ty.childType(zcu);
		1229	break :legal dest_elem_ty.bitSize(zcu) == 8 * dest_elem_ty.abiSize(zcu);
		1230	},
		1231	};
		1232
		1233	const operand_ty = l.typeOf(ty_op.operand);
		1234	const operand_legal = switch (operand_ty.zigTypeTag(zcu)) {
		1235	else => true,
		1236	.array, .vector => legal: {
		1237	if (operand_ty.arrayLen(zcu) == 1) break :legal true;
		1238	const operand_elem_ty = operand_ty.childType(zcu);
		1239	break :legal operand_elem_ty.bitSize(zcu) == 8 * operand_elem_ty.abiSize(zcu);
		1240	},
		1241	};
		1242
		1243	if (dest_legal and operand_legal) return null;
		1244
		1245	if (!operand_legal and !dest_legal and operand_ty.arrayLen(zcu) == dest_ty.arrayLen(zcu)) {
		1246	// from_ty and to_ty are both arrays or vectors of types with the same bit size,
		1247	// so we can do an elementwise bitcast.
		1248	return try l.scalarizeBlockPayload(orig_inst, .ty_op);
		1249	}
		1250
		1251	// Fallback path. Our strategy is to use an unsigned integer type as an intermediate
		1252	// "bag of bits" representation which can be manipulated by bitwise operations.
		1253
		1254	const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
		1255	assert(operand_ty.bitSize(zcu) == num_bits);
		1256	const uint_ty = try pt.intType(.unsigned, num_bits);
		1257	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
		1258
		1259	var inst_buf: [39]Air.Inst.Index = undefined;
		1260	var main_block: Block = .init(&inst_buf);
1580	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	1261	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1581		1262
1582	var res_block: Block = .init(&inst_buf);	1263	// First, convert `operand_ty` to `uint_ty` (`uN`).
1583	{	1264
1584	const res_alloc_inst = res_block.add(l, .{	1265	const uint_val: Air.Inst.Ref = uint_val: {
1585	.tag = .alloc,	1266	if (operand_legal) {
1586	.data = .{ .ty = try pt.singleMutPtrType(res_ty) },	1267	_ = main_block.stealCapacity(19);
1587	});	1268	break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1588	const int_ref = res_block.addBitCast(l, int_ty, orig_ty_op.operand);	1269	}
1589	const index_alloc_inst = res_block.add(l, .{	1270
1590	.tag = .alloc,	1271	// %1 = block({
1591	.data = .{ .ty = .ptr_usize },	1272	// %2 = alloc(*usize)
1592	});	1273	// %3 = alloc(*uN)
1593	_ = res_block.add(l, .{	1274	// %4 = store(%2, <usize, operand_len>)
1594	.tag = .store,	1275	// %5 = store(%3, <uN, 0>)
1595	.data = .{ .bin_op = .{	1276	// %6 = loop({
1596	.lhs = index_alloc_inst.toRef(),	1277	// %7 = load(%2)
1597	.rhs = .zero_usize,	1278	// %8 = array_elem_val(orig_operand, %7)
		1279	// %9 = bitcast(uE, %8)
		1280	// %10 = intcast(uN, %9)
		1281	// %11 = load(%3)
		1282	// %12 = shl_exact(%11, <uS, E>)
		1283	// %13 = bit_or(%12, %10)
		1284	// %14 = cmp_eq(%4, @zero_usize)
		1285	// %15 = cond_br(%14, {
		1286	// %16 = br(%1, %13)
		1287	// }, {
		1288	// %17 = store(%3, %13)
		1289	// %18 = sub(%7, @one_usize)
		1290	// %19 = store(%2, %18)
		1291	// %20 = repeat(%6)
		1292	// })
		1293	// })
		1294	// })
		1295
		1296	const elem_bits = operand_ty.childType(zcu).bitSize(zcu);
		1297	const elem_bits_val = try pt.intValue(shift_ty, elem_bits);
		1298	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
		1299
		1300	const uint_block_inst = main_block.add(l, .{
		1301	.tag = .block,
		1302	.data = .{ .ty_pl = .{
		1303	.ty = .fromType(uint_ty),
		1304	.payload = undefined,
1598	} },	1305	} },
1599	});	1306	});
		1307	var uint_block: Block = .init(main_block.stealCapacity(19));
		1308
		1309	const index_ptr = uint_block.addTy(l, .alloc, .ptr_usize).toRef();
		1310	const result_ptr = uint_block.addTy(l, .alloc, try pt.singleMutPtrType(uint_ty)).toRef();
		1311	_ = uint_block.addBinOp(
		1312	l,
		1313	.store,
		1314	index_ptr,
		1315	.fromValue(try pt.intValue(.usize, operand_ty.arrayLen(zcu))),
		1316	);
		1317	_ = uint_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.intValue(uint_ty, 0)));
		1318
		1319	var loop: Loop = .init(l, &uint_block);
		1320	loop.block = .init(uint_block.stealRemainingCapacity());
		1321
		1322	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
		1323	const raw_elem = loop.block.addBinOp(
		1324	l,
		1325	if (operand_ty.zigTypeTag(zcu) == .vector) .legalize_vec_elem_val else .array_elem_val,
		1326	ty_op.operand,
		1327	index_val,
		1328	).toRef();
		1329	const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
		1330	const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
		1331	const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
		1332	const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
		1333	const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
		1334
		1335	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .zero_usize).toRef();
		1336	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
		1337
		1338	condbr.then_block = .init(loop.block.stealRemainingCapacity());
		1339	condbr.then_block.addBr(l, uint_block_inst, new_result);
1600		1340
1601	var loop: Loop = .init(l, &res_block);	1341	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1602	loop.block = .init(res_block.stealRemainingCapacity());	1342	_ = condbr.else_block.addBinOp(l, .store, result_ptr, new_result);
1603	{	1343	const new_index_val = condbr.else_block.addBinOp(l, .sub, index_val, .one_usize).toRef();
1604	const cur_index_inst = loop.block.add(l, .{	1344	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1605	.tag = .load,	1345	_ = condbr.else_block.add(l, .{
1606	.data = .{ .ty_op = .{	1346	.tag = .repeat,
1607	.ty = .usize_type,	1347	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1608	.operand = index_alloc_inst.toRef(),	1348	});
1609	} },
1610	});
1611	_ = loop.block.add(l, .{
1612	.tag = .vector_store_elem,
1613	.data = .{ .vector_store_elem = .{
1614	.vector_ptr = res_alloc_inst.toRef(),
1615	.payload = try l.addExtra(Air.Bin, .{
1616	.lhs = cur_index_inst.toRef(),
1617	.rhs = loop.block.addBitCast(l, res_elem_ty, loop.block.add(l, .{
1618	.tag = .trunc,
1619	.data = .{ .ty_op = .{
1620	.ty = Air.internedToRef(elem_int_ty.toIntern()),
1621	.operand = loop.block.add(l, .{
1622	.tag = .shr,
1623	.data = .{ .bin_op = .{
1624	.lhs = int_ref,
1625	.rhs = loop.block.add(l, .{
1626	.tag = .mul,
1627	.data = .{ .bin_op = .{
1628	.lhs = loop.block.add(l, .{
1629	.tag = .intcast,
1630	.data = .{ .ty_op = .{
1631	.ty = Air.internedToRef(shift_ty.toIntern()),
1632	.operand = cur_index_inst.toRef(),
1633	} },
1634	}).toRef(),
1635	.rhs = try pt.intRef(shift_ty, elem_bits),
1636	} },
1637	}).toRef(),
1638	} },
1639	}).toRef(),
1640	} },
1641	}).toRef()),
1642	}),
1643	} },
1644	});
1645		1349
1646	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(	1350	try condbr.finish(l);
1647	l,	1351	try loop.finish(l);
1648	.lt,	1352
1649	cur_index_inst.toRef(),	1353	const inst_data = l.air_instructions.items(.data);
1650	try pt.intRef(.usize, res_ty.vectorLen(zcu) - 1),	1354	inst_data[@intFromEnum(uint_block_inst)].ty_pl.payload = try l.addBlockBody(uint_block.body());
1651	.{},	1355
1652	)).toRef(), &loop.block, .{});	1356	break :uint_val uint_block_inst.toRef();
1653	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());	1357	};
1654	{	1358
1655	_ = loop_cond_br.then_block.add(l, .{	1359	// Now convert `uint_ty` (`uN`) to `dest_ty`.
1656	.tag = .store,	1360
1657	.data = .{ .bin_op = .{	1361	if (dest_legal) {
1658	.lhs = index_alloc_inst.toRef(),	1362	_ = main_block.stealCapacity(17);
1659	.rhs = loop_cond_br.then_block.add(l, .{	1363	const result = main_block.addBitCast(l, dest_ty, uint_val);
1660	.tag = .add,	1364	main_block.addBr(l, orig_inst, result);
1661	.data = .{ .bin_op = .{	1365	} else {
1662	.lhs = cur_index_inst.toRef(),	1366	// %1 = alloc(*usize)
1663	.rhs = .one_usize,	1367	// %2 = alloc(*@Vector(N, Result))
1664	} },	1368	// %3 = store(%1, @zero_usize)
1665	}).toRef(),	1369	// %4 = loop({
		1370	// %5 = load(%1)
		1371	// %6 = mul(%5, <usize, E>)
		1372	// %7 = intcast(uS, %6)
		1373	// %8 = shr(uint_val, %7)
		1374	// %9 = trunc(uE, %8)
		1375	// %10 = bitcast(Result, %9)
		1376	// %11 = legalize_vec_store_elem(%2, %5, %10)
		1377	// %12 = cmp_eq(%5, <usize, vec_len>)
		1378	// %13 = cond_br(%12, {
		1379	// %14 = load(%2)
		1380	// %15 = br(%0, %14)
		1381	// }, {
		1382	// %16 = add(%5, @one_usize)
		1383	// %17 = store(%1, %16)
		1384	// %18 = repeat(%4)
		1385	// })
		1386	// })
		1387	//
		1388	// The result might be an array, in which case `legalize_vec_store_elem`
		1389	// becomes `ptr_elem_ptr` followed by `store`.
		1390
		1391	const elem_ty = dest_ty.childType(zcu);
		1392	const elem_bits = elem_ty.bitSize(zcu);
		1393	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
		1394
		1395	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
		1396	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(dest_ty)).toRef();
		1397	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
		1398
		1399	var loop: Loop = .init(l, &main_block);
		1400	loop.block = .init(main_block.stealRemainingCapacity());
		1401
		1402	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
		1403	const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
		1404	const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
		1405	const shifted_uint = loop.block.addBinOp(l, .shr, index_val, casted_bit_offset).toRef();
		1406	const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
		1407	const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
		1408	switch (dest_ty.zigTypeTag(zcu)) {
		1409	.array => {
		1410	const elem_ptr = loop.block.add(l, .{
		1411	.tag = .ptr_elem_ptr,
		1412	.data = .{ .ty_pl = .{
		1413	.ty = .fromType(try pt.singleMutPtrType(elem_ty)),
		1414	.payload = try l.addExtra(Air.Bin, .{
		1415	.lhs = result_ptr,
		1416	.rhs = index_val,
		1417	}),
		1418	} },
		1419	}).toRef();
		1420	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
		1421	},
		1422	.vector => {
		1423	_ = loop.block.add(l, .{
		1424	.tag = .legalize_vec_store_elem,
		1425	.data = .{ .pl_op = .{
		1426	.operand = result_ptr,
		1427	.payload = try l.addExtra(Air.Bin, .{
		1428	.lhs = index_val,
		1429	.rhs = elem_val,
		1430	}),
1666	} },	1431	} },
1667	});	1432	});
1668	_ = loop_cond_br.then_block.add(l, .{	1433	_ = loop.block.stealCapacity(1);
1669	.tag = .repeat,	1434	},
1670	.data = .{ .repeat = .{ .loop_inst = loop.inst } },	1435	else => unreachable,
1671	});
1672	}
1673	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1674	_ = loop_cond_br.else_block.add(l, .{
1675	.tag = .br,
1676	.data = .{ .br = .{
1677	.block_inst = orig_inst,
1678	.operand = loop_cond_br.else_block.add(l, .{
1679	.tag = .load,
1680	.data = .{ .ty_op = .{
1681	.ty = Air.internedToRef(res_ty.toIntern()),
1682	.operand = res_alloc_inst.toRef(),
1683	} },
1684	}).toRef(),
1685	} },
1686	});
1687	try loop_cond_br.finish(l);
1688	}	1436	}
		1437
		1438	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, dest_ty.arrayLen(zcu) - 1))).toRef();
		1439
		1440	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
		1441
		1442	condbr.then_block = .init(loop.block.stealRemainingCapacity());
		1443	const result_val = condbr.then_block.addTyOp(l, .load, dest_ty, result_ptr).toRef();
		1444	condbr.then_block.addBr(l, orig_inst, result_val);
		1445
		1446	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
		1447	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
		1448	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
		1449	_ = condbr.else_block.add(l, .{
		1450	.tag = .repeat,
		1451	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
		1452	});
		1453
		1454	try condbr.finish(l);
1689	try loop.finish(l);	1455	try loop.finish(l);
1690	}	1456	}
		1457
1691	return .{ .ty_pl = .{	1458	return .{ .ty_pl = .{
1692	.ty = Air.internedToRef(res_ty.toIntern()),	1459	.ty = .fromType(dest_ty),
1693	.payload = try l.addBlockBody(res_block.body()),	1460	.payload = try l.addBlockBody(main_block.body()),
1694	} };	1461	} };
1695	}	1462	}
1696	fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	1463	fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
...	@@ -1698,169 +1465,145 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!	...	@@ -1698,169 +1465,145 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!
1698	const zcu = pt.zcu;	1465	const zcu = pt.zcu;
1699		1466
1700	const orig = l.air_instructions.get(@intFromEnum(orig_inst));	1467	const orig = l.air_instructions.get(@intFromEnum(orig_inst));
1701	const res_ty = l.typeOfIndex(orig_inst);	1468	const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1702	const wrapped_res_ty = res_ty.fieldType(0, zcu);	1469
1703	const wrapped_res_scalar_ty = wrapped_res_ty.childType(zcu);	1470	const vec_tuple_ty = l.typeOfIndex(orig_inst);
1704	const res_len = wrapped_res_ty.vectorLen(zcu);	1471	const vec_int_ty = vec_tuple_ty.fieldType(0, zcu);
		1472	const vec_overflow_ty = vec_tuple_ty.fieldType(1, zcu);
		1473
		1474	assert(l.typeOf(orig_operands.lhs).toIntern() == vec_int_ty.toIntern());
		1475	if (orig.tag != .shl_with_overflow) {
		1476	assert(l.typeOf(orig_operands.rhs).toIntern() == vec_int_ty.toIntern());
		1477	}
		1478
		1479	const scalar_int_ty = vec_int_ty.childType(zcu);
		1480	const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
		1481
		1482	// %1 = block(struct { @Vector(N, Int), @Vector(N, u1) }, {
		1483	// %2 = alloc(*usize)
		1484	// %3 = alloc(*struct { @Vector(N, Int), @Vector(N, u1) })
		1485	// %4 = struct_field_ptr_index_0(*@Vector(N, Int), %3)
		1486	// %5 = struct_field_ptr_index_1(*@Vector(N, u1), %3)
		1487	// %6 = store(%2, @zero_usize)
		1488	// %7 = loop({
		1489	// %8 = load(%2)
		1490	// %9 = legalize_vec_elem_val(orig_lhs, %8)
		1491	// %10 = legalize_vec_elem_val(orig_rhs, %8)
		1492	// %11 = ???_with_overflow(struct { Int, u1 }, %9, %10)
		1493	// %12 = struct_field_val(%11, 0)
		1494	// %13 = struct_field_val(%11, 1)
		1495	// %14 = legalize_vec_store_elem(%4, %8, %12)
		1496	// %15 = legalize_vec_store_elem(%4, %8, %13)
		1497	// %16 = cmp_eq(%8, <usize, N-1>)
		1498	// %17 = cond_br(%16, {
		1499	// %18 = load(%3)
		1500	// %19 = br(%1, %18)
		1501	// }, {
		1502	// %20 = add(%8, @one_usize)
		1503	// %21 = store(%2, %20)
		1504	// %22 = repeat(%7)
		1505	// })
		1506	// })
		1507	// })
		1508
		1509	const elems_len = vec_int_ty.vectorLen(zcu);
1705		1510
1706	var inst_buf: [21]Air.Inst.Index = undefined;	1511	var inst_buf: [21]Air.Inst.Index = undefined;
		1512	var main_block: Block = .init(&inst_buf);
1707	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	1513	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1708		1514
1709	var res_block: Block = .init(&inst_buf);	1515	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1710	{	1516	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(vec_tuple_ty)).toRef();
1711	const res_alloc_inst = res_block.add(l, .{	1517	const result_int_ptr = main_block.addTyOp(
1712	.tag = .alloc,	1518	l,
1713	.data = .{ .ty = try pt.singleMutPtrType(res_ty) },	1519	.struct_field_ptr_index_0,
1714	});	1520	try pt.singleMutPtrType(vec_int_ty),
1715	const ptr_wrapped_res_inst = res_block.add(l, .{	1521	result_ptr,
1716	.tag = .struct_field_ptr_index_0,	1522	).toRef();
1717	.data = .{ .ty_op = .{	1523	const result_overflow_ptr = main_block.addTyOp(
1718	.ty = Air.internedToRef((try pt.singleMutPtrType(wrapped_res_ty)).toIntern()),	1524	l,
1719	.operand = res_alloc_inst.toRef(),	1525	.struct_field_ptr_index_1,
1720	} },	1526	try pt.singleMutPtrType(vec_overflow_ty),
1721	});	1527	result_ptr,
1722	const ptr_overflow_res_inst = res_block.add(l, .{	1528	).toRef();
1723	.tag = .struct_field_ptr_index_1,	1529
1724	.data = .{ .ty_op = .{	1530	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1725	.ty = Air.internedToRef((try pt.singleMutPtrType(res_ty.fieldType(1, zcu))).toIntern()),	1531
1726	.operand = res_alloc_inst.toRef(),	1532	var loop: Loop = .init(l, &main_block);
1727	} },	1533	loop.block = .init(main_block.stealRemainingCapacity());
1728	});	1534
1729	const index_alloc_inst = res_block.add(l, .{	1535	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1730	.tag = .alloc,	1536	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.lhs, index_val).toRef();
1731	.data = .{ .ty = .ptr_usize },	1537	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.rhs, index_val).toRef();
1732	});	1538	const elem_result = loop.block.add(l, .{
1733	_ = res_block.add(l, .{	1539	.tag = orig.tag,
1734	.tag = .store,	1540	.data = .{ .ty_pl = .{
1735	.data = .{ .bin_op = .{	1541	.ty = .fromType(scalar_tuple_ty),
1736	.lhs = index_alloc_inst.toRef(),	1542	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1737	.rhs = .zero_usize,	1543	} },
1738	} },	1544	}).toRef();
1739	});	1545	const int_elem = loop.block.add(l, .{
		1546	.tag = .struct_field_val,
		1547	.data = .{ .ty_pl = .{
		1548	.ty = .fromType(scalar_int_ty),
		1549	.payload = try l.addExtra(Air.StructField, .{
		1550	.struct_operand = elem_result,
		1551	.field_index = 0,
		1552	}),
		1553	} },
		1554	}).toRef();
		1555	const overflow_elem = loop.block.add(l, .{
		1556	.tag = .struct_field_val,
		1557	.data = .{ .ty_pl = .{
		1558	.ty = .u1_type,
		1559	.payload = try l.addExtra(Air.StructField, .{
		1560	.struct_operand = elem_result,
		1561	.field_index = 1,
		1562	}),
		1563	} },
		1564	}).toRef();
		1565	_ = loop.block.add(l, .{
		1566	.tag = .legalize_vec_store_elem,
		1567	.data = .{ .pl_op = .{
		1568	.operand = result_int_ptr,
		1569	.payload = try l.addExtra(Air.Bin, .{
		1570	.lhs = index_val,
		1571	.rhs = int_elem,
		1572	}),
		1573	} },
		1574	});
		1575	_ = loop.block.add(l, .{
		1576	.tag = .legalize_vec_store_elem,
		1577	.data = .{ .pl_op = .{
		1578	.operand = result_overflow_ptr,
		1579	.payload = try l.addExtra(Air.Bin, .{
		1580	.lhs = index_val,
		1581	.rhs = overflow_elem,
		1582	}),
		1583	} },
		1584	});
1740		1585
1741	var loop: Loop = .init(l, &res_block);	1586	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, elems_len - 1))).toRef();
1742	loop.block = .init(res_block.stealRemainingCapacity());	1587	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1743	{	1588
1744	const cur_index_inst = loop.block.add(l, .{	1589	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1745	.tag = .load,	1590	const result_val = condbr.then_block.addTyOp(l, .load, vec_tuple_ty, result_ptr).toRef();
1746	.data = .{ .ty_op = .{	1591	condbr.then_block.addBr(l, orig_inst, result_val);
1747	.ty = .usize_type,	1592
1748	.operand = index_alloc_inst.toRef(),	1593	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1749	} },	1594	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1750	});	1595	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1751	const extra = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;	1596	_ = condbr.else_block.add(l, .{
1752	const res_elem = loop.block.add(l, .{	1597	.tag = .repeat,
1753	.tag = orig.tag,	1598	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1754	.data = .{ .ty_pl = .{	1599	});
1755	.ty = Air.internedToRef(try zcu.intern_pool.getTupleType(zcu.gpa, pt.tid, .{	1600
1756	.types = &.{ wrapped_res_scalar_ty.toIntern(), .u1_type },	1601	try condbr.finish(l);
1757	.values = &(.{.none} ** 2),	1602	try loop.finish(l);
1758	})),
1759	.payload = try l.addExtra(Air.Bin, .{
1760	.lhs = loop.block.add(l, .{
1761	.tag = .array_elem_val,
1762	.data = .{ .bin_op = .{
1763	.lhs = extra.lhs,
1764	.rhs = cur_index_inst.toRef(),
1765	} },
1766	}).toRef(),
1767	.rhs = loop.block.add(l, .{
1768	.tag = .array_elem_val,
1769	.data = .{ .bin_op = .{
1770	.lhs = extra.rhs,
1771	.rhs = cur_index_inst.toRef(),
1772	} },
1773	}).toRef(),
1774	}),
1775	} },
1776	});
1777	_ = loop.block.add(l, .{
1778	.tag = .vector_store_elem,
1779	.data = .{ .vector_store_elem = .{
1780	.vector_ptr = ptr_overflow_res_inst.toRef(),
1781	.payload = try l.addExtra(Air.Bin, .{
1782	.lhs = cur_index_inst.toRef(),
1783	.rhs = loop.block.add(l, .{
1784	.tag = .struct_field_val,
1785	.data = .{ .ty_pl = .{
1786	.ty = .u1_type,
1787	.payload = try l.addExtra(Air.StructField, .{
1788	.struct_operand = res_elem.toRef(),
1789	.field_index = 1,
1790	}),
1791	} },
1792	}).toRef(),
1793	}),
1794	} },
1795	});
1796	_ = loop.block.add(l, .{
1797	.tag = .vector_store_elem,
1798	.data = .{ .vector_store_elem = .{
1799	.vector_ptr = ptr_wrapped_res_inst.toRef(),
1800	.payload = try l.addExtra(Air.Bin, .{
1801	.lhs = cur_index_inst.toRef(),
1802	.rhs = loop.block.add(l, .{
1803	.tag = .struct_field_val,
1804	.data = .{ .ty_pl = .{
1805	.ty = Air.internedToRef(wrapped_res_scalar_ty.toIntern()),
1806	.payload = try l.addExtra(Air.StructField, .{
1807	.struct_operand = res_elem.toRef(),
1808	.field_index = 0,
1809	}),
1810	} },
1811	}).toRef(),
1812	}),
1813	} },
1814	});
1815		1603
1816	var loop_cond_br: CondBr = .init(l, (try loop.block.addCmp(
1817	l,
1818	.lt,
1819	cur_index_inst.toRef(),
1820	try pt.intRef(.usize, res_len - 1),
1821	.{},
1822	)).toRef(), &loop.block, .{});
1823	loop_cond_br.then_block = .init(loop.block.stealRemainingCapacity());
1824	{
1825	_ = loop_cond_br.then_block.add(l, .{
1826	.tag = .store,
1827	.data = .{ .bin_op = .{
1828	.lhs = index_alloc_inst.toRef(),
1829	.rhs = loop_cond_br.then_block.add(l, .{
1830	.tag = .add,
1831	.data = .{ .bin_op = .{
1832	.lhs = cur_index_inst.toRef(),
1833	.rhs = .one_usize,
1834	} },
1835	}).toRef(),
1836	} },
1837	});
1838	_ = loop_cond_br.then_block.add(l, .{
1839	.tag = .repeat,
1840	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1841	});
1842	}
1843	loop_cond_br.else_block = .init(loop_cond_br.then_block.stealRemainingCapacity());
1844	_ = loop_cond_br.else_block.add(l, .{
1845	.tag = .br,
1846	.data = .{ .br = .{
1847	.block_inst = orig_inst,
1848	.operand = loop_cond_br.else_block.add(l, .{
1849	.tag = .load,
1850	.data = .{ .ty_op = .{
1851	.ty = Air.internedToRef(res_ty.toIntern()),
1852	.operand = res_alloc_inst.toRef(),
1853	} },
1854	}).toRef(),
1855	} },
1856	});
1857	try loop_cond_br.finish(l);
1858	}
1859	try loop.finish(l);
1860	}
1861	return .{ .ty_pl = .{	1604	return .{ .ty_pl = .{
1862	.ty = Air.internedToRef(res_ty.toIntern()),	1605	.ty = .fromType(vec_tuple_ty),
1863	.payload = try l.addBlockBody(res_block.body()),	1606	.payload = try l.addBlockBody(main_block.body()),
1864	} };	1607	} };
1865	}	1608	}
1866		1609
...	@@ -2047,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz	...	@@ -2047,7 +1790,7 @@ fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimiz
2047		1790
2048	// We emit 9 instructions in the worst case.	1791	// We emit 9 instructions in the worst case.
2049	var inst_buf: [9]Air.Inst.Index = undefined;	1792	var inst_buf: [9]Air.Inst.Index = undefined;
2050	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	1793	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2051	var main_block: Block = .init(&inst_buf);	1794	var main_block: Block = .init(&inst_buf);
2052		1795
2053	// This check is a bit annoying because of floating-point rounding and the fact that this	1796	// This check is a bit annoying because of floating-point rounding and the fact that this
...	@@ -2231,37 +1974,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_	...	@@ -2231,37 +1974,6 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2231	} };	1974	} };
2232	}	1975	}
2233		1976
2234	fn expandBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2235	const pt = l.pt;
2236	const zcu = pt.zcu;
2237	const ip = &zcu.intern_pool;
2238
2239	const orig_ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
2240	const res_ty = orig_ty_op.ty.toType();
2241	const res_ty_key = ip.indexToKey(res_ty.toIntern());
2242	const operand_ty = l.typeOf(orig_ty_op.operand);
2243	const operand_ty_key = ip.indexToKey(operand_ty.toIntern());
2244	_ = res_ty_key;
2245	_ = operand_ty_key;
2246
2247	var inst_buf: [1]Air.Inst.Index = undefined;
2248	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2249
2250	var res_block: Block = .init(&inst_buf);
2251	{
2252	_ = res_block.add(l, .{
2253	.tag = .br,
2254	.data = .{ .br = .{
2255	.block_inst = orig_inst,
2256	.operand = try pt.undefRef(res_ty),
2257	} },
2258	});
2259	}
2260	return .{ .ty_pl = .{
2261	.ty = Air.internedToRef(res_ty.toIntern()),
2262	.payload = try l.addBlockBody(res_block.body()),
2263	} };
2264	}
2265	fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	1977	fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2266	const pt = l.pt;	1978	const pt = l.pt;
2267	const zcu = pt.zcu;	1979	const zcu = pt.zcu;
...	@@ -2431,89 +2143,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err	...	@@ -2431,89 +2143,73 @@ fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Err
2431	const field_ty = orig_ty_pl.ty.toType();	2143	const field_ty = orig_ty_pl.ty.toType();
2432	const agg_ty = l.typeOf(orig_extra.struct_operand);	2144	const agg_ty = l.typeOf(orig_extra.struct_operand);
2433		2145
		2146	const agg_bits: u16 = @intCast(agg_ty.bitSize(zcu));
		2147	const bit_offset = zcu.structPackedFieldBitOffset(zcu.typeToStruct(agg_ty).?, orig_extra.field_index);
		2148
		2149	const agg_int_ty = try pt.intType(.unsigned, agg_bits);
		2150	const field_int_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
		2151
		2152	const agg_shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, agg_bits));
		2153	const bit_offset_ref: Air.Inst.Ref = .fromValue(try pt.intValue(agg_shift_ty, bit_offset));
		2154
2434	var inst_buf: [5]Air.Inst.Index = undefined;	2155	var inst_buf: [5]Air.Inst.Index = undefined;
		2156	var main_block: Block = .init(&inst_buf);
2435	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);	2157	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2436		2158
2437	var res_block: Block = .init(&inst_buf);	2159	const agg_int = main_block.addBitCast(l, agg_int_ty, orig_extra.struct_operand);
2438	{	2160	const shifted_agg_int = main_block.addBinOp(l, .shr, agg_int, bit_offset_ref).toRef();
2439	const agg_alloc_inst = res_block.add(l, .{	2161	const field_int = main_block.addTyOp(l, .trunc, field_int_ty, shifted_agg_int).toRef();
2440	.tag = .alloc,	2162	const field_val = main_block.addBitCast(l, field_ty, field_int);
2441	.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },	2163	main_block.addBr(l, orig_inst, field_val);
2442	});	2164
2443	_ = res_block.add(l, .{
2444	.tag = .store,
2445	.data = .{ .bin_op = .{
2446	.lhs = agg_alloc_inst.toRef(),
2447	.rhs = orig_extra.struct_operand,
2448	} },
2449	});
2450	_ = res_block.add(l, .{
2451	.tag = .br,
2452	.data = .{ .br = .{
2453	.block_inst = orig_inst,
2454	.operand = res_block.add(l, .{
2455	.tag = .load,
2456	.data = .{ .ty_op = .{
2457	.ty = Air.internedToRef(field_ty.toIntern()),
2458	.operand = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), orig_extra.field_index)).toRef(),
2459	} },
2460	}).toRef(),
2461	} },
2462	});
2463	}
2464	return .{ .ty_pl = .{	2165	return .{ .ty_pl = .{
2465	.ty = Air.internedToRef(field_ty.toIntern()),	2166	.ty = .fromType(field_ty),
2466	.payload = try l.addBlockBody(res_block.body()),	2167	.payload = try l.addBlockBody(main_block.body()),
2467	} };	2168	} };
2468	}	2169	}
2469	fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {	2170	fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2470	const pt = l.pt;	2171	const pt = l.pt;
2471	const zcu = pt.zcu;	2172	const zcu = pt.zcu;
		2173	const gpa = zcu.gpa;
2472		2174
2473	const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;	2175	const orig_ty_pl = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_pl;
2474	const field_ty = orig_ty_pl.ty.toType();
2475	const agg_ty = orig_ty_pl.ty.toType();	2176	const agg_ty = orig_ty_pl.ty.toType();
2476	const agg_field_count = agg_ty.structFieldCount(zcu);	2177	const agg_field_count = agg_ty.structFieldCount(zcu);
2477		2178
2478	const ExpectedContents = [1 + 2 * 32 + 2]Air.Inst.Index;	2179	var sfba_state = std.heap.stackFallback(@sizeOf([4 * 32 + 2]Air.Inst.Index), gpa);
2479	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =	2180	const sfba = sfba_state.get();
2480	std.heap.stackFallback(@sizeOf(ExpectedContents), zcu.gpa);
2481	const gpa = stack.get();
2482		2181
2483	const inst_buf = try gpa.alloc(Air.Inst.Index, 1 + 2 * agg_field_count + 2);	2182	const inst_buf = try sfba.alloc(Air.Inst.Index, 4 * agg_field_count + 2);
2484	defer gpa.free(inst_buf);	2183	defer sfba.free(inst_buf);
2485	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2486		2184
2487	var res_block: Block = .init(inst_buf);	2185	var main_block: Block = .init(inst_buf);
2488	{	2186	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2489	const agg_alloc_inst = res_block.add(l, .{	2187
2490	.tag = .alloc,	2188	const num_bits: u16 = @intCast(agg_ty.bitSize(zcu));
2491	.data = .{ .ty = try pt.singleMutPtrType(agg_ty) },	2189	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
2492	});	2190	const uint_ty = try pt.intType(.unsigned, num_bits);
2493	for (0..agg_field_count, orig_ty_pl.payload..) \|field_index, extra_index\| _ = res_block.add(l, .{	2191	var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
2494	.tag = .store,	2192
2495	.data = .{ .bin_op = .{	2193	var field_idx = agg_field_count;
2496	.lhs = (try res_block.addStructFieldPtr(l, agg_alloc_inst.toRef(), field_index)).toRef(),	2194	while (field_idx > 0) {
2497	.rhs = @enumFromInt(l.air_extra.items[extra_index]),	2195	field_idx -= 1;
2498	} },	2196	const field_ty = agg_ty.fieldType(field_idx, zcu);
2499	});	2197	const field_uint_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
2500	_ = res_block.add(l, .{	2198	const field_bit_size_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, field_ty.bitSize(zcu)));
2501	.tag = .br,	2199	const field_val: Air.Inst.Ref = @enumFromInt(l.air_extra.items[orig_ty_pl.payload + field_idx]);
2502	.data = .{ .br = .{	2200
2503	.block_inst = orig_inst,	2201	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2504	.operand = res_block.add(l, .{	2202	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2505	.tag = .load,	2203	const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
2506	.data = .{ .ty_op = .{	2204	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2507	.ty = Air.internedToRef(field_ty.toIntern()),
2508	.operand = agg_alloc_inst.toRef(),
2509	} },
2510	}).toRef(),
2511	} },
2512	});
2513	}	2205	}
		2206
		2207	const result = main_block.addBitCast(l, agg_ty, cur_uint);
		2208	main_block.addBr(l, orig_inst, result);
		2209
2514	return .{ .ty_pl = .{	2210	return .{ .ty_pl = .{
2515	.ty = Air.internedToRef(field_ty.toIntern()),	2211	.ty = .fromType(agg_ty),
2516	.payload = try l.addBlockBody(res_block.body()),	2212	.payload = try l.addBlockBody(main_block.body()),
2517	} };	2213	} };
2518	}	2214	}
2519		2215
...	@@ -2571,6 +2267,36 @@ const Block = struct {	...	@@ -2571,6 +2267,36 @@ const Block = struct {
2571	b.len += 1;	2267	b.len += 1;
2572	return inst;	2268	return inst;
2573	}	2269	}
		2270	fn addBr(b: Block, l: Legalize, target: Air.Inst.Index, operand: Air.Inst.Ref) void {
		2271	_ = b.add(l, .{
		2272	.tag = .br,
		2273	.data = .{ .br = .{ .block_inst = target, .operand = operand } },
		2274	});
		2275	}
		2276	fn addTy(b: Block, l: Legalize, tag: Air.Inst.Tag, ty: Type) Air.Inst.Index {
		2277	return b.add(l, .{ .tag = tag, .data = .{ .ty = ty } });
		2278	}
		2279	fn addBinOp(b: Block, l: Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
		2280	return b.add(l, .{
		2281	.tag = tag,
		2282	.data = .{ .bin_op = .{ .lhs = lhs, .rhs = rhs } },
		2283	});
		2284	}
		2285	fn addUnOp(b: Block, l: Legalize, tag: Air.Inst.Tag, operand: Air.Inst.Ref) Air.Inst.Index {
		2286	return b.add(l, .{
		2287	.tag = tag,
		2288	.data = .{ .un_op = operand },
		2289	});
		2290	}
		2291	fn addTyOp(b: Block, l: Legalize, tag: Air.Inst.Tag, ty: Type, operand: Air.Inst.Ref) Air.Inst.Index {
		2292	return b.add(l, .{
		2293	.tag = tag,
		2294	.data = .{ .ty_op = .{
		2295	.ty = .fromType(ty),
		2296	.operand = operand,
		2297	} },
		2298	});
		2299	}
2574		2300
2575	/// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,	2301	/// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
2576	/// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.	2302	/// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
...	@@ -2625,14 +2351,27 @@ const Block = struct {	...	@@ -2625,14 +2351,27 @@ const Block = struct {
2625	} },	2351	} },
2626	});	2352	});
2627	}	2353	}
		2354	return addCmpScalar(b, l, op, lhs, rhs, opts.optimized);
		2355	}
		2356
		2357	/// Similar to `addCmp`, but for scalars only. Unlike `addCmp`, this function is
		2358	/// infallible, because it doesn't need to add entries to `extra`.
		2359	fn addCmpScalar(
		2360	b: *Block,
		2361	l: *Legalize,
		2362	op: std.math.CompareOperator,
		2363	lhs: Air.Inst.Ref,
		2364	rhs: Air.Inst.Ref,
		2365	optimized: bool,
		2366	) Air.Inst.Index {
2628	return b.add(l, .{	2367	return b.add(l, .{
2629	.tag = switch (op) {	2368	.tag = switch (op) {
2630	.lt => if (opts.optimized) .cmp_lt_optimized else .cmp_lt,	2369	.lt => if (optimized) .cmp_lt_optimized else .cmp_lt,
2631	.lte => if (opts.optimized) .cmp_lte_optimized else .cmp_lte,	2370	.lte => if (optimized) .cmp_lte_optimized else .cmp_lte,
2632	.eq => if (opts.optimized) .cmp_eq_optimized else .cmp_eq,	2371	.eq => if (optimized) .cmp_eq_optimized else .cmp_eq,
2633	.gte => if (opts.optimized) .cmp_gte_optimized else .cmp_gte,	2372	.gte => if (optimized) .cmp_gte_optimized else .cmp_gte,
2634	.gt => if (opts.optimized) .cmp_gt_optimized else .cmp_gt,	2373	.gt => if (optimized) .cmp_gt_optimized else .cmp_gt,
2635	.neq => if (opts.optimized) .cmp_neq_optimized else .cmp_neq,	2374	.neq => if (optimized) .cmp_neq_optimized else .cmp_neq,
2636	},	2375	},
2637	.data = .{ .bin_op = .{	2376	.data = .{ .bin_op = .{
2638	.lhs = lhs,	2377	.lhs = lhs,
...	@@ -2641,93 +2380,6 @@ const Block = struct {	...	@@ -2641,93 +2380,6 @@ const Block = struct {
2641	});	2380	});
2642	}	2381	}
2643		2382
2644	/// Adds a `struct_field_ptr*` instruction to `b`. This is a fairly thin wrapper around `add`
2645	/// that selects the optimized instruction encoding to use, although it does compute the
2646	/// proper field pointer type.
2647	fn addStructFieldPtr(
2648	b: *Block,
2649	l: *Legalize,
2650	struct_operand: Air.Inst.Ref,
2651	field_index: usize,
2652	) Error!Air.Inst.Index {
2653	const pt = l.pt;
2654	const zcu = pt.zcu;
2655
2656	const agg_ptr_ty = l.typeOf(struct_operand);
2657	const agg_ptr_info = agg_ptr_ty.ptrInfo(zcu);
2658	const agg_ty: Type = .fromInterned(agg_ptr_info.child);
2659	const agg_ptr_align = switch (agg_ptr_info.flags.alignment) {
2660	.none => agg_ty.abiAlignment(zcu),
2661	else => \|agg_ptr_align\| agg_ptr_align,
2662	};
2663	const agg_layout = agg_ty.containerLayout(zcu);
2664	const field_ty = agg_ty.fieldType(field_index, zcu);
2665	var field_ptr_info: InternPool.Key.PtrType = .{
2666	.child = field_ty.toIntern(),
2667	.flags = .{
2668	.is_const = agg_ptr_info.flags.is_const,
2669	.is_volatile = agg_ptr_info.flags.is_volatile,
2670	.address_space = agg_ptr_info.flags.address_space,
2671	},
2672	};
2673	field_ptr_info.flags.alignment = field_ptr_align: switch (agg_layout) {
2674	.auto => agg_ty.fieldAlignment(field_index, zcu).min(agg_ptr_align),
2675	.@"extern" => switch (agg_ty.zigTypeTag(zcu)) {
2676	else => unreachable,
2677	.@"struct" => .fromLog2Units(@min(
2678	agg_ptr_align.toLog2Units(),
2679	@ctz(agg_ty.structFieldOffset(field_index, zcu)),
2680	)),
2681	.@"union" => agg_ptr_align,
2682	},
2683	.@"packed" => switch (agg_ty.zigTypeTag(zcu)) {
2684	else => unreachable,
2685	.@"struct" => {
2686	const packed_offset = agg_ty.packedStructFieldPtrInfo(agg_ptr_ty, @intCast(field_index), pt);
2687	field_ptr_info.packed_offset = packed_offset;
2688	break :field_ptr_align agg_ptr_align;
2689	},
2690	.@"union" => {
2691	field_ptr_info.packed_offset = .{
2692	.host_size = switch (agg_ptr_info.packed_offset.host_size) {
2693	0 => @intCast(agg_ty.abiSize(zcu)),
2694	else => \|host_size\| host_size,
2695	},
2696	.bit_offset = agg_ptr_info.packed_offset.bit_offset,
2697	};
2698	break :field_ptr_align agg_ptr_align;
2699	},
2700	},
2701	};
2702	const field_ptr_ty = try pt.ptrType(field_ptr_info);
2703	const field_ptr_ty_ref = Air.internedToRef(field_ptr_ty.toIntern());
2704	return switch (field_index) {
2705	inline 0...3 => \|ct_field_index\| b.add(l, .{
2706	.tag = switch (ct_field_index) {
2707	0 => .struct_field_ptr_index_0,
2708	1 => .struct_field_ptr_index_1,
2709	2 => .struct_field_ptr_index_2,
2710	3 => .struct_field_ptr_index_3,
2711	else => comptime unreachable,
2712	},
2713	.data = .{ .ty_op = .{
2714	.ty = field_ptr_ty_ref,
2715	.operand = struct_operand,
2716	} },
2717	}),
2718	else => b.add(l, .{
2719	.tag = .struct_field_ptr,
2720	.data = .{ .ty_pl = .{
2721	.ty = field_ptr_ty_ref,
2722	.payload = try l.addExtra(Air.StructField, .{
2723	.struct_operand = struct_operand,
2724	.field_index = @intCast(field_index),
2725	}),
2726	} },
2727	}),
2728	};
2729	}
2730
2731	/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for	2383	/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
2732	/// no-op casts.	2384	/// no-op casts.
2733	fn addBitCast(	2385	fn addBitCast(
...	@@ -2774,31 +2426,6 @@ const Block = struct {	...	@@ -2774,31 +2426,6 @@ const Block = struct {
2774	}	2426	}
2775	};	2427	};
2776		2428
2777	const Result = struct {
2778	inst: Air.Inst.Index,
2779	block: Block,
2780
2781	/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
2782	/// to initialize it.
2783	fn init(l: Legalize, ty: Type, parent_block: Block) Result {
2784	return .{
2785	.inst = parent_block.add(l, .{
2786	.tag = .block,
2787	.data = .{ .ty_pl = .{
2788	.ty = Air.internedToRef(ty.toIntern()),
2789	.payload = undefined,
2790	} },
2791	}),
2792	.block = undefined,
2793	};
2794	}
2795
2796	fn finish(res: Result, l: *Legalize) Error!void {
2797	const data = &l.air_instructions.items(.data)[@intFromEnum(res.inst)];
2798	data.ty_pl.payload = try l.addBlockBody(res.block.body());
2799	}
2800	};
2801
2802	const Loop = struct {	2429	const Loop = struct {
2803	inst: Air.Inst.Index,	2430	inst: Air.Inst.Index,
2804	block: Block,	2431	block: Block,

src/Air/Liveness.zig+7-6

 @@ -458,17 +458,12 @@ fn analyzeInst(
         .memset_safe,
         .memcpy,
         .memmove,
+        .legalize_vec_elem_val,
         => {
             const o = inst_datas[@intFromEnum(inst)].bin_op;
             return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none });
         },
-        .vector_store_elem => {
-            const o = inst_datas[@intFromEnum(inst)].vector_store_elem;
-            const extra = a.air.extraData(Air.Bin, o.payload).data;
-            return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs });
-        },
         .arg,
         .alloc,
         .ret_ptr,
 @@ -775,6 +770,12 @@ fn analyzeInst(
             const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
             return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, .none, .none });
         },
+        .legalize_vec_store_elem => {
+            const pl_op = inst_datas[@intFromEnum(inst)].pl_op;
+            const bin = a.air.extraData(Air.Bin, pl_op.payload).data;
+            return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });
+        },
     }
 }

src/Air/Liveness/Verify.zig+6-5

 @@ -272,6 +272,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
             .memset_safe,
             .memcpy,
             .memmove,
+            .legalize_vec_elem_val,
             => {
                 const bin_op = data[@intFromEnum(inst)].bin_op;
                 try self.verifyInstOperands(inst, .{ bin_op.lhs, bin_op.rhs, .none });
 @@ -322,11 +323,6 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
                 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
                 try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand });
             },
-            .vector_store_elem => {
-                const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem;
-                const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data;
-                try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs });
-            },
             .cmpxchg_strong,
             .cmpxchg_weak,
             => {
 @@ -582,6 +578,11 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
                 try self.verifyInst(inst);
             },
+            .legalize_vec_store_elem => {
+                const pl_op = data[@intFromEnum(inst)].pl_op;
+                const bin = self.air.extraData(Air.Bin, pl_op.payload).data;
+                try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
+            },
         }
     }
 }

src/Air/print.zig+14-12

 @@ -171,6 +171,7 @@ const Writer = struct {
             .memmove,
             .memset,
             .memset_safe,
+            .legalize_vec_elem_val,
             => try w.writeBinOp(s, inst),
             .is_null,
 @@ -330,8 +331,8 @@ const Writer = struct {
             .shuffle_two => try w.writeShuffleTwo(s, inst),
             .reduce, .reduce_optimized => try w.writeReduce(s, inst),
             .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
-            .vector_store_elem => try w.writeVectorStoreElem(s, inst),
             .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),
+            .legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),
             .work_item_id,
             .work_group_size,
 @@ -509,6 +510,18 @@ const Writer = struct {
         try w.writeOperand(s, inst, 2, pl_op.operand);
     }
+    fn writeLegalizeVecStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
+        const pl_op = w.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
+        const bin = w.air.extraData(Air.Bin, pl_op.payload).data;
+        try w.writeOperand(s, inst, 0, pl_op.operand);
+        try s.writeAll(", ");
+        try w.writeOperand(s, inst, 1, bin.lhs);
+        try s.writeAll(", ");
+        try w.writeOperand(s, inst, 2, bin.rhs);
+        try s.writeAll(", ");
+    }
     fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
         const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
         try w.writeType(s, unwrapped.result_ty);
 @@ -576,17 +589,6 @@ const Writer = struct {
         try w.writeOperand(s, inst, 1, extra.rhs);
     }
-    fn writeVectorStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
-        const data = w.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
-        const extra = w.air.extraData(Air.VectorCmp, data.payload).data;
-        try w.writeOperand(s, inst, 0, data.vector_ptr);
-        try s.writeAll(", ");
-        try w.writeOperand(s, inst, 1, extra.lhs);
-        try s.writeAll(", ");
-        try w.writeOperand(s, inst, 2, extra.rhs);
-    }
     fn writeRuntimeNavPtr(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
         const ip = &w.pt.zcu.intern_pool;
         const ty_nav = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;

src/Air/types_resolved.zig+2-7

 @@ -88,6 +88,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
             .atomic_store_monotonic,
             .atomic_store_release,
             .atomic_store_seq_cst,
+            .legalize_vec_elem_val,
             => {
                 if (!checkRef(data.bin_op.lhs, zcu)) return false;
                 if (!checkRef(data.bin_op.rhs, zcu)) return false;
 @@ -316,19 +317,13 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
                 if (!checkRef(data.prefetch.ptr, zcu)) return false;
             },
-            .vector_store_elem => {
-                const bin = air.extraData(Air.Bin, data.vector_store_elem.payload).data;
-                if (!checkRef(data.vector_store_elem.vector_ptr, zcu)) return false;
-                if (!checkRef(bin.lhs, zcu)) return false;
-                if (!checkRef(bin.rhs, zcu)) return false;
-            },
             .runtime_nav_ptr => {
                 if (!checkType(.fromInterned(data.ty_nav.ty), zcu)) return false;
             },
             .select,
             .mul_add,
+            .legalize_vec_store_elem,
             => {
                 const bin = air.extraData(Air.Bin, data.pl_op.payload).data;
                 if (!checkRef(data.pl_op.operand, zcu)) return false;

src/InternPool.zig+2-5

...	@@ -2104,7 +2104,6 @@ pub const Key = union(enum) {	...	@@ -2104,7 +2104,6 @@ pub const Key = union(enum) {
2104		2104
2105	pub const VectorIndex = enum(u16) {	2105	pub const VectorIndex = enum(u16) {
2106	none = std.math.maxInt(u16),	2106	none = std.math.maxInt(u16),
2107	runtime = std.math.maxInt(u16) - 1,
2108	_,	2107	_,
2109	};	2108	};
2110		2109
...	@@ -3739,10 +3738,8 @@ pub const LoadedStructType = struct {	...	@@ -3739,10 +3738,8 @@ pub const LoadedStructType = struct {
3739	return s.field_inits.get(ip)[i];	3738	return s.field_inits.get(ip)[i];
3740	}	3739	}
3741		3740
3742	/// Returns `none` in the case the struct is a tuple.	3741	pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) NullTerminatedString {
3743	pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) OptionalNullTerminatedString {	3742	return s.field_names.get(ip)[i];
3744	if (s.field_names.len == 0) return .none;
3745	return s.field_names.get(ip)[i].toOptional();
3746	}	3743	}
3747		3744
3748	pub fn fieldIsComptime(s: LoadedStructType, ip: *const InternPool, i: usize) bool {	3745	pub fn fieldIsComptime(s: LoadedStructType, ip: *const InternPool, i: usize) bool {

src/Sema.zig+31-94

...	@@ -15919,24 +15919,30 @@ fn zirOverflowArithmetic(	...	@@ -15919,24 +15919,30 @@ fn zirOverflowArithmetic(
15919	},	15919	},
15920	.mul_with_overflow => {	15920	.mul_with_overflow => {
15921	// If either of the arguments is zero, the result is zero and no overflow occured.	15921	// If either of the arguments is zero, the result is zero and no overflow occured.
15922	// If either of the arguments is one, the result is the other and no overflow occured.
15923	// Otherwise, if either of the arguments is undefined, both results are undefined.
15924	const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1);
15925	if (maybe_lhs_val) \|lhs_val\| {	15922	if (maybe_lhs_val) \|lhs_val\| {
15926	if (!lhs_val.isUndef(zcu)) {	15923	if (!lhs_val.isUndef(zcu) and try lhs_val.compareAllWithZeroSema(.eq, pt)) {
15927	if (try lhs_val.compareAllWithZeroSema(.eq, pt)) {	15924	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15928	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15929	} else if (try sema.compareAll(lhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) {
15930	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15931	}
15932	}	15925	}
15933	}	15926	}
15934
15935	if (maybe_rhs_val) \|rhs_val\| {	15927	if (maybe_rhs_val) \|rhs_val\| {
15936	if (!rhs_val.isUndef(zcu)) {	15928	if (!rhs_val.isUndef(zcu) and try rhs_val.compareAllWithZeroSema(.eq, pt)) {
15937	if (try rhs_val.compareAllWithZeroSema(.eq, pt)) {	15929	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
		15930	}
		15931	}
		15932	// If either of the arguments is one, the result is the other and no overflow occured.
		15933	const dest_scalar_ty = dest_ty.scalarType(zcu);
		15934	const dest_scalar_int = dest_scalar_ty.intInfo(zcu);
		15935	// We could still be working with i1, where '1' is not a legal value!
		15936	if (!(dest_scalar_int.bits == 1 and dest_scalar_int.signedness == .signed)) {
		15937	const scalar_one = try pt.intValue(dest_scalar_ty, 1);
		15938	const vec_one = try sema.splat(dest_ty, scalar_one);
		15939	if (maybe_lhs_val) \|lhs_val\| {
		15940	if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) {
15938	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };	15941	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs };
15939	} else if (try sema.compareAll(rhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) {	15942	}
		15943	}
		15944	if (maybe_rhs_val) \|rhs_val\| {
		15945	if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) {
15940	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };	15946	break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs };
15941	}	15947	}
15942	}	15948	}
...	@@ -15947,7 +15953,6 @@ fn zirOverflowArithmetic(	...	@@ -15947,7 +15953,6 @@ fn zirOverflowArithmetic(
15947	if (lhs_val.isUndef(zcu) or rhs_val.isUndef(zcu)) {	15953	if (lhs_val.isUndef(zcu) or rhs_val.isUndef(zcu)) {
15948	break :result .{ .overflow_bit = .undef, .wrapped = .undef };	15954	break :result .{ .overflow_bit = .undef, .wrapped = .undef };
15949	}	15955	}
15950
15951	const result = try arith.mulWithOverflow(sema, dest_ty, lhs_val, rhs_val);	15956	const result = try arith.mulWithOverflow(sema, dest_ty, lhs_val, rhs_val);
15952	break :result .{ .overflow_bit = result.overflow_bit, .wrapped = result.wrapped_result };	15957	break :result .{ .overflow_bit = result.overflow_bit, .wrapped = result.wrapped_result };
15953	}	15958	}
...	@@ -17751,10 +17756,7 @@ fn zirTypeInfo(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai	...	@@ -17751,10 +17756,7 @@ fn zirTypeInfo(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
17751	try ty.resolveStructFieldInits(pt);	17756	try ty.resolveStructFieldInits(pt);
17752		17757
17753	for (struct_field_vals, 0..) \|*field_val, field_index\| {	17758	for (struct_field_vals, 0..) \|*field_val, field_index\| {
17754	const field_name = if (struct_type.fieldName(ip, field_index).unwrap()) \|field_name\|	17759	const field_name = struct_type.fieldName(ip, field_index);
17755	field_name
17756	else
17757	try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
17758	const field_name_len = field_name.length(ip);	17760	const field_name_len = field_name.length(ip);
17759	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);	17761	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
17760	const field_init = struct_type.fieldInit(ip, field_index);	17762	const field_init = struct_type.fieldInit(ip, field_index);
...	@@ -28345,6 +28347,10 @@ fn elemPtrArray(	...	@@ -28345,6 +28347,10 @@ fn elemPtrArray(
28345	break :o index;	28347	break :o index;
28346	} else null;	28348	} else null;
28347		28349
		28350	if (offset == null and array_ty.zigTypeTag(zcu) == .vector) {
		28351	return sema.fail(block, elem_index_src, "vector index not comptime known", .{});
		28352	}
		28353
28348	const elem_ptr_ty = try array_ptr_ty.elemPtrType(offset, pt);	28354	const elem_ptr_ty = try array_ptr_ty.elemPtrType(offset, pt);
28349		28355
28350	if (maybe_undef_array_ptr_val) \|array_ptr_val\| {	28356	if (maybe_undef_array_ptr_val) \|array_ptr_val\| {
...	@@ -28362,10 +28368,6 @@ fn elemPtrArray(	...	@@ -28362,10 +28368,6 @@ fn elemPtrArray(
28362	try sema.validateRuntimeValue(block, array_ptr_src, array_ptr);	28368	try sema.validateRuntimeValue(block, array_ptr_src, array_ptr);
28363	}	28369	}
28364		28370
28365	if (offset == null and array_ty.zigTypeTag(zcu) == .vector) {
28366	return sema.fail(block, elem_index_src, "vector index not comptime known", .{});
28367	}
28368
28369	// Runtime check is only needed if unable to comptime check.	28371	// Runtime check is only needed if unable to comptime check.
28370	if (oob_safety and block.wantSafety() and offset == null) {	28372	if (oob_safety and block.wantSafety() and offset == null) {
28371	const len_inst = try pt.intRef(.usize, array_len);	28373	const len_inst = try pt.intRef(.usize, array_len);
...	@@ -30397,22 +30399,6 @@ fn storePtr2(	...	@@ -30397,22 +30399,6 @@ fn storePtr2(
30397		30399
30398	const is_ret = air_tag == .ret_ptr;	30400	const is_ret = air_tag == .ret_ptr;
30399		30401
30400	// Detect if we are storing an array operand to a bitcasted vector pointer.
30401	// If so, we instead reach through the bitcasted pointer to the vector pointer,
30402	// bitcast the array operand to a vector, and then lower this as a store of
30403	// a vector value to a vector pointer. This generally results in better code,
30404	// as well as working around an LLVM bug:
30405	// https://github.com/ziglang/zig/issues/11154
30406	if (sema.obtainBitCastedVectorPtr(ptr)) \|vector_ptr\| {
30407	const vector_ty = sema.typeOf(vector_ptr).childType(zcu);
30408	const vector = sema.coerceExtra(block, vector_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch \|err\| switch (err) {
30409	error.NotCoercible => unreachable,
30410	else => \|e\| return e,
30411	};
30412	try sema.storePtr2(block, src, vector_ptr, ptr_src, vector, operand_src, .store);
30413	return;
30414	}
30415
30416	const operand = sema.coerceExtra(block, elem_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch \|err\| switch (err) {	30402	const operand = sema.coerceExtra(block, elem_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch \|err\| switch (err) {
30417	error.NotCoercible => unreachable,	30403	error.NotCoercible => unreachable,
30418	else => \|e\| return e,	30404	else => \|e\| return e,
...	@@ -30445,29 +30431,6 @@ fn storePtr2(	...	@@ -30445,29 +30431,6 @@ fn storePtr2(
30445		30431
30446	try sema.requireRuntimeBlock(block, src, runtime_src);	30432	try sema.requireRuntimeBlock(block, src, runtime_src);
30447		30433
30448	if (ptr_ty.ptrInfo(zcu).flags.vector_index == .runtime) {
30449	const ptr_inst = ptr.toIndex().?;
30450	const air_tags = sema.air_instructions.items(.tag);
30451	if (air_tags[@intFromEnum(ptr_inst)] == .ptr_elem_ptr) {
30452	const ty_pl = sema.air_instructions.items(.data)[@intFromEnum(ptr_inst)].ty_pl;
30453	const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data;
30454	_ = try block.addInst(.{
30455	.tag = .vector_store_elem,
30456	.data = .{ .vector_store_elem = .{
30457	.vector_ptr = bin_op.lhs,
30458	.payload = try block.sema.addExtra(Air.Bin{
30459	.lhs = bin_op.rhs,
30460	.rhs = operand,
30461	}),
30462	} },
30463	});
30464	return;
30465	}
30466	return sema.fail(block, ptr_src, "unable to determine vector element index of type '{f}'", .{
30467	ptr_ty.fmt(pt),
30468	});
30469	}
30470
30471	const store_inst = if (is_ret)	30434	const store_inst = if (is_ret)
30472	try block.addBinOp(.store, ptr, operand)	30435	try block.addBinOp(.store, ptr, operand)
30473	else	30436	else
...	@@ -30567,37 +30530,6 @@ fn markMaybeComptimeAllocRuntime(sema: Sema, block: Block, alloc_inst: Air.Ins	...	@@ -30567,37 +30530,6 @@ fn markMaybeComptimeAllocRuntime(sema: Sema, block: Block, alloc_inst: Air.Ins
30567	}	30530	}
30568	}	30531	}
30569		30532
30570	/// Traverse an arbitrary number of bitcasted pointers and return the underyling vector
30571	/// pointer. Only if the final element type matches the vector element type, and the
30572	/// lengths match.
30573	fn obtainBitCastedVectorPtr(sema: *Sema, ptr: Air.Inst.Ref) ?Air.Inst.Ref {
30574	const pt = sema.pt;
30575	const zcu = pt.zcu;
30576	const array_ty = sema.typeOf(ptr).childType(zcu);
30577	if (array_ty.zigTypeTag(zcu) != .array) return null;
30578	var ptr_ref = ptr;
30579	var ptr_inst = ptr_ref.toIndex() orelse return null;
30580	const air_datas = sema.air_instructions.items(.data);
30581	const air_tags = sema.air_instructions.items(.tag);
30582	const vector_ty = while (air_tags[@intFromEnum(ptr_inst)] == .bitcast) {
30583	ptr_ref = air_datas[@intFromEnum(ptr_inst)].ty_op.operand;
30584	if (!sema.isKnownZigType(ptr_ref, .pointer)) return null;
30585	const child_ty = sema.typeOf(ptr_ref).childType(zcu);
30586	if (child_ty.zigTypeTag(zcu) == .vector) break child_ty;
30587	ptr_inst = ptr_ref.toIndex() orelse return null;
30588	} else return null;
30589
30590	// We have a pointer-to-array and a pointer-to-vector. If the elements and
30591	// lengths match, return the result.
30592	if (array_ty.childType(zcu).eql(vector_ty.childType(zcu), zcu) and
30593	array_ty.arrayLen(zcu) == vector_ty.vectorLen(zcu))
30594	{
30595	return ptr_ref;
30596	} else {
30597	return null;
30598	}
30599	}
30600
30601	/// Call when you have Value objects rather than Air instructions, and you want to	30533	/// Call when you have Value objects rather than Air instructions, and you want to
30602	/// assert the store must be done at comptime.	30534	/// assert the store must be done at comptime.
30603	fn storePtrVal(	30535	fn storePtrVal(
...	@@ -35577,8 +35509,13 @@ fn structFieldInits(	...	@@ -35577,8 +35509,13 @@ fn structFieldInits(
35577	const default_val = try sema.resolveConstValue(&block_scope, init_src, coerced, null);	35509	const default_val = try sema.resolveConstValue(&block_scope, init_src, coerced, null);
35578		35510
35579	if (default_val.canMutateComptimeVarState(zcu)) {	35511	if (default_val.canMutateComptimeVarState(zcu)) {
35580	const field_name = struct_type.fieldName(ip, field_i).unwrap().?;	35512	return sema.failWithContainsReferenceToComptimeVar(
35581	return sema.failWithContainsReferenceToComptimeVar(&block_scope, init_src, field_name, "field default value", default_val);	35513	&block_scope,
		35514	init_src,
		35515	struct_type.fieldName(ip, field_i),
		35516	"field default value",
		35517	default_val,
		35518	);
35582	}	35519	}
35583	struct_type.field_inits.get(ip)[field_i] = default_val.toIntern();	35520	struct_type.field_inits.get(ip)[field_i] = default_val.toIntern();
35584	}	35521	}

src/Sema/comptime_ptr_access.zig-2

src/Type.zig+4-6

...	@@ -198,9 +198,7 @@ pub fn print(ty: Type, writer: *std.Io.Writer, pt: Zcu.PerThread) std.Io.Writer.	...	@@ -198,9 +198,7 @@ pub fn print(ty: Type, writer: *std.Io.Writer, pt: Zcu.PerThread) std.Io.Writer.
198	info.packed_offset.bit_offset, info.packed_offset.host_size,	198	info.packed_offset.bit_offset, info.packed_offset.host_size,
199	});	199	});
200	}	200	}
201	if (info.flags.vector_index == .runtime) {	201	if (info.flags.vector_index != .none) {
202	try writer.writeAll(":?");
203	} else if (info.flags.vector_index != .none) {
204	try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)});	202	try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)});
205	}	203	}
206	try writer.writeAll(") ");	204	try writer.writeAll(") ");
...	@@ -3113,7 +3111,7 @@ pub fn enumTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {	...	@@ -3113,7 +3111,7 @@ pub fn enumTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {
3113	pub fn structFieldName(ty: Type, index: usize, zcu: *const Zcu) InternPool.OptionalNullTerminatedString {	3111	pub fn structFieldName(ty: Type, index: usize, zcu: *const Zcu) InternPool.OptionalNullTerminatedString {
3114	const ip = &zcu.intern_pool;	3112	const ip = &zcu.intern_pool;
3115	return switch (ip.indexToKey(ty.toIntern())) {	3113	return switch (ip.indexToKey(ty.toIntern())) {
3116	.struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index),	3114	.struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index).toOptional(),
3117	.tuple_type => .none,	3115	.tuple_type => .none,
3118	else => unreachable,	3116	else => unreachable,
3119	};	3117	};
...	@@ -3558,7 +3556,7 @@ pub fn packedStructFieldPtrInfo(	...	@@ -3558,7 +3556,7 @@ pub fn packedStructFieldPtrInfo(
3558	} else .{	3556	} else .{
3559	switch (zcu.comp.getZigBackend()) {	3557	switch (zcu.comp.getZigBackend()) {
3560	else => (running_bits + 7) / 8,	3558	else => (running_bits + 7) / 8,
3561	.stage2_x86_64 => @intCast(struct_ty.abiSize(zcu)),	3559	.stage2_x86_64, .stage2_c => @intCast(struct_ty.abiSize(zcu)),
3562	},	3560	},
3563	bit_offset,	3561	bit_offset,
3564	};	3562	};
...	@@ -3985,7 +3983,7 @@ pub fn elemPtrType(ptr_ty: Type, offset: ?usize, pt: Zcu.PerThread) !Type {	...	@@ -3985,7 +3983,7 @@ pub fn elemPtrType(ptr_ty: Type, offset: ?usize, pt: Zcu.PerThread) !Type {
3985	break :blk .{	3983	break :blk .{
3986	.host_size = @intCast(parent_ty.arrayLen(zcu)),	3984	.host_size = @intCast(parent_ty.arrayLen(zcu)),
3987	.alignment = parent_ty.abiAlignment(zcu),	3985	.alignment = parent_ty.abiAlignment(zcu),
3988	.vector_index = if (offset) \|some\| @enumFromInt(some) else .runtime,	3986	.vector_index = @enumFromInt(offset.?),
3989	};	3987	};
3990	} else .{};	3988	} else .{};
3991		3989

src/Value.zig+21-150

...	@@ -574,166 +574,37 @@ pub fn writeToPackedMemory(	...	@@ -574,166 +574,37 @@ pub fn writeToPackedMemory(
574	}	574	}
575	}	575	}
576		576
577	/// Load a Value from the contents of `buffer`.	577	/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.
578	///	578	///
579	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past	579	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
580	/// the end of the value in memory.	580	/// the end of the value in memory.
581	pub fn readFromMemory(	581	pub fn readUintFromMemory(
582	ty: Type,	582	ty: Type,
583	pt: Zcu.PerThread,	583	pt: Zcu.PerThread,
584	buffer: []const u8,	584	buffer: []const u8,
585	arena: Allocator,	585	arena: Allocator,
586	) error{	586	) Allocator.Error!Value {
587	IllDefinedMemoryLayout,
588	Unimplemented,
589	OutOfMemory,
590	}!Value {
591	const zcu = pt.zcu;	587	const zcu = pt.zcu;
592	const ip = &zcu.intern_pool;	588	const endian = zcu.getTarget().cpu.arch.endian();
593	const target = zcu.getTarget();
594	const endian = target.cpu.arch.endian();
595	switch (ty.zigTypeTag(zcu)) {
596	.void => return Value.void,
597	.bool => {
598	if (buffer[0] == 0) {
599	return Value.false;
600	} else {
601	return Value.true;
602	}
603	},
604	.int, .@"enum" => \|ty_tag\| {
605	const int_ty = switch (ty_tag) {
606	.int => ty,
607	.@"enum" => ty.intTagType(zcu),
608	else => unreachable,
609	};
610	const int_info = int_ty.intInfo(zcu);
611	const bits = int_info.bits;
612	const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
613	if (bits == 0 or buffer.len == 0) return zcu.getCoerced(try zcu.intValue(int_ty, 0), ty);
614		589
615	if (bits <= 64) switch (int_info.signedness) { // Fast path for integers <= u64	590	assert(ty.isUnsignedInt(zcu));
616	.signed => {	591	const bits = ty.intInfo(zcu).bits;
617	const val = std.mem.readVarInt(i64, buffer[0..byte_count], endian);	592	const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
618	const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
619	return zcu.getCoerced(try zcu.intValue(int_ty, result), ty);
620	},
621	.unsigned => {
622	const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
623	const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
624	return zcu.getCoerced(try zcu.intValue(int_ty, result), ty);
625	},
626	} else { // Slow path, we have to construct a big-int
627	const Limb = std.math.big.Limb;
628	const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
629	const limbs_buffer = try arena.alloc(Limb, limb_count);
630
631	var bigint = BigIntMutable.init(limbs_buffer, 0);
632	bigint.readTwosComplement(buffer[0..byte_count], bits, endian, int_info.signedness);
633	return zcu.getCoerced(try zcu.intValue_big(int_ty, bigint.toConst()), ty);
634	}
635	},
636	.float => return Value.fromInterned(try pt.intern(.{ .float = .{
637	.ty = ty.toIntern(),
638	.storage = switch (ty.floatBits(target)) {
639	16 => .{ .f16 = @bitCast(std.mem.readInt(u16, buffer[0..2], endian)) },
640	32 => .{ .f32 = @bitCast(std.mem.readInt(u32, buffer[0..4], endian)) },
641	64 => .{ .f64 = @bitCast(std.mem.readInt(u64, buffer[0..8], endian)) },
642	80 => .{ .f80 = @bitCast(std.mem.readInt(u80, buffer[0..10], endian)) },
643	128 => .{ .f128 = @bitCast(std.mem.readInt(u128, buffer[0..16], endian)) },
644	else => unreachable,
645	},
646	} })),
647	.array => {
648	const elem_ty = ty.childType(zcu);
649	const elem_size = elem_ty.abiSize(zcu);
650	const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
651	var offset: usize = 0;
652	for (elems) \|*elem\| {
653	elem.* = (try readFromMemory(elem_ty, zcu, buffer[offset..], arena)).toIntern();
654	offset += @intCast(elem_size);
655	}
656	return pt.aggregateValue(ty, elems);
657	},
658	.vector => {
659	// We use byte_count instead of abi_size here, so that any padding bytes
660	// follow the data bytes, on both big- and little-endian systems.
661	const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
662	return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
663	},
664	.@"struct" => {
665	const struct_type = zcu.typeToStruct(ty).?;
666	switch (struct_type.layout) {
667	.auto => unreachable, // Sema is supposed to have emitted a compile error already
668	.@"extern" => {
669	const field_types = struct_type.field_types;
670	const field_vals = try arena.alloc(InternPool.Index, field_types.len);
671	for (field_vals, 0..) \|*field_val, i\| {
672	const field_ty = Type.fromInterned(field_types.get(ip)[i]);
673	const off: usize = @intCast(ty.structFieldOffset(i, zcu));
674	const sz: usize = @intCast(field_ty.abiSize(zcu));
675	field_val.* = (try readFromMemory(field_ty, zcu, buffer[off..(off + sz)], arena)).toIntern();
676	}
677	return pt.aggregateValue(ty, field_vals);
678	},
679	.@"packed" => {
680	const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
681	return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
682	},
683	}
684	},
685	.error_set => {
686	const bits = zcu.errorSetBits();
687	const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
688	const int = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
689	const index = (int << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
690	const name = zcu.global_error_set.keys()[@intCast(index)];
691		593
692	return Value.fromInterned(try pt.intern(.{ .err = .{	594	assert(buffer.len >= byte_count);
693	.ty = ty.toIntern(),	595
694	.name = name,	596	if (bits <= 64) {
695	} }));	597	const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
696	},	598	const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
697	.@"union" => switch (ty.containerLayout(zcu)) {	599	return pt.intValue(ty, result);
698	.auto => return error.IllDefinedMemoryLayout,	600	} else {
699	.@"extern" => {	601	const Limb = std.math.big.Limb;
700	const union_size = ty.abiSize(zcu);	602	const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
701	const array_ty = try zcu.arrayType(.{ .len = union_size, .child = .u8_type });	603	const limbs_buffer = try arena.alloc(Limb, limb_count);
702	const val = (try readFromMemory(array_ty, zcu, buffer, arena)).toIntern();	604
703	return Value.fromInterned(try pt.internUnion(.{	605	var bigint: BigIntMutable = .init(limbs_buffer, 0);
704	.ty = ty.toIntern(),	606	bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);
705	.tag = .none,	607	return pt.intValue_big(ty, bigint.toConst());
706	.val = val,
707	}));
708	},
709	.@"packed" => {
710	const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
711	return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena);
712	},
713	},
714	.pointer => {
715	assert(!ty.isSlice(zcu)); // No well defined layout.
716	const int_val = try readFromMemory(Type.usize, zcu, buffer, arena);
717	return Value.fromInterned(try pt.intern(.{ .ptr = .{
718	.ty = ty.toIntern(),
719	.base_addr = .int,
720	.byte_offset = int_val.toUnsignedInt(zcu),
721	} }));
722	},
723	.optional => {
724	assert(ty.isPtrLikeOptional(zcu));
725	const child_ty = ty.optionalChild(zcu);
726	const child_val = try readFromMemory(child_ty, zcu, buffer, arena);
727	return Value.fromInterned(try pt.intern(.{ .opt = .{
728	.ty = ty.toIntern(),
729	.val = switch (child_val.orderAgainstZero(pt)) {
730	.lt => unreachable,
731	.eq => .none,
732	.gt => child_val.toIntern(),
733	},
734	} }));
735	},
736	else => return error.Unimplemented,
737	}	608	}
738	}	609	}
739		610

src/Zcu/PerThread.zig+22-4

 @@ -3512,7 +3512,6 @@ pub fn ptrType(pt: Zcu.PerThread, info: InternPool.Key.PtrType) Allocator.Error!
                 canon_info.packed_offset.host_size = 0;
             }
         },
-        .runtime => {},
         _ => assert(@intFromEnum(info.flags.vector_index) < info.packed_offset.host_size),
     }
 @@ -3663,21 +3662,40 @@ pub fn intRef(pt: Zcu.PerThread, ty: Type, x: anytype) Allocator.Error!Air.Inst.
 }
 pub fn intValue_big(pt: Zcu.PerThread, ty: Type, x: BigIntConst) Allocator.Error!Value {
-    return Value.fromInterned(try pt.intern(.{ .int = .{
+    if (ty.toIntern() != .comptime_int_type) {
+        const int_info = ty.intInfo(pt.zcu);
+        assert(x.fitsInTwosComp(int_info.signedness, int_info.bits));
+    }
+    return .fromInterned(try pt.intern(.{ .int = .{
         .ty = ty.toIntern(),
         .storage = .{ .big_int = x },
     } }));
 }
 pub fn intValue_u64(pt: Zcu.PerThread, ty: Type, x: u64) Allocator.Error!Value {
-    return Value.fromInterned(try pt.intern(.{ .int = .{
+    if (ty.toIntern() != .comptime_int_type and x != 0) {
+        const int_info = ty.intInfo(pt.zcu);
+        const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed);
+        assert(unsigned_bits >= std.math.log2(x) + 1);
+    }
+    return .fromInterned(try pt.intern(.{ .int = .{
         .ty = ty.toIntern(),
         .storage = .{ .u64 = x },
     } }));
 }
 pub fn intValue_i64(pt: Zcu.PerThread, ty: Type, x: i64) Allocator.Error!Value {
-    return Value.fromInterned(try pt.intern(.{ .int = .{
+    if (ty.toIntern() != .comptime_int_type and x != 0) {
+        const int_info = ty.intInfo(pt.zcu);
+        const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed);
+        if (x > 0) {
+            assert(unsigned_bits >= std.math.log2(x) + 1);
+        } else {
+            assert(int_info.signedness == .signed);
+            assert(unsigned_bits >= std.math.log2_int_ceil(u64, @abs(x)));
+        }
+    }
+    return .fromInterned(try pt.intern(.{ .int = .{
         .ty = ty.toIntern(),
         .storage = .{ .i64 = x },
     } }));

src/codegen/aarch64/Select.zig+9-12

 @@ -134,6 +134,10 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
     var air_inst_index = air_body[air_body_index];
     const initial_def_order_len = isel.def_order.count();
     air_tag: switch (air_tags[@intFromEnum(air_inst_index)]) {
+        // No "scalarize" legalizations are enabled, so these instructions never appear.
+        .legalize_vec_elem_val => unreachable,
+        .legalize_vec_store_elem => unreachable,
         .arg,
         .ret_addr,
         .frame_addr,
 @@ -826,18 +830,6 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
             try isel.analyzeUse(un_op);
-            air_body_index += 1;
-            air_inst_index = air_body[air_body_index];
-            continue :air_tag air_tags[@intFromEnum(air_inst_index)];
-        },
-        .vector_store_elem => {
-            const vector_store_elem = air_data[@intFromEnum(air_inst_index)].vector_store_elem;
-            const bin_op = isel.air.extraData(Air.Bin, vector_store_elem.payload).data;
-            try isel.analyzeUse(vector_store_elem.vector_ptr);
-            try isel.analyzeUse(bin_op.lhs);
-            try isel.analyzeUse(bin_op.rhs);
             air_body_index += 1;
             air_inst_index = air_body[air_body_index];
             continue :air_tag air_tags[@intFromEnum(air_inst_index)];
 @@ -962,6 +954,11 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
     };
     air_tag: switch (air.next().?) {
         else => |air_tag| return isel.fail("unimplemented {t}", .{air_tag}),
+        // No "scalarize" legalizations are enabled, so these instructions never appear.
+        .legalize_vec_elem_val => unreachable,
+        .legalize_vec_store_elem => unreachable,
         .arg => {
             const arg_vi = isel.live_values.fetchRemove(air.inst_index).?.value;
             defer arg_vi.deref(isel);

src/codegen/c.zig+135-450

...	@@ -37,6 +37,7 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {	...	@@ -37,6 +37,7 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
37	.expand_packed_load = true,	37	.expand_packed_load = true,
38	.expand_packed_store = true,	38	.expand_packed_store = true,
39	.expand_packed_struct_field_val = true,	39	.expand_packed_struct_field_val = true,
		40	.expand_packed_aggregate_init = true,
40	}),	41	}),
41	};	42	};
42	}	43	}
...	@@ -1392,114 +1393,21 @@ pub const DeclGen = struct {	...	@@ -1392,114 +1393,21 @@ pub const DeclGen = struct {
1392	try w.writeByte('}');	1393	try w.writeByte('}');
1393	},	1394	},
1394	.@"packed" => {	1395	.@"packed" => {
1395	const int_info = ty.intInfo(zcu);	1396	// https://github.com/ziglang/zig/issues/24657 will eliminate most of the
1396		1397	// following logic, leaving only the recursive `renderValue` call. Once
1397	const bits = Type.smallestUnsignedBits(int_info.bits - 1);	1398	// that proposal is implemented, a `packed struct` will literally be
1398	const bit_offset_ty = try pt.intType(.unsigned, bits);	1399	// represented in the InternPool by its comptime-known backing integer.
1399		1400	var arena: std.heap.ArenaAllocator = .init(zcu.gpa);
1400	var bit_offset: u64 = 0;	1401	defer arena.deinit();
1401	var eff_num_fields: usize = 0;	1402	const backing_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip));
1402		1403	const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu)));
1403	for (0..loaded_struct.field_types.len) \|field_index\| {	1404	val.writeToMemory(pt, buf) catch \|err\| switch (err) {
1404	const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);	1405	error.IllDefinedMemoryLayout => unreachable,
1405	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;	1406	error.OutOfMemory => \|e\| return e,
1406	eff_num_fields += 1;	1407	error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed struct value", .{}),
1407	}	1408	};
1408		1409	const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator());
1409	if (eff_num_fields == 0) {	1410	return dg.renderValue(w, backing_val, location);
1410	try w.writeByte('(');
1411	try dg.renderUndefValue(w, ty, location);
1412	try w.writeByte(')');
1413	} else if (ty.bitSize(zcu) > 64) {
1414	// zig_or_u128(zig_or_u128(zig_shl_u128(a, a_off), zig_shl_u128(b, b_off)), zig_shl_u128(c, c_off))
1415	var num_or = eff_num_fields - 1;
1416	while (num_or > 0) : (num_or -= 1) {
1417	try w.writeAll("zig_or_");
1418	try dg.renderTypeForBuiltinFnName(w, ty);
1419	try w.writeByte('(');
1420	}
1421
1422	var eff_index: usize = 0;
1423	var needs_closing_paren = false;
1424	for (0..loaded_struct.field_types.len) \|field_index\| {
1425	const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1426	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1427
1428	const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
1429	.bytes => \|bytes\| try pt.intern(.{ .int = .{
1430	.ty = field_ty.toIntern(),
1431	.storage = .{ .u64 = bytes.at(field_index, ip) },
1432	} }),
1433	.elems => \|elems\| elems[field_index],
1434	.repeated_elem => \|elem\| elem,
1435	};
1436	const cast_context = IntCastContext{ .value = .{ .value = Value.fromInterned(field_val) } };
1437	if (bit_offset != 0) {
1438	try w.writeAll("zig_shl_");
1439	try dg.renderTypeForBuiltinFnName(w, ty);
1440	try w.writeByte('(');
1441	try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument);
1442	try w.writeAll(", ");
1443	try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument);
1444	try w.writeByte(')');
1445	} else {
1446	try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument);
1447	}
1448
1449	if (needs_closing_paren) try w.writeByte(')');
1450	if (eff_index != eff_num_fields - 1) try w.writeAll(", ");
1451
1452	bit_offset += field_ty.bitSize(zcu);
1453	needs_closing_paren = true;
1454	eff_index += 1;
1455	}
1456	} else {
1457	try w.writeByte('(');
1458	// a << a_off \| b << b_off \| c << c_off
1459	var empty = true;
1460	for (0..loaded_struct.field_types.len) \|field_index\| {
1461	const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
1462	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1463
1464	if (!empty) try w.writeAll(" \| ");
1465	try w.writeByte('(');
1466	try dg.renderCType(w, ctype);
1467	try w.writeByte(')');
1468
1469	const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
1470	.bytes => \|bytes\| try pt.intern(.{ .int = .{
1471	.ty = field_ty.toIntern(),
1472	.storage = .{ .u64 = bytes.at(field_index, ip) },
1473	} }),
1474	.elems => \|elems\| elems[field_index],
1475	.repeated_elem => \|elem\| elem,
1476	};
1477
1478	const field_int_info: std.builtin.Type.Int = if (field_ty.isAbiInt(zcu))
1479	field_ty.intInfo(zcu)
1480	else
1481	.{ .signedness = .unsigned, .bits = undefined };
1482	switch (field_int_info.signedness) {
1483	.signed => {
1484	try w.writeByte('(');
1485	try dg.renderValue(w, Value.fromInterned(field_val), .Other);
1486	try w.writeAll(" & ");
1487	const field_uint_ty = try pt.intType(.unsigned, field_int_info.bits);
1488	try dg.renderValue(w, try field_uint_ty.maxIntScalar(pt, field_uint_ty), .Other);
1489	try w.writeByte(')');
1490	},
1491	.unsigned => try dg.renderValue(w, Value.fromInterned(field_val), .Other),
1492	}
1493	if (bit_offset != 0) {
1494	try w.writeAll(" << ");
1495	try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument);
1496	}
1497
1498	bit_offset += field_ty.bitSize(zcu);
1499	empty = false;
1500	}
1501	try w.writeByte(')');
1502	}
1503	},	1411	},
1504	}	1412	}
1505	},	1413	},
...	@@ -1507,33 +1415,38 @@ pub const DeclGen = struct {	...	@@ -1507,33 +1415,38 @@ pub const DeclGen = struct {
1507	},	1415	},
1508	.un => \|un\| {	1416	.un => \|un\| {
1509	const loaded_union = ip.loadUnionType(ty.toIntern());	1417	const loaded_union = ip.loadUnionType(ty.toIntern());
		1418	if (loaded_union.flagsUnordered(ip).layout == .@"packed") {
		1419	// https://github.com/ziglang/zig/issues/24657 will eliminate most of the
		1420	// following logic, leaving only the recursive `renderValue` call. Once
		1421	// that proposal is implemented, a `packed union` will literally be
		1422	// represented in the InternPool by its comptime-known backing integer.
		1423	var arena: std.heap.ArenaAllocator = .init(zcu.gpa);
		1424	defer arena.deinit();
		1425	const backing_ty = try ty.unionBackingType(pt);
		1426	const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu)));
		1427	val.writeToMemory(pt, buf) catch \|err\| switch (err) {
		1428	error.IllDefinedMemoryLayout => unreachable,
		1429	error.OutOfMemory => \|e\| return e,
		1430	error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed union value", .{}),
		1431	};
		1432	const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator());
		1433	return dg.renderValue(w, backing_val, location);
		1434	}
1510	if (un.tag == .none) {	1435	if (un.tag == .none) {
1511	const backing_ty = try ty.unionBackingType(pt);	1436	const backing_ty = try ty.unionBackingType(pt);
1512	switch (loaded_union.flagsUnordered(ip).layout) {	1437	assert(loaded_union.flagsUnordered(ip).layout == .@"extern");
1513	.@"packed" => {	1438	if (location == .StaticInitializer) {
1514	if (!location.isInitializer()) {	1439	return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{});
1515	try w.writeByte('(');
1516	try dg.renderType(w, backing_ty);
1517	try w.writeByte(')');
1518	}
1519	try dg.renderValue(w, Value.fromInterned(un.val), location);
1520	},
1521	.@"extern" => {
1522	if (location == .StaticInitializer) {
1523	return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{});
1524	}
1525
1526	const ptr_ty = try pt.singleConstPtrType(ty);
1527	try w.writeAll("*((");
1528	try dg.renderType(w, ptr_ty);
1529	try w.writeAll(")(");
1530	try dg.renderType(w, backing_ty);
1531	try w.writeAll("){");
1532	try dg.renderValue(w, Value.fromInterned(un.val), location);
1533	try w.writeAll("})");
1534	},
1535	else => unreachable,
1536	}	1440	}
		1441
		1442	const ptr_ty = try pt.singleConstPtrType(ty);
		1443	try w.writeAll("*((");
		1444	try dg.renderType(w, ptr_ty);
		1445	try w.writeAll(")(");
		1446	try dg.renderType(w, backing_ty);
		1447	try w.writeAll("){");
		1448	try dg.renderValue(w, Value.fromInterned(un.val), location);
		1449	try w.writeAll("})");
1537	} else {	1450	} else {
1538	if (!location.isInitializer()) {	1451	if (!location.isInitializer()) {
1539	try w.writeByte('(');	1452	try w.writeByte('(');
...	@@ -1544,21 +1457,6 @@ pub const DeclGen = struct {	...	@@ -1544,21 +1457,6 @@ pub const DeclGen = struct {
1544	const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?;	1457	const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?;
1545	const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]);	1458	const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]);
1546	const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index];	1459	const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index];
1547	if (loaded_union.flagsUnordered(ip).layout == .@"packed") {
1548	if (field_ty.hasRuntimeBits(zcu)) {
1549	if (field_ty.isPtrAtRuntime(zcu)) {
1550	try w.writeByte('(');
1551	try dg.renderCType(w, ctype);
1552	try w.writeByte(')');
1553	} else if (field_ty.zigTypeTag(zcu) == .float) {
1554	try w.writeByte('(');
1555	try dg.renderCType(w, ctype);
1556	try w.writeByte(')');
1557	}
1558	try dg.renderValue(w, Value.fromInterned(un.val), location);
1559	} else try w.writeByte('0');
1560	return;
1561	}
1562		1460
1563	const has_tag = loaded_union.hasTag(ip);	1461	const has_tag = loaded_union.hasTag(ip);
1564	if (has_tag) try w.writeByte('{');	1462	if (has_tag) try w.writeByte('{');
...	@@ -1745,9 +1643,11 @@ pub const DeclGen = struct {	...	@@ -1745,9 +1643,11 @@ pub const DeclGen = struct {
1745	}	1643	}
1746	return w.writeByte('}');	1644	return w.writeByte('}');
1747	},	1645	},
1748	.@"packed" => return w.print("{f}", .{	1646	.@"packed" => return dg.renderUndefValue(
1749	try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other),	1647	w,
1750	}),	1648	.fromInterned(loaded_struct.backingIntTypeUnordered(ip)),
		1649	location,
		1650	),
1751	}	1651	}
1752	},	1652	},
1753	.tuple_type => \|tuple_info\| {	1653	.tuple_type => \|tuple_info\| {
...	@@ -1815,9 +1715,11 @@ pub const DeclGen = struct {	...	@@ -1815,9 +1715,11 @@ pub const DeclGen = struct {
1815	}	1715	}
1816	if (has_tag) try w.writeByte('}');	1716	if (has_tag) try w.writeByte('}');
1817	},	1717	},
1818	.@"packed" => return w.print("{f}", .{	1718	.@"packed" => return dg.renderUndefValue(
1819	try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other),	1719	w,
1820	}),	1720	try ty.unionBackingType(pt),
		1721	location,
		1722	),
1821	}	1723	}
1822	},	1724	},
1823	.error_union_type => \|error_union_type\| switch (ctype.info(ctype_pool)) {	1725	.error_union_type => \|error_union_type\| switch (ctype.info(ctype_pool)) {
...	@@ -2445,10 +2347,7 @@ pub const DeclGen = struct {	...	@@ -2445,10 +2347,7 @@ pub const DeclGen = struct {
2445	const ty = val.typeOf(zcu);	2347	const ty = val.typeOf(zcu);
2446	return .{ .data = .{	2348	return .{ .data = .{
2447	.dg = dg,	2349	.dg = dg,
2448	.int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed")	2350	.int_info = ty.intInfo(zcu),
2449	.{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) }
2450	else
2451	ty.intInfo(zcu),
2452	.kind = kind,	2351	.kind = kind,
2453	.ctype = try dg.ctypeFromType(ty, kind),	2352	.ctype = try dg.ctypeFromType(ty, kind),
2454	.val = val,	2353	.val = val,
...	@@ -3426,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {	...	@@ -3426,6 +3325,10 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3426	// zig fmt: off	3325	// zig fmt: off
3427	.inferred_alloc, .inferred_alloc_comptime => unreachable,	3326	.inferred_alloc, .inferred_alloc_comptime => unreachable,
3428		3327
		3328	// No "scalarize" legalizations are enabled, so these instructions never appear.
		3329	.legalize_vec_elem_val => unreachable,
		3330	.legalize_vec_store_elem => unreachable,
		3331
3429	.arg => try airArg(f, inst),	3332	.arg => try airArg(f, inst),
3430		3333
3431	.breakpoint => try airBreakpoint(f),	3334	.breakpoint => try airBreakpoint(f),
...	@@ -3656,7 +3559,6 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {	...	@@ -3656,7 +3559,6 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3656		3559
3657	.is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}),	3560	.is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}),
3658	.error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}),	3561	.error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}),
3659	.vector_store_elem => return f.fail("TODO: C backend: implement vector_store_elem", .{}),
3660		3562
3661	.runtime_nav_ptr => try airRuntimeNavPtr(f, inst),	3563	.runtime_nav_ptr => try airRuntimeNavPtr(f, inst),
3662		3564
...	@@ -3899,6 +3801,24 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -3899,6 +3801,24 @@ fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3899	});	3801	});
3900	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });	3802	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });
3901	try f.allocs.put(zcu.gpa, local.new_local, true);	3803	try f.allocs.put(zcu.gpa, local.new_local, true);
		3804
		3805	switch (elem_ty.zigTypeTag(zcu)) {
		3806	.@"struct", .@"union" => switch (elem_ty.containerLayout(zcu)) {
		3807	.@"packed" => {
		3808	// For packed aggregates, we zero-initialize to try and work around a design flaw
		3809	// related to how `packed`, `undefined`, and RLS interact. See comment in `airStore`
		3810	// for details.
		3811	const w = &f.object.code.writer;
		3812	try w.print("memset(&t{d}, 0x00, sizeof(", .{local.new_local});
		3813	try f.renderType(w, elem_ty);
		3814	try w.writeAll("));");
		3815	try f.object.newline();
		3816	},
		3817	.auto, .@"extern" => {},
		3818	},
		3819	else => {},
		3820	}
		3821
3902	return .{ .local_ref = local.new_local };	3822	return .{ .local_ref = local.new_local };
3903	}	3823	}
3904		3824
...	@@ -3918,6 +3838,24 @@ fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -3918,6 +3838,24 @@ fn airRetPtr(f: *Function, inst: Air.Inst.Index) !CValue {
3918	});	3838	});
3919	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });	3839	log.debug("%{d}: allocated unfreeable t{d}", .{ inst, local.new_local });
3920	try f.allocs.put(zcu.gpa, local.new_local, true);	3840	try f.allocs.put(zcu.gpa, local.new_local, true);
		3841
		3842	switch (elem_ty.zigTypeTag(zcu)) {
		3843	.@"struct", .@"union" => switch (elem_ty.containerLayout(zcu)) {
		3844	.@"packed" => {
		3845	// For packed aggregates, we zero-initialize to try and work around a design flaw
		3846	// related to how `packed`, `undefined`, and RLS interact. See comment in `airStore`
		3847	// for details.
		3848	const w = &f.object.code.writer;
		3849	try w.print("memset(&t{d}, 0x00, sizeof(", .{local.new_local});
		3850	try f.renderType(w, elem_ty);
		3851	try w.writeAll("));");
		3852	try f.object.newline();
		3853	},
		3854	.auto, .@"extern" => {},
		3855	},
		3856	else => {},
		3857	}
		3858
3921	return .{ .local_ref = local.new_local };	3859	return .{ .local_ref = local.new_local };
3922	}	3860	}
3923		3861
...	@@ -3956,6 +3894,10 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -3956,6 +3894,10 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3956	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);	3894	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);
3957	const src_ty: Type = .fromInterned(ptr_info.child);	3895	const src_ty: Type = .fromInterned(ptr_info.child);
3958		3896
		3897	// `Air.Legalize.Feature.expand_packed_load` should ensure that the only
		3898	// bit-pointers we see here are vector element pointers.
		3899	assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none);
		3900
3959	if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) {	3901	if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
3960	try reap(f, inst, &.{ty_op.operand});	3902	try reap(f, inst, &.{ty_op.operand});
3961	return .none;	3903	return .none;
...	@@ -3987,40 +3929,6 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -3987,40 +3929,6 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3987	try w.writeAll(", sizeof(");	3929	try w.writeAll(", sizeof(");
3988	try f.renderType(w, src_ty);	3930	try f.renderType(w, src_ty);
3989	try w.writeAll("))");	3931	try w.writeAll("))");
3990	} else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
3991	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
3992	const host_ty = try pt.intType(.unsigned, host_bits);
3993
3994	const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1));
3995	const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset);
3996
3997	const field_ty = try pt.intType(.unsigned, @as(u16, @intCast(src_ty.bitSize(zcu))));
3998
3999	try f.writeCValue(w, local, .Other);
4000	try v.elem(f, w);
4001	try w.writeAll(" = (");
4002	try f.renderType(w, src_ty);
4003	try w.writeAll(")zig_wrap_");
4004	try f.object.dg.renderTypeForBuiltinFnName(w, field_ty);
4005	try w.writeAll("((");
4006	try f.renderType(w, field_ty);
4007	try w.writeByte(')');
4008	const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64;
4009	if (cant_cast) {
4010	if (field_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
4011	try w.writeAll("zig_lo_");
4012	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4013	try w.writeByte('(');
4014	}
4015	try w.writeAll("zig_shr_");
4016	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4017	try w.writeByte('(');
4018	try f.writeCValueDeref(w, operand);
4019	try v.elem(f, w);
4020	try w.print(", {f})", .{try f.fmtIntLiteralDec(bit_offset_val)});
4021	if (cant_cast) try w.writeByte(')');
4022	try f.object.dg.renderBuiltinInfo(w, field_ty, .bits);
4023	try w.writeByte(')');
4024	} else {	3932	} else {
4025	try f.writeCValue(w, local, .Other);	3933	try f.writeCValue(w, local, .Other);
4026	try v.elem(f, w);	3934	try v.elem(f, w);
...	@@ -4213,6 +4121,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {	...	@@ -4213,6 +4121,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4213	const ptr_scalar_ty = ptr_ty.scalarType(zcu);	4121	const ptr_scalar_ty = ptr_ty.scalarType(zcu);
4214	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);	4122	const ptr_info = ptr_scalar_ty.ptrInfo(zcu);
4215		4123
		4124	// `Air.Legalize.Feature.expand_packed_store` should ensure that the only
		4125	// bit-pointers we see here are vector element pointers.
		4126	assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none);
		4127
4216	const ptr_val = try f.resolveInst(bin_op.lhs);	4128	const ptr_val = try f.resolveInst(bin_op.lhs);
4217	const src_ty = f.typeOf(bin_op.rhs);	4129	const src_ty = f.typeOf(bin_op.rhs);
4218		4130
...	@@ -4222,9 +4134,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {	...	@@ -4222,9 +4134,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4222	if (val_is_undef) {	4134	if (val_is_undef) {
4223	try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });	4135	try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
4224	if (safety and ptr_info.packed_offset.host_size == 0) {	4136	if (safety and ptr_info.packed_offset.host_size == 0) {
		4137	// If the thing we're initializing is a packed struct/union, we set to 0 instead of
		4138	// 0xAA. This is a hack to work around a problem with partially-undefined packed
		4139	// aggregates. If we used 0xAA here, then a later initialization through RLS would
		4140	// not zero the high padding bits (for a packed type which is not 8/16/32/64/etc bits),
		4141	// so we would get a miscompilation. Using 0x00 here avoids this bug in some cases. It
		4142	// is not a correct fix; for instance it misses any case where packed structs are
		4143	// nested in other aggregates. A proper fix for this will involve changing the language,
		4144	// such as to remove RLS. This just prevents miscompilations in some common cases.
		4145	const byte_str: []const u8 = switch (src_ty.zigTypeTag(zcu)) {
		4146	else => "0xaa",
		4147	.@"struct", .@"union" => switch (src_ty.containerLayout(zcu)) {
		4148	.auto, .@"extern" => "0xaa",
		4149	.@"packed" => "0x00",
		4150	},
		4151	};
4225	try w.writeAll("memset(");	4152	try w.writeAll("memset(");
4226	try f.writeCValue(w, ptr_val, .FunctionArgument);	4153	try f.writeCValue(w, ptr_val, .FunctionArgument);
4227	try w.writeAll(", 0xaa, sizeof(");	4154	try w.print(", {s}, sizeof(", .{byte_str});
4228	try f.renderType(w, .fromInterned(ptr_info.child));	4155	try f.renderType(w, .fromInterned(ptr_info.child));
4229	try w.writeAll("));");	4156	try w.writeAll("));");
4230	try f.object.newline();	4157	try f.object.newline();
...	@@ -4277,66 +4204,6 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {	...	@@ -4277,66 +4204,6 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
4277	try w.writeByte(';');	4204	try w.writeByte(';');
4278	try f.object.newline();	4205	try f.object.newline();
4279	try v.end(f, inst, w);	4206	try v.end(f, inst, w);
4280	} else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
4281	const host_bits = ptr_info.packed_offset.host_size * 8;
4282	const host_ty = try pt.intType(.unsigned, host_bits);
4283
4284	const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1));
4285	const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset);
4286
4287	const src_bits = src_ty.bitSize(zcu);
4288
4289	const ExpectedContents = [BigInt.Managed.default_capacity]BigIntLimb;
4290	var stack align(@alignOf(ExpectedContents)) =
4291	std.heap.stackFallback(@sizeOf(ExpectedContents), f.object.dg.gpa);
4292
4293	var mask = try BigInt.Managed.initCapacity(stack.get(), BigInt.calcTwosCompLimbCount(host_bits));
4294	defer mask.deinit();
4295
4296	try mask.setTwosCompIntLimit(.max, .unsigned, @intCast(src_bits));
4297	try mask.shiftLeft(&mask, ptr_info.packed_offset.bit_offset);
4298	try mask.bitNotWrap(&mask, .unsigned, host_bits);
4299
4300	const mask_val = try pt.intValue_big(host_ty, mask.toConst());
4301
4302	const v = try Vectorize.start(f, inst, w, ptr_ty);
4303	const a = try Assignment.start(f, w, src_scalar_ctype);
4304	try f.writeCValueDeref(w, ptr_val);
4305	try v.elem(f, w);
4306	try a.assign(f, w);
4307	try w.writeAll("zig_or_");
4308	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4309	try w.writeAll("(zig_and_");
4310	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4311	try w.writeByte('(');
4312	try f.writeCValueDeref(w, ptr_val);
4313	try v.elem(f, w);
4314	try w.print(", {f}), zig_shl_", .{try f.fmtIntLiteralHex(mask_val)});
4315	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4316	try w.writeByte('(');
4317	const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64;
4318	if (cant_cast) {
4319	if (src_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
4320	try w.writeAll("zig_make_");
4321	try f.object.dg.renderTypeForBuiltinFnName(w, host_ty);
4322	try w.writeAll("(0, ");
4323	} else {
4324	try w.writeByte('(');
4325	try f.renderType(w, host_ty);
4326	try w.writeByte(')');
4327	}
4328
4329	if (src_ty.isPtrAtRuntime(zcu)) {
4330	try w.writeByte('(');
4331	try f.renderType(w, .usize);
4332	try w.writeByte(')');
4333	}
4334	try f.writeCValue(w, src_val, .Other);
4335	try v.elem(f, w);
4336	if (cant_cast) try w.writeByte(')');
4337	try w.print(", {f}))", .{try f.fmtIntLiteralDec(bit_offset_val)});
4338	try a.end(f, w);
4339	try v.end(f, inst, w);
4340	} else {	4207	} else {
4341	switch (ptr_val) {	4208	switch (ptr_val) {
4342	.local_ref => \|ptr_local_index\| switch (src_val) {	4209	.local_ref => \|ptr_local_index\| switch (src_val) {
...	@@ -6015,10 +5882,7 @@ fn fieldLocation(	...	@@ -6015,10 +5882,7 @@ fn fieldLocation(
6015	else if (!field_ptr_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu))	5882	else if (!field_ptr_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu))
6016	.{ .byte_offset = loaded_struct.offsets.get(ip)[field_index] }	5883	.{ .byte_offset = loaded_struct.offsets.get(ip)[field_index] }
6017	else	5884	else
6018	.{ .field = if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\|	5885	.{ .field = .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) } },
6019	.{ .identifier = field_name.toSlice(ip) }
6020	else
6021	.{ .field = field_index } },
6022	.@"packed" => if (field_ptr_ty.ptrInfo(zcu).packed_offset.host_size == 0)	5886	.@"packed" => if (field_ptr_ty.ptrInfo(zcu).packed_offset.host_size == 0)
6023	.{ .byte_offset = @divExact(zcu.structPackedFieldBitOffset(loaded_struct, field_index) +	5887	.{ .byte_offset = @divExact(zcu.structPackedFieldBitOffset(loaded_struct, field_index) +
6024	container_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset, 8) }	5888	container_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset, 8) }
...	@@ -6202,115 +6066,20 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -6202,115 +6066,20 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue {
6202	// Ensure complete type definition is visible before accessing fields.	6066	// Ensure complete type definition is visible before accessing fields.
6203	_ = try f.ctypeFromType(struct_ty, .complete);	6067	_ = try f.ctypeFromType(struct_ty, .complete);
6204		6068
		6069	assert(struct_ty.containerLayout(zcu) != .@"packed"); // `Air.Legalize.Feature.expand_packed_struct_field_val` handles this case
6205	const field_name: CValue = switch (ip.indexToKey(struct_ty.toIntern())) {	6070	const field_name: CValue = switch (ip.indexToKey(struct_ty.toIntern())) {
6206	.struct_type => field_name: {	6071	.struct_type => .{ .identifier = struct_ty.structFieldName(extra.field_index, zcu).unwrap().?.toSlice(ip) },
6207	const loaded_struct = ip.loadStructType(struct_ty.toIntern());	6072	.union_type => name: {
6208	switch (loaded_struct.layout) {	6073	const union_type = ip.loadUnionType(struct_ty.toIntern());
6209	.auto, .@"extern" => break :field_name if (loaded_struct.fieldName(ip, extra.field_index).unwrap()) \|field_name\|	6074	const enum_tag_ty: Type = .fromInterned(union_type.enum_tag_ty);
6210	.{ .identifier = field_name.toSlice(ip) }	6075	const field_name_str = enum_tag_ty.enumFieldName(extra.field_index, zcu).toSlice(ip);
6211	else	6076	if (union_type.hasTag(ip)) {
6212	.{ .field = extra.field_index },	6077	break :name .{ .payload_identifier = field_name_str };
6213	.@"packed" => {	6078	} else {
6214	const int_info = struct_ty.intInfo(zcu);	6079	break :name .{ .identifier = field_name_str };
6215
6216	const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1));
6217
6218	const bit_offset = zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index);
6219
6220	const field_int_signedness = if (inst_ty.isAbiInt(zcu))
6221	inst_ty.intInfo(zcu).signedness
6222	else
6223	.unsigned;
6224	const field_int_ty = try pt.intType(field_int_signedness, @as(u16, @intCast(inst_ty.bitSize(zcu))));
6225
6226	const temp_local = try f.allocLocal(inst, field_int_ty);
6227	try f.writeCValue(w, temp_local, .Other);
6228	try w.writeAll(" = zig_wrap_");
6229	try f.object.dg.renderTypeForBuiltinFnName(w, field_int_ty);
6230	try w.writeAll("((");
6231	try f.renderType(w, field_int_ty);
6232	try w.writeByte(')');
6233	const cant_cast = int_info.bits > 64;
6234	if (cant_cast) {
6235	if (field_int_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{});
6236	try w.writeAll("zig_lo_");
6237	try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty);
6238	try w.writeByte('(');
6239	}
6240	if (bit_offset > 0) {
6241	try w.writeAll("zig_shr_");
6242	try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty);
6243	try w.writeByte('(');
6244	}
6245	try f.writeCValue(w, struct_byval, .Other);
6246	if (bit_offset > 0) try w.print(", {f})", .{
6247	try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)),
6248	});
6249	if (cant_cast) try w.writeByte(')');
6250	try f.object.dg.renderBuiltinInfo(w, field_int_ty, .bits);
6251	try w.writeAll(");");
6252	try f.object.newline();
6253	if (inst_ty.eql(field_int_ty, zcu)) return temp_local;
6254
6255	const local = try f.allocLocal(inst, inst_ty);
6256	if (local.new_local != temp_local.new_local) {
6257	try w.writeAll("memcpy(");
6258	try f.writeCValue(w, .{ .local_ref = local.new_local }, .FunctionArgument);
6259	try w.writeAll(", ");
6260	try f.writeCValue(w, .{ .local_ref = temp_local.new_local }, .FunctionArgument);
6261	try w.writeAll(", sizeof(");
6262	try f.renderType(w, inst_ty);
6263	try w.writeAll("));");
6264	try f.object.newline();
6265	}
6266	try freeLocal(f, inst, temp_local.new_local, null);
6267	return local;
6268	},
6269	}	6080	}
6270	},	6081	},
6271	.tuple_type => .{ .field = extra.field_index },	6082	.tuple_type => .{ .field = extra.field_index },
6272	.union_type => field_name: {
6273	const loaded_union = ip.loadUnionType(struct_ty.toIntern());
6274	switch (loaded_union.flagsUnordered(ip).layout) {
6275	.auto, .@"extern" => {
6276	const name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index];
6277	break :field_name if (loaded_union.hasTag(ip))
6278	.{ .payload_identifier = name.toSlice(ip) }
6279	else
6280	.{ .identifier = name.toSlice(ip) };
6281	},
6282	.@"packed" => {
6283	const operand_lval = if (struct_byval == .constant) blk: {
6284	const operand_local = try f.allocLocal(inst, struct_ty);
6285	try f.writeCValue(w, operand_local, .Other);
6286	try w.writeAll(" = ");
6287	try f.writeCValue(w, struct_byval, .Other);
6288	try w.writeByte(';');
6289	try f.object.newline();
6290	break :blk operand_local;
6291	} else struct_byval;
6292	const local = try f.allocLocal(inst, inst_ty);
6293	if (switch (local) {
6294	.new_local, .local => \|local_index\| switch (operand_lval) {
6295	.new_local, .local => \|operand_local_index\| local_index != operand_local_index,
6296	else => true,
6297	},
6298	else => true,
6299	}) {
6300	try w.writeAll("memcpy(&");
6301	try f.writeCValue(w, local, .Other);
6302	try w.writeAll(", &");
6303	try f.writeCValue(w, operand_lval, .Other);
6304	try w.writeAll(", sizeof(");
6305	try f.renderType(w, inst_ty);
6306	try w.writeAll("));");
6307	try f.object.newline();
6308	}
6309	try f.freeCValue(inst, operand_lval);
6310	return local;
6311	},
6312	}
6313	},
6314	else => unreachable,	6083	else => unreachable,
6315	};	6084	};
6316		6085
...	@@ -7702,98 +7471,13 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -7702,98 +7471,13 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue {
7702	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;	7471	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7703		7472
7704	const a = try Assignment.start(f, w, try f.ctypeFromType(field_ty, .complete));	7473	const a = try Assignment.start(f, w, try f.ctypeFromType(field_ty, .complete));
7705	try f.writeCValueMember(w, local, if (loaded_struct.fieldName(ip, field_index).unwrap()) \|field_name\|	7474	try f.writeCValueMember(w, local, .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) });
7706	.{ .identifier = field_name.toSlice(ip) }
7707	else
7708	.{ .field = field_index });
7709	try a.assign(f, w);	7475	try a.assign(f, w);
7710	try f.writeCValue(w, resolved_elements[field_index], .Other);	7476	try f.writeCValue(w, resolved_elements[field_index], .Other);
7711	try a.end(f, w);	7477	try a.end(f, w);
7712	}	7478	}
7713	},	7479	},
7714	.@"packed" => {	7480	.@"packed" => unreachable, // `Air.Legalize.Feature.expand_packed_struct_init` handles this case
7715	try f.writeCValue(w, local, .Other);
7716	try w.writeAll(" = ");
7717
7718	const backing_int_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip));
7719	const int_info = backing_int_ty.intInfo(zcu);
7720
7721	const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1));
7722
7723	var bit_offset: u64 = 0;
7724
7725	var empty = true;
7726	for (0..elements.len) \|field_index\| {
7727	if (inst_ty.structFieldIsComptime(field_index, zcu)) continue;
7728	const field_ty = inst_ty.fieldType(field_index, zcu);
7729	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7730
7731	if (!empty) {
7732	try w.writeAll("zig_or_");
7733	try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7734	try w.writeByte('(');
7735	}
7736	empty = false;
7737	}
7738	empty = true;
7739	for (resolved_elements, 0..) \|element, field_index\| {
7740	if (inst_ty.structFieldIsComptime(field_index, zcu)) continue;
7741	const field_ty = inst_ty.fieldType(field_index, zcu);
7742	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
7743
7744	if (!empty) try w.writeAll(", ");
7745	// TODO: Skip this entire shift if val is 0?
7746	try w.writeAll("zig_shlw_");
7747	try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7748	try w.writeByte('(');
7749
7750	if (field_ty.isAbiInt(zcu)) {
7751	try w.writeAll("zig_and_");
7752	try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty);
7753	try w.writeByte('(');
7754	}
7755
7756	if (inst_ty.isAbiInt(zcu) and (field_ty.isAbiInt(zcu) or field_ty.isPtrAtRuntime(zcu))) {
7757	try f.renderIntCast(w, inst_ty, element, .{}, field_ty, .FunctionArgument);
7758	} else {
7759	try w.writeByte('(');
7760	try f.renderType(w, inst_ty);
7761	try w.writeByte(')');
7762	if (field_ty.isPtrAtRuntime(zcu)) {
7763	try w.writeByte('(');
7764	try f.renderType(w, switch (int_info.signedness) {
7765	.unsigned => .usize,
7766	.signed => .isize,
7767	});
7768	try w.writeByte(')');
7769	}
7770	try f.writeCValue(w, element, .Other);
7771	}
7772
7773	if (field_ty.isAbiInt(zcu)) {
7774	try w.writeAll(", ");
7775	const field_int_info = field_ty.intInfo(zcu);
7776	const field_mask = if (int_info.signedness == .signed and int_info.bits == field_int_info.bits)
7777	try pt.intValue(backing_int_ty, -1)
7778	else
7779	try (try pt.intType(.unsigned, field_int_info.bits)).maxIntScalar(pt, backing_int_ty);
7780	try f.object.dg.renderValue(w, field_mask, .FunctionArgument);
7781	try w.writeByte(')');
7782	}
7783
7784	try w.print(", {f}", .{
7785	try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)),
7786	});
7787	try f.object.dg.renderBuiltinInfo(w, inst_ty, .bits);
7788	try w.writeByte(')');
7789	if (!empty) try w.writeByte(')');
7790
7791	bit_offset += field_ty.bitSize(zcu);
7792	empty = false;
7793	}
7794	try w.writeByte(';');
7795	try f.object.newline();
7796	},
7797	}	7481	}
7798	},	7482	},
7799	.tuple_type => \|tuple_info\| for (0..tuple_info.types.len) \|field_index\| {	7483	.tuple_type => \|tuple_info\| for (0..tuple_info.types.len) \|field_index\| {
...	@@ -7828,9 +7512,10 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue {	...	@@ -7828,9 +7512,10 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue {
7828	try reap(f, inst, &.{extra.init});	7512	try reap(f, inst, &.{extra.init});
7829		7513
7830	const w = &f.object.code.writer;	7514	const w = &f.object.code.writer;
7831	const local = try f.allocLocal(inst, union_ty);
7832	if (loaded_union.flagsUnordered(ip).layout == .@"packed") return f.moveCValue(inst, union_ty, payload);	7515	if (loaded_union.flagsUnordered(ip).layout == .@"packed") return f.moveCValue(inst, union_ty, payload);
7833		7516
		7517	const local = try f.allocLocal(inst, union_ty);
		7518
7834	const field: CValue = if (union_ty.unionTagTypeSafety(zcu)) \|tag_ty\| field: {	7519	const field: CValue = if (union_ty.unionTagTypeSafety(zcu)) \|tag_ty\| field: {
7835	const layout = union_ty.unionGetLayout(zcu);	7520	const layout = union_ty.unionGetLayout(zcu);
7836	if (layout.tag_size != 0) {	7521	if (layout.tag_size != 0) {

src/codegen/c/Type.zig+1-5

 @@ -2514,11 +2514,7 @@ pub const Pool = struct {
                                     kind.noParameter(),
                                 );
                                 if (field_ctype.index == .void) continue;
-                                const field_name = if (loaded_struct.fieldName(ip, field_index)
+                                const field_name = try pool.string(allocator, loaded_struct.fieldName(ip, field_index).toSlice(ip));
-                                    .unwrap()) |field_name|
-                                    try pool.string(allocator, field_name.toSlice(ip))
-                                else
-                                    String.fromUnnamed(@intCast(field_index));
                                 const field_alignas = AlignAs.fromAlignment(.{
                                     .@"align" = loaded_struct.fieldAlign(ip, field_index),
                                     .abi = field_type.abiAlignment(zcu),

src/codegen/llvm.zig+13-45

...	@@ -2409,8 +2409,7 @@ pub const Object = struct {	...	@@ -2409,8 +2409,7 @@ pub const Object = struct {
2409	const field_size = field_ty.abiSize(zcu);	2409	const field_size = field_ty.abiSize(zcu);
2410	const field_align = ty.fieldAlignment(field_index, zcu);	2410	const field_align = ty.fieldAlignment(field_index, zcu);
2411	const field_offset = ty.structFieldOffset(field_index, zcu);	2411	const field_offset = ty.structFieldOffset(field_index, zcu);
2412	const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse	2412	const field_name = struct_type.fieldName(ip, field_index);
2413	try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
2414	fields.appendAssumeCapacity(try o.builder.debugMemberType(	2413	fields.appendAssumeCapacity(try o.builder.debugMemberType(
2415	try o.builder.metadataString(field_name.toSlice(ip)),	2414	try o.builder.metadataString(field_name.toSlice(ip)),
2416	null, // File	2415	null, // File
...	@@ -4885,6 +4884,11 @@ pub const FuncGen = struct {	...	@@ -4885,6 +4884,11 @@ pub const FuncGen = struct {
4885		4884
4886	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {	4885	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4887	// zig fmt: off	4886	// zig fmt: off
		4887
		4888	// No "scalarize" legalizations are enabled, so these instructions never appear.
		4889	.legalize_vec_elem_val => unreachable,
		4890	.legalize_vec_store_elem => unreachable,
		4891
4888	.add => try self.airAdd(inst, .normal),	4892	.add => try self.airAdd(inst, .normal),
4889	.add_optimized => try self.airAdd(inst, .fast),	4893	.add_optimized => try self.airAdd(inst, .fast),
4890	.add_wrap => try self.airAddWrap(inst),	4894	.add_wrap => try self.airAddWrap(inst),
...	@@ -5091,8 +5095,6 @@ pub const FuncGen = struct {	...	@@ -5091,8 +5095,6 @@ pub const FuncGen = struct {
5091	.wasm_memory_size => try self.airWasmMemorySize(inst),	5095	.wasm_memory_size => try self.airWasmMemorySize(inst),
5092	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),	5096	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
5093		5097
5094	.vector_store_elem => try self.airVectorStoreElem(inst),
5095
5096	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),	5098	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
5097		5099
5098	.inferred_alloc, .inferred_alloc_comptime => unreachable,	5100	.inferred_alloc, .inferred_alloc_comptime => unreachable,
...	@@ -6871,16 +6873,14 @@ pub const FuncGen = struct {	...	@@ -6871,16 +6873,14 @@ pub const FuncGen = struct {
6871	const array_llvm_ty = try o.lowerType(pt, array_ty);	6873	const array_llvm_ty = try o.lowerType(pt, array_ty);
6872	const elem_ty = array_ty.childType(zcu);	6874	const elem_ty = array_ty.childType(zcu);
6873	if (isByRef(array_ty, zcu)) {	6875	if (isByRef(array_ty, zcu)) {
6874	const indices: [2]Builder.Value = .{	6876	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
6875	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs,	6877	try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
6876	};	6878	rhs,
		6879	}, "");
6877	if (isByRef(elem_ty, zcu)) {	6880	if (isByRef(elem_ty, zcu)) {
6878	const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6879	const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();	6881	const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6880	return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);	6882	return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6881	} else {	6883	} else {
6882	const elem_ptr =
6883	try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, "");
6884	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);	6884	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
6885	}	6885	}
6886	}	6886	}
...	@@ -8138,33 +8138,6 @@ pub const FuncGen = struct {	...	@@ -8138,33 +8138,6 @@ pub const FuncGen = struct {
8138	}, "");	8138	}, "");
8139	}	8139	}
8140		8140
8141	fn airVectorStoreElem(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8142	const o = self.ng.object;
8143	const pt = self.ng.pt;
8144	const zcu = pt.zcu;
8145	const data = self.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
8146	const extra = self.air.extraData(Air.Bin, data.payload).data;
8147
8148	const vector_ptr = try self.resolveInst(data.vector_ptr);
8149	const vector_ptr_ty = self.typeOf(data.vector_ptr);
8150	const index = try self.resolveInst(extra.lhs);
8151	const operand = try self.resolveInst(extra.rhs);
8152
8153	self.maybeMarkAllowZeroAccess(vector_ptr_ty.ptrInfo(zcu));
8154
8155	// TODO: Emitting a load here is a violation of volatile semantics. Not fixable in general.
8156	// https://github.com/ziglang/zig/issues/18652#issuecomment-2452844908
8157	const access_kind: Builder.MemoryAccessKind =
8158	if (vector_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
8159	const elem_llvm_ty = try o.lowerType(pt, vector_ptr_ty.childType(zcu));
8160	const alignment = vector_ptr_ty.ptrAlignment(zcu).toLlvm();
8161	const loaded = try self.wip.load(access_kind, elem_llvm_ty, vector_ptr, alignment, "");
8162
8163	const new_vector = try self.wip.insertElement(loaded, operand, index, "");
8164	_ = try self.store(vector_ptr, vector_ptr_ty, new_vector, .none);
8165	return .none;
8166	}
8167
8168	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {	8141	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8169	const o = fg.ng.object;	8142	const o = fg.ng.object;
8170	const pt = fg.ng.pt;	8143	const pt = fg.ng.pt;
...	@@ -8301,8 +8274,7 @@ pub const FuncGen = struct {	...	@@ -8301,8 +8274,7 @@ pub const FuncGen = struct {
8301	const rhs = try self.resolveInst(bin_op.rhs);	8274	const rhs = try self.resolveInst(bin_op.rhs);
8302	const inst_ty = self.typeOfIndex(inst);	8275	const inst_ty = self.typeOfIndex(inst);
8303	const scalar_ty = inst_ty.scalarType(zcu);	8276	const scalar_ty = inst_ty.scalarType(zcu);
8304		8277	assert(scalar_ty.zigTypeTag(zcu) == .int);
8305	if (scalar_ty.isAnyFloat()) return self.todo("saturating float add", .{});
8306	return self.wip.callIntrinsic(	8278	return self.wip.callIntrinsic(
8307	.normal,	8279	.normal,
8308	.none,	8280	.none,
...	@@ -8342,8 +8314,7 @@ pub const FuncGen = struct {	...	@@ -8342,8 +8314,7 @@ pub const FuncGen = struct {
8342	const rhs = try self.resolveInst(bin_op.rhs);	8314	const rhs = try self.resolveInst(bin_op.rhs);
8343	const inst_ty = self.typeOfIndex(inst);	8315	const inst_ty = self.typeOfIndex(inst);
8344	const scalar_ty = inst_ty.scalarType(zcu);	8316	const scalar_ty = inst_ty.scalarType(zcu);
8345		8317	assert(scalar_ty.zigTypeTag(zcu) == .int);
8346	if (scalar_ty.isAnyFloat()) return self.todo("saturating float sub", .{});
8347	return self.wip.callIntrinsic(	8318	return self.wip.callIntrinsic(
8348	.normal,	8319	.normal,
8349	.none,	8320	.none,
...	@@ -8383,8 +8354,7 @@ pub const FuncGen = struct {	...	@@ -8383,8 +8354,7 @@ pub const FuncGen = struct {
8383	const rhs = try self.resolveInst(bin_op.rhs);	8354	const rhs = try self.resolveInst(bin_op.rhs);
8384	const inst_ty = self.typeOfIndex(inst);	8355	const inst_ty = self.typeOfIndex(inst);
8385	const scalar_ty = inst_ty.scalarType(zcu);	8356	const scalar_ty = inst_ty.scalarType(zcu);
8386		8357	assert(scalar_ty.zigTypeTag(zcu) == .int);
8387	if (scalar_ty.isAnyFloat()) return self.todo("saturating float mul", .{});
8388	return self.wip.callIntrinsic(	8358	return self.wip.callIntrinsic(
8389	.normal,	8359	.normal,
8390	.none,	8360	.none,
...	@@ -11452,7 +11422,6 @@ pub const FuncGen = struct {	...	@@ -11452,7 +11422,6 @@ pub const FuncGen = struct {
11452	const access_kind: Builder.MemoryAccessKind =	11422	const access_kind: Builder.MemoryAccessKind =
11453	if (info.flags.is_volatile) .@"volatile" else .normal;	11423	if (info.flags.is_volatile) .@"volatile" else .normal;
11454		11424
11455	assert(info.flags.vector_index != .runtime);
11456	if (info.flags.vector_index != .none) {	11425	if (info.flags.vector_index != .none) {
11457	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);	11426	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11458	const vec_elem_ty = try o.lowerType(pt, elem_ty);	11427	const vec_elem_ty = try o.lowerType(pt, elem_ty);
...	@@ -11522,7 +11491,6 @@ pub const FuncGen = struct {	...	@@ -11522,7 +11491,6 @@ pub const FuncGen = struct {
11522	const access_kind: Builder.MemoryAccessKind =	11491	const access_kind: Builder.MemoryAccessKind =
11523	if (info.flags.is_volatile) .@"volatile" else .normal;	11492	if (info.flags.is_volatile) .@"volatile" else .normal;
11524		11493
11525	assert(info.flags.vector_index != .runtime);
11526	if (info.flags.vector_index != .none) {	11494	if (info.flags.vector_index != .none) {
11527	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);	11495	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11528	const vec_elem_ty = try o.lowerType(pt, elem_ty);	11496	const vec_elem_ty = try o.lowerType(pt, elem_ty);

src/codegen/riscv64/CodeGen.zig+5-1

 @@ -1391,6 +1391,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
         const tag = air_tags[@intFromEnum(inst)];
         switch (tag) {
             // zig fmt: off
+            // No "scalarize" legalizations are enabled, so these instructions never appear.
+            .legalize_vec_elem_val   => unreachable,
+            .legalize_vec_store_elem => unreachable,
             .add,
             .add_wrap,
             .sub,
 @@ -1633,7 +1638,6 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
             .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}),
             .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}),
-            .vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}),
             .c_va_arg => return func.fail("TODO implement c_va_arg", .{}),
             .c_va_copy => return func.fail("TODO implement c_va_copy", .{}),

src/codegen/sparc64/CodeGen.zig+5-1

 @@ -479,6 +479,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
         self.reused_operands = @TypeOf(self.reused_operands).initEmpty();
         switch (air_tags[@intFromEnum(inst)]) {
             // zig fmt: off
+            // No "scalarize" legalizations are enabled, so these instructions never appear.
+            .legalize_vec_elem_val   => unreachable,
+            .legalize_vec_store_elem => unreachable,
             .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
             .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
 @@ -702,7 +707,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
             .is_named_enum_value => @panic("TODO implement is_named_enum_value"),
             .error_set_has_value => @panic("TODO implement error_set_has_value"),
-            .vector_store_elem => @panic("TODO implement vector_store_elem"),
             .runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"),
             .c_va_arg => return self.fail("TODO implement c_va_arg", .{}),

src/codegen/spirv/CodeGen.zig+1-27

...	@@ -1520,8 +1520,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {	...	@@ -1520,8 +1520,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1520	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);	1520	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1521	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;	1521	if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue;
1522		1522
1523	const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse	1523	const field_name = struct_type.fieldName(ip, field_index);
1524	try ip.getOrPutStringFmt(zcu.gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls);
1525	try member_types.append(try cg.resolveType(field_ty, .indirect));	1524	try member_types.append(try cg.resolveType(field_ty, .indirect));
1526	try member_names.append(field_name.toSlice(ip));	1525	try member_names.append(field_name.toSlice(ip));
1527	try member_offsets.append(@intCast(ty.structFieldOffset(field_index, zcu)));	1526	try member_offsets.append(@intCast(ty.structFieldOffset(field_index, zcu)));
...	@@ -2726,8 +2725,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {	...	@@ -2726,8 +2725,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
2726	.ptr_elem_val => try cg.airPtrElemVal(inst),	2725	.ptr_elem_val => try cg.airPtrElemVal(inst),
2727	.array_elem_val => try cg.airArrayElemVal(inst),	2726	.array_elem_val => try cg.airArrayElemVal(inst),
2728		2727
2729	.vector_store_elem => return cg.airVectorStoreElem(inst),
2730
2731	.set_union_tag => return cg.airSetUnionTag(inst),	2728	.set_union_tag => return cg.airSetUnionTag(inst),
2732	.get_union_tag => try cg.airGetUnionTag(inst),	2729	.get_union_tag => try cg.airGetUnionTag(inst),
2733	.union_init => try cg.airUnionInit(inst),	2730	.union_init => try cg.airUnionInit(inst),
...	@@ -4446,29 +4443,6 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {	...	@@ -4446,29 +4443,6 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
4446	return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });	4443	return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
4447	}	4444	}
4448		4445
4449	fn airVectorStoreElem(cg: *CodeGen, inst: Air.Inst.Index) !void {
4450	const zcu = cg.module.zcu;
4451	const data = cg.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem;
4452	const extra = cg.air.extraData(Air.Bin, data.payload).data;
4453
4454	const vector_ptr_ty = cg.typeOf(data.vector_ptr);
4455	const vector_ty = vector_ptr_ty.childType(zcu);
4456	const scalar_ty = vector_ty.scalarType(zcu);
4457
4458	const scalar_ty_id = try cg.resolveType(scalar_ty, .indirect);
4459	const storage_class = cg.module.storageClass(vector_ptr_ty.ptrAddressSpace(zcu));
4460	const scalar_ptr_ty_id = try cg.module.ptrType(scalar_ty_id, storage_class);
4461
4462	const vector_ptr = try cg.resolve(data.vector_ptr);
4463	const index = try cg.resolve(extra.lhs);
4464	const operand = try cg.resolve(extra.rhs);
4465
4466	const elem_ptr_id = try cg.accessChainId(scalar_ptr_ty_id, vector_ptr, &.{index});
4467	try cg.store(scalar_ty, elem_ptr_id, operand, .{
4468	.is_volatile = vector_ptr_ty.isVolatilePtr(zcu),
4469	});
4470	}
4471
4472	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void {	4446	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void {
4473	const zcu = cg.module.zcu;	4447	const zcu = cg.module.zcu;
4474	const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;	4448	const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;

src/codegen/wasm/CodeGen.zig+4-1

...	@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum	...	@@ -1786,6 +1786,10 @@ fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum
1786	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	1786	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1787	const air_tags = cg.air.instructions.items(.tag);	1787	const air_tags = cg.air.instructions.items(.tag);
1788	return switch (air_tags[@intFromEnum(inst)]) {	1788	return switch (air_tags[@intFromEnum(inst)]) {
		1789	// No "scalarize" legalizations are enabled, so these instructions never appear.
		1790	.legalize_vec_elem_val => unreachable,
		1791	.legalize_vec_store_elem => unreachable,
		1792
1789	.inferred_alloc, .inferred_alloc_comptime => unreachable,	1793	.inferred_alloc, .inferred_alloc_comptime => unreachable,
1790		1794
1791	.add => cg.airBinOp(inst, .add),	1795	.add => cg.airBinOp(inst, .add),
...	@@ -1978,7 +1982,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {	...	@@ -1978,7 +1982,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1978	.save_err_return_trace_index,	1982	.save_err_return_trace_index,
1979	.is_named_enum_value,	1983	.is_named_enum_value,
1980	.addrspace_cast,	1984	.addrspace_cast,
1981	.vector_store_elem,
1982	.c_va_arg,	1985	.c_va_arg,
1983	.c_va_copy,	1986	.c_va_copy,
1984	.c_va_end,	1987	.c_va_end,

src/codegen/x86_64/CodeGen.zig+65-972

...	@@ -854,12 +854,6 @@ const FrameAlloc = struct {	...	@@ -854,12 +854,6 @@ const FrameAlloc = struct {
854	}	854	}
855	};	855	};
856		856
857	const StackAllocation = struct {
858	inst: ?Air.Inst.Index,
859	/// TODO do we need size? should be determined by inst.ty.abiSize(zcu)
860	size: u32,
861	};
862
863	const BlockData = struct {	857	const BlockData = struct {
864	relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,	858	relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
865	state: State,	859	state: State,
...	@@ -89326,7 +89320,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -89326,7 +89320,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89326	error.SelectFailed => res[0] = try ops[0].load(val_ty, .{	89320	error.SelectFailed => res[0] = try ops[0].load(val_ty, .{
89327	.disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) {	89321	.disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) {
89328	.none => 0,	89322	.none => 0,
89329	.runtime => unreachable,
89330	else => \|vector_index\| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),	89323	else => \|vector_index\| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)),
89331	},	89324	},
89332	}, cg),	89325	}, cg),
...	@@ -89569,7 +89562,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -89569,7 +89562,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89569	error.SelectFailed => try ops[0].store(&ops[1], .{	89562	error.SelectFailed => try ops[0].store(&ops[1], .{
89570	.disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) {	89563	.disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) {
89571	.none => 0,	89564	.none => 0,
89572	.runtime => unreachable,
89573	else => \|vector_index\| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)),	89565	else => \|vector_index\| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)),
89574	},	89566	},
89575	.safe = switch (air_tag) {	89567	.safe = switch (air_tag) {
...	@@ -103934,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -103934,7 +103926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
103934	try ops[0].toOffset(0, cg);	103926	try ops[0].toOffset(0, cg);
103935	try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);	103927	try ops[0].finish(inst, &.{ty_op.operand}, &ops, cg);
103936	},	103928	},
103937	.array_elem_val => {	103929	.array_elem_val, .legalize_vec_elem_val => {
103938	const bin_op = air_datas[@intFromEnum(inst)].bin_op;	103930	const bin_op = air_datas[@intFromEnum(inst)].bin_op;
103939	const array_ty = cg.typeOf(bin_op.lhs);	103931	const array_ty = cg.typeOf(bin_op.lhs);
103940	const res_ty = array_ty.elemType2(zcu);	103932	const res_ty = array_ty.elemType2(zcu);
...	@@ -171402,8 +171394,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -171402,8 +171394,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
171402	.aggregate_init => \|air_tag\| fallback: {	171394	.aggregate_init => \|air_tag\| fallback: {
171403	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;	171395	const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
171404	const agg_ty = ty_pl.ty.toType();	171396	const agg_ty = ty_pl.ty.toType();
171405	if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or	171397	if (agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) {
171406	(agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst);	171398	break :fallback try cg.airAggregateInitBoolVec(inst);
		171399	}
171407	var res = try cg.tempAllocMem(agg_ty);	171400	var res = try cg.tempAllocMem(agg_ty);
171408	const reset_index = cg.next_temp_index;	171401	const reset_index = cg.next_temp_index;
171409	var bt = cg.liveness.iterateBigTomb(inst);	171402	var bt = cg.liveness.iterateBigTomb(inst);
...	@@ -171441,10 +171434,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -171441,10 +171434,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
171441	}	171434	}
171442	}	171435	}
171443	},	171436	},
171444	.@"packed" => return cg.fail("failed to select {s} {f}", .{	171437	.@"packed" => unreachable,
171445	@tagName(air_tag),
171446	agg_ty.fmt(pt),
171447	}),
171448	}	171438	}
171449	},	171439	},
171450	.tuple_type => \|tuple_type\| {	171440	.tuple_type => \|tuple_type\| {
...	@@ -173054,10 +173044,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -173054,10 +173044,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173054	try ert.die(cg);	173044	try ert.die(cg);
173055	try res.finish(inst, &.{}, &.{}, cg);	173045	try res.finish(inst, &.{}, &.{}, cg);
173056	},	173046	},
173057	.vector_store_elem => {	173047	.runtime_nav_ptr => {
173058	const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem;	173048	const ty_nav = air_datas[@intFromEnum(inst)].ty_nav;
173059	const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data;	173049	const nav = ip.getNav(ty_nav.nav);
173060	var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs });	173050	const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip);
		173051	if (is_threadlocal) if (cg.target.ofmt == .coff or cg.mod.pic) {
		173052	try cg.spillRegisters(&.{ .rdi, .rax });
		173053	} else {
		173054	try cg.spillRegisters(&.{.rax});
		173055	};
		173056	var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav });
		173057	if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {};
		173058	try res.finish(inst, &.{}, &.{}, cg);
		173059	},
		173060	.c_va_arg => try cg.airVaArg(inst),
		173061	.c_va_copy => try cg.airVaCopy(inst),
		173062	.c_va_end => try cg.airVaEnd(inst),
		173063	.c_va_start => try cg.airVaStart(inst),
		173064	.legalize_vec_store_elem => {
		173065	const pl_op = air_datas[@intFromEnum(inst)].pl_op;
		173066	const bin = cg.air.extraData(Air.Bin, pl_op.payload).data;
		173067	// vector_ptr, index, elem_val
		173068	var ops = try cg.tempsFromOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
173061	cg.select(&.{}, &.{}, &ops, comptime &.{ .{	173069	cg.select(&.{}, &.{}, &ops, comptime &.{ .{
173062	.src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },	173070	.src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool },
173063	.patterns = &.{	173071	.patterns = &.{
...	@@ -173639,7 +173647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -173639,7 +173647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173639	} },	173647	} },
173640	} }) catch \|err\| switch (err) {	173648	} }) catch \|err\| switch (err) {
173641	error.SelectFailed => {	173649	error.SelectFailed => {
173642	const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu);	173650	const elem_size = cg.typeOf(bin.rhs).abiSize(zcu);
173643	while (try ops[0].toRegClass(true, .general_purpose, cg) or	173651	while (try ops[0].toRegClass(true, .general_purpose, cg) or
173644	try ops[1].toRegClass(true, .general_purpose, cg))	173652	try ops[1].toRegClass(true, .general_purpose, cg))
173645	{}	173653	{}
...	@@ -173681,23 +173689,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {	...	@@ -173681,23 +173689,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173681	};	173689	};
173682	for (ops) \|op\| try op.die(cg);	173690	for (ops) \|op\| try op.die(cg);
173683	},	173691	},
173684	.runtime_nav_ptr => {
173685	const ty_nav = air_datas[@intFromEnum(inst)].ty_nav;
173686	const nav = ip.getNav(ty_nav.nav);
173687	const is_threadlocal = zcu.comp.config.any_non_single_threaded and nav.isThreadlocal(ip);
173688	if (is_threadlocal) if (cg.target.ofmt == .coff or cg.mod.pic) {
173689	try cg.spillRegisters(&.{ .rdi, .rax });
173690	} else {
173691	try cg.spillRegisters(&.{.rax});
173692	};
173693	var res = try cg.tempInit(.fromInterned(ty_nav.ty), .{ .lea_nav = ty_nav.nav });
173694	if (is_threadlocal) while (try res.toRegClass(true, .general_purpose, cg)) {};
173695	try res.finish(inst, &.{}, &.{}, cg);
173696	},
173697	.c_va_arg => try cg.airVaArg(inst),
173698	.c_va_copy => try cg.airVaCopy(inst),
173699	.c_va_end => try cg.airVaEnd(inst),
173700	.c_va_start => try cg.airVaStart(inst),
173701	.work_item_id, .work_group_size, .work_group_id => unreachable,	173692	.work_item_id, .work_group_size, .work_group_id => unreachable,
173702	}	173693	}
173703	try cg.resetTemps(@enumFromInt(0));	173694	try cg.resetTemps(@enumFromInt(0));
...	@@ -180646,944 +180637,57 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {	...	@@ -180646,944 +180637,57 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
180646	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });	180637	return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs });
180647	}	180638	}
180648		180639
180649	fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void {	180640	fn airAggregateInitBoolVec(self: *CodeGen, inst: Air.Inst.Index) !void {
180650	const pt = self.pt;	180641	const pt = self.pt;
180651	const zcu = pt.zcu;	180642	const zcu = pt.zcu;
		180643	const result_ty = self.typeOfIndex(inst);
		180644	const len: usize = @intCast(result_ty.arrayLen(zcu));
180652	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;	180645	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
180653	const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;	180646	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
180654
180655	const dst_ty = self.typeOfIndex(inst);
180656	const elem_ty = dst_ty.childType(zcu);
180657	const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu));
180658	const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
180659	const lhs_ty = self.typeOf(extra.a);
180660	const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu));
180661	const rhs_ty = self.typeOf(extra.b);
180662	const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu));
180663	const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);
180664
180665	const ExpectedContents = [32]?i32;
180666	var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
180667	std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
180668	const allocator = stack.get();
180669
180670	const mask_elems = try allocator.alloc(?i32, extra.mask_len);
180671	defer allocator.free(mask_elems);
180672	for (mask_elems, 0..) \|*mask_elem, elem_index\| {
180673	const mask_elem_val =
180674	Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable;
180675	mask_elem.* = if (mask_elem_val.isUndef(zcu))
180676	null
180677	else
180678	@intCast(mask_elem_val.toSignedInt(zcu));
180679	}
180680
180681	const has_avx = self.hasFeature(.avx);
180682	const result = @as(?MCValue, result: {
180683	for (mask_elems) \|mask_elem\| {
180684	if (mask_elem) \|_\| break;
180685	} else break :result try self.allocRegOrMem(inst, true);
180686
180687	for (mask_elems, 0..) \|mask_elem, elem_index\| {
180688	if (mask_elem orelse continue != elem_index) break;
180689	} else {
180690	const lhs_mcv = try self.resolveInst(extra.a);
180691	if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;
180692	const dst_mcv = try self.allocRegOrMem(inst, true);
180693	try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});
180694	break :result dst_mcv;
180695	}
180696
180697	for (mask_elems, 0..) \|mask_elem, elem_index\| {
180698	if (~(mask_elem orelse continue) != elem_index) break;
180699	} else {
180700	const rhs_mcv = try self.resolveInst(extra.b);
180701	if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;
180702	const dst_mcv = try self.allocRegOrMem(inst, true);
180703	try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});
180704	break :result dst_mcv;
180705	}
180706
180707	for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) \|variant\| unpck: {
180708	if (elem_abi_size > 8) break :unpck;
180709	if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck;
180710
180711	var sources: [2]?u1 = @splat(null);
180712	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180713	const mask_elem = maybe_mask_elem orelse continue;
180714	const mask_elem_index =
180715	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck;
180716	const elem_byte = (elem_index >> 1) * elem_abi_size;
180717	if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) \| @as(u4, switch (variant) {
180718	.unpckl => 0b0000,
180719	.unpckh => 0b1000,
180720	else => unreachable,
180721	}) \| (elem_byte << 1 & 0b10000)) break :unpck;
180722
180723	const source = @intFromBool(mask_elem < 0);
180724	if (sources[elem_index & 0b00001]) \|prev_source\| {
180725	if (source != prev_source) break :unpck;
180726	} else sources[elem_index & 0b00001] = source;
180727	}
180728	if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck;
180729
180730	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180731	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180732	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180733	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180734
180735	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180736	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180737	lhs_mcv
180738	else if (has_avx and lhs_mcv.isRegister())
180739	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180740	else
180741	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180742	const dst_reg = dst_mcv.getReg().?;
180743	const dst_alias = registerAlias(dst_reg, max_abi_size);
180744
180745	const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
180746	(dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) {
180747	4 => if (has_avx) .v_ps else ._ps,
180748	8 => if (has_avx) .v_pd else ._pd,
180749	else => unreachable,
180750	}, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) {
180751	.unpckl => switch (elem_abi_size) {
180752	1 => .unpcklbw,
180753	2 => .unpcklwd,
180754	4 => .unpckldq,
180755	8 => .unpcklqdq,
180756	else => unreachable,
180757	},
180758	.unpckh => switch (elem_abi_size) {
180759	1 => .unpckhbw,
180760	2 => .unpckhwd,
180761	4 => .unpckhdq,
180762	8 => .unpckhqdq,
180763	else => unreachable,
180764	},
180765	else => unreachable,
180766	} };
180767	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory(
180768	mir_tag,
180769	dst_alias,
180770	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180771	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180772	) else try self.asmRegisterRegisterRegister(
180773	mir_tag,
180774	dst_alias,
180775	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180776	registerAlias(if (rhs_mcv.isRegister())
180777	rhs_mcv.getReg().?
180778	else
180779	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180780	) else if (rhs_mcv.isBase()) try self.asmRegisterMemory(
180781	mir_tag,
180782	dst_alias,
180783	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180784	) else try self.asmRegisterRegister(
180785	mir_tag,
180786	dst_alias,
180787	registerAlias(if (rhs_mcv.isRegister())
180788	rhs_mcv.getReg().?
180789	else
180790	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180791	);
180792	break :result dst_mcv;
180793	}
180794
180795	pshufd: {
180796	if (elem_abi_size != 4) break :pshufd;
180797	if (max_abi_size > self.vectorSize(.float)) break :pshufd;
180798
180799	var control: u8 = 0b00_00_00_00;
180800	var sources: [1]?u1 = @splat(null);
180801	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180802	const mask_elem = maybe_mask_elem orelse continue;
180803	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180804	if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd;
180805
180806	const source = @intFromBool(mask_elem < 0);
180807	if (sources[0]) \|prev_source\| {
180808	if (source != prev_source) break :pshufd;
180809	} else sources[(elem_index & 0b010) >> 1] = source;
180810
180811	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
180812	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
180813	if (elem_index & 0b100 == 0)
180814	control \|= select_mask
180815	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd;
180816	}
180817
180818	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180819	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180820	const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]);
180821
180822	const dst_reg = if (src_mcv.isRegister() and
180823	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv))
180824	src_mcv.getReg().?
180825	else
180826	try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
180827	const dst_alias = registerAlias(dst_reg, max_abi_size);
180828
180829	if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180830	.{ if (has_avx) .vp_d else .p_d, .shuf },
180831	dst_alias,
180832	try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180833	.u(control),
180834	) else try self.asmRegisterRegisterImmediate(
180835	.{ if (has_avx) .vp_d else .p_d, .shuf },
180836	dst_alias,
180837	registerAlias(if (src_mcv.isRegister())
180838	src_mcv.getReg().?
180839	else
180840	try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size),
180841	.u(control),
180842	);
180843	break :result .{ .register = dst_reg };
180844	}
180845
180846	shufps: {
180847	if (elem_abi_size != 4) break :shufps;
180848	if (max_abi_size > self.vectorSize(.float)) break :shufps;
180849
180850	var control: u8 = 0b00_00_00_00;
180851	var sources: [2]?u1 = @splat(null);
180852	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180853	const mask_elem = maybe_mask_elem orelse continue;
180854	const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180855	if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps;
180856
180857	const source = @intFromBool(mask_elem < 0);
180858	if (sources[(elem_index & 0b010) >> 1]) \|prev_source\| {
180859	if (source != prev_source) break :shufps;
180860	} else sources[(elem_index & 0b010) >> 1] = source;
180861
180862	const select_bit: u3 = @intCast((elem_index & 0b011) << 1);
180863	const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit;
180864	if (elem_index & 0b100 == 0)
180865	control \|= select_mask
180866	else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps;
180867	}
180868	if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps;
180869
180870	const operands = [2]Air.Inst.Ref{ extra.a, extra.b };
180871	const operand_tys = [2]Type{ lhs_ty, rhs_ty };
180872	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180873	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180874
180875	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180876	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180877	lhs_mcv
180878	else if (has_avx and lhs_mcv.isRegister())
180879	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180880	else
180881	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180882	const dst_reg = dst_mcv.getReg().?;
180883	const dst_alias = registerAlias(dst_reg, max_abi_size);
180884
180885	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
180886	.{ .v_ps, .shuf },
180887	dst_alias,
180888	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180889	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180890	.u(control),
180891	) else try self.asmRegisterRegisterRegisterImmediate(
180892	.{ .v_ps, .shuf },
180893	dst_alias,
180894	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180895	registerAlias(if (rhs_mcv.isRegister())
180896	rhs_mcv.getReg().?
180897	else
180898	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180899	.u(control),
180900	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180901	.{ ._ps, .shuf },
180902	dst_alias,
180903	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180904	.u(control),
180905	) else try self.asmRegisterRegisterImmediate(
180906	.{ ._ps, .shuf },
180907	dst_alias,
180908	registerAlias(if (rhs_mcv.isRegister())
180909	rhs_mcv.getReg().?
180910	else
180911	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180912	.u(control),
180913	);
180914	break :result dst_mcv;
180915	}
180916
180917	shufpd: {
180918	if (elem_abi_size != 8) break :shufpd;
180919	if (max_abi_size > self.vectorSize(.float)) break :shufpd;
180920
180921	var control: u4 = 0b0_0_0_0;
180922	var sources: [2]?u1 = @splat(null);
180923	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180924	const mask_elem = maybe_mask_elem orelse continue;
180925	const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);
180926	if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;
180927
180928	const source = @intFromBool(mask_elem < 0);
180929	if (sources[elem_index & 0b01]) \|prev_source\| {
180930	if (source != prev_source) break :shufpd;
180931	} else sources[elem_index & 0b01] = source;
180932
180933	control \|= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);
180934	}
180935	if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;
180936
180937	const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b };
180938	const operand_tys: [2]Type = .{ lhs_ty, rhs_ty };
180939	const lhs_mcv = try self.resolveInst(operands[sources[0].?]);
180940	const rhs_mcv = try self.resolveInst(operands[sources[1].?]);
180941
180942	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
180943	self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))
180944	lhs_mcv
180945	else if (has_avx and lhs_mcv.isRegister())
180946	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
180947	else
180948	try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);
180949	const dst_reg = dst_mcv.getReg().?;
180950	const dst_alias = registerAlias(dst_reg, max_abi_size);
180951
180952	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
180953	.{ .v_pd, .shuf },
180954	dst_alias,
180955	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180956	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180957	.u(control),
180958	) else try self.asmRegisterRegisterRegisterImmediate(
180959	.{ .v_pd, .shuf },
180960	dst_alias,
180961	registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),
180962	registerAlias(if (rhs_mcv.isRegister())
180963	rhs_mcv.getReg().?
180964	else
180965	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180966	.u(control),
180967	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
180968	.{ ._pd, .shuf },
180969	dst_alias,
180970	try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }),
180971	.u(control),
180972	) else try self.asmRegisterRegisterImmediate(
180973	.{ ._pd, .shuf },
180974	dst_alias,
180975	registerAlias(if (rhs_mcv.isRegister())
180976	rhs_mcv.getReg().?
180977	else
180978	try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),
180979	.u(control),
180980	);
180981	break :result dst_mcv;
180982	}
180983
180984	blend: {
180985	if (elem_abi_size < 2) break :blend;
180986	if (dst_abi_size > self.vectorSize(.float)) break :blend;
180987	if (!self.hasFeature(.sse4_1)) break :blend;
180988
180989	var control: u8 = 0b0_0_0_0_0_0_0_0;
180990	for (mask_elems, 0..) \|maybe_mask_elem, elem_index\| {
180991	const mask_elem = maybe_mask_elem orelse continue;
180992	const mask_elem_index =
180993	std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend;
180994	if (mask_elem_index != elem_index) break :blend;
180995
180996	const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index);
180997	if (elem_index & 0b1000 == 0)
180998	control \|= select_mask
180999	else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend;
181000	}
181001
181002	if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: {
181003	const expanded_control = switch (elem_abi_size) {
181004	4 => control,
181005	8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
181006	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
181007	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
181008	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00),
181009	else => break :vpblendd,
181010	};
181011
181012	const lhs_mcv = try self.resolveInst(extra.a);
181013	const lhs_reg = if (lhs_mcv.isRegister())
181014	lhs_mcv.getReg().?
181015	else
181016	try self.copyToTmpRegister(dst_ty, lhs_mcv);
181017	const lhs_lock = self.register_manager.lockReg(lhs_reg);
181018	defer if (lhs_lock) \|lock\| self.register_manager.unlockReg(lock);
181019
181020	const rhs_mcv = try self.resolveInst(extra.b);
181021	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
181022	if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181023	.{ .vp_d, .blend },
181024	registerAlias(dst_reg, dst_abi_size),
181025	registerAlias(lhs_reg, dst_abi_size),
181026	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181027	.u(expanded_control),
181028	) else try self.asmRegisterRegisterRegisterImmediate(
181029	.{ .vp_d, .blend },
181030	registerAlias(dst_reg, dst_abi_size),
181031	registerAlias(lhs_reg, dst_abi_size),
181032	registerAlias(if (rhs_mcv.isRegister())
181033	rhs_mcv.getReg().?
181034	else
181035	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181036	.u(expanded_control),
181037	);
181038	break :result .{ .register = dst_reg };
181039	}
181040
181041	if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: {
181042	const expanded_control = switch (elem_abi_size) {
181043	2 => control,
181044	4 => if (dst_abi_size <= 16 or
181045	@as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0)))
181046	@as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) \|
181047	@as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) \|
181048	@as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) \|
181049	@as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00)
181050	else
181051	break :pblendw,
181052	8 => if (dst_abi_size <= 16 or
181053	@as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0)))
181054	@as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) \|
181055	@as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000)
181056	else
181057	break :pblendw,
181058	16 => break :pblendw,
181059	else => unreachable,
181060	};
181061
181062	const lhs_mcv = try self.resolveInst(extra.a);
181063	const rhs_mcv = try self.resolveInst(extra.b);
181064
181065	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181066	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181067	lhs_mcv
181068	else if (has_avx and lhs_mcv.isRegister())
181069	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181070	else
181071	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181072	const dst_reg = dst_mcv.getReg().?;
181073
181074	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181075	.{ .vp_w, .blend },
181076	registerAlias(dst_reg, dst_abi_size),
181077	registerAlias(if (lhs_mcv.isRegister())
181078	lhs_mcv.getReg().?
181079	else
181080	dst_reg, dst_abi_size),
181081	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181082	.u(expanded_control),
181083	) else try self.asmRegisterRegisterRegisterImmediate(
181084	.{ .vp_w, .blend },
181085	registerAlias(dst_reg, dst_abi_size),
181086	registerAlias(if (lhs_mcv.isRegister())
181087	lhs_mcv.getReg().?
181088	else
181089	dst_reg, dst_abi_size),
181090	registerAlias(if (rhs_mcv.isRegister())
181091	rhs_mcv.getReg().?
181092	else
181093	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181094	.u(expanded_control),
181095	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
181096	.{ .p_w, .blend },
181097	registerAlias(dst_reg, dst_abi_size),
181098	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181099	.u(expanded_control),
181100	) else try self.asmRegisterRegisterImmediate(
181101	.{ .p_w, .blend },
181102	registerAlias(dst_reg, dst_abi_size),
181103	registerAlias(if (rhs_mcv.isRegister())
181104	rhs_mcv.getReg().?
181105	else
181106	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181107	.u(expanded_control),
181108	);
181109	break :result .{ .register = dst_reg };
181110	}
181111
181112	const expanded_control = switch (elem_abi_size) {
181113	4, 8 => control,
181114	16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) \|
181115	@as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00),
181116	else => unreachable,
181117	};
181118
181119	const lhs_mcv = try self.resolveInst(extra.a);
181120	const rhs_mcv = try self.resolveInst(extra.b);
181121
181122	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181123	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181124	lhs_mcv
181125	else if (has_avx and lhs_mcv.isRegister())
181126	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181127	else
181128	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181129	const dst_reg = dst_mcv.getReg().?;
181130
181131	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
181132	switch (elem_abi_size) {
181133	4 => .{ .v_ps, .blend },
181134	8, 16 => .{ .v_pd, .blend },
181135	else => unreachable,
181136	},
181137	registerAlias(dst_reg, dst_abi_size),
181138	registerAlias(if (lhs_mcv.isRegister())
181139	lhs_mcv.getReg().?
181140	else
181141	dst_reg, dst_abi_size),
181142	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181143	.u(expanded_control),
181144	) else try self.asmRegisterRegisterRegisterImmediate(
181145	switch (elem_abi_size) {
181146	4 => .{ .v_ps, .blend },
181147	8, 16 => .{ .v_pd, .blend },
181148	else => unreachable,
181149	},
181150	registerAlias(dst_reg, dst_abi_size),
181151	registerAlias(if (lhs_mcv.isRegister())
181152	lhs_mcv.getReg().?
181153	else
181154	dst_reg, dst_abi_size),
181155	registerAlias(if (rhs_mcv.isRegister())
181156	rhs_mcv.getReg().?
181157	else
181158	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181159	.u(expanded_control),
181160	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate(
181161	switch (elem_abi_size) {
181162	4 => .{ ._ps, .blend },
181163	8, 16 => .{ ._pd, .blend },
181164	else => unreachable,
181165	},
181166	registerAlias(dst_reg, dst_abi_size),
181167	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181168	.u(expanded_control),
181169	) else try self.asmRegisterRegisterImmediate(
181170	switch (elem_abi_size) {
181171	4 => .{ ._ps, .blend },
181172	8, 16 => .{ ._pd, .blend },
181173	else => unreachable,
181174	},
181175	registerAlias(dst_reg, dst_abi_size),
181176	registerAlias(if (rhs_mcv.isRegister())
181177	rhs_mcv.getReg().?
181178	else
181179	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181180	.u(expanded_control),
181181	);
181182	break :result .{ .register = dst_reg };
181183	}
181184
181185	blendv: {
181186	if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv;
181187
181188	const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8);
181189	const select_mask_ty = try pt.vectorType(.{
181190	.len = @intCast(mask_elems.len),
181191	.child = select_mask_elem_ty.toIntern(),
181192	});
181193	var select_mask_elems: [32]InternPool.Index = undefined;
181194	for (
181195	select_mask_elems[0..mask_elems.len],
181196	mask_elems,
181197	0..,
181198	) \|*select_mask_elem, maybe_mask_elem, elem_index\| {
181199	const mask_elem = maybe_mask_elem orelse continue;
181200	const mask_elem_index =
181201	std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv;
181202	if (mask_elem_index != elem_index) break :blendv;
181203
181204	select_mask_elem.* = (if (mask_elem < 0)
181205	try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty)
181206	else
181207	try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern();
181208	}
181209	const select_mask_mcv = try self.lowerValue(
181210	try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]),
181211	);
181212		180647
181213	if (self.hasFeature(.sse4_1)) {	180648	assert(result_ty.zigTypeTag(zcu) == .vector);
181214	const mir_tag: Mir.Inst.FixedTag = .{	180649	assert(result_ty.childType(zcu).toIntern() == .bool_type);
181215	if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or
181216	(dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) {
181217	4 => if (has_avx) .v_ps else ._ps,
181218	8 => if (has_avx) .v_pd else ._pd,
181219	else => unreachable,
181220	} else if (has_avx) .vp_b else .p_b,
181221	.blendv,
181222	};
181223		180650
181224	const select_mask_reg = if (!has_avx) reg: {	180651	const result_size = result_ty.abiSize(zcu);
181225	try self.register_manager.getKnownReg(.xmm0, null);	180652	if (result_size > 8) return self.fail("TODO airAggregateInitBoolVec over 8 bytes", .{});
181226	try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{});
181227	break :reg .xmm0;
181228	} else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);
181229	const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size);
181230	const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg);
181231	defer self.register_manager.unlockReg(select_mask_lock);
181232
181233	const lhs_mcv = try self.resolveInst(extra.a);
181234	const rhs_mcv = try self.resolveInst(extra.b);
181235
181236	const dst_mcv: MCValue = if (lhs_mcv.isRegister() and
181237	self.reuseOperand(inst, extra.a, 0, lhs_mcv))
181238	lhs_mcv
181239	else if (has_avx and lhs_mcv.isRegister())
181240	.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
181241	else
181242	try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv);
181243	const dst_reg = dst_mcv.getReg().?;
181244	const dst_alias = registerAlias(dst_reg, dst_abi_size);
181245		180653
181246	if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister(	180654	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
181247	mir_tag,
181248	dst_alias,
181249	if (lhs_mcv.isRegister())
181250	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
181251	else
181252	dst_alias,
181253	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181254	select_mask_alias,
181255	) else try self.asmRegisterRegisterRegisterRegister(
181256	mir_tag,
181257	dst_alias,
181258	if (lhs_mcv.isRegister())
181259	registerAlias(lhs_mcv.getReg().?, dst_abi_size)
181260	else
181261	dst_alias,
181262	registerAlias(if (rhs_mcv.isRegister())
181263	rhs_mcv.getReg().?
181264	else
181265	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181266	select_mask_alias,
181267	) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister(
181268	mir_tag,
181269	dst_alias,
181270	try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181271	select_mask_alias,
181272	) else try self.asmRegisterRegisterRegister(
181273	mir_tag,
181274	dst_alias,
181275	registerAlias(if (rhs_mcv.isRegister())
181276	rhs_mcv.getReg().?
181277	else
181278	try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size),
181279	select_mask_alias,
181280	);
181281	break :result dst_mcv;
181282	}
181283		180655
181284	const lhs_mcv = try self.resolveInst(extra.a);	180656	{
181285	const rhs_mcv = try self.resolveInst(extra.b);	180657	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
		180658	defer self.register_manager.unlockReg(dst_lock);
181286		180659
181287	const dst_mcv: MCValue = if (rhs_mcv.isRegister() and	180660	try self.spillEflagsIfOccupied();
181288	self.reuseOperand(inst, extra.b, 1, rhs_mcv))	180661	try self.asmRegisterRegister(
181289	rhs_mcv	180662	.{ ._, .xor },
181290	else	180663	registerAlias(dst_reg, @min(result_size, 4)),
181291	try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv);	180664	registerAlias(dst_reg, @min(result_size, 4)),
181292	const dst_reg = dst_mcv.getReg().?;	180665	);
181293	const dst_alias = registerAlias(dst_reg, dst_abi_size);
181294		180666
181295	const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv);	180667	for (elements, 0..) \|elem, elem_i\| {
181296	const mask_alias = registerAlias(mask_reg, dst_abi_size);	180668	const elem_reg = try self.copyToTmpRegister(.bool, .{ .air_ref = elem });
181297	const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg);	180669	const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
181298	defer self.register_manager.unlockReg(mask_lock);	180670	defer self.register_manager.unlockReg(elem_lock);
181299		180671
181300	const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat())	180672	try self.asmRegisterImmediate(
181301	switch (elem_ty.floatBits(self.target)) {	180673	.{ ._, .@"and" },
181302	16, 80, 128 => .p_,	180674	registerAlias(elem_reg, @min(result_size, 4)),
181303	32 => ._ps,	180675	.u(1),
181304	64 => ._pd,
181305	else => unreachable,
181306	}
181307	else
181308	.p_;
181309	try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias);
181310	if (lhs_mcv.isBase()) try self.asmRegisterMemory(
181311	.{ mir_fixes, .andn },
181312	mask_alias,
181313	try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }),
181314	) else try self.asmRegisterRegister(
181315	.{ mir_fixes, .andn },
181316	mask_alias,
181317	if (lhs_mcv.isRegister())
181318	lhs_mcv.getReg().?
181319	else
181320	try self.copyToTmpRegister(dst_ty, lhs_mcv),
181321	);	180676	);
181322	try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias);	180677	if (elem_i > 0) try self.asmRegisterImmediate(
181323	break :result dst_mcv;	180678	.{ ._l, .sh },
181324	}	180679	registerAlias(elem_reg, @intCast(result_size)),
181325		180680	.u(@intCast(elem_i)),
181326	pshufb: {
181327	if (max_abi_size > 16) break :pshufb;
181328	if (!self.hasFeature(.ssse3)) break :pshufb;
181329
181330	const temp_regs =
181331	try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);
181332	const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);
181333	defer for (temp_locks) \|lock\| self.register_manager.unlockReg(lock);
181334
181335	const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);
181336	try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});
181337
181338	const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);
181339	try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});
181340
181341	var lhs_mask_elems: [16]InternPool.Index = undefined;
181342	for (lhs_mask_elems[0..max_abi_size], 0..) \|*lhs_mask_elem, byte_index\| {
181343	const elem_index = byte_index / elem_abi_size;
181344	lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
181345	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
181346	if (mask_elem < 0) break :elem 0b1_00_00000;
181347	const mask_elem_index: u31 = @intCast(mask_elem);
181348	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
181349	break :elem mask_elem_index * elem_abi_size + byte_off;
181350	})).toIntern();
181351	}
181352	const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
181353	const lhs_mask_mcv = try self.lowerValue(
181354	try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]),
181355	);	180681	);
181356	const lhs_mask_mem: Memory = .{	180682	try self.asmRegisterRegister(
181357	.base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) },	180683	.{ ._, .@"or" },
181358	.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },	180684	registerAlias(dst_reg, @intCast(result_size)),
181359	};	180685	registerAlias(elem_reg, @intCast(result_size)),
181360	if (has_avx) try self.asmRegisterRegisterMemory(
181361	.{ .vp_b, .shuf },
181362	lhs_temp_alias,
181363	lhs_temp_alias,
181364	lhs_mask_mem,
181365	) else try self.asmRegisterMemory(
181366	.{ .p_b, .shuf },
181367	lhs_temp_alias,
181368	lhs_mask_mem,
181369	);	180686	);
181370
181371	var rhs_mask_elems: [16]InternPool.Index = undefined;
181372	for (rhs_mask_elems[0..max_abi_size], 0..) \|*rhs_mask_elem, byte_index\| {
181373	const elem_index = byte_index / elem_abi_size;
181374	rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {
181375	const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;
181376	if (mask_elem >= 0) break :elem 0b1_00_00000;
181377	const mask_elem_index: u31 = @intCast(~mask_elem);
181378	const byte_off: u32 = @intCast(byte_index % elem_abi_size);
181379	break :elem mask_elem_index * elem_abi_size + byte_off;
181380	})).toIntern();
181381	}
181382	const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type });
181383	const rhs_mask_mcv = try self.lowerValue(
181384	try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]),
181385	);
181386	const rhs_mask_mem: Memory = .{
181387	.base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) },
181388	.mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } },
181389	};
181390	if (has_avx) try self.asmRegisterRegisterMemory(
181391	.{ .vp_b, .shuf },
181392	rhs_temp_alias,
181393	rhs_temp_alias,
181394	rhs_mask_mem,
181395	) else try self.asmRegisterMemory(
181396	.{ .p_b, .shuf },
181397	rhs_temp_alias,
181398	rhs_mask_mem,
181399	);
181400
181401	if (has_avx) try self.asmRegisterRegisterRegister(
181402	.{ switch (elem_ty.zigTypeTag(zcu)) {
181403	else => break :result null,
181404	.int => .vp_,
181405	.float => switch (elem_ty.floatBits(self.target)) {
181406	32 => .v_ps,
181407	64 => .v_pd,
181408	16, 80, 128 => break :result null,
181409	else => unreachable,
181410	},
181411	}, .@"or" },
181412	lhs_temp_alias,
181413	lhs_temp_alias,
181414	rhs_temp_alias,
181415	) else try self.asmRegisterRegister(
181416	.{ switch (elem_ty.zigTypeTag(zcu)) {
181417	else => break :result null,
181418	.int => .p_,
181419	.float => switch (elem_ty.floatBits(self.target)) {
181420	32 => ._ps,
181421	64 => ._pd,
181422	16, 80, 128 => break :result null,
181423	else => unreachable,
181424	},
181425	}, .@"or" },
181426	lhs_temp_alias,
181427	rhs_temp_alias,
181428	);
181429	break :result .{ .register = temp_regs[0] };
181430	}	180687	}
		180688	}
181431		180689
181432	break :result null;	180690	const result: MCValue = .{ .register = dst_reg };
181433	}) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{
181434	lhs_ty.fmt(pt),
181435	rhs_ty.fmt(pt),
181436	dst_ty.fmt(pt),
181437	Value.fromInterned(extra.mask).fmtValue(pt),
181438	});
181439	return self.finishAir(inst, result, .{ extra.a, extra.b, .none });
181440	}
181441
181442	fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
181443	const pt = self.pt;
181444	const zcu = pt.zcu;
181445	const result_ty = self.typeOfIndex(inst);
181446	const len: usize = @intCast(result_ty.arrayLen(zcu));
181447	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
181448	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
181449	const result: MCValue = result: {
181450	switch (result_ty.zigTypeTag(zcu)) {
181451	.@"struct" => {
181452	if (result_ty.containerLayout(zcu) == .@"packed") return self.fail(
181453	"TODO implement airAggregateInit for {f}",
181454	.{result_ty.fmt(pt)},
181455	);
181456	const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
181457	const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
181458	try self.genInlineMemset(
181459	.{ .lea_frame = .{ .index = frame_index } },
181460	.{ .immediate = 0 },
181461	.{ .immediate = result_ty.abiSize(zcu) },
181462	.{},
181463	);
181464	for (elements, 0..) \|elem, elem_i_usize\| {
181465	const elem_i: u32 = @intCast(elem_i_usize);
181466	if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
181467
181468	const elem_ty = result_ty.fieldType(elem_i, zcu);
181469	const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
181470	if (elem_bit_size > 64) {
181471	return self.fail(
181472	"TODO airAggregateInit implement packed structs with large fields",
181473	.{},
181474	);
181475	}
181476	const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
181477	const elem_abi_bits = elem_abi_size * 8;
181478	const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i);
181479	const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
181480	const elem_bit_off = elem_off % elem_abi_bits;
181481	const elem_mcv = try self.resolveInst(elem);
181482	const elem_lock = switch (elem_mcv) {
181483	.register => \|reg\| self.register_manager.lockReg(reg),
181484	.immediate => \|imm\| lock: {
181485	if (imm == 0) continue;
181486	break :lock null;
181487	},
181488	else => null,
181489	};
181490	defer if (elem_lock) \|lock\| self.register_manager.unlockReg(lock);
181491
181492	const elem_extra_bits = self.regExtraBits(elem_ty);
181493	{
181494	const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181495	const temp_alias = registerAlias(temp_reg, elem_abi_size);
181496	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181497	defer self.register_manager.unlockReg(temp_lock);
181498
181499	if (elem_bit_off < elem_extra_bits) {
181500	try self.truncateRegister(elem_ty, temp_alias);
181501	}
181502	if (elem_bit_off > 0) try self.genShiftBinOpMir(
181503	.{ ._l, .sh },
181504	elem_ty,
181505	.{ .register = temp_alias },
181506	.u8,
181507	.{ .immediate = elem_bit_off },
181508	);
181509	try self.genBinOpMir(
181510	.{ ._, .@"or" },
181511	elem_ty,
181512	.{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
181513	.{ .register = temp_alias },
181514	);
181515	}
181516	if (elem_bit_off > elem_extra_bits) {
181517	const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181518	const temp_alias = registerAlias(temp_reg, elem_abi_size);
181519	const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181520	defer self.register_manager.unlockReg(temp_lock);
181521
181522	if (elem_extra_bits > 0) {
181523	try self.truncateRegister(elem_ty, temp_alias);
181524	}
181525	try self.genShiftBinOpMir(
181526	.{ ._r, .sh },
181527	elem_ty,
181528	.{ .register = temp_reg },
181529	.u8,
181530	.{ .immediate = elem_abi_bits - elem_bit_off },
181531	);
181532	try self.genBinOpMir(
181533	.{ ._, .@"or" },
181534	elem_ty,
181535	.{ .load_frame = .{
181536	.index = frame_index,
181537	.off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
181538	} },
181539	.{ .register = temp_alias },
181540	);
181541	}
181542	}
181543	break :result .{ .load_frame = .{ .index = frame_index } };
181544	},
181545	.vector => {
181546	const elem_ty = result_ty.childType(zcu);
181547	if (elem_ty.toIntern() != .bool_type) return self.fail(
181548	"TODO implement airAggregateInit for {f}",
181549	.{result_ty.fmt(pt)},
181550	);
181551	const result_size: u32 = @intCast(result_ty.abiSize(zcu));
181552	const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
181553	const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
181554	defer self.register_manager.unlockReg(dst_lock);
181555	try self.asmRegisterRegister(
181556	.{ ._, .xor },
181557	registerAlias(dst_reg, @min(result_size, 4)),
181558	registerAlias(dst_reg, @min(result_size, 4)),
181559	);
181560
181561	for (elements, 0..) \|elem, elem_i\| {
181562	const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
181563	const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
181564	defer self.register_manager.unlockReg(elem_lock);
181565
181566	try self.asmRegisterImmediate(
181567	.{ ._, .@"and" },
181568	registerAlias(elem_reg, @min(result_size, 4)),
181569	.u(1),
181570	);
181571	if (elem_i > 0) try self.asmRegisterImmediate(
181572	.{ ._l, .sh },
181573	registerAlias(elem_reg, result_size),
181574	.u(@intCast(elem_i)),
181575	);
181576	try self.asmRegisterRegister(
181577	.{ ._, .@"or" },
181578	registerAlias(dst_reg, result_size),
181579	registerAlias(elem_reg, result_size),
181580	);
181581	}
181582	break :result .{ .register = dst_reg };
181583	},
181584	else => unreachable,
181585	}
181586	};
181587		180691
181588	if (elements.len <= Air.Liveness.bpi - 1) {	180692	if (elements.len <= Air.Liveness.bpi - 1) {
181589	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);	180693	var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none);
...	@@ -182269,15 +181373,6 @@ fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMe	...	@@ -182269,15 +181373,6 @@ fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMe
182269	};	181373	};
182270	}	181374	}
182271		181375
182272	fn failMsg(cg: CodeGen, msg: Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } {
182273	@branchHint(.cold);
182274	const zcu = cg.pt.zcu;
182275	return switch (cg.owner) {
182276	.nav_index => \|i\| zcu.codegenFailMsg(i, msg),
182277	.lazy_sym => \|s\| zcu.codegenFailTypeMsg(s.ty, msg),
182278	};
182279	}
182280
182281	fn parseRegName(name: []const u8) ?Register {	181376	fn parseRegName(name: []const u8) ?Register {
182282	if (std.mem.startsWith(u8, name, "db")) return @enumFromInt(	181377	if (std.mem.startsWith(u8, name, "db")) return @enumFromInt(
182283	@intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null),	181378	@intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null),
...	@@ -188819,7 +187914,6 @@ const Select = struct {	...	@@ -188819,7 +187914,6 @@ const Select = struct {
188819	const ptr_info = ty.ptrInfo(zcu);	187914	const ptr_info = ty.ptrInfo(zcu);
188820	return switch (ptr_info.flags.vector_index) {	187915	return switch (ptr_info.flags.vector_index) {
188821	.none => false,	187916	.none => false,
188822	.runtime => unreachable,
188823	else => ptr_info.child == .bool_type,	187917	else => ptr_info.child == .bool_type,
188824	};	187918	};
188825	},	187919	},
...	@@ -188827,7 +187921,6 @@ const Select = struct {	...	@@ -188827,7 +187921,6 @@ const Select = struct {
188827	const ptr_info = ty.ptrInfo(zcu);	187921	const ptr_info = ty.ptrInfo(zcu);
188828	return switch (ptr_info.flags.vector_index) {	187922	return switch (ptr_info.flags.vector_index) {
188829	.none => false,	187923	.none => false,
188830	.runtime => unreachable,
188831	else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size,	187924	else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size,
188832	};	187925	};
188833	},	187926	},
...	@@ -190814,7 +189907,7 @@ const Select = struct {	...	@@ -190814,7 +189907,7 @@ const Select = struct {
190814	.src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *	189907	.src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) *
190815	Select.Operand.Ref.src1.valueOf(s).immediate),	189908	Select.Operand.Ref.src1.valueOf(s).immediate),
190816	.vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) {	189909	.vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) {
190817	.none, .runtime => unreachable,	189910	.none => unreachable,
190818	else => \|vector_index\| @intFromEnum(vector_index),	189911	else => \|vector_index\| @intFromEnum(vector_index),
190819	},	189912	},
190820	.src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate),	189913	.src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate),

src/link/Dwarf.zig+6-22

 @@ -3158,11 +3158,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo
                                     .struct_field
                             else
                                 .struct_field);
-                            if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else {
+                            try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
-                                var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
-                                const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
-                                try wip_nav.strp(field_name);
-                            }
                             try wip_nav.refType(field_type);
                             if (!is_comptime) {
                                 try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
 @@ -3187,7 +3183,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo
                     var field_bit_offset: u16 = 0;
                     for (0..loaded_struct.field_types.len) |field_index| {
                         try wip_nav.abbrevCode(.packed_struct_field);
-                        try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip));
+                        try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
                         const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
                         try wip_nav.refType(field_type);
                         try diw.writeUleb128(field_bit_offset);
 @@ -4269,11 +4265,7 @@ fn updateLazyValue(
                             .comptime_value_field_runtime_bits
                         else
                             continue);
-                        if (loaded_struct_type.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else {
+                        try wip_nav.strp(loaded_struct_type.fieldName(ip, field_index).toSlice(ip));
-                            var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
-                            const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
-                            try wip_nav.strp(field_name);
-                        }
                         const field_value: Value = .fromInterned(switch (aggregate.storage) {
                             .bytes => unreachable,
                             .elems => |elems| elems[field_index],
 @@ -4467,11 +4459,7 @@ fn updateContainerTypeWriterError(
                         .struct_field
                 else
                     .struct_field);
-                if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else {
+                try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
-                    var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
-                    const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
-                    try wip_nav.strp(field_name);
-                }
                 try wip_nav.refType(field_type);
                 if (!is_comptime) {
                     try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
 @@ -4573,11 +4561,7 @@ fn updateContainerTypeWriterError(
                                         .struct_field
                                 else
                                     .struct_field);
-                                if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else {
+                                try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
-                                    var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined;
-                                    const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable;
-                                    try wip_nav.strp(field_name);
-                                }
                                 try wip_nav.refType(field_type);
                                 if (!is_comptime) {
                                     try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]);
 @@ -4600,7 +4584,7 @@ fn updateContainerTypeWriterError(
                         var field_bit_offset: u16 = 0;
                         for (0..loaded_struct.field_types.len) |field_index| {
                             try wip_nav.abbrevCode(.packed_struct_field);
-                            try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip));
+                            try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip));
                             const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]);
                             try wip_nav.refType(field_type);
                             try diw.writeUleb128(field_bit_offset);

stage1/zig.h+25-34

 @@ -40,6 +40,8 @@
 #elif defined(__mips__)
 #define zig_mips32
 #define zig_mips
+#elif defined(__or1k__)
+#define zig_or1k
 #elif defined(__powerpc64__)
 #define zig_powerpc64
 #define zig_powerpc
 @@ -72,6 +74,9 @@
 #elif defined (__x86_64__) || (defined(zig_msvc) && defined(_M_X64))
 #define zig_x86_64
 #define zig_x86
+#elif defined(__I86__)
+#define zig_x86_16
+#define zig_x86
 #endif
 #if defined(zig_msvc) || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 @@ -82,9 +87,7 @@
 #define zig_big_endian 1
 #endif
-#if defined(_AIX)
+#if defined(__MACH__)
-#define zig_aix
-#elif defined(__MACH__)
 #define zig_darwin
 #elif defined(__DragonFly__)
 #define zig_dragonfly
 @@ -114,20 +117,14 @@
 #define zig_wasi
 #elif defined(_WIN32)
 #define zig_windows
-#elif defined(__MVS__)
-#define zig_zos
 #endif
 #if defined(zig_windows)
 #define zig_coff
 #elif defined(__ELF__)
 #define zig_elf
-#elif defined(zig_zos)
-#define zig_goff
 #elif defined(zig_darwin)
 #define zig_macho
-#elif defined(zig_aix)
-#define zig_xcoff
 #endif
 #define zig_concat(lhs, rhs) lhs##rhs
 @@ -390,12 +387,16 @@
 #define zig_trap() __asm__ volatile(".word 0x0")
 #elif defined(zig_mips)
 #define zig_trap() __asm__ volatile(".word 0x3d")
+#elif defined(zig_or1k)
+#define zig_trap() __asm__ volatile("l.cust8")
 #elif defined(zig_riscv)
 #define zig_trap() __asm__ volatile("unimp")
 #elif defined(zig_s390x)
 #define zig_trap() __asm__ volatile("j 0x2")
 #elif defined(zig_sparc)
 #define zig_trap() __asm__ volatile("illtrap")
+#elif defined(zig_x86_16)
+#define zig_trap() __asm__ volatile("int $0x3")
 #elif defined(zig_x86)
 #define zig_trap() __asm__ volatile("ud2")
 #else
 @@ -422,6 +423,8 @@
 #define zig_breakpoint() __asm__ volatile("break 0x0")
 #elif defined(zig_mips)
 #define zig_breakpoint() __asm__ volatile("break")
+#elif defined(zig_or1k)
+#define zig_breakpoint() __asm__ volatile("l.trap 0x0")
 #elif defined(zig_powerpc)
 #define zig_breakpoint() __asm__ volatile("trap")
 #elif defined(zig_riscv)
 @@ -804,15 +807,13 @@ static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
 #endif
 }
-zig_extern int32_t  __addosi4(int32_t lhs, int32_t rhs, int *overflow);
 static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow) || defined(zig_gcc)
     int32_t full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int32_t full_res = (int32_t)((uint32_t)lhs + (uint32_t)rhs);
-    int32_t full_res = __addosi4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i32(full_res, bits);
     return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits);
 @@ -830,15 +831,13 @@ static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
 #endif
 }
-zig_extern int64_t  __addodi4(int64_t lhs, int64_t rhs, int *overflow);
 static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow) || defined(zig_gcc)
     int64_t full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int64_t full_res = (int64_t)((uint64_t)lhs + (uint64_t)rhs);
-    int64_t full_res = __addodi4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i64(full_res, bits);
     return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits);
 @@ -912,15 +911,13 @@ static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8
 #endif
 }
-zig_extern int32_t  __subosi4(int32_t lhs, int32_t rhs, int *overflow);
 static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow) || defined(zig_gcc)
     int32_t full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int32_t full_res = (int32_t)((uint32_t)lhs - (uint32_t)rhs);
-    int32_t full_res = __subosi4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i32(full_res, bits);
     return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits);
 @@ -938,15 +935,13 @@ static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8
 #endif
 }
-zig_extern int64_t  __subodi4(int64_t lhs, int64_t rhs, int *overflow);
 static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow) || defined(zig_gcc)
     int64_t full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    int64_t full_res = (int64_t)((uint64_t)lhs - (uint64_t)rhs);
-    int64_t full_res = __subodi4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i64(full_res, bits);
     return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits);
 @@ -1750,15 +1745,13 @@ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
 #endif
 }
-zig_extern zig_i128  __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
 static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
 #if zig_has_builtin(add_overflow)
     zig_i128 full_res;
     bool overflow = __builtin_add_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    zig_i128 full_res = (zig_i128)((zig_u128)lhs + (zig_u128)rhs);
-    zig_i128 full_res =  __addoti4(lhs, rhs, &overflow_int);
+    bool overflow = ((full_res ^ lhs) & (full_res ^ rhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i128(full_res, bits);
     return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits);
 @@ -1776,15 +1769,13 @@ static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint
 #endif
 }
-zig_extern zig_i128  __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow);
 static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) {
 #if zig_has_builtin(sub_overflow)
     zig_i128 full_res;
     bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res);
 #else
-    int overflow_int;
+    zig_i128 full_res = (zig_i128)((zig_u128)lhs - (zig_u128)rhs);
-    zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int);
+    bool overflow = ((lhs ^ rhs) & (full_res ^ lhs)) < 0;
-    bool overflow = overflow_int != 0;
 #endif
     *res = zig_wrap_i128(full_res, bits);
     return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits);
 @@ -4213,7 +4204,7 @@ static inline void zig_loongarch_cpucfg(uint32_t word, uint32_t* result) {
 #endif
 }
-#elif defined(zig_x86)
+#elif defined(zig_x86) && !defined(zig_x86_16)
 static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) {
 #if defined(zig_msvc)

test/behavior/union.zig+4-1

 @@ -218,10 +218,13 @@ test "union with specified enum tag" {
 }
 test "packed union generates correctly aligned type" {
+    // This test will be removed after the following accepted proposal is implemented:
+    // https://github.com/ziglang/zig/issues/24657
     if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
     if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
     if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
+    if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
     const U = packed union {
         f1: *const fn () error{TestUnexpectedResult}!void,
 @@ -1544,7 +1547,7 @@ test "packed union field pointer has correct alignment" {
     const host_size = switch (builtin.zig_backend) {
         else => comptime std.math.divCeil(comptime_int, @bitSizeOf(S), 8) catch unreachable,
-        .stage2_x86_64 => @sizeOf(S),
+        .stage2_x86_64, .stage2_c => @sizeOf(S),
     };
     comptime assert(@TypeOf(ap) == *align(4:2:host_size) u20);
     comptime assert(@TypeOf(bp) == *align(1:2:host_size) u20);

...	@@ -24,7 +24,6 @@ pub fn loadComptimePtr(sema: Sema, block: Block, src: LazySrcLoc, ptr: Value)	...	@@ -24,7 +24,6 @@ pub fn loadComptimePtr(sema: Sema, block: Block, src: LazySrcLoc, ptr: Value)
24	const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);	24	const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
25	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {	25	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
26	.none => 0,	26	.none => 0,
27	.runtime => return .runtime_load,
28	else => \|idx\| switch (pt.zcu.getTarget().cpu.arch.endian()) {	27	else => \|idx\| switch (pt.zcu.getTarget().cpu.arch.endian()) {
29	.little => child_bits * @intFromEnum(idx),	28	.little => child_bits * @intFromEnum(idx),
30	.big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian	29	.big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
...	@@ -81,7 +80,6 @@ pub fn storeComptimePtr(	...	@@ -81,7 +80,6 @@ pub fn storeComptimePtr(
81	};	80	};
82	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {	81	const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
83	.none => 0,	82	.none => 0,
84	.runtime => return .runtime_store,
85	else => \|idx\| switch (zcu.getTarget().cpu.arch.endian()) {	83	else => \|idx\| switch (zcu.getTarget().cpu.arch.endian()) {
86	.little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),	84	.little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
87	.big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian	85	.big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian