Meg's Mirrors

zig

1	pt: Zcu.PerThread,
2	air_instructions: std.MultiArrayList(Air.Inst),
3	air_extra: std.ArrayList(u32),
4	features: if (switch (dev.env) {
5	.bootstrap => @import("../codegen/c.zig").legalizeFeatures(undefined),
6	else => null,
7	}) |bootstrap_features| struct {
8	fn init(features: *const Features) @This() {
9	assert(features.eql(bootstrap_features.*));
10	return .{};
11	}
12	/// `inline` to propagate comptime-known result.
13	inline fn has(_: @This(), comptime feature: Feature) bool {
14	return comptime bootstrap_features.contains(feature);
15	}
16	/// `inline` to propagate comptime-known result.
17	inline fn hasAny(_: @This(), comptime features: []const Feature) bool {
18	return comptime !bootstrap_features.intersectWith(.initMany(features)).eql(.empty);
19	}
20	} else struct {
21	features: *const Features,
22	/// `inline` to propagate whether `dev.check` returns.
23	inline fn init(features: *const Features) @This() {
24	dev.check(.legalize);
25	return .{ .features = features };
26	}
27	fn has(rt: @This(), comptime feature: Feature) bool {
28	return rt.features.contains(feature);
29	}
30	fn hasAny(rt: @This(), comptime features: []const Feature) bool {
31	return !rt.features.intersectWith(comptime .initMany(features)).eql(.empty);
32	}
33	},
34
35	pub const Feature = enum {
36	scalarize_add,
37	scalarize_add_safe,
38	scalarize_add_optimized,
39	scalarize_add_wrap,
40	scalarize_add_sat,
41	scalarize_sub,
42	scalarize_sub_safe,
43	scalarize_sub_optimized,
44	scalarize_sub_wrap,
45	scalarize_sub_sat,
46	scalarize_mul,
47	scalarize_mul_safe,
48	scalarize_mul_optimized,
49	scalarize_mul_wrap,
50	scalarize_mul_sat,
51	scalarize_div_float,
52	scalarize_div_float_optimized,
53	scalarize_div_trunc,
54	scalarize_div_trunc_optimized,
55	scalarize_div_floor,
56	scalarize_div_floor_optimized,
57	scalarize_div_ceil,
58	scalarize_div_ceil_optimized,
59	scalarize_div_exact,
60	scalarize_div_exact_optimized,
61	scalarize_rem,
62	scalarize_rem_optimized,
63	scalarize_mod,
64	scalarize_mod_optimized,
65	scalarize_max,
66	scalarize_min,
67	scalarize_add_with_overflow,
68	scalarize_sub_with_overflow,
69	scalarize_mul_with_overflow,
70	scalarize_shl_with_overflow,
71	scalarize_bit_and,
72	scalarize_bit_or,
73	scalarize_shr,
74	scalarize_shr_exact,
75	scalarize_shl,
76	scalarize_shl_exact,
77	scalarize_shl_sat,
78	scalarize_xor,
79	scalarize_not,
80	scalarize_ptr_cast,
81	scalarize_ptr_from_int,
82	scalarize_int_from_ptr,
83	scalarize_clz,
84	scalarize_ctz,
85	scalarize_popcount,
86	scalarize_byte_swap,
87	scalarize_bit_reverse,
88	scalarize_sqrt,
89	scalarize_sin,
90	scalarize_cos,
91	scalarize_tan,
92	scalarize_exp,
93	scalarize_exp2,
94	scalarize_log,
95	scalarize_log2,
96	scalarize_log10,
97	scalarize_abs,
98	scalarize_floor,
99	scalarize_ceil,
100	scalarize_round,
101	scalarize_trunc_float,
102	scalarize_neg,
103	scalarize_neg_optimized,
104	scalarize_cmp_vector,
105	scalarize_cmp_vector_optimized,
106	scalarize_fptrunc,
107	scalarize_fpext,
108	scalarize_int_cast,
109	scalarize_int_cast_safe,
110	scalarize_trunc,
111	scalarize_int_from_float,
112	scalarize_int_from_float_optimized,
113	scalarize_int_from_float_safe,
114	scalarize_int_from_float_optimized_safe,
115	scalarize_float_from_int,
116	scalarize_reduce,
117	scalarize_reduce_optimized,
118	scalarize_shuffle_one,
119	scalarize_shuffle_two,
120	scalarize_select,
121	scalarize_mul_add,
122
123	// Below are several different features for scalarizing `bit_cast` in different scenarios. It is
124	// valid to enable any combination of these features.
125
126	/// Scalarize `bit_cast` where the operand or result type is an array.
127	scalarize_bit_cast_array,
128	/// Scalarize `bit_cast` where either:
129	///
130	/// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
131	/// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
132	///
133	/// This effectively scalarizes any `bit_cast` to/from a vector, *unless* the operation can be
134	/// performed by bitcasting each vector element and returning a vector of the results.
135	///
136	/// If this feature is enabled, the following AIR instruction tags may be emitted:
137	/// * `.legalize_vec_elem_val`
138	/// * `.legalize_vec_store_elem`
139	scalarize_bit_cast_vector_non_elementwise,
140	/// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
141	/// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
142	/// need to sign- or zero-extend multiple elements to populate "padding" bits.
143	///
144	/// Enabling this feature requires changing the behavior of `@bitSize` on arrays in `Type.zig`
145	/// to conform to the new bitcast semantics.
146	///
147	/// If this feature is enabled, the following AIR instruction tags may be emitted:
148	/// * `.legalize_vec_elem_val`
149	/// * `.legalize_vec_store_elem`
150	scalarize_bit_cast_padded_elems,
151
152	/// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
153	unsplat_shift_rhs,
154	/// Legalize reduce of a one element vector to a bitcast.
155	reduce_one_elem_to_bit_cast,
156	/// Legalize splat to a one element vector to a bitcast.
157	splat_one_elem_to_bit_cast,
158
159	/// Replace `bit_cast_safe` with an explicit safety check which `call`s the panic function on failure.
160	/// `scalarize_*` variants for `bit_cast_safe` do not exist since the safety check is only desired if the result
161	/// type is a scalar enum type, so the scalarizatins for regular `bit_cast` are exactly equivalent.
162	expand_bit_cast_safe,
163	/// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
164	/// Not compatible with `scalarize_int_cast_safe`.
165	expand_int_cast_safe,
166	/// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
167	/// Not compatible with `scalarize_int_from_float_safe`.
168	expand_int_from_float_safe,
169	/// Replace `int_from_float_optimized_safe` with an explicit safety check which `call`s the panic function on failure.
170	/// Not compatible with `scalarize_int_from_float_optimized_safe`.
171	expand_int_from_float_optimized_safe,
172	/// Replace `add_safe` with an explicit safety check which `call`s the panic function on failure.
173	/// Not compatible with `scalarize_add_safe`.
174	expand_add_safe,
175	/// Replace `sub_safe` with an explicit safety check which `call`s the panic function on failure.
176	/// Not compatible with `scalarize_sub_safe`.
177	expand_sub_safe,
178	/// Replace `mul_safe` with an explicit safety check which `call`s the panic function on failure.
179	/// Not compatible with `scalarize_mul_safe`.
180	expand_mul_safe,
181
182	/// Replace `div_ceil` with truncating division followed by a remainder based adjustment for integers,
183	/// or division followed by ceil for floats.
184	/// Not compatible with `scalarize_div_ceil`.
185	expand_div_ceil,
186	/// Replace `div_ceil_optimized` with truncating division followed by a remainder based adjustment for integers,
187	/// or division followed by ceil for floats.
188	/// Not compatible with `scalarize_div_ceil_optimized`.
189	expand_div_ceil_optimized,
190
191	/// Replace `load` from a packed pointer with a non-packed `load`, `shr`, `truncate`.
192	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
193	/// first byte of memory until bit pointers know their backing type.
194	expand_packed_load,
195	/// Replace `store` and `store_safe` to a packed pointer with a non-packed `load`/`store`, `bit_and`, `bit_or`, and `shl`.
196	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
197	/// first byte of memory until bit pointers know their backing type.
198	expand_packed_store,
199	/// Replace `agg_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
200	expand_packed_agg_field_val,
201	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
202	expand_packed_aggregate_init,
203	/// Replace `splat` of an array with an `aggregate_init`.
204	expand_array_splat,
205	/// Replace `array_to_vector` with an `array_elem_val` per element followed by an `aggregate_init`.
206	expand_array_to_vector,
207
208	/// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
209	/// routines in compiler_rt, including `fptrunc`/`fpext`/`float_from_int`/`int_from_float`
210	/// where the operand or target type is a 16-bit floating-point type. This feature implies:
211	///
212	/// * scalarization of 16-bit float vector operations
213	/// * expansion of safety-checked 16-bit float operations
214	///
215	/// If this feature is enabled, the following AIR instruction tags may be emitted:
216	/// * `.legalize_vec_elem_val`
217	/// * `.legalize_vec_store_elem`
218	/// * `.legalize_compiler_rt_call`
219	soft_f16,
220	/// Like `soft_f16`, but for 32-bit floating-point types.
221	soft_f32,
222	/// Like `soft_f16`, but for 64-bit floating-point types.
223	soft_f64,
224	/// Like `soft_f16`, but for 80-bit floating-point types.
225	soft_f80,
226	/// Like `soft_f16`, but for 128-bit floating-point types.
227	soft_f128,
228
229	fn scalarize(tag: Air.Inst.Tag) Feature {
230	return switch (tag) {
231	else => unreachable,
232	.add => .scalarize_add,
233	.add_safe => .scalarize_add_safe,
234	.add_optimized => .scalarize_add_optimized,
235	.add_wrap => .scalarize_add_wrap,
236	.add_sat => .scalarize_add_sat,
237	.sub => .scalarize_sub,
238	.sub_safe => .scalarize_sub_safe,
239	.sub_optimized => .scalarize_sub_optimized,
240	.sub_wrap => .scalarize_sub_wrap,
241	.sub_sat => .scalarize_sub_sat,
242	.mul => .scalarize_mul,
243	.mul_safe => .scalarize_mul_safe,
244	.mul_optimized => .scalarize_mul_optimized,
245	.mul_wrap => .scalarize_mul_wrap,
246	.mul_sat => .scalarize_mul_sat,
247	.div_float => .scalarize_div_float,
248	.div_float_optimized => .scalarize_div_float_optimized,
249	.div_trunc => .scalarize_div_trunc,
250	.div_trunc_optimized => .scalarize_div_trunc_optimized,
251	.div_floor => .scalarize_div_floor,
252	.div_floor_optimized => .scalarize_div_floor_optimized,
253	.div_ceil => .scalarize_div_ceil,
254	.div_ceil_optimized => .scalarize_div_ceil_optimized,
255	.div_exact => .scalarize_div_exact,
256	.div_exact_optimized => .scalarize_div_exact_optimized,
257	.rem => .scalarize_rem,
258	.rem_optimized => .scalarize_rem_optimized,
259	.mod => .scalarize_mod,
260	.mod_optimized => .scalarize_mod_optimized,
261	.max => .scalarize_max,
262	.min => .scalarize_min,
263	.add_with_overflow => .scalarize_add_with_overflow,
264	.sub_with_overflow => .scalarize_sub_with_overflow,
265	.mul_with_overflow => .scalarize_mul_with_overflow,
266	.shl_with_overflow => .scalarize_shl_with_overflow,
267	.bit_and => .scalarize_bit_and,
268	.bit_or => .scalarize_bit_or,
269	.shr => .scalarize_shr,
270	.shr_exact => .scalarize_shr_exact,
271	.shl => .scalarize_shl,
272	.shl_exact => .scalarize_shl_exact,
273	.shl_sat => .scalarize_shl_sat,
274	.xor => .scalarize_xor,
275	.not => .scalarize_not,
276	.clz => .scalarize_clz,
277	.ctz => .scalarize_ctz,
278	.popcount => .scalarize_popcount,
279	.byte_swap => .scalarize_byte_swap,
280	.bit_reverse => .scalarize_bit_reverse,
281	.sqrt => .scalarize_sqrt,
282	.sin => .scalarize_sin,
283	.cos => .scalarize_cos,
284	.tan => .scalarize_tan,
285	.exp => .scalarize_exp,
286	.exp2 => .scalarize_exp2,
287	.log => .scalarize_log,
288	.log2 => .scalarize_log2,
289	.log10 => .scalarize_log10,
290	.abs => .scalarize_abs,
291	.floor => .scalarize_floor,
292	.ceil => .scalarize_ceil,
293	.round => .scalarize_round,
294	.trunc_float => .scalarize_trunc_float,
295	.neg => .scalarize_neg,
296	.neg_optimized => .scalarize_neg_optimized,
297	.cmp_vector => .scalarize_cmp_vector,
298	.cmp_vector_optimized => .scalarize_cmp_vector_optimized,
299	.fptrunc => .scalarize_fptrunc,
300	.fpext => .scalarize_fpext,
301	.int_cast => .scalarize_int_cast,
302	.int_cast_safe => .scalarize_int_cast_safe,
303	.ptr_cast => .scalarize_ptr_cast,
304	.ptr_from_int => .scalarize_ptr_from_int,
305	.int_from_ptr => .scalarize_int_from_ptr,
306	.trunc => .scalarize_trunc,
307	.int_from_float => .scalarize_int_from_float,
308	.int_from_float_optimized => .scalarize_int_from_float_optimized,
309	.int_from_float_safe => .scalarize_int_from_float_safe,
310	.int_from_float_optimized_safe => .scalarize_int_from_float_optimized_safe,
311	.float_from_int => .scalarize_float_from_int,
312	.reduce => .scalarize_reduce,
313	.reduce_optimized => .scalarize_reduce_optimized,
314	.shuffle_one => .scalarize_shuffle_one,
315	.shuffle_two => .scalarize_shuffle_two,
316	.select => .scalarize_select,
317	.mul_add => .scalarize_mul_add,
318	};
319	}
320	};
321
322	pub const Features = std.enums.EnumSet(Feature);
323
324	pub const Error = std.mem.Allocator.Error;
325
326	pub fn legalize(air: *Air, pt: Zcu.PerThread, features: *const Features) Error!void {
327	assert(!features.eql(.empty)); // backend asked to run legalize, but no features were enabled
328	var l: Legalize = .{
329	.pt = pt,
330	.air_instructions = air.instructions.toMultiArrayList(),
331	.air_extra = air.extra,
332	.features = .init(features),
333	};
334	defer air.* = l.getTmpAir();
335	const main_extra = l.extraData(Air.Block, l.air_extra.items[@backingInt(Air.ExtraIndex.main_block)]);
336	try l.legalizeBody(main_extra.end, main_extra.data.body_len);
337	}
338
339	fn getTmpAir(l: *const Legalize) Air {
340	return .{
341	.instructions = l.air_instructions.slice(),
342	.extra = l.air_extra,
343	};
344	}
345
346	fn typeOf(l: *const Legalize, ref: Air.Inst.Ref) Type {
347	return l.getTmpAir().typeOf(ref, &l.pt.zcu.intern_pool);
348	}
349
350	fn typeOfIndex(l: *const Legalize, inst: Air.Inst.Index) Type {
351	return l.getTmpAir().typeOfIndex(inst, &l.pt.zcu.intern_pool);
352	}
353
354	fn extraData(l: *const Legalize, comptime T: type, index: usize) @TypeOf(Air.extraData(undefined, T, undefined)) {
355	return l.getTmpAir().extraData(T, index);
356	}
357
358	fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
359	// In zig1, this function needs a lot of eval branch quota, because all of the inlined feature
360	// checks are comptime-evaluated (to ensure unused features are not included in the binary).
361	@setEvalBranchQuota(4000);
362
363	const zcu = l.pt.zcu;
364	const ip = &zcu.intern_pool;
365	for (0..body_len) |body_index| {
366	const inst: Air.Inst.Index = @fromBackingInt(@intCast(l.air_extra.items[body_start + body_index]));
367	inst: switch (l.air_instructions.items(.tag)[@backingInt(inst)]) {
368	.arg => {},
369	inline .add,
370	.add_optimized,
371	.sub,
372	.sub_optimized,
373	.mul,
374	.mul_optimized,
375	.div_float,
376	.div_float_optimized,
377	.div_exact,
378	.div_exact_optimized,
379	.rem,
380	.rem_optimized,
381	.min,
382	.max,
383	=> |air_tag| {
384	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
385	const ty = l.typeOf(bin_op.lhs);
386	switch (l.wantScalarizeOrSoftFloat(air_tag, ty)) {
387	.none => {},
388	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
389	.soft_float => continue :inst try l.compilerRtCall(
390	inst,
391	softFloatFunc(air_tag, ty, zcu),
392	&.{ bin_op.lhs, bin_op.rhs },
393	l.typeOf(bin_op.lhs),
394	),
395	}
396	},
397	inline .div_trunc,
398	.div_trunc_optimized,
399	.div_floor,
400	.div_floor_optimized,
401	=> |air_tag| {
402	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
403	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(bin_op.lhs))) {
404	.none => {},
405	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
406	.soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatDivTruncFloorCeilBlockPayload(
407	inst,
408	bin_op.lhs,
409	bin_op.rhs,
410	air_tag,
411	)),
412	}
413	},
414	inline .mod, .mod_optimized => |air_tag| {
415	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
416	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(bin_op.lhs))) {
417	.none => {},
418	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
419	.soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatModBlockPayload(
420	inst,
421	bin_op.lhs,
422	bin_op.rhs,
423	)),
424	}
425	},
426	inline .add_wrap,
427	.add_sat,
428	.sub_wrap,
429	.sub_sat,
430	.mul_wrap,
431	.mul_sat,
432	.bit_and,
433	.bit_or,
434	.xor,
435	=> |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
436	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
437	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
438	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
439	}
440	},
441	.add_safe => if (l.features.has(.expand_add_safe)) {
442	assert(!l.features.has(.scalarize_add_safe)); // it doesn't make sense to do both
443	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .add_with_overflow));
444	} else if (l.features.has(.scalarize_add_safe)) {
445	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
446	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
447	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
448	}
449	},
450	.sub_safe => if (l.features.has(.expand_sub_safe)) {
451	assert(!l.features.has(.scalarize_sub_safe)); // it doesn't make sense to do both
452	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .sub_with_overflow));
453	} else if (l.features.has(.scalarize_sub_safe)) {
454	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
455	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
456	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
457	}
458	},
459	.mul_safe => if (l.features.has(.expand_mul_safe)) {
460	assert(!l.features.has(.scalarize_mul_safe)); // it doesn't make sense to do both
461	continue :inst l.replaceInst(inst, .block, try l.safeArithmeticBlockPayload(inst, .mul_with_overflow));
462	} else if (l.features.has(.scalarize_mul_safe)) {
463	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
464	if (l.typeOf(bin_op.lhs).isVector(zcu)) {
465	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
466	}
467	},
468	.ptr_add, .ptr_sub => {},
469	inline .add_with_overflow,
470	.sub_with_overflow,
471	.mul_with_overflow,
472	.shl_with_overflow,
473	=> |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
474	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
475	if (ty_pl.ty.fieldType(0, zcu).isVector(zcu)) {
476	continue :inst l.replaceInst(inst, .block, try l.scalarizeOverflowBlockPayload(inst));
477	}
478	},
479	.alloc => {},
480	.inferred_alloc, .inferred_alloc_comptime => unreachable,
481	.ret_ptr, .assembly => {},
482	inline .shr,
483	.shr_exact,
484	.shl,
485	.shl_exact,
486	.shl_sat,
487	=> |air_tag| if (l.features.hasAny(&.{
488	.unsplat_shift_rhs,
489	.scalarize(air_tag),
490	})) {
491	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
492	if (l.typeOf(bin_op.rhs).isVector(zcu)) {
493	if (l.features.has(.unsplat_shift_rhs)) {
494	if (bin_op.rhs.toInterned()) |rhs_ip_index| switch (ip.indexToKey(rhs_ip_index)) {
495	else => {},
496	.aggregate => |aggregate| switch (aggregate.storage) {
497	else => {},
498	.repeated_elem => |splat| continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
499	.lhs = bin_op.lhs,
500	.rhs = Air.internedToRef(splat),
501	} }),
502	},
503	} else {
504	const rhs_inst = bin_op.rhs.toIndex().?;
505	switch (l.air_instructions.items(.tag)[@backingInt(rhs_inst)]) {
506	else => {},
507	.splat => continue :inst l.replaceInst(inst, air_tag, .{ .bin_op = .{
508	.lhs = bin_op.lhs,
509	.rhs = l.air_instructions.items(.data)[@backingInt(rhs_inst)].ty_op.operand,
510	} }),
511	}
512	}
513	}
514	if (l.features.has(comptime .scalarize(air_tag))) {
515	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op));
516	}
517	}
518	},
519	inline .not,
520	.clz,
521	.ctz,
522	.popcount,
523	.byte_swap,
524	.bit_reverse,
525	.int_cast,
526	.ptr_cast,
527	.ptr_from_int,
528	.int_from_ptr,
529	.trunc,
530	=> |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
531	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
532	if (ty_op.ty.isVector(zcu)) {
533	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
534	}
535	},
536	.abs => {
537	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
538	switch (l.wantScalarizeOrSoftFloat(.abs, ty_op.ty)) {
539	.none => {},
540	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
541	.soft_float => continue :inst try l.compilerRtCall(
542	inst,
543	softFloatFunc(.abs, ty_op.ty, zcu),
544	&.{ty_op.operand},
545	ty_op.ty,
546	),
547	}
548	},
549	.fptrunc => {
550	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
551	const src_ty = l.typeOf(ty_op.operand);
552	const dest_ty = ty_op.ty;
553	if (src_ty.zigTypeTag(zcu) == .vector) {
554	if (l.features.has(.scalarize_fptrunc) or
555	l.wantSoftFloatScalar(src_ty.childType(zcu)) or
556	l.wantSoftFloatScalar(dest_ty.childType(zcu)))
557	{
558	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
559	}
560	} else if (l.wantSoftFloatScalar(src_ty) or l.wantSoftFloatScalar(dest_ty)) {
561	continue :inst try l.compilerRtCall(inst, l.softFptruncFunc(src_ty, dest_ty), &.{ty_op.operand}, dest_ty);
562	}
563	},
564	.fpext => {
565	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
566	const src_ty = l.typeOf(ty_op.operand);
567	const dest_ty = ty_op.ty;
568	if (src_ty.zigTypeTag(zcu) == .vector) {
569	if (l.features.has(.scalarize_fpext) or
570	l.wantSoftFloatScalar(src_ty.childType(zcu)) or
571	l.wantSoftFloatScalar(dest_ty.childType(zcu)))
572	{
573	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
574	}
575	} else if (l.wantSoftFloatScalar(src_ty) or l.wantSoftFloatScalar(dest_ty)) {
576	continue :inst try l.compilerRtCall(inst, l.softFpextFunc(src_ty, dest_ty), &.{ty_op.operand}, dest_ty);
577	}
578	},
579	inline .int_from_float, .int_from_float_optimized => |air_tag| {
580	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
581	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(ty_op.operand))) {
582	.none => {},
583	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
584	.soft_float => switch (try l.softIntFromFloat(inst)) {
585	.call => |func| continue :inst try l.compilerRtCall(inst, func, &.{ty_op.operand}, ty_op.ty),
586	.block_payload => |data| continue :inst l.replaceInst(inst, .block, data),
587	},
588	}
589	},
590	.float_from_int => {
591	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
592	const dest_ty = ty_op.ty;
593	switch (l.wantScalarizeOrSoftFloat(.float_from_int, dest_ty)) {
594	.none => {},
595	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
596	.soft_float => switch (try l.softFloatFromInt(inst)) {
597	.call => |func| continue :inst try l.compilerRtCall(inst, func, &.{ty_op.operand}, dest_ty),
598	.block_payload => |data| continue :inst l.replaceInst(inst, .block, data),
599	},
600	}
601	},
602	.bit_cast => if (l.features.hasAny(&.{
603	.scalarize_bit_cast_array,
604	.scalarize_bit_cast_vector_non_elementwise,
605	.scalarize_bit_cast_padded_elems,
606	})) {
607	if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
608	continue :inst l.replaceInst(inst, .block, payload);
609	}
610	},
611	.bit_cast_safe => if (l.features.has(.expand_bit_cast_safe)) {
612	if (try l.safeBitcastBlockPayload(inst)) |payload| {
613	continue :inst l.replaceInst(inst, .block, payload);
614	}
615	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
616	continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = ty_op });
617	} else if (l.features.hasAny(&.{
618	.scalarize_bit_cast_array,
619	.scalarize_bit_cast_vector_non_elementwise,
620	.scalarize_bit_cast_padded_elems,
621	})) {
622	if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
623	continue :inst l.replaceInst(inst, .block, payload);
624	}
625	},
626	.int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
627	assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
628	if (try l.safeIntcastBlockPayload(inst)) |payload| {
629	continue :inst l.replaceInst(inst, .block, payload);
630	}
631	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
632	continue :inst l.replaceInst(inst, .int_cast, .{ .ty_op = ty_op });
633	} else if (l.features.has(.scalarize_int_cast_safe)) {
634	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
635	if (ty_op.ty.isVector(zcu)) {
636	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
637	}
638	},
639	inline .div_ceil, .div_ceil_optimized => |air_tag| {
640	const expand_feature: Feature = switch (air_tag) {
641	.div_ceil => .expand_div_ceil,
642	.div_ceil_optimized => .expand_div_ceil_optimized,
643	else => unreachable,
644	};
645
646	if (l.features.has(expand_feature)) {
647	assert(!l.features.has(.scalarize(air_tag))); // it doesn't make sense to do both
648	continue :inst l.replaceInst(inst, .block, try l.divCeilBlockPayload(inst, air_tag));
649	} else {
650	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
651	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(bin_op.lhs))) {
652	.none => {},
653	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
654	.soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatDivTruncFloorCeilBlockPayload(
655	inst,
656	bin_op.lhs,
657	bin_op.rhs,
658	air_tag,
659	)),
660	}
661	}
662	},
663	inline .int_from_float_safe,
664	.int_from_float_optimized_safe,
665	=> |air_tag| {
666	const optimized = air_tag == .int_from_float_optimized_safe;
667	const expand_feature = switch (air_tag) {
668	.int_from_float_safe => .expand_int_from_float_safe,
669	.int_from_float_optimized_safe => .expand_int_from_float_optimized_safe,
670	else => unreachable,
671	};
672	if (l.features.has(expand_feature)) {
673	assert(!l.features.has(.scalarize(air_tag)));
674	continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, optimized));
675	}
676	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
677	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(ty_op.operand))) {
678	.none => {},
679	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
680	// Expand the safety check so that soft-float can rewrite the unchecked operation.
681	.soft_float => continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, optimized)),
682	}
683	},
684	.block, .loop => {
685	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
686	const extra = l.extraData(Air.Block, ty_pl.payload);
687	try l.legalizeBody(extra.end, extra.data.body_len);
688	},
689	.repeat,
690	.br,
691	.trap,
692	.breakpoint,
693	.ret_addr,
694	.frame_addr,
695	.call,
696	.call_always_tail,
697	.call_never_tail,
698	.call_never_inline,
699	=> {},
700	inline .sqrt,
701	.sin,
702	.cos,
703	.tan,
704	.exp,
705	.exp2,
706	.log,
707	.log2,
708	.log10,
709	.floor,
710	.ceil,
711	.round,
712	.trunc_float,
713	=> |air_tag| {
714	const operand = l.air_instructions.items(.data)[@backingInt(inst)].un_op;
715	const ty = l.typeOf(operand);
716	switch (l.wantScalarizeOrSoftFloat(air_tag, ty)) {
717	.none => {},
718	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op)),
719	.soft_float => continue :inst try l.compilerRtCall(
720	inst,
721	softFloatFunc(air_tag, ty, zcu),
722	&.{operand},
723	l.typeOf(operand),
724	),
725	}
726	},
727	inline .neg, .neg_optimized => |air_tag| {
728	const operand = l.air_instructions.items(.data)[@backingInt(inst)].un_op;
729	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(operand))) {
730	.none => {},
731	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op)),
732	.soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatNegBlockPayload(inst, operand)),
733	}
734	},
735	.cmp_lt,
736	.cmp_lt_optimized,
737	.cmp_lte,
738	.cmp_lte_optimized,
739	.cmp_eq,
740	.cmp_eq_optimized,
741	.cmp_gte,
742	.cmp_gte_optimized,
743	.cmp_gt,
744	.cmp_gt_optimized,
745	.cmp_neq,
746	.cmp_neq_optimized,
747	=> |air_tag| {
748	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
749	const ty = l.typeOf(bin_op.lhs);
750	if (l.wantSoftFloatScalar(ty)) {
751	continue :inst l.replaceInst(
752	inst,
753	.block,
754	try l.softFloatCmpBlockPayload(inst, ty, air_tag.toCmpOp().?, bin_op.lhs, bin_op.rhs),
755	);
756	}
757	},
758	inline .cmp_vector, .cmp_vector_optimized => |air_tag| {
759	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
760	const payload = l.extraData(Air.VectorCmp, ty_pl.payload).data;
761	switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(payload.lhs))) {
762	.none => {},
763	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector)),
764	.soft_float => unreachable, // the operand is not a scalar
765	}
766	},
767	.cond_br => {
768	const pl_op = l.air_instructions.items(.data)[@backingInt(inst)].pl_op;
769	const extra = l.extraData(Air.CondBr, pl_op.payload);
770	try l.legalizeBody(extra.end, extra.data.then_body_len);
771	try l.legalizeBody(extra.end + extra.data.then_body_len, extra.data.else_body_len);
772	},
773	.switch_br, .loop_switch_br => {
774	const pl_op = l.air_instructions.items(.data)[@backingInt(inst)].pl_op;
775	const extra = l.extraData(Air.SwitchBr, pl_op.payload);
776	const hint_bag_count = @divCeil(extra.data.cases_len + 1, 10);
777	var extra_index = extra.end + hint_bag_count;
778	for (0..extra.data.cases_len) |_| {
779	const case_extra = l.extraData(Air.SwitchBr.Case, extra_index);
780	const case_body_start = case_extra.end + case_extra.data.items_len + case_extra.data.ranges_len * 2;
781	try l.legalizeBody(case_body_start, case_extra.data.body_len);
782	extra_index = case_body_start + case_extra.data.body_len;
783	}
784	try l.legalizeBody(extra_index, extra.data.else_body_len);
785	},
786	.switch_dispatch => {},
787	.@"try", .try_cold => {
788	const pl_op = l.air_instructions.items(.data)[@backingInt(inst)].pl_op;
789	const extra = l.extraData(Air.Try, pl_op.payload);
790	try l.legalizeBody(extra.end, extra.data.body_len);
791	},
792	.try_ptr, .try_ptr_cold => {
793	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
794	const extra = l.extraData(Air.TryPtr, ty_pl.payload);
795	try l.legalizeBody(extra.end, extra.data.body_len);
796	},
797	.dbg_stmt, .dbg_empty_stmt => {},
798	.dbg_inline_block => {
799	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
800	const extra = l.extraData(Air.DbgInlineBlock, ty_pl.payload);
801	try l.legalizeBody(extra.end, extra.data.body_len);
802	},
803	.dbg_var_ptr,
804	.dbg_var_val,
805	.dbg_arg_inline,
806	.is_null,
807	.is_non_null,
808	.is_null_ptr,
809	.is_non_null_ptr,
810	.is_err,
811	.is_non_err,
812	.is_err_ptr,
813	.is_non_err_ptr,
814	=> {},
815	.load => if (l.features.has(.expand_packed_load)) {
816	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
817	const ptr_info = l.typeOf(ty_op.operand).ptrInfo(zcu);
818	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
819	continue :inst l.replaceInst(inst, .block, try l.packedLoadBlockPayload(inst));
820	}
821	},
822	.ret, .ret_safe, .ret_load => {},
823	.store, .store_safe => if (l.features.has(.expand_packed_store)) {
824	const bin_op = l.air_instructions.items(.data)[@backingInt(inst)].bin_op;
825	const ptr_info = l.typeOf(bin_op.lhs).ptrInfo(zcu);
826	if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) {
827	continue :inst l.replaceInst(inst, .block, try l.packedStoreBlockPayload(inst));
828	}
829	},
830	.unreach,
831	.optional_payload,
832	.optional_payload_ptr,
833	.optional_payload_ptr_set,
834	.wrap_optional,
835	.unwrap_errunion_payload,
836	.unwrap_errunion_err,
837	.unwrap_errunion_payload_ptr,
838	.unwrap_errunion_err_ptr,
839	.errunion_payload_ptr_set,
840	.wrap_errunion_payload,
841	.wrap_errunion_err,
842	.struct_field_ptr,
843	.struct_field_ptr_index_0,
844	.struct_field_ptr_index_1,
845	.struct_field_ptr_index_2,
846	.struct_field_ptr_index_3,
847	=> {},
848	.agg_field_val => if (l.features.has(.expand_packed_agg_field_val)) {
849	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
850	const extra = l.extraData(Air.StructField, ty_pl.payload).data;
851	switch (l.typeOf(extra.struct_operand).containerLayout(zcu)) {
852	.auto, .@"extern" => {},
853	.@"packed" => continue :inst l.replaceInst(inst, .block, try l.packedStructFieldValBlockPayload(inst)),
854	}
855	},
856	.set_union_tag,
857	.get_union_tag,
858	.slice,
859	.slice_len,
860	.slice_ptr,
861	.ptr_slice_len_ptr,
862	.ptr_slice_ptr_ptr,
863	.array_elem_val,
864	.slice_elem_val,
865	.slice_elem_ptr,
866	.ptr_elem_val,
867	.ptr_elem_ptr,
868	.array_to_slice,
869	=> {},
870	.array_to_vector => if (l.features.has(.expand_array_to_vector)) {
871	continue :inst l.replaceInst(inst, .block, try l.arrayToVectorBlockPayload(inst));
872	},
873	inline .reduce, .reduce_optimized => |air_tag| {
874	const reduce = l.air_instructions.items(.data)[@backingInt(inst)].reduce;
875	const vector_ty = l.typeOf(reduce.operand);
876	if (l.features.has(.reduce_one_elem_to_bit_cast)) {
877	switch (vector_ty.vectorLen(zcu)) {
878	0 => unreachable,
879	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
880	.ty = vector_ty.childType(zcu),
881	.operand = reduce.operand,
882	} }),
883	else => {},
884	}
885	}
886	switch (l.wantScalarizeOrSoftFloat(air_tag, vector_ty)) {
887	.none => {},
888	.scalarize => continue :inst l.replaceInst(
889	inst,
890	.block,
891	try l.scalarizeReduceBlockPayload(inst, air_tag == .reduce_optimized),
892	),
893	.soft_float => unreachable, // the operand is not a scalar
894	}
895	},
896	.splat => {
897	const ty_op = l.air_instructions.items(.data)[@backingInt(inst)].ty_op;
898	switch (ty_op.ty.zigTypeTag(zcu)) {
899	.vector => switch (ty_op.ty.vectorLen(zcu)) {
900	0 => unreachable,
901	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
902	.ty = ty_op.ty,
903	.operand = ty_op.operand,
904	} }),
905	else => {},
906	},
907	.array => if (l.features.has(.expand_array_splat)) {
908	const len: usize = @intCast(ty_op.ty.arrayLen(zcu));
909	const elems_start: u32 = @intCast(l.air_extra.items.len);
910	try l.air_extra.appendNTimes(l.pt.zcu.gpa, @backingInt(ty_op.operand), len);
911	continue :inst l.replaceInst(inst, .aggregate_init, .{ .ty_pl = .{
912	.ty = ty_op.ty,
913	.payload = elems_start,
914	} });
915	},
916	else => unreachable,
917	}
918	},
919	.shuffle_one => {
920	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
921	switch (l.wantScalarizeOrSoftFloat(.shuffle_one, ty_pl.ty)) {
922	.none => {},
923	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst)),
924	.soft_float => unreachable, // the operand is not a scalar
925	}
926	},
927	.shuffle_two => {
928	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
929	switch (l.wantScalarizeOrSoftFloat(.shuffle_two, ty_pl.ty)) {
930	.none => {},
931	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst)),
932	.soft_float => unreachable, // the operand is not a scalar
933	}
934	},
935	.select => {
936	const pl_op = l.air_instructions.items(.data)[@backingInt(inst)].pl_op;
937	const bin = l.extraData(Air.Bin, pl_op.payload).data;
938	switch (l.wantScalarizeOrSoftFloat(.select, l.typeOf(bin.lhs))) {
939	.none => {},
940	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select)),
941	.soft_float => unreachable, // the operand is not a scalar
942	}
943	},
944	.memset,
945	.memset_safe,
946	.memcpy,
947	.memmove,
948	.cmpxchg_weak,
949	.cmpxchg_strong,
950	.atomic_load,
951	.atomic_store_unordered,
952	.atomic_store_monotonic,
953	.atomic_store_release,
954	.atomic_store_seq_cst,
955	.atomic_rmw,
956	.is_named_enum_value,
957	.tag_name,
958	.error_name,
959	.error_set_has_value,
960	=> {},
961	.aggregate_init => if (l.features.has(.expand_packed_aggregate_init)) {
962	const ty_pl = l.air_instructions.items(.data)[@backingInt(inst)].ty_pl;
963	const agg_ty = ty_pl.ty;
964	switch (agg_ty.zigTypeTag(zcu)) {
965	else => {},
966	.@"union" => unreachable,
967	.@"struct" => switch (agg_ty.containerLayout(zcu)) {
968	.auto, .@"extern" => {},
969	.@"packed" => {
970	// If any field accounts for the full bit size of the struct, this init
971	// is just equivalent to a bitcast of that field. This usually means the
972	// field count is 1, but not always, as there could be zero-bit fields.
973	const struct_bits = agg_ty.bitSize(zcu);
974	for (0..agg_ty.structFieldCount(zcu)) |field_index| {
975	const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
976	if (field_bits == struct_bits) {
977	// Just bitcast this field.
978	continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
979	.ty = agg_ty,
980	.operand = @fromBackingInt(@intCast(l.air_extra.items[ty_pl.payload + field_index])),
981	} });
982	}
983	}
984	// Otherwise, we will need to use a sequence of bitcasts and shifts to
985	// combine multiple values' bits.
986	continue :inst l.replaceInst(inst, .block, try l.packedAggregateInitBlockPayload(inst));
987	},
988	},
989	}
990	},
991	.union_init, .prefetch => {},
992	.mul_add => {
993	const pl_op = l.air_instructions.items(.data)[@backingInt(inst)].pl_op;
994	const ty = l.typeOf(pl_op.operand);
995	switch (l.wantScalarizeOrSoftFloat(.mul_add, ty)) {
996	.none => {},
997	.scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin)),
998	.soft_float => {
999	const bin = l.extraData(Air.Bin, pl_op.payload).data;
1000	const func = softFloatFunc(.mul_add, ty, zcu);
1001	continue :inst try l.compilerRtCall(inst, func, &.{ bin.lhs, bin.rhs, pl_op.operand }, ty);
1002	},
1003	}
1004	},
1005	.field_parent_ptr,
1006	.wasm_memory_size,
1007	.wasm_memory_grow,
1008	.cmp_lte_errors_len,
1009	.err_return_trace,
1010	.set_err_return_trace,
1011	.addrspace_cast,
1012	.save_err_return_trace_index,
1013	.runtime_nav_ptr,
1014	.c_va_arg,
1015	.c_va_copy,
1016	.c_va_end,
1017	.c_va_start,
1018	.work_item_id,
1019	.work_group_size,
1020	.work_group_id,
1021	.legalize_vec_elem_val,
1022	.legalize_vec_store_elem,
1023	.legalize_compiler_rt_call,
1024	.spirv_runtime_array_len,
1025	.error_cast,
1026	.error_from_int,
1027	.int_from_error,
1028	.union_from_enum,
1029	=> {},
1030	}
1031	}
1032	}
1033
1034	const ScalarizeForm = enum { un_op, ty_op, bin_op, pl_op_bin, cmp_vector, select };
1035	fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: ScalarizeForm) Error!Air.Inst.Data {
1036	const pt = l.pt;
1037	const zcu = pt.zcu;
1038
1039	const orig = l.air_instructions.get(@backingInt(orig_inst));
1040	const res_ty = l.typeOfIndex(orig_inst);
1041	const result_is_array = switch (res_ty.zigTypeTag(zcu)) {
1042	.vector => false,
1043	.array => true,
1044	else => unreachable,
1045	};
1046	const res_len = res_ty.arrayLen(zcu);
1047	const res_elem_ty = res_ty.childType(zcu);
1048
1049	if (result_is_array) {
1050	// This is only allowed when legalizing an elementwise bitcast.
1051	switch (orig.tag) {
1052	.bit_cast, .bit_cast_safe => {},
1053	else => unreachable,
1054	}
1055	assert(form == .ty_op);
1056	}
1057
1058	// Our output will be a loop doing elementwise stores:
1059	//
1060	// %1 = block(@Vector(N, Scalar), {
1061	// %2 = alloc(*usize)
1062	// %3 = alloc(*@Vector(N, Scalar))
1063	// %4 = store(%2, @zero_usize)
1064	// %5 = loop({
1065	// %6 = load(%2)
1066	// %7 = <scalar result of operation at index %5>
1067	// %8 = legalize_vec_store_elem(%3, %5, %6)
1068	// %9 = cmp_eq(%6, <usize, N-1>)
1069	// %10 = cond_br(%9, {
1070	// %11 = load(%3)
1071	// %12 = br(%1, %11)
1072	// }, {
1073	// %13 = add(%6, @one_usize)
1074	// %14 = store(%2, %13)
1075	// %15 = repeat(%5)
1076	// })
1077	// })
1078	// })
1079	//
1080	// If scalarizing an elementwise bitcast, the result might be an array, in which case
1081	// `legalize_vec_store_elem` becomes two instructions (`ptr_elem_ptr` and `store`).
1082	// Therefore, there are 13 or 14 instructions in the block, plus however many are
1083	// needed to compute each result element for `form`.
1084	const inst_per_form: usize = switch (form) {
1085	.un_op, .ty_op => 2,
1086	.bin_op, .cmp_vector => 3,
1087	.pl_op_bin => 4,
1088	.select => 7,
1089	};
1090	const max_inst_per_form = 7; // maximum value in the above switch
1091	var inst_buf: [14 + max_inst_per_form]Air.Inst.Index = undefined;
1092
1093	var main_block: Block = .init(&inst_buf);
1094	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1095
1096	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1097	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(res_ty)).toRef();
1098
1099	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1100
1101	var loop: Loop = .init(l, &main_block);
1102	loop.block = .init(main_block.stealRemainingCapacity());
1103
1104	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1105	const elem_val: Air.Inst.Ref = switch (form) {
1106	.un_op => elem: {
1107	const orig_operand = orig.data.un_op;
1108	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
1109	break :elem loop.block.addUnOp(l, orig.tag, operand).toRef();
1110	},
1111	.ty_op => elem: {
1112	const orig_operand = orig.data.ty_op.operand;
1113	const operand_is_array = switch (l.typeOf(orig_operand).zigTypeTag(zcu)) {
1114	.vector => false,
1115	.array => true,
1116	else => unreachable,
1117	};
1118	const operand = loop.block.addBinOp(
1119	l,
1120	if (operand_is_array) .array_elem_val else .legalize_vec_elem_val,
1121	orig_operand,
1122	index_val,
1123	).toRef();
1124	const scalar_tag: Air.Inst.Tag = switch (orig.tag) {
1125	.bit_cast_safe => .bit_cast, // safety check is not supposed to be elementwise
1126	else => orig.tag,
1127	};
1128	break :elem loop.block.addTyOp(l, scalar_tag, res_elem_ty, operand).toRef();
1129	},
1130	.bin_op => elem: {
1131	const orig_bin = orig.data.bin_op;
1132	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
1133	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
1134	break :elem loop.block.addBinOp(l, orig.tag, lhs, rhs).toRef();
1135	},
1136	.pl_op_bin => elem: {
1137	const orig_operand = orig.data.pl_op.operand;
1138	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1139	const operand = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operand, index_val).toRef();
1140	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
1141	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
1142	break :elem loop.block.add(l, .{
1143	.tag = orig.tag,
1144	.data = .{ .pl_op = .{
1145	.operand = operand,
1146	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1147	} },
1148	}).toRef();
1149	},
1150	.cmp_vector => elem: {
1151	const orig_payload = l.extraData(Air.VectorCmp, orig.data.ty_pl.payload).data;
1152	const cmp_op = orig_payload.compareOperator();
1153	const optimized = switch (orig.tag) {
1154	.cmp_vector => false,
1155	.cmp_vector_optimized => true,
1156	else => unreachable,
1157	};
1158	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.lhs, index_val).toRef();
1159	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_payload.rhs, index_val).toRef();
1160	break :elem loop.block.addCmpScalar(l, cmp_op, lhs, rhs, optimized).toRef();
1161	},
1162	.select => elem: {
1163	const orig_cond = orig.data.pl_op.operand;
1164	const orig_bin = l.extraData(Air.Bin, orig.data.pl_op.payload).data;
1165
1166	const elem_block_inst = loop.block.add(l, .{
1167	.tag = .block,
1168	.data = .{ .ty_pl = .{
1169	.ty = res_elem_ty,
1170	.payload = undefined,
1171	} },
1172	});
1173	var elem_block: Block = .init(loop.block.stealCapacity(2));
1174	const cond = elem_block.addBinOp(l, .legalize_vec_elem_val, orig_cond, index_val).toRef();
1175
1176	var condbr: CondBr = .init(l, cond, &elem_block, .{});
1177
1178	condbr.then_block = .init(loop.block.stealCapacity(2));
1179	const lhs = condbr.then_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.lhs, index_val).toRef();
1180	condbr.then_block.addBr(l, elem_block_inst, lhs);
1181
1182	condbr.else_block = .init(loop.block.stealCapacity(2));
1183	const rhs = condbr.else_block.addBinOp(l, .legalize_vec_elem_val, orig_bin.rhs, index_val).toRef();
1184	condbr.else_block.addBr(l, elem_block_inst, rhs);
1185
1186	try condbr.finish(l);
1187
1188	const inst_data = l.air_instructions.items(.data);
1189	inst_data[@backingInt(elem_block_inst)].ty_pl.payload = try l.addBlockBody(elem_block.body());
1190
1191	break :elem elem_block_inst.toRef();
1192	},
1193	};
1194	_ = loop.block.stealCapacity(max_inst_per_form - inst_per_form);
1195	if (result_is_array) {
1196	const elem_ptr = loop.block.add(l, .{
1197	.tag = .ptr_elem_ptr,
1198	.data = .{ .ty_pl = .{
1199	.ty = try pt.singleMutPtrType(res_elem_ty),
1200	.payload = try l.addExtra(Air.Bin, .{
1201	.lhs = result_ptr,
1202	.rhs = index_val,
1203	}),
1204	} },
1205	}).toRef();
1206	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
1207	} else {
1208	_ = loop.block.add(l, .{
1209	.tag = .legalize_vec_store_elem,
1210	.data = .{ .pl_op = .{
1211	.operand = result_ptr,
1212	.payload = try l.addExtra(Air.Bin, .{
1213	.lhs = index_val,
1214	.rhs = elem_val,
1215	}),
1216	} },
1217	});
1218	_ = loop.block.stealCapacity(1);
1219	}
1220	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, res_len - 1))).toRef();
1221
1222	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1223	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1224	const result_val = condbr.then_block.addTyOp(l, .load, res_ty, result_ptr).toRef();
1225	condbr.then_block.addBr(l, orig_inst, result_val);
1226
1227	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1228	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1229	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1230	_ = condbr.else_block.add(l, .{
1231	.tag = .repeat,
1232	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1233	});
1234
1235	try condbr.finish(l);
1236
1237	try loop.finish(l);
1238
1239	return .{ .ty_pl = .{
1240	.ty = res_ty,
1241	.payload = try l.addBlockBody(main_block.body()),
1242	} };
1243	}
1244	fn scalarizeShuffleOneBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1245	const pt = l.pt;
1246	const zcu = pt.zcu;
1247	const gpa = zcu.gpa;
1248
1249	const shuffle = l.getTmpAir().unwrapShuffleOne(zcu, orig_inst);
1250
1251	// We're going to emit something like this:
1252	//
1253	// var x: @Vector(N, T) = all_comptime_known_elems;
1254	// for (out_idxs, in_idxs) |i, j| x[i] = operand[j];
1255	//
1256	// So we must first compute `out_idxs` and `in_idxs`.
1257
1258	var bfa_buf: [512]u8 = undefined;
1259	var bfa_state: std.heap.BufferFirstAllocator = .init(&bfa_buf, gpa);
1260	const bfa = bfa_state.allocator();
1261
1262	const out_idxs_buf = try bfa.alloc(InternPool.Index, shuffle.mask.len);
1263	defer bfa.free(out_idxs_buf);
1264
1265	const in_idxs_buf = try bfa.alloc(InternPool.Index, shuffle.mask.len);
1266	defer bfa.free(in_idxs_buf);
1267
1268	var n: usize = 0;
1269	for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1270	.value => {},
1271	.elem => |in_idx| {
1272	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1273	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1274	n += 1;
1275	},
1276	};
1277
1278	const init_val: Value = init: {
1279	const undef_val = try pt.undefValue(shuffle.result_ty.childType(zcu));
1280	const elems = try bfa.alloc(InternPool.Index, shuffle.mask.len);
1281	defer bfa.free(elems);
1282	for (shuffle.mask, elems) |mask, *elem| elem.* = switch (mask.unwrap()) {
1283	.value => |ip_index| ip_index,
1284	.elem => undef_val.toIntern(),
1285	};
1286	break :init try pt.aggregateValue(shuffle.result_ty, elems);
1287	};
1288
1289	// %1 = block(@Vector(N, T), {
1290	// %2 = alloc(*@Vector(N, T))
1291	// %3 = alloc(*usize)
1292	// %4 = store(%2, <init_val>)
1293	// %5 = [addScalarizedShuffle]
1294	// %6 = load(%2)
1295	// %7 = br(%1, %6)
1296	// })
1297
1298	var inst_buf: [6]Air.Inst.Index = undefined;
1299	var main_block: Block = .init(&inst_buf);
1300	try l.air_instructions.ensureUnusedCapacity(gpa, 19);
1301
1302	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
1303	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1304
1305	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(init_val));
1306
1307	try l.addScalarizedShuffle(
1308	&main_block,
1309	shuffle.operand,
1310	result_ptr,
1311	index_ptr,
1312	out_idxs_buf[0..n],
1313	in_idxs_buf[0..n],
1314	);
1315
1316	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
1317	main_block.addBr(l, orig_inst, result_val);
1318
1319	return .{ .ty_pl = .{
1320	.ty = shuffle.result_ty,
1321	.payload = try l.addBlockBody(main_block.body()),
1322	} };
1323	}
1324	fn scalarizeShuffleTwoBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1325	const pt = l.pt;
1326	const zcu = pt.zcu;
1327	const gpa = zcu.gpa;
1328
1329	const shuffle = l.getTmpAir().unwrapShuffleTwo(zcu, orig_inst);
1330
1331	// We're going to emit something like this:
1332	//
1333	// var x: @Vector(N, T) = undefined;
1334	// for (out_idxs_a, in_idxs_a) |i, j| x[i] = operand_a[j];
1335	// for (out_idxs_b, in_idxs_b) |i, j| x[i] = operand_b[j];
1336	//
1337	// The AIR will look like this:
1338	//
1339	// %1 = block(@Vector(N, T), {
1340	// %2 = alloc(*@Vector(N, T))
1341	// %3 = alloc(*usize)
1342	// %4 = store(%2, <@Vector(N, T), undefined>)
1343	// %5 = [addScalarizedShuffle]
1344	// %6 = [addScalarizedShuffle]
1345	// %7 = load(%2)
1346	// %8 = br(%1, %7)
1347	// })
1348
1349	var bfa_buf: [512]u8 = undefined;
1350	var bfa_state: std.heap.BufferFirstAllocator = .init(&bfa_buf, gpa);
1351	const bfa = bfa_state.allocator();
1352
1353	const out_idxs_buf = try bfa.alloc(InternPool.Index, shuffle.mask.len);
1354	defer bfa.free(out_idxs_buf);
1355
1356	const in_idxs_buf = try bfa.alloc(InternPool.Index, shuffle.mask.len);
1357	defer bfa.free(in_idxs_buf);
1358
1359	// Iterate `shuffle.mask` before doing anything, because modifying AIR invalidates it.
1360	const out_idxs_a, const in_idxs_a, const out_idxs_b, const in_idxs_b = idxs: {
1361	var n: usize = 0;
1362	for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1363	.undef, .b_elem => {},
1364	.a_elem => |in_idx| {
1365	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1366	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1367	n += 1;
1368	},
1369	};
1370	const a_len = n;
1371	for (shuffle.mask, 0..) |mask, out_idx| switch (mask.unwrap()) {
1372	.undef, .a_elem => {},
1373	.b_elem => |in_idx| {
1374	out_idxs_buf[n] = (try pt.intValue(.usize, out_idx)).toIntern();
1375	in_idxs_buf[n] = (try pt.intValue(.usize, in_idx)).toIntern();
1376	n += 1;
1377	},
1378	};
1379	break :idxs .{
1380	out_idxs_buf[0..a_len],
1381	in_idxs_buf[0..a_len],
1382	out_idxs_buf[a_len..n],
1383	in_idxs_buf[a_len..n],
1384	};
1385	};
1386
1387	var inst_buf: [7]Air.Inst.Index = undefined;
1388	var main_block: Block = .init(&inst_buf);
1389	try l.air_instructions.ensureUnusedCapacity(gpa, 33);
1390
1391	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(shuffle.result_ty)).toRef();
1392	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1393
1394	_ = main_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.undefValue(shuffle.result_ty)));
1395
1396	if (out_idxs_a.len == 0) {
1397	_ = main_block.stealCapacity(1);
1398	} else {
1399	try l.addScalarizedShuffle(
1400	&main_block,
1401	shuffle.operand_a,
1402	result_ptr,
1403	index_ptr,
1404	out_idxs_a,
1405	in_idxs_a,
1406	);
1407	}
1408
1409	if (out_idxs_b.len == 0) {
1410	_ = main_block.stealCapacity(1);
1411	} else {
1412	try l.addScalarizedShuffle(
1413	&main_block,
1414	shuffle.operand_b,
1415	result_ptr,
1416	index_ptr,
1417	out_idxs_b,
1418	in_idxs_b,
1419	);
1420	}
1421
1422	const result_val = main_block.addTyOp(l, .load, shuffle.result_ty, result_ptr).toRef();
1423	main_block.addBr(l, orig_inst, result_val);
1424
1425	return .{ .ty_pl = .{
1426	.ty = shuffle.result_ty,
1427	.payload = try l.addBlockBody(main_block.body()),
1428	} };
1429	}
1430	/// Adds code to `parent_block` which behaves like this loop:
1431	///
1432	/// for (out_idxs, in_idxs) |i, j| result_vec_ptr[i] = operand_vec[j];
1433	///
1434	/// The actual AIR adds exactly one instruction to `parent_block` itself and 14 instructions
1435	/// overall, and is as follows:
1436	///
1437	/// %1 = block(void, {
1438	/// %2 = store(index_ptr, @zero_usize)
1439	/// %3 = loop({
1440	/// %4 = load(index_ptr)
1441	/// %5 = ptr_elem_val(out_idxs_ptr, %4)
1442	/// %6 = ptr_elem_val(in_idxs_ptr, %4)
1443	/// %7 = legalize_vec_elem_val(operand_vec, %6)
1444	/// %8 = legalize_vec_store_elem(result_vec_ptr, %4, %7)
1445	/// %9 = cmp_eq(%4, <usize, out_idxs.len-1>)
1446	/// %10 = cond_br(%9, {
1447	/// %11 = br(%1, @void_value)
1448	/// }, {
1449	/// %12 = add(%4, @one_usize)
1450	/// %13 = store(index_ptr, %12)
1451	/// %14 = repeat(%3)
1452	/// })
1453	/// })
1454	/// })
1455	///
1456	/// The caller is responsible for reserving space in `l.air_instructions`.
1457	fn addScalarizedShuffle(
1458	l: *Legalize,
1459	parent_block: *Block,
1460	operand_vec: Air.Inst.Ref,
1461	result_vec_ptr: Air.Inst.Ref,
1462	index_ptr: Air.Inst.Ref,
1463	out_idxs: []const InternPool.Index,
1464	in_idxs: []const InternPool.Index,
1465	) Error!void {
1466	const pt = l.pt;
1467
1468	assert(out_idxs.len == in_idxs.len);
1469	const n = out_idxs.len;
1470
1471	const idxs_ty = try pt.arrayType(.{ .len = n, .child = .usize_type });
1472	const idxs_ptr_ty = try pt.singleConstPtrType(idxs_ty);
1473	const manyptr_usize_ty = try pt.manyConstPtrType(.usize);
1474
1475	const out_idxs_ptr = try pt.intern(.{ .ptr = .{
1476	.ty = manyptr_usize_ty.toIntern(),
1477	.base_addr = .{ .uav = .{
1478	.val = (try pt.aggregateValue(idxs_ty, out_idxs)).toIntern(),
1479	.orig_ty = idxs_ptr_ty.toIntern(),
1480	} },
1481	.byte_offset = 0,
1482	} });
1483	const in_idxs_ptr = try pt.intern(.{ .ptr = .{
1484	.ty = manyptr_usize_ty.toIntern(),
1485	.base_addr = .{ .uav = .{
1486	.val = (try pt.aggregateValue(idxs_ty, in_idxs)).toIntern(),
1487	.orig_ty = idxs_ptr_ty.toIntern(),
1488	} },
1489	.byte_offset = 0,
1490	} });
1491
1492	const main_block_inst = parent_block.add(l, .{
1493	.tag = .block,
1494	.data = .{ .ty_pl = .{
1495	.ty = .void,
1496	.payload = undefined,
1497	} },
1498	});
1499
1500	var inst_buf: [13]Air.Inst.Index = undefined;
1501	var main_block: Block = .init(&inst_buf);
1502
1503	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1504
1505	var loop: Loop = .init(l, &main_block);
1506	loop.block = .init(main_block.stealRemainingCapacity());
1507
1508	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1509	const in_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(in_idxs_ptr), index_val).toRef();
1510	const out_idx_val = loop.block.addBinOp(l, .ptr_elem_val, .fromIntern(out_idxs_ptr), index_val).toRef();
1511
1512	const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, operand_vec, in_idx_val).toRef();
1513	_ = loop.block.add(l, .{
1514	.tag = .legalize_vec_store_elem,
1515	.data = .{ .pl_op = .{
1516	.operand = result_vec_ptr,
1517	.payload = try l.addExtra(Air.Bin, .{
1518	.lhs = out_idx_val,
1519	.rhs = elem_val,
1520	}),
1521	} },
1522	});
1523
1524	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, n - 1))).toRef();
1525	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1526	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1527	condbr.then_block.addBr(l, main_block_inst, .void_value);
1528
1529	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1530	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1531	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1532	_ = condbr.else_block.add(l, .{
1533	.tag = .repeat,
1534	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1535	});
1536
1537	try condbr.finish(l);
1538	try loop.finish(l);
1539
1540	const inst_data = l.air_instructions.items(.data);
1541	inst_data[@backingInt(main_block_inst)].ty_pl.payload = try l.addBlockBody(main_block.body());
1542	}
1543	fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
1544	const pt = l.pt;
1545	const zcu = pt.zcu;
1546
1547	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
1548
1549	const dest_ty = ty_op.ty;
1550	const operand_ty = l.typeOf(ty_op.operand);
1551
1552	// We exit this block only if the scalarization is actually necessary. Otherwise we will return
1553	// `null` from within the block.
1554	const operand_to_int_ok: bool, const int_to_dest_ok: bool = int_ok: {
1555	const operand_tag = operand_ty.zigTypeTag(zcu);
1556	const dest_tag = dest_ty.zigTypeTag(zcu);
1557
1558	if (operand_tag != .array and
1559	operand_tag != .vector and
1560	dest_tag != .array and
1561	dest_tag != .vector)
1562	{
1563	return null;
1564	}
1565
1566	// We track the validity of 3 different bitcast operations:
1567	// * operand -> dest
1568	// * operand -> uint
1569	// * uint -> dest
1570	// If operand->dest turns out to be valid, we don't need to scalarize. Otherwise, knowing
1571	// the validity of the other operations helps us lower the scalarization efficiently.
1572	var operand_to_dest: bool = true;
1573	var operand_to_int: bool = true;
1574	var int_to_dest: bool = true;
1575
1576	if (l.features.has(.scalarize_bit_cast_array)) {
1577	if (operand_tag == .array) {
1578	operand_to_dest = false;
1579	operand_to_int = false;
1580	}
1581	if (dest_tag == .array) {
1582	operand_to_dest = false;
1583	int_to_dest = false;
1584	}
1585	}
1586
1587	if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
1588	if (operand_tag == .vector) operand_to_int = false;
1589	if (dest_tag == .vector) int_to_dest = false;
1590
1591	if (operand_tag == .vector or dest_tag == .vector) {
1592	if (operand_tag != .vector or
1593	dest_tag != .vector or
1594	operand_ty.vectorLen(zcu) != dest_ty.vectorLen(zcu))
1595	{
1596	operand_to_dest = false;
1597	}
1598	}
1599	}
1600
1601	if (l.features.has(.scalarize_bit_cast_padded_elems)) {
1602	if (operand_tag == .array or operand_tag == .vector) {
1603	const elem_ty = operand_ty.childType(zcu);
1604	if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
1605	operand_to_int = false;
1606	operand_to_dest = false;
1607	}
1608	}
1609	if (dest_tag == .array or dest_tag == .vector) {
1610	const elem_ty = dest_ty.childType(zcu);
1611	if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
1612	int_to_dest = false;
1613	operand_to_dest = false;
1614	}
1615	}
1616	}
1617
1618	if (operand_to_dest) {
1619	return null; // no scalarization needed!
1620	}
1621
1622	// We need a scalarization, but before breaking from the block, check if we can do it
1623	// elementwise---if we can, that's preferable to the generic lowering.
1624	if ((operand_tag == .array or operand_tag == .vector) and
1625	(dest_tag == .array or dest_tag == .vector) and
1626	operand_ty.arrayLenIncludingSentinel(zcu) == dest_ty.arrayLenIncludingSentinel(zcu))
1627	{
1628	// Operand and result types are both arrays/vectors whose element types have the same
1629	// bit size, so we can do an elementwise bitcast.
1630	return try l.scalarizeBlockPayload(orig_inst, .ty_op);
1631	}
1632
1633	break :int_ok .{ operand_to_int, int_to_dest };
1634	};
1635
1636	// Generic scalarization implementation. Our strategy is to use an unsigned integer type as an
1637	// intermediate "bag of bits" representation which can be manipulated by bitwise operations.
1638
1639	const num_bits: u16 = @intCast(dest_ty.bitSize(zcu));
1640	assert(operand_ty.bitSize(zcu) == num_bits);
1641	const uint_ty = try pt.intType(.unsigned, num_bits);
1642	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
1643
1644	var inst_buf: [39]Air.Inst.Index = undefined;
1645	var main_block: Block = .init(&inst_buf);
1646	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1647
1648	// First, convert `operand_ty` to `uint_ty` (`uN`).
1649
1650	const uint_val: Air.Inst.Ref = uint_val: {
1651	if (operand_to_int_ok) {
1652	_ = main_block.stealCapacity(19);
1653	break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1654	}
1655
1656	if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
1657	_ = main_block.stealCapacity(18);
1658	const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
1659	break :uint_val main_block.addBitCast(l, uint_ty, elem);
1660	}
1661
1662	// %1 = block({
1663	// %2 = alloc(*usize)
1664	// %3 = alloc(*uN)
1665	// %4 = store(%2, <usize, operand_len>)
1666	// %5 = store(%3, <uN, 0>)
1667	// %6 = loop({
1668	// %7 = load(%2)
1669	// %8 = array_elem_val(orig_operand, %7)
1670	// %9 = bit_cast(uE, %8)
1671	// %10 = int_cast(uN, %9)
1672	// %11 = load(%3)
1673	// %12 = shl_exact(%11, <uS, E>)
1674	// %13 = bit_or(%12, %10)
1675	// %14 = cmp_eq(%4, @zero_usize)
1676	// %15 = cond_br(%14, {
1677	// %16 = br(%1, %13)
1678	// }, {
1679	// %17 = store(%3, %13)
1680	// %18 = sub(%7, @one_usize)
1681	// %19 = store(%2, %18)
1682	// %20 = repeat(%6)
1683	// })
1684	// })
1685	// })
1686
1687	const elem_bits = operand_ty.childType(zcu).bitSize(zcu);
1688	const elem_bits_val = try pt.intValue(shift_ty, elem_bits);
1689	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
1690
1691	const uint_block_inst = main_block.add(l, .{
1692	.tag = .block,
1693	.data = .{ .ty_pl = .{
1694	.ty = uint_ty,
1695	.payload = undefined,
1696	} },
1697	});
1698	var uint_block: Block = .init(main_block.stealCapacity(19));
1699
1700	const index_ptr = uint_block.addTy(l, .alloc, .ptr_usize).toRef();
1701	const result_ptr = uint_block.addTy(l, .alloc, try pt.singleMutPtrType(uint_ty)).toRef();
1702	_ = uint_block.addBinOp(
1703	l,
1704	.store,
1705	index_ptr,
1706	.fromValue(try pt.intValue(.usize, operand_ty.arrayLen(zcu) - 1)),
1707	);
1708	_ = uint_block.addBinOp(l, .store, result_ptr, .fromValue(try pt.intValue(uint_ty, 0)));
1709
1710	var loop: Loop = .init(l, &uint_block);
1711	loop.block = .init(uint_block.stealRemainingCapacity());
1712
1713	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1714	const raw_elem = loop.block.addBinOp(
1715	l,
1716	if (operand_ty.zigTypeTag(zcu) == .vector) .legalize_vec_elem_val else .array_elem_val,
1717	ty_op.operand,
1718	index_val,
1719	).toRef();
1720	const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1721	const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
1722	const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1723	const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1724	const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
1725
1726	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .zero_usize).toRef();
1727	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1728
1729	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1730	condbr.then_block.addBr(l, uint_block_inst, new_result);
1731
1732	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1733	_ = condbr.else_block.addBinOp(l, .store, result_ptr, new_result);
1734	const new_index_val = condbr.else_block.addBinOp(l, .sub, index_val, .one_usize).toRef();
1735	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1736	_ = condbr.else_block.add(l, .{
1737	.tag = .repeat,
1738	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1739	});
1740
1741	try condbr.finish(l);
1742	try loop.finish(l);
1743
1744	const inst_data = l.air_instructions.items(.data);
1745	inst_data[@backingInt(uint_block_inst)].ty_pl.payload = try l.addBlockBody(uint_block.body());
1746
1747	break :uint_val uint_block_inst.toRef();
1748	};
1749
1750	// Now convert `uint_ty` (`uN`) to `dest_ty`.
1751
1752	// We omit the safety check when casting to an array or a vector since it's
1753	// not supposed to be elementwise.
1754	if (dest_ty.zigTypeTag(zcu) == .@"enum") assert(int_to_dest_ok);
1755
1756	if (int_to_dest_ok) {
1757	_ = main_block.stealCapacity(17);
1758	const result = switch (l.air_instructions.items(.tag)[@backingInt(orig_inst)]) {
1759	.bit_cast => main_block.addBitCast(l, dest_ty, uint_val),
1760	.bit_cast_safe => main_block.add(l, .{
1761	.tag = .bit_cast_safe,
1762	.data = .{ .ty_op = .{
1763	.ty = dest_ty,
1764	.operand = uint_val,
1765	} },
1766	}).toRef(),
1767	else => unreachable,
1768	};
1769	main_block.addBr(l, orig_inst, result);
1770	} else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
1771	_ = main_block.stealCapacity(16);
1772	const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
1773	const aggregate_init_payload_start = l.air_extra.items.len;
1774	try l.air_extra.append(zcu.gpa, @backingInt(elem));
1775	const result = main_block.add(l, .{
1776	.tag = .aggregate_init,
1777	.data = .{ .ty_pl = .{
1778	.ty = dest_ty,
1779	.payload = @intCast(aggregate_init_payload_start),
1780	} },
1781	}).toRef();
1782	main_block.addBr(l, orig_inst, result);
1783	} else {
1784	// %1 = alloc(*usize)
1785	// %2 = alloc(*@Vector(N, Result))
1786	// %3 = store(%1, @zero_usize)
1787	// %4 = loop({
1788	// %5 = load(%1)
1789	// %6 = mul(%5, <usize, E>)
1790	// %7 = int_cast(uS, %6)
1791	// %8 = shr(uint_val, %7)
1792	// %9 = trunc(uE, %8)
1793	// %10 = bit_cast(Result, %9)
1794	// %11 = legalize_vec_store_elem(%2, %5, %10)
1795	// %12 = cmp_eq(%5, <usize, vec_len>)
1796	// %13 = cond_br(%12, {
1797	// %14 = load(%2)
1798	// %15 = br(%0, %14)
1799	// }, {
1800	// %16 = add(%5, @one_usize)
1801	// %17 = store(%1, %16)
1802	// %18 = repeat(%4)
1803	// })
1804	// })
1805	//
1806	// The result might be an array, in which case `legalize_vec_store_elem`
1807	// becomes `ptr_elem_ptr` followed by `store`.
1808
1809	const elem_ty = dest_ty.childType(zcu);
1810	const elem_bits = elem_ty.bitSize(zcu);
1811	const elem_uint_ty = try pt.intType(.unsigned, @intCast(elem_bits));
1812
1813	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1814	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(dest_ty)).toRef();
1815	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1816
1817	var loop: Loop = .init(l, &main_block);
1818	loop.block = .init(main_block.stealRemainingCapacity());
1819
1820	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1821	const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1822	const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
1823	const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
1824	const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1825	const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
1826	switch (dest_ty.zigTypeTag(zcu)) {
1827	.array => {
1828	const elem_ptr = loop.block.add(l, .{
1829	.tag = .ptr_elem_ptr,
1830	.data = .{ .ty_pl = .{
1831	.ty = try pt.singleMutPtrType(elem_ty),
1832	.payload = try l.addExtra(Air.Bin, .{
1833	.lhs = result_ptr,
1834	.rhs = index_val,
1835	}),
1836	} },
1837	}).toRef();
1838	_ = loop.block.addBinOp(l, .store, elem_ptr, elem_val);
1839	},
1840	.vector => {
1841	_ = loop.block.add(l, .{
1842	.tag = .legalize_vec_store_elem,
1843	.data = .{ .pl_op = .{
1844	.operand = result_ptr,
1845	.payload = try l.addExtra(Air.Bin, .{
1846	.lhs = index_val,
1847	.rhs = elem_val,
1848	}),
1849	} },
1850	});
1851	_ = loop.block.stealCapacity(1);
1852	},
1853	else => unreachable,
1854	}
1855
1856	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, dest_ty.arrayLen(zcu) - 1))).toRef();
1857
1858	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1859
1860	condbr.then_block = .init(loop.block.stealRemainingCapacity());
1861	const result_val = condbr.then_block.addTyOp(l, .load, dest_ty, result_ptr).toRef();
1862	condbr.then_block.addBr(l, orig_inst, result_val);
1863
1864	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1865	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1866	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1867	_ = condbr.else_block.add(l, .{
1868	.tag = .repeat,
1869	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
1870	});
1871
1872	try condbr.finish(l);
1873	try loop.finish(l);
1874	}
1875
1876	return .{ .ty_pl = .{
1877	.ty = dest_ty,
1878	.payload = try l.addBlockBody(main_block.body()),
1879	} };
1880	}
1881	fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1882	const pt = l.pt;
1883	const zcu = pt.zcu;
1884
1885	const orig = l.air_instructions.get(@backingInt(orig_inst));
1886	const orig_operands = l.extraData(Air.Bin, orig.data.ty_pl.payload).data;
1887
1888	const vec_tuple_ty = l.typeOfIndex(orig_inst);
1889	const vec_int_ty = vec_tuple_ty.fieldType(0, zcu);
1890	const vec_overflow_ty = vec_tuple_ty.fieldType(1, zcu);
1891
1892	assert(l.typeOf(orig_operands.lhs).toIntern() == vec_int_ty.toIntern());
1893	if (orig.tag != .shl_with_overflow) {
1894	assert(l.typeOf(orig_operands.rhs).toIntern() == vec_int_ty.toIntern());
1895	}
1896
1897	const scalar_int_ty = vec_int_ty.childType(zcu);
1898	const scalar_tuple_ty = try pt.overflowArithmeticTupleType(scalar_int_ty);
1899
1900	// %1 = block(struct { @Vector(N, Int), @Vector(N, u1) }, {
1901	// %2 = alloc(*usize)
1902	// %3 = alloc(*struct { @Vector(N, Int), @Vector(N, u1) })
1903	// %4 = struct_field_ptr_index_0(*@Vector(N, Int), %3)
1904	// %5 = struct_field_ptr_index_1(*@Vector(N, u1), %3)
1905	// %6 = store(%2, @zero_usize)
1906	// %7 = loop({
1907	// %8 = load(%2)
1908	// %9 = legalize_vec_elem_val(orig_lhs, %8)
1909	// %10 = legalize_vec_elem_val(orig_rhs, %8)
1910	// %11 = ???_with_overflow(struct { Int, u1 }, %9, %10)
1911	// %12 = agg_field_val(%11, 0)
1912	// %13 = agg_field_val(%11, 1)
1913	// %14 = legalize_vec_store_elem(%4, %8, %12)
1914	// %15 = legalize_vec_store_elem(%4, %8, %13)
1915	// %16 = cmp_eq(%8, <usize, N-1>)
1916	// %17 = cond_br(%16, {
1917	// %18 = load(%3)
1918	// %19 = br(%1, %18)
1919	// }, {
1920	// %20 = add(%8, @one_usize)
1921	// %21 = store(%2, %20)
1922	// %22 = repeat(%7)
1923	// })
1924	// })
1925	// })
1926
1927	const elems_len = vec_int_ty.vectorLen(zcu);
1928
1929	var inst_buf: [21]Air.Inst.Index = undefined;
1930	var main_block: Block = .init(&inst_buf);
1931	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1932
1933	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1934	const result_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(vec_tuple_ty)).toRef();
1935	const result_int_ptr = main_block.addTyOp(
1936	l,
1937	.struct_field_ptr_index_0,
1938	try pt.singleMutPtrType(vec_int_ty),
1939	result_ptr,
1940	).toRef();
1941	const result_overflow_ptr = main_block.addTyOp(
1942	l,
1943	.struct_field_ptr_index_1,
1944	try pt.singleMutPtrType(vec_overflow_ty),
1945	result_ptr,
1946	).toRef();
1947
1948	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1949
1950	var loop: Loop = .init(l, &main_block);
1951	loop.block = .init(main_block.stealRemainingCapacity());
1952
1953	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1954	const lhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.lhs, index_val).toRef();
1955	const rhs = loop.block.addBinOp(l, .legalize_vec_elem_val, orig_operands.rhs, index_val).toRef();
1956	const elem_result = loop.block.add(l, .{
1957	.tag = orig.tag,
1958	.data = .{ .ty_pl = .{
1959	.ty = scalar_tuple_ty,
1960	.payload = try l.addExtra(Air.Bin, .{ .lhs = lhs, .rhs = rhs }),
1961	} },
1962	}).toRef();
1963	const int_elem = loop.block.add(l, .{
1964	.tag = .agg_field_val,
1965	.data = .{ .ty_pl = .{
1966	.ty = scalar_int_ty,
1967	.payload = try l.addExtra(Air.StructField, .{
1968	.struct_operand = elem_result,
1969	.field_index = 0,
1970	}),
1971	} },
1972	}).toRef();
1973	const overflow_elem = loop.block.add(l, .{
1974	.tag = .agg_field_val,
1975	.data = .{ .ty_pl = .{
1976	.ty = .u1,
1977	.payload = try l.addExtra(Air.StructField, .{
1978	.struct_operand = elem_result,
1979	.field_index = 1,
1980	}),
1981	} },
1982	}).toRef();
1983	_ = loop.block.add(l, .{
1984	.tag = .legalize_vec_store_elem,
1985	.data = .{ .pl_op = .{
1986	.operand = result_int_ptr,
1987	.payload = try l.addExtra(Air.Bin, .{
1988	.lhs = index_val,
1989	.rhs = int_elem,
1990	}),
1991	} },
1992	});
1993	_ = loop.block.add(l, .{
1994	.tag = .legalize_vec_store_elem,
1995	.data = .{ .pl_op = .{
1996	.operand = result_overflow_ptr,
1997	.payload = try l.addExtra(Air.Bin, .{
1998	.lhs = index_val,
1999	.rhs = overflow_elem,
2000	}),
2001	} },
2002	});
2003
2004	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, elems_len - 1))).toRef();
2005	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
2006
2007	condbr.then_block = .init(loop.block.stealRemainingCapacity());
2008	const result_val = condbr.then_block.addTyOp(l, .load, vec_tuple_ty, result_ptr).toRef();
2009	condbr.then_block.addBr(l, orig_inst, result_val);
2010
2011	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2012	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
2013	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
2014	_ = condbr.else_block.add(l, .{
2015	.tag = .repeat,
2016	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
2017	});
2018
2019	try condbr.finish(l);
2020	try loop.finish(l);
2021
2022	return .{ .ty_pl = .{
2023	.ty = vec_tuple_ty,
2024	.payload = try l.addBlockBody(main_block.body()),
2025	} };
2026	}
2027	fn scalarizeReduceBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimized: bool) Error!Air.Inst.Data {
2028	const pt = l.pt;
2029	const zcu = pt.zcu;
2030
2031	const reduce = l.air_instructions.items(.data)[@backingInt(orig_inst)].reduce;
2032
2033	const vector_ty = l.typeOf(reduce.operand);
2034	const scalar_ty = vector_ty.childType(zcu);
2035
2036	const ident_val: Value = switch (reduce.operation) {
2037	// identity for add is 0; identity for OR and XOR is all 0 bits
2038	.Or, .Xor, .Add => switch (scalar_ty.zigTypeTag(zcu)) {
2039	.int => try pt.intValue(scalar_ty, 0),
2040	.float => try pt.floatValue(scalar_ty, 0.0),
2041	.bool => .false,
2042	else => unreachable,
2043	},
2044	// identity for multiplication is 1
2045	.Mul => switch (scalar_ty.zigTypeTag(zcu)) {
2046	.int => try pt.intValue(scalar_ty, 1),
2047	.float => try pt.floatValue(scalar_ty, 1.0),
2048	else => unreachable,
2049	},
2050	// identity for AND is all 1 bits
2051	.And => switch (scalar_ty.zigTypeTag(zcu)) {
2052	.int => switch (scalar_ty.intInfo(zcu).signedness) {
2053	.unsigned => try scalar_ty.maxIntScalar(pt, scalar_ty),
2054	.signed => try pt.intValue(scalar_ty, -1),
2055	},
2056	.bool => .true,
2057	else => unreachable,
2058	},
2059	// identity for @min is maximum value
2060	.Min => switch (scalar_ty.zigTypeTag(zcu)) {
2061	.int => try scalar_ty.maxIntScalar(pt, scalar_ty),
2062	.float => try pt.floatValue(scalar_ty, std.math.inf(f32)),
2063	else => unreachable,
2064	},
2065	// identity for @max is minimum value
2066	.Max => switch (scalar_ty.zigTypeTag(zcu)) {
2067	.int => try scalar_ty.minIntScalar(pt, scalar_ty),
2068	.float => try pt.floatValue(scalar_ty, -std.math.inf(f32)),
2069	else => unreachable,
2070	},
2071	};
2072
2073	const op_tag: Air.Inst.Tag = switch (reduce.operation) {
2074	.Or => .bit_or,
2075	.And => .bit_and,
2076	.Xor => .xor,
2077	.Min => .min,
2078	.Max => .max,
2079	.Add => switch (scalar_ty.zigTypeTag(zcu)) {
2080	.int => .add_wrap,
2081	.float => if (optimized) .add_optimized else .add,
2082	else => unreachable,
2083	},
2084	.Mul => switch (scalar_ty.zigTypeTag(zcu)) {
2085	.int => .mul_wrap,
2086	.float => if (optimized) .mul_optimized else .mul,
2087	else => unreachable,
2088	},
2089	};
2090
2091	// %1 = block(Scalar, {
2092	// %2 = alloc(*usize)
2093	// %3 = alloc(*Scalar)
2094	// %4 = store(%2, @zero_usize)
2095	// %5 = store(%3, <Scalar, 0>) // or whatever the identity is for this operator
2096	// %6 = loop({
2097	// %7 = load(%2)
2098	// %8 = legalize_vec_elem_val(orig_operand, %7)
2099	// %9 = load(%3)
2100	// %10 = add(%8, %9) // or whatever the operator is
2101	// %11 = cmp_eq(%7, <usize, N-1>)
2102	// %12 = cond_br(%11, {
2103	// %13 = br(%1, %10)
2104	// }, {
2105	// %14 = store(%3, %10)
2106	// %15 = add(%7, @one_usize)
2107	// %16 = store(%2, %15)
2108	// %17 = repeat(%6)
2109	// })
2110	// })
2111	// })
2112
2113	var inst_buf: [16]Air.Inst.Index = undefined;
2114	var main_block: Block = .init(&inst_buf);
2115	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2116
2117	const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
2118	const accum_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(scalar_ty)).toRef();
2119	_ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
2120	_ = main_block.addBinOp(l, .store, accum_ptr, .fromValue(ident_val));
2121
2122	var loop: Loop = .init(l, &main_block);
2123	loop.block = .init(main_block.stealRemainingCapacity());
2124
2125	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
2126	const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, reduce.operand, index_val).toRef();
2127	const old_accum = loop.block.addTyOp(l, .load, scalar_ty, accum_ptr).toRef();
2128	const new_accum = loop.block.addBinOp(l, op_tag, old_accum, elem_val).toRef();
2129
2130	const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, vector_ty.vectorLen(zcu) - 1))).toRef();
2131
2132	var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
2133
2134	condbr.then_block = .init(loop.block.stealRemainingCapacity());
2135	condbr.then_block.addBr(l, orig_inst, new_accum);
2136
2137	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2138	_ = condbr.else_block.addBinOp(l, .store, accum_ptr, new_accum);
2139	const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
2140	_ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
2141	_ = condbr.else_block.add(l, .{
2142	.tag = .repeat,
2143	.data = .{ .repeat = .{ .loop_inst = loop.inst } },
2144	});
2145
2146	try condbr.finish(l);
2147	try loop.finish(l);
2148
2149	return .{ .ty_pl = .{
2150	.ty = scalar_ty,
2151	.payload = try l.addBlockBody(main_block.body()),
2152	} };
2153	}
2154
2155	fn safeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
2156	const pt = l.pt;
2157	const zcu = pt.zcu;
2158	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
2159
2160	const operand_ref = ty_op.operand;
2161	const dest_ty = ty_op.ty;
2162
2163	if (dest_ty.zigTypeTag(zcu) != .@"enum" or
2164	dest_ty.isNonexhaustiveEnum(zcu) or
2165	!zcu.backendSupportsFeature(.is_named_enum_value))
2166	{
2167	return null;
2168	}
2169
2170	// We are building this:
2171	//
2172	// %x = block({
2173	// %1 = bit_cast(@res_ty, %y)
2174	// %2 = is_named_enum_value(%1)
2175	// %3 = cond_br(%2, {
2176	// %4 = br(%x, %1)
2177	// }, {
2178	// %5 = call(@panic.invalidEnumValue, [])
2179	// %6 = unreach()
2180	// })
2181	// })
2182
2183	var inst_buf: [6]Air.Inst.Index = undefined;
2184	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2185
2186	var block: Block = .init(&inst_buf);
2187
2188	const cast_inst = block.addBitCast(l, dest_ty, operand_ref);
2189	const is_named_inst = block.add(l, .{
2190	.tag = .is_named_enum_value,
2191	.data = .{ .un_op = cast_inst },
2192	});
2193
2194	var condbr: CondBr = .init(l, is_named_inst.toRef(), &block, .{ .false = .cold });
2195
2196	condbr.then_block = .init(block.stealRemainingCapacity());
2197	condbr.then_block.addBr(l, orig_inst, cast_inst);
2198
2199	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2200	try condbr.else_block.addPanic(l, .invalid_enum_value);
2201
2202	try condbr.finish(l);
2203
2204	return .{ .ty_pl = .{
2205	.ty = dest_ty,
2206	.payload = try l.addBlockBody(block.body()),
2207	} };
2208	}
2209
2210	fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?Air.Inst.Data {
2211	const pt = l.pt;
2212	const zcu = pt.zcu;
2213	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
2214
2215	const operand_ref = ty_op.operand;
2216	const operand_ty = l.typeOf(operand_ref);
2217	const dest_ty = ty_op.ty;
2218
2219	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
2220	const operand_scalar_ty = operand_ty.scalarType(zcu);
2221	const dest_scalar_ty = dest_ty.scalarType(zcu);
2222
2223	assert(operand_scalar_ty.zigTypeTag(zcu) == .int);
2224	const dest_is_enum = switch (dest_scalar_ty.zigTypeTag(zcu)) {
2225	.int => false,
2226	.@"enum" => true,
2227	else => unreachable,
2228	};
2229	const have_enum_value_check = dest_is_enum and
2230	!dest_ty.isNonexhaustiveEnum(zcu) and
2231	zcu.backendSupportsFeature(.is_named_enum_value);
2232
2233	const operand_info = operand_scalar_ty.intInfo(zcu);
2234	const dest_info = dest_scalar_ty.intInfo(zcu);
2235
2236	const have_min_check, const have_max_check = c: {
2237	const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
2238	const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
2239	const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
2240	const operand_allows_neg = operand_info.signedness == .signed and operand_info.bits > 0;
2241	break :c .{
2242	operand_allows_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
2243	dest_pos_bits < operand_pos_bits,
2244	};
2245	};
2246
2247	if (!have_enum_value_check and !have_min_check and !have_max_check) {
2248	return null;
2249	}
2250
2251	// The worst-case scenario in terms of total instructions and total condbrs is the case where
2252	// the result type is an exhaustive enum whose tag type is smaller than the operand type:
2253	//
2254	// %x = block({
2255	// %1 = cmp_lt(%y, @min_allowed_int)
2256	// %2 = cmp_gt(%y, @max_allowed_int)
2257	// %3 = bool_or(%1, %2)
2258	// %4 = cond_br(%3, {
2259	// %5 = call(@panic.invalidEnumValue, [])
2260	// %6 = unreach()
2261	// }, {
2262	// %7 = int_cast(@res_ty, %y)
2263	// %8 = is_named_enum_value(%7)
2264	// %9 = cond_br(%8, {
2265	// %10 = br(%x, %7)
2266	// }, {
2267	// %11 = call(@panic.invalidEnumValue, [])
2268	// %12 = unreach()
2269	// })
2270	// })
2271	// })
2272	//
2273	// Note that vectors of enums don't exist -- the worst case for vectors is this:
2274	//
2275	// %x = block({
2276	// %1 = cmp_lt(%y, @min_allowed_int)
2277	// %2 = cmp_gt(%y, @max_allowed_int)
2278	// %3 = bool_or(%1, %2)
2279	// %4 = reduce(%3, .@"or")
2280	// %5 = cond_br(%4, {
2281	// %6 = call(@panic.invalidEnumValue, [])
2282	// %7 = unreach()
2283	// }, {
2284	// %8 = int_cast(@res_ty, %y)
2285	// %9 = br(%x, %8)
2286	// })
2287	// })
2288
2289	var inst_buf: [12]Air.Inst.Index = undefined;
2290	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2291	var condbr_buf: [2]CondBr = undefined;
2292	var condbr_idx: usize = 0;
2293
2294	var main_block: Block = .init(&inst_buf);
2295	var cur_block: *Block = &main_block;
2296
2297	const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
2298
2299	if (have_min_check or have_max_check) {
2300	const dest_int_ty = if (dest_is_enum) dest_ty.backingIntType(zcu) else dest_ty;
2301	const condbr = &condbr_buf[condbr_idx];
2302	condbr_idx += 1;
2303	const below_min_inst: Air.Inst.Index = if (have_min_check) inst: {
2304	const min_val_ref = Air.internedToRef((try dest_int_ty.minInt(pt, operand_ty)).toIntern());
2305	break :inst try cur_block.addCmp(l, .lt, operand_ref, min_val_ref, .{ .vector = is_vector });
2306	} else undefined;
2307	const above_max_inst: Air.Inst.Index = if (have_max_check) inst: {
2308	const max_val_ref = Air.internedToRef((try dest_int_ty.maxInt(pt, operand_ty)).toIntern());
2309	break :inst try cur_block.addCmp(l, .gt, operand_ref, max_val_ref, .{ .vector = is_vector });
2310	} else undefined;
2311	const out_of_range_inst: Air.Inst.Index = inst: {
2312	if (have_min_check and have_max_check) break :inst cur_block.add(l, .{
2313	.tag = .bit_or,
2314	.data = .{ .bin_op = .{
2315	.lhs = below_min_inst.toRef(),
2316	.rhs = above_max_inst.toRef(),
2317	} },
2318	});
2319	if (have_min_check) break :inst below_min_inst;
2320	if (have_max_check) break :inst above_max_inst;
2321	unreachable;
2322	};
2323	const scalar_out_of_range_inst: Air.Inst.Index = if (is_vector) cur_block.add(l, .{
2324	.tag = .reduce,
2325	.data = .{ .reduce = .{
2326	.operand = out_of_range_inst.toRef(),
2327	.operation = .Or,
2328	} },
2329	}) else out_of_range_inst;
2330	condbr.* = .init(l, scalar_out_of_range_inst.toRef(), cur_block, .{ .true = .cold });
2331	condbr.then_block = .init(cur_block.stealRemainingCapacity());
2332	try condbr.then_block.addPanic(l, panic_id);
2333	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2334	cur_block = &condbr.else_block;
2335	}
2336
2337	// Now we know we're in-range, we can int_cast:
2338	const cast_inst = cur_block.add(l, .{
2339	.tag = .int_cast,
2340	.data = .{ .ty_op = .{
2341	.ty = dest_ty,
2342	.operand = operand_ref,
2343	} },
2344	});
2345	// For ints we're already done, but for exhaustive enums we must check this is a valid tag.
2346	if (have_enum_value_check) {
2347	assert(!is_vector); // vectors of enums don't exist
2348	// We are building this:
2349	// %1 = is_named_enum_value(%cast_inst)
2350	// %2 = cond_br(%1, {
2351	// <new cursor>
2352	// }, {
2353	// <panic>
2354	// })
2355	const is_named_inst = cur_block.add(l, .{
2356	.tag = .is_named_enum_value,
2357	.data = .{ .un_op = cast_inst.toRef() },
2358	});
2359	const condbr = &condbr_buf[condbr_idx];
2360	condbr_idx += 1;
2361	condbr.* = .init(l, is_named_inst.toRef(), cur_block, .{ .false = .cold });
2362	condbr.else_block = .init(cur_block.stealRemainingCapacity());
2363	try condbr.else_block.addPanic(l, panic_id);
2364	condbr.then_block = .init(condbr.else_block.stealRemainingCapacity());
2365	cur_block = &condbr.then_block;
2366	}
2367	// Finally, just `br` to our outer `block`.
2368	cur_block.addBr(l, orig_inst, cast_inst.toRef());
2369
2370	// We might not have used all of the instructions; that's intentional.
2371	_ = cur_block.stealRemainingCapacity();
2372
2373	assert(condbr_idx != 0); // should have already returned `null`
2374	for (condbr_buf[0..condbr_idx]) |*condbr| try condbr.finish(l);
2375	return .{ .ty_pl = .{
2376	.ty = dest_ty,
2377	.payload = try l.addBlockBody(main_block.body()),
2378	} };
2379	}
2380	fn safeIntFromFloatBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimized: bool) Error!Air.Inst.Data {
2381	const pt = l.pt;
2382	const zcu = pt.zcu;
2383	const gpa = zcu.gpa;
2384	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
2385
2386	const operand_ref = ty_op.operand;
2387	const operand_ty = l.typeOf(operand_ref);
2388	const dest_ty = ty_op.ty;
2389
2390	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
2391	const dest_scalar_ty = dest_ty.scalarType(zcu);
2392	const int_info = dest_scalar_ty.intInfo(zcu);
2393
2394	// We emit 9 instructions in the worst case.
2395	var inst_buf: [9]Air.Inst.Index = undefined;
2396	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2397	var main_block: Block = .init(&inst_buf);
2398
2399	// This check is a bit annoying because of floating-point rounding and the fact that this
2400	// builtin truncates. We'll use a bigint for our calculations, because we need to construct
2401	// integers exceeding the bounds of the result integer type, and we need to convert it to a
2402	// float with a specific rounding mode to avoid errors.
2403	// Our bigint may exceed the twos complement limit by one, so add an extra limb.
2404	const limbs = try gpa.alloc(
2405	std.math.big.Limb,
2406	std.math.big.int.calcTwosCompLimbCount(int_info.bits) + 1,
2407	);
2408	defer gpa.free(limbs);
2409	var big: std.math.big.int.Mutable = .init(limbs, 0);
2410
2411	// Check if the operand is lower than `min_int` when truncated to an integer.
2412	big.setTwosCompIntLimit(.min, int_info.signedness, int_info.bits);
2413	const below_min_inst: Air.Inst.Index = if (!big.positive or big.eqlZero()) bad: {
2414	// `min_int <= 0`, so check for `x <= min_int - 1`.
2415	big.addScalar(big.toConst(), -1);
2416	// For `<=`, we must round the RHS down, so that this value is the first `x` which returns `true`.
2417	const limit_val = try floatFromBigIntVal(pt, is_vector, operand_ty, big.toConst(), .floor);
2418	break :bad try main_block.addCmp(l, .lte, operand_ref, Air.internedToRef(limit_val.toIntern()), .{
2419	.vector = is_vector,
2420	.optimized = optimized,
2421	});
2422	} else {
2423	// `min_int > 0`, which is currently impossible. It would become possible under #3806, in
2424	// which case we must detect `x < min_int`.
2425	unreachable;
2426	};
2427
2428	// Check if the operand is greater than `max_int` when truncated to an integer.
2429	big.setTwosCompIntLimit(.max, int_info.signedness, int_info.bits);
2430	const above_max_inst: Air.Inst.Index = if (big.positive or big.eqlZero()) bad: {
2431	// `max_int >= 0`, so check for `x >= max_int + 1`.
2432	big.addScalar(big.toConst(), 1);
2433	// For `>=`, we must round the RHS up, so that this value is the first `x` which returns `true`.
2434	const limit_val = try floatFromBigIntVal(pt, is_vector, operand_ty, big.toConst(), .ceil);
2435	break :bad try main_block.addCmp(l, .gte, operand_ref, Air.internedToRef(limit_val.toIntern()), .{
2436	.vector = is_vector,
2437	.optimized = optimized,
2438	});
2439	} else {
2440	// `max_int < 0`, which is currently impossible. It would become possible under #3806, in
2441	// which case we must detect `x > max_int`.
2442	unreachable;
2443	};
2444
2445	// Combine the conditions.
2446	const out_of_bounds_inst: Air.Inst.Index = main_block.add(l, .{
2447	.tag = .bit_or,
2448	.data = .{ .bin_op = .{
2449	.lhs = below_min_inst.toRef(),
2450	.rhs = above_max_inst.toRef(),
2451	} },
2452	});
2453	const scalar_out_of_bounds_inst: Air.Inst.Index = if (is_vector) main_block.add(l, .{
2454	.tag = .reduce,
2455	.data = .{ .reduce = .{
2456	.operand = out_of_bounds_inst.toRef(),
2457	.operation = .Or,
2458	} },
2459	}) else out_of_bounds_inst;
2460
2461	// Now emit the actual condbr. "true" will be safety panic. "false" will be "ok", meaning we do
2462	// the `int_from_float` and `br` the result to `orig_inst`.
2463	var condbr: CondBr = .init(l, scalar_out_of_bounds_inst.toRef(), &main_block, .{ .true = .cold });
2464	condbr.then_block = .init(main_block.stealRemainingCapacity());
2465	try condbr.then_block.addPanic(l, .integer_part_out_of_bounds);
2466	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2467	const cast_inst = condbr.else_block.add(l, .{
2468	.tag = if (optimized) .int_from_float_optimized else .int_from_float,
2469	.data = .{ .ty_op = .{
2470	.ty = dest_ty,
2471	.operand = operand_ref,
2472	} },
2473	});
2474	_ = condbr.else_block.add(l, .{
2475	.tag = .br,
2476	.data = .{ .br = .{
2477	.block_inst = orig_inst,
2478	.operand = cast_inst.toRef(),
2479	} },
2480	});
2481	_ = condbr.else_block.stealRemainingCapacity(); // we might not have used it all
2482	try condbr.finish(l);
2483
2484	return .{ .ty_pl = .{
2485	.ty = dest_ty,
2486	.payload = try l.addBlockBody(main_block.body()),
2487	} };
2488	}
2489	fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_op_tag: Air.Inst.Tag) Error!Air.Inst.Data {
2490	const pt = l.pt;
2491	const zcu = pt.zcu;
2492	const bin_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].bin_op;
2493
2494	const operand_ty = l.typeOf(bin_op.lhs);
2495	assert(l.typeOf(bin_op.rhs).toIntern() == operand_ty.toIntern());
2496	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
2497
2498	const overflow_tuple_ty = try pt.overflowArithmeticTupleType(operand_ty);
2499	const overflow_bits_ty = overflow_tuple_ty.fieldType(1, zcu);
2500
2501	// The worst-case scenario is a vector operand:
2502	//
2503	// %1 = add_with_overflow(%x, %y)
2504	// %2 = agg_field_val(%1, .@"1")
2505	// %3 = reduce(%2, .@"or")
2506	// %4 = bit_cast(%3, @bool_type)
2507	// %5 = cond_br(%4, {
2508	// %6 = call(@panic.integerOverflow, [])
2509	// %7 = unreach()
2510	// }, {
2511	// %8 = agg_field_val(%1, .@"0")
2512	// %9 = br(%z, %8)
2513	// })
2514	var inst_buf: [9]Air.Inst.Index = undefined;
2515	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2516
2517	var main_block: Block = .init(&inst_buf);
2518
2519	const overflow_op_inst = main_block.add(l, .{
2520	.tag = overflow_op_tag,
2521	.data = .{ .ty_pl = .{
2522	.ty = overflow_tuple_ty,
2523	.payload = try l.addExtra(Air.Bin, .{
2524	.lhs = bin_op.lhs,
2525	.rhs = bin_op.rhs,
2526	}),
2527	} },
2528	});
2529	const overflow_bits_inst = main_block.add(l, .{
2530	.tag = .agg_field_val,
2531	.data = .{ .ty_pl = .{
2532	.ty = overflow_bits_ty,
2533	.payload = try l.addExtra(Air.StructField, .{
2534	.struct_operand = overflow_op_inst.toRef(),
2535	.field_index = 1,
2536	}),
2537	} },
2538	});
2539	const any_overflow_bit_inst = if (is_vector) main_block.add(l, .{
2540	.tag = .reduce,
2541	.data = .{ .reduce = .{
2542	.operand = overflow_bits_inst.toRef(),
2543	.operation = .Or,
2544	} },
2545	}) else overflow_bits_inst;
2546	const any_overflow_inst = try main_block.addCmp(l, .eq, any_overflow_bit_inst.toRef(), .one_u1, .{});
2547
2548	var condbr: CondBr = .init(l, any_overflow_inst.toRef(), &main_block, .{ .true = .cold });
2549	condbr.then_block = .init(main_block.stealRemainingCapacity());
2550	try condbr.then_block.addPanic(l, .integer_overflow);
2551	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
2552
2553	const result_inst = condbr.else_block.add(l, .{
2554	.tag = .agg_field_val,
2555	.data = .{ .ty_pl = .{
2556	.ty = operand_ty,
2557	.payload = try l.addExtra(Air.StructField, .{
2558	.struct_operand = overflow_op_inst.toRef(),
2559	.field_index = 0,
2560	}),
2561	} },
2562	});
2563	_ = condbr.else_block.add(l, .{
2564	.tag = .br,
2565	.data = .{ .br = .{
2566	.block_inst = orig_inst,
2567	.operand = result_inst.toRef(),
2568	} },
2569	});
2570	// We might not have used all of the instructions; that's intentional.
2571	_ = condbr.else_block.stealRemainingCapacity();
2572
2573	try condbr.finish(l);
2574	return .{ .ty_pl = .{
2575	.ty = operand_ty,
2576	.payload = try l.addBlockBody(main_block.body()),
2577	} };
2578	}
2579
2580	fn divCeilBlockPayload(
2581	l: *Legalize,
2582	orig_inst: Air.Inst.Index,
2583	air_tag: Air.Inst.Tag,
2584	) Error!Air.Inst.Data {
2585	const pt = l.pt;
2586	const zcu = pt.zcu;
2587	const gpa = zcu.gpa;
2588
2589	const bin_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].bin_op;
2590	const operand_ty = l.typeOf(bin_op.lhs);
2591	assert(l.typeOf(bin_op.rhs).toIntern() == operand_ty.toIntern());
2592
2593	const scalar_ty = operand_ty.scalarType(zcu);
2594	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
2595
2596	switch (scalar_ty.zigTypeTag(zcu)) {
2597	.float => {
2598	// %result = ceil(lhs / rhs)
2599
2600	var inst_buf: [3]Air.Inst.Index = undefined;
2601	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2602
2603	var main_block: Block = .init(&inst_buf);
2604
2605	const div_tag: Air.Inst.Tag = switch (air_tag) {
2606	.div_ceil => .div_float,
2607	.div_ceil_optimized => .div_float_optimized,
2608	else => unreachable,
2609	};
2610
2611	const div_inst = main_block.add(l, .{
2612	.tag = div_tag,
2613	.data = .{ .bin_op = bin_op },
2614	});
2615
2616	const ceil_inst = main_block.add(l, .{
2617	.tag = .ceil,
2618	.data = .{ .un_op = div_inst.toRef() },
2619	});
2620
2621	main_block.addBr(l, orig_inst, ceil_inst.toRef());
2622
2623	_ = main_block.stealRemainingCapacity();
2624	return .{ .ty_pl = .{
2625	.ty = operand_ty,
2626	.payload = try l.addBlockBody(main_block.body()),
2627	} };
2628	},
2629
2630	.int => {
2631	// Integer div_ceil:
2632	//
2633	// q = div_trunc(lhs, rhs)
2634	// r = rem(lhs, rhs)
2635	//
2636	// unsigned:
2637	// q + int(r != 0)
2638	//
2639	// signed:
2640	// q + int(r != 0 and same_sign(lhs, rhs))
2641	//
2642	// same_sign is `(lhs ^ rhs) >= 0`.
2643
2644	var inst_buf: [10]Air.Inst.Index = undefined;
2645	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2646
2647	var main_block: Block = .init(&inst_buf);
2648
2649	const q_inst = main_block.add(l, .{
2650	.tag = .div_trunc,
2651	.data = .{ .bin_op = bin_op },
2652	});
2653
2654	const r_inst = main_block.add(l, .{
2655	.tag = .rem,
2656	.data = .{ .bin_op = bin_op },
2657	});
2658
2659	const zero_ref: Air.Inst.Ref = if (is_vector) zero: {
2660	const zero_scalar = try pt.intValue(scalar_ty, 0);
2661	const zero_vec = try pt.aggregateSplatValue(operand_ty, zero_scalar);
2662	break :zero Air.internedToRef(zero_vec.toIntern());
2663	} else Air.internedToRef((try pt.intValue(operand_ty, 0)).toIntern());
2664
2665	const r_nonzero_inst = try main_block.addCmp(
2666	l,
2667	.neq,
2668	r_inst.toRef(),
2669	zero_ref,
2670	.{ .vector = is_vector },
2671	);
2672
2673	const int_info = scalar_ty.intInfo(zcu);
2674
2675	const need_adjust_inst: Air.Inst.Index = if (int_info.signedness == .unsigned) r_nonzero_inst else inst: {
2676	const sign_xor_inst = main_block.add(l, .{
2677	.tag = .xor,
2678	.data = .{ .bin_op = .{
2679	.lhs = bin_op.lhs,
2680	.rhs = bin_op.rhs,
2681	} },
2682	});
2683
2684	const signs_same_inst = try main_block.addCmp(
2685	l,
2686	.gte,
2687	sign_xor_inst.toRef(),
2688	zero_ref,
2689	.{ .vector = is_vector },
2690	);
2691
2692	break :inst main_block.add(l, .{
2693	.tag = .bit_and,
2694	.data = .{ .bin_op = .{
2695	.lhs = r_nonzero_inst.toRef(),
2696	.rhs = signs_same_inst.toRef(),
2697	} },
2698	});
2699	};
2700
2701	const adjust_u1_ty = if (is_vector)
2702	try pt.vectorType(.{
2703	.len = operand_ty.vectorLen(zcu),
2704	.child = Type.u1.toIntern(),
2705	})
2706	else
2707	Type.u1;
2708
2709	const adjust_u1_ref = main_block.addBitCast(l, adjust_u1_ty, need_adjust_inst.toRef());
2710	const adjust_inst = main_block.addTyOp(l, .int_cast, operand_ty, adjust_u1_ref);
2711
2712	const result_inst = main_block.add(l, .{
2713	.tag = .add,
2714	.data = .{ .bin_op = .{
2715	.lhs = q_inst.toRef(),
2716	.rhs = adjust_inst.toRef(),
2717	} },
2718	});
2719
2720	main_block.addBr(l, orig_inst, result_inst.toRef());
2721
2722	_ = main_block.stealRemainingCapacity();
2723	return .{ .ty_pl = .{
2724	.ty = operand_ty,
2725	.payload = try l.addBlockBody(main_block.body()),
2726	} };
2727	},
2728
2729	else => unreachable,
2730	}
2731	}
2732
2733	fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2734	const pt = l.pt;
2735	const zcu = pt.zcu;
2736
2737	const orig_ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
2738	const res_ty = orig_ty_op.ty;
2739	const res_int_ty = try pt.intType(.unsigned, @intCast(res_ty.bitSize(zcu)));
2740	const ptr_ty = l.typeOf(orig_ty_op.operand);
2741	const ptr_info = ptr_ty.ptrInfo(zcu);
2742	// This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type.
2743	const load_bits = 8 * ptr_info.packed_offset.host_size;
2744	const load_ty = try pt.intType(.unsigned, load_bits);
2745
2746	var inst_buf: [6]Air.Inst.Index = undefined;
2747	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2748
2749	var res_block: Block = .init(&inst_buf);
2750	_ = res_block.add(l, .{
2751	.tag = .br,
2752	.data = .{ .br = .{
2753	.block_inst = orig_inst,
2754	.operand = res_block.addBitCast(l, res_ty, res_block.add(l, .{
2755	.tag = .trunc,
2756	.data = .{ .ty_op = .{
2757	.ty = res_int_ty,
2758	.operand = res_block.add(l, .{
2759	.tag = .shr,
2760	.data = .{ .bin_op = .{
2761	.lhs = res_block.add(l, .{
2762	.tag = .load,
2763	.data = .{ .ty_op = .{
2764	.ty = load_ty,
2765	.operand = res_block.addPtrCast(l, load_ptr_ty: {
2766	var load_ptr_info = ptr_info;
2767	load_ptr_info.child = load_ty.toIntern();
2768	load_ptr_info.flags.vector_index = .none;
2769	load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2770	break :load_ptr_ty try pt.ptrType(load_ptr_info);
2771	}, orig_ty_op.operand),
2772	} },
2773	}).toRef(),
2774	.rhs = try pt.intRef(
2775	try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_bits)),
2776	ptr_info.packed_offset.bit_offset,
2777	),
2778	} },
2779	}).toRef(),
2780	} },
2781	}).toRef()),
2782	} },
2783	});
2784	return .{ .ty_pl = .{
2785	.ty = res_ty,
2786	.payload = try l.addBlockBody(res_block.body()),
2787	} };
2788	}
2789	fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2790	const pt = l.pt;
2791	const zcu = pt.zcu;
2792
2793	const orig_bin_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].bin_op;
2794	const ptr_ty = l.typeOf(orig_bin_op.lhs);
2795	const ptr_info = ptr_ty.ptrInfo(zcu);
2796	const operand_ty = l.typeOf(orig_bin_op.rhs);
2797	const operand_bits: u16 = @intCast(operand_ty.bitSize(zcu));
2798	const operand_int_ty = try pt.intType(.unsigned, operand_bits);
2799	// This relies on a heap of possibly invalid assumptions to work around not knowing the actual backing type.
2800	const load_store_bits = 8 * ptr_info.packed_offset.host_size;
2801	const load_store_ty = try pt.intType(.unsigned, load_store_bits);
2802
2803	var inst_buf: [9]Air.Inst.Index = undefined;
2804	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2805
2806	var res_block: Block = .init(&inst_buf);
2807	{
2808	const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
2809	var load_ptr_info = ptr_info;
2810	load_ptr_info.child = load_store_ty.toIntern();
2811	load_ptr_info.flags.vector_index = .none;
2812	load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2813	break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2814	}, orig_bin_op.lhs);
2815	_ = res_block.add(l, .{
2816	.tag = .store,
2817	.data = .{ .bin_op = .{
2818	.lhs = backing_ptr,
2819	.rhs = res_block.add(l, .{
2820	.tag = .bit_or,
2821	.data = .{ .bin_op = .{
2822	.lhs = res_block.add(l, .{
2823	.tag = .bit_and,
2824	.data = .{ .bin_op = .{
2825	.lhs = res_block.add(l, .{
2826	.tag = .load,
2827	.data = .{ .ty_op = .{
2828	.ty = load_store_ty,
2829	.operand = backing_ptr,
2830	} },
2831	}).toRef(),
2832	.rhs = Air.internedToRef((keep_mask: {
2833	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
2834	var bfa_buf: ExpectedContents = undefined;
2835	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), zcu.gpa);
2836	const gpa = bfa.allocator();
2837
2838	var mask_big_int: std.math.big.int.Mutable = .{
2839	.limbs = try gpa.alloc(
2840	std.math.big.Limb,
2841	std.math.big.int.calcTwosCompLimbCount(load_store_bits),
2842	),
2843	.len = undefined,
2844	.positive = undefined,
2845	};
2846	defer gpa.free(mask_big_int.limbs);
2847	mask_big_int.setTwosCompIntLimit(.max, .unsigned, operand_bits);
2848	mask_big_int.shiftLeft(mask_big_int.toConst(), ptr_info.packed_offset.bit_offset);
2849	mask_big_int.bitNotWrap(mask_big_int.toConst(), .unsigned, load_store_bits);
2850	break :keep_mask try pt.intValue_big(load_store_ty, mask_big_int.toConst());
2851	}).toIntern()),
2852	} },
2853	}).toRef(),
2854	.rhs = res_block.add(l, .{
2855	.tag = .shl_exact,
2856	.data = .{ .bin_op = .{
2857	.lhs = res_block.add(l, .{
2858	.tag = .int_cast,
2859	.data = .{ .ty_op = .{
2860	.ty = load_store_ty,
2861	.operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
2862	} },
2863	}).toRef(),
2864	.rhs = try pt.intRef(
2865	try pt.intType(.unsigned, std.math.log2_int_ceil(u16, load_store_bits)),
2866	ptr_info.packed_offset.bit_offset,
2867	),
2868	} },
2869	}).toRef(),
2870	} },
2871	}).toRef(),
2872	} },
2873	});
2874	_ = res_block.add(l, .{
2875	.tag = .br,
2876	.data = .{ .br = .{
2877	.block_inst = orig_inst,
2878	.operand = .void_value,
2879	} },
2880	});
2881	}
2882	return .{ .ty_pl = .{
2883	.ty = .void,
2884	.payload = try l.addBlockBody(res_block.body()),
2885	} };
2886	}
2887	fn packedStructFieldValBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2888	const pt = l.pt;
2889	const zcu = pt.zcu;
2890
2891	const orig_ty_pl = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_pl;
2892	const orig_extra = l.extraData(Air.StructField, orig_ty_pl.payload).data;
2893	const field_ty = orig_ty_pl.ty;
2894	const agg_ty = l.typeOf(orig_extra.struct_operand);
2895
2896	const agg_bits: u16 = @intCast(agg_ty.bitSize(zcu));
2897	const bit_offset = zcu.structPackedFieldBitOffset(zcu.typeToStruct(agg_ty).?, orig_extra.field_index);
2898
2899	const agg_int_ty = try pt.intType(.unsigned, agg_bits);
2900	const field_int_ty = try pt.intType(.unsigned, @intCast(field_ty.bitSize(zcu)));
2901
2902	const agg_shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, agg_bits));
2903	const bit_offset_ref: Air.Inst.Ref = .fromValue(try pt.intValue(agg_shift_ty, bit_offset));
2904
2905	var inst_buf: [5]Air.Inst.Index = undefined;
2906	var main_block: Block = .init(&inst_buf);
2907	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
2908
2909	const agg_int = main_block.addBitCast(l, agg_int_ty, orig_extra.struct_operand);
2910	const shifted_agg_int = main_block.addBinOp(l, .shr, agg_int, bit_offset_ref).toRef();
2911	const field_int = main_block.addTyOp(l, .trunc, field_int_ty, shifted_agg_int).toRef();
2912	const field_val = main_block.addBitCast(l, field_ty, field_int);
2913	main_block.addBr(l, orig_inst, field_val);
2914
2915	return .{ .ty_pl = .{
2916	.ty = field_ty,
2917	.payload = try l.addBlockBody(main_block.body()),
2918	} };
2919	}
2920	fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2921	const pt = l.pt;
2922	const zcu = pt.zcu;
2923	const gpa = zcu.gpa;
2924
2925	const orig_ty_pl = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_pl;
2926	const agg_ty = orig_ty_pl.ty;
2927	const agg_field_count = agg_ty.structFieldCount(zcu);
2928	var opv_field_count: u32 = 0;
2929	for (0..agg_field_count) |field_idx| {
2930	const field_ty = agg_ty.fieldType(field_idx, zcu);
2931	const field_bits: u16 = @intCast(field_ty.bitSize(zcu));
2932	if (field_bits == 0) opv_field_count += 1;
2933	}
2934
2935	var bfa_buf: [4 * 32 + 2]Air.Inst.Index = undefined;
2936	var bfa_state: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), gpa);
2937	const bfa = bfa_state.allocator();
2938
2939	const inst_buf = try bfa.alloc(Air.Inst.Index, 4 * (agg_field_count - opv_field_count) + 2);
2940	defer bfa.free(inst_buf);
2941
2942	var main_block: Block = .init(inst_buf);
2943	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2944
2945	const num_bits: u16 = @intCast(agg_ty.bitSize(zcu));
2946	const shift_ty = try pt.intType(.unsigned, std.math.log2_int_ceil(u16, num_bits));
2947	const uint_ty = try pt.intType(.unsigned, num_bits);
2948	var cur_uint: Air.Inst.Ref = .fromValue(try pt.intValue(uint_ty, 0));
2949
2950	var field_idx = agg_field_count;
2951	while (field_idx > 0) {
2952	field_idx -= 1;
2953	const field_ty = agg_ty.fieldType(field_idx, zcu);
2954	const field_bits: u16 = @intCast(field_ty.bitSize(zcu));
2955	if (field_bits == 0) continue;
2956	assert(field_bits < num_bits);
2957	const field_uint_ty = try pt.intType(.unsigned, field_bits);
2958	const field_bit_size_ref: Air.Inst.Ref = .fromValue(try pt.intValue(shift_ty, field_bits));
2959	const field_val: Air.Inst.Ref = @fromBackingInt(@intCast(l.air_extra.items[orig_ty_pl.payload + field_idx]));
2960
2961	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2962	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2963	const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
2964	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2965	}
2966
2967	const result = main_block.addBitCast(l, agg_ty, cur_uint);
2968	main_block.addBr(l, orig_inst, result);
2969
2970	return .{ .ty_pl = .{
2971	.ty = agg_ty,
2972	.payload = try l.addBlockBody(main_block.body()),
2973	} };
2974	}
2975
2976	fn arrayToVectorBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
2977	const pt = l.pt;
2978	const zcu = pt.zcu;
2979	const gpa = zcu.gpa;
2980
2981	const orig_ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
2982	const vec_ty = orig_ty_op.ty;
2983	const len: usize = @intCast(vec_ty.vectorLen(zcu));
2984
2985	var bfa_buf: [64 + 2]Air.Inst.Index = undefined;
2986	var bfa_state: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), gpa);
2987	const bfa = bfa_state.allocator();
2988
2989	const inst_buf = try bfa.alloc(Air.Inst.Index, len + 2);
2990	defer bfa.free(inst_buf);
2991
2992	var main_block: Block = .init(inst_buf);
2993	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2994	try l.air_extra.ensureUnusedCapacity(gpa, len);
2995
2996	const elems_start: u32 = @intCast(l.air_extra.items.len);
2997	for (0..len) |elem_index| {
2998	const index_ref: Air.Inst.Ref = .fromValue(try pt.intValue(.usize, elem_index));
2999	const elem = main_block.addBinOp(l, .array_elem_val, orig_ty_op.operand, index_ref).toRef();
3000	l.air_extra.appendAssumeCapacity(@backingInt(elem));
3001	}
3002
3003	const result = main_block.add(l, .{
3004	.tag = .aggregate_init,
3005	.data = .{ .ty_pl = .{
3006	.ty = vec_ty,
3007	.payload = elems_start,
3008	} },
3009	}).toRef();
3010	main_block.addBr(l, orig_inst, result);
3011
3012	return .{ .ty_pl = .{
3013	.ty = vec_ty,
3014	.payload = try l.addBlockBody(main_block.body()),
3015	} };
3016	}
3017
3018	/// Given a `std.math.big.int.Const`, converts it to a `Value` which is a float of type `float_ty`
3019	/// representing the same numeric value. If the integer cannot be exactly represented, `round`
3020	/// decides whether the value should be rounded up or down. If `is_vector`, then `float_ty` is
3021	/// instead a vector of floats, and the result value is a vector containing the converted scalar
3022	/// repeated N times.
3023	fn floatFromBigIntVal(
3024	pt: Zcu.PerThread,
3025	is_vector: bool,
3026	float_ty: Type,
3027	x: std.math.big.int.Const,
3028	round: std.math.big.int.Round,
3029	) Error!Value {
3030	const zcu = pt.zcu;
3031	const scalar_ty = switch (is_vector) {
3032	true => float_ty.childType(zcu),
3033	false => float_ty,
3034	};
3035	assert(scalar_ty.zigTypeTag(zcu) == .float);
3036	const scalar_val: Value = switch (scalar_ty.floatBits(zcu.getTarget())) {
3037	16 => try pt.floatValue(scalar_ty, x.toFloat(f16, round)[0]),
3038	32 => try pt.floatValue(scalar_ty, x.toFloat(f32, round)[0]),
3039	64 => try pt.floatValue(scalar_ty, x.toFloat(f64, round)[0]),
3040	80 => try pt.floatValue(scalar_ty, x.toFloat(f80, round)[0]),
3041	128 => try pt.floatValue(scalar_ty, x.toFloat(f128, round)[0]),
3042	else => unreachable,
3043	};
3044	if (is_vector) {
3045	return pt.aggregateSplatValue(float_ty, scalar_val);
3046	} else {
3047	return scalar_val;
3048	}
3049	}
3050
3051	const Block = struct {
3052	instructions: []Air.Inst.Index,
3053	len: usize,
3054
3055	/// There are two common usages of the API:
3056	/// * `buf.len` is exactly the number of instructions which will be in this block
3057	/// * `buf.len` is no smaller than necessary, and `b.stealRemainingCapacity` will be used
3058	fn init(buf: []Air.Inst.Index) Block {
3059	return .{
3060	.instructions = buf,
3061	.len = 0,
3062	};
3063	}
3064
3065	/// Like `Legalize.addInstAssumeCapacity`, but also appends the instruction to `b`.
3066	fn add(b: *Block, l: *Legalize, inst_data: Air.Inst) Air.Inst.Index {
3067	const inst = l.addInstAssumeCapacity(inst_data);
3068	b.instructions[b.len] = inst;
3069	b.len += 1;
3070	return inst;
3071	}
3072	fn addBr(b: *Block, l: *Legalize, target: Air.Inst.Index, operand: Air.Inst.Ref) void {
3073	_ = b.add(l, .{
3074	.tag = .br,
3075	.data = .{ .br = .{ .block_inst = target, .operand = operand } },
3076	});
3077	}
3078	fn addTy(b: *Block, l: *Legalize, tag: Air.Inst.Tag, ty: Type) Air.Inst.Index {
3079	return b.add(l, .{ .tag = tag, .data = .{ .ty = ty } });
3080	}
3081	fn addBinOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) Air.Inst.Index {
3082	return b.add(l, .{
3083	.tag = tag,
3084	.data = .{ .bin_op = .{ .lhs = lhs, .rhs = rhs } },
3085	});
3086	}
3087	fn addUnOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, operand: Air.Inst.Ref) Air.Inst.Index {
3088	return b.add(l, .{
3089	.tag = tag,
3090	.data = .{ .un_op = operand },
3091	});
3092	}
3093	fn addTyOp(b: *Block, l: *Legalize, tag: Air.Inst.Tag, ty: Type, operand: Air.Inst.Ref) Air.Inst.Index {
3094	return b.add(l, .{
3095	.tag = tag,
3096	.data = .{ .ty_op = .{
3097	.ty = ty,
3098	.operand = operand,
3099	} },
3100	});
3101	}
3102
3103	fn addCompilerRtCall(b: *Block, l: *Legalize, func: Air.CompilerRtFunc, args: []const Air.Inst.Ref) Error!Air.Inst.Index {
3104	return b.add(l, .{
3105	.tag = .legalize_compiler_rt_call,
3106	.data = .{ .legalize_compiler_rt_call = .{
3107	.func = func,
3108	.payload = payload: {
3109	const extra_len = @typeInfo(Air.Call).@"struct".field_names.len + args.len;
3110	try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, extra_len);
3111	const index = l.addExtra(Air.Call, .{ .args_len = @intCast(args.len) }) catch unreachable;
3112	l.air_extra.appendSliceAssumeCapacity(@ptrCast(args));
3113	break :payload index;
3114	},
3115	} },
3116	});
3117	}
3118
3119	/// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
3120	/// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
3121	fn addPanic(b: *Block, l: *Legalize, panic_id: Zcu.SimplePanicId) Error!void {
3122	const zcu = l.pt.zcu;
3123	if (!zcu.backendSupportsFeature(.panic_fn)) {
3124	_ = b.add(l, .{
3125	.tag = .trap,
3126	.data = .{ .no_op = {} },
3127	});
3128	return;
3129	}
3130	const panic_fn_val = zcu.std_lang_decl_values.get(panic_id.toStdLangDecl());
3131	_ = b.add(l, .{
3132	.tag = .call,
3133	.data = .{ .pl_op = .{
3134	.operand = Air.internedToRef(panic_fn_val),
3135	.payload = try l.addExtra(Air.Call, .{ .args_len = 0 }),
3136	} },
3137	});
3138	_ = b.add(l, .{
3139	.tag = .unreach,
3140	.data = .{ .no_op = {} },
3141	});
3142	}
3143
3144	/// Adds a `cmp_*` instruction (including maybe `cmp_vector`) to `b`. This is a fairly thin wrapper
3145	/// around `add`, although it does compute the result type if `is_vector` (`@Vector(n, bool)`).
3146	fn addCmp(
3147	b: *Block,
3148	l: *Legalize,
3149	op: std.math.CompareOperator,
3150	lhs: Air.Inst.Ref,
3151	rhs: Air.Inst.Ref,
3152	opts: struct { optimized: bool = false, vector: bool = false },
3153	) Error!Air.Inst.Index {
3154	const pt = l.pt;
3155	if (opts.vector) {
3156	const bool_vec_ty = try pt.vectorType(.{
3157	.child = .bool_type,
3158	.len = l.typeOf(lhs).vectorLen(pt.zcu),
3159	});
3160	return b.add(l, .{
3161	.tag = if (opts.optimized) .cmp_vector_optimized else .cmp_vector,
3162	.data = .{ .ty_pl = .{
3163	.ty = bool_vec_ty,
3164	.payload = try l.addExtra(Air.VectorCmp, .{
3165	.lhs = lhs,
3166	.rhs = rhs,
3167	.op = Air.VectorCmp.encodeOp(op),
3168	}),
3169	} },
3170	});
3171	}
3172	return addCmpScalar(b, l, op, lhs, rhs, opts.optimized);
3173	}
3174
3175	/// Similar to `addCmp`, but for scalars only. Unlike `addCmp`, this function is
3176	/// infallible, because it doesn't need to add entries to `extra`.
3177	fn addCmpScalar(
3178	b: *Block,
3179	l: *Legalize,
3180	op: std.math.CompareOperator,
3181	lhs: Air.Inst.Ref,
3182	rhs: Air.Inst.Ref,
3183	optimized: bool,
3184	) Air.Inst.Index {
3185	return b.add(l, .{
3186	.tag = .fromCmpOp(op, optimized),
3187	.data = .{ .bin_op = .{
3188	.lhs = lhs,
3189	.rhs = rhs,
3190	} },
3191	});
3192	}
3193
3194	/// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
3195	/// no-op casts.
3196	fn addBitCast(
3197	b: *Block,
3198	l: *Legalize,
3199	result_ty: Type,
3200	operand: Air.Inst.Ref,
3201	) Air.Inst.Ref {
3202	const zcu = l.pt.zcu;
3203	const operand_ty = l.typeOf(operand);
3204	assert(!operand_ty.isPtrAtRuntime(zcu));
3205	assert(!operand_ty.isSliceAtRuntime(zcu));
3206	assert(!result_ty.isPtrAtRuntime(zcu));
3207	assert(!result_ty.isSliceAtRuntime(zcu));
3208	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
3209	.tag = .bit_cast,
3210	.data = .{ .ty_op = .{
3211	.ty = result_ty,
3212	.operand = operand,
3213	} },
3214	}).toRef();
3215	_ = b.stealCapacity(1);
3216	return operand;
3217	}
3218
3219	/// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
3220	/// no-op casts.
3221	fn addPtrCast(
3222	b: *Block,
3223	l: *Legalize,
3224	result_ty: Type,
3225	operand: Air.Inst.Ref,
3226	) Air.Inst.Ref {
3227	const zcu = l.pt.zcu;
3228	const operand_ty = l.typeOf(operand);
3229	if (operand_ty.isSliceAtRuntime(zcu)) {
3230	assert(result_ty.isSliceAtRuntime(zcu));
3231	} else {
3232	assert(operand_ty.isPtrAtRuntime(zcu));
3233	assert(result_ty.isPtrAtRuntime(zcu));
3234	}
3235	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
3236	.tag = .ptr_cast,
3237	.data = .{ .ty_op = .{
3238	.ty = result_ty,
3239	.operand = operand,
3240	} },
3241	}).toRef();
3242	_ = b.stealCapacity(1);
3243	return operand;
3244	}
3245
3246	/// This function emits *two* instructions.
3247	fn addSoftFloatCmp(
3248	b: *Block,
3249	l: *Legalize,
3250	float_ty: Type,
3251	op: std.math.CompareOperator,
3252	lhs: Air.Inst.Ref,
3253	rhs: Air.Inst.Ref,
3254	) Error!Air.Inst.Ref {
3255	const pt = l.pt;
3256	const target = pt.zcu.getTarget();
3257	const use_aeabi = target.cpu.arch.isArm() and switch (target.abi) {
3258	.eabi,
3259	.eabihf,
3260	.musleabi,
3261	.musleabihf,
3262	.gnueabi,
3263	.gnueabihf,
3264	.android,
3265	.androideabi,
3266	=> true,
3267	else => false,
3268	};
3269	const func: Air.CompilerRtFunc, const ret_cmp_op: std.math.CompareOperator = switch (float_ty.floatBits(target)) {
3270	// zig fmt: off
3271	16 => switch (op) {
3272	.eq => .{ .__eqhf2, .eq },
3273	.neq => .{ .__nehf2, .neq },
3274	.lt => .{ .__lthf2, .lt },
3275	.lte => .{ .__lehf2, .lte },
3276	.gt => .{ .__gthf2, .gt },
3277	.gte => .{ .__gehf2, .gte },
3278	},
3279	32 => switch (op) {
3280	.eq => if (use_aeabi) .{ .__aeabi_fcmpeq, .neq } else .{ .__eqsf2, .eq },
3281	.neq => if (use_aeabi) .{ .__aeabi_fcmpeq, .eq } else .{ .__nesf2, .neq },
3282	.lt => if (use_aeabi) .{ .__aeabi_fcmplt, .neq } else .{ .__ltsf2, .lt },
3283	.lte => if (use_aeabi) .{ .__aeabi_fcmple, .neq } else .{ .__lesf2, .lte },
3284	.gt => if (use_aeabi) .{ .__aeabi_fcmpgt, .neq } else .{ .__gtsf2, .gt },
3285	.gte => if (use_aeabi) .{ .__aeabi_fcmpge, .neq } else .{ .__gesf2, .gte },
3286	},
3287	64 => switch (op) {
3288	.eq => if (use_aeabi) .{ .__aeabi_dcmpeq, .neq } else .{ .__eqdf2, .eq },
3289	.neq => if (use_aeabi) .{ .__aeabi_dcmpeq, .eq } else .{ .__nedf2, .neq },
3290	.lt => if (use_aeabi) .{ .__aeabi_dcmplt, .neq } else .{ .__ltdf2, .lt },
3291	.lte => if (use_aeabi) .{ .__aeabi_dcmple, .neq } else .{ .__ledf2, .lte },
3292	.gt => if (use_aeabi) .{ .__aeabi_dcmpgt, .neq } else .{ .__gtdf2, .gt },
3293	.gte => if (use_aeabi) .{ .__aeabi_dcmpge, .neq } else .{ .__gedf2, .gte },
3294	},
3295	80 => switch (op) {
3296	.eq => .{ .__eqxf2, .eq },
3297	.neq => .{ .__nexf2, .neq },
3298	.lt => .{ .__ltxf2, .lt },
3299	.lte => .{ .__lexf2, .lte },
3300	.gt => .{ .__gtxf2, .gt },
3301	.gte => .{ .__gexf2, .gte },
3302	},
3303	128 => switch (op) {
3304	.eq => .{ .__eqtf2, .eq },
3305	.neq => .{ .__netf2, .neq },
3306	.lt => .{ .__lttf2, .lt },
3307	.lte => .{ .__letf2, .lte },
3308	.gt => .{ .__gttf2, .gt },
3309	.gte => .{ .__getf2, .gte },
3310	},
3311	else => unreachable,
3312	// zig fmt: on
3313	};
3314	const call_inst = try b.addCompilerRtCall(l, func, &.{ lhs, rhs });
3315	const raw_result = call_inst.toRef();
3316	assert(l.typeOf(raw_result).toIntern() == .i32_type);
3317	const zero_i32: Air.Inst.Ref = .fromValue(try pt.intValue(.i32, 0));
3318	const ret_cmp_tag: Air.Inst.Tag = .fromCmpOp(ret_cmp_op, false);
3319	return b.addBinOp(l, ret_cmp_tag, raw_result, zero_i32).toRef();
3320	}
3321
3322	/// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`.
3323	/// This is useful when you've provided a buffer big enough for all your instructions, but you are
3324	/// now starting a new block and some of them need to live there instead.
3325	fn stealRemainingCapacity(b: *Block) []Air.Inst.Index {
3326	return b.stealFrom(b.len);
3327	}
3328
3329	/// Returns `len` elements taken from the unused capacity of `b.instructions`, and shrinks
3330	/// `b.instructions` down to not include them anymore.
3331	/// This is useful when you've provided a buffer big enough for all your instructions, but you are
3332	/// now starting a new block and some of them need to live there instead.
3333	fn stealCapacity(b: *Block, len: usize) []Air.Inst.Index {
3334	return b.stealFrom(b.instructions.len - len);
3335	}
3336
3337	fn stealFrom(b: *Block, start: usize) []Air.Inst.Index {
3338	assert(start >= b.len);
3339	defer b.instructions.len = start;
3340	return b.instructions[start..];
3341	}
3342
3343	fn body(b: *const Block) []const Air.Inst.Index {
3344	assert(b.len == b.instructions.len);
3345	return b.instructions;
3346	}
3347	};
3348
3349	const Loop = struct {
3350	inst: Air.Inst.Index,
3351	block: Block,
3352
3353	/// The return value has `block` initialized to `undefined`; it is the caller's reponsibility
3354	/// to initialize it.
3355	fn init(l: *Legalize, parent_block: *Block) Loop {
3356	return .{
3357	.inst = parent_block.add(l, .{
3358	.tag = .loop,
3359	.data = .{ .ty_pl = .{
3360	.ty = .noreturn,
3361	.payload = undefined,
3362	} },
3363	}),
3364	.block = undefined,
3365	};
3366	}
3367
3368	fn finish(loop: Loop, l: *Legalize) Error!void {
3369	const data = &l.air_instructions.items(.data)[@backingInt(loop.inst)];
3370	data.ty_pl.payload = try l.addBlockBody(loop.block.body());
3371	}
3372	};
3373
3374	const CondBr = struct {
3375	inst: Air.Inst.Index,
3376	hints: Air.CondBr.BranchHints,
3377	then_block: Block,
3378	else_block: Block,
3379
3380	/// The return value has `then_block` and `else_block` initialized to `undefined`; it is the
3381	/// caller's reponsibility to initialize them.
3382	fn init(l: *Legalize, operand: Air.Inst.Ref, parent_block: *Block, hints: Air.CondBr.BranchHints) CondBr {
3383	return .{
3384	.inst = parent_block.add(l, .{
3385	.tag = .cond_br,
3386	.data = .{ .pl_op = .{
3387	.operand = operand,
3388	.payload = undefined,
3389	} },
3390	}),
3391	.hints = hints,
3392	.then_block = undefined,
3393	.else_block = undefined,
3394	};
3395	}
3396
3397	fn finish(cond_br: CondBr, l: *Legalize) Error!void {
3398	const then_body = cond_br.then_block.body();
3399	const else_body = cond_br.else_block.body();
3400	try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 3 + then_body.len + else_body.len);
3401
3402	const data = &l.air_instructions.items(.data)[@backingInt(cond_br.inst)];
3403	data.pl_op.payload = @intCast(l.air_extra.items.len);
3404	l.air_extra.appendSliceAssumeCapacity(&.{
3405	@intCast(then_body.len),
3406	@intCast(else_body.len),
3407	@bitCast(cond_br.hints),
3408	});
3409	l.air_extra.appendSliceAssumeCapacity(@ptrCast(then_body));
3410	l.air_extra.appendSliceAssumeCapacity(@ptrCast(else_body));
3411	}
3412	};
3413
3414	fn addInstAssumeCapacity(l: *Legalize, inst: Air.Inst) Air.Inst.Index {
3415	defer l.air_instructions.appendAssumeCapacity(inst);
3416	return @fromBackingInt(@intCast(l.air_instructions.len));
3417	}
3418
3419	fn addExtra(l: *Legalize, comptime Extra: type, extra: Extra) Error!u32 {
3420	const extra_info = @typeInfo(Extra).@"struct";
3421	try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, extra_info.field_names.len);
3422	defer inline for (extra_info.field_names, extra_info.field_types) |field_name, field_type| l.air_extra.appendAssumeCapacity(switch (field_type) {
3423	u32 => @field(extra, field_name),
3424	Air.Inst.Ref => @backingInt(@field(extra, field_name)),
3425	else => @compileError(@typeName(field_type)),
3426	});
3427	return @intCast(l.air_extra.items.len);
3428	}
3429
3430	fn addBlockBody(l: *Legalize, body: []const Air.Inst.Index) Error!u32 {
3431	try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, 1 + body.len);
3432	defer {
3433	l.air_extra.appendAssumeCapacity(@intCast(body.len));
3434	l.air_extra.appendSliceAssumeCapacity(@ptrCast(body));
3435	}
3436	return @intCast(l.air_extra.items.len);
3437	}
3438
3439	/// Returns `tag` to remind the caller to `continue :inst` the result.
3440	/// `inline` to propagate the comptime-known `tag` result.
3441	inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst.Tag, data: Air.Inst.Data) Air.Inst.Tag {
3442	const orig_ty = if (std.debug.runtime_safety) l.typeOfIndex(inst) else {};
3443	l.air_instructions.set(@backingInt(inst), .{ .tag = tag, .data = data });
3444	if (std.debug.runtime_safety) assert(l.typeOfIndex(inst).toIntern() == orig_ty.toIntern());
3445	return tag;
3446	}
3447
3448	fn compilerRtCall(
3449	l: *Legalize,
3450	orig_inst: Air.Inst.Index,
3451	func: Air.CompilerRtFunc,
3452	args: []const Air.Inst.Ref,
3453	result_ty: Type,
3454	) Error!Air.Inst.Tag {
3455	const zcu = l.pt.zcu;
3456	const gpa = zcu.gpa;
3457
3458	const func_ret_ty = func.returnType();
3459
3460	if (func_ret_ty.toIntern() == result_ty.toIntern()) {
3461	try l.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.Call).@"struct".field_names.len + args.len);
3462	const payload = l.addExtra(Air.Call, .{ .args_len = @intCast(args.len) }) catch unreachable;
3463	l.air_extra.appendSliceAssumeCapacity(@ptrCast(args));
3464	return l.replaceInst(orig_inst, .legalize_compiler_rt_call, .{ .legalize_compiler_rt_call = .{
3465	.func = func,
3466	.payload = payload,
3467	} });
3468	}
3469
3470	// We need to bitcast the result to an "alias" type (e.g. c_int/i32, c_longdouble/f128).
3471
3472	assert(func_ret_ty.bitSize(zcu) == result_ty.bitSize(zcu));
3473
3474	var inst_buf: [3]Air.Inst.Index = undefined;
3475	var main_block: Block = .init(&inst_buf);
3476	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3477
3478	const call_inst = try main_block.addCompilerRtCall(l, func, args);
3479	const casted_result = main_block.addBitCast(l, result_ty, call_inst.toRef());
3480	main_block.addBr(l, orig_inst, casted_result);
3481
3482	return l.replaceInst(orig_inst, .block, .{ .ty_pl = .{
3483	.ty = result_ty,
3484	.payload = try l.addBlockBody(main_block.body()),
3485	} });
3486	}
3487
3488	fn softFptruncFunc(l: *const Legalize, src_ty: Type, dst_ty: Type) Air.CompilerRtFunc {
3489	const target = l.pt.zcu.getTarget();
3490	const src_bits = src_ty.floatBits(target);
3491	const dst_bits = dst_ty.floatBits(target);
3492	assert(dst_bits < src_bits);
3493	const to_f16_func: Air.CompilerRtFunc = switch (src_bits) {
3494	128 => .__trunctfhf2,
3495	80 => .__truncxfhf2,
3496	64 => .__truncdfhf2,
3497	32 => .__truncsfhf2,
3498	else => unreachable,
3499	};
3500	const offset: u8 = switch (dst_bits) {
3501	16 => 0,
3502	32 => 1,
3503	64 => 2,
3504	80 => 3,
3505	else => unreachable,
3506	};
3507	return @fromBackingInt(@intCast(@backingInt(to_f16_func) + offset));
3508	}
3509	fn softFpextFunc(l: *const Legalize, src_ty: Type, dst_ty: Type) Air.CompilerRtFunc {
3510	const target = l.pt.zcu.getTarget();
3511	const src_bits = src_ty.floatBits(target);
3512	const dst_bits = dst_ty.floatBits(target);
3513	assert(dst_bits > src_bits);
3514	const to_f128_func: Air.CompilerRtFunc = switch (src_bits) {
3515	16 => .__extendhftf2,
3516	32 => .__extendsftf2,
3517	64 => .__extenddftf2,
3518	80 => .__extendxftf2,
3519	else => unreachable,
3520	};
3521	const offset: u8 = switch (dst_bits) {
3522	128 => 0,
3523	80 => 1,
3524	64 => 2,
3525	32 => 3,
3526	else => unreachable,
3527	};
3528	return @fromBackingInt(@intCast(@backingInt(to_f128_func) + offset));
3529	}
3530	fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3531	call: Air.CompilerRtFunc,
3532	block_payload: Air.Inst.Data,
3533	} {
3534	const pt = l.pt;
3535	const zcu = pt.zcu;
3536	const target = zcu.getTarget();
3537
3538	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
3539	const dest_ty = ty_op.ty;
3540	const src_ty = l.typeOf(ty_op.operand);
3541
3542	const src_info = src_ty.intInfo(zcu);
3543	const float_off: u32 = switch (dest_ty.floatBits(target)) {
3544	16 => 0,
3545	32 => 1,
3546	64 => 2,
3547	80 => 3,
3548	128 => 4,
3549	else => unreachable,
3550	};
3551	const base: Air.CompilerRtFunc = switch (src_info.signedness) {
3552	.signed => .__floatsihf,
3553	.unsigned => .__floatunsihf,
3554	};
3555	fixed: {
3556	const extended_int_bits: u16, const int_bits_off: u32 = switch (src_info.bits) {
3557	0...32 => .{ 32, 0 },
3558	33...64 => .{ 64, 5 },
3559	65...128 => .{ 128, 10 },
3560	else => break :fixed,
3561	};
3562	// x86_64-windows uses an odd callconv for 128-bit integers, so we use the
3563	// arbitrary-precision routine in that case for simplicity.
3564	if (target.cpu.arch == .x86_64 and target.os.tag == .windows and extended_int_bits == 128) {
3565	break :fixed;
3566	}
3567
3568	const func: Air.CompilerRtFunc = @fromBackingInt(@intCast(@backingInt(base) + int_bits_off + float_off));
3569	if (extended_int_bits == src_info.bits) return .{ .call = func };
3570
3571	// We need to emit a block which first sign/zero-extends to the right type and *then* calls
3572	// the required routine.
3573	const extended_ty = try l.pt.intType(src_info.signedness, extended_int_bits);
3574
3575	var inst_buf: [4]Air.Inst.Index = undefined;
3576	var main_block: Block = .init(&inst_buf);
3577	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3578
3579	const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3580	const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3581	const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3582	main_block.addBr(l, orig_inst, casted_result);
3583
3584	return .{ .block_payload = .{ .ty_pl = .{
3585	.ty = dest_ty,
3586	.payload = try l.addBlockBody(main_block.body()),
3587	} } };
3588	}
3589
3590	// We need to emit a block which puts the integer into an `alloc` (possibly sign/zero-extended)
3591	// and calls an arbitrary-width conversion routine.
3592
3593	const func: Air.CompilerRtFunc = @fromBackingInt(@intCast(@backingInt(base) + 15 + float_off));
3594
3595	// The extended integer routines expect the integer representation where the integer is
3596	// effectively zero- or sign-extended to its ABI size. We represent that by intcasting to
3597	// such an integer type and passing a pointer to *that*.
3598	const extended_ty = try pt.intType(src_info.signedness, @intCast(src_ty.abiSize(zcu) * 8));
3599	assert(extended_ty.abiSize(zcu) == src_ty.abiSize(zcu));
3600
3601	var inst_buf: [6]Air.Inst.Index = undefined;
3602	var main_block: Block = .init(&inst_buf);
3603	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3604
3605	const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3606	break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3607	} else ext: {
3608	_ = main_block.stealCapacity(1);
3609	break :ext ty_op.operand;
3610	};
3611	const extended_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(extended_ty)).toRef();
3612	_ = main_block.addBinOp(l, .store, extended_ptr, extended_val);
3613	const bits_val = try pt.intValue(.usize, src_info.bits);
3614	const call_inst = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val) });
3615	const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3616	main_block.addBr(l, orig_inst, casted_result);
3617
3618	return .{ .block_payload = .{ .ty_pl = .{
3619	.ty = dest_ty,
3620	.payload = try l.addBlockBody(main_block.body()),
3621	} } };
3622	}
3623	fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3624	call: Air.CompilerRtFunc,
3625	block_payload: Air.Inst.Data,
3626	} {
3627	const pt = l.pt;
3628	const zcu = pt.zcu;
3629	const target = zcu.getTarget();
3630
3631	const ty_op = l.air_instructions.items(.data)[@backingInt(orig_inst)].ty_op;
3632	const src_ty = l.typeOf(ty_op.operand);
3633	const dest_ty = ty_op.ty;
3634
3635	const dest_info = dest_ty.intInfo(zcu);
3636	const float_off: u32 = switch (src_ty.floatBits(target)) {
3637	16 => 0,
3638	32 => 1,
3639	64 => 2,
3640	80 => 3,
3641	128 => 4,
3642	else => unreachable,
3643	};
3644	const base: Air.CompilerRtFunc = switch (dest_info.signedness) {
3645	.signed => .__fixhfsi,
3646	.unsigned => .__fixunshfsi,
3647	};
3648	fixed: {
3649	const extended_int_bits: u16, const int_bits_off: u32 = switch (dest_info.bits) {
3650	0...32 => .{ 32, 0 },
3651	33...64 => .{ 64, 5 },
3652	65...128 => .{ 128, 10 },
3653	else => break :fixed,
3654	};
3655	// x86_64-windows uses an odd callconv for 128-bit integers, so we use the
3656	// arbitrary-precision routine in that case for simplicity.
3657	if (target.cpu.arch == .x86_64 and target.os.tag == .windows and extended_int_bits == 128) {
3658	break :fixed;
3659	}
3660
3661	const func: Air.CompilerRtFunc = @fromBackingInt(@intCast(@backingInt(base) + int_bits_off + float_off));
3662	if (extended_int_bits == dest_info.bits) return .{ .call = func };
3663
3664	// We need to emit a block which calls the routine and then casts to the required type.
3665
3666	var inst_buf: [3]Air.Inst.Index = undefined;
3667	var main_block: Block = .init(&inst_buf);
3668	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3669
3670	const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3671	const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
3672	main_block.addBr(l, orig_inst, casted_val);
3673
3674	return .{ .block_payload = .{ .ty_pl = .{
3675	.ty = dest_ty,
3676	.payload = try l.addBlockBody(main_block.body()),
3677	} } };
3678	}
3679
3680	// We need to emit a block which calls an arbitrary-width conversion routine, then loads the
3681	// integer from an `alloc` and possibly truncates it.
3682	const func: Air.CompilerRtFunc = @fromBackingInt(@intCast(@backingInt(base) + 15 + float_off));
3683
3684	const extended_ty = try pt.intType(dest_info.signedness, @intCast(dest_ty.abiSize(zcu) * 8));
3685	assert(extended_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
3686
3687	var inst_buf: [5]Air.Inst.Index = undefined;
3688	var main_block: Block = .init(&inst_buf);
3689	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3690
3691	const extended_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(extended_ty)).toRef();
3692	const bits_val = try pt.intValue(.usize, dest_info.bits);
3693	_ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3694	const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3695	const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
3696	main_block.addBr(l, orig_inst, result_val);
3697
3698	return .{ .block_payload = .{ .ty_pl = .{
3699	.ty = dest_ty,
3700	.payload = try l.addBlockBody(main_block.body()),
3701	} } };
3702	}
3703	fn softFloatFunc(op: Air.Inst.Tag, float_ty: Type, zcu: *const Zcu) Air.CompilerRtFunc {
3704	const f16_func: Air.CompilerRtFunc = switch (op) {
3705	.add, .add_optimized => .__addhf3,
3706	.sub, .sub_optimized => .__subhf3,
3707	.mul, .mul_optimized => .__mulhf3,
3708
3709	.div_float,
3710	.div_float_optimized,
3711	.div_exact,
3712	.div_exact_optimized,
3713	=> .__divhf3,
3714
3715	.min => .__fminh,
3716	.max => .__fmaxh,
3717
3718	.ceil => .__ceilh,
3719	.floor => .__floorh,
3720	.trunc_float => .__trunch,
3721	.round => .__roundh,
3722
3723	.log => .__logh,
3724	.log2 => .__log2h,
3725	.log10 => .__log10h,
3726
3727	.exp => .__exph,
3728	.exp2 => .__exp2h,
3729
3730	.sin => .__sinh,
3731	.cos => .__cosh,
3732	.tan => .__tanh,
3733
3734	.abs => .__fabsh,
3735	.sqrt => .__sqrth,
3736	.rem, .rem_optimized => .__fmodh,
3737	.mul_add => .__fmah,
3738
3739	else => unreachable,
3740	};
3741	const offset: u8 = switch (float_ty.floatBits(zcu.getTarget())) {
3742	16 => 0,
3743	32 => 1,
3744	64 => 2,
3745	80 => 3,
3746	128 => 4,
3747	else => unreachable,
3748	};
3749	return @fromBackingInt(@intCast(@backingInt(f16_func) + offset));
3750	}
3751
3752	fn softFloatNegBlockPayload(
3753	l: *Legalize,
3754	orig_inst: Air.Inst.Index,
3755	operand: Air.Inst.Ref,
3756	) Error!Air.Inst.Data {
3757	const pt = l.pt;
3758	const zcu = pt.zcu;
3759	const gpa = zcu.gpa;
3760
3761	const float_ty = l.typeOfIndex(orig_inst);
3762
3763	const int_ty: Type, const sign_bit: Value = switch (float_ty.floatBits(zcu.getTarget())) {
3764	16 => .{ .u16, try pt.intValue(.u16, @as(u16, 1) << 15) },
3765	32 => .{ .u32, try pt.intValue(.u32, @as(u32, 1) << 31) },
3766	64 => .{ .u64, try pt.intValue(.u64, @as(u64, 1) << 63) },
3767	80 => .{ .u80, try pt.intValue(.u80, @as(u80, 1) << 79) },
3768	128 => .{ .u128, try pt.intValue(.u128, @as(u128, 1) << 127) },
3769	else => unreachable,
3770	};
3771
3772	const sign_bit_ref: Air.Inst.Ref = .fromValue(sign_bit);
3773
3774	var inst_buf: [4]Air.Inst.Index = undefined;
3775	var main_block: Block = .init(&inst_buf);
3776	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3777
3778	const operand_as_int = main_block.addBitCast(l, int_ty, operand);
3779	const result_as_int = main_block.addBinOp(l, .xor, operand_as_int, sign_bit_ref).toRef();
3780	const result = main_block.addBitCast(l, float_ty, result_as_int);
3781	main_block.addBr(l, orig_inst, result);
3782
3783	return .{ .ty_pl = .{
3784	.ty = float_ty,
3785	.payload = try l.addBlockBody(main_block.body()),
3786	} };
3787	}
3788
3789	fn softFloatDivTruncFloorCeilBlockPayload(
3790	l: *Legalize,
3791	orig_inst: Air.Inst.Index,
3792	lhs: Air.Inst.Ref,
3793	rhs: Air.Inst.Ref,
3794	air_tag: Air.Inst.Tag,
3795	) Error!Air.Inst.Data {
3796	const zcu = l.pt.zcu;
3797	const gpa = zcu.gpa;
3798
3799	const float_ty = l.typeOfIndex(orig_inst);
3800
3801	const floor_tag: Air.Inst.Tag = switch (air_tag) {
3802	.div_trunc, .div_trunc_optimized => .trunc_float,
3803	.div_floor, .div_floor_optimized => .floor,
3804	.div_ceil, .div_ceil_optimized => .ceil,
3805	else => unreachable,
3806	};
3807
3808	var inst_buf: [4]Air.Inst.Index = undefined;
3809	var main_block: Block = .init(&inst_buf);
3810	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3811
3812	const div_inst = try main_block.addCompilerRtCall(l, softFloatFunc(.div_float, float_ty, zcu), &.{ lhs, rhs });
3813	const floor_inst = try main_block.addCompilerRtCall(l, softFloatFunc(floor_tag, float_ty, zcu), &.{div_inst.toRef()});
3814	const casted_result = main_block.addBitCast(l, float_ty, floor_inst.toRef());
3815	main_block.addBr(l, orig_inst, casted_result);
3816
3817	return .{ .ty_pl = .{
3818	.ty = float_ty,
3819	.payload = try l.addBlockBody(main_block.body()),
3820	} };
3821	}
3822	fn softFloatModBlockPayload(
3823	l: *Legalize,
3824	orig_inst: Air.Inst.Index,
3825	lhs: Air.Inst.Ref,
3826	rhs: Air.Inst.Ref,
3827	) Error!Air.Inst.Data {
3828	const pt = l.pt;
3829	const zcu = pt.zcu;
3830	const gpa = zcu.gpa;
3831
3832	const float_ty = l.typeOfIndex(orig_inst);
3833
3834	var inst_buf: [10]Air.Inst.Index = undefined;
3835	var main_block: Block = .init(&inst_buf);
3836	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3837
3838	const rem = try main_block.addCompilerRtCall(l, softFloatFunc(.rem, float_ty, zcu), &.{ lhs, rhs });
3839	const lhs_lt_zero = try main_block.addSoftFloatCmp(l, float_ty, .lt, lhs, .fromValue(try pt.floatValue(float_ty, 0.0)));
3840
3841	var condbr: CondBr = .init(l, lhs_lt_zero, &main_block, .{});
3842	condbr.then_block = .init(main_block.stealRemainingCapacity());
3843	{
3844	const add = try condbr.then_block.addCompilerRtCall(l, softFloatFunc(.add, float_ty, zcu), &.{ rem.toRef(), rhs });
3845	const inner_rem = try condbr.then_block.addCompilerRtCall(l, softFloatFunc(.rem, float_ty, zcu), &.{ add.toRef(), rhs });
3846	const casted_result = condbr.then_block.addBitCast(l, float_ty, inner_rem.toRef());
3847	condbr.then_block.addBr(l, orig_inst, casted_result);
3848	}
3849	condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
3850	{
3851	const casted_result = condbr.else_block.addBitCast(l, float_ty, rem.toRef());
3852	condbr.else_block.addBr(l, orig_inst, casted_result);
3853	}
3854
3855	try condbr.finish(l);
3856
3857	return .{ .ty_pl = .{
3858	.ty = float_ty,
3859	.payload = try l.addBlockBody(main_block.body()),
3860	} };
3861	}
3862	fn softFloatCmpBlockPayload(
3863	l: *Legalize,
3864	orig_inst: Air.Inst.Index,
3865	float_ty: Type,
3866	op: std.math.CompareOperator,
3867	lhs: Air.Inst.Ref,
3868	rhs: Air.Inst.Ref,
3869	) Error!Air.Inst.Data {
3870	const pt = l.pt;
3871	const gpa = pt.zcu.gpa;
3872
3873	var inst_buf: [3]Air.Inst.Index = undefined;
3874	var main_block: Block = .init(&inst_buf);
3875	try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3876
3877	const result = try main_block.addSoftFloatCmp(l, float_ty, op, lhs, rhs);
3878	main_block.addBr(l, orig_inst, result);
3879
3880	return .{ .ty_pl = .{
3881	.ty = .bool,
3882	.payload = try l.addBlockBody(main_block.body()),
3883	} };
3884	}
3885
3886	/// `inline` to propagate potentially comptime-known return value.
3887	inline fn wantScalarizeOrSoftFloat(
3888	l: *const Legalize,
3889	comptime air_tag: Air.Inst.Tag,
3890	ty: Type,
3891	) enum {
3892	none,
3893	scalarize,
3894	soft_float,
3895	} {
3896	const zcu = l.pt.zcu;
3897	const is_vec, const scalar_ty = switch (ty.zigTypeTag(zcu)) {
3898	.vector => .{ true, ty.childType(zcu) },
3899	else => .{ false, ty },
3900	};
3901
3902	if (is_vec and l.features.has(.scalarize(air_tag))) return .scalarize;
3903
3904	if (l.wantSoftFloatScalar(scalar_ty)) {
3905	return if (is_vec) .scalarize else .soft_float;
3906	}
3907	return .none;
3908	}
3909
3910	/// `inline` to propagate potentially comptime-known return value.
3911	inline fn wantSoftFloatScalar(l: *const Legalize, ty: Type) bool {
3912	const zcu = l.pt.zcu;
3913	return switch (ty.zigTypeTag(zcu)) {
3914	.vector => unreachable,
3915	.float => switch (ty.floatBits(zcu.getTarget())) {
3916	16 => l.features.has(.soft_f16),
3917	32 => l.features.has(.soft_f32),
3918	64 => l.features.has(.soft_f64),
3919	80 => l.features.has(.soft_f80),
3920	128 => l.features.has(.soft_f128),
3921	else => unreachable,
3922	},
3923	else => false,
3924	};
3925	}
3926
3927	const Air = @import("../Air.zig");
3928	const assert = std.debug.assert;
3929	const dev = @import("../dev.zig");
3930	const InternPool = @import("../InternPool.zig");
3931	const Legalize = @This();
3932	const std = @import("std");
3933	const Type = @import("../Type.zig");
3934	const Value = @import("../Value.zig");
3935	const Zcu = @import("../Zcu.zig");