Meg's Mirrors

zig

1	const std = @import("std");
2	const builtin = @import("builtin");
3	const Allocator = std.mem.Allocator;
4	const assert = std.debug.assert;
5	const testing = std.testing;
6	const math = std.math;
7	const mem = std.mem;
8	const log = std.log.scoped(.codegen);
9
10	const CodeGen = @This();
11	const codegen = @import("../../codegen.zig");
12	const Zcu = @import("../../Zcu.zig");
13	const InternPool = @import("../../InternPool.zig");
14	const Decl = Zcu.Decl;
15	const Type = @import("../../Type.zig");
16	const Value = @import("../../Value.zig");
17	const Compilation = @import("../../Compilation.zig");
18	const link = @import("../../link.zig");
19	const Air = @import("../../Air.zig");
20	const Mir = @import("Mir.zig");
21	const assembly = @import("assembly.zig");
22	const abi = @import("../../codegen/wasm/abi.zig");
23	const Alignment = InternPool.Alignment;
24	const errUnionPayloadOffset = codegen.errUnionPayloadOffset;
25	const errUnionErrorOffset = codegen.errUnionErrorOffset;
26
27	pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
28	return comptime &.initMany(&.{
29	.expand_bit_cast_safe,
30	.expand_int_cast_safe,
31	.expand_int_from_float_safe,
32	.expand_int_from_float_optimized_safe,
33	.expand_add_safe,
34	.expand_sub_safe,
35	.expand_mul_safe,
36
37	.expand_packed_load,
38	.expand_packed_store,
39	.expand_packed_agg_field_val,
40	.expand_packed_aggregate_init,
41	.expand_array_splat,
42	.expand_array_to_vector,
43
44	.scalarize_add,
45	.scalarize_add_optimized,
46	.scalarize_add_wrap,
47	.scalarize_add_sat,
48	.scalarize_sub,
49	.scalarize_sub_optimized,
50	.scalarize_sub_wrap,
51	.scalarize_sub_sat,
52	.scalarize_mul,
53	.scalarize_mul_optimized,
54	.scalarize_mul_wrap,
55	.scalarize_mul_sat,
56	.scalarize_div_float,
57	.scalarize_div_float_optimized,
58	.scalarize_div_trunc,
59	.scalarize_div_trunc_optimized,
60	.scalarize_div_floor,
61	.scalarize_div_floor_optimized,
62	.scalarize_div_ceil,
63	.scalarize_div_ceil_optimized,
64	.scalarize_div_exact,
65	.scalarize_div_exact_optimized,
66	.scalarize_rem,
67	.scalarize_rem_optimized,
68	.scalarize_mod,
69	.scalarize_mod_optimized,
70	.scalarize_max,
71	.scalarize_min,
72	.scalarize_add_with_overflow,
73	.scalarize_sub_with_overflow,
74	.scalarize_mul_with_overflow,
75	.scalarize_shl_with_overflow,
76	.scalarize_bit_and,
77	.scalarize_bit_or,
78	.scalarize_shr,
79	.scalarize_shr_exact,
80	.scalarize_shl,
81	.scalarize_shl_exact,
82	.scalarize_shl_sat,
83	.scalarize_xor,
84	.scalarize_not,
85	.scalarize_clz,
86	.scalarize_ctz,
87	.scalarize_popcount,
88	.scalarize_byte_swap,
89	.scalarize_bit_reverse,
90	.scalarize_sqrt,
91	.scalarize_sin,
92	.scalarize_cos,
93	.scalarize_tan,
94	.scalarize_exp,
95	.scalarize_exp2,
96	.scalarize_log,
97	.scalarize_log2,
98	.scalarize_log10,
99	.scalarize_abs,
100	.scalarize_floor,
101	.scalarize_ceil,
102	.scalarize_round,
103	.scalarize_trunc_float,
104	.scalarize_neg,
105	.scalarize_neg_optimized,
106	.scalarize_cmp_vector,
107	.scalarize_cmp_vector_optimized,
108	.scalarize_fptrunc,
109	.scalarize_fpext,
110	.scalarize_int_cast,
111	.scalarize_ptr_cast,
112	.scalarize_ptr_from_int,
113	.scalarize_int_from_ptr,
114	.scalarize_trunc,
115	.scalarize_int_from_float,
116	.scalarize_int_from_float_optimized,
117	.scalarize_float_from_int,
118	.scalarize_reduce,
119	.scalarize_reduce_optimized,
120	.scalarize_shuffle_one,
121	.scalarize_shuffle_two,
122	.scalarize_select,
123	.scalarize_mul_add,
124
125	.scalarize_bit_cast_padded_elems,
126	});
127	}
128
129	/// Reference to the function declaration the code
130	/// section belongs to
131	owner_nav: InternPool.Nav.Index,
132	/// Current block depth. Used to calculate the relative difference between a break
133	/// and block
134	block_depth: u32 = 0,
135	air: Air,
136	liveness: Air.Liveness,
137	gpa: mem.Allocator,
138	func_index: InternPool.Index,
139	/// Contains a list of current branches.
140	/// When we return from a branch, the branch will be popped from this list,
141	/// which means branches can only contain references from within its own branch,
142	/// or a branch higher (lower index) in the tree.
143	branches: std.ArrayList(Branch) = .empty,
144	/// Table to save `WValue`'s generated by an `Air.Inst`
145	// values: ValueTable,
146	/// Mapping from Air.Inst.Index to block ids
147	blocks: std.array_hash_map.Auto(Air.Inst.Index, struct {
148	label: u32,
149	value: WValue,
150	}) = .{},
151	/// Maps `loop` instructions to their label. `br` to here repeats the loop.
152	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, u32) = .empty,
153	/// The index the next local generated will have
154	/// NOTE: arguments share the index with locals therefore the first variable
155	/// will have the index that comes after the last argument's index
156	local_index: u32,
157	/// The index of the current argument.
158	/// Used to track which argument is being referenced in `airArg`.
159	arg_index: u32 = 0,
160	/// List of simd128 immediates. Each value is stored as an array of bytes.
161	/// This list will only be populated for 128bit-simd values when the target features
162	/// are enabled also.
163	simd_immediates: std.ArrayList([16]u8) = .empty,
164	/// The Target we're emitting (used to call intInfo)
165	target: *const std.Target,
166	ptr_size: enum { wasm32, wasm64 },
167	pt: Zcu.PerThread,
168	/// List of MIR Instructions
169	mir_instructions: std.MultiArrayList(Mir.Inst),
170	/// Contains extra data for MIR
171	mir_extra: std.ArrayList(u32),
172	/// List of all locals' types generated throughout this declaration
173	/// used to emit locals count at start of 'code' section.
174	mir_locals: std.ArrayList(std.wasm.Valtype),
175	/// Set of all UAVs referenced by this function. Key is the UAV value, value is the alignment.
176	/// `.none` means naturally aligned. An explicit alignment is never less than the natural alignment.
177	mir_uavs: std.array_hash_map.Auto(InternPool.Index, Alignment),
178	/// Set of all functions whose address this function has taken and which therefore might be called
179	/// via a `call_indirect` function.
180	mir_indirect_function_set: std.array_hash_map.Auto(InternPool.Nav.Index, void),
181	/// Set of all function types used by this function. These must be interned by the linker.
182	mir_func_tys: std.array_hash_map.Auto(InternPool.Index, void),
183	/// The number of `error_name_table_ref` instructions emitted.
184	error_name_table_ref_count: u32,
185	/// When a function is executing, we store the the current stack pointer's value within this local.
186	/// This value is then used to restore the stack pointer to the original value at the return of the function.
187	initial_stack_value: WValue = .none,
188	/// The current stack pointer subtracted with the stack size. From this value, we will calculate
189	/// all offsets of the stack values.
190	bottom_stack_value: WValue = .none,
191	/// Arguments of this function declaration
192	/// This will be set after `resolveCallingConventionValues`
193	args: []WValue,
194	/// This will only be `.none` if the function returns void, or returns an immediate.
195	/// When it returns a pointer to the stack, the `.local` tag will be active and must be populated
196	/// before this function returns its execution to the caller.
197	return_value: WValue,
198	/// Only populated for variadic functions.
199	/// Holds the hidden final parameter pointing to the varargs buffer.
200	varargs: WValue,
201	/// The size of the stack this function occupies. In the function prologue
202	/// we will move the stack pointer by this number, forward aligned with the `stack_alignment`.
203	stack_size: u32 = 0,
204	/// The stack alignment, which is 16 bytes by default. This is specified by the
205	/// tool-conventions: https://github.com/WebAssembly/tool-conventions/blob/main/BasicCABI.md
206	/// and also what the llvm backend will emit.
207	/// However, local variables or the usage of `incoming_stack_alignment` in a `CallingConvention` can overwrite this default.
208	stack_alignment: Alignment = .@"16",
209
210	// For each individual Wasm valtype we store a seperate free list which
211	// allows us to re-use locals that are no longer used. e.g. a temporary local.
212	/// A list of indexes which represents a local of valtype `i32`.
213	/// It is illegal to store a non-i32 valtype in this list.
214	free_locals_i32: std.ArrayList(u32) = .empty,
215	/// A list of indexes which represents a local of valtype `i64`.
216	/// It is illegal to store a non-i64 valtype in this list.
217	free_locals_i64: std.ArrayList(u32) = .empty,
218	/// A list of indexes which represents a local of valtype `f32`.
219	/// It is illegal to store a non-f32 valtype in this list.
220	free_locals_f32: std.ArrayList(u32) = .empty,
221	/// A list of indexes which represents a local of valtype `f64`.
222	/// It is illegal to store a non-f64 valtype in this list.
223	free_locals_f64: std.ArrayList(u32) = .empty,
224	/// A list of indexes which represents a local of valtype `v127`.
225	/// It is illegal to store a non-v128 valtype in this list.
226	free_locals_v128: std.ArrayList(u32) = .empty,
227
228	/// When in debug mode, this tracks if no `finishAir` was missed.
229	/// Forgetting to call `finishAir` will cause the result to not be
230	/// stored in our `values` map and therefore cause bugs.
231	air_bookkeeping: @TypeOf(bookkeeping_init) = bookkeeping_init,
232
233	/// Wasm Value, created when generating an instruction
234	const WValue = union(enum) {
235	/// `WValue` which has been freed and may no longer hold
236	/// any references.
237	dead: void,
238	/// May be referenced but is unused
239	none: void,
240	/// The value lives on top of the stack
241	stack: void,
242	/// Index of the local
243	local: struct {
244	/// Contains the index to the local
245	value: u32,
246	/// The amount of instructions referencing this `WValue`
247	references: u32,
248	},
249	/// An immediate 32bit value
250	imm32: u32,
251	/// An immediate 64bit value
252	imm64: u64,
253	/// Index into the list of simd128 immediates. This `WValue` is
254	/// only possible in very rare cases, therefore it would be
255	/// a waste of memory to store the value in a 128 bit integer.
256	imm128: u32,
257	/// A constant 32bit float value
258	float32: f32,
259	/// A constant 64bit float value
260	float64: f64,
261	nav_ref: struct {
262	nav_index: InternPool.Nav.Index,
263	offset: i32 = 0,
264	},
265	uav_ref: struct {
266	ip_index: InternPool.Index,
267	offset: i32 = 0,
268	orig_ptr_ty: InternPool.Index = .none,
269	},
270	/// Offset from the bottom of the virtual stack, with the offset
271	/// pointing to where the value lives.
272	stack_offset: struct {
273	/// Contains the actual value of the offset
274	value: u32,
275	/// The amount of instructions referencing this `WValue`
276	references: u32,
277	},
278
279	/// Returns the offset from the bottom of the stack. This is useful when
280	/// we use the load or store instruction to ensure we retrieve the value
281	/// from the correct position, rather than the value that lives at the
282	/// bottom of the stack. For instances where `WValue` is not `stack_value`
283	/// this will return 0, which allows us to simply call this function for all
284	/// loads and stores without requiring checks everywhere.
285	fn offset(value: WValue) u32 {
286	switch (value) {
287	.stack_offset => |stack_offset| return stack_offset.value,
288	.dead => unreachable,
289	else => return 0,
290	}
291	}
292
293	/// Promotes a `WValue` to a local when given value is on top of the stack.
294	/// When encountering a `local` or `stack_offset` this is essentially a no-op.
295	/// All other tags are illegal.
296	fn toLocal(value: WValue, gen: *CodeGen, ty: Type) InnerError!WValue {
297	switch (value) {
298	.stack => {
299	const new_local = try gen.allocLocal(ty);
300	try gen.addLocal(.local_set, new_local.local.value);
301	return new_local;
302	},
303	else => return value,
304	}
305	}
306
307	/// Marks a local as no longer being referenced and essentially allows
308	/// us to re-use it somewhere else within the function.
309	/// The valtype of the local is deducted by using the index of the given `WValue`.
310	fn free(value: *WValue, gen: *CodeGen) void {
311	if (value.* != .local) return;
312	const local_value = value.local.value;
313	const reserved = gen.args.len + @intFromBool(gen.return_value != .none) + @intFromBool(gen.varargs != .none);
314	if (local_value < reserved + 2) return; // reserved locals may never be re-used. Also accounts for 2 stack locals.
315
316	const index = local_value - reserved;
317	const valtype = gen.mir_locals.items[index];
318	switch (valtype) {
319	.i32 => gen.free_locals_i32.append(gen.gpa, local_value) catch return, // It's ok to fail any of those, a new local can be allocated instead
320	.i64 => gen.free_locals_i64.append(gen.gpa, local_value) catch return,
321	.f32 => gen.free_locals_f32.append(gen.gpa, local_value) catch return,
322	.f64 => gen.free_locals_f64.append(gen.gpa, local_value) catch return,
323	.v128 => gen.free_locals_v128.append(gen.gpa, local_value) catch return,
324	}
325	log.debug("freed local ({d}) of type {}", .{ local_value, valtype });
326	value.* = .dead;
327	}
328	};
329
330	/// Hashmap to store generated `WValue` for each `Air.Inst.Ref`
331	const ValueTable = std.array_hash_map.Auto(Air.Inst.Ref, WValue);
332
333	const bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {};
334
335	const InnerError = error{
336	OutOfMemory,
337	/// An error occurred when trying to lower AIR to MIR.
338	AlreadyReported,
339	/// Compiler implementation could not handle a large integer.
340	Overflow,
341	};
342
343	pub fn deinit(cg: *CodeGen) void {
344	const gpa = cg.gpa;
345	for (cg.branches.items) |*branch| branch.deinit(gpa);
346	cg.branches.deinit(gpa);
347	cg.blocks.deinit(gpa);
348	cg.loops.deinit(gpa);
349	cg.simd_immediates.deinit(gpa);
350	cg.free_locals_i32.deinit(gpa);
351	cg.free_locals_i64.deinit(gpa);
352	cg.free_locals_f32.deinit(gpa);
353	cg.free_locals_f64.deinit(gpa);
354	cg.free_locals_v128.deinit(gpa);
355	cg.mir_instructions.deinit(gpa);
356	cg.mir_extra.deinit(gpa);
357	cg.mir_locals.deinit(gpa);
358	cg.mir_uavs.deinit(gpa);
359	cg.mir_indirect_function_set.deinit(gpa);
360	cg.mir_func_tys.deinit(gpa);
361	cg.* = undefined;
362	}
363
364	pub fn fail(cg: *CodeGen, comptime fmt: []const u8, args: anytype) error{ OutOfMemory, AlreadyReported } {
365	const zcu = cg.pt.zcu;
366	const func = zcu.funcInfo(cg.func_index);
367	return zcu.codegenFail(func.owner_nav, fmt, args);
368	}
369
370	/// Resolves the `WValue` for the given instruction `inst`
371	/// When the given instruction has a `Value`, it returns a constant instead
372	fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
373	var branch_index = cg.branches.items.len;
374	while (branch_index > 0) : (branch_index -= 1) {
375	const branch = cg.branches.items[branch_index - 1];
376	if (branch.values.get(ref)) |value| {
377	return value;
378	}
379	}
380
381	// when we did not find an existing instruction, it
382	// means we must generate it from a constant.
383	// We always store constants in the most outer branch as they must never
384	// be removed. The most outer branch is always at index 0.
385	const gop = try cg.branches.items[0].values.getOrPut(cg.gpa, ref);
386	assert(!gop.found_existing);
387
388	const pt = cg.pt;
389	const zcu = pt.zcu;
390	const val: Value = .fromInterned(ref.toInterned().?);
391	const ty = cg.typeOf(ref);
392	if (!ty.hasRuntimeBits(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) {
393	gop.value_ptr.* = .none;
394	return .none;
395	}
396
397	// When we need to pass the value by reference (such as a struct), we will
398	// leverage `generateSymbol` to lower the constant to bytes and emit it
399	// to the 'rodata' section. We then return the index into the section as `WValue`.
400	//
401	// In the other cases, we will simply lower the constant to a value that fits
402	// into a single local (such as a pointer, integer, bool, etc).
403	const result: WValue = if (isByRef(ty, zcu, cg.target))
404	.{ .uav_ref = .{ .ip_index = val.toIntern() } }
405	else
406	try cg.lowerConstant(val);
407
408	gop.value_ptr.* = result;
409	return result;
410	}
411
412	fn resolveValue(cg: *CodeGen, val: Value) InnerError!WValue {
413	const zcu = cg.pt.zcu;
414	const ty = val.typeOf(zcu);
415
416	return if (isByRef(ty, zcu, cg.target))
417	.{ .uav_ref = .{ .ip_index = val.toIntern() } }
418	else
419	try cg.lowerConstant(val);
420	}
421
422	/// NOTE: if result == .stack, it will be stored in .local
423	fn finishAir(cg: *CodeGen, inst: Air.Inst.Index, result: WValue, operands: []const Air.Inst.Ref) InnerError!void {
424	assert(operands.len <= Air.Liveness.bpi - 1);
425	var tomb_bits = cg.liveness.getTombBits(inst);
426	for (operands) |operand| {
427	const dies = @as(u1, @truncate(tomb_bits)) != 0;
428	tomb_bits >>= 1;
429	if (!dies) continue;
430	processDeath(cg, operand);
431	}
432	try cg.finishAirResult(inst, result);
433	}
434
435	fn finishAirResult(cg: *CodeGen, inst: Air.Inst.Index, result: WValue) InnerError!void {
436	// results of `none` can never be referenced.
437	if (result != .none) {
438	const trackable_result = if (result != .stack)
439	result
440	else
441	try result.toLocal(cg, cg.typeOfIndex(inst));
442	const branch = cg.currentBranch();
443	branch.values.putAssumeCapacityNoClobber(inst.toRef(), trackable_result);
444	}
445
446	if (std.debug.runtime_safety) {
447	cg.air_bookkeeping += 1;
448	}
449	}
450
451	const Branch = struct {
452	values: ValueTable = .{},
453
454	fn deinit(branch: *Branch, gpa: Allocator) void {
455	branch.values.deinit(gpa);
456	branch.* = undefined;
457	}
458	};
459
460	inline fn currentBranch(cg: *CodeGen) *Branch {
461	return &cg.branches.items[cg.branches.items.len - 1];
462	}
463
464	fn feed(cg: *CodeGen, bt: *Air.Liveness.BigTomb, operand: Air.Inst.Ref) void {
465	if (bt.feed()) {
466	cg.processDeath(operand);
467	}
468	}
469
470	fn processDeath(cg: *CodeGen, ref: Air.Inst.Ref) void {
471	if (ref.toIndex() == null) return;
472	// Branches are currently only allowed to free locals allocated
473	// within their own branch.
474	// TODO: Upon branch consolidation free any locals if needed.
475	const value = cg.currentBranch().values.getPtr(ref) orelse return;
476	if (value.* != .local) return;
477	const reserved_indexes = cg.args.len + @intFromBool(cg.return_value != .none);
478	if (value.local.value < reserved_indexes) {
479	return; // function arguments can never be re-used
480	}
481	log.debug("Decreasing reference for ref: %{d}, using local '{d}'", .{ @backingInt(ref.toIndex().?), value.local.value });
482	value.local.references -= 1; // if this panics, a call to `reuseOperand` was forgotten by the developer
483	if (value.local.references == 0) {
484	value.free(cg);
485	}
486	}
487
488	pub fn addInst(cg: *CodeGen, inst: Mir.Inst) error{OutOfMemory}!void {
489	try cg.mir_instructions.append(cg.gpa, inst);
490	}
491
492	pub fn addTag(cg: *CodeGen, tag: Mir.Inst.Tag) error{OutOfMemory}!void {
493	try cg.addInst(.{ .tag = tag, .data = .{ .tag = {} } });
494	}
495
496	pub fn addExtended(cg: *CodeGen, opcode: std.wasm.MiscOpcode) error{OutOfMemory}!void {
497	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
498	try cg.mir_extra.append(cg.gpa, @backingInt(opcode));
499	try cg.addInst(.{ .tag = .misc_prefix, .data = .{ .payload = extra_index } });
500	}
501
502	pub fn addLabel(cg: *CodeGen, tag: Mir.Inst.Tag, label: u32) error{OutOfMemory}!void {
503	try cg.addInst(.{ .tag = tag, .data = .{ .label = label } });
504	}
505
506	pub fn addLocal(cg: *CodeGen, tag: Mir.Inst.Tag, local: u32) error{OutOfMemory}!void {
507	try cg.addInst(.{ .tag = tag, .data = .{ .local = local } });
508	}
509
510	/// Accepts an unsigned 32bit integer rather than a signed integer to
511	/// prevent us from having to bitcast multiple times as most values
512	/// within codegen are represented as unsigned rather than signed.
513	pub fn addImm32(cg: *CodeGen, imm: u32) error{OutOfMemory}!void {
514	try cg.addInst(.{ .tag = .i32_const, .data = .{ .imm32 = @bitCast(imm) } });
515	}
516
517	/// Accepts an unsigned 64bit integer rather than a signed integer to
518	/// prevent us from having to bitcast multiple times as most values
519	/// within codegen are represented as unsigned rather than signed.
520	pub fn addImm64(cg: *CodeGen, imm: u64) error{OutOfMemory}!void {
521	const extra_index = try cg.addExtra(Mir.Imm64.init(imm));
522	try cg.addInst(.{ .tag = .i64_const, .data = .{ .payload = extra_index } });
523	}
524
525	/// Accepts the index into the list of 128bit-immediates
526	pub fn addImm128(cg: *CodeGen, index: u32) error{OutOfMemory}!void {
527	const simd_values = cg.simd_immediates.items[index];
528	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
529	// tag + 128bit value
530	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, 5);
531	cg.mir_extra.appendAssumeCapacity(@backingInt(std.wasm.SimdOpcode.v128_const));
532	cg.mir_extra.appendSliceAssumeCapacity(@alignCast(mem.bytesAsSlice(u32, &simd_values)));
533	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
534	}
535
536	pub fn addFloat32(cg: *CodeGen, float: f32) error{OutOfMemory}!void {
537	try cg.addInst(.{ .tag = .f32_const, .data = .{ .float32 = float } });
538	}
539
540	pub fn addFloat64(cg: *CodeGen, float: f64) error{OutOfMemory}!void {
541	const extra_index = try cg.addExtra(Mir.Float64.init(float));
542	try cg.addInst(.{ .tag = .f64_const, .data = .{ .payload = extra_index } });
543	}
544
545	/// Inserts an instruction to load/store from/to wasm's linear memory dependent on the given `tag`.
546	pub fn addMemArg(cg: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
547	const extra_index = try cg.addExtra(mem_arg);
548	try cg.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } });
549	}
550
551	/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the
552	/// given `tag`.
553	pub fn addAtomicMemArg(cg: *CodeGen, tag: std.wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void {
554	const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @backingInt(tag) }));
555	_ = try cg.addExtra(mem_arg);
556	try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
557	}
558
559	/// Helper function to emit atomic mir opcodes.
560	pub fn addAtomicTag(cg: *CodeGen, tag: std.wasm.AtomicsOpcode) error{OutOfMemory}!void {
561	const extra_index = try cg.addExtra(@as(struct { val: u32 }, .{ .val = @backingInt(tag) }));
562	try cg.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } });
563	}
564
565	fn addCallIntrinsic(cg: *CodeGen, intrinsic: Mir.Intrinsic) error{OutOfMemory}!void {
566	try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });
567	}
568
569	/// Appends entries to `mir_extra` based on the type of `extra`.
570	/// Returns the index into `mir_extra`
571	fn addExtra(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
572	const field_count = @typeInfo(@TypeOf(extra)).@"struct".field_names.len;
573	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, field_count);
574	return cg.addExtraAssumeCapacity(extra);
575	}
576
577	/// Appends entries to `mir_extra` based on the type of `extra`.
578	/// Returns the index into `mir_extra`
579	fn addExtraAssumeCapacity(cg: *CodeGen, extra: anytype) error{OutOfMemory}!u32 {
580	const info = @typeInfo(@TypeOf(extra)).@"struct";
581	const result: u32 = @intCast(cg.mir_extra.items.len);
582	inline for (info.field_names, info.field_types) |field_name, field_type| {
583	cg.mir_extra.appendAssumeCapacity(switch (field_type) {
584	u32 => @field(extra, field_name),
585	i32 => @bitCast(@field(extra, field_name)),
586	InternPool.Index,
587	InternPool.Nav.Index,
588	=> @backingInt(@field(extra, field_name)),
589	else => @compileError("Unsupported field type " ++ @typeName(field_type)),
590	});
591	}
592	return result;
593	}
594
595	/// For `std.lang.CallingConvention.auto`.
596	pub fn typeToValtype(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.Valtype {
597	return switch (ty.zigTypeTag(zcu)) {
598	.float => switch (ty.floatBits(target)) {
599	16 => .i32, // stored/loaded as u16
600	32 => .f32,
601	64 => .f64,
602	80, 128 => .i32,
603	else => unreachable,
604	},
605	.int, .@"enum" => switch (ty.intInfo(zcu).bits) {
606	0...32 => .i32,
607	33...64 => .i64,
608	else => .i32,
609	},
610	.vector => switch (CodeGen.determineSimdStoreStrategy(ty, zcu, target)) {
611	.direct => .v128,
612	.unrolled => .i32,
613	},
614	.@"union", .@"struct" => switch (ty.containerLayout(zcu)) {
615	.@"packed" => typeToValtype(ty.backingIntType(zcu), zcu, target),
616	.auto, .@"extern" => .i32,
617	},
618	else => .i32, // all represented as reference/immediate
619	};
620	}
621
622	/// Using a given `Type`, returns the corresponding wasm value type
623	/// Differently from `typeToValtype` this also allows `void` to create a block
624	/// with no return type
625	fn genBlockType(ty: Type, zcu: *const Zcu, target: *const std.Target) std.wasm.BlockType {
626	return switch (ty.ip_index) {
627	.void_type, .noreturn_type => .empty,
628	else => .fromValtype(typeToValtype(ty, zcu, target)),
629	};
630	}
631
632	/// Writes the bytecode depending on the given `WValue` in `val`
633	fn emitWValue(cg: *CodeGen, value: WValue) InnerError!void {
634	switch (value) {
635	.dead => unreachable, // reference to free'd `WValue` (missing reuseOperand?)
636	.none, .stack => {}, // no-op
637	.local => |idx| try cg.addLocal(.local_get, idx.value),
638	.imm32 => |val| try cg.addImm32(val),
639	.imm64 => |val| try cg.addImm64(val),
640	.imm128 => |val| try cg.addImm128(val),
641	.float32 => |val| try cg.addFloat32(val),
642	.float64 => |val| try cg.addFloat64(val),
643	.nav_ref => |nav_ref| {
644	const zcu = cg.pt.zcu;
645	const ip = &zcu.intern_pool;
646	if (ip.zigTypeTag(ip.getNav(nav_ref.nav_index).resolved.?.type) == .@"fn") {
647	assert(nav_ref.offset == 0);
648	try cg.mir_indirect_function_set.put(cg.gpa, nav_ref.nav_index, {});
649	try cg.addInst(.{ .tag = .func_ref, .data = .{ .nav_index = nav_ref.nav_index } });
650	} else if (nav_ref.offset == 0) {
651	try cg.addInst(.{ .tag = .nav_ref, .data = .{ .nav_index = nav_ref.nav_index } });
652	} else {
653	try cg.addInst(.{
654	.tag = .nav_ref_off,
655	.data = .{
656	.payload = try cg.addExtra(Mir.NavRefOff{
657	.nav_index = nav_ref.nav_index,
658	.offset = nav_ref.offset,
659	}),
660	},
661	});
662	}
663	},
664	.uav_ref => |uav| {
665	const zcu = cg.pt.zcu;
666	const ip = &zcu.intern_pool;
667	assert(!ip.isFunctionType(ip.typeOf(uav.ip_index)));
668	const gop = try cg.mir_uavs.getOrPut(cg.gpa, uav.ip_index);
669	const this_align: Alignment = a: {
670	if (uav.orig_ptr_ty == .none) break :a .none;
671	const ptr_type = ip.indexToKey(uav.orig_ptr_ty).ptr_type;
672	const this_align = ptr_type.flags.alignment;
673	if (this_align == .none) break :a .none;
674	const abi_align = Type.fromInterned(ptr_type.child).abiAlignment(zcu);
675	if (this_align.compare(.lte, abi_align)) break :a .none;
676	break :a this_align;
677	};
678	if (!gop.found_existing or
679	gop.value_ptr.* == .none or
680	(this_align != .none and this_align.compare(.gt, gop.value_ptr.*)))
681	{
682	gop.value_ptr.* = this_align;
683	}
684	if (uav.offset == 0) {
685	try cg.addInst(.{
686	.tag = .uav_ref,
687	.data = .{ .ip_index = uav.ip_index },
688	});
689	} else {
690	try cg.addInst(.{
691	.tag = .uav_ref_off,
692	.data = .{ .payload = try cg.addExtra(@as(Mir.UavRefOff, .{
693	.value = uav.ip_index,
694	.offset = uav.offset,
695	})) },
696	});
697	}
698	},
699	.stack_offset => try cg.addLocal(.local_get, cg.bottom_stack_value.local.value), // caller must ensure to address the offset
700	}
701	}
702
703	/// If given a local or stack-offset, increases the reference count by 1.
704	/// The old `WValue` found at instruction `ref` is then replaced by the
705	/// modified `WValue` and returned. When given a non-local or non-stack-offset,
706	/// returns the given `operand` itfunc instead.
707	fn reuseOperand(cg: *CodeGen, ref: Air.Inst.Ref, operand: WValue) WValue {
708	if (operand != .local and operand != .stack_offset) return operand;
709	var new_value = operand;
710	switch (new_value) {
711	.local => |*local| local.references += 1,
712	.stack_offset => |*stack_offset| stack_offset.references += 1,
713	else => unreachable,
714	}
715	const old_value = cg.getResolvedInst(ref);
716	old_value.* = new_value;
717	return new_value;
718	}
719
720	/// From a reference, returns its resolved `WValue`.
721	/// It's illegal to provide a `Air.Inst.Ref` that hasn't been resolved yet.
722	fn getResolvedInst(cg: *CodeGen, ref: Air.Inst.Ref) *WValue {
723	var index = cg.branches.items.len;
724	while (index > 0) : (index -= 1) {
725	const branch = cg.branches.items[index - 1];
726	if (branch.values.getPtr(ref)) |value| {
727	return value;
728	}
729	}
730	unreachable; // developer-error: This can only be called on resolved instructions. Use `resolveInst` instead.
731	}
732
733	/// Creates one locals for a given `Type`.
734	/// Returns a corresponding `Wvalue` with `local` as active tag
735	fn allocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
736	const zcu = cg.pt.zcu;
737	const valtype = typeToValtype(ty, zcu, cg.target);
738	const index_or_null = switch (valtype) {
739	.i32 => cg.free_locals_i32.pop(),
740	.i64 => cg.free_locals_i64.pop(),
741	.f32 => cg.free_locals_f32.pop(),
742	.f64 => cg.free_locals_f64.pop(),
743	.v128 => cg.free_locals_v128.pop(),
744	};
745	if (index_or_null) |index| {
746	log.debug("reusing local ({d}) of type {}", .{ index, valtype });
747	return .{ .local = .{ .value = index, .references = 1 } };
748	}
749	log.debug("new local of type {}", .{valtype});
750	return cg.ensureAllocLocal(ty);
751	}
752
753	/// Ensures a new local will be created. This is useful when it's useful
754	/// to use a zero-initialized local.
755	fn ensureAllocLocal(cg: *CodeGen, ty: Type) InnerError!WValue {
756	const zcu = cg.pt.zcu;
757	try cg.mir_locals.append(cg.gpa, typeToValtype(ty, zcu, cg.target));
758	const initial_index = cg.local_index;
759	cg.local_index += 1;
760	return .{ .local = .{ .value = initial_index, .references = 1 } };
761	}
762
763	pub const Error = error{
764	OutOfMemory,
765	/// Indicates the error is already stored in Zcu `failed_codegen`.
766	AlreadyReported,
767	};
768
769	pub fn generate(
770	bin_file: *link.File,
771	pt: Zcu.PerThread,
772	func_index: InternPool.Index,
773	air: *const Air,
774	liveness: *const ?Air.Liveness,
775	) Error!Mir {
776	_ = bin_file;
777	const zcu = pt.zcu;
778	const gpa = zcu.gpa;
779	const cg = zcu.funcInfo(func_index);
780	const file_scope = zcu.navFileScope(cg.owner_nav);
781	const target = &file_scope.mod.?.resolved_target.result;
782	const fn_ty = zcu.navValue(cg.owner_nav).typeOf(zcu);
783	const fn_info = zcu.typeToFunc(fn_ty).?;
784	const ret_ty: Type = .fromInterned(fn_info.return_type);
785	const any_returns = !firstParamSRet(fn_info.cc, ret_ty, zcu, target) and ret_ty.hasRuntimeBits(zcu);
786
787	var cc_result = try resolveCallingConventionValues(zcu, fn_ty, target);
788	defer cc_result.deinit(gpa);
789
790	var code_gen: CodeGen = .{
791	.gpa = gpa,
792	.pt = pt,
793	.air = air.*,
794	.liveness = liveness.*.?,
795	.owner_nav = cg.owner_nav,
796	.target = target,
797	.ptr_size = switch (target.cpu.arch) {
798	.wasm32 => .wasm32,
799	.wasm64 => .wasm64,
800	else => unreachable,
801	},
802	.func_index = func_index,
803	.args = cc_result.args,
804	.return_value = cc_result.return_value,
805	.varargs = cc_result.varargs,
806	.local_index = cc_result.local_index,
807	.mir_instructions = .empty,
808	.mir_extra = .empty,
809	.mir_locals = .empty,
810	.mir_uavs = .empty,
811	.mir_indirect_function_set = .empty,
812	.mir_func_tys = .empty,
813	.error_name_table_ref_count = 0,
814	};
815	defer code_gen.deinit();
816
817	try code_gen.mir_func_tys.putNoClobber(gpa, fn_ty.toIntern(), {});
818
819	return generateInner(&code_gen, any_returns) catch |err| switch (err) {
820	error.AlreadyReported,
821	error.OutOfMemory,
822	=> |e| return e,
823	else => |e| return code_gen.fail("failed to generate function: {s}", .{@errorName(e)}),
824	};
825	}
826
827	fn generateInner(cg: *CodeGen, any_returns: bool) InnerError!Mir {
828	const zcu = cg.pt.zcu;
829	// branch used for const values
830	try cg.branches.append(cg.gpa, .{});
831	// func scope branch
832	try cg.branches.append(cg.gpa, .{});
833	defer {
834	var func_branch = cg.branches.pop().?;
835	func_branch.deinit(cg.gpa);
836	var const_branch = cg.branches.pop().?;
837	const_branch.deinit(cg.gpa);
838	assert(cg.branches.items.len == 0); // missing branch merge
839	}
840	// Generate MIR for function body
841	try cg.genBody(cg.air.getMainBody());
842
843	// In case we have a return value, but the last instruction is a noreturn (such as a while loop)
844	// we emit an unreachable instruction to tell the stack validator that part will never be reached.
845	if (any_returns and cg.air.instructions.len > 0) {
846	const main_body = cg.air.getMainBody();
847	const inst: Air.Inst.Index = main_body[main_body.len - 1];
848	const last_inst_ty = cg.typeOfIndex(inst);
849	if (!last_inst_ty.hasRuntimeBits(zcu)) {
850	try cg.addTag(.@"unreachable");
851	}
852	}
853	// End of function body
854	try cg.addTag(.end);
855	try cg.addTag(.dbg_epilogue_begin);
856
857	try cg.mir_extra.shrinkToLen(cg.gpa);
858	try cg.mir_locals.shrinkToLen(cg.gpa);
859
860	return .{
861	.instructions = cg.mir_instructions.toOwnedSlice(),
862	.extra = cg.mir_extra.toOwnedSliceAssert(),
863	.locals = cg.mir_locals.toOwnedSliceAssert(),
864	.prologue = if (cg.initial_stack_value == .none) .none else .{
865	.sp_local = cg.initial_stack_value.local.value,
866	.flags = .{ .stack_alignment = cg.stack_alignment },
867	.stack_size = cg.stack_size,
868	.bottom_stack_local = cg.bottom_stack_value.local.value,
869	},
870	.uavs = cg.mir_uavs.move(),
871	.indirect_function_set = cg.mir_indirect_function_set.move(),
872	.func_tys = cg.mir_func_tys.move(),
873	.error_name_table_ref_count = cg.error_name_table_ref_count,
874	};
875	}
876
877	const CallWValues = struct {
878	args: []WValue,
879	return_value: WValue,
880	varargs: WValue,
881	local_index: u32,
882
883	fn deinit(values: *CallWValues, gpa: Allocator) void {
884	gpa.free(values.args);
885	values.* = undefined;
886	}
887	};
888
889	fn resolveCallingConventionValues(
890	zcu: *const Zcu,
891	fn_ty: Type,
892	target: *const std.Target,
893	) Allocator.Error!CallWValues {
894	const gpa = zcu.gpa;
895	const ip = &zcu.intern_pool;
896	const fn_info = zcu.typeToFunc(fn_ty).?;
897	const cc = fn_info.cc;
898
899	var result: CallWValues = .{
900	.args = &.{},
901	.return_value = .none,
902	.varargs = .none,
903	.local_index = 0,
904	};
905	if (cc == .naked) return result;
906
907	var args = std.array_list.Managed(WValue).init(gpa);
908	defer args.deinit();
909
910	// Check if we store the result as a pointer to the stack rather than
911	// by value
912	if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, target)) {
913	// the sret arg will be passed as first argument, therefore we
914	// set the `return_value` before allocating locals for regular args.
915	result.return_value = .{ .local = .{ .value = result.local_index, .references = 1 } };
916	result.local_index += 1;
917	}
918
919	switch (cc) {
920	.auto => {
921	for (fn_info.param_types.get(ip)) |ty| {
922	if (!Type.fromInterned(ty).hasRuntimeBits(zcu)) {
923	continue;
924	}
925
926	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
927	result.local_index += 1;
928	}
929	},
930	.wasm_mvp => {
931	for (fn_info.param_types.get(ip)) |ty| {
932	const param_ty: Type = .fromInterned(ty);
933	if (!param_ty.hasRuntimeBits(zcu)) {
934	continue;
935	}
936
937	switch (abi.classifyType(param_ty, zcu, target)) {
938	.direct, .indirect => {
939	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
940	result.local_index += 1;
941	},
942	.double_i64 => {
943	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
944	try args.append(.{ .local = .{ .value = result.local_index + 1, .references = 1 } });
945	result.local_index += 2;
946	},
947	.unrolled => |vector| {
948	for (0..vector.len) |_| {
949	try args.append(.{ .local = .{ .value = result.local_index, .references = 1 } });
950	result.local_index += 1;
951	}
952	},
953	}
954	}
955	},
956	else => unreachable, // Frontend is responsible for emitting an error earlier.
957	}
958
959	if (fn_info.is_var_args) {
960	result.varargs = .{ .local = .{ .value = result.local_index, .references = 1 } };
961	result.local_index += 1;
962	}
963
964	result.args = try args.toOwnedSlice();
965	return result;
966	}
967
968	pub fn firstParamSRet(
969	cc: std.lang.CallingConvention,
970	return_type: Type,
971	zcu: *const Zcu,
972	target: *const std.Target,
973	) bool {
974	if (!return_type.hasRuntimeBits(zcu)) return false;
975	switch (cc) {
976	.@"inline" => unreachable,
977	.auto => return isByRef(return_type, zcu, target),
978	.wasm_mvp => switch (abi.classifyType(return_type, zcu, target)) {
979	.direct => return false,
980	.double_i64, .indirect => return true,
981	.unrolled => |vector| return vector.len > 1,
982	},
983	else => return false,
984	}
985	}
986
987	/// Lowers a Zig type and its value based on a given calling convention to ensure
988	/// it matches the ABI.
989	fn lowerArg(cg: *CodeGen, cc: std.lang.CallingConvention, ty: Type, value: WValue) !void {
990	if (cc != .wasm_mvp) {
991	return cg.lowerToStack(value);
992	}
993
994	const zcu = cg.pt.zcu;
995
996	switch (abi.classifyType(ty, zcu, cg.target)) {
997	.direct => |scalar_ty| {
998	if (!isByRef(ty, zcu, cg.target)) {
999	return cg.lowerToStack(value);
1000	} else {
1001	_ = try cg.load(value, scalar_ty, 0);
1002	}
1003	},
1004	.double_i64 => {
1005	assert(ty.abiSize(zcu) == 16);
1006	// in this case we have an integer or float that must be lowered as 2 i64's.
1007	try cg.emitWValue(value);
1008	try cg.addMemArg(.i64_load, .{ .offset = value.offset(), .alignment = 8 });
1009	try cg.emitWValue(value);
1010	try cg.addMemArg(.i64_load, .{ .offset = value.offset() + 8, .alignment = 8 });
1011	},
1012	.indirect => {
1013	const stack_copy = try cg.allocStack(ty);
1014	try cg.store(stack_copy, value, ty, 0);
1015	return cg.lowerToStack(stack_copy);
1016	},
1017	.unrolled => |vector| {
1018	const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
1019	for (0..vector.len) |index| {
1020	_ = try cg.load(value, vector.elem_type, @intCast(index * elem_size));
1021	}
1022	},
1023	}
1024	}
1025
1026	/// Lowers a `WValue` to the stack. This means when the `value` results in
1027	/// `.stack_offset` we calculate the pointer of this offset and use that.
1028	/// The value is left on the stack, and not stored in any temporary.
1029	fn lowerToStack(cg: *CodeGen, value: WValue) !void {
1030	switch (value) {
1031	.stack_offset => |offset| {
1032	try cg.emitWValue(value);
1033	if (offset.value > 0) {
1034	switch (cg.ptr_size) {
1035	.wasm32 => {
1036	try cg.addImm32(offset.value);
1037	try cg.addTag(.i32_add);
1038	},
1039	.wasm64 => {
1040	try cg.addImm64(offset.value);
1041	try cg.addTag(.i64_add);
1042	},
1043	}
1044	}
1045	},
1046	else => try cg.emitWValue(value),
1047	}
1048	}
1049
1050	/// Creates a local for the initial stack value
1051	/// Asserts `initial_stack_value` is `.none`
1052	fn initializeStack(cg: *CodeGen) !void {
1053	assert(cg.initial_stack_value == .none);
1054	// Reserve a local to store the current stack pointer
1055	// We can later use this local to set the stack pointer back to the value
1056	// we have stored here.
1057	cg.initial_stack_value = try cg.ensureAllocLocal(Type.usize);
1058	// Also reserve a local to store the bottom stack value
1059	cg.bottom_stack_value = try cg.ensureAllocLocal(Type.usize);
1060	}
1061
1062	/// Reads the stack pointer from `Context.initial_stack_value` and writes it
1063	/// to the global stack pointer variable
1064	fn restoreStackPointer(cg: *CodeGen) !void {
1065	// only restore the pointer if it was initialized
1066	if (cg.initial_stack_value == .none) return;
1067	// Get the original stack pointer's value
1068	try cg.emitWValue(cg.initial_stack_value);
1069
1070	try cg.addTag(.global_set_sp);
1071	}
1072
1073	/// From a given type, will create space on the virtual stack to store the value of such type.
1074	/// This returns a `WValue` with its active tag set to `local`, containing the index to the local
1075	/// that points to the position on the virtual stack. This function should be used instead of
1076	/// moveStack unless a local was already created to store the pointer.
1077	///
1078	/// Asserts Type has codegenbits
1079	fn allocStack(cg: *CodeGen, ty: Type) !WValue {
1080	const pt = cg.pt;
1081	const zcu = pt.zcu;
1082	assert(ty.hasRuntimeBits(zcu));
1083
1084	const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse {
1085	return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
1086	ty.fmt(pt), ty.abiSize(zcu),
1087	});
1088	};
1089	const abi_align = ty.abiAlignment(zcu);
1090
1091	return cg.allocStackBytes(abi_size, abi_align);
1092	}
1093
1094	fn allocInt(cg: *CodeGen, int_ty: IntType) !WValue {
1095	const abi_size = std.math.cast(u32, std.zig.target.intByteSize(cg.target, int_ty.bits)) orelse {
1096	return cg.fail("Integer ABI size exceeds max stack size", .{});
1097	};
1098	const abi_align: Alignment = .fromByteUnits(std.zig.target.intAlignment(cg.target, int_ty.bits));
1099
1100	return cg.allocStackBytes(abi_size, abi_align);
1101	}
1102
1103	fn allocStackBytes(cg: *CodeGen, size: u32, alignment: Alignment) !WValue {
1104	assert(size > 0);
1105
1106	if (cg.initial_stack_value == .none) {
1107	try cg.initializeStack();
1108	}
1109
1110	cg.stack_alignment = cg.stack_alignment.max(alignment);
1111
1112	const offset: u32 = @intCast(alignment.forward(cg.stack_size));
1113	defer cg.stack_size = offset + size;
1114
1115	return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1116	}
1117
1118	/// From a given AIR instruction generates a pointer to the stack where
1119	/// the value of its type will live.
1120	/// This is different from allocStack where this will use the pointer's alignment
1121	/// if it is set, to ensure the stack alignment will be set correctly.
1122	fn allocStackPtr(cg: *CodeGen, inst: Air.Inst.Index) !WValue {
1123	const pt = cg.pt;
1124	const zcu = pt.zcu;
1125	const ptr_ty = cg.typeOfIndex(inst);
1126	const pointee_ty = ptr_ty.childType(zcu);
1127
1128	if (cg.initial_stack_value == .none) {
1129	try cg.initializeStack();
1130	}
1131
1132	const abi_alignment = ptr_ty.ptrAlignment(zcu);
1133	const abi_size = std.math.cast(u32, pointee_ty.abiSize(zcu)) orelse {
1134	return cg.fail("Type {f} with ABI size of {d} exceeds stack frame size", .{
1135	pointee_ty.fmt(pt), pointee_ty.abiSize(zcu),
1136	});
1137	};
1138	cg.stack_alignment = cg.stack_alignment.max(abi_alignment);
1139
1140	const offset: u32 = @intCast(abi_alignment.forward(cg.stack_size));
1141	defer cg.stack_size = offset + abi_size;
1142
1143	return .{ .stack_offset = .{ .value = offset, .references = 1 } };
1144	}
1145
1146	fn emitMemoryCopy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1147	const len_known_neq_0 = switch (len) {
1148	.imm32 => |val| if (val != 0) true else return,
1149	.imm64 => |val| if (val != 0) true else return,
1150	else => false,
1151	};
1152	const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
1153	const emit_check = !(len0_ok or len_known_neq_0);
1154
1155	if (emit_check) {
1156	try cg.startBlock(.block, .empty);
1157
1158	// Even if `len` is zero, the spec requires an implementation to trap if `src + len` or
1159	// `dst + len` are out of memory bounds. This can easily happen in Zig in a case such
1160	// as:
1161	//
1162	// const dst: [*]u8 = undefined;
1163	// const src: [*]u8 = undefined;
1164	// var len: usize = runtime_zero();
1165	// @memcpy(dst[0..len], src[0..len]);
1166	//
1167	// So explicitly avoid using `memory.copy` in the `len == 0` case. Lovely design.
1168	try cg.emitWValue(len);
1169	try cg.addTag(.i32_eqz);
1170	try cg.addLabel(.br_if, 0);
1171	}
1172
1173	try cg.lowerToStack(dst);
1174	try cg.lowerToStack(src);
1175	try cg.emitWValue(len);
1176	try cg.addExtended(.memory_copy);
1177
1178	if (emit_check) {
1179	try cg.endBlock();
1180	}
1181	}
1182
1183	fn memcpy(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1184	if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1185	try cg.emitMemoryCopy(dst, src, len);
1186	return;
1187	}
1188
1189	try cg.lowerToStack(dst);
1190	try cg.lowerToStack(src);
1191	try cg.emitWValue(len);
1192	try cg.addCallIntrinsic(.memcpy);
1193	try cg.addTag(.drop);
1194	}
1195
1196	fn memmove(cg: *CodeGen, dst: WValue, src: WValue, len: WValue) !void {
1197	if (cg.target.cpu.has(.wasm, .bulk_memory)) {
1198	try cg.emitMemoryCopy(dst, src, len);
1199	return;
1200	}
1201
1202	try cg.lowerToStack(dst);
1203	try cg.lowerToStack(src);
1204	try cg.emitWValue(len);
1205	try cg.addCallIntrinsic(.memmove);
1206	try cg.addTag(.drop);
1207	}
1208
1209	fn ptrSize(cg: *const CodeGen) u16 {
1210	return @divExact(cg.target.ptrBitWidth(), 8);
1211	}
1212
1213	/// For a given `Type`, will return true when the type will be passed
1214	/// by reference, rather than by value
1215	fn isByRef(ty: Type, zcu: *const Zcu, target: *const std.Target) bool {
1216	switch (ty.zigTypeTag(zcu)) {
1217	.type,
1218	.comptime_int,
1219	.comptime_float,
1220	.enum_literal,
1221	.undefined,
1222	.null,
1223	.@"opaque",
1224	.spirv,
1225	=> unreachable,
1226
1227	.noreturn,
1228	.void,
1229	.bool,
1230	.error_set,
1231	.@"fn",
1232	.@"anyframe",
1233	=> return false,
1234
1235	.array,
1236	.frame,
1237	=> return ty.hasRuntimeBits(zcu),
1238	.@"struct", .@"union" => switch (ty.containerLayout(zcu)) {
1239	.@"packed" => return isByRef(ty.backingIntType(zcu), zcu, target),
1240	.@"extern", .auto => return ty.hasRuntimeBits(zcu),
1241	},
1242	.vector => return determineSimdStoreStrategy(ty, zcu, target) == .unrolled,
1243	.int => return ty.intInfo(zcu).bits > 64,
1244	.@"enum" => return ty.intInfo(zcu).bits > 64,
1245	.float => return ty.floatBits(target) > 64,
1246	.error_union => {
1247	const pl_ty = ty.errorUnionPayload(zcu);
1248	if (!pl_ty.hasRuntimeBits(zcu)) {
1249	return false;
1250	}
1251	return true;
1252	},
1253	.optional => {
1254	if (ty.isPtrLikeOptional(zcu)) return false;
1255	const pl_type = ty.optionalChild(zcu);
1256	if (pl_type.zigTypeTag(zcu) == .error_set) return false;
1257	return pl_type.hasRuntimeBits(zcu);
1258	},
1259	.pointer => {
1260	// Slices act like struct and will be passed by reference
1261	if (ty.isSlice(zcu)) return true;
1262	return false;
1263	},
1264	}
1265	}
1266
1267	const SimdStoreStrategy = enum {
1268	direct,
1269	unrolled,
1270	};
1271
1272	/// For a given vector type, returns the `SimdStoreStrategy`.
1273	/// This means when a given type is 128 bits and either the simd128 or relaxed-simd
1274	/// features are enabled, the function will return `.direct`. This would allow to store
1275	/// it using a instruction, rather than an unrolled version.
1276	pub fn determineSimdStoreStrategy(ty: Type, zcu: *const Zcu, target: *const std.Target) SimdStoreStrategy {
1277	assert(ty.zigTypeTag(zcu) == .vector);
1278	if (ty.bitSize(zcu) != 128) return .unrolled;
1279	if (target.cpu.has(.wasm, .relaxed_simd) or target.cpu.has(.wasm, .simd128)) {
1280	return .direct;
1281	}
1282	return .unrolled;
1283	}
1284
1285	/// Creates a new local for a pointer that points to memory with given offset.
1286	/// This can be used to get a pointer to a struct field, error payload, etc.
1287	/// By providing `modify` as action, it will modify the given `ptr_value` instead of making a new
1288	/// local value to store the pointer. This allows for local re-use and improves binary size.
1289	fn buildPointerOffset(cg: *CodeGen, ptr_value: WValue, offset: u64, action: enum { modify, new }) InnerError!WValue {
1290	// do not perform arithmetic when offset is 0.
1291	if (offset == 0 and ptr_value.offset() == 0 and action == .modify) return ptr_value;
1292	const result_ptr: WValue = switch (action) {
1293	.new => try cg.ensureAllocLocal(Type.usize),
1294	.modify => ptr_value,
1295	};
1296	try cg.emitWValue(ptr_value);
1297	if (offset + ptr_value.offset() > 0) {
1298	switch (cg.ptr_size) {
1299	.wasm32 => {
1300	try cg.addImm32(@intCast(offset + ptr_value.offset()));
1301	try cg.addTag(.i32_add);
1302	},
1303	.wasm64 => {
1304	try cg.addImm64(offset + ptr_value.offset());
1305	try cg.addTag(.i64_add);
1306	},
1307	}
1308	}
1309	try cg.addLocal(.local_set, result_ptr.local.value);
1310	return result_ptr;
1311	}
1312
1313	fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1314	const zcu = cg.pt.zcu;
1315	const air_tags = cg.air.instructions.items(.tag);
1316	return switch (air_tags[@backingInt(inst)]) {
1317	// No soft float legalizations are enabled.
1318	.legalize_compiler_rt_call => unreachable,
1319
1320	.inferred_alloc, .inferred_alloc_comptime => unreachable,
1321
1322	.legalize_vec_elem_val => cg.airArrayElemVal(inst),
1323	.legalize_vec_store_elem => {
1324	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1325	const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
1326	const vec_ptr = try cg.resolveInst(pl_op.operand);
1327	const elem_idx = try cg.resolveInst(bin_op.lhs);
1328	const elem_val = try cg.resolveInst(bin_op.rhs);
1329
1330	const elem_ty = cg.typeOf(bin_op.rhs);
1331	const elem_size = elem_ty.abiSize(zcu);
1332
1333	try cg.lowerToStack(vec_ptr);
1334	try cg.emitWValue(elem_idx);
1335	try cg.addImm32(@intCast(elem_size));
1336	try cg.addTag(.i32_mul);
1337	try cg.addTag(.i32_add);
1338	const ptr = try WValue.toLocal(.stack, cg, Type.usize);
1339
1340	try cg.store(ptr, elem_val, elem_ty, 0);
1341
1342	return cg.finishAir(inst, .none, &.{ pl_op.operand, bin_op.lhs, bin_op.rhs });
1343	},
1344
1345	.add,
1346	.sub,
1347	.mul,
1348	.rem,
1349	.mod,
1350	.max,
1351	.min,
1352	.div_exact,
1353	.div_trunc,
1354	.div_floor,
1355	.div_ceil,
1356	=> |tag| {
1357	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1358	const lhs = try cg.resolveInst(bin_op.lhs);
1359	const rhs = try cg.resolveInst(bin_op.rhs);
1360
1361	const ty = cg.typeOfIndex(inst);
1362	const type_tag = ty.zigTypeTag(zcu);
1363
1364	if (type_tag == .vector) {
1365	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1366	}
1367
1368	if (type_tag == .float) {
1369	const float_ty: FloatType = .fromType(cg, ty);
1370
1371	const result = switch (tag) {
1372	.add => try cg.floatAdd(float_ty, lhs, rhs),
1373	.sub => try cg.floatSub(float_ty, lhs, rhs),
1374	.mul => try cg.floatMul(float_ty, lhs, rhs),
1375	.rem => try cg.floatRem(float_ty, lhs, rhs),
1376	.mod => try cg.floatMod(float_ty, lhs, rhs),
1377	.max => try cg.floatMax(float_ty, lhs, rhs),
1378	.min => try cg.floatMin(float_ty, lhs, rhs),
1379	.div_exact => try cg.floatDiv(float_ty, lhs, rhs),
1380	.div_trunc => try cg.floatDivTrunc(float_ty, lhs, rhs),
1381	.div_floor => try cg.floatDivFloor(float_ty, lhs, rhs),
1382	.div_ceil => try cg.floatDivCeil(float_ty, lhs, rhs),
1383	else => unreachable,
1384	};
1385
1386	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1387	} else if (type_tag == .int) {
1388	const int_ty: IntType = .fromType(cg, ty);
1389
1390	const result = switch (tag) {
1391	.add => try cg.intAdd(int_ty, lhs, rhs),
1392	.sub => try cg.intSub(int_ty, lhs, rhs),
1393	.mul => try cg.intMul(int_ty, lhs, rhs),
1394	.rem => try cg.intRem(int_ty, lhs, rhs),
1395	.mod => try cg.intMod(int_ty, lhs, rhs),
1396	.max => try cg.intMax(int_ty, lhs, rhs),
1397	.min => try cg.intMin(int_ty, lhs, rhs),
1398	.div_exact => try cg.intDiv(int_ty, lhs, rhs),
1399	.div_trunc => try cg.intDiv(int_ty, lhs, rhs),
1400	.div_floor => try cg.intDivFloor(int_ty, lhs, rhs),
1401	.div_ceil => try cg.intDivCeil(int_ty, lhs, rhs),
1402	else => unreachable,
1403	};
1404
1405	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1406	} else {
1407	unreachable;
1408	}
1409	},
1410	.div_float => {
1411	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1412	const lhs = try cg.resolveInst(bin_op.lhs);
1413	const rhs = try cg.resolveInst(bin_op.rhs);
1414	const ty = cg.typeOfIndex(inst);
1415
1416	if (ty.zigTypeTag(zcu) == .vector) {
1417	return cg.fail("TODO: implement AIR op: div_float for vectors", .{});
1418	}
1419
1420	const result = try cg.floatDiv(.fromType(cg, ty), lhs, rhs);
1421	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1422	},
1423	.abs => {
1424	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1425	const operand = try cg.resolveInst(ty_op.operand);
1426
1427	const ty = cg.typeOf(ty_op.operand);
1428	const type_tag = ty.zigTypeTag(zcu);
1429
1430	if (type_tag == .vector) {
1431	return cg.fail("TODO: implement AIR op: abs for vectors", .{});
1432	}
1433
1434	if (type_tag == .float) {
1435	const result = try cg.floatAbs(.fromType(cg, ty), operand);
1436	return cg.finishAir(inst, result, &.{ty_op.operand});
1437	} else if (type_tag == .int) {
1438	const result = try cg.intAbs(.fromType(cg, ty), operand);
1439	return cg.finishAir(inst, result, &.{ty_op.operand});
1440	} else {
1441	unreachable;
1442	}
1443	},
1444	.mul_add => {
1445	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
1446	const bin_op = cg.air.extraData(Air.Bin, pl_op.payload).data;
1447	const addend = try cg.resolveInst(pl_op.operand);
1448	const lhs = try cg.resolveInst(bin_op.lhs);
1449	const rhs = try cg.resolveInst(bin_op.rhs);
1450	const ty = cg.typeOfIndex(inst);
1451
1452	if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1453	return cg.fail("TODO: implement AIR op: mul_add for vectors", .{});
1454	}
1455
1456	const result = try cg.floatMulAdd(.fromType(cg, ty), lhs, rhs, addend);
1457	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs, pl_op.operand });
1458	},
1459
1460	.add_sat,
1461	.sub_sat,
1462	.mul_sat,
1463	.shl_sat,
1464	=> |tag| {
1465	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1466	const lhs = try cg.resolveInst(bin_op.lhs);
1467	const rhs = try cg.resolveInst(bin_op.rhs);
1468	const ty = cg.typeOfIndex(inst);
1469
1470	if (ty.zigTypeTag(cg.pt.zcu) == .vector) {
1471	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1472	}
1473
1474	const int_ty: IntType = .fromType(cg, ty);
1475	const result = switch (tag) {
1476	.add_sat => try cg.intAddSat(int_ty, lhs, rhs),
1477	.sub_sat => try cg.intSubSat(int_ty, lhs, rhs),
1478	.mul_sat => try cg.intMulSat(int_ty, lhs, rhs),
1479	.shl_sat => try cg.intShlSat(int_ty, lhs, rhs),
1480	else => unreachable,
1481	};
1482
1483	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1484	},
1485
1486	.add_with_overflow,
1487	.sub_with_overflow,
1488	.mul_with_overflow,
1489	.shl_with_overflow,
1490	=> |tag| {
1491	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
1492	const extra = cg.air.extraData(Air.Bin, ty_pl.payload).data;
1493
1494	const lhs = try cg.resolveInst(extra.lhs);
1495	const rhs = try cg.resolveInst(extra.rhs);
1496
1497	const ty = cg.typeOf(extra.lhs);
1498	const int_ty: IntType = .fromType(cg, ty);
1499
1500	const out = switch (tag) {
1501	.add_with_overflow => try cg.intAddOverflow(int_ty, lhs, rhs),
1502	.sub_with_overflow => try cg.intSubOverflow(int_ty, lhs, rhs),
1503	.mul_with_overflow => try cg.intMulOverflow(int_ty, lhs, rhs),
1504	.shl_with_overflow => try cg.intShlOverflow(int_ty, lhs, rhs),
1505	else => unreachable,
1506	};
1507
1508	var ov_tmp = try out.ov.toLocal(cg, Type.u1);
1509	defer ov_tmp.free(cg);
1510
1511	var res_tmp = try out.result.toLocal(cg, ty);
1512	defer res_tmp.free(cg);
1513
1514	const result = try cg.allocStack(cg.typeOfIndex(inst));
1515	const offset: u32 = @intCast(ty.abiSize(cg.pt.zcu));
1516
1517	try cg.store(result, res_tmp, ty, 0);
1518	try cg.store(result, ov_tmp, Type.u1, offset);
1519
1520	try cg.finishAir(inst, result, &.{ extra.lhs, extra.rhs });
1521	},
1522
1523	.add_wrap, .sub_wrap, .mul_wrap, .shl => |tag| {
1524	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1525	const lhs = try cg.resolveInst(bin_op.lhs);
1526	const rhs = try cg.resolveInst(bin_op.rhs);
1527	const ty = cg.typeOfIndex(inst);
1528
1529	if (ty.zigTypeTag(zcu) == .vector) {
1530	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1531	}
1532
1533	const int_ty: IntType = .fromType(cg, ty);
1534	const raw_result = switch (tag) {
1535	.add_wrap => try cg.intAdd(int_ty, lhs, rhs),
1536	.sub_wrap => try cg.intSub(int_ty, lhs, rhs),
1537	.mul_wrap => try cg.intMul(int_ty, lhs, rhs),
1538	.shl => try cg.intShl(int_ty, lhs, rhs),
1539	else => unreachable,
1540	};
1541	const result = try cg.intWrap(int_ty, raw_result);
1542
1543	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1544	},
1545
1546	.bit_and, .bit_or, .xor, .shl_exact, .shr, .shr_exact => |tag| {
1547	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
1548	const lhs = try cg.resolveInst(bin_op.lhs);
1549	const rhs = try cg.resolveInst(bin_op.rhs);
1550	const ty = cg.typeOfIndex(inst);
1551
1552	if (ty.zigTypeTag(zcu) == .vector) {
1553	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1554	}
1555
1556	const int_ty: IntType = .fromType(cg, ty);
1557	const result = switch (tag) {
1558	.bit_and => try cg.intAnd(int_ty, lhs, rhs),
1559	.bit_or => try cg.intOr(int_ty, lhs, rhs),
1560	.xor => try cg.intXor(int_ty, lhs, rhs),
1561	.shl_exact => try cg.intShl(int_ty, lhs, rhs),
1562	.shr, .shr_exact => try cg.intShr(int_ty, lhs, rhs),
1563	else => unreachable,
1564	};
1565
1566	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
1567	},
1568
1569	.not => {
1570	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1571	const operand = try cg.resolveInst(ty_op.operand);
1572	const ty = cg.typeOf(ty_op.operand);
1573
1574	if (ty.zigTypeTag(zcu) == .vector) {
1575	return cg.fail("TODO: implement AIR op: not for vectors", .{});
1576	}
1577
1578	const result = try cg.intNot(.fromType(cg, ty), operand);
1579	try cg.finishAir(inst, result, &.{ty_op.operand});
1580	},
1581
1582	.ptr_cast => cg.airNopCast(inst),
1583	.error_cast => cg.airNopCast(inst),
1584	.error_from_int => cg.airNopCast(inst),
1585	.int_from_error => cg.airNopCast(inst),
1586	.ptr_from_int => cg.airNopCast(inst),
1587	.int_from_ptr => cg.airIntFromPtr(inst),
1588
1589	.bit_cast => cg.airBitcast(inst),
1590	.union_from_enum => cg.airUnionFromEnum(inst),
1591
1592	.int_cast => {
1593	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1594
1595	const dest_ty = ty_op.ty;
1596	const operand = try cg.resolveInst(ty_op.operand);
1597	const src_ty = cg.typeOf(ty_op.operand);
1598
1599	if (dest_ty.zigTypeTag(zcu) == .vector) {
1600	return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1601	}
1602
1603	const src_int_ty: IntType = .fromType(cg, src_ty);
1604	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1605
1606	const src_bits = src_int_ty.bits;
1607	const dest_bits = dest_int_ty.bits;
1608
1609	const same_class: bool = (src_bits <= 32 and dest_bits <= 32) or
1610	(src_bits >= 33 and src_bits <= 64 and dest_bits >= 33 and dest_bits <= 64) or
1611	(src_bits >= 65 and src_bits <= 128 and dest_bits >= 65 and dest_bits <= 128);
1612
1613	const result = if (same_class)
1614	cg.reuseOperand(ty_op.operand, operand)
1615	else
1616	try cg.intCast(dest_int_ty, src_int_ty, operand);
1617
1618	try cg.finishAir(inst, result, &.{ty_op.operand});
1619	},
1620	.trunc => {
1621	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1622
1623	const operand = try cg.resolveInst(ty_op.operand);
1624	const dest_ty = ty_op.ty;
1625	const src_ty = cg.typeOf(ty_op.operand);
1626
1627	if (dest_ty.zigTypeTag(zcu) == .vector or src_ty.zigTypeTag(zcu) == .vector) {
1628	return cg.fail("TODO: implement AIR op: trunc for vectors", .{});
1629	}
1630
1631	const src_int_ty: IntType = .fromType(cg, src_ty);
1632	const dest_int_ty: IntType = .fromType(cg, dest_ty);
1633
1634	const result = if (src_int_ty.bits == dest_int_ty.bits)
1635	cg.reuseOperand(ty_op.operand, operand)
1636	else blk: {
1637	break :blk try cg.intTrunc(dest_int_ty, src_int_ty, operand);
1638	};
1639
1640	try cg.finishAir(inst, result, &.{ty_op.operand});
1641	},
1642
1643	.fptrunc, .fpext => |tag| {
1644	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1645
1646	const operand = try cg.resolveInst(ty_op.operand);
1647	const src_ty = cg.typeOf(ty_op.operand);
1648	const dest_ty = cg.typeOfIndex(inst);
1649
1650	if (dest_ty.zigTypeTag(cg.pt.zcu) == .vector) {
1651	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1652	}
1653
1654	const src_float_ty: FloatType = .fromType(cg, src_ty);
1655	const dest_float_ty: FloatType = .fromType(cg, dest_ty);
1656
1657	const result = switch (tag) {
1658	.fptrunc => try cg.floatTruncCast(dest_float_ty, src_float_ty, operand),
1659	.fpext => try cg.floatExtendCast(dest_float_ty, src_float_ty, operand),
1660	else => unreachable,
1661	};
1662
1663	try cg.finishAir(inst, result, &.{ty_op.operand});
1664	},
1665
1666	.int_from_float => {
1667	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1668	const operand = try cg.resolveInst(ty_op.operand);
1669	const src_ty = cg.typeOf(ty_op.operand);
1670	const dest_ty = cg.typeOfIndex(inst);
1671
1672	if (src_ty.zigTypeTag(zcu) == .vector) {
1673	return cg.fail("TODO: implement AIR op: int_from_float for vectors", .{});
1674	}
1675
1676	const result = try cg.intFromFloat(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1677	try cg.finishAir(inst, result, &.{ty_op.operand});
1678	},
1679	.float_from_int => {
1680	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1681	const operand = try cg.resolveInst(ty_op.operand);
1682	const src_ty = cg.typeOf(ty_op.operand);
1683	const dest_ty = cg.typeOfIndex(inst);
1684
1685	if (src_ty.zigTypeTag(zcu) == .vector) {
1686	return cg.fail("TODO: implement AIR op: float_from_int for vectors", .{});
1687	}
1688
1689	const result = try cg.floatFromInt(.fromType(cg, dest_ty), .fromType(cg, src_ty), operand);
1690	try cg.finishAir(inst, result, &.{ty_op.operand});
1691	},
1692
1693	.clz, .ctz, .popcount, .byte_swap, .bit_reverse => |tag| {
1694	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
1695	const operand = try cg.resolveInst(ty_op.operand);
1696
1697	const ty = cg.typeOf(ty_op.operand);
1698
1699	if (ty.zigTypeTag(zcu) == .vector) {
1700	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1701	}
1702
1703	const int_ty: IntType = .fromType(cg, ty);
1704	const result = switch (tag) {
1705	.clz => try cg.intClz(int_ty, operand),
1706	.ctz => try cg.intCtz(int_ty, operand),
1707	.popcount => try cg.intPopCount(int_ty, operand),
1708	.byte_swap => try cg.intByteSwap(int_ty, operand),
1709	.bit_reverse => try cg.intBitReverse(int_ty, operand),
1710	else => unreachable,
1711	};
1712	try cg.finishAir(inst, result, &.{ty_op.operand});
1713	},
1714
1715	.sqrt, .sin, .cos, .tan, .exp, .exp2, .log, .log2, .log10, .floor, .ceil, .round, .trunc_float, .neg => |tag| {
1716	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
1717	const operand = try cg.resolveInst(un_op);
1718	const ty = cg.typeOfIndex(inst);
1719
1720	if (ty.zigTypeTag(zcu) == .vector) {
1721	return cg.fail("TODO: implement AIR op: {s} for vectors", .{@tagName(tag)});
1722	}
1723
1724	const float_ty: FloatType = .fromType(cg, ty);
1725	const result = switch (tag) {
1726	.sqrt => try cg.floatSqrt(float_ty, operand),
1727	.sin => try cg.floatSin(float_ty, operand),
1728	.cos => try cg.floatCos(float_ty, operand),
1729	.tan => try cg.floatTan(float_ty, operand),
1730	.exp => try cg.floatExp(float_ty, operand),
1731	.exp2 => try cg.floatExp2(float_ty, operand),
1732	.log => try cg.floatLog(float_ty, operand),
1733	.log2 => try cg.floatLog2(float_ty, operand),
1734	.log10 => try cg.floatLog10(float_ty, operand),
1735	.floor => try cg.floatFloor(float_ty, operand),
1736	.ceil => try cg.floatCeil(float_ty, operand),
1737	.round => try cg.floatRound(float_ty, operand),
1738	.trunc_float => try cg.floatTrunc(float_ty, operand),
1739	.neg => try cg.floatNeg(float_ty, operand),
1740	else => unreachable,
1741	};
1742
1743	try cg.finishAir(inst, result, &.{un_op});
1744	},
1745
1746	.cmp_eq => cg.airCmp(inst, .eq),
1747	.cmp_gte => cg.airCmp(inst, .gte),
1748	.cmp_gt => cg.airCmp(inst, .gt),
1749	.cmp_lte => cg.airCmp(inst, .lte),
1750	.cmp_lt => cg.airCmp(inst, .lt),
1751	.cmp_neq => cg.airCmp(inst, .neq),
1752
1753	.cmp_vector => cg.airCmpVector(inst),
1754	.cmp_lte_errors_len => cg.airCmpLteErrorsLen(inst),
1755
1756	.array_elem_val => cg.airArrayElemVal(inst),
1757	.array_to_slice => cg.airArrayToSlice(inst),
1758	.array_to_vector => unreachable, // legalize .expand_array_to_vector
1759	.alloc => cg.airAlloc(inst),
1760	.arg => cg.airArg(inst),
1761	.block => cg.airBlock(inst),
1762	.trap => cg.airTrap(inst),
1763	.unreach => cg.airUnreachable(inst),
1764	.breakpoint => cg.airBreakpoint(inst),
1765	.br => cg.airBr(inst),
1766	.repeat => cg.airRepeat(inst),
1767	.switch_dispatch => cg.airSwitchDispatch(inst),
1768	.cond_br => cg.airCondBr(inst),
1769
1770	.@"try" => cg.airTry(inst),
1771	.try_cold => cg.airTry(inst),
1772	.try_ptr => cg.airTryPtr(inst),
1773	.try_ptr_cold => cg.airTryPtr(inst),
1774
1775	.dbg_stmt => cg.airDbgStmt(inst),
1776	.dbg_empty_stmt => try cg.finishAir(inst, .none, &.{}),
1777	.dbg_inline_block => cg.airDbgInlineBlock(inst),
1778	.dbg_var_ptr => cg.airDbgVar(inst, .local_var, true),
1779	.dbg_var_val => cg.airDbgVar(inst, .local_var, false),
1780	.dbg_arg_inline => cg.airDbgVar(inst, .arg, false),
1781
1782	.call => cg.airCall(inst, .auto),
1783	.call_always_tail => cg.airCall(inst, .always_tail),
1784	.call_never_tail => cg.airCall(inst, .never_tail),
1785	.call_never_inline => cg.airCall(inst, .never_inline),
1786
1787	.is_err => cg.airIsErr(inst, .i32_ne, .value),
1788	.is_non_err => cg.airIsErr(inst, .i32_eq, .value),
1789	.is_err_ptr => cg.airIsErr(inst, .i32_ne, .ptr),
1790	.is_non_err_ptr => cg.airIsErr(inst, .i32_eq, .ptr),
1791
1792	.is_null => cg.airIsNull(inst, .i32_eq, .value),
1793	.is_non_null => cg.airIsNull(inst, .i32_ne, .value),
1794	.is_null_ptr => cg.airIsNull(inst, .i32_eq, .ptr),
1795	.is_non_null_ptr => cg.airIsNull(inst, .i32_ne, .ptr),
1796
1797	.load => cg.airLoad(inst),
1798	.loop => cg.airLoop(inst),
1799	.memset => cg.airMemset(inst, false),
1800	.memset_safe => cg.airMemset(inst, true),
1801	.optional_payload => cg.airOptionalPayload(inst),
1802	.optional_payload_ptr => cg.airOptionalPayloadPtr(inst),
1803	.optional_payload_ptr_set => cg.airOptionalPayloadPtrSet(inst),
1804	.ptr_add => cg.airPtrBinOp(inst, .add),
1805	.ptr_sub => cg.airPtrBinOp(inst, .sub),
1806	.ptr_elem_ptr => cg.airPtrElemPtr(inst),
1807	.ptr_elem_val => cg.airPtrElemVal(inst),
1808	.ret => cg.airRet(inst),
1809	.ret_safe => cg.airRet(inst), // TODO
1810	.ret_ptr => cg.airRetPtr(inst),
1811	.ret_load => cg.airRetLoad(inst),
1812	.splat => cg.airSplat(inst),
1813	.select => cg.airSelect(inst),
1814	.shuffle_one => cg.airShuffleOne(inst),
1815	.shuffle_two => cg.airShuffleTwo(inst),
1816	.reduce => cg.airReduce(inst),
1817	.aggregate_init => cg.airAggregateInit(inst),
1818	.union_init => cg.airUnionInit(inst),
1819	.prefetch => cg.airPrefetch(inst),
1820
1821	.slice => cg.airSlice(inst),
1822	.slice_len => cg.airSliceLen(inst),
1823	.slice_elem_val => cg.airSliceElemVal(inst),
1824	.slice_elem_ptr => cg.airSliceElemPtr(inst),
1825	.slice_ptr => cg.airSlicePtr(inst),
1826	.ptr_slice_len_ptr => cg.airPtrSliceFieldPtr(inst, cg.ptrSize()),
1827	.ptr_slice_ptr_ptr => cg.airPtrSliceFieldPtr(inst, 0),
1828	.store => cg.airStore(inst, false),
1829	.store_safe => cg.airStore(inst, true),
1830
1831	.set_union_tag => cg.airSetUnionTag(inst),
1832	.get_union_tag => cg.airGetUnionTag(inst),
1833	.struct_field_ptr => cg.airStructFieldPtr(inst),
1834	.struct_field_ptr_index_0 => cg.airStructFieldPtrIndex(inst, 0),
1835	.struct_field_ptr_index_1 => cg.airStructFieldPtrIndex(inst, 1),
1836	.struct_field_ptr_index_2 => cg.airStructFieldPtrIndex(inst, 2),
1837	.struct_field_ptr_index_3 => cg.airStructFieldPtrIndex(inst, 3),
1838	.agg_field_val => cg.airAggFieldVal(inst),
1839	.field_parent_ptr => cg.airFieldParentPtr(inst),
1840
1841	.switch_br => cg.airSwitchBr(inst, false),
1842	.loop_switch_br => cg.airSwitchBr(inst, true),
1843
1844	.wrap_optional => cg.airWrapOptional(inst),
1845	.unwrap_errunion_payload => cg.airUnwrapErrUnionPayload(inst, false),
1846	.unwrap_errunion_payload_ptr => cg.airUnwrapErrUnionPayload(inst, true),
1847	.unwrap_errunion_err => cg.airUnwrapErrUnionError(inst, false),
1848	.unwrap_errunion_err_ptr => cg.airUnwrapErrUnionError(inst, true),
1849	.wrap_errunion_payload => cg.airWrapErrUnionPayload(inst),
1850	.wrap_errunion_err => cg.airWrapErrUnionErr(inst),
1851	.errunion_payload_ptr_set => cg.airErrUnionPayloadPtrSet(inst),
1852	.error_name => cg.airErrorName(inst),
1853
1854	.wasm_memory_size => cg.airWasmMemorySize(inst),
1855	.wasm_memory_grow => cg.airWasmMemoryGrow(inst),
1856
1857	.memcpy => cg.airMemcpy(inst),
1858	.memmove => cg.airMemmove(inst),
1859
1860	.ret_addr => cg.airRetAddr(inst),
1861	.tag_name => cg.airTagName(inst),
1862
1863	.error_set_has_value => cg.airErrorSetHasValue(inst),
1864	.frame_addr => cg.airFrameAddress(inst),
1865
1866	.runtime_nav_ptr => cg.airRuntimeNavPtr(inst),
1867
1868	.assembly => cg.airAsm(inst),
1869
1870	.c_va_arg => try cg.airVaArg(inst),
1871	.c_va_copy => try cg.airVaCopy(inst),
1872	.c_va_end => try cg.airVaEnd(inst),
1873	.c_va_start => try cg.airVaStart(inst),
1874
1875	.is_named_enum_value => try cg.airIsNamedEnumValue(inst),
1876
1877	.err_return_trace,
1878	.set_err_return_trace,
1879	.save_err_return_trace_index,
1880	.addrspace_cast,
1881	=> |tag| return cg.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}),
1882
1883	.atomic_load => cg.airAtomicLoad(inst),
1884	.atomic_store_unordered,
1885	.atomic_store_monotonic,
1886	.atomic_store_release,
1887	.atomic_store_seq_cst,
1888	// in WebAssembly, all atomic instructions are sequentially ordered.
1889	=> cg.airAtomicStore(inst),
1890	.atomic_rmw => cg.airAtomicRmw(inst),
1891	.cmpxchg_weak => cg.airCmpxchg(inst),
1892	.cmpxchg_strong => cg.airCmpxchg(inst),
1893
1894	.add_optimized,
1895	.sub_optimized,
1896	.mul_optimized,
1897	.div_float_optimized,
1898	.div_trunc_optimized,
1899	.div_floor_optimized,
1900	.div_ceil_optimized,
1901	.div_exact_optimized,
1902	.rem_optimized,
1903	.mod_optimized,
1904	.neg_optimized,
1905	.cmp_lt_optimized,
1906	.cmp_lte_optimized,
1907	.cmp_eq_optimized,
1908	.cmp_gte_optimized,
1909	.cmp_gt_optimized,
1910	.cmp_neq_optimized,
1911	.cmp_vector_optimized,
1912	.reduce_optimized,
1913	.int_from_float_optimized,
1914	=> return cg.fail("TODO implement optimized float mode", .{}),
1915
1916	.add_safe,
1917	.sub_safe,
1918	.mul_safe,
1919	.bit_cast_safe,
1920	.int_cast_safe,
1921	.int_from_float_safe,
1922	.int_from_float_optimized_safe,
1923	=> return cg.fail("TODO implement safety_checked_instructions", .{}),
1924
1925	.work_item_id,
1926	.work_group_size,
1927	.work_group_id,
1928	.spirv_runtime_array_len,
1929	=> unreachable,
1930	};
1931	}
1932
1933	fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1934	const zcu = cg.pt.zcu;
1935	const ip = &zcu.intern_pool;
1936
1937	for (body) |inst| {
1938	if (cg.liveness.isUnused(inst) and !cg.air.mustLower(inst, ip)) {
1939	continue;
1940	}
1941	const old_bookkeeping_value = cg.air_bookkeeping;
1942	try cg.currentBranch().values.ensureUnusedCapacity(cg.gpa, 1);
1943	try cg.genInst(inst);
1944
1945	if (std.debug.runtime_safety and cg.air_bookkeeping < old_bookkeeping_value + 1) {
1946	std.debug.panic("Missing call to `finishAir` in AIR instruction %{d} ('{t}')", .{
1947	inst,
1948	cg.air.instructions.items(.tag)[@backingInt(inst)],
1949	});
1950	}
1951	}
1952	}
1953
1954	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1955	const zcu = cg.pt.zcu;
1956	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
1957	const operand = try cg.resolveInst(un_op);
1958	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
1959	const ret_ty = Type.fromInterned(fn_info.return_type);
1960
1961	// result must be stored in the stack and we return a pointer
1962	// to the stack instead
1963	if (cg.return_value != .none) {
1964	try cg.store(cg.return_value, operand, ret_ty, 0);
1965	} else if (fn_info.cc == .wasm_mvp and ret_ty.hasRuntimeBits(zcu)) {
1966	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
1967	.direct => |scalar_type| {
1968	if (!isByRef(ret_ty, zcu, cg.target)) {
1969	try cg.emitWValue(operand);
1970	} else {
1971	_ = try cg.load(operand, scalar_type, 0);
1972	}
1973	},
1974	.double_i64, .indirect => unreachable,
1975	.unrolled => |vector| {
1976	assert(vector.len == 1);
1977	_ = try cg.load(operand, vector.elem_type, 0);
1978	},
1979	}
1980	} else {
1981	if (!ret_ty.hasRuntimeBits(zcu) and ret_ty.isError(zcu)) {
1982	try cg.addImm32(0);
1983	} else {
1984	try cg.emitWValue(operand);
1985	}
1986	}
1987	try cg.restoreStackPointer();
1988	try cg.addTag(.@"return");
1989
1990	return cg.finishAir(inst, .none, &.{un_op});
1991	}
1992
1993	fn airRetPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1994	const zcu = cg.pt.zcu;
1995	const child_type = cg.typeOfIndex(inst).childType(zcu);
1996
1997	const result = result: {
1998	if (!child_type.hasRuntimeBits(zcu)) {
1999	break :result try cg.allocStack(Type.usize); // create pointer to void
2000	}
2001
2002	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
2003	if (firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2004	break :result cg.return_value;
2005	}
2006
2007	break :result try cg.allocStackPtr(inst);
2008	};
2009
2010	return cg.finishAir(inst, result, &.{});
2011	}
2012
2013	fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2014	const zcu = cg.pt.zcu;
2015	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
2016	const operand = try cg.resolveInst(un_op);
2017	const ret_ty = cg.typeOf(un_op).childType(zcu);
2018
2019	const fn_info = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?;
2020	if (!ret_ty.hasRuntimeBits(zcu)) {
2021	if (ret_ty.isError(zcu)) {
2022	try cg.addImm32(0);
2023	}
2024	} else if (!firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target)) {
2025	if (fn_info.cc == .wasm_mvp) {
2026	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2027	.direct => |scalar_type| _ = try cg.load(operand, scalar_type, 0),
2028	.double_i64, .indirect => unreachable,
2029	.unrolled => |vector| {
2030	assert(vector.len == 1);
2031	_ = try cg.load(operand, vector.elem_type, 0);
2032	},
2033	}
2034	} else {
2035	_ = try cg.load(operand, ret_ty, 0);
2036	}
2037	}
2038
2039	try cg.restoreStackPointer();
2040	try cg.addTag(.@"return");
2041	return cg.finishAir(inst, .none, &.{un_op});
2042	}
2043
2044	fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier) InnerError!void {
2045	if (modifier == .always_tail) return cg.fail("TODO implement tail calls for wasm", .{});
2046	const call = cg.air.unwrapCall(inst);
2047	const args = call.args;
2048	const ty = cg.typeOf(call.callee);
2049
2050	const pt = cg.pt;
2051	const zcu = pt.zcu;
2052	const ip = &zcu.intern_pool;
2053	const fn_ty = switch (ty.zigTypeTag(zcu)) {
2054	.@"fn" => ty,
2055	.pointer => ty.childType(zcu),
2056	else => unreachable,
2057	};
2058	const ret_ty = fn_ty.fnReturnType(zcu);
2059	const fn_info = zcu.typeToFunc(fn_ty).?;
2060	const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target);
2061
2062	const callee: ?InternPool.Nav.Index = blk: {
2063	const func_val: Value = .fromInterned(call.callee.toInterned() orelse break :blk null);
2064
2065	switch (ip.indexToKey(func_val.toIntern())) {
2066	inline .func, .@"extern" => |x| break :blk x.owner_nav,
2067	.ptr => |ptr| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
2068	.nav => |nav| break :blk nav,
2069	else => {},
2070	},
2071	else => {},
2072	}
2073	return cg.fail("unable to lower callee to a function index", .{});
2074	};
2075
2076	const sret: WValue = if (first_param_sret)
2077	try cg.allocStack(ret_ty)
2078	else
2079	.none;
2080
2081	const fixed_arg_count = fn_info.param_types.len;
2082
2083	const varargs_buf: WValue = if (fn_info.is_var_args) buf: {
2084	var varargs_size: u32 = 0;
2085	var varargs_align: Alignment = .fromByteUnits(1);
2086
2087	for (args[fixed_arg_count..]) |arg| {
2088	const arg_ty = cg.typeOf(arg);
2089	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2090
2091	const arg_size = std.math.cast(u32, arg_ty.abiSize(zcu)) orelse {
2092	return cg.fail("argument type {f} too large for wasm varargs buffer", .{arg_ty.fmt(pt)});
2093	};
2094	const arg_align = arg_ty.abiAlignment(zcu);
2095
2096	varargs_align = varargs_align.max(arg_align);
2097	varargs_size = @intCast(arg_align.forward(varargs_size));
2098	varargs_size += arg_size;
2099	}
2100
2101	if (varargs_size == 0) varargs_size = 1;
2102
2103	const buffer = try cg.allocStackBytes(varargs_size, varargs_align);
2104
2105	var offset: u32 = 0;
2106	for (args[fixed_arg_count..]) |arg| {
2107	const arg_ty = cg.typeOf(arg);
2108	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2109
2110	const arg_val = try cg.resolveInst(arg);
2111	const arg_size = std.math.cast(u32, arg_ty.abiSize(zcu)) orelse {
2112	return cg.fail("argument type {f} too large for wasm varargs buffer", .{arg_ty.fmt(pt)});
2113	};
2114	const arg_align = arg_ty.abiAlignment(zcu);
2115
2116	offset = @intCast(arg_align.forward(offset));
2117	try cg.store(buffer, arg_val, arg_ty, offset);
2118	offset += arg_size;
2119	}
2120
2121	break :buf buffer;
2122	} else .none;
2123
2124	if (first_param_sret) {
2125	try cg.lowerToStack(sret);
2126	}
2127
2128	for (args, 0..) |arg, arg_i| {
2129	if (fn_info.is_var_args and arg_i >= fixed_arg_count) break;
2130
2131	const arg_ty = cg.typeOf(arg);
2132	if (!arg_ty.hasRuntimeBits(zcu)) continue;
2133
2134	const arg_val = try cg.resolveInst(arg);
2135	try cg.lowerArg(fn_info.cc, arg_ty, arg_val);
2136	}
2137
2138	if (fn_info.is_var_args) {
2139	try cg.lowerToStack(varargs_buf);
2140	}
2141
2142	if (callee) |nav_index| {
2143	try cg.addInst(.{ .tag = .call_nav, .data = .{ .nav_index = nav_index } });
2144	} else {
2145	// in this case we call a function pointer
2146	// so load its value onto the stack
2147	assert(ty.zigTypeTag(zcu) == .pointer);
2148	const operand = try cg.resolveInst(call.callee);
2149	try cg.emitWValue(operand);
2150
2151	try cg.mir_func_tys.put(cg.gpa, fn_ty.toIntern(), {});
2152	try cg.addInst(.{
2153	.tag = .call_indirect,
2154	.data = .{ .ip_index = fn_ty.toIntern() },
2155	});
2156	}
2157
2158	const result_value = result_value: {
2159	if (!ret_ty.hasRuntimeBits(zcu) and !ret_ty.isError(zcu)) {
2160	break :result_value .none;
2161	} else if (first_param_sret) {
2162	break :result_value sret;
2163	} else if (zcu.typeToFunc(fn_ty).?.cc == .wasm_mvp) {
2164	switch (abi.classifyType(ret_ty, zcu, cg.target)) {
2165	.direct => |scalar_type| {
2166	if (!isByRef(ret_ty, zcu, cg.target)) {
2167	const result_local = try cg.allocLocal(ret_ty);
2168	try cg.addLocal(.local_set, result_local.local.value);
2169	break :result_value result_local;
2170	} else {
2171	const result_local = try cg.allocLocal(scalar_type);
2172	try cg.addLocal(.local_set, result_local.local.value);
2173	const result = try cg.allocStack(ret_ty);
2174	try cg.store(result, result_local, scalar_type, 0);
2175	break :result_value result;
2176	}
2177	},
2178	.double_i64, .indirect => unreachable,
2179	.unrolled => |vector| {
2180	assert(vector.len == 1);
2181	const result_local = try cg.allocLocal(vector.elem_type);
2182	// save call result from operand stack
2183	try cg.addLocal(.local_set, result_local.local.value);
2184	const result = try cg.allocStack(ret_ty);
2185	try cg.store(result, result_local, vector.elem_type, 0);
2186	break :result_value result;
2187	},
2188	}
2189	} else {
2190	const result_local = try cg.allocLocal(ret_ty);
2191	try cg.addLocal(.local_set, result_local.local.value);
2192	break :result_value result_local;
2193	}
2194	};
2195
2196	var bt = cg.liveness.iterateBigTomb(inst);
2197	cg.feed(&bt, call.callee);
2198	for (args) |arg| cg.feed(&bt, arg);
2199	return cg.finishAirResult(inst, result_value);
2200	}
2201
2202	fn airVaStart(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2203	try cg.emitWValue(cg.varargs);
2204	return cg.finishAir(inst, .stack, &.{});
2205	}
2206
2207	fn airVaEnd(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2208	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
2209	return cg.finishAir(inst, .none, &.{un_op});
2210	}
2211
2212	fn airVaCopy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2213	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2214	const operand = try cg.resolveInst(ty_op.operand);
2215
2216	const result = try cg.load(operand, .usize, 0);
2217
2218	return cg.finishAir(inst, result, &.{ty_op.operand});
2219	}
2220
2221	fn airVaArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2222	const zcu = cg.pt.zcu;
2223	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2224	const operand = try cg.resolveInst(ty_op.operand);
2225
2226	const ty = cg.typeOfIndex(inst);
2227
2228	if (!ty.hasRuntimeBits(zcu)) {
2229	return cg.finishAir(inst, .none, &.{ty_op.operand});
2230	}
2231
2232	const is_f32_va_arg = ty.toIntern() == .f32_type;
2233	const load_ty: Type = if (is_f32_va_arg) Type.f64 else ty;
2234
2235	const abi_size: u32 = @intCast(load_ty.abiSize(zcu));
2236	const abi_align: u32 = @intCast(load_ty.abiAlignment(zcu).toByteUnits().?);
2237
2238	const arg_ptr = try cg.allocLocal(.usize);
2239	_ = try cg.load(operand, .usize, 0);
2240
2241	if (abi_align > 1) {
2242	switch (cg.ptr_size) {
2243	.wasm32 => {
2244	try cg.addImm32(abi_align - 1);
2245	try cg.addTag(.i32_add);
2246	try cg.addImm32(~(abi_align - 1));
2247	try cg.addTag(.i32_and);
2248	},
2249	.wasm64 => {
2250	try cg.addImm64(abi_align - 1);
2251	try cg.addTag(.i64_add);
2252	try cg.addImm64(~@as(u64, abi_align - 1));
2253	try cg.addTag(.i64_and);
2254	},
2255	}
2256	}
2257
2258	try cg.addLocal(.local_set, arg_ptr.local.value);
2259
2260	try cg.lowerToStack(operand);
2261	try cg.lowerToStack(arg_ptr);
2262	switch (cg.ptr_size) {
2263	.wasm32 => {
2264	try cg.addImm32(abi_size);
2265	try cg.addTag(.i32_add);
2266	},
2267	.wasm64 => {
2268	try cg.addImm64(abi_size);
2269	try cg.addTag(.i64_add);
2270	},
2271	}
2272	try cg.store(.stack, .stack, .usize, 0);
2273
2274	const result = if (is_f32_va_arg) result: {
2275	const promoted = try cg.load(arg_ptr, Type.f64, 0);
2276	try cg.emitWValue(promoted);
2277	try cg.addTag(.f32_demote_f64);
2278
2279	const result_local = try cg.allocLocal(Type.f32);
2280	try cg.addLocal(.local_set, result_local.local.value);
2281	break :result result_local;
2282	} else try cg.load(arg_ptr, ty, 0);
2283
2284	return cg.finishAir(inst, result, &.{ty_op.operand});
2285	}
2286
2287	fn airAlloc(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2288	const value = try cg.allocStackPtr(inst);
2289	return cg.finishAir(inst, value, &.{});
2290	}
2291
2292	fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2293	const pt = cg.pt;
2294	const zcu = pt.zcu;
2295	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
2296
2297	const lhs = try cg.resolveInst(bin_op.lhs);
2298	const rhs = try cg.resolveInst(bin_op.rhs);
2299	const ptr_ty = cg.typeOf(bin_op.lhs);
2300	const ptr_info = ptr_ty.ptrInfo(zcu);
2301	const elem_ty = ptr_ty.childType(zcu);
2302
2303	if (!safety and bin_op.rhs == .undef) {
2304	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2305	}
2306
2307	const offset: u32 = switch (ptr_info.flags.vector_index) {
2308	.none => offset: {
2309	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
2310	break :offset 0;
2311	},
2312	else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
2313	};
2314	try cg.store(lhs, rhs, elem_ty, offset);
2315	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2316	}
2317
2318	fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerError!void {
2319	assert(!(lhs != .stack and rhs == .stack));
2320	const pt = cg.pt;
2321	const zcu = pt.zcu;
2322	const abi_size = ty.abiSize(zcu);
2323
2324	if (!ty.hasRuntimeBits(zcu)) return;
2325
2326	if (isByRef(ty, zcu, cg.target)) {
2327	const offset_ptr: WValue = switch (offset + lhs.offset()) {
2328	0 => lhs,
2329	else => |total_offset| ptr: {
2330	try cg.emitWValue(lhs);
2331	try cg.addImm32(total_offset);
2332	try cg.addTag(.i32_add);
2333	break :ptr .stack;
2334	},
2335	};
2336	return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2337	}
2338
2339	if (ty.zigTypeTag(zcu) == .vector) {
2340	try cg.emitWValue(lhs);
2341	try cg.lowerToStack(rhs);
2342	// TODO: Add helper functions for simd opcodes
2343	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2344	// stores as := opcode, offset, alignment (opcode::memarg)
2345	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2346	@backingInt(std.wasm.SimdOpcode.v128_store),
2347	offset + lhs.offset(),
2348	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2349	});
2350	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2351	}
2352
2353	const store_opcode: Mir.Inst.Tag = opcode: {
2354	if (ty.isAnyFloat()) {
2355	break :opcode switch (abi_size) {
2356	2 => .i32_store16,
2357	4 => .f32_store,
2358	8 => .f64_store,
2359	else => unreachable,
2360	};
2361	} else {
2362	break :opcode switch (abi_size) {
2363	1 => .i32_store8,
2364	2 => .i32_store16,
2365	4 => .i32_store,
2366	8 => .i64_store,
2367	else => unreachable,
2368	};
2369	}
2370	};
2371
2372	try cg.emitWValue(lhs);
2373	try cg.lowerToStack(rhs);
2374
2375	try cg.addMemArg(
2376	store_opcode,
2377	.{
2378	.offset = offset + lhs.offset(),
2379	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2380	},
2381	);
2382	}
2383
2384	fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2385	const pt = cg.pt;
2386	const zcu = pt.zcu;
2387	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
2388	const operand = try cg.resolveInst(ty_op.operand);
2389	const elem_ty = ty_op.ty;
2390	const ptr_ty = cg.typeOf(ty_op.operand);
2391	const ptr_info = ptr_ty.ptrInfo(zcu);
2392
2393	assert(elem_ty.hasRuntimeBits(zcu));
2394
2395	const offset: u32 = switch (ptr_info.flags.vector_index) {
2396	.none => offset: {
2397	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2398	break :offset 0;
2399	},
2400	else => |index| @intCast(@backingInt(index) * elem_ty.abiSize(zcu)),
2401	};
2402	const result = try cg.load(operand, elem_ty, offset);
2403	return cg.finishAir(inst, result, &.{ty_op.operand});
2404	}
2405
2406	/// Loads an operand from the linear memory section.
2407	/// NOTE: Leaves the value on the stack, if isByRef == false.
2408	fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2409	const zcu = cg.pt.zcu;
2410	if (isByRef(ty, zcu, cg.target)) {
2411	const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
2412	0 => operand,
2413	else => |total_offset| ptr: {
2414	try cg.emitWValue(operand);
2415	try cg.addImm32(total_offset);
2416	try cg.addTag(.i32_add);
2417	break :ptr .stack;
2418	},
2419	};
2420	const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2421	const new_ptr = try cg.allocStack(ty);
2422	try cg.store(new_ptr, src_ptr, ty, 0);
2423	return new_ptr;
2424	}
2425
2426	// load local's value from memory by its stack position
2427	try cg.emitWValue(operand);
2428
2429	if (ty.zigTypeTag(zcu) == .vector) {
2430	// TODO: Add helper functions for simd opcodes
2431	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
2432	// stores as := opcode, offset, alignment (opcode::memarg)
2433	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2434	@backingInt(std.wasm.SimdOpcode.v128_load),
2435	offset + operand.offset(),
2436	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2437	});
2438	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2439	return .stack;
2440	}
2441
2442	const abi_size = ty.abiSize(zcu);
2443	const load_opcode: Mir.Inst.Tag = opcode: {
2444	if (ty.isAnyFloat()) {
2445	break :opcode switch (abi_size) {
2446	2 => .i32_load16_u,
2447	4 => .f32_load,
2448	8 => .f64_load,
2449	else => unreachable,
2450	};
2451	} else {
2452	const is_signed = if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness == .signed else false;
2453	break :opcode switch (abi_size) {
2454	1 => if (is_signed) .i32_load8_s else .i32_load8_u,
2455	2 => if (is_signed) .i32_load16_s else .i32_load16_u,
2456	4 => .i32_load,
2457	8 => .i64_load,
2458	else => unreachable,
2459	};
2460	}
2461	};
2462
2463	try cg.addMemArg(
2464	load_opcode,
2465	.{
2466	.offset = offset + operand.offset(),
2467	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2468	},
2469	);
2470
2471	if (ty.isAbiInt(zcu)) {
2472	const int_info: IntType = .fromType(cg, ty);
2473	switch (int_info.bits) {
2474	8, 16, 32, 64 => {},
2475	else => _ = try cg.intWrap(int_info, .stack),
2476	}
2477	}
2478
2479	return .stack;
2480	}
2481
2482	fn airArg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2483	const pt = cg.pt;
2484	const zcu = pt.zcu;
2485	const arg_index = cg.arg_index;
2486	const arg = cg.args[arg_index];
2487	const cc = zcu.typeToFunc(zcu.navValue(cg.owner_nav).typeOf(zcu)).?.cc;
2488	const arg_ty = cg.typeOfIndex(inst);
2489	if (cc == .wasm_mvp) {
2490	switch (abi.classifyType(arg_ty, zcu, cg.target)) {
2491	.direct => |scalar_type| {
2492	cg.arg_index += 1;
2493	if (isByRef(arg_ty, zcu, cg.target)) {
2494	const result = try cg.allocStack(arg_ty);
2495	try cg.store(result, arg, scalar_type, 0);
2496	return cg.finishAir(inst, result, &.{});
2497	}
2498	},
2499	.indirect => cg.arg_index += 1,
2500	.double_i64 => {
2501	cg.arg_index += 2;
2502	const result = try cg.allocStack(arg_ty);
2503	try cg.store(result, arg, Type.u64, 0);
2504	try cg.store(result, cg.args[arg_index + 1], Type.u64, 8);
2505	return cg.finishAir(inst, result, &.{});
2506	},
2507	.unrolled => |vector| {
2508	const result = try cg.allocStack(arg_ty);
2509	const elem_size: u32 = @intCast(vector.elem_type.abiSize(zcu));
2510	for (0..vector.len) |index| {
2511	try cg.store(result, cg.args[cg.arg_index], vector.elem_type, @intCast(index * elem_size));
2512	cg.arg_index += 1;
2513	}
2514	return cg.finishAir(inst, result, &.{});
2515	},
2516	}
2517	} else {
2518	cg.arg_index += 1;
2519	}
2520
2521	return cg.finishAir(inst, arg, &.{});
2522	}
2523
2524	const IntType = struct {
2525	is_signed: bool,
2526	bits: u16,
2527
2528	const @"i32": IntType = .{ .is_signed = true, .bits = 32 };
2529	const @"i64": IntType = .{ .is_signed = true, .bits = 64 };
2530	const @"u32": IntType = .{ .is_signed = false, .bits = 32 };
2531	const @"u64": IntType = .{ .is_signed = false, .bits = 64 };
2532
2533	// Adapted from x86_64 backend
2534	// Differ from Type.intInfo as it treats pointers/booleans/packed/enums/errors as integer
2535	fn fromType(cg: *CodeGen, ty: Type) IntType {
2536	const zcu = cg.pt.zcu;
2537	const ip = &zcu.intern_pool;
2538	var ty_index = ty.ip_index;
2539	while (true) switch (ip.indexToKey(ty_index)) {
2540	.int_type => |int_type| return .{ .is_signed = int_type.signedness == .signed, .bits = int_type.bits },
2541	.ptr_type => |ptr_type| return switch (ptr_type.flags.size) {
2542	.one, .many, .c => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2543	.slice => unreachable,
2544	},
2545	.opt_type => |opt_child| return if (!Type.fromInterned(opt_child).hasRuntimeBits(zcu))
2546	.{ .is_signed = false, .bits = 1 }
2547	else switch (ip.indexToKey(opt_child)) {
2548	.ptr_type => |ptr_type| switch (ptr_type.flags.size) {
2549	.one, .many => switch (ptr_type.flags.is_allowzero) {
2550	false => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2551	true => unreachable,
2552	},
2553	.slice, .c => unreachable,
2554	},
2555	else => unreachable,
2556	},
2557	.error_union_type => |error_union_type| return if (!Type.fromInterned(error_union_type.payload_type)
2558	.hasRuntimeBits(zcu)) .{ .is_signed = false, .bits = zcu.errorSetBits() } else unreachable,
2559	.simple_type => |simple_type| return switch (simple_type) {
2560	.bool => .{ .is_signed = false, .bits = 1 },
2561	.anyerror, .adhoc_inferred_error_set => .{ .is_signed = false, .bits = zcu.errorSetBits() },
2562	.isize => .{ .is_signed = true, .bits = cg.target.ptrBitWidth() },
2563	.usize => .{ .is_signed = false, .bits = cg.target.ptrBitWidth() },
2564	.c_char => .{ .is_signed = cg.target.cCharSignedness().? == .signed, .bits = cg.target.cTypeBitSize(.char).? },
2565	.c_short => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.short).? },
2566	.c_ushort => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.short).? },
2567	.c_int => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.int).? },
2568	.c_uint => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.int).? },
2569	.c_long => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.long).? },
2570	.c_ulong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.long).? },
2571	.c_longlong => .{ .is_signed = true, .bits = cg.target.cTypeBitSize(.longlong).? },
2572	.c_ulonglong => .{ .is_signed = false, .bits = cg.target.cTypeBitSize(.longlong).? },
2573	.f16, .f32, .f64, .f80, .f128, .c_longdouble => unreachable,
2574	.anyopaque, .void, .type, .comptime_int, .comptime_float, .noreturn, .null, .undefined, .enum_literal, .generic_poison => unreachable,
2575	},
2576	.enum_type,
2577	.struct_type,
2578	.union_type,
2579	=> ty_index = Type.fromInterned(ty_index).backingIntType(zcu).toIntern(),
2580	.error_set_type, .inferred_error_set_type => return .{ .is_signed = false, .bits = zcu.errorSetBits() },
2581	else => unreachable,
2582	};
2583	}
2584	};
2585
2586	fn intBackingBits(cg: *CodeGen, bits: u16) u16 {
2587	return switch (bits) {
2588	0 => unreachable,
2589	1...32 => 32,
2590	33...64 => 64,
2591	else => std.zig.target.intByteSize(cg.target, bits) * 8,
2592	};
2593	}
2594
2595	fn intAdd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2596	switch (ty.bits) {
2597	0 => unreachable,
2598	1...32 => {
2599	try cg.emitWValue(lhs);
2600	try cg.emitWValue(rhs);
2601	try cg.addTag(.i32_add);
2602	return .stack;
2603	},
2604	33...64 => {
2605	try cg.emitWValue(lhs);
2606	try cg.emitWValue(rhs);
2607	try cg.addTag(.i64_add);
2608	return .stack;
2609	},
2610	65...128 => {
2611	const result = try cg.allocStack(Type.u128);
2612
2613	var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2614	defer lhs_lsb.free(cg);
2615	var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2616	defer rhs_lsb.free(cg);
2617	var op_lsb = try (try cg.intAdd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2618	defer op_lsb.free(cg);
2619
2620	const lhs_msb = try cg.load(lhs, Type.u64, 8);
2621	const rhs_msb = try cg.load(rhs, Type.u64, 8);
2622	const op_msb = try cg.intAdd(.u64, lhs_msb, rhs_msb);
2623
2624	const lt = try cg.intCmp(.u64, .lt, op_lsb, rhs_lsb);
2625	const tmp = try cg.intCast(.u64, .u32, lt);
2626	var tmp_op = try (try cg.intAdd(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2627	defer tmp_op.free(cg);
2628
2629	try cg.store(result, op_lsb, Type.u64, 0);
2630	try cg.store(result, tmp_op, Type.u64, 8);
2631	return result;
2632	},
2633	else => {
2634	const result = try cg.allocInt(ty);
2635
2636	try cg.lowerToStack(result);
2637	try cg.lowerToStack(lhs);
2638	try cg.lowerToStack(rhs);
2639	try cg.addImm32(@intFromBool(ty.is_signed));
2640	try cg.addImm32(ty.bits);
2641	try cg.addCallIntrinsic(.__addo_limb64);
2642	try cg.addTag(.drop);
2643	return result;
2644	},
2645	}
2646	}
2647
2648	fn intSub(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2649	switch (ty.bits) {
2650	0 => unreachable,
2651	1...32 => {
2652	try cg.emitWValue(lhs);
2653	try cg.emitWValue(rhs);
2654	try cg.addTag(.i32_sub);
2655	return .stack;
2656	},
2657	33...64 => {
2658	try cg.emitWValue(lhs);
2659	try cg.emitWValue(rhs);
2660	try cg.addTag(.i64_sub);
2661	return .stack;
2662	},
2663	65...128 => {
2664	const result = try cg.allocStack(Type.u128);
2665
2666	var lhs_lsb = try (try cg.load(lhs, Type.u64, 0)).toLocal(cg, Type.u64);
2667	defer lhs_lsb.free(cg);
2668	var rhs_lsb = try (try cg.load(rhs, Type.u64, 0)).toLocal(cg, Type.u64);
2669	defer rhs_lsb.free(cg);
2670	var op_lsb = try (try cg.intSub(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
2671	defer op_lsb.free(cg);
2672
2673	const lhs_msb = try cg.load(lhs, Type.u64, 8);
2674	const rhs_msb = try cg.load(rhs, Type.u64, 8);
2675	const op_msb = try cg.intSub(.u64, lhs_msb, rhs_msb);
2676
2677	const lt = try cg.intCmp(.u64, .lt, lhs_lsb, rhs_lsb);
2678	const tmp = try cg.intCast(.u64, .u32, lt);
2679	var tmp_op = try (try cg.intSub(.u64, op_msb, tmp)).toLocal(cg, Type.u64);
2680	defer tmp_op.free(cg);
2681
2682	try cg.store(result, op_lsb, Type.u64, 0);
2683	try cg.store(result, tmp_op, Type.u64, 8);
2684	return result;
2685	},
2686	else => {
2687	const result = try cg.allocInt(ty);
2688
2689	try cg.lowerToStack(result);
2690	try cg.lowerToStack(lhs);
2691	try cg.lowerToStack(rhs);
2692	try cg.addImm32(@intFromBool(ty.is_signed));
2693	try cg.addImm32(ty.bits);
2694	try cg.addCallIntrinsic(.__subo_limb64);
2695	try cg.addTag(.drop);
2696
2697	return result;
2698	},
2699	}
2700	}
2701
2702	fn intMul(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2703	switch (ty.bits) {
2704	0 => unreachable,
2705	1...32 => {
2706	try cg.emitWValue(lhs);
2707	try cg.emitWValue(rhs);
2708	try cg.addTag(.i32_mul);
2709	return .stack;
2710	},
2711	33...64 => {
2712	try cg.emitWValue(lhs);
2713	try cg.emitWValue(rhs);
2714	try cg.addTag(.i64_mul);
2715	return .stack;
2716	},
2717	65...128 => return cg.callIntrinsic(.__multi3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs }),
2718	else => {
2719	const result = try cg.allocInt(ty);
2720
2721	try cg.lowerToStack(result);
2722	try cg.lowerToStack(lhs);
2723	try cg.lowerToStack(rhs);
2724	try cg.addImm32(@intFromBool(ty.is_signed));
2725	try cg.addImm32(ty.bits);
2726	try cg.addCallIntrinsic(.__mulo_limb64);
2727	try cg.addTag(.drop);
2728
2729	return result;
2730	},
2731	}
2732	}
2733
2734	fn intDiv(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2735	switch (ty.bits) {
2736	0 => unreachable,
2737	1...32 => {
2738	try cg.emitWValue(lhs);
2739	try cg.emitWValue(rhs);
2740	try cg.addTag(if (ty.is_signed) .i32_div_s else .i32_div_u);
2741	return .stack;
2742	},
2743	33...64 => {
2744	try cg.emitWValue(lhs);
2745	try cg.emitWValue(rhs);
2746	try cg.addTag(if (ty.is_signed) .i64_div_s else .i64_div_u);
2747	return .stack;
2748	},
2749	65...128 => {
2750	if (ty.is_signed) {
2751	return cg.callIntrinsic(.__divti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2752	} else {
2753	return cg.callIntrinsic(.__udivti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2754	}
2755	},
2756	else => {
2757	const result = try cg.allocInt(ty);
2758	const bits = cg.intBackingBits(ty.bits);
2759	var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
2760	if (ty.is_signed) {
2761	_ = try cg.callIntrinsic(
2762	.__divei5,
2763	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2764	.void,
2765	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2766	);
2767	} else {
2768	_ = try cg.callIntrinsic(
2769	.__udivei5,
2770	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2771	.void,
2772	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2773	);
2774	}
2775	tmp.free(cg);
2776	return result;
2777	},
2778	}
2779	}
2780
2781	fn intDivFloor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2782	if (!ty.is_signed) {
2783	return cg.intDiv(ty, lhs, rhs);
2784	}
2785
2786	switch (ty.bits) {
2787	0 => unreachable,
2788	1...32 => {
2789	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
2790	defer q.free(cg);
2791
2792	const zero: WValue = .{ .imm32 = 0 };
2793
2794	const r = try cg.intRem(ty, lhs, rhs);
2795	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2796	defer r_nonzero.free(cg);
2797
2798	const sign_xor = try cg.intXor(ty, lhs, rhs);
2799	var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2800	defer sign_diff.free(cg);
2801
2802	try cg.emitWValue(q);
2803	const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2804	try cg.emitWValue(need_adjust);
2805	try cg.addTag(.i32_sub);
2806	return .stack;
2807	},
2808	33...64 => {
2809	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
2810	defer q.free(cg);
2811
2812	const zero: WValue = .{ .imm64 = 0 };
2813
2814	const r = try cg.intRem(ty, lhs, rhs);
2815	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2816	defer r_nonzero.free(cg);
2817
2818	const sign_xor = try cg.intXor(ty, lhs, rhs);
2819	var sign_diff = try (try cg.intCmp(ty, .lt, sign_xor, zero)).toLocal(cg, Type.i32);
2820	defer sign_diff.free(cg);
2821
2822	try cg.emitWValue(q);
2823	const need_adjust = try cg.intAnd(.u32, r_nonzero, sign_diff);
2824	try cg.emitWValue(need_adjust);
2825	try cg.addTag(.i64_extend_i32_u);
2826	try cg.addTag(.i64_sub);
2827	return .stack;
2828	},
2829	else => {
2830	const q = try cg.intDiv(ty, lhs, rhs);
2831
2832	const zero = try cg.intZeroValue(ty);
2833
2834	const r = try cg.intRem(ty, lhs, rhs);
2835	_ = try cg.intCmp(ty, .neq, r, zero);
2836
2837	const sign_xor = try cg.intXor(ty, lhs, rhs);
2838	_ = try cg.intCmp(ty, .lt, sign_xor, zero);
2839	var adjust = try (try cg.intAnd(.u32, .stack, .stack)).toLocal(cg, Type.u32);
2840
2841	const adjust_bigint = try cg.intCast(ty, .u32, adjust);
2842	adjust.free(cg);
2843	return try cg.intSub(ty, q, adjust_bigint);
2844	},
2845	}
2846	}
2847
2848	fn intDivCeil(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2849	switch (ty.bits) {
2850	0 => unreachable,
2851	1...32 => {
2852	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i32);
2853	defer q.free(cg);
2854
2855	const zero: WValue = .{ .imm32 = 0 };
2856
2857	const r = try cg.intRem(ty, lhs, rhs);
2858	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2859	defer r_nonzero.free(cg);
2860
2861	if (!ty.is_signed) {
2862	try cg.emitWValue(q);
2863	try cg.emitWValue(r_nonzero);
2864	try cg.addTag(.i32_add);
2865	return .stack;
2866	}
2867
2868	const sign_xor = try cg.intXor(ty, lhs, rhs);
2869	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.i32);
2870	defer same_sign.free(cg);
2871
2872	try cg.emitWValue(q);
2873	const need_adjust = try cg.intAnd(.u32, r_nonzero, same_sign);
2874	try cg.emitWValue(need_adjust);
2875	try cg.addTag(.i32_add);
2876	return .stack;
2877	},
2878	33...64 => {
2879	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.i64);
2880	defer q.free(cg);
2881
2882	const zero: WValue = .{ .imm64 = 0 };
2883
2884	const r = try cg.intRem(ty, lhs, rhs);
2885	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.i32);
2886	defer r_nonzero.free(cg);
2887
2888	if (!ty.is_signed) {
2889	try cg.emitWValue(q);
2890	try cg.emitWValue(r_nonzero);
2891	try cg.addTag(.i64_extend_i32_u);
2892	try cg.addTag(.i64_add);
2893	return .stack;
2894	}
2895
2896	const sign_xor = try cg.intXor(ty, lhs, rhs);
2897	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.i32);
2898	defer same_sign.free(cg);
2899
2900	try cg.emitWValue(q);
2901	const need_adjust = try cg.intAnd(.u32, r_nonzero, same_sign);
2902	try cg.emitWValue(need_adjust);
2903	try cg.addTag(.i64_extend_i32_u);
2904	try cg.addTag(.i64_add);
2905	return .stack;
2906	},
2907	else => {
2908	var q = try (try cg.intDiv(ty, lhs, rhs)).toLocal(cg, Type.usize);
2909	defer q.free(cg);
2910
2911	const zero = try cg.intZeroValue(ty);
2912
2913	const r = try cg.intRem(ty, lhs, rhs);
2914	var r_nonzero = try (try cg.intCmp(ty, .neq, r, zero)).toLocal(cg, Type.u32);
2915	defer r_nonzero.free(cg);
2916
2917	if (!ty.is_signed) {
2918	var adjust_bigint = try (try cg.intCast(ty, .u32, r_nonzero)).toLocal(cg, Type.usize);
2919	defer adjust_bigint.free(cg);
2920
2921	return try cg.intAdd(ty, q, adjust_bigint);
2922	}
2923
2924	const sign_xor = try cg.intXor(ty, lhs, rhs);
2925	var same_sign = try (try cg.intCmp(ty, .gte, sign_xor, zero)).toLocal(cg, Type.u32);
2926	defer same_sign.free(cg);
2927
2928	var adjust = try (try cg.intAnd(.u32, r_nonzero, same_sign)).toLocal(cg, Type.u32);
2929	defer adjust.free(cg);
2930
2931	var adjust_bigint = try (try cg.intCast(ty, .u32, adjust)).toLocal(cg, Type.usize);
2932	defer adjust_bigint.free(cg);
2933
2934	return try cg.intAdd(ty, q, adjust_bigint);
2935	},
2936	}
2937	}
2938
2939	fn intRem(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2940	switch (ty.bits) {
2941	0 => unreachable,
2942	1...32 => {
2943	try cg.emitWValue(lhs);
2944	try cg.emitWValue(rhs);
2945	try cg.addTag(if (ty.is_signed) .i32_rem_s else .i32_rem_u);
2946	return .stack;
2947	},
2948	33...64 => {
2949	try cg.emitWValue(lhs);
2950	try cg.emitWValue(rhs);
2951	try cg.addTag(if (ty.is_signed) .i64_rem_s else .i64_rem_u);
2952	return .stack;
2953	},
2954	65...128 => {
2955	if (ty.is_signed) {
2956	return cg.callIntrinsic(.__modti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2957	} else {
2958	return cg.callIntrinsic(.__umodti3, &.{ .i128_type, .i128_type }, Type.i128, &.{ lhs, rhs });
2959	}
2960	},
2961	else => {
2962	const result = try cg.allocInt(ty);
2963	const bits = cg.intBackingBits(ty.bits);
2964	var tmp = try cg.allocInt(.{ .is_signed = false, .bits = bits * 2 });
2965	if (ty.is_signed) {
2966	_ = try cg.callIntrinsic(
2967	.__modei5,
2968	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2969	.void,
2970	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2971	);
2972	} else {
2973	_ = try cg.callIntrinsic(
2974	.__umodei5,
2975	&.{ .usize_type, .usize_type, .usize_type, .usize_type, .usize_type },
2976	.void,
2977	&.{ result, lhs, rhs, tmp, .{ .imm32 = ty.bits } },
2978	);
2979	}
2980	tmp.free(cg);
2981	return result;
2982	},
2983	}
2984	}
2985
2986	fn intMod(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2987	if (!ty.is_signed) {
2988	return cg.intRem(ty, lhs, rhs);
2989	}
2990
2991	// mod_s(a, b) = rem_s(rem_s(a, b) + b, b)
2992	const rem = try cg.intRem(ty, lhs, rhs);
2993	const sum = try cg.intAdd(ty, rem, rhs);
2994	return cg.intRem(ty, sum, rhs);
2995	}
2996
2997	fn intAnd(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
2998	switch (ty.bits) {
2999	0 => unreachable,
3000	1...32 => {
3001	try cg.emitWValue(lhs);
3002	try cg.emitWValue(rhs);
3003	try cg.addTag(.i32_and);
3004	return .stack;
3005	},
3006	33...64 => {
3007	try cg.emitWValue(lhs);
3008	try cg.emitWValue(rhs);
3009	try cg.addTag(.i64_and);
3010	return .stack;
3011	},
3012	65...128 => {
3013	const result = try cg.allocStack(Type.u128);
3014
3015	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3016	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3017	const and_lsb = try (try cg.intAnd(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3018	try cg.store(result, and_lsb, Type.u64, 0);
3019
3020	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3021	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3022	const and_msb = try (try cg.intAnd(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3023	try cg.store(result, and_msb, Type.u64, 8);
3024
3025	return result;
3026	},
3027	else => {
3028	const result = try cg.allocInt(ty);
3029
3030	try cg.lowerToStack(result);
3031	try cg.lowerToStack(lhs);
3032	try cg.lowerToStack(rhs);
3033	try cg.addImm32(ty.bits);
3034	try cg.addCallIntrinsic(.__and_limb64);
3035
3036	return result;
3037	},
3038	}
3039	}
3040
3041	fn intOr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3042	switch (ty.bits) {
3043	0 => unreachable,
3044	1...32 => {
3045	try cg.emitWValue(lhs);
3046	try cg.emitWValue(rhs);
3047	try cg.addTag(.i32_or);
3048	return .stack;
3049	},
3050	33...64 => {
3051	try cg.emitWValue(lhs);
3052	try cg.emitWValue(rhs);
3053	try cg.addTag(.i64_or);
3054	return .stack;
3055	},
3056	65...128 => {
3057	const result = try cg.allocStack(Type.u128);
3058
3059	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3060	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3061	const or_lsb = try (try cg.intOr(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3062	try cg.store(result, or_lsb, Type.u64, 0);
3063
3064	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3065	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3066	const or_msb = try (try cg.intOr(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3067	try cg.store(result, or_msb, Type.u64, 8);
3068
3069	return result;
3070	},
3071	else => {
3072	const result = try cg.allocInt(ty);
3073
3074	try cg.lowerToStack(result);
3075	try cg.lowerToStack(lhs);
3076	try cg.lowerToStack(rhs);
3077	try cg.addImm32(ty.bits);
3078	try cg.addCallIntrinsic(.__or_limb64);
3079
3080	return result;
3081	},
3082	}
3083	}
3084
3085	fn intXor(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3086	switch (ty.bits) {
3087	0 => unreachable,
3088	1...32 => {
3089	try cg.emitWValue(lhs);
3090	try cg.emitWValue(rhs);
3091	try cg.addTag(.i32_xor);
3092	return .stack;
3093	},
3094	33...64 => {
3095	try cg.emitWValue(lhs);
3096	try cg.emitWValue(rhs);
3097	try cg.addTag(.i64_xor);
3098	return .stack;
3099	},
3100	65...128 => {
3101	const result = try cg.allocStack(Type.u128);
3102
3103	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
3104	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
3105	const xor_lsb = try (try cg.intXor(.u64, lhs_lsb, rhs_lsb)).toLocal(cg, Type.u64);
3106	try cg.store(result, xor_lsb, Type.u64, 0);
3107
3108	const lhs_msb = try cg.load(lhs, Type.u64, 8);
3109	const rhs_msb = try cg.load(rhs, Type.u64, 8);
3110	const xor_msb = try (try cg.intXor(.u64, lhs_msb, rhs_msb)).toLocal(cg, Type.u64);
3111	try cg.store(result, xor_msb, Type.u64, 8);
3112
3113	return result;
3114	},
3115	else => {
3116	const result = try cg.allocInt(ty);
3117
3118	try cg.lowerToStack(result);
3119	try cg.lowerToStack(lhs);
3120	try cg.lowerToStack(rhs);
3121	try cg.addImm32(ty.bits);
3122	try cg.addCallIntrinsic(.__xor_limb64);
3123
3124	return result;
3125	},
3126	}
3127	}
3128
3129	fn intNot(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3130	switch (ty.bits) {
3131	0 => unreachable,
3132	1 => {
3133	try cg.emitWValue(operand);
3134	if (ty.is_signed) {
3135	try cg.addImm32(~@as(u32, 0));
3136	try cg.addTag(.i32_xor);
3137	} else {
3138	try cg.addTag(.i32_eqz);
3139	}
3140	return .stack;
3141	},
3142	2...32 => {
3143	const mask: u32 = if (ty.is_signed)
3144	~@as(u32, 0)
3145	else
3146	~@as(u32, 0) >> @intCast(32 - ty.bits);
3147	try cg.emitWValue(operand);
3148	try cg.addImm32(mask);
3149	try cg.addTag(.i32_xor);
3150	return .stack;
3151	},
3152	33...64 => {
3153	const mask: u64 = if (ty.is_signed)
3154	~@as(u64, 0)
3155	else
3156	~@as(u64, 0) >> @intCast(64 - ty.bits);
3157	try cg.emitWValue(operand);
3158	try cg.addImm64(mask);
3159	try cg.addTag(.i64_xor);
3160	return .stack;
3161	},
3162	65...128 => {
3163	const result = try cg.allocStack(Type.u128);
3164
3165	try cg.emitWValue(result);
3166	_ = try cg.load(operand, Type.u64, 0);
3167	try cg.addImm64(~@as(u64, 0));
3168	try cg.addTag(.i64_xor);
3169	try cg.store(.stack, .stack, Type.u64, result.offset());
3170
3171	try cg.emitWValue(result);
3172	_ = try cg.load(operand, Type.u64, 8);
3173	const high_mask: u64 = if (ty.is_signed)
3174	~@as(u64, 0)
3175	else
3176	~@as(u64, 0) >> @intCast(128 - ty.bits);
3177	try cg.addImm64(high_mask);
3178	try cg.addTag(.i64_xor);
3179	try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
3180
3181	return result;
3182	},
3183	else => {
3184	const result = try cg.allocInt(ty);
3185
3186	try cg.lowerToStack(result);
3187	try cg.lowerToStack(operand);
3188	try cg.addImm32(@intFromBool(ty.is_signed));
3189	try cg.addImm32(ty.bits);
3190	try cg.addCallIntrinsic(.__not_limb64);
3191
3192	return result;
3193	},
3194	}
3195	}
3196
3197	// rhs is a shift count, pointing to i32 value
3198	// does not perform wrapping, padding bits does not satisfy invariant
3199	fn intShl(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3200	switch (ty.bits) {
3201	0 => unreachable,
3202	1...32 => {
3203	try cg.emitWValue(lhs);
3204	try cg.emitWValue(rhs);
3205	try cg.addTag(.i32_shl);
3206	return .stack;
3207	},
3208	33...64 => {
3209	try cg.emitWValue(lhs);
3210	try cg.emitWValue(rhs);
3211	try cg.addTag(.i64_extend_i32_u);
3212	try cg.addTag(.i64_shl);
3213	return .stack;
3214	},
3215	65...128 => return cg.callIntrinsic(.__ashlti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs }),
3216	else => {
3217	const result = try cg.allocInt(ty);
3218
3219	try cg.lowerToStack(result);
3220	try cg.lowerToStack(lhs);
3221	try cg.lowerToStack(rhs);
3222	try cg.addImm32(@intFromBool(ty.is_signed));
3223	try cg.addImm32(ty.bits);
3224	try cg.addCallIntrinsic(.__shlo_limb64);
3225	try cg.addTag(.drop);
3226
3227	return result;
3228	},
3229	}
3230	}
3231
3232	// rhs is a shift count, pointing to i32 value
3233	fn intShr(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3234	switch (ty.bits) {
3235	0 => unreachable,
3236	1...32 => {
3237	try cg.emitWValue(lhs);
3238	try cg.emitWValue(rhs);
3239	try cg.addTag(if (ty.is_signed) .i32_shr_s else .i32_shr_u);
3240	return .stack;
3241	},
3242	33...64 => {
3243	try cg.emitWValue(lhs);
3244	try cg.emitWValue(rhs);
3245	try cg.addTag(.i64_extend_i32_u);
3246	try cg.addTag(if (ty.is_signed) .i64_shr_s else .i64_shr_u);
3247	return .stack;
3248	},
3249	65...128 => {
3250	if (ty.is_signed) {
3251	return cg.callIntrinsic(.__ashrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3252	} else {
3253	return cg.callIntrinsic(.__lshrti3, &.{ .i128_type, .i32_type }, Type.i128, &.{ lhs, rhs });
3254	}
3255	},
3256	else => {
3257	const result = try cg.allocInt(ty);
3258
3259	try cg.lowerToStack(result);
3260	try cg.lowerToStack(lhs);
3261	try cg.lowerToStack(rhs);
3262	try cg.addImm32(@intFromBool(ty.is_signed));
3263	try cg.addImm32(ty.bits);
3264	try cg.addCallIntrinsic(.__shr_limb64);
3265
3266	return result;
3267	},
3268	}
3269	}
3270
3271	fn intAbs(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3272	if (!ty.is_signed) return operand;
3273	switch (ty.bits) {
3274	0 => unreachable,
3275	1...32 => {
3276	try cg.emitWValue(operand);
3277	try cg.addImm32(31);
3278	try cg.addTag(.i32_shr_s);
3279
3280	var mask = try cg.allocLocal(Type.i32);
3281	defer mask.free(cg);
3282	try cg.addLocal(.local_tee, mask.local.value);
3283
3284	try cg.emitWValue(operand);
3285	try cg.addTag(.i32_xor);
3286	try cg.emitWValue(mask);
3287	try cg.addTag(.i32_sub);
3288	return .stack;
3289	},
3290	33...64 => {
3291	try cg.emitWValue(operand);
3292	try cg.addImm64(63);
3293	try cg.addTag(.i64_shr_s);
3294
3295	var mask = try cg.allocLocal(Type.i64);
3296	defer mask.free(cg);
3297	try cg.addLocal(.local_tee, mask.local.value);
3298
3299	try cg.emitWValue(operand);
3300	try cg.addTag(.i64_xor);
3301	try cg.emitWValue(mask);
3302	try cg.addTag(.i64_sub);
3303	return .stack;
3304	},
3305	65...128 => {
3306	const u128_ty: IntType = .{ .is_signed = false, .bits = 128 };
3307
3308	const mask = try cg.allocStack(Type.u128);
3309	try cg.emitWValue(mask);
3310	try cg.emitWValue(mask);
3311
3312	_ = try cg.load(operand, Type.u64, 8);
3313	try cg.addImm64(63);
3314	try cg.addTag(.i64_shr_s);
3315
3316	var tmp = try cg.allocLocal(Type.u64);
3317	defer tmp.free(cg);
3318	try cg.addLocal(.local_tee, tmp.local.value);
3319	try cg.store(.stack, .stack, Type.u64, mask.offset() + 0);
3320	try cg.emitWValue(tmp);
3321	try cg.store(.stack, .stack, Type.u64, mask.offset() + 8);
3322
3323	const a = try cg.intXor(u128_ty, operand, mask);
3324	const b = try cg.intSub(u128_ty, a, mask);
3325	return b;
3326	},
3327	else => {
3328	const result = try cg.allocInt(ty);
3329
3330	try cg.lowerToStack(result);
3331	try cg.lowerToStack(operand);
3332	try cg.addImm32(ty.bits);
3333	try cg.addCallIntrinsic(.__abs_limb64);
3334
3335	return result;
3336	},
3337	}
3338	}
3339
3340	fn intMax(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3341	try cg.lowerToStack(lhs);
3342	try cg.lowerToStack(rhs);
3343	_ = try cg.intCmp(ty, .gt, lhs, rhs);
3344	try cg.addTag(.select);
3345	return .stack;
3346	}
3347
3348	fn intMin(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3349	try cg.lowerToStack(lhs);
3350	try cg.lowerToStack(rhs);
3351	_ = try cg.intCmp(ty, .lt, lhs, rhs);
3352	try cg.addTag(.select);
3353	return .stack;
3354	}
3355
3356	fn intClz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3357	switch (ty.bits) {
3358	0 => unreachable,
3359	1...32 => {
3360	if (ty.is_signed and ty.bits < 32) {
3361	const mask: u32 = ~@as(u32, 0) >> @intCast(32 - ty.bits);
3362	_ = try cg.intAnd(.u32, operand, .{ .imm32 = mask });
3363	} else {
3364	try cg.emitWValue(operand);
3365	}
3366	try cg.addTag(.i32_clz);
3367	if (ty.bits < 32) {
3368	try cg.addImm32(32 - ty.bits);
3369	try cg.addTag(.i32_sub);
3370	}
3371	return .stack;
3372	},
3373	33...64 => {
3374	if (ty.is_signed and ty.bits < 64) {
3375	const mask: u64 = ~@as(u64, 0) >> @intCast(64 - ty.bits);
3376	_ = try cg.intAnd(.u64, operand, .{ .imm64 = mask });
3377	} else {
3378	try cg.emitWValue(operand);
3379	}
3380	try cg.addTag(.i64_clz);
3381	try cg.addTag(.i32_wrap_i64);
3382	if (ty.bits < 64) {
3383	try cg.addImm32(64 - ty.bits);
3384	try cg.addTag(.i32_sub);
3385	}
3386	return .stack;
3387	},
3388	65...128 => {
3389	var msb = try (try cg.load(operand, Type.u64, 8)).toLocal(cg, Type.u64);
3390	defer msb.free(cg);
3391
3392	if (ty.is_signed and ty.bits < 128) {
3393	const mask: u64 = ~@as(u64, 0) >> @intCast(128 - ty.bits);
3394	_ = try cg.intAnd(.u64, msb, .{ .imm64 = mask });
3395	} else {
3396	try cg.emitWValue(msb);
3397	}
3398
3399	try cg.addTag(.i64_clz);
3400	_ = try cg.load(operand, Type.u64, 0);
3401	try cg.addTag(.i64_clz);
3402	try cg.emitWValue(.{ .imm64 = 64 });
3403	try cg.addTag(.i64_add);
3404	_ = try cg.intCmp(.u64, .neq, msb, .{ .imm64 = 0 });
3405	try cg.addTag(.select);
3406	try cg.addTag(.i32_wrap_i64);
3407
3408	if (ty.bits < 128) {
3409	try cg.addImm32(128 - ty.bits);
3410	try cg.addTag(.i32_sub);
3411	}
3412
3413	return .stack;
3414	},
3415	else => {
3416	try cg.lowerToStack(operand);
3417	try cg.addImm32(ty.bits);
3418	try cg.addCallIntrinsic(.__clz_limb64);
3419
3420	return .stack;
3421	},
3422	}
3423	}
3424
3425	fn intCtz(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3426	switch (ty.bits) {
3427	0 => unreachable,
3428	1...32 => {
3429	if (ty.bits < 32) {
3430	_ = try cg.intOr(.u32, operand, .{ .imm32 = @as(u32, 1) << @intCast(ty.bits) });
3431	} else {
3432	try cg.emitWValue(operand);
3433	}
3434	try cg.addTag(.i32_ctz);
3435	return .stack;
3436	},
3437	33...64 => {
3438	if (ty.bits < 64) {
3439	_ = try cg.intOr(.u64, operand, .{ .imm64 = @as(u64, 1) << @intCast(ty.bits) });
3440	} else {
3441	try cg.emitWValue(operand);
3442	}
3443	try cg.addTag(.i64_ctz);
3444	try cg.addTag(.i32_wrap_i64);
3445	return .stack;
3446	},
3447	65...128 => {
3448	var lsb = try (try cg.load(operand, Type.u64, 0)).toLocal(cg, Type.u64);
3449	defer lsb.free(cg);
3450
3451	try cg.emitWValue(lsb);
3452	try cg.addTag(.i64_ctz);
3453
3454	_ = try cg.load(operand, Type.u64, 8);
3455	if (ty.bits < 128) {
3456	try cg.addImm64(@as(u64, 1) << @intCast(ty.bits - 64));
3457	try cg.addTag(.i64_or);
3458	}
3459	try cg.addTag(.i64_ctz);
3460	try cg.addImm64(64);
3461	try cg.addTag(.i64_add);
3462	_ = try cg.intCmp(.u64, .neq, lsb, .{ .imm64 = 0 });
3463	try cg.addTag(.select);
3464	try cg.addTag(.i32_wrap_i64);
3465	return .stack;
3466	},
3467	else => {
3468	try cg.lowerToStack(operand);
3469	try cg.addImm32(ty.bits);
3470	try cg.addCallIntrinsic(.__ctz_limb64);
3471
3472	return .stack;
3473	},
3474	}
3475	}
3476
3477	fn intPopCount(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3478	switch (ty.bits) {
3479	0 => unreachable,
3480	1...32 => {
3481	try cg.emitWValue(operand);
3482	if (ty.is_signed and ty.bits < 32) {
3483	try cg.addImm32(32 - ty.bits);
3484	try cg.addTag(.i32_shl);
3485	}
3486	try cg.addTag(.i32_popcnt);
3487	return .stack;
3488	},
3489	33...64 => {
3490	try cg.emitWValue(operand);
3491	if (ty.is_signed and ty.bits < 64) {
3492	try cg.addImm64(64 - ty.bits);
3493	try cg.addTag(.i64_shl);
3494	}
3495	try cg.addTag(.i64_popcnt);
3496	try cg.addTag(.i32_wrap_i64);
3497	return .stack;
3498	},
3499	65...128 => {
3500	_ = try cg.load(operand, Type.u64, 0);
3501	try cg.addTag(.i64_popcnt);
3502	_ = try cg.load(operand, Type.u64, 8);
3503	if (ty.is_signed and ty.bits < 128) {
3504	try cg.addImm64(128 - ty.bits);
3505	try cg.addTag(.i64_shl);
3506	}
3507	try cg.addTag(.i64_popcnt);
3508
3509	try cg.addTag(.i64_add);
3510	try cg.addTag(.i32_wrap_i64);
3511	return .stack;
3512	},
3513	else => {
3514	try cg.lowerToStack(operand);
3515	try cg.addImm32(ty.bits);
3516	try cg.addCallIntrinsic(.__popcount_limb64);
3517
3518	return .stack;
3519	},
3520	}
3521	}
3522
3523	fn intBitReverse(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3524	switch (ty.bits) {
3525	0 => unreachable,
3526	1...32 => {
3527	const intrin_ret = try cg.callIntrinsic(
3528	.__bitreversesi2,
3529	&.{.u32_type},
3530	Type.u32,
3531	&.{operand},
3532	);
3533	if (ty.bits == 32) return intrin_ret;
3534	return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3535	},
3536	33...64 => {
3537	const intrin_ret = try cg.callIntrinsic(
3538	.__bitreversedi2,
3539	&.{.u64_type},
3540	Type.u64,
3541	&.{operand},
3542	);
3543	if (ty.bits == 64) return intrin_ret;
3544	return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3545	},
3546	65...128 => {
3547	const tmp = try cg.allocStack(Type.u128);
3548
3549	try cg.emitWValue(tmp);
3550	const hi = try cg.load(operand, Type.u64, 8);
3551	const hi_rev = try cg.callIntrinsic(
3552	.__bitreversedi2,
3553	&.{.u64_type},
3554	Type.u64,
3555	&.{hi},
3556	);
3557	try cg.emitWValue(hi_rev);
3558	try cg.store(.stack, .stack, Type.u64, tmp.offset());
3559
3560	try cg.emitWValue(tmp);
3561	const lo = try cg.load(operand, Type.u64, 0);
3562	const lo_rev = try cg.callIntrinsic(
3563	.__bitreversedi2,
3564	&.{.u64_type},
3565	Type.u64,
3566	&.{lo},
3567	);
3568	try cg.emitWValue(lo_rev);
3569	try cg.store(.stack, .stack, Type.u64, tmp.offset() + 8);
3570
3571	if (ty.bits < 128) {
3572	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3573	return cg.intShr(shift_ty, tmp, .{ .imm32 = 128 - ty.bits });
3574	} else {
3575	return tmp;
3576	}
3577	},
3578	else => {
3579	const result = try cg.allocInt(ty);
3580
3581	try cg.lowerToStack(result);
3582	try cg.lowerToStack(operand);
3583	try cg.addImm32(@intFromBool(ty.is_signed));
3584	try cg.addImm32(ty.bits);
3585	try cg.addCallIntrinsic(.__bitreverse_limb64);
3586
3587	return result;
3588	},
3589	}
3590	}
3591
3592	fn intByteSwap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3593	switch (ty.bits) {
3594	0 => unreachable,
3595	1...32 => {
3596	const intrin_ret = try cg.callIntrinsic(
3597	.__bswapsi2,
3598	&.{.u32_type},
3599	Type.u32,
3600	&.{operand},
3601	);
3602	if (ty.bits == 32) return intrin_ret;
3603	return cg.intShr(ty, intrin_ret, .{ .imm32 = 32 - ty.bits });
3604	},
3605	33...64 => {
3606	const intrin_ret = try cg.callIntrinsic(
3607	.__bswapdi2,
3608	&.{.u64_type},
3609	Type.u64,
3610	&.{operand},
3611	);
3612	if (ty.bits == 64) return intrin_ret;
3613	return cg.intShr(ty, intrin_ret, .{ .imm32 = 64 - ty.bits });
3614	},
3615	65...128 => {
3616	const result = try cg.allocStack(Type.u128);
3617
3618	try cg.emitWValue(result);
3619
3620	const low = try cg.load(operand, Type.u64, 0);
3621	const swap_low = try cg.callIntrinsic(
3622	.__bswapdi2,
3623	&.{.u64_type},
3624	Type.u64,
3625	&.{low},
3626	);
3627	try cg.store(.stack, swap_low, Type.u64, result.offset() + 8);
3628
3629	try cg.emitWValue(result);
3630
3631	const high = try cg.load(operand, Type.u64, 8);
3632	const swap_high = try cg.callIntrinsic(
3633	.__bswapdi2,
3634	&.{.u64_type},
3635	Type.u64,
3636	&.{high},
3637	);
3638	try cg.store(.stack, swap_high, Type.u64, result.offset());
3639
3640	if (ty.bits < 128) {
3641	const shift_ty: IntType = .{ .is_signed = ty.is_signed, .bits = 128 };
3642	return cg.intShr(shift_ty, result, .{ .imm32 = 128 - ty.bits });
3643	} else {
3644	return result;
3645	}
3646	},
3647	else => {
3648	const result = try cg.allocInt(ty);
3649
3650	try cg.lowerToStack(result);
3651	try cg.lowerToStack(operand);
3652	try cg.addImm32(@intFromBool(ty.is_signed));
3653	try cg.addImm32(ty.bits);
3654	try cg.addCallIntrinsic(.__byteswap_limb64);
3655
3656	return result;
3657	},
3658	}
3659	}
3660
3661	fn intWrap(cg: *CodeGen, ty: IntType, operand: WValue) InnerError!WValue {
3662	switch (ty.bits) {
3663	0 => unreachable,
3664	1...31 => {
3665	try cg.emitWValue(operand);
3666	if (ty.is_signed) {
3667	try cg.addImm32(32 - ty.bits);
3668	try cg.addTag(.i32_shl);
3669	try cg.addImm32(32 - ty.bits);
3670	try cg.addTag(.i32_shr_s);
3671	} else {
3672	try cg.addImm32(~@as(u32, 0) >> @intCast(32 - ty.bits));
3673	try cg.addTag(.i32_and);
3674	}
3675	return .stack;
3676	},
3677	32 => return operand,
3678	33...63 => {
3679	try cg.emitWValue(operand);
3680	if (ty.is_signed) {
3681	try cg.addImm64(64 - ty.bits);
3682	try cg.addTag(.i64_shl);
3683	try cg.addImm64(64 - ty.bits);
3684	try cg.addTag(.i64_shr_s);
3685	} else {
3686	try cg.addImm64(~@as(u64, 0) >> @intCast(64 - ty.bits));
3687	try cg.addTag(.i64_and);
3688	}
3689	return .stack;
3690	},
3691	64 => return operand,
3692	65...127 => {
3693	const result = try cg.allocStack(Type.u128);
3694
3695	try cg.emitWValue(result);
3696	_ = try cg.load(operand, Type.u64, 0);
3697	try cg.store(.stack, .stack, Type.u64, result.offset());
3698
3699	try cg.emitWValue(result);
3700	_ = try cg.load(operand, Type.u64, 8);
3701	if (ty.is_signed) {
3702	try cg.addImm64(128 - ty.bits);
3703	try cg.addTag(.i64_shl);
3704	try cg.addImm64(128 - ty.bits);
3705	try cg.addTag(.i64_shr_s);
3706	} else {
3707	try cg.addImm64(~@as(u64, 0) >> @intCast(128 - ty.bits));
3708	try cg.addTag(.i64_and);
3709	}
3710	try cg.store(.stack, .stack, Type.u64, result.offset() + 8);
3711
3712	return result;
3713	},
3714	128 => return operand,
3715	else => {
3716	const bits = cg.intBackingBits(ty.bits);
3717	if (ty.bits == bits) return operand;
3718
3719	const result = try cg.allocInt(ty);
3720
3721	const used_len = @divCeil(ty.bits, 64) * 8;
3722
3723	if (ty.bits % 64 != 0) {
3724	try cg.memcpy(result, operand, .{ .imm32 = used_len - 8 });
3725	const pad = 64 - ty.bits % 64;
3726
3727	try cg.emitWValue(result);
3728	_ = try cg.load(operand, Type.u64, used_len - 8);
3729	if (ty.is_signed) {
3730	try cg.addImm64(pad);
3731	try cg.addTag(.i64_shl);
3732	try cg.addImm64(pad);
3733	try cg.addTag(.i64_shr_s);
3734	} else {
3735	try cg.addImm64(~@as(u64, 0) >> @intCast(pad));
3736	try cg.addTag(.i64_and);
3737	}
3738	try cg.store(.stack, .stack, Type.u64, result.offset() + used_len - 8);
3739	} else {
3740	try cg.memcpy(result, operand, .{ .imm32 = used_len });
3741	}
3742
3743	const full_len = @divExact(bits, 8);
3744	if (used_len + 8 == full_len) { // last limb needs sign extended
3745	try cg.emitWValue(result);
3746	if (ty.is_signed) {
3747	_ = try cg.load(result, Type.u64, used_len - 8);
3748	try cg.addImm64(63);
3749	try cg.addTag(.i64_shr_s);
3750	} else {
3751	try cg.addImm64(0);
3752	}
3753	try cg.store(.stack, .stack, Type.u64, result.offset() + used_len);
3754	}
3755
3756	return result;
3757	},
3758	}
3759	}
3760
3761	fn intMaxValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3762	if (int_ty.bits <= 32) {
3763	if (int_ty.is_signed) {
3764	return .{ .imm32 = (~@as(u32, 0) >> @intCast(32 - int_ty.bits)) >> 1 };
3765	} else {
3766	return .{ .imm32 = ~@as(u32, 0) >> @intCast(32 - int_ty.bits) };
3767	}
3768	} else if (int_ty.bits <= 64) {
3769	if (int_ty.is_signed) {
3770	return .{ .imm64 = (~@as(u64, 0) >> @intCast(64 - int_ty.bits)) >> 1 };
3771	} else {
3772	return .{ .imm64 = ~@as(u64, 0) >> @intCast(64 - int_ty.bits) };
3773	}
3774	} else if (int_ty.bits <= 128) {
3775	const result = try cg.allocInt(int_ty);
3776	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, 0);
3777
3778	if (int_ty.is_signed) {
3779	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(128 - int_ty.bits)) >> 1 }, Type.u64, 8);
3780	} else {
3781	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(128 - int_ty.bits) }, Type.u64, 8);
3782	}
3783	return result;
3784	} else {
3785	const result = try cg.allocInt(int_ty);
3786	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3787	const used_len = @divCeil(int_ty.bits, 64) * 8;
3788
3789	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0xFF });
3790
3791	if (int_ty.is_signed) {
3792	try cg.store(result, .{ .imm64 = (~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits)) >> 1 }, Type.u64, used_len - 8);
3793	} else {
3794	try cg.store(result, .{ .imm64 = ~@as(u64, 0) >> @intCast(used_len * 8 - int_ty.bits) }, Type.u64, used_len - 8);
3795	}
3796
3797	if (used_len + 8 == full_len) {
3798	try cg.store(result, .{ .imm64 = 0 }, Type.u64, full_len - 8);
3799	}
3800
3801	return result;
3802	}
3803	}
3804
3805	fn intMinValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3806	if (!int_ty.is_signed) {
3807	return cg.intZeroValue(int_ty);
3808	}
3809	if (int_ty.bits <= 32) {
3810	return .{ .imm32 = ~@as(u32, 0) << @intCast(int_ty.bits - 1) };
3811	} else if (int_ty.bits <= 64) {
3812	return .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 1) };
3813	} else if (int_ty.bits <= 128) {
3814	const result = try cg.allocInt(int_ty);
3815	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3816	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - 65) }, Type.u64, 8);
3817	return result;
3818	} else {
3819	const result = try cg.allocInt(int_ty);
3820	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3821	const used_len = @divCeil(int_ty.bits, 64) * 8;
3822
3823	try cg.memset(Type.u8, result, .{ .imm32 = used_len - 8 }, .{ .imm32 = 0 });
3824	try cg.store(result, .{ .imm64 = ~@as(u64, 0) << @intCast(int_ty.bits - (used_len - 8) * 8 - 1) }, Type.u64, used_len - 8);
3825
3826	if (used_len + 8 == full_len) {
3827	try cg.store(result, .{ .imm64 = ~@as(u64, 0) }, Type.u64, full_len - 8);
3828	}
3829
3830	return result;
3831	}
3832	}
3833
3834	fn intAddSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3835	const raw_val = try cg.intAdd(int_ty, lhs, rhs);
3836	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3837	defer op_val.free(cg);
3838
3839	const max_val = try cg.intMaxValue(int_ty);
3840
3841	if (int_ty.is_signed) {
3842	const zero = try cg.intZeroValue(int_ty);
3843	var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3844	defer rhs_is_neg.free(cg);
3845	const min_val = try cg.intMinValue(int_ty);
3846
3847	try cg.lowerToStack(min_val);
3848	try cg.lowerToStack(max_val);
3849	try cg.emitWValue(rhs_is_neg);
3850	try cg.addTag(.select);
3851
3852	try cg.lowerToStack(op_val);
3853	const overflow_cmp = try cg.intCmp(int_ty, .lt, op_val, lhs);
3854	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3855	try cg.emitWValue(is_overflow);
3856	try cg.addTag(.select);
3857	return .stack;
3858	} else {
3859	try cg.lowerToStack(max_val);
3860	try cg.lowerToStack(op_val);
3861
3862	const is_overflow = try cg.intCmp(int_ty, .lt, op_val, lhs);
3863	try cg.emitWValue(is_overflow);
3864	try cg.addTag(.select);
3865	return .stack;
3866	}
3867	}
3868
3869	fn intSubSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3870	const raw_val = try cg.intSub(int_ty, lhs, rhs);
3871	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3872	defer op_val.free(cg);
3873
3874	if (int_ty.is_signed) {
3875	const zero = try cg.intZeroValue(int_ty);
3876	var rhs_is_neg = try cg.toLocalInt(try cg.intCmp(int_ty, .lt, rhs, zero), .u32);
3877	defer rhs_is_neg.free(cg);
3878	const max_val = try cg.intMaxValue(int_ty);
3879	const min_val = try cg.intMinValue(int_ty);
3880
3881	try cg.lowerToStack(max_val);
3882	try cg.lowerToStack(min_val);
3883	try cg.emitWValue(rhs_is_neg);
3884	try cg.addTag(.select);
3885
3886	try cg.lowerToStack(op_val);
3887	const overflow_cmp = try cg.intCmp(int_ty, .gt, op_val, lhs);
3888	const is_overflow = try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
3889	try cg.emitWValue(is_overflow);
3890	try cg.addTag(.select);
3891	return .stack;
3892	} else {
3893	const zero = try cg.intZeroValue(int_ty);
3894
3895	try cg.lowerToStack(zero);
3896	try cg.lowerToStack(op_val);
3897	const is_overflow = try cg.intCmp(int_ty, .lt, lhs, rhs);
3898	try cg.emitWValue(is_overflow);
3899	try cg.addTag(.select);
3900	return .stack;
3901	}
3902	}
3903
3904	fn intMulSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3905	const ext_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = int_ty.bits * 2 };
3906
3907	const lhs_ext = try cg.intCast(ext_ty, int_ty, lhs);
3908	const rhs_ext = try cg.intCast(ext_ty, int_ty, rhs);
3909
3910	var mul_ext = try cg.toLocalInt(try cg.intMul(ext_ty, lhs_ext, rhs_ext), ext_ty);
3911	defer mul_ext.free(cg);
3912
3913	var op_val = try cg.toLocalInt(try cg.intTrunc(int_ty, ext_ty, mul_ext), int_ty);
3914	defer op_val.free(cg);
3915	const max_val = try cg.intMaxValue(int_ty);
3916
3917	if (int_ty.is_signed) {
3918	const min_val = try cg.intMinValue(int_ty);
3919
3920	try cg.lowerToStack(min_val);
3921
3922	try cg.lowerToStack(max_val);
3923	try cg.lowerToStack(op_val);
3924	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3925	const ov_pos = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3926	try cg.emitWValue(ov_pos);
3927	try cg.addTag(.select);
3928
3929	const min_ext = try cg.intCast(ext_ty, int_ty, min_val);
3930	const ov_neg = try cg.intCmp(ext_ty, .gt, min_ext, mul_ext);
3931	try cg.lowerToStack(ov_neg);
3932	try cg.addTag(.select);
3933	return .stack;
3934	} else {
3935	try cg.lowerToStack(max_val);
3936	try cg.lowerToStack(op_val);
3937	const max_ext = try cg.intCast(ext_ty, int_ty, max_val);
3938	const is_overflow = try cg.intCmp(ext_ty, .lt, max_ext, mul_ext);
3939	try cg.emitWValue(is_overflow);
3940	try cg.addTag(.select);
3941	return .stack;
3942	}
3943	}
3944
3945	fn intShlSat(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!WValue {
3946	const raw_val = try cg.intShl(int_ty, lhs, rhs);
3947	var op_val = try cg.toLocalInt(try cg.intWrap(int_ty, raw_val), int_ty);
3948	defer op_val.free(cg);
3949
3950	var check_val = try cg.toLocalInt(try cg.intShr(int_ty, op_val, rhs), int_ty);
3951	defer check_val.free(cg);
3952
3953	const max_val = try cg.intMaxValue(int_ty);
3954
3955	if (int_ty.is_signed) {
3956	const zero = try cg.intZeroValue(int_ty);
3957	const min_val = try cg.intMinValue(int_ty);
3958
3959	try cg.lowerToStack(min_val);
3960	try cg.lowerToStack(max_val);
3961	const lhs_is_neg = try cg.intCmp(int_ty, .lt, lhs, zero);
3962	try cg.emitWValue(lhs_is_neg);
3963	try cg.addTag(.select);
3964
3965	try cg.lowerToStack(op_val);
3966	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3967	try cg.emitWValue(is_overflow);
3968	try cg.addTag(.select);
3969	return .stack;
3970	} else {
3971	try cg.lowerToStack(max_val);
3972	try cg.lowerToStack(op_val);
3973	const is_overflow = try cg.intCmp(int_ty, .neq, check_val, lhs);
3974	try cg.emitWValue(is_overflow);
3975	try cg.addTag(.select);
3976	return .stack;
3977	}
3978	}
3979
3980	fn intZeroValue(cg: *CodeGen, int_ty: IntType) InnerError!WValue {
3981	switch (int_ty.bits) {
3982	0 => unreachable,
3983	1...32 => return .{ .imm32 = 0 },
3984	33...64 => return .{ .imm64 = 0 },
3985	65...128 => {
3986	const result = try cg.allocInt(int_ty);
3987	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 0);
3988	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
3989	return result;
3990	},
3991	else => {
3992	const result = try cg.allocInt(int_ty);
3993	const full_len = @divExact(cg.intBackingBits(int_ty.bits), 8);
3994	try cg.memset(Type.u8, result, .{ .imm32 = full_len }, .{ .imm32 = 0 });
3995	return result;
3996	},
3997	}
3998	}
3999
4000	fn toLocalInt(cg: *CodeGen, value: WValue, int_ty: IntType) InnerError!WValue {
4001	switch (value) {
4002	.stack => {
4003	const ty: Type = switch (int_ty.bits) {
4004	0 => unreachable,
4005	1...32 => .u32,
4006	33...64 => .u64,
4007	65...128 => .u128,
4008	else => return cg.fail("TODO: Support toLocalInt for integer bitsize: {d}", .{int_ty.bits}),
4009	};
4010	const new_local = try cg.allocLocal(ty);
4011	try cg.addLocal(.local_set, new_local.local.value);
4012	return new_local;
4013	},
4014	.local, .stack_offset => return value,
4015	else => unreachable,
4016	}
4017	}
4018
4019	const OverflowResult = struct {
4020	result: WValue,
4021	ov: WValue,
4022	};
4023
4024	fn intAddOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4025	switch (ty.bits) {
4026	0 => unreachable,
4027	1...128 => {
4028	const raw_result = try cg.intAdd(ty, lhs, rhs);
4029	const op_result = try cg.intWrap(ty, raw_result);
4030	const op_tmp = try cg.toLocalInt(op_result, ty);
4031
4032	const overflow_bit = if (ty.is_signed) blk: {
4033	const zero = try cg.intZeroValue(ty);
4034	const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
4035	const overflow_cmp = try cg.intCmp(ty, .lt, op_tmp, lhs);
4036	break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
4037	} else try cg.intCmp(ty, .lt, op_tmp, lhs);
4038
4039	return .{ .result = op_tmp, .ov = overflow_bit };
4040	},
4041	else => {
4042	const result = try cg.allocInt(ty);
4043
4044	try cg.lowerToStack(result);
4045	try cg.lowerToStack(lhs);
4046	try cg.lowerToStack(rhs);
4047	try cg.addImm32(@intFromBool(ty.is_signed));
4048	try cg.addImm32(ty.bits);
4049	try cg.addCallIntrinsic(.__addo_limb64);
4050
4051	return .{ .result = result, .ov = .stack };
4052	},
4053	}
4054	}
4055
4056	fn intSubOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4057	switch (ty.bits) {
4058	0 => unreachable,
4059	1...128 => {
4060	const raw_result = try cg.intSub(ty, lhs, rhs);
4061	const op_result = try cg.intWrap(ty, raw_result);
4062	const op_tmp = try cg.toLocalInt(op_result, ty);
4063
4064	const overflow_bit = if (ty.is_signed) blk: {
4065	const zero = try cg.intZeroValue(ty);
4066	const rhs_is_neg = try cg.intCmp(ty, .lt, rhs, zero);
4067	const overflow_cmp = try cg.intCmp(ty, .gt, op_tmp, lhs);
4068	break :blk try cg.intCmp(.u32, .neq, rhs_is_neg, overflow_cmp);
4069	} else try cg.intCmp(ty, .gt, op_tmp, lhs);
4070
4071	return .{ .result = op_tmp, .ov = overflow_bit };
4072	},
4073	else => {
4074	const result = try cg.allocInt(ty);
4075
4076	try cg.lowerToStack(result);
4077	try cg.lowerToStack(lhs);
4078	try cg.lowerToStack(rhs);
4079	try cg.addImm32(@intFromBool(ty.is_signed));
4080	try cg.addImm32(ty.bits);
4081	try cg.addCallIntrinsic(.__subo_limb64);
4082	return .{ .result = result, .ov = .stack };
4083	},
4084	}
4085	}
4086
4087	fn intMulOverflow(cg: *CodeGen, int_ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4088	const overflow_bit = try cg.allocLocal(Type.u32);
4089	try cg.addImm32(0);
4090	try cg.addLocal(.local_set, overflow_bit.local.value);
4091
4092	const result_val = if (int_ty.bits <= 32) blk: {
4093	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 64 };
4094	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4095	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4096	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4097	const bin_op = try cg.toLocalInt(mul_raw, new_ty);
4098
4099	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4100	const res_tmp = try cg.toLocalInt(res, int_ty);
4101
4102	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4103	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4104	try cg.addLocal(.local_set, overflow_bit.local.value);
4105	break :blk res_tmp;
4106	} else if (int_ty.bits <= 64) blk: {
4107	const new_ty: IntType = .{ .is_signed = int_ty.is_signed, .bits = 128 };
4108	const lhs_upcast = try cg.intCast(new_ty, int_ty, lhs);
4109	const rhs_upcast = try cg.intCast(new_ty, int_ty, rhs);
4110	const mul_raw = try cg.intMul(new_ty, lhs_upcast, rhs_upcast);
4111	const bin_op = try cg.toLocalInt(mul_raw, new_ty);
4112
4113	const res = try cg.intTrunc(int_ty, new_ty, bin_op);
4114	const res_tmp = try cg.toLocalInt(res, int_ty);
4115
4116	const res_upcast = try cg.intCast(new_ty, int_ty, res_tmp);
4117	_ = try cg.intCmp(new_ty, .neq, res_upcast, bin_op);
4118	try cg.addLocal(.local_set, overflow_bit.local.value);
4119	break :blk res_tmp;
4120	} else if (int_ty.bits == 128 and int_ty.is_signed) blk: {
4121	const overflow_ret = try cg.allocStack(Type.i32);
4122	const res = try cg.callIntrinsic(
4123	.__muloti4,
4124	&[_]InternPool.Index{ .i128_type, .i128_type, .usize_type },
4125	Type.i128,
4126	&.{ lhs, rhs, overflow_ret },
4127	);
4128	_ = try cg.load(overflow_ret, Type.i32, 0);
4129	try cg.addLocal(.local_set, overflow_bit.local.value);
4130	break :blk res;
4131	} else {
4132	const result = try cg.allocInt(int_ty);
4133
4134	try cg.lowerToStack(result);
4135	try cg.lowerToStack(lhs);
4136	try cg.lowerToStack(rhs);
4137	try cg.addImm32(@intFromBool(int_ty.is_signed));
4138	try cg.addImm32(int_ty.bits);
4139	try cg.addCallIntrinsic(.__mulo_limb64);
4140
4141	return .{ .result = result, .ov = .stack };
4142	};
4143
4144	return .{ .result = result_val, .ov = .{ .local = overflow_bit.local } };
4145	}
4146
4147	fn intShlOverflow(cg: *CodeGen, ty: IntType, lhs: WValue, rhs: WValue) InnerError!OverflowResult {
4148	switch (ty.bits) {
4149	0 => unreachable,
4150	1...128 => {
4151	const raw_shl = try cg.intShl(ty, lhs, rhs);
4152	const wrapped_shl = try cg.intWrap(ty, raw_shl);
4153	const shl_tmp = try cg.toLocalInt(wrapped_shl, ty);
4154
4155	const shr = try cg.intShr(ty, shl_tmp, rhs);
4156	const overflow_bit = try cg.intCmp(ty, .neq, shr, lhs);
4157
4158	return .{ .result = shl_tmp, .ov = overflow_bit };
4159	},
4160	else => {
4161	const result = try cg.allocInt(ty);
4162
4163	try cg.lowerToStack(result);
4164	try cg.lowerToStack(lhs);
4165	try cg.lowerToStack(rhs);
4166	try cg.addImm32(@intFromBool(ty.is_signed));
4167	try cg.addImm32(ty.bits);
4168	try cg.addCallIntrinsic(.__shlo_limb64);
4169
4170	return .{ .result = result, .ov = .stack };
4171	},
4172	}
4173	}
4174
4175	fn intCast(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
4176	const src_bits: u16 = cg.intBackingBits(src_ty.bits);
4177	const dest_bits: u16 = cg.intBackingBits(dest_ty.bits);
4178
4179	if (src_bits == dest_bits) {
4180	return operand;
4181	}
4182
4183	if (src_bits == 64 and dest_bits == 32) {
4184	try cg.emitWValue(operand);
4185	try cg.addTag(.i32_wrap_i64);
4186	return .stack;
4187	} else if (src_bits == 32 and dest_bits == 64) {
4188	try cg.emitWValue(operand);
4189	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4190	return .stack;
4191	} else if (dest_bits >= 128) {
4192	const result = try cg.allocInt(dest_ty);
4193
4194	const dest_len = dest_bits / 8;
4195
4196	if (dest_bits <= src_bits) {
4197	assert(src_bits >= 128);
4198	try cg.memcpy(result, operand, .{ .imm32 = dest_len });
4199	} else {
4200	var src_len: u32 = undefined;
4201	if (src_bits == 32) {
4202	try cg.emitWValue(result);
4203	try cg.emitWValue(operand);
4204	try cg.addTag(if (src_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4205	try cg.store(.stack, .stack, Type.u64, result.offset());
4206	src_len = 8;
4207	} else if (src_bits == 64) {
4208	try cg.emitWValue(result);
4209	try cg.emitWValue(operand);
4210	try cg.store(.stack, .stack, Type.u64, result.offset());
4211	src_len = 8;
4212	} else {
4213	src_len = src_bits / 8;
4214	try cg.memcpy(result, operand, .{ .imm32 = src_len });
4215	}
4216
4217	if (dest_bits == 128) {
4218	if (src_ty.is_signed) {
4219	try cg.emitWValue(result);
4220	if (src_bits == 32) {
4221	try cg.emitWValue(operand);
4222	try cg.addTag(if (dest_ty.is_signed) .i64_extend_i32_s else .i64_extend_i32_u);
4223	} else if (src_bits == 64) {
4224	try cg.emitWValue(operand);
4225	} else unreachable;
4226	const shr = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4227	try cg.store(.stack, shr, Type.u64, 8 + result.offset());
4228	} else {
4229	try cg.store(result, .{ .imm64 = 0 }, Type.u64, 8);
4230	}
4231	} else {
4232	var pad = result;
4233	pad.stack_offset.value += src_len;
4234	const memset_len = dest_len - src_len;
4235	if (src_ty.is_signed) {
4236	if (src_bits == 32) {
4237	try cg.emitWValue(operand);
4238	_ = try cg.intShr(IntType.i32, .stack, .{ .imm32 = 31 });
4239	} else if (src_bits == 64) {
4240	try cg.emitWValue(operand);
4241	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4242	try cg.addTag(.i32_wrap_i64);
4243	} else {
4244	_ = try cg.load(operand, Type.u64, src_len - 8);
4245	_ = try cg.intShr(IntType.i64, .stack, .{ .imm32 = 63 });
4246	try cg.addTag(.i32_wrap_i64);
4247	}
4248	var sign_byte = try @as(WValue, .stack).toLocal(cg, Type.u32);
4249	try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, sign_byte);
4250	sign_byte.free(cg);
4251	} else {
4252	try cg.memset(Type.u8, pad, .{ .imm32 = memset_len }, .{ .imm32 = 0 });
4253	}
4254	}
4255	}
4256
4257	return result;
4258	} else {
4259	assert(dest_bits <= 64);
4260	assert(src_bits >= 128);
4261	const load_ty = if (dest_bits == 32) Type.u32 else Type.u64;
4262	return cg.load(operand, load_ty, 0);
4263	}
4264	}
4265
4266	fn intTrunc(cg: *CodeGen, dest_ty: IntType, src_ty: IntType, operand: WValue) InnerError!WValue {
4267	var result = try cg.intCast(dest_ty, src_ty, operand);
4268
4269	const dest_wasm_bits = cg.intBackingBits(dest_ty.bits);
4270
4271	if (dest_wasm_bits != dest_ty.bits) {
4272	result = try cg.intWrap(dest_ty, result);
4273	}
4274
4275	return result;
4276	}
4277
4278	const FloatType = enum {
4279	f16,
4280	f32,
4281	f64,
4282	f80,
4283	f128,
4284
4285	fn fromType(cg: *CodeGen, ty: Type) FloatType {
4286	assert(ty.isRuntimeFloat());
4287	return switch (ty.floatBits(cg.target)) {
4288	16 => .f16,
4289	32 => .f32,
4290	64 => .f64,
4291	80 => .f80,
4292	128 => .f128,
4293	else => unreachable,
4294	};
4295	}
4296	};
4297
4298	fn floatAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4299	switch (ty) {
4300	.f16 => return cg.callIntrinsic(.__addhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4301	.f32 => {
4302	try cg.emitWValue(lhs);
4303	try cg.emitWValue(rhs);
4304	try cg.addTag(.f32_add);
4305	return .stack;
4306	},
4307	.f64 => {
4308	try cg.emitWValue(lhs);
4309	try cg.emitWValue(rhs);
4310	try cg.addTag(.f64_add);
4311	return .stack;
4312	},
4313	.f80 => return cg.callIntrinsic(.__addxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4314	.f128 => return cg.callIntrinsic(.__addtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4315	}
4316	}
4317
4318	fn floatSub(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4319	switch (ty) {
4320	.f16 => return cg.callIntrinsic(.__subhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4321	.f32 => {
4322	try cg.emitWValue(lhs);
4323	try cg.emitWValue(rhs);
4324	try cg.addTag(.f32_sub);
4325	return .stack;
4326	},
4327	.f64 => {
4328	try cg.emitWValue(lhs);
4329	try cg.emitWValue(rhs);
4330	try cg.addTag(.f64_sub);
4331	return .stack;
4332	},
4333	.f80 => return cg.callIntrinsic(.__subxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4334	.f128 => return cg.callIntrinsic(.__subtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4335	}
4336	}
4337
4338	fn floatMul(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4339	switch (ty) {
4340	.f16 => return cg.callIntrinsic(.__mulhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4341	.f32 => {
4342	try cg.emitWValue(lhs);
4343	try cg.emitWValue(rhs);
4344	try cg.addTag(.f32_mul);
4345	return .stack;
4346	},
4347	.f64 => {
4348	try cg.emitWValue(lhs);
4349	try cg.emitWValue(rhs);
4350	try cg.addTag(.f64_mul);
4351	return .stack;
4352	},
4353	.f80 => return cg.callIntrinsic(.__mulxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4354	.f128 => return cg.callIntrinsic(.__multf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4355	}
4356	}
4357
4358	fn floatMulAdd(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue, addend: WValue) InnerError!WValue {
4359	const mul_result = try cg.floatMul(ty, lhs, rhs);
4360	return cg.floatAdd(ty, mul_result, addend);
4361	}
4362
4363	fn floatDiv(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4364	switch (ty) {
4365	.f16 => return cg.callIntrinsic(.__divhf3, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4366	.f32 => {
4367	try cg.emitWValue(lhs);
4368	try cg.emitWValue(rhs);
4369	try cg.addTag(.f32_div);
4370	return .stack;
4371	},
4372	.f64 => {
4373	try cg.emitWValue(lhs);
4374	try cg.emitWValue(rhs);
4375	try cg.addTag(.f64_div);
4376	return .stack;
4377	},
4378	.f80 => return cg.callIntrinsic(.__divxf3, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4379	.f128 => return cg.callIntrinsic(.__divtf3, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4380	}
4381	}
4382
4383	fn floatRem(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4384	switch (ty) {
4385	.f16 => return cg.callIntrinsic(.__fmodh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4386	.f32 => return cg.callIntrinsic(.fmodf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4387	.f64 => return cg.callIntrinsic(.fmod, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4388	.f80 => return cg.callIntrinsic(.__fmodx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4389	.f128 => return cg.callIntrinsic(.fmodf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4390	}
4391	}
4392
4393	// div_trunc(a, b) = trunc(a / b)
4394	fn floatDivTrunc(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4395	const div_result = try cg.floatDiv(ty, lhs, rhs);
4396	return cg.floatTrunc(ty, div_result);
4397	}
4398
4399	// div_floor(a, b) = floor(a / b)
4400	fn floatDivFloor(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4401	const div_result = try cg.floatDiv(ty, lhs, rhs);
4402	return cg.floatFloor(ty, div_result);
4403	}
4404
4405	// div_ceil(a, b) = ceil(a / b)
4406	fn floatDivCeil(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4407	const div_result = try cg.floatDiv(ty, lhs, rhs);
4408	return cg.floatCeil(ty, div_result);
4409	}
4410
4411	// mod(a, b) = fmod(fmod(a, b) + b, b)
4412	fn floatMod(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4413	const r = try cg.floatRem(ty, lhs, rhs);
4414	const s = try cg.floatAdd(ty, r, rhs);
4415	return cg.floatRem(ty, s, rhs);
4416	}
4417
4418	// wasm fN_max NaN semantics differ with Zig
4419	fn floatMax(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4420	switch (ty) {
4421	.f16 => return cg.callIntrinsic(.__fmaxh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4422	.f32 => return cg.callIntrinsic(.fmaxf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4423	.f64 => return cg.callIntrinsic(.fmax, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4424	.f80 => return cg.callIntrinsic(.__fmaxx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4425	.f128 => return cg.callIntrinsic(.fmaxf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4426	}
4427	}
4428
4429	// wasm fN_min NaN semantics differ with Zig
4430	fn floatMin(cg: *CodeGen, ty: FloatType, lhs: WValue, rhs: WValue) InnerError!WValue {
4431	switch (ty) {
4432	.f16 => return cg.callIntrinsic(.__fminh, &.{ .f16_type, .f16_type }, Type.f16, &.{ lhs, rhs }),
4433	.f32 => return cg.callIntrinsic(.fminf, &.{ .f32_type, .f32_type }, Type.f32, &.{ lhs, rhs }),
4434	.f64 => return cg.callIntrinsic(.fmin, &.{ .f64_type, .f64_type }, Type.f64, &.{ lhs, rhs }),
4435	.f80 => return cg.callIntrinsic(.__fminx, &.{ .f80_type, .f80_type }, Type.f80, &.{ lhs, rhs }),
4436	.f128 => return cg.callIntrinsic(.fminf128, &.{ .f128_type, .f128_type }, Type.f128, &.{ lhs, rhs }),
4437	}
4438	}
4439
4440	fn floatSqrt(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4441	switch (ty) {
4442	.f16 => return cg.callIntrinsic(.__sqrth, &.{.f16_type}, Type.f16, &.{arg}),
4443	.f32 => {
4444	try cg.emitWValue(arg);
4445	try cg.addTag(.f32_sqrt);
4446	return .stack;
4447	},
4448	.f64 => {
4449	try cg.emitWValue(arg);
4450	try cg.addTag(.f64_sqrt);
4451	return .stack;
4452	},
4453	.f80 => return cg.callIntrinsic(.__sqrtx, &.{.f80_type}, Type.f80, &.{arg}),
4454	.f128 => return cg.callIntrinsic(.sqrtf128, &.{.f128_type}, Type.f128, &.{arg}),
4455	}
4456	}
4457
4458	fn floatSin(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4459	switch (ty) {
4460	.f16 => return cg.callIntrinsic(.__sinh, &.{.f16_type}, Type.f16, &.{arg}),
4461	.f32 => return cg.callIntrinsic(.sinf, &.{.f32_type}, Type.f32, &.{arg}),
4462	.f64 => return cg.callIntrinsic(.sin, &.{.f64_type}, Type.f64, &.{arg}),
4463	.f80 => return cg.callIntrinsic(.__sinx, &.{.f80_type}, Type.f80, &.{arg}),
4464	.f128 => return cg.callIntrinsic(.sinf128, &.{.f128_type}, Type.f128, &.{arg}),
4465	}
4466	}
4467
4468	fn floatCos(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4469	switch (ty) {
4470	.f16 => return cg.callIntrinsic(.__cosh, &.{.f16_type}, Type.f16, &.{arg}),
4471	.f32 => return cg.callIntrinsic(.cosf, &.{.f32_type}, Type.f32, &.{arg}),
4472	.f64 => return cg.callIntrinsic(.cos, &.{.f64_type}, Type.f64, &.{arg}),
4473	.f80 => return cg.callIntrinsic(.__cosx, &.{.f80_type}, Type.f80, &.{arg}),
4474	.f128 => return cg.callIntrinsic(.cosf128, &.{.f128_type}, Type.f128, &.{arg}),
4475	}
4476	}
4477
4478	fn floatTan(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4479	switch (ty) {
4480	.f16 => return cg.callIntrinsic(.__tanh, &.{.f16_type}, Type.f16, &.{arg}),
4481	.f32 => return cg.callIntrinsic(.tanf, &.{.f32_type}, Type.f32, &.{arg}),
4482	.f64 => return cg.callIntrinsic(.tan, &.{.f64_type}, Type.f64, &.{arg}),
4483	.f80 => return cg.callIntrinsic(.__tanx, &.{.f80_type}, Type.f80, &.{arg}),
4484	.f128 => return cg.callIntrinsic(.tanf128, &.{.f128_type}, Type.f128, &.{arg}),
4485	}
4486	}
4487
4488	fn floatExp(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4489	switch (ty) {
4490	.f16 => return cg.callIntrinsic(.__exph, &.{.f16_type}, Type.f16, &.{arg}),
4491	.f32 => return cg.callIntrinsic(.expf, &.{.f32_type}, Type.f32, &.{arg}),
4492	.f64 => return cg.callIntrinsic(.exp, &.{.f64_type}, Type.f64, &.{arg}),
4493	.f80 => return cg.callIntrinsic(.__expx, &.{.f80_type}, Type.f80, &.{arg}),
4494	.f128 => return cg.callIntrinsic(.expf128, &.{.f128_type}, Type.f128, &.{arg}),
4495	}
4496	}
4497
4498	fn floatExp2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4499	switch (ty) {
4500	.f16 => return cg.callIntrinsic(.__exp2h, &.{.f16_type}, Type.f16, &.{arg}),
4501	.f32 => return cg.callIntrinsic(.exp2f, &.{.f32_type}, Type.f32, &.{arg}),
4502	.f64 => return cg.callIntrinsic(.exp2, &.{.f64_type}, Type.f64, &.{arg}),
4503	.f80 => return cg.callIntrinsic(.__exp2x, &.{.f80_type}, Type.f80, &.{arg}),
4504	.f128 => return cg.callIntrinsic(.exp2f128, &.{.f128_type}, Type.f128, &.{arg}),
4505	}
4506	}
4507
4508	fn floatLog(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4509	switch (ty) {
4510	.f16 => return cg.callIntrinsic(.__logh, &.{.f16_type}, Type.f16, &.{arg}),
4511	.f32 => return cg.callIntrinsic(.logf, &.{.f32_type}, Type.f32, &.{arg}),
4512	.f64 => return cg.callIntrinsic(.log, &.{.f64_type}, Type.f64, &.{arg}),
4513	.f80 => return cg.callIntrinsic(.__logx, &.{.f80_type}, Type.f80, &.{arg}),
4514	.f128 => return cg.callIntrinsic(.logf128, &.{.f128_type}, Type.f128, &.{arg}),
4515	}
4516	}
4517
4518	fn floatLog2(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4519	switch (ty) {
4520	.f16 => return cg.callIntrinsic(.__log2h, &.{.f16_type}, Type.f16, &.{arg}),
4521	.f32 => return cg.callIntrinsic(.log2f, &.{.f32_type}, Type.f32, &.{arg}),
4522	.f64 => return cg.callIntrinsic(.log2, &.{.f64_type}, Type.f64, &.{arg}),
4523	.f80 => return cg.callIntrinsic(.__log2x, &.{.f80_type}, Type.f80, &.{arg}),
4524	.f128 => return cg.callIntrinsic(.log2f128, &.{.f128_type}, Type.f128, &.{arg}),
4525	}
4526	}
4527
4528	fn floatLog10(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4529	switch (ty) {
4530	.f16 => return cg.callIntrinsic(.__log10h, &.{.f16_type}, Type.f16, &.{arg}),
4531	.f32 => return cg.callIntrinsic(.log10f, &.{.f32_type}, Type.f32, &.{arg}),
4532	.f64 => return cg.callIntrinsic(.log10, &.{.f64_type}, Type.f64, &.{arg}),
4533	.f80 => return cg.callIntrinsic(.__log10x, &.{.f80_type}, Type.f80, &.{arg}),
4534	.f128 => return cg.callIntrinsic(.log10f128, &.{.f128_type}, Type.f128, &.{arg}),
4535	}
4536	}
4537
4538	fn floatFloor(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4539	switch (ty) {
4540	.f16 => return cg.callIntrinsic(.__floorh, &.{.f16_type}, Type.f16, &.{arg}),
4541	.f32 => {
4542	try cg.emitWValue(arg);
4543	try cg.addTag(.f32_floor);
4544	return .stack;
4545	},
4546	.f64 => {
4547	try cg.emitWValue(arg);
4548	try cg.addTag(.f64_floor);
4549	return .stack;
4550	},
4551	.f80 => return cg.callIntrinsic(.__floorx, &.{.f80_type}, Type.f80, &.{arg}),
4552	.f128 => return cg.callIntrinsic(.floorf128, &.{.f128_type}, Type.f128, &.{arg}),
4553	}
4554	}
4555
4556	fn floatCeil(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4557	switch (ty) {
4558	.f16 => return cg.callIntrinsic(.__ceilh, &.{.f16_type}, Type.f16, &.{arg}),
4559	.f32 => {
4560	try cg.emitWValue(arg);
4561	try cg.addTag(.f32_ceil);
4562	return .stack;
4563	},
4564	.f64 => {
4565	try cg.emitWValue(arg);
4566	try cg.addTag(.f64_ceil);
4567	return .stack;
4568	},
4569	.f80 => return cg.callIntrinsic(.__ceilx, &.{.f80_type}, Type.f80, &.{arg}),
4570	.f128 => return cg.callIntrinsic(.ceilf128, &.{.f128_type}, Type.f128, &.{arg}),
4571	}
4572	}
4573
4574	fn floatRound(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4575	switch (ty) {
4576	.f16 => return cg.callIntrinsic(.__roundh, &.{.f16_type}, Type.f16, &.{arg}),
4577	.f32 => {
4578	try cg.emitWValue(arg);
4579	try cg.addTag(.f32_nearest);
4580	return .stack;
4581	},
4582	.f64 => {
4583	try cg.emitWValue(arg);
4584	try cg.addTag(.f64_nearest);
4585	return .stack;
4586	},
4587	.f80 => return cg.callIntrinsic(.__roundx, &.{.f80_type}, Type.f80, &.{arg}),
4588	.f128 => return cg.callIntrinsic(.roundf128, &.{.f128_type}, Type.f128, &.{arg}),
4589	}
4590	}
4591
4592	fn floatTrunc(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4593	switch (ty) {
4594	.f16 => return cg.callIntrinsic(.__trunch, &.{.f16_type}, Type.f16, &.{arg}),
4595	.f32 => {
4596	try cg.emitWValue(arg);
4597	try cg.addTag(.f32_trunc);
4598	return .stack;
4599	},
4600	.f64 => {
4601	try cg.emitWValue(arg);
4602	try cg.addTag(.f64_trunc);
4603	return .stack;
4604	},
4605	.f80 => return cg.callIntrinsic(.__truncx, &.{.f80_type}, Type.f80, &.{arg}),
4606	.f128 => return cg.callIntrinsic(.truncf128, &.{.f128_type}, Type.f128, &.{arg}),
4607	}
4608	}
4609
4610	fn floatNeg(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4611	switch (ty) {
4612	.f16 => {
4613	try cg.emitWValue(arg);
4614	try cg.addImm32(0x8000);
4615	try cg.addTag(.i32_xor);
4616	return .stack;
4617	},
4618	.f32 => {
4619	try cg.emitWValue(arg);
4620	try cg.addTag(.f32_neg);
4621	return .stack;
4622	},
4623	.f64 => {
4624	try cg.emitWValue(arg);
4625	try cg.addTag(.f64_neg);
4626	return .stack;
4627	},
4628	.f80, .f128 => {
4629	const result = try cg.allocStack(Type.f128);
4630	try cg.emitWValue(result);
4631	try cg.emitWValue(arg);
4632	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4633	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4634	try cg.emitWValue(result);
4635	try cg.emitWValue(arg);
4636	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4637	if (ty == .f80) {
4638	try cg.addImm64(0x8000);
4639	} else {
4640	try cg.addImm64(0x8000000000000000);
4641	}
4642	try cg.addTag(.i64_xor);
4643	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
4644	return result;
4645	},
4646	}
4647	}
4648
4649	fn floatAbs(cg: *CodeGen, ty: FloatType, arg: WValue) InnerError!WValue {
4650	switch (ty) {
4651	.f16 => {
4652	try cg.emitWValue(arg);
4653	try cg.addImm32(0x7FFF);
4654	try cg.addTag(.i32_and);
4655	return .stack;
4656	},
4657	.f32 => {
4658	try cg.emitWValue(arg);
4659	try cg.addTag(.f32_abs);
4660	return .stack;
4661	},
4662	.f64 => {
4663	try cg.emitWValue(arg);
4664	try cg.addTag(.f64_abs);
4665	return .stack;
4666	},
4667	.f80, .f128 => {
4668	const result = try cg.allocStack(Type.f128);
4669	try cg.emitWValue(result);
4670	try cg.emitWValue(arg);
4671	try cg.addMemArg(.i64_load, .{ .offset = 0 + arg.offset(), .alignment = 2 });
4672	try cg.addMemArg(.i64_store, .{ .offset = 0 + result.offset(), .alignment = 2 });
4673	try cg.emitWValue(result);
4674	try cg.emitWValue(arg);
4675	try cg.addMemArg(.i64_load, .{ .offset = 8 + arg.offset(), .alignment = 2 });
4676	if (ty == .f80) {
4677	try cg.addImm64(0x7FFF);
4678	} else {
4679	try cg.addImm64(0x7FFFFFFFFFFFFFFF);
4680	}
4681	try cg.addTag(.i64_and);
4682	try cg.addMemArg(.i64_store, .{ .offset = 8 + result.offset(), .alignment = 2 });
4683	return result;
4684	},
4685	}
4686	}
4687
4688	fn floatExtendCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4689	switch (dest_ty) {
4690	.f16 => unreachable,
4691	.f32 => switch (src_ty) {
4692	.f16 => {
4693	_ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4694	return .stack;
4695	},
4696	else => unreachable,
4697	},
4698	.f64 => switch (src_ty) {
4699	.f16 => {
4700	_ = try cg.callIntrinsic(.__extendhfsf2, &.{.f16_type}, Type.f32, &.{operand});
4701	try cg.addTag(.f64_promote_f32);
4702	return .stack;
4703	},
4704	.f32 => {
4705	try cg.emitWValue(operand);
4706	try cg.addTag(.f64_promote_f32);
4707	return .stack;
4708	},
4709	else => unreachable,
4710	},
4711	.f80 => switch (src_ty) {
4712	.f16 => return cg.callIntrinsic(.__extendhfxf2, &.{.f16_type}, Type.f80, &.{operand}),
4713	.f32 => return cg.callIntrinsic(.__extendsfxf2, &.{.f32_type}, Type.f80, &.{operand}),
4714	.f64 => return cg.callIntrinsic(.__extenddfxf2, &.{.f64_type}, Type.f80, &.{operand}),
4715	else => unreachable,
4716	},
4717	.f128 => switch (src_ty) {
4718	.f16 => return cg.callIntrinsic(.__extendhftf2, &.{.f16_type}, Type.f128, &.{operand}),
4719	.f32 => return cg.callIntrinsic(.__extendsftf2, &.{.f32_type}, Type.f128, &.{operand}),
4720	.f64 => return cg.callIntrinsic(.__extenddftf2, &.{.f64_type}, Type.f128, &.{operand}),
4721	.f80 => return cg.callIntrinsic(.__extendxftf2, &.{.f80_type}, Type.f128, &.{operand}),
4722	else => unreachable,
4723	},
4724	}
4725	}
4726
4727	fn floatTruncCast(cg: *CodeGen, dest_ty: FloatType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4728	switch (dest_ty) {
4729	.f16 => switch (src_ty) {
4730	.f32 => return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{operand}),
4731	.f64 => {
4732	try cg.emitWValue(operand);
4733	try cg.addTag(.f32_demote_f64);
4734	return cg.callIntrinsic(.__truncsfhf2, &.{.f32_type}, Type.f16, &.{.stack});
4735	},
4736	.f80 => return cg.callIntrinsic(.__truncxfhf2, &.{.f80_type}, Type.f16, &.{operand}),
4737	.f128 => return cg.callIntrinsic(.__trunctfhf2, &.{.f128_type}, Type.f16, &.{operand}),
4738	else => unreachable,
4739	},
4740	.f32 => switch (src_ty) {
4741	.f64 => {
4742	try cg.emitWValue(operand);
4743	try cg.addTag(.f32_demote_f64);
4744	return .stack;
4745	},
4746	.f80 => return cg.callIntrinsic(.__truncxfsf2, &.{.f80_type}, Type.f32, &.{operand}),
4747	.f128 => return cg.callIntrinsic(.__trunctfsf2, &.{.f128_type}, Type.f32, &.{operand}),
4748	else => unreachable,
4749	},
4750	.f64 => switch (src_ty) {
4751	.f80 => return cg.callIntrinsic(.__truncxfdf2, &.{.f80_type}, Type.f64, &.{operand}),
4752	.f128 => return cg.callIntrinsic(.__trunctfdf2, &.{.f128_type}, Type.f64, &.{operand}),
4753	else => unreachable,
4754	},
4755	.f80 => switch (src_ty) {
4756	.f128 => return cg.callIntrinsic(.__trunctfxf2, &.{.f128_type}, Type.f80, &.{operand}),
4757	else => unreachable,
4758	},
4759	.f128 => unreachable,
4760	}
4761	}
4762
4763	fn intFromFloat(cg: *CodeGen, dest_ty: IntType, src_ty: FloatType, operand: WValue) InnerError!WValue {
4764	switch (dest_ty.bits) {
4765	0 => unreachable,
4766	1...32 => switch (src_ty) {
4767	.f16 => {
4768	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfsi else .__fixunshfsi;
4769	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u32, &.{operand});
4770	},
4771	.f32 => {
4772	try cg.emitWValue(operand);
4773	try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f32_s else .i32_trunc_f32_u);
4774	return .stack;
4775	},
4776	.f64 => {
4777	try cg.emitWValue(operand);
4778	try cg.addTag(if (dest_ty.is_signed) .i32_trunc_f64_s else .i32_trunc_f64_u);
4779	return .stack;
4780	},
4781	.f80 => {
4782	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfsi else .__fixunsxfsi;
4783	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u32, &.{operand});
4784	},
4785	.f128 => {
4786	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfsi else .__fixunstfsi;
4787	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u32, &.{operand});
4788	},
4789	},
4790	33...64 => switch (src_ty) {
4791	.f16 => {
4792	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfdi else .__fixunshfdi;
4793	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u64, &.{operand});
4794	},
4795	.f32 => {
4796	try cg.emitWValue(operand);
4797	try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f32_s else .i64_trunc_f32_u);
4798	return .stack;
4799	},
4800	.f64 => {
4801	try cg.emitWValue(operand);
4802	try cg.addTag(if (dest_ty.is_signed) .i64_trunc_f64_s else .i64_trunc_f64_u);
4803	return .stack;
4804	},
4805	.f80 => {
4806	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfdi else .__fixunsxfdi;
4807	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u64, &.{operand});
4808	},
4809	.f128 => {
4810	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfdi else .__fixunstfdi;
4811	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u64, &.{operand});
4812	},
4813	},
4814	65...128 => switch (src_ty) {
4815	.f16 => {
4816	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfti else .__fixunshfti;
4817	return cg.callIntrinsic(intrinsic, &.{.f16_type}, Type.u128, &.{operand});
4818	},
4819	.f32 => {
4820	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfti else .__fixunssfti;
4821	return cg.callIntrinsic(intrinsic, &.{.f32_type}, Type.u128, &.{operand});
4822	},
4823	.f64 => {
4824	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfti else .__fixunsdfti;
4825	return cg.callIntrinsic(intrinsic, &.{.f64_type}, Type.u128, &.{operand});
4826	},
4827	.f80 => {
4828	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfti else .__fixunsxfti;
4829	return cg.callIntrinsic(intrinsic, &.{.f80_type}, Type.u128, &.{operand});
4830	},
4831	.f128 => {
4832	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfti else .__fixunstfti;
4833	return cg.callIntrinsic(intrinsic, &.{.f128_type}, Type.u128, &.{operand});
4834	},
4835	},
4836	else => {
4837	const result = try cg.allocInt(dest_ty);
4838
4839	switch (src_ty) {
4840	.f16 => {
4841	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixhfei else .__fixunshfei;
4842	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f16_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4843	},
4844	.f32 => {
4845	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixsfei else .__fixunssfei;
4846	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f32_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4847	},
4848	.f64 => {
4849	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixdfei else .__fixunsdfei;
4850	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f64_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4851	},
4852	.f80 => {
4853	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixxfei else .__fixunsxfei;
4854	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f80_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4855	},
4856	.f128 => {
4857	const intrinsic: Mir.Intrinsic = if (dest_ty.is_signed) .__fixtfei else .__fixunstfei;
4858	_ = try cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type, .f128_type }, .void, &.{ result, .{ .imm32 = dest_ty.bits }, operand });
4859	},
4860	}
4861
4862	return result;
4863	},
4864	}
4865	}
4866
4867	fn floatFromInt(cg: *CodeGen, dest_ty: FloatType, src_ty: IntType, operand: WValue) InnerError!WValue {
4868	switch (dest_ty) {
4869	.f16 => switch (src_ty.bits) {
4870	0 => unreachable,
4871	1...32 => {
4872	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsihf else .__floatunsihf;
4873	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f16, &.{operand});
4874	},
4875	33...64 => {
4876	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdihf else .__floatundihf;
4877	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f16, &.{operand});
4878	},
4879	65...128 => {
4880	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattihf else .__floatuntihf;
4881	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f16, &.{operand});
4882	},
4883	else => {
4884	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateihf else .__floatuneihf;
4885	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f16, &.{ operand, .{ .imm32 = src_ty.bits } });
4886	},
4887	},
4888	.f32 => switch (src_ty.bits) {
4889	0 => unreachable,
4890	1...32 => {
4891	try cg.emitWValue(operand);
4892	try cg.addTag(if (src_ty.is_signed) .f32_convert_i32_s else .f32_convert_i32_u);
4893	return .stack;
4894	},
4895	33...64 => {
4896	try cg.emitWValue(operand);
4897	try cg.addTag(if (src_ty.is_signed) .f32_convert_i64_s else .f32_convert_i64_u);
4898	return .stack;
4899	},
4900	65...128 => {
4901	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattisf else .__floatuntisf;
4902	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f32, &.{operand});
4903	},
4904	else => {
4905	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateisf else .__floatuneisf;
4906	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f32, &.{ operand, .{ .imm32 = src_ty.bits } });
4907	},
4908	},
4909	.f64 => switch (src_ty.bits) {
4910	0 => unreachable,
4911	1...32 => {
4912	try cg.emitWValue(operand);
4913	try cg.addTag(if (src_ty.is_signed) .f64_convert_i32_s else .f64_convert_i32_u);
4914	return .stack;
4915	},
4916	33...64 => {
4917	try cg.emitWValue(operand);
4918	try cg.addTag(if (src_ty.is_signed) .f64_convert_i64_s else .f64_convert_i64_u);
4919	return .stack;
4920	},
4921	65...128 => {
4922	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattidf else .__floatuntidf;
4923	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f64, &.{operand});
4924	},
4925	else => {
4926	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateidf else .__floatuneidf;
4927	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f64, &.{ operand, .{ .imm32 = src_ty.bits } });
4928	},
4929	},
4930	.f80 => switch (src_ty.bits) {
4931	0 => unreachable,
4932	1...32 => {
4933	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsixf else .__floatunsixf;
4934	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f80, &.{operand});
4935	},
4936	33...64 => {
4937	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatdixf else .__floatundixf;
4938	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f80, &.{operand});
4939	},
4940	65...128 => {
4941	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattixf else .__floatuntixf;
4942	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f80, &.{operand});
4943	},
4944	else => {
4945	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateixf else .__floatuneixf;
4946	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f80, &.{ operand, .{ .imm32 = src_ty.bits } });
4947	},
4948	},
4949	.f128 => switch (src_ty.bits) {
4950	0 => unreachable,
4951	1...32 => {
4952	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatsitf else .__floatunsitf;
4953	return cg.callIntrinsic(intrinsic, &.{.i32_type}, Type.f128, &.{operand});
4954	},
4955	33...64 => {
4956	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floatditf else .__floatunditf;
4957	return cg.callIntrinsic(intrinsic, &.{.i64_type}, Type.f128, &.{operand});
4958	},
4959	65...128 => {
4960	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floattitf else .__floatuntitf;
4961	return cg.callIntrinsic(intrinsic, &.{.i128_type}, Type.f128, &.{operand});
4962	},
4963	else => {
4964	const intrinsic: Mir.Intrinsic = if (src_ty.is_signed) .__floateitf else .__floatuneitf;
4965	return cg.callIntrinsic(intrinsic, &.{ .usize_type, .usize_type }, Type.f128, &.{ operand, .{ .imm32 = src_ty.bits } });
4966	},
4967	},
4968	}
4969	}
4970
4971	fn lowerPtr(cg: *CodeGen, ptr_val: InternPool.Index, prev_offset: u64) InnerError!WValue {
4972	const pt = cg.pt;
4973	const zcu = pt.zcu;
4974	const ip = &zcu.intern_pool;
4975	const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
4976	const offset: u64 = prev_offset + ptr.byte_offset;
4977	return switch (ptr.base_addr) {
4978	.nav => |nav| return if (ip.getNav(nav).getExtern(ip) != null or
4979	Type.fromInterned(ip.getNav(nav).resolved.?.type).isRuntimeFnOrHasRuntimeBits(zcu))
4980	.{ .nav_ref = .{ .nav_index = nav, .offset = @intCast(offset) } }
4981	else
4982	.{ .imm32 = @intCast(zcu.navAlignment(nav).forward(@as(u32, 0xaaaaaaaa))) },
4983	.uav => |uav| return if (Type.fromInterned(ip.typeOf(uav.val)).isRuntimeFnOrHasRuntimeBits(zcu))
4984	.{ .uav_ref = .{ .ip_index = uav.val, .offset = @intCast(offset), .orig_ptr_ty = uav.orig_ty } }
4985	else
4986	.{ .imm32 = @intCast(Type.fromInterned(uav.orig_ty).ptrAlignment(zcu).forward(@as(u32, 0xaaaaaaaa))) },
4987	.int => return cg.lowerConstant(try pt.intValue(.usize, offset)),
4988	.eu_payload => |eu_ptr| try cg.lowerPtr(
4989	eu_ptr,
4990	offset + codegen.errUnionPayloadOffset(
4991	Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
4992	zcu,
4993	),
4994	),
4995	.opt_payload => |opt_ptr| return cg.lowerPtr(opt_ptr, offset),
4996	.field => |field| {
4997	const base_ptr = Value.fromInterned(field.base);
4998	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
4999	const field_off: u64 = switch (base_ty.zigTypeTag(zcu)) {
5000	.pointer => off: {
5001	assert(base_ty.isSlice(zcu));
5002	break :off switch (field.index) {
5003	Value.slice_ptr_index => 0,
5004	Value.slice_len_index => @divExact(cg.target.ptrBitWidth(), 8),
5005	else => unreachable,
5006	};
5007	},
5008	.@"struct" => switch (base_ty.containerLayout(zcu)) {
5009	.auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
5010	.@"extern", .@"packed" => unreachable,
5011	},
5012	.@"union" => switch (base_ty.containerLayout(zcu)) {
5013	.auto => base_ty.structFieldOffset(@intCast(field.index), zcu),
5014	.@"extern", .@"packed" => unreachable,
5015	},
5016	else => unreachable,
5017	};
5018	return cg.lowerPtr(field.base, offset + field_off);
5019	},
5020	.arr_elem, .comptime_field, .comptime_alloc => unreachable,
5021	};
5022	}
5023
5024	/// Asserts that `isByRef` returns `false` for `val.typeOf(zcu)`.
5025	fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
5026	const pt = cg.pt;
5027	const zcu = pt.zcu;
5028	const ty = val.typeOf(zcu);
5029	assert(!isByRef(ty, zcu, cg.target));
5030	const ip = &zcu.intern_pool;
5031	if (val.isUndef(zcu)) return cg.emitUndefined(ty);
5032
5033	switch (ip.indexToKey(val.ip_index)) {
5034	.int_type,
5035	.ptr_type,
5036	.array_type,
5037	.vector_type,
5038	.opt_type,
5039	.anyframe_type,
5040	.error_union_type,
5041	.simple_type,
5042	.struct_type,
5043	.tuple_type,
5044	.union_type,
5045	.opaque_type,
5046	.spirv_type,
5047	.enum_type,
5048	.func_type,
5049	.error_set_type,
5050	.inferred_error_set_type,
5051	=> unreachable, // types, not values
5052
5053	.undef => unreachable, // handled above
5054	.simple_value => |simple_value| switch (simple_value) {
5055	.void,
5056	.null,
5057	.@"unreachable",
5058	=> unreachable, // non-runtime values
5059	.false, .true => return .{ .imm32 = switch (simple_value) {
5060	.false => 0,
5061	.true => 1,
5062	else => unreachable,
5063	} },
5064	},
5065	.@"extern",
5066	.func,
5067	.enum_literal,
5068	=> unreachable, // non-runtime values
5069	.int => {
5070	const int_info = ty.intInfo(zcu);
5071	switch (int_info.signedness) {
5072	.signed => switch (int_info.bits) {
5073	0...32 => return .{ .imm32 = @bitCast(@as(i32, @intCast(val.toSignedInt(zcu)))) },
5074	33...64 => return .{ .imm64 = @bitCast(val.toSignedInt(zcu)) },
5075	else => unreachable,
5076	},
5077	.unsigned => switch (int_info.bits) {
5078	0...32 => return .{ .imm32 = @intCast(val.toUnsignedInt(zcu)) },
5079	33...64 => return .{ .imm64 = val.toUnsignedInt(zcu) },
5080	else => unreachable,
5081	},
5082	}
5083	},
5084	.err => |err| {
5085	const int = try pt.getErrorValue(err.name);
5086	return .{ .imm32 = int };
5087	},
5088	.error_union => |error_union| {
5089	const err_int_ty = try pt.errorIntType();
5090	const err_val: Value = switch (error_union.val) {
5091	.err_name => |err_name| .fromInterned(try pt.intern(.{ .err = .{
5092	.ty = ty.errorUnionSet(zcu).toIntern(),
5093	.name = err_name,
5094	} })),
5095	.payload => try pt.intValue(err_int_ty, 0),
5096	};
5097	const payload_type = ty.errorUnionPayload(zcu);
5098	if (!payload_type.hasRuntimeBits(zcu)) {
5099	// We use the error type directly as the type.
5100	return cg.lowerConstant(err_val);
5101	}
5102
5103	return cg.fail("Wasm TODO: lowerConstant error union with non-zero-bit payload type", .{});
5104	},
5105	.enum_tag => |enum_tag| return cg.lowerConstant(.fromInterned(enum_tag.int)),
5106	.float => |float| switch (float.storage) {
5107	.f16 => |f16_val| return .{ .imm32 = @as(u16, @bitCast(f16_val)) },
5108	.f32 => |f32_val| return .{ .float32 = f32_val },
5109	.f64 => |f64_val| return .{ .float64 = f64_val },
5110	else => unreachable,
5111	},
5112	.slice => unreachable, // isByRef == true
5113	.ptr => return cg.lowerPtr(val.toIntern(), 0),
5114	.opt => if (ty.optionalReprIsPayload(zcu)) {
5115	if (val.optionalValue(zcu)) |payload| {
5116	return cg.lowerConstant(payload);
5117	} else {
5118	return .{ .imm32 = 0 };
5119	}
5120	} else {
5121	return .{ .imm32 = @intFromBool(!val.isNull(zcu)) };
5122	},
5123	.aggregate => switch (ip.indexToKey(ty.ip_index)) {
5124	.array_type => return cg.fail("Wasm TODO: LowerConstant for {f}", .{ty.fmt(pt)}),
5125	.vector_type => {
5126	assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
5127	var buf: [16]u8 = undefined;
5128	val.writeToMemory(zcu, &buf) catch unreachable;
5129	return cg.storeSimdImmd(buf);
5130	},
5131	.struct_type => unreachable, // packed structs use `bitpack`
5132	else => unreachable,
5133	},
5134	.un => unreachable, // packed unions use `bitpack`
5135	.bitpack => |bitpack| return cg.lowerConstant(.fromInterned(bitpack.backing_int_val)),
5136	.memoized_call => unreachable,
5137	}
5138	}
5139
5140	/// Stores the value as a 128bit-immediate value by storing it inside
5141	/// the list and returning the index into this list as `WValue`.
5142	fn storeSimdImmd(cg: *CodeGen, value: [16]u8) !WValue {
5143	const index = @as(u32, @intCast(cg.simd_immediates.items.len));
5144	try cg.simd_immediates.append(cg.gpa, value);
5145	return .{ .imm128 = index };
5146	}
5147
5148	fn emitUndefined(cg: *CodeGen, ty: Type) InnerError!WValue {
5149	const zcu = cg.pt.zcu;
5150	switch (ty.zigTypeTag(zcu)) {
5151	.bool, .error_set => return .{ .imm32 = 0xaaaaaaaa },
5152	.int, .@"enum" => switch (ty.intInfo(zcu).bits) {
5153	0...32 => return .{ .imm32 = 0xaaaaaaaa },
5154	33...64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
5155	else => unreachable,
5156	},
5157	.float => switch (ty.floatBits(cg.target)) {
5158	16 => return .{ .imm32 = 0xaaaaaaaa },
5159	32 => return .{ .float32 = @as(f32, @bitCast(@as(u32, 0xaaaaaaaa))) },
5160	64 => return .{ .float64 = @as(f64, @bitCast(@as(u64, 0xaaaaaaaaaaaaaaaa))) },
5161	else => unreachable,
5162	},
5163	.pointer => switch (cg.ptr_size) {
5164	.wasm32 => return .{ .imm32 = 0xaaaaaaaa },
5165	.wasm64 => return .{ .imm64 = 0xaaaaaaaaaaaaaaaa },
5166	},
5167	.optional => {
5168	const pl_ty = ty.optionalChild(zcu);
5169	if (ty.optionalReprIsPayload(zcu)) {
5170	return cg.emitUndefined(pl_ty);
5171	}
5172	return .{ .imm32 = 0xaaaaaaaa };
5173	},
5174	.error_union => {
5175	return .{ .imm32 = 0xaaaaaaaa };
5176	},
5177	.@"struct", .@"union" => {
5178	const backing_int_ty = ty.backingIntType(zcu);
5179	return cg.emitUndefined(backing_int_ty);
5180	},
5181	else => return cg.fail("Wasm TODO: emitUndefined for type: {t}\n", .{ty.zigTypeTag(zcu)}),
5182	}
5183	}
5184
5185	fn airBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5186	const block = cg.air.unwrapBlock(inst);
5187	try cg.lowerBlock(inst, block.ty, block.body);
5188	}
5189
5190	fn lowerBlock(cg: *CodeGen, inst: Air.Inst.Index, block_ty: Type, body: []const Air.Inst.Index) InnerError!void {
5191	const zcu = cg.pt.zcu;
5192	// if wasm_block_ty is non-empty, we create a register to store the temporary value
5193	const block_result: WValue = if (block_ty.hasRuntimeBits(zcu))
5194	try cg.allocLocal(block_ty)
5195	else
5196	.none;
5197
5198	try cg.startBlock(.block, .empty);
5199	// Here we set the current block idx, so breaks know the depth to jump
5200	// to when breaking out.
5201	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5202	.label = cg.block_depth,
5203	.value = block_result,
5204	});
5205
5206	{
5207	try cg.branches.append(cg.gpa, .{});
5208	defer {
5209	var branch = cg.branches.pop().?;
5210	branch.deinit(cg.gpa);
5211	}
5212	try cg.genBody(body);
5213	try cg.endBlock();
5214	}
5215
5216	return cg.finishAir(inst, block_result, &.{});
5217	}
5218
5219	/// appends a new wasm block to the code section and increases the `block_depth` by 1
5220	fn startBlock(cg: *CodeGen, block_tag: std.wasm.Opcode, block_type: std.wasm.BlockType) !void {
5221	cg.block_depth += 1;
5222	try cg.addInst(.{
5223	.tag = Mir.Inst.Tag.fromOpcode(block_tag),
5224	.data = .{ .block_type = block_type },
5225	});
5226	}
5227
5228	/// Ends the current wasm block and decreases the `block_depth` by 1
5229	fn endBlock(cg: *CodeGen) !void {
5230	try cg.addTag(.end);
5231	cg.block_depth -= 1;
5232	}
5233
5234	fn airLoop(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5235	const block = cg.air.unwrapBlock(inst);
5236
5237	// result type of loop is always 'noreturn', meaning we can always
5238	// emit the wasm type 'block_empty'.
5239	try cg.startBlock(.loop, .empty);
5240
5241	try cg.loops.putNoClobber(cg.gpa, inst, cg.block_depth);
5242	defer assert(cg.loops.remove(inst));
5243
5244	try cg.genBody(block.body);
5245	try cg.endBlock();
5246
5247	return cg.finishAir(inst, .none, &.{});
5248	}
5249
5250	fn airCondBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5251	const cond_br = cg.air.unwrapCondBr(inst);
5252	const condition = try cg.resolveInst(cond_br.condition);
5253	const then_body = cond_br.then_body;
5254	const else_body = cond_br.else_body;
5255
5256	// result type is always noreturn, so use `block_empty` as type.
5257	try cg.startBlock(.block, .empty);
5258	// emit the conditional value
5259	try cg.emitWValue(condition);
5260
5261	// we inserted the block in front of the condition
5262	// so now check if condition matches. If not, break outside this block
5263	// and continue with the then codepath
5264	try cg.addLabel(.br_if, 0);
5265
5266	try cg.branches.ensureUnusedCapacity(cg.gpa, 2);
5267	{
5268	cg.branches.appendAssumeCapacity(.{});
5269	defer {
5270	var else_stack = cg.branches.pop().?;
5271	else_stack.deinit(cg.gpa);
5272	}
5273	try cg.genBody(else_body);
5274	try cg.endBlock();
5275	}
5276
5277	// Outer block that matches the condition
5278	{
5279	cg.branches.appendAssumeCapacity(.{});
5280	defer {
5281	var then_stack = cg.branches.pop().?;
5282	then_stack.deinit(cg.gpa);
5283	}
5284	try cg.genBody(then_body);
5285	}
5286
5287	return cg.finishAir(inst, .none, &.{});
5288	}
5289
5290	fn airCmp(cg: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) InnerError!void {
5291	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
5292	const lhs = try cg.resolveInst(bin_op.lhs);
5293	const rhs = try cg.resolveInst(bin_op.rhs);
5294	const operand_ty = cg.typeOf(bin_op.lhs);
5295	const zcu = cg.pt.zcu;
5296
5297	const type_tag = operand_ty.zigTypeTag(zcu);
5298
5299	if (type_tag == .vector) {
5300	return cg.fail("TODO: implement AIR op: cmp for vectors", .{});
5301	}
5302
5303	if (type_tag == .optional and !operand_ty.optionalReprIsPayload(zcu)) {
5304	const payload_ty = operand_ty.optionalChild(zcu);
5305
5306	if (payload_ty.hasRuntimeBits(zcu)) {
5307	assert(op == .eq or op == .neq);
5308	assert(!isByRef(payload_ty, zcu, cg.target));
5309
5310	var result = try cg.allocLocal(Type.i32);
5311	defer result.free(cg);
5312
5313	var lhs_null = try cg.allocLocal(Type.i32);
5314	defer lhs_null.free(cg);
5315
5316	try cg.startBlock(.block, .empty);
5317
5318	try cg.addImm32(if (op == .eq) 0 else 1);
5319	try cg.addLocal(.local_set, result.local.value);
5320
5321	_ = try cg.isNull(lhs, operand_ty, .i32_eq, .value);
5322	try cg.addLocal(.local_tee, lhs_null.local.value);
5323	_ = try cg.isNull(rhs, operand_ty, .i32_eq, .value);
5324	try cg.addTag(.i32_ne);
5325	try cg.addLabel(.br_if, 0);
5326
5327	try cg.addImm32(if (op == .eq) 1 else 0);
5328	try cg.addLocal(.local_set, result.local.value);
5329
5330	try cg.addLocal(.local_get, lhs_null.local.value);
5331	try cg.addLabel(.br_if, 0);
5332
5333	_ = try cg.load(lhs, payload_ty, 0);
5334	_ = try cg.load(rhs, payload_ty, 0);
5335
5336	if (payload_ty.isAnyFloat()) {
5337	_ = try cg.floatCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5338	} else {
5339	_ = try cg.intCmp(.fromType(cg, payload_ty), op, .stack, .stack);
5340	}
5341
5342	try cg.addLocal(.local_set, result.local.value);
5343	try cg.endBlock();
5344
5345	try cg.addLocal(.local_get, result.local.value);
5346	try cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
5347	} else {
5348	const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5349	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5350	}
5351	} else if (type_tag == .float) {
5352	const result = try cg.floatCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5353	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5354	} else {
5355	const result = try cg.intCmp(.fromType(cg, operand_ty), op, lhs, rhs);
5356	try cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
5357	}
5358	}
5359
5360	fn intCmp(cg: *CodeGen, ty: IntType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
5361	switch (ty.bits) {
5362	0 => unreachable,
5363	1...32 => {
5364	// lhs or rhs could be stack pointers
5365	try cg.lowerToStack(lhs);
5366	try cg.lowerToStack(rhs);
5367	const opcode: Mir.Inst.Tag = switch (op) {
5368	.eq => .i32_eq,
5369	.neq => .i32_ne,
5370	.lt => if (ty.is_signed) .i32_lt_s else .i32_lt_u,
5371	.lte => if (ty.is_signed) .i32_le_s else .i32_le_u,
5372	.gte => if (ty.is_signed) .i32_ge_s else .i32_ge_u,
5373	.gt => if (ty.is_signed) .i32_gt_s else .i32_gt_u,
5374	};
5375	try cg.addTag(opcode);
5376	return .stack;
5377	},
5378	33...64 => {
5379	// lhs or rhs could be stack pointers
5380	try cg.lowerToStack(lhs);
5381	try cg.lowerToStack(rhs);
5382	const opcode: Mir.Inst.Tag = switch (op) {
5383	.eq => .i64_eq,
5384	.neq => .i64_ne,
5385	.lt => if (ty.is_signed) .i64_lt_s else .i64_lt_u,
5386	.lte => if (ty.is_signed) .i64_le_s else .i64_le_u,
5387	.gte => if (ty.is_signed) .i64_ge_s else .i64_ge_u,
5388	.gt => if (ty.is_signed) .i64_gt_s else .i64_gt_u,
5389	};
5390	try cg.addTag(opcode);
5391	return .stack;
5392	},
5393	65...128 => {
5394	var lhs_msb = try (try cg.load(lhs, Type.u64, 8)).toLocal(cg, Type.u64);
5395	defer lhs_msb.free(cg);
5396	var rhs_msb = try (try cg.load(rhs, Type.u64, 8)).toLocal(cg, Type.u64);
5397	defer rhs_msb.free(cg);
5398
5399	switch (op) {
5400	.eq, .neq => {
5401	const xor_high = try cg.intXor(.u64, lhs_msb, rhs_msb);
5402	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5403	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5404	const xor_low = try cg.intXor(.u64, lhs_lsb, rhs_lsb);
5405	const or_result = try cg.intOr(.u64, xor_high, xor_low);
5406
5407	switch (op) {
5408	.eq => return cg.intCmp(.u64, .eq, or_result, .{ .imm64 = 0 }),
5409	.neq => return cg.intCmp(.u64, .neq, or_result, .{ .imm64 = 0 }),
5410	else => unreachable,
5411	}
5412	},
5413	else => {
5414	const word_int_ty: IntType = if (ty.is_signed) .i64 else .u64;
5415
5416	const lhs_lsb = try cg.load(lhs, Type.u64, 0);
5417	const rhs_lsb = try cg.load(rhs, Type.u64, 0);
5418
5419	// leave values on stack for 'select'
5420	_ = try cg.intCmp(.u64, op, lhs_lsb, rhs_lsb);
5421	_ = try cg.intCmp(word_int_ty, op, lhs_msb, rhs_msb);
5422	_ = try cg.intCmp(word_int_ty, .eq, lhs_msb, rhs_msb);
5423	try cg.addTag(.select);
5424	},
5425	}
5426
5427	return .stack;
5428	},
5429	else => {
5430	try cg.lowerToStack(lhs);
5431	try cg.lowerToStack(rhs);
5432	try cg.addImm32(@intFromBool(ty.is_signed));
5433	try cg.addImm32(ty.bits);
5434	try cg.addCallIntrinsic(.__cmp_limb64);
5435	try cg.addImm32(0);
5436	try cg.addTag(switch (op) {
5437	.eq => .i32_eq,
5438	.neq => .i32_ne,
5439	.lt => .i32_lt_s,
5440	.lte => .i32_le_s,
5441	.gte => .i32_ge_s,
5442	.gt => .i32_gt_s,
5443	});
5444	return .stack;
5445	},
5446	}
5447	}
5448
5449	fn floatCmp(cg: *CodeGen, ty: FloatType, op: std.math.CompareOperator, lhs: WValue, rhs: WValue) InnerError!WValue {
5450	switch (ty) {
5451	.f16 => {
5452	_ = try cg.floatExtendCast(.f32, .f16, lhs);
5453	_ = try cg.floatExtendCast(.f32, .f16, rhs);
5454	try cg.addTag(switch (op) {
5455	.eq => .f32_eq,
5456	.neq => .f32_ne,
5457	.lt => .f32_lt,
5458	.lte => .f32_le,
5459	.gte => .f32_ge,
5460	.gt => .f32_gt,
5461	});
5462	return .stack;
5463	},
5464	.f32 => {
5465	try cg.emitWValue(lhs);
5466	try cg.emitWValue(rhs);
5467	try cg.addTag(switch (op) {
5468	.eq => .f32_eq,
5469	.neq => .f32_ne,
5470	.lt => .f32_lt,
5471	.lte => .f32_le,
5472	.gte => .f32_ge,
5473	.gt => .f32_gt,
5474	});
5475	return .stack;
5476	},
5477	.f64 => {
5478	try cg.emitWValue(lhs);
5479	try cg.emitWValue(rhs);
5480	try cg.addTag(switch (op) {
5481	.eq => .f64_eq,
5482	.neq => .f64_ne,
5483	.lt => .f64_lt,
5484	.lte => .f64_le,
5485	.gte => .f64_ge,
5486	.gt => .f64_gt,
5487	});
5488	return .stack;
5489	},
5490	.f80 => {
5491	const intrinsic: Mir.Intrinsic = switch (op) {
5492	.lt => .__ltxf2,
5493	.lte => .__lexf2,
5494	.eq => .__eqxf2,
5495	.neq => .__nexf2,
5496	.gte => .__gexf2,
5497	.gt => .__gtxf2,
5498	};
5499	const result = try cg.callIntrinsic(intrinsic, &.{ .f80_type, .f80_type }, Type.bool, &.{ lhs, rhs });
5500	return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
5501	},
5502	.f128 => {
5503	const intrinsic: Mir.Intrinsic = switch (op) {
5504	.lt => .__lttf2,
5505	.lte => .__letf2,
5506	.eq => .__eqtf2,
5507	.neq => .__netf2,
5508	.gte => .__getf2,
5509	.gt => .__gttf2,
5510	};
5511	const result = try cg.callIntrinsic(intrinsic, &.{ .f128_type, .f128_type }, Type.bool, &.{ lhs, rhs });
5512	return cg.intCmp(.i32, op, result, .{ .imm32 = 0 });
5513	},
5514	}
5515	}
5516
5517	fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5518	_ = inst;
5519	return cg.fail("TODO implement airCmpVector for wasm", .{});
5520	}
5521
5522	fn airCmpLteErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5523	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
5524	const operand = try cg.resolveInst(un_op);
5525
5526	try cg.emitWValue(operand);
5527	const pt = cg.pt;
5528	const err_int_ty = try pt.errorIntType();
5529	try cg.addTag(.errors_len);
5530	const result = try cg.intCmp(.fromType(cg, err_int_ty), .lt, .stack, .stack);
5531
5532	return cg.finishAir(inst, result, &.{un_op});
5533	}
5534
5535	fn airBr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5536	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
5537	const block = cg.blocks.get(br.block_inst).?;
5538
5539	// if operand has codegen bits we should break with a value
5540	if (block.value != .none) {
5541	const operand = try cg.resolveInst(br.operand);
5542	try cg.lowerToStack(operand);
5543	try cg.addLocal(.local_set, block.value.local.value);
5544	}
5545
5546	// We map every block to its block index.
5547	// We then determine how far we have to jump to it by subtracting it from current block depth
5548	const idx: u32 = cg.block_depth - block.label;
5549	try cg.addLabel(.br, idx);
5550
5551	return cg.finishAir(inst, .none, &.{br.operand});
5552	}
5553
5554	fn airRepeat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5555	const repeat = cg.air.instructions.items(.data)[@backingInt(inst)].repeat;
5556	const loop_label = cg.loops.get(repeat.loop_inst).?;
5557
5558	const idx: u32 = cg.block_depth - loop_label;
5559	try cg.addLabel(.br, idx);
5560
5561	return cg.finishAir(inst, .none, &.{});
5562	}
5563
5564	fn airTrap(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5565	try cg.addTag(.@"unreachable");
5566	return cg.finishAir(inst, .none, &.{});
5567	}
5568
5569	fn airBreakpoint(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5570	// unsupported by wasm itfunc. Can be implemented once we support DWARF
5571	// for wasm
5572	try cg.addTag(.@"unreachable");
5573	return cg.finishAir(inst, .none, &.{});
5574	}
5575
5576	fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5577	try cg.addTag(.@"unreachable");
5578	return cg.finishAir(inst, .none, &.{});
5579	}
5580
5581	fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5582	const zcu = cg.pt.zcu;
5583	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5584
5585	const operand_ty = cg.typeOf(ty_op.operand);
5586	const dest_ty = cg.typeOfIndex(inst);
5587	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5588	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5589	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5590
5591	const operand = try cg.resolveInst(ty_op.operand);
5592	const result = cg.reuseOperand(ty_op.operand, operand);
5593	return cg.finishAir(inst, result, &.{ty_op.operand});
5594	}
5595
5596	fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5597	const zcu = cg.pt.zcu;
5598	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5599
5600	const operand_ty = cg.typeOf(ty_op.operand);
5601	const dest_ty = cg.typeOfIndex(inst);
5602	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5603	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5604	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5605
5606	const operand = try cg.resolveInst(ty_op.operand);
5607	const result = switch (operand) {
5608	.stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5609	else => cg.reuseOperand(ty_op.operand, operand),
5610	};
5611	return cg.finishAir(inst, result, &.{ty_op.operand});
5612	}
5613
5614	fn airUnionFromEnum(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5615	const zcu = cg.pt.zcu;
5616	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5617
5618	const union_ty = cg.typeOfIndex(inst);
5619	const enum_ty = cg.typeOf(ty_op.operand);
5620	const layout = union_ty.unionGetLayout(zcu);
5621
5622	const enum_value = try cg.resolveInst(ty_op.operand);
5623	const result = try cg.allocStack(union_ty);
5624	try cg.store(result, enum_value, enum_ty, @intCast(layout.tagOffset()));
5625
5626	return cg.finishAir(inst, result, &.{ty_op.operand});
5627	}
5628
5629	fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5630	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5631	const operand = try cg.resolveInst(ty_op.operand);
5632	const dest_ty = cg.typeOfIndex(inst);
5633	const src_ty = cg.typeOf(ty_op.operand);
5634
5635	const result = (try cg.bitcast(dest_ty, src_ty, operand)) orelse cg.reuseOperand(ty_op.operand, operand);
5636
5637	return cg.finishAir(inst, result, &.{ty_op.operand});
5638	}
5639
5640	const BitcastClass = union(enum) {
5641	int: IntType,
5642	float: FloatType,
5643	aggregate, // arrays and vectors.
5644	};
5645
5646	fn bitcastClass(cg: *CodeGen, ty: Type) BitcastClass {
5647	const zcu = cg.pt.zcu;
5648	return switch (ty.zigTypeTag(zcu)) {
5649	.bool,
5650	.int,
5651	.@"enum",
5652	.error_set,
5653	.@"struct",
5654	.@"union",
5655	=> .{ .int = .fromType(cg, ty) },
5656	.float => .{ .float = .fromType(cg, ty) },
5657	.array, .vector => .aggregate,
5658	else => unreachable,
5659	};
5660	}
5661
5662	fn bitcast(cg: *CodeGen, dest_ty: Type, src_ty: Type, operand: WValue) InnerError!?WValue {
5663	if (dest_ty.eql(src_ty)) return null;
5664
5665	const zcu = cg.pt.zcu;
5666	const src_class = cg.bitcastClass(src_ty);
5667	const dest_class = cg.bitcastClass(dest_ty);
5668	const src_by_ref = isByRef(src_ty, zcu, cg.target);
5669	const dest_by_ref = isByRef(dest_ty, zcu, cg.target);
5670
5671	const needs_wrapping = switch (dest_class) {
5672	.int => |dest_int| dest_int.bits != cg.intBackingBits(dest_int.bits) and
5673	switch (src_class) {
5674	.int => |src_int| src_int.is_signed != dest_int.is_signed,
5675	.float, .aggregate => true,
5676	},
5677	.float, .aggregate => false,
5678	};
5679
5680	if (src_by_ref and dest_by_ref) {
5681	if (needs_wrapping) return try cg.intWrap(dest_class.int, operand);
5682	return null;
5683	}
5684
5685	if (dest_by_ref) {
5686	const result = try cg.allocStack(src_ty);
5687	try cg.store(result, operand, src_ty, 0);
5688	return result;
5689	}
5690
5691	if (src_by_ref) {
5692	return try cg.load(operand, dest_ty, 0);
5693	}
5694
5695	switch (src_class) {
5696	.float => |float_ty| switch (float_ty) {
5697	.f16 => return try cg.intWrap(dest_class.int, operand),
5698	.f32 => {
5699	try cg.emitWValue(operand);
5700	try cg.addTag(.i32_reinterpret_f32);
5701	return .stack;
5702	},
5703	.f64 => {
5704	try cg.emitWValue(operand);
5705	try cg.addTag(.i64_reinterpret_f64);
5706	return .stack;
5707	},
5708	.f80, .f128 => unreachable,
5709	},
5710	.int => {},
5711	.aggregate => unreachable,
5712	}
5713
5714	switch (dest_class) {
5715	.float => |float_ty| switch (float_ty) {
5716	.f16 => return null,
5717	.f32 => {
5718	try cg.emitWValue(operand);
5719	try cg.addTag(.f32_reinterpret_i32);
5720	return .stack;
5721	},
5722	.f64 => {
5723	try cg.emitWValue(operand);
5724	try cg.addTag(.f64_reinterpret_i64);
5725	return .stack;
5726	},
5727	.f80, .f128 => unreachable,
5728	},
5729	.int => {},
5730	.aggregate => unreachable,
5731	}
5732
5733	if (needs_wrapping) {
5734	return try cg.intWrap(dest_class.int, operand);
5735	}
5736
5737	return null;
5738	}
5739
5740	fn airStructFieldPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5741	const zcu = cg.pt.zcu;
5742	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5743	const extra = cg.air.extraData(Air.StructField, ty_pl.payload);
5744
5745	const struct_ptr = try cg.resolveInst(extra.data.struct_operand);
5746	const struct_ptr_ty = cg.typeOf(extra.data.struct_operand);
5747	const struct_ty = struct_ptr_ty.childType(zcu);
5748	const result = try cg.structFieldPtr(inst, extra.data.struct_operand, struct_ptr, struct_ptr_ty, struct_ty, extra.data.field_index);
5749	return cg.finishAir(inst, result, &.{extra.data.struct_operand});
5750	}
5751
5752	fn airStructFieldPtrIndex(cg: *CodeGen, inst: Air.Inst.Index, index: u32) InnerError!void {
5753	const zcu = cg.pt.zcu;
5754	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
5755	const struct_ptr = try cg.resolveInst(ty_op.operand);
5756	const struct_ptr_ty = cg.typeOf(ty_op.operand);
5757	const struct_ty = struct_ptr_ty.childType(zcu);
5758
5759	const result = try cg.structFieldPtr(inst, ty_op.operand, struct_ptr, struct_ptr_ty, struct_ty, index);
5760	return cg.finishAir(inst, result, &.{ty_op.operand});
5761	}
5762
5763	fn structFieldPtr(
5764	cg: *CodeGen,
5765	inst: Air.Inst.Index,
5766	ref: Air.Inst.Ref,
5767	struct_ptr: WValue,
5768	struct_ptr_ty: Type,
5769	struct_ty: Type,
5770	index: u32,
5771	) InnerError!WValue {
5772	const pt = cg.pt;
5773	const zcu = pt.zcu;
5774	const result_ty = cg.typeOfIndex(inst);
5775	const struct_ptr_ty_info = struct_ptr_ty.ptrInfo(zcu);
5776
5777	const offset = switch (struct_ty.containerLayout(zcu)) {
5778	.@"packed" => switch (struct_ty.zigTypeTag(zcu)) {
5779	.@"struct" => offset: {
5780	if (result_ty.ptrInfo(zcu).packed_offset.host_size != 0) {
5781	break :offset @as(u32, 0);
5782	}
5783	const struct_type = zcu.typeToStruct(struct_ty).?;
5784	break :offset @divExact(zcu.structPackedFieldBitOffset(struct_type, index) + struct_ptr_ty_info.packed_offset.bit_offset, 8);
5785	},
5786	.@"union" => 0,
5787	else => unreachable,
5788	},
5789	else => struct_ty.structFieldOffset(index, zcu),
5790	};
5791	// save a load and store when we can simply reuse the operand
5792	if (offset == 0) {
5793	return cg.reuseOperand(ref, struct_ptr);
5794	}
5795	switch (struct_ptr) {
5796	.stack_offset => |stack_offset| {
5797	return .{ .stack_offset = .{ .value = stack_offset.value + @as(u32, @intCast(offset)), .references = 1 } };
5798	},
5799	else => return cg.buildPointerOffset(struct_ptr, offset, .new),
5800	}
5801	}
5802
5803	fn airAggFieldVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5804	const pt = cg.pt;
5805	const zcu = pt.zcu;
5806	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
5807	const struct_field = cg.air.extraData(Air.StructField, ty_pl.payload).data;
5808
5809	const struct_ty = cg.typeOf(struct_field.struct_operand);
5810	const operand = try cg.resolveInst(struct_field.struct_operand);
5811	const field_index = struct_field.field_index;
5812	const field_ty = struct_ty.fieldType(field_index, zcu);
5813	if (!field_ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{struct_field.struct_operand});
5814
5815	const result: WValue = switch (struct_ty.containerLayout(zcu)) {
5816	.@"packed" => unreachable, // legalize .expand_packed_agg_field_val
5817	else => result: {
5818	const offset = std.math.cast(u32, struct_ty.structFieldOffset(field_index, zcu)) orelse {
5819	return cg.fail("Field type '{f}' too big to fit into stack frame", .{field_ty.fmt(pt)});
5820	};
5821	if (isByRef(field_ty, zcu, cg.target)) {
5822	switch (operand) {
5823	.stack_offset => |stack_offset| {
5824	break :result .{ .stack_offset = .{ .value = stack_offset.value + offset, .references = 1 } };
5825	},
5826	else => break :result try cg.buildPointerOffset(operand, offset, .new),
5827	}
5828	}
5829	break :result try cg.load(operand, field_ty, offset);
5830	},
5831	};
5832
5833	return cg.finishAir(inst, result, &.{struct_field.struct_operand});
5834	}
5835
5836	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index, is_dispatch_loop: bool) InnerError!void {
5837	const pt = cg.pt;
5838	const zcu = pt.zcu;
5839
5840	const switch_br = cg.air.unwrapSwitch(inst);
5841	const target_ty = cg.typeOf(switch_br.operand);
5842
5843	assert(target_ty.hasRuntimeBits(zcu));
5844
5845	// swap target value with placeholder local, for dispatching
5846	const target = if (is_dispatch_loop) target: {
5847	const initial_target = try cg.resolveInst(switch_br.operand);
5848	const target: WValue = try cg.allocLocal(target_ty);
5849	try cg.lowerToStack(initial_target);
5850	try cg.addLocal(.local_set, target.local.value);
5851
5852	try cg.startBlock(.loop, .empty); // dispatch loop start
5853	try cg.blocks.putNoClobber(cg.gpa, inst, .{
5854	.label = cg.block_depth,
5855	.value = target,
5856	});
5857
5858	break :target target;
5859	} else try cg.resolveInst(switch_br.operand);
5860
5861	const has_else_body = switch_br.else_body_len != 0;
5862	const branch_count = switch_br.cases_len + 1; // if else branch is missing, we trap when failing all conditions
5863	try cg.branches.ensureUnusedCapacity(cg.gpa, switch_br.cases_len + @intFromBool(has_else_body));
5864
5865	if (switch_br.cases_len == 0) {
5866	assert(has_else_body);
5867
5868	var it = switch_br.iterateCases();
5869	const else_body = it.elseBody();
5870
5871	cg.branches.appendAssumeCapacity(.{});
5872	defer {
5873	var else_branch = cg.branches.pop().?;
5874	else_branch.deinit(cg.gpa);
5875	}
5876	try cg.genBody(else_body);
5877
5878	if (is_dispatch_loop) {
5879	try cg.endBlock(); // dispatch loop end
5880	}
5881	return cg.finishAir(inst, .none, &.{});
5882	}
5883
5884	var min: ?Value = null;
5885	var max: ?Value = null;
5886	var branching_size: u32 = 0; // single item +1, range +2
5887
5888	{
5889	var cases_it = switch_br.iterateCases();
5890	while (cases_it.next()) |case| {
5891	for (case.items) |item| {
5892	const val = Value.fromInterned(item.toInterned().?);
5893	if (min == null or val.compareHetero(.lt, min.?, zcu)) min = val;
5894	if (max == null or val.compareHetero(.gt, max.?, zcu)) max = val;
5895	branching_size += 1;
5896	}
5897	for (case.ranges) |range| {
5898	const low = Value.fromInterned(range[0].toInterned().?);
5899	if (min == null or low.compareHetero(.lt, min.?, zcu)) min = low;
5900	const high = Value.fromInterned(range[1].toInterned().?);
5901	if (max == null or high.compareHetero(.gt, max.?, zcu)) max = high;
5902	branching_size += 2;
5903	}
5904	}
5905	}
5906
5907	var min_space: Value.BigIntSpace = undefined;
5908	const min_bigint = min.?.toBigInt(&min_space, zcu);
5909	var max_space: Value.BigIntSpace = undefined;
5910	const max_bigint = max.?.toBigInt(&max_space, zcu);
5911	const limbs = try cg.gpa.alloc(
5912	std.math.big.Limb,
5913	@max(min_bigint.limbs.len, max_bigint.limbs.len) + 1,
5914	);
5915	defer cg.gpa.free(limbs);
5916
5917	const width_maybe: ?u32 = width: {
5918	var width_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5919	width_bigint.sub(max_bigint, min_bigint);
5920	width_bigint.addScalar(width_bigint.toConst(), 1);
5921	break :width width_bigint.toConst().toInt(u32) catch null;
5922	};
5923
5924	try cg.startBlock(.block, .empty); // whole switch block start
5925
5926	for (0..branch_count) |_| {
5927	try cg.startBlock(.block, .empty);
5928	}
5929
5930	// Heuristic on deciding when to use .br_table instead of .br_if jump table
5931	// 1. Differences between lowest and highest values should fit into u32
5932	// 2. .br_table should be applied for "dense" switch, we test it by checking .br_if jumps will need more instructions
5933	// 3. Do not use .br_table for tiny switches
5934	const use_br_table = cond: {
5935	const width = width_maybe orelse break :cond false;
5936	if (width > 2 * branching_size) break :cond false;
5937	if (width < 2 or branch_count < 2) break :cond false;
5938	break :cond true;
5939	};
5940
5941	const int_ty: IntType = .fromType(cg, target_ty);
5942
5943	if (use_br_table) {
5944	const width = width_maybe.?;
5945
5946	const br_value_original = try cg.intSub(int_ty, target, try cg.resolveValue(min.?));
5947	_ = try cg.intCast(.u32, int_ty, br_value_original);
5948
5949	const jump_table: Mir.JumpTable = .{ .length = width + 1 };
5950	const table_extra_index = try cg.addExtra(jump_table);
5951	try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
5952
5953	const branch_list = try cg.mir_extra.addManyAsSlice(cg.gpa, width + 1);
5954	@memset(branch_list, branch_count - 1);
5955
5956	var cases_it = switch_br.iterateCases();
5957	while (cases_it.next()) |case| {
5958	for (case.items) |item| {
5959	const val = Value.fromInterned(item.toInterned().?);
5960	var val_space: Value.BigIntSpace = undefined;
5961	const val_bigint = val.toBigInt(&val_space, zcu);
5962	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5963	index_bigint.sub(val_bigint, min_bigint);
5964	branch_list[index_bigint.toConst().toInt(u32) catch unreachable] = case.idx;
5965	}
5966	for (case.ranges) |range| {
5967	var low_space: Value.BigIntSpace = undefined;
5968	const low_bigint = Value.fromInterned(range[0].toInterned().?).toBigInt(&low_space, zcu);
5969	var high_space: Value.BigIntSpace = undefined;
5970	const high_bigint = Value.fromInterned(range[1].toInterned().?).toBigInt(&high_space, zcu);
5971	var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
5972	index_bigint.sub(low_bigint, min_bigint);
5973	const start = index_bigint.toConst().toInt(u32) catch unreachable;
5974	index_bigint.sub(high_bigint, min_bigint);
5975	const end = (index_bigint.toConst().toInt(u32) catch unreachable) + 1;
5976	@memset(branch_list[start..end], case.idx);
5977	}
5978	}
5979	} else {
5980	var cases_it = switch_br.iterateCases();
5981	while (cases_it.next()) |case| {
5982	for (case.items) |ref| {
5983	const val = try cg.resolveInst(ref);
5984	_ = try cg.intCmp(int_ty, .eq, target, val);
5985	try cg.addLabel(.br_if, case.idx); // item match found
5986	}
5987	for (case.ranges) |range| {
5988	const low = try cg.resolveInst(range[0]);
5989	const high = try cg.resolveInst(range[1]);
5990
5991	const gte = try cg.intCmp(int_ty, .gte, target, low);
5992	const lte = try cg.intCmp(int_ty, .lte, target, high);
5993	_ = try cg.intAnd(.u32, gte, lte);
5994	try cg.addLabel(.br_if, case.idx); // range match found
5995	}
5996	}
5997	try cg.addLabel(.br, branch_count - 1);
5998	}
5999
6000	var cases_it = switch_br.iterateCases();
6001	while (cases_it.next()) |case| {
6002	try cg.endBlock();
6003
6004	cg.branches.appendAssumeCapacity(.{});
6005	defer {
6006	var case_branch = cg.branches.pop().?;
6007	case_branch.deinit(cg.gpa);
6008	}
6009	try cg.genBody(case.body);
6010
6011	try cg.addLabel(.br, branch_count - case.idx - 1); // matching case found and executed => exit switch
6012	}
6013
6014	try cg.endBlock();
6015	if (has_else_body) {
6016	const else_body = cases_it.elseBody();
6017
6018	cg.branches.appendAssumeCapacity(.{});
6019	defer {
6020	var else_branch = cg.branches.pop().?;
6021	else_branch.deinit(cg.gpa);
6022	}
6023	try cg.genBody(else_body);
6024	} else {
6025	try cg.addTag(.@"unreachable");
6026	}
6027
6028	try cg.endBlock(); // whole switch block end
6029
6030	if (is_dispatch_loop) {
6031	try cg.endBlock(); // dispatch loop end
6032	}
6033
6034	return cg.finishAir(inst, .none, &.{});
6035	}
6036
6037	fn airSwitchDispatch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6038	const br = cg.air.instructions.items(.data)[@backingInt(inst)].br;
6039	const switch_loop = cg.blocks.get(br.block_inst).?;
6040
6041	const operand = try cg.resolveInst(br.operand);
6042	try cg.lowerToStack(operand);
6043	try cg.addLocal(.local_set, switch_loop.value.local.value);
6044
6045	const idx: u32 = cg.block_depth - switch_loop.label;
6046	try cg.addLabel(.br, idx);
6047
6048	return cg.finishAir(inst, .none, &.{br.operand});
6049	}
6050
6051	fn airIsErr(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: enum { value, ptr }) InnerError!void {
6052	const zcu = cg.pt.zcu;
6053	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
6054	const operand = try cg.resolveInst(un_op);
6055	const err_union_ty = switch (op_kind) {
6056	.value => cg.typeOf(un_op),
6057	.ptr => cg.typeOf(un_op).childType(zcu),
6058	};
6059	const pl_ty = err_union_ty.errorUnionPayload(zcu);
6060
6061	const result: WValue = result: {
6062	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6063	switch (opcode) {
6064	.i32_ne => break :result .{ .imm32 = 0 },
6065	.i32_eq => break :result .{ .imm32 = 1 },
6066	else => unreachable,
6067	}
6068	}
6069
6070	try cg.emitWValue(operand);
6071	if (op_kind == .ptr or pl_ty.hasRuntimeBits(zcu)) {
6072	try cg.addMemArg(.i32_load16_u, .{
6073	.offset = operand.offset() + @as(u32, @intCast(errUnionErrorOffset(pl_ty, zcu))),
6074	.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
6075	});
6076	}
6077
6078	// Compare the error value with '0'
6079	try cg.addImm32(0);
6080	try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
6081	break :result .stack;
6082	};
6083	return cg.finishAir(inst, result, &.{un_op});
6084	}
6085
6086	/// E!T -> T op_is_ptr == false
6087	/// *(E!T) -> *T op_is_prt == true
6088	fn airUnwrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
6089	const zcu = cg.pt.zcu;
6090	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6091
6092	const operand = try cg.resolveInst(ty_op.operand);
6093	const op_ty = cg.typeOf(ty_op.operand);
6094	const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
6095	const payload_ty = eu_ty.errorUnionPayload(zcu);
6096
6097	const result: WValue = result: {
6098	if (!payload_ty.hasRuntimeBits(zcu)) {
6099	if (op_is_ptr) {
6100	break :result cg.reuseOperand(ty_op.operand, operand);
6101	} else {
6102	break :result .none;
6103	}
6104	}
6105
6106	const pl_offset: u32 = @intCast(errUnionPayloadOffset(payload_ty, zcu));
6107	if (op_is_ptr or isByRef(payload_ty, zcu, cg.target)) {
6108	break :result try cg.buildPointerOffset(operand, pl_offset, .new);
6109	} else {
6110	assert(isByRef(eu_ty, zcu, cg.target));
6111	break :result try cg.load(operand, payload_ty, pl_offset);
6112	}
6113	};
6114	return cg.finishAir(inst, result, &.{ty_op.operand});
6115	}
6116
6117	/// E!T -> E op_is_ptr == false
6118	/// *(E!T) -> E op_is_ptr == true
6119	/// NOTE: op_is_ptr will not change return type
6120	fn airUnwrapErrUnionError(cg: *CodeGen, inst: Air.Inst.Index, op_is_ptr: bool) InnerError!void {
6121	const zcu = cg.pt.zcu;
6122	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6123
6124	const operand = try cg.resolveInst(ty_op.operand);
6125	const op_ty = cg.typeOf(ty_op.operand);
6126	const eu_ty = if (op_is_ptr) op_ty.childType(zcu) else op_ty;
6127	const payload_ty = eu_ty.errorUnionPayload(zcu);
6128
6129	const result: WValue = result: {
6130	if (eu_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6131	break :result .{ .imm32 = 0 };
6132	}
6133
6134	const err_offset: u32 = @intCast(errUnionErrorOffset(payload_ty, zcu));
6135	if (op_is_ptr or isByRef(eu_ty, zcu, cg.target)) {
6136	break :result try cg.load(operand, Type.anyerror, err_offset);
6137	} else {
6138	assert(!payload_ty.hasRuntimeBits(zcu));
6139	break :result cg.reuseOperand(ty_op.operand, operand);
6140	}
6141	};
6142	return cg.finishAir(inst, result, &.{ty_op.operand});
6143	}
6144
6145	fn airWrapErrUnionPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6146	const zcu = cg.pt.zcu;
6147	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6148
6149	const operand = try cg.resolveInst(ty_op.operand);
6150	const err_ty = cg.typeOfIndex(inst);
6151
6152	const pl_ty = cg.typeOf(ty_op.operand);
6153	const result = result: {
6154	if (!pl_ty.hasRuntimeBits(zcu)) {
6155	break :result cg.reuseOperand(ty_op.operand, operand);
6156	}
6157
6158	const err_union = try cg.allocStack(err_ty);
6159	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6160	try cg.store(payload_ptr, operand, pl_ty, 0);
6161
6162	// ensure we also write '0' to the error part, so any present stack value gets overwritten by it.
6163	try cg.emitWValue(err_union);
6164	try cg.addImm32(0);
6165	const err_val_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
6166	try cg.addMemArg(.i32_store16, .{
6167	.offset = err_union.offset() + err_val_offset,
6168	.alignment = 2,
6169	});
6170	break :result err_union;
6171	};
6172	return cg.finishAir(inst, result, &.{ty_op.operand});
6173	}
6174
6175	fn airWrapErrUnionErr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6176	const zcu = cg.pt.zcu;
6177	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6178
6179	const operand = try cg.resolveInst(ty_op.operand);
6180	const err_ty = ty_op.ty;
6181	const pl_ty = err_ty.errorUnionPayload(zcu);
6182
6183	const result = result: {
6184	if (!pl_ty.hasRuntimeBits(zcu)) {
6185	break :result cg.reuseOperand(ty_op.operand, operand);
6186	}
6187
6188	const err_union = try cg.allocStack(err_ty);
6189	// store error value
6190	try cg.store(err_union, operand, Type.anyerror, @intCast(errUnionErrorOffset(pl_ty, zcu)));
6191
6192	// write 'undefined' to the payload
6193	const payload_ptr = try cg.buildPointerOffset(err_union, @as(u32, @intCast(errUnionPayloadOffset(pl_ty, zcu))), .new);
6194	const len = @as(u32, @intCast(err_ty.errorUnionPayload(zcu).abiSize(zcu)));
6195	try cg.memset(Type.u8, payload_ptr, .{ .imm32 = len }, .{ .imm32 = 0xaa });
6196
6197	break :result err_union;
6198	};
6199	return cg.finishAir(inst, result, &.{ty_op.operand});
6200	}
6201
6202	const OpKind = enum { value, ptr };
6203
6204	fn airIsNull(cg: *CodeGen, inst: Air.Inst.Index, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!void {
6205	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
6206	const operand = try cg.resolveInst(un_op);
6207
6208	const op_ty = cg.typeOf(un_op);
6209	const result = try cg.isNull(operand, op_ty, opcode, op_kind);
6210	return cg.finishAir(inst, result, &.{un_op});
6211	}
6212
6213	/// For a given type and operand, checks if it's considered `null`.
6214	/// NOTE: Leaves the result on the stack
6215	fn isNull(cg: *CodeGen, operand: WValue, op_ty: Type, opcode: std.wasm.Opcode, op_kind: OpKind) InnerError!WValue {
6216	const pt = cg.pt;
6217	const zcu = pt.zcu;
6218	try cg.emitWValue(operand);
6219	const optional_ty = switch (op_kind) {
6220	.value => op_ty,
6221	.ptr => op_ty.childType(zcu),
6222	};
6223	const payload_ty = optional_ty.optionalChild(zcu);
6224	if (!optional_ty.optionalReprIsPayload(zcu)) {
6225	// When payload is zero-bits, we can treat operand as a value, rather than
6226	// a pointer to the stack value
6227	if (payload_ty.hasRuntimeBits(zcu)) {
6228	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6229	return cg.fail("Optional type {f} too big to fit into stack frame", .{optional_ty.fmt(pt)});
6230	};
6231	try cg.addMemArg(.i32_load8_u, .{ .offset = operand.offset() + offset, .alignment = 1 });
6232	}
6233	} else if (payload_ty.isSlice(zcu)) {
6234	switch (cg.ptr_size) {
6235	.wasm32 => try cg.addMemArg(.i32_load, .{ .offset = operand.offset(), .alignment = 4 }),
6236	.wasm64 => try cg.addMemArg(.i64_load, .{ .offset = operand.offset(), .alignment = 8 }),
6237	}
6238	} else {
6239	if (op_kind == .ptr) {
6240	try cg.addMemArg(.i32_load, .{
6241	.offset = operand.offset(),
6242	.alignment = 4,
6243	});
6244	}
6245	}
6246
6247	// Compare the null value with '0'
6248	try cg.addImm32(0);
6249	try cg.addTag(Mir.Inst.Tag.fromOpcode(opcode));
6250
6251	return .stack;
6252	}
6253
6254	fn airOptionalPayload(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6255	const zcu = cg.pt.zcu;
6256	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6257	const opt_ty = cg.typeOf(ty_op.operand);
6258	const payload_ty = cg.typeOfIndex(inst);
6259	if (!payload_ty.hasRuntimeBits(zcu)) {
6260	return cg.finishAir(inst, .none, &.{ty_op.operand});
6261	}
6262
6263	const result = result: {
6264	const operand = try cg.resolveInst(ty_op.operand);
6265	if (opt_ty.optionalReprIsPayload(zcu)) break :result cg.reuseOperand(ty_op.operand, operand);
6266
6267	if (isByRef(payload_ty, zcu, cg.target)) {
6268	break :result try cg.buildPointerOffset(operand, 0, .new);
6269	}
6270
6271	break :result try cg.load(operand, payload_ty, 0);
6272	};
6273	return cg.finishAir(inst, result, &.{ty_op.operand});
6274	}
6275
6276	fn airOptionalPayloadPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6277	const zcu = cg.pt.zcu;
6278	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6279	const operand = try cg.resolveInst(ty_op.operand);
6280	const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
6281
6282	const result = result: {
6283	const payload_ty = opt_ty.optionalChild(zcu);
6284	if (!payload_ty.hasRuntimeBits(zcu) or opt_ty.optionalReprIsPayload(zcu)) {
6285	break :result cg.reuseOperand(ty_op.operand, operand);
6286	}
6287
6288	break :result try cg.buildPointerOffset(operand, 0, .new);
6289	};
6290	return cg.finishAir(inst, result, &.{ty_op.operand});
6291	}
6292
6293	fn airOptionalPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6294	const pt = cg.pt;
6295	const zcu = pt.zcu;
6296	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6297	const operand = try cg.resolveInst(ty_op.operand);
6298	const opt_ty = cg.typeOf(ty_op.operand).childType(zcu);
6299	const payload_ty = opt_ty.optionalChild(zcu);
6300
6301	if (opt_ty.optionalReprIsPayload(zcu)) {
6302	return cg.finishAir(inst, operand, &.{ty_op.operand});
6303	}
6304
6305	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6306	return cg.fail("Optional type {f} too big to fit into stack frame", .{opt_ty.fmt(pt)});
6307	};
6308
6309	try cg.emitWValue(operand);
6310	try cg.addImm32(1);
6311	try cg.addMemArg(.i32_store8, .{ .offset = operand.offset() + offset, .alignment = 1 });
6312
6313	const result = try cg.buildPointerOffset(operand, 0, .new);
6314	return cg.finishAir(inst, result, &.{ty_op.operand});
6315	}
6316
6317	fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6318	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6319	const payload_ty = cg.typeOf(ty_op.operand);
6320	const pt = cg.pt;
6321	const zcu = pt.zcu;
6322
6323	const result = result: {
6324	if (!payload_ty.hasRuntimeBits(zcu)) {
6325	const non_null_bit = try cg.allocStack(Type.u1);
6326	try cg.emitWValue(non_null_bit);
6327	try cg.addImm32(1);
6328	try cg.addMemArg(.i32_store8, .{ .offset = non_null_bit.offset(), .alignment = 1 });
6329	break :result non_null_bit;
6330	}
6331
6332	const operand = try cg.resolveInst(ty_op.operand);
6333	const op_ty = cg.typeOfIndex(inst);
6334	if (op_ty.optionalReprIsPayload(zcu)) {
6335	break :result cg.reuseOperand(ty_op.operand, operand);
6336	}
6337	const offset = std.math.cast(u32, payload_ty.abiSize(zcu)) orelse {
6338	return cg.fail("Optional type {f} too big to fit into stack frame", .{op_ty.fmt(pt)});
6339	};
6340
6341	// Create optional type, set the non-null bit, and store the operand inside the optional type
6342	const result_ptr = try cg.allocStack(op_ty);
6343	try cg.emitWValue(result_ptr);
6344	try cg.addImm32(1);
6345	try cg.addMemArg(.i32_store8, .{ .offset = result_ptr.offset() + offset, .alignment = 1 });
6346
6347	const payload_ptr = try cg.buildPointerOffset(result_ptr, 0, .new);
6348	try cg.store(payload_ptr, operand, payload_ty, 0);
6349	break :result result_ptr;
6350	};
6351
6352	return cg.finishAir(inst, result, &.{ty_op.operand});
6353	}
6354
6355	fn airSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6356	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6357	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6358
6359	const lhs = try cg.resolveInst(bin_op.lhs);
6360	const rhs = try cg.resolveInst(bin_op.rhs);
6361	const slice_ty = cg.typeOfIndex(inst);
6362
6363	const slice = try cg.allocStack(slice_ty);
6364	try cg.store(slice, lhs, Type.usize, 0);
6365	try cg.store(slice, rhs, Type.usize, cg.ptrSize());
6366
6367	return cg.finishAir(inst, slice, &.{ bin_op.lhs, bin_op.rhs });
6368	}
6369
6370	fn airSliceLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6371	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6372
6373	const operand = try cg.resolveInst(ty_op.operand);
6374	return cg.finishAir(inst, try cg.sliceLen(operand), &.{ty_op.operand});
6375	}
6376
6377	fn airSliceElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6378	const zcu = cg.pt.zcu;
6379	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6380
6381	const slice_ty = cg.typeOf(bin_op.lhs);
6382	const slice = try cg.resolveInst(bin_op.lhs);
6383	const index = try cg.resolveInst(bin_op.rhs);
6384	const elem_ty = slice_ty.childType(zcu);
6385	const elem_size = elem_ty.abiSize(zcu);
6386
6387	// load pointer onto stack
6388	_ = try cg.load(slice, Type.usize, 0);
6389
6390	// calculate index into slice
6391	try cg.emitWValue(index);
6392	try cg.addImm32(@intCast(elem_size));
6393	try cg.addTag(.i32_mul);
6394	try cg.addTag(.i32_add);
6395
6396	const elem_result = try cg.load(.stack, elem_ty, 0);
6397
6398	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6399	}
6400
6401	fn airSliceElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6402	const zcu = cg.pt.zcu;
6403	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6404	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6405
6406	const elem_ty = ty_pl.ty.childType(zcu);
6407	const elem_size = elem_ty.abiSize(zcu);
6408
6409	const slice = try cg.resolveInst(bin_op.lhs);
6410	const index = try cg.resolveInst(bin_op.rhs);
6411
6412	_ = try cg.load(slice, Type.usize, 0);
6413
6414	// calculate index into slice
6415	try cg.emitWValue(index);
6416	try cg.addImm32(@intCast(elem_size));
6417	try cg.addTag(.i32_mul);
6418	try cg.addTag(.i32_add);
6419
6420	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6421	}
6422
6423	fn airSlicePtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6424	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6425	const operand = try cg.resolveInst(ty_op.operand);
6426	return cg.finishAir(inst, try cg.slicePtr(operand), &.{ty_op.operand});
6427	}
6428
6429	fn slicePtr(cg: *CodeGen, operand: WValue) InnerError!WValue {
6430	const ptr = try cg.load(operand, Type.usize, 0);
6431	return ptr.toLocal(cg, Type.usize);
6432	}
6433
6434	fn sliceLen(cg: *CodeGen, operand: WValue) InnerError!WValue {
6435	const len = try cg.load(operand, Type.usize, cg.ptrSize());
6436	return len.toLocal(cg, Type.usize);
6437	}
6438
6439	fn airArrayToSlice(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6440	const zcu = cg.pt.zcu;
6441	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6442
6443	const operand = try cg.resolveInst(ty_op.operand);
6444	const array_ty = cg.typeOf(ty_op.operand).childType(zcu);
6445	const slice_ty = ty_op.ty;
6446
6447	// create a slice on the stack
6448	const slice_local = try cg.allocStack(slice_ty);
6449
6450	try cg.store(slice_local, operand, Type.usize, 0);
6451
6452	// store the length of the array in the slice
6453	const array_len: u32 = @intCast(array_ty.arrayLen(zcu));
6454	try cg.store(slice_local, .{ .imm32 = array_len }, Type.usize, cg.ptrSize());
6455
6456	return cg.finishAir(inst, slice_local, &.{ty_op.operand});
6457	}
6458
6459	fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6460	const zcu = cg.pt.zcu;
6461	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6462
6463	const ptr_ty = cg.typeOf(bin_op.lhs);
6464	const ptr = try cg.resolveInst(bin_op.lhs);
6465	const index = try cg.resolveInst(bin_op.rhs);
6466	const elem_ty = ptr_ty.childType(zcu);
6467	const elem_size = elem_ty.abiSize(zcu);
6468
6469	// load pointer onto the stack
6470	if (ptr_ty.isSlice(zcu)) {
6471	_ = try cg.load(ptr, Type.usize, 0);
6472	} else {
6473	try cg.lowerToStack(ptr);
6474	}
6475
6476	// calculate index into slice
6477	try cg.emitWValue(index);
6478	try cg.addImm32(@intCast(elem_size));
6479	try cg.addTag(.i32_mul);
6480	try cg.addTag(.i32_add);
6481
6482	const elem_result = try cg.load(.stack, elem_ty, 0);
6483
6484	return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6485	}
6486
6487	fn airPtrElemPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6488	const zcu = cg.pt.zcu;
6489	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6490	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6491
6492	const ptr_ty = cg.typeOf(bin_op.lhs);
6493	const elem_ty = ty_pl.ty.childType(zcu);
6494	const elem_size = elem_ty.abiSize(zcu);
6495
6496	const ptr = try cg.resolveInst(bin_op.lhs);
6497	const index = try cg.resolveInst(bin_op.rhs);
6498
6499	// load pointer onto the stack
6500	if (ptr_ty.isSlice(zcu)) {
6501	_ = try cg.load(ptr, Type.usize, 0);
6502	} else {
6503	try cg.lowerToStack(ptr);
6504	}
6505
6506	// calculate index into ptr
6507	try cg.emitWValue(index);
6508	try cg.addImm32(@intCast(elem_size));
6509	try cg.addTag(.i32_mul);
6510	try cg.addTag(.i32_add);
6511
6512	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6513	}
6514
6515	fn airPtrBinOp(cg: *CodeGen, inst: Air.Inst.Index, op: enum { add, sub }) InnerError!void {
6516	const zcu = cg.pt.zcu;
6517	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6518	const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data;
6519
6520	const ptr = try cg.resolveInst(bin_op.lhs);
6521	const offset = try cg.resolveInst(bin_op.rhs);
6522	const ptr_ty = cg.typeOf(bin_op.lhs);
6523	const pointee_ty = switch (ptr_ty.ptrSize(zcu)) {
6524	.one => ptr_ty.childType(zcu).childType(zcu), // ptr to array, so get array element type
6525	else => ptr_ty.childType(zcu),
6526	};
6527
6528	try cg.lowerToStack(ptr);
6529	try cg.emitWValue(offset);
6530
6531	switch (cg.ptr_size) {
6532	.wasm32 => {
6533	try cg.addImm32(@intCast(pointee_ty.abiSize(zcu)));
6534	try cg.addTag(.i32_mul);
6535	try cg.addTag(switch (op) {
6536	.add => .i32_add,
6537	.sub => .i32_sub,
6538	});
6539	},
6540	.wasm64 => {
6541	try cg.addImm64(pointee_ty.abiSize(zcu));
6542	try cg.addTag(.i64_mul);
6543	try cg.addTag(switch (op) {
6544	.add => .i64_add,
6545	.sub => .i64_sub,
6546	});
6547	},
6548	}
6549
6550	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6551	}
6552
6553	fn airMemset(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
6554	const zcu = cg.pt.zcu;
6555	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6556
6557	const ptr = try cg.resolveInst(bin_op.lhs);
6558	const ptr_ty = cg.typeOf(bin_op.lhs);
6559	const value = try cg.resolveInst(bin_op.rhs);
6560	const len = switch (ptr_ty.ptrSize(zcu)) {
6561	.slice => try cg.sliceLen(ptr),
6562	.one => @as(WValue, .{ .imm32 = @as(u32, @intCast(ptr_ty.childType(zcu).arrayLen(zcu))) }),
6563	.c, .many => unreachable,
6564	};
6565
6566	const elem_ty = if (ptr_ty.ptrSize(zcu) == .one)
6567	ptr_ty.childType(zcu).childType(zcu)
6568	else
6569	ptr_ty.childType(zcu);
6570
6571	if (!safety and bin_op.rhs == .undef) {
6572	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6573	}
6574
6575	const dst_ptr = try cg.sliceOrArrayPtr(ptr, ptr_ty);
6576	try cg.memset(elem_ty, dst_ptr, len, value);
6577
6578	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
6579	}
6580
6581	/// Sets a region of memory at `ptr` to the value of `value`
6582	/// When the user has enabled the bulk_memory feature, we lower
6583	/// this to wasm's memset instruction. When the feature is not present,
6584	/// we implement it manually.
6585	fn memset(cg: *CodeGen, elem_ty: Type, ptr: WValue, len: WValue, value: WValue) InnerError!void {
6586	const zcu = cg.pt.zcu;
6587	const abi_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6588
6589	// When bulk_memory is enabled, we lower it to wasm's memset instruction.
6590	// If not, we lower it ourselves.
6591	if (cg.target.cpu.has(.wasm, .bulk_memory) and abi_size == 1) {
6592	const len0_ok = cg.target.cpu.has(.wasm, .nontrapping_bulk_memory_len0);
6593
6594	if (!len0_ok) {
6595	try cg.startBlock(.block, .empty);
6596
6597	// Even if `len` is zero, the spec requires an implementation to trap if `ptr + len` is
6598	// out of memory bounds. This can easily happen in Zig in a case such as:
6599	//
6600	// const ptr: [*]u8 = undefined;
6601	// var len: usize = runtime_zero();
6602	// @memset(ptr[0..len], 42);
6603	//
6604	// So explicitly avoid using `memory.fill` in the `len == 0` case. Lovely design.
6605	try cg.emitWValue(len);
6606	try cg.addTag(.i32_eqz);
6607	try cg.addLabel(.br_if, 0);
6608	}
6609
6610	try cg.lowerToStack(ptr);
6611	try cg.emitWValue(value);
6612	try cg.emitWValue(len);
6613	try cg.addExtended(.memory_fill);
6614
6615	if (!len0_ok) {
6616	try cg.endBlock();
6617	}
6618
6619	return;
6620	}
6621
6622	const final_len: WValue = switch (len) {
6623	.imm32 => |val| .{ .imm32 = val * abi_size },
6624	.imm64 => |val| .{ .imm64 = val * abi_size },
6625	else => if (abi_size != 1) blk: {
6626	const new_len = try cg.ensureAllocLocal(Type.usize);
6627	try cg.emitWValue(len);
6628	switch (cg.ptr_size) {
6629	.wasm32 => {
6630	try cg.emitWValue(.{ .imm32 = abi_size });
6631	try cg.addTag(.i32_mul);
6632	},
6633	.wasm64 => {
6634	try cg.emitWValue(.{ .imm64 = abi_size });
6635	try cg.addTag(.i64_mul);
6636	},
6637	}
6638	try cg.addLocal(.local_set, new_len.local.value);
6639	break :blk new_len;
6640	} else len,
6641	};
6642
6643	var end_ptr = try cg.allocLocal(Type.usize);
6644	defer end_ptr.free(cg);
6645	var new_ptr = try cg.buildPointerOffset(ptr, 0, .new);
6646	defer new_ptr.free(cg);
6647
6648	// get the loop conditional: if current pointer address equals final pointer's address
6649	try cg.lowerToStack(ptr);
6650	try cg.emitWValue(final_len);
6651	switch (cg.ptr_size) {
6652	.wasm32 => try cg.addTag(.i32_add),
6653	.wasm64 => try cg.addTag(.i64_add),
6654	}
6655	try cg.addLocal(.local_set, end_ptr.local.value);
6656
6657	// outer block to jump to when loop is done
6658	try cg.startBlock(.block, .empty);
6659	try cg.startBlock(.loop, .empty);
6660
6661	// check for condition for loop end
6662	try cg.emitWValue(new_ptr);
6663	try cg.emitWValue(end_ptr);
6664	switch (cg.ptr_size) {
6665	.wasm32 => try cg.addTag(.i32_eq),
6666	.wasm64 => try cg.addTag(.i64_eq),
6667	}
6668	try cg.addLabel(.br_if, 1); // jump out of loop into outer block (finished)
6669
6670	// store the value at the current position of the pointer
6671	try cg.store(new_ptr, value, elem_ty, 0);
6672
6673	// move the pointer to the next element
6674	try cg.emitWValue(new_ptr);
6675	switch (cg.ptr_size) {
6676	.wasm32 => {
6677	try cg.emitWValue(.{ .imm32 = abi_size });
6678	try cg.addTag(.i32_add);
6679	},
6680	.wasm64 => {
6681	try cg.emitWValue(.{ .imm64 = abi_size });
6682	try cg.addTag(.i64_add);
6683	},
6684	}
6685	try cg.addLocal(.local_set, new_ptr.local.value);
6686
6687	// end of loop
6688	try cg.addLabel(.br, 0); // jump to start of loop
6689	try cg.endBlock();
6690	try cg.endBlock();
6691	}
6692
6693	fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6694	const zcu = cg.pt.zcu;
6695	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
6696
6697	const array_ty = cg.typeOf(bin_op.lhs);
6698	const array = try cg.resolveInst(bin_op.lhs);
6699	const index = try cg.resolveInst(bin_op.rhs);
6700	const elem_ty = array_ty.childType(zcu);
6701	const elem_size = elem_ty.abiSize(zcu);
6702
6703	if (isByRef(array_ty, zcu, cg.target)) {
6704	try cg.lowerToStack(array);
6705	try cg.emitWValue(index);
6706	try cg.addImm32(@intCast(elem_size));
6707	try cg.addTag(.i32_mul);
6708	try cg.addTag(.i32_add);
6709	} else {
6710	assert(array_ty.zigTypeTag(zcu) == .vector);
6711
6712	switch (index) {
6713	inline .imm32, .imm64 => |lane| {
6714	const opcode: std.wasm.SimdOpcode = switch (elem_ty.bitSize(zcu)) {
6715	8 => if (elem_ty.isSignedInt(zcu)) .i8x16_extract_lane_s else .i8x16_extract_lane_u,
6716	16 => if (elem_ty.isSignedInt(zcu)) .i16x8_extract_lane_s else .i16x8_extract_lane_u,
6717	32 => if (elem_ty.isInt(zcu)) .i32x4_extract_lane else .f32x4_extract_lane,
6718	64 => if (elem_ty.isInt(zcu)) .i64x2_extract_lane else .f64x2_extract_lane,
6719	else => unreachable,
6720	};
6721
6722	var operands = [_]u32{ @backingInt(opcode), @as(u8, @intCast(lane)) };
6723
6724	try cg.emitWValue(array);
6725
6726	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6727	try cg.mir_extra.appendSlice(cg.gpa, &operands);
6728	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6729
6730	return cg.finishAir(inst, .stack, &.{ bin_op.lhs, bin_op.rhs });
6731	},
6732	else => {
6733	const stack_vec = try cg.allocStack(array_ty);
6734	try cg.store(stack_vec, array, array_ty, 0);
6735
6736	// Is a non-unrolled vector (v128)
6737	try cg.lowerToStack(stack_vec);
6738	try cg.emitWValue(index);
6739	try cg.addImm32(@intCast(elem_size));
6740	try cg.addTag(.i32_mul);
6741	try cg.addTag(.i32_add);
6742	},
6743	}
6744	}
6745
6746	const result = if (isByRef(elem_ty, zcu, cg.target))
6747	.stack
6748	else
6749	try cg.load(.stack, elem_ty, 0);
6750	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6751	}
6752
6753	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6754	const zcu = cg.pt.zcu;
6755	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
6756	const operand = try cg.resolveInst(ty_op.operand);
6757	const ty = cg.typeOfIndex(inst);
6758	const elem_ty = ty.childType(zcu);
6759
6760	if (determineSimdStoreStrategy(ty, zcu, cg.target) == .direct) blk: {
6761	switch (operand) {
6762	// when the operand lives in the linear memory section, we can directly
6763	// load and splat the value at once. Meaning we do not first have to load
6764	// the scalar value onto the stack.
6765	.stack_offset, .nav_ref, .uav_ref => {
6766	const opcode = switch (elem_ty.bitSize(zcu)) {
6767	8 => @backingInt(std.wasm.SimdOpcode.v128_load8_splat),
6768	16 => @backingInt(std.wasm.SimdOpcode.v128_load16_splat),
6769	32 => @backingInt(std.wasm.SimdOpcode.v128_load32_splat),
6770	64 => @backingInt(std.wasm.SimdOpcode.v128_load64_splat),
6771	else => break :blk, // Cannot make use of simd-instructions
6772	};
6773	try cg.emitWValue(operand);
6774	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6775	// stores as := opcode, offset, alignment (opcode::memarg)
6776	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
6777	opcode,
6778	operand.offset(),
6779	@intCast(elem_ty.abiAlignment(zcu).toByteUnits().?),
6780	});
6781	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6782	return cg.finishAir(inst, .stack, &.{ty_op.operand});
6783	},
6784	.local => {
6785	const opcode = switch (elem_ty.bitSize(zcu)) {
6786	8 => @backingInt(std.wasm.SimdOpcode.i8x16_splat),
6787	16 => @backingInt(std.wasm.SimdOpcode.i16x8_splat),
6788	32 => if (elem_ty.isInt(zcu)) @backingInt(std.wasm.SimdOpcode.i32x4_splat) else @backingInt(std.wasm.SimdOpcode.f32x4_splat),
6789	64 => if (elem_ty.isInt(zcu)) @backingInt(std.wasm.SimdOpcode.i64x2_splat) else @backingInt(std.wasm.SimdOpcode.f64x2_splat),
6790	else => break :blk, // Cannot make use of simd-instructions
6791	};
6792	try cg.emitWValue(operand);
6793	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6794	try cg.mir_extra.append(cg.gpa, opcode);
6795	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6796	return cg.finishAir(inst, .stack, &.{ty_op.operand});
6797	},
6798	else => unreachable,
6799	}
6800	}
6801
6802	const vector_len = @as(usize, @intCast(ty.vectorLen(zcu)));
6803	const result = try cg.allocStack(ty);
6804	const elem_byte_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6805	var index: usize = 0;
6806	var offset: u32 = 0;
6807	while (index < vector_len) : (index += 1) {
6808	try cg.store(result, operand, elem_ty, offset);
6809	offset += elem_byte_size;
6810	}
6811
6812	return cg.finishAir(inst, result, &.{ty_op.operand});
6813	}
6814
6815	fn airSelect(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6816	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
6817	const operand = try cg.resolveInst(pl_op.operand);
6818
6819	_ = operand;
6820	return cg.fail("TODO: Implement wasm airSelect", .{});
6821	}
6822
6823	fn airShuffleOne(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6824	const pt = cg.pt;
6825	const zcu = pt.zcu;
6826
6827	const unwrapped = cg.air.unwrapShuffleOne(zcu, inst);
6828	const result_ty = unwrapped.result_ty;
6829	const mask = unwrapped.mask;
6830	const operand = try cg.resolveInst(unwrapped.operand);
6831
6832	const elem_ty = result_ty.childType(zcu);
6833	const elem_size = elem_ty.abiSize(zcu);
6834
6835	// TODO: this function could have an `i8x16_shuffle` fast path like `airShuffleTwo` if we were
6836	// to lower the comptime-known operands to a non-by-ref vector value.
6837
6838	// TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6839	// I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6840	if (!isByRef(result_ty, zcu, cg.target) or
6841	!isByRef(cg.typeOf(unwrapped.operand), zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
6842
6843	const dest_alloc = try cg.allocStack(result_ty);
6844	for (mask, 0..) |mask_elem, out_idx| {
6845	try cg.emitWValue(dest_alloc);
6846	const elem_val = switch (mask_elem.unwrap()) {
6847	.elem => |idx| try cg.load(operand, elem_ty, @intCast(elem_size * idx)),
6848	.value => |val| try cg.lowerConstant(.fromInterned(val)),
6849	};
6850	try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6851	}
6852	return cg.finishAir(inst, dest_alloc, &.{unwrapped.operand});
6853	}
6854
6855	fn airShuffleTwo(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6856	const pt = cg.pt;
6857	const zcu = pt.zcu;
6858
6859	const unwrapped = cg.air.unwrapShuffleTwo(zcu, inst);
6860	const result_ty = unwrapped.result_ty;
6861	const mask = unwrapped.mask;
6862	const operand_a = try cg.resolveInst(unwrapped.operand_a);
6863	const operand_b = try cg.resolveInst(unwrapped.operand_b);
6864
6865	const a_ty = cg.typeOf(unwrapped.operand_a);
6866	const b_ty = cg.typeOf(unwrapped.operand_b);
6867	const elem_ty = result_ty.childType(zcu);
6868	const elem_size = elem_ty.abiSize(zcu);
6869
6870	// WASM has `i8x16_shuffle`, which we can apply if the element type bit size is a multiple of 8
6871	// and the input and output vectors have a bit size of 128 (and are hence not by-ref). Otherwise,
6872	// we fall back to a naive loop lowering.
6873	if (!isByRef(a_ty, zcu, cg.target) and
6874	!isByRef(b_ty, zcu, cg.target) and
6875	!isByRef(result_ty, zcu, cg.target) and
6876	elem_ty.bitSize(zcu) % 8 == 0)
6877	{
6878	var lane_map: [16]u8 align(4) = undefined;
6879	const lanes_per_elem: usize = @intCast(elem_ty.bitSize(zcu) / 8);
6880	for (mask, 0..) |mask_elem, out_idx| {
6881	const out_first_lane = out_idx * lanes_per_elem;
6882	const in_first_lane = switch (mask_elem.unwrap()) {
6883	.a_elem => |i| i * lanes_per_elem,
6884	.b_elem => |i| i * lanes_per_elem + 16,
6885	.undef => 0, // doesn't matter
6886	};
6887	for (lane_map[out_first_lane..][0..lanes_per_elem], in_first_lane..) |*out, in| {
6888	out.* = @intCast(in);
6889	}
6890	}
6891	try cg.emitWValue(operand_a);
6892	try cg.emitWValue(operand_b);
6893	const extra_index: u32 = @intCast(cg.mir_extra.items.len);
6894	try cg.mir_extra.appendSlice(cg.gpa, &.{
6895	@backingInt(std.wasm.SimdOpcode.i8x16_shuffle),
6896	@bitCast(lane_map[0..4].*),
6897	@bitCast(lane_map[4..8].*),
6898	@bitCast(lane_map[8..12].*),
6899	@bitCast(lane_map[12..].*),
6900	});
6901	try cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
6902	return cg.finishAir(inst, .stack, &.{ unwrapped.operand_a, unwrapped.operand_b });
6903	}
6904
6905	// TODO: this is incorrect if either operand or the result is *not* by-ref, which is possible.
6906	// I tried to fix it, but I couldn't make much sense of how this backend handles memory.
6907	if (!isByRef(result_ty, zcu, cg.target) or
6908	!isByRef(a_ty, zcu, cg.target) or
6909	!isByRef(b_ty, zcu, cg.target)) return cg.fail("TODO: handle mixed by-ref shuffle", .{});
6910
6911	const dest_alloc = try cg.allocStack(result_ty);
6912	for (mask, 0..) |mask_elem, out_idx| {
6913	try cg.emitWValue(dest_alloc);
6914	const elem_val = switch (mask_elem.unwrap()) {
6915	.a_elem => |idx| try cg.load(operand_a, elem_ty, @intCast(elem_size * idx)),
6916	.b_elem => |idx| try cg.load(operand_b, elem_ty, @intCast(elem_size * idx)),
6917	.undef => try cg.emitUndefined(elem_ty),
6918	};
6919	try cg.store(.stack, elem_val, elem_ty, @intCast(dest_alloc.offset() + elem_size * out_idx));
6920	}
6921	return cg.finishAir(inst, dest_alloc, &.{ unwrapped.operand_a, unwrapped.operand_b });
6922	}
6923
6924	fn airReduce(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6925	const reduce = cg.air.instructions.items(.data)[@backingInt(inst)].reduce;
6926	const operand = try cg.resolveInst(reduce.operand);
6927
6928	_ = operand;
6929	return cg.fail("TODO: Implement wasm airReduce", .{});
6930	}
6931
6932	fn airAggregateInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6933	const pt = cg.pt;
6934	const zcu = pt.zcu;
6935	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
6936	const result_ty = cg.typeOfIndex(inst);
6937	const len = @as(usize, @intCast(result_ty.arrayLen(zcu)));
6938	const elements: []const Air.Inst.Ref = @ptrCast(cg.air.extra.items[ty_pl.payload..][0..len]);
6939
6940	const result: WValue = result_value: {
6941	switch (result_ty.zigTypeTag(zcu)) {
6942	.array, .vector => {
6943	const result = try cg.allocStack(result_ty);
6944	const elem_ty = result_ty.childType(zcu);
6945	const elem_size = @as(u32, @intCast(elem_ty.abiSize(zcu)));
6946	const sentinel = result_ty.sentinel(zcu);
6947
6948	// When the element type is by reference, we must copy the entire
6949	// value. It is therefore safer to move the offset pointer and store
6950	// each value individually, instead of using store offsets.
6951	if (isByRef(elem_ty, zcu, cg.target)) {
6952	// copy stack pointer into a temporary local, which is
6953	// moved for each element to store each value in the right position.
6954	const offset = try cg.buildPointerOffset(result, 0, .new);
6955	for (elements, 0..) |elem, elem_index| {
6956	const elem_val = try cg.resolveInst(elem);
6957	try cg.store(offset, elem_val, elem_ty, 0);
6958
6959	if (elem_index < elements.len - 1 or sentinel != null) {
6960	_ = try cg.buildPointerOffset(offset, elem_size, .modify);
6961	}
6962	}
6963	if (sentinel) |s| {
6964	const val = try cg.resolveValue(s);
6965	try cg.store(offset, val, elem_ty, 0);
6966	}
6967	} else {
6968	var offset: u32 = 0;
6969	for (elements) |elem| {
6970	const elem_val = try cg.resolveInst(elem);
6971	try cg.store(result, elem_val, elem_ty, offset);
6972	offset += elem_size;
6973	}
6974	if (sentinel) |s| {
6975	const val = try cg.resolveValue(s);
6976	try cg.store(result, val, elem_ty, offset);
6977	}
6978	}
6979	break :result_value result;
6980	},
6981	.@"struct" => switch (result_ty.containerLayout(zcu)) {
6982	.@"packed" => unreachable, // legalize .expand_packed_aggregate_init
6983	else => {
6984	const result = try cg.allocStack(result_ty);
6985	for (elements, 0..) |elem, elem_index| {
6986	if (try result_ty.structFieldValueComptime(pt, elem_index) != null) continue;
6987
6988	const elem_ty = result_ty.fieldType(elem_index, zcu);
6989	const field_offset = result_ty.structFieldOffset(elem_index, zcu);
6990	const offset = try cg.buildPointerOffset(result, field_offset, .new);
6991
6992	const value = try cg.resolveInst(elem);
6993	try cg.store(offset, value, elem_ty, 0);
6994	}
6995
6996	break :result_value result;
6997	},
6998	},
6999	else => unreachable,
7000	}
7001	};
7002
7003	var bt = cg.liveness.iterateBigTomb(inst);
7004	for (elements) |arg| cg.feed(&bt, arg);
7005	return cg.finishAirResult(inst, result);
7006	}
7007
7008	fn airUnionInit(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7009	const pt = cg.pt;
7010	const zcu = pt.zcu;
7011	const ip = &zcu.intern_pool;
7012	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7013	const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
7014
7015	const result = result: {
7016	const union_ty = cg.typeOfIndex(inst);
7017	const layout = union_ty.unionGetLayout(zcu);
7018	const union_obj = zcu.typeToUnion(union_ty).?;
7019	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
7020	const field_name = ip.loadEnumType(union_obj.enum_tag_type).field_names.get(ip)[extra.field_index];
7021
7022	const tag_int = blk: {
7023	const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
7024	const enum_field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
7025	const tag_val = try pt.enumValueFieldIndex(tag_ty, enum_field_index);
7026	break :blk try cg.lowerConstant(tag_val);
7027	};
7028	if (layout.payload_size == 0) {
7029	if (layout.tag_size == 0) {
7030	break :result .none;
7031	}
7032	assert(!isByRef(union_ty, zcu, cg.target));
7033	break :result tag_int;
7034	}
7035
7036	if (isByRef(union_ty, zcu, cg.target)) {
7037	const result_ptr = try cg.allocStack(union_ty);
7038	const payload = try cg.resolveInst(extra.init);
7039	if (layout.tag_align.compare(.gte, layout.payload_align)) {
7040	if (isByRef(field_ty, zcu, cg.target)) {
7041	const payload_ptr = try cg.buildPointerOffset(result_ptr, layout.tag_size, .new);
7042	try cg.store(payload_ptr, payload, field_ty, 0);
7043	} else {
7044	try cg.store(result_ptr, payload, field_ty, @intCast(layout.tag_size));
7045	}
7046
7047	if (layout.tag_size > 0) {
7048	try cg.store(result_ptr, tag_int, .fromInterned(union_obj.enum_tag_type), 0);
7049	}
7050	} else {
7051	try cg.store(result_ptr, payload, field_ty, 0);
7052	if (layout.tag_size > 0) {
7053	try cg.store(
7054	result_ptr,
7055	tag_int,
7056	.fromInterned(union_obj.enum_tag_type),
7057	@intCast(layout.payload_size),
7058	);
7059	}
7060	}
7061	break :result result_ptr;
7062	} else {
7063	unreachable;
7064	}
7065	};
7066
7067	return cg.finishAir(inst, result, &.{extra.init});
7068	}
7069
7070	fn airPrefetch(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7071	const prefetch = cg.air.instructions.items(.data)[@backingInt(inst)].prefetch;
7072	return cg.finishAir(inst, .none, &.{prefetch.ptr});
7073	}
7074
7075	fn airWasmMemorySize(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7076	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7077
7078	try cg.addLabel(.memory_size, pl_op.payload);
7079	return cg.finishAir(inst, .stack, &.{pl_op.operand});
7080	}
7081
7082	fn airWasmMemoryGrow(cg: *CodeGen, inst: Air.Inst.Index) !void {
7083	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7084
7085	const operand = try cg.resolveInst(pl_op.operand);
7086	try cg.emitWValue(operand);
7087	try cg.addLabel(.memory_grow, pl_op.payload);
7088	return cg.finishAir(inst, .stack, &.{pl_op.operand});
7089	}
7090
7091	fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7092	const pt = cg.pt;
7093	const zcu = pt.zcu;
7094	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7095	const un_ty = cg.typeOf(bin_op.lhs).childType(zcu);
7096	const tag_ty = cg.typeOf(bin_op.rhs);
7097	const layout = un_ty.unionGetLayout(zcu);
7098	if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7099
7100	const union_ptr = try cg.resolveInst(bin_op.lhs);
7101	const new_tag = try cg.resolveInst(bin_op.rhs);
7102	if (layout.payload_size == 0) {
7103	try cg.store(union_ptr, new_tag, tag_ty, 0);
7104	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7105	}
7106
7107	// when the tag alignment is smaller than the payload, the field will be stored
7108	// after the payload.
7109	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align)) blk: {
7110	break :blk @intCast(layout.payload_size);
7111	} else 0;
7112	try cg.store(union_ptr, new_tag, tag_ty, offset);
7113	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7114	}
7115
7116	fn airGetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7117	const zcu = cg.pt.zcu;
7118	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7119
7120	const un_ty = cg.typeOf(ty_op.operand);
7121	const tag_ty = cg.typeOfIndex(inst);
7122	const layout = un_ty.unionGetLayout(zcu);
7123	if (layout.tag_size == 0) return cg.finishAir(inst, .none, &.{ty_op.operand});
7124
7125	const operand = try cg.resolveInst(ty_op.operand);
7126	// when the tag alignment is smaller than the payload, the field will be stored
7127	// after the payload.
7128	const offset: u32 = if (layout.tag_align.compare(.lt, layout.payload_align))
7129	@intCast(layout.payload_size)
7130	else
7131	0;
7132	const result = try cg.load(operand, tag_ty, offset);
7133	return cg.finishAir(inst, result, &.{ty_op.operand});
7134	}
7135
7136	fn airErrUnionPayloadPtrSet(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7137	const zcu = cg.pt.zcu;
7138	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7139
7140	const err_set_ty = cg.typeOf(ty_op.operand).childType(zcu);
7141	const payload_ty = err_set_ty.errorUnionPayload(zcu);
7142	const operand = try cg.resolveInst(ty_op.operand);
7143
7144	// set error-tag to '0' to annotate error union is non-error
7145	try cg.store(
7146	operand,
7147	.{ .imm32 = 0 },
7148	Type.anyerror,
7149	@intCast(errUnionErrorOffset(payload_ty, zcu)),
7150	);
7151
7152	const result = result: {
7153	if (!payload_ty.hasRuntimeBits(zcu)) {
7154	break :result cg.reuseOperand(ty_op.operand, operand);
7155	}
7156
7157	break :result try cg.buildPointerOffset(operand, @as(u32, @intCast(errUnionPayloadOffset(payload_ty, zcu))), .new);
7158	};
7159	return cg.finishAir(inst, result, &.{ty_op.operand});
7160	}
7161
7162	fn airFieldParentPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7163	const pt = cg.pt;
7164	const zcu = pt.zcu;
7165	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7166	const extra = cg.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
7167
7168	const field_ptr = try cg.resolveInst(extra.field_ptr);
7169	const parent_ptr_ty = cg.typeOfIndex(inst);
7170	const parent_ty = parent_ptr_ty.childType(zcu);
7171	const field_ptr_ty = cg.typeOf(extra.field_ptr);
7172	const field_index = extra.field_index;
7173	const field_offset = switch (parent_ty.containerLayout(zcu)) {
7174	.auto, .@"extern" => parent_ty.structFieldOffset(field_index, zcu),
7175	.@"packed" => offset: {
7176	const parent_ptr_offset = parent_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
7177	const field_offset = if (zcu.typeToStruct(parent_ty)) |loaded_struct| zcu.structPackedFieldBitOffset(loaded_struct, field_index) else 0;
7178	const field_ptr_offset = field_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset;
7179	break :offset @divExact(parent_ptr_offset + field_offset - field_ptr_offset, 8);
7180	},
7181	};
7182
7183	const result = if (field_offset != 0) result: {
7184	const base = try cg.buildPointerOffset(field_ptr, 0, .new);
7185	try cg.addLocal(.local_get, base.local.value);
7186	try cg.addImm32(@intCast(field_offset));
7187	try cg.addTag(.i32_sub);
7188	try cg.addLocal(.local_set, base.local.value);
7189	break :result base;
7190	} else cg.reuseOperand(extra.field_ptr, field_ptr);
7191
7192	return cg.finishAir(inst, result, &.{extra.field_ptr});
7193	}
7194
7195	fn sliceOrArrayPtr(cg: *CodeGen, ptr: WValue, ptr_ty: Type) InnerError!WValue {
7196	const zcu = cg.pt.zcu;
7197	if (ptr_ty.isSlice(zcu)) {
7198	return cg.slicePtr(ptr);
7199	} else {
7200	return ptr;
7201	}
7202	}
7203
7204	fn airMemcpy(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7205	const zcu = cg.pt.zcu;
7206	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7207	const dst = try cg.resolveInst(bin_op.lhs);
7208	const dst_ty = cg.typeOf(bin_op.lhs);
7209	const ptr_elem_ty = dst_ty.childType(zcu);
7210	const src = try cg.resolveInst(bin_op.rhs);
7211	const src_ty = cg.typeOf(bin_op.rhs);
7212	const len = switch (dst_ty.ptrSize(zcu)) {
7213	.slice => blk: {
7214	const slice_len = try cg.sliceLen(dst);
7215	if (ptr_elem_ty.abiSize(zcu) != 1) {
7216	try cg.emitWValue(slice_len);
7217	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7218	try cg.addTag(.i32_mul);
7219	try cg.addLocal(.local_set, slice_len.local.value);
7220	}
7221	break :blk slice_len;
7222	},
7223	.one => @as(WValue, .{
7224	.imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
7225	}),
7226	.c, .many => unreachable,
7227	};
7228	const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7229	const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7230	try cg.memcpy(dst_ptr, src_ptr, len);
7231
7232	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7233	}
7234
7235	fn airMemmove(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7236	const zcu = cg.pt.zcu;
7237	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7238	const dst = try cg.resolveInst(bin_op.lhs);
7239	const dst_ty = cg.typeOf(bin_op.lhs);
7240	const ptr_elem_ty = dst_ty.childType(zcu);
7241	const src = try cg.resolveInst(bin_op.rhs);
7242	const src_ty = cg.typeOf(bin_op.rhs);
7243	const len = switch (dst_ty.ptrSize(zcu)) {
7244	.slice => blk: {
7245	const slice_len = try cg.sliceLen(dst);
7246	if (ptr_elem_ty.abiSize(zcu) != 1) {
7247	try cg.emitWValue(slice_len);
7248	try cg.emitWValue(.{ .imm32 = @as(u32, @intCast(ptr_elem_ty.abiSize(zcu))) });
7249	try cg.addTag(.i32_mul);
7250	try cg.addLocal(.local_set, slice_len.local.value);
7251	}
7252	break :blk slice_len;
7253	},
7254	.one => @as(WValue, .{
7255	.imm32 = @as(u32, @intCast(ptr_elem_ty.arrayLen(zcu) * ptr_elem_ty.childType(zcu).abiSize(zcu))),
7256	}),
7257	.c, .many => unreachable,
7258	};
7259	const dst_ptr = try cg.sliceOrArrayPtr(dst, dst_ty);
7260	const src_ptr = try cg.sliceOrArrayPtr(src, src_ty);
7261	try cg.memmove(dst_ptr, src_ptr, len);
7262
7263	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7264	}
7265
7266	fn airRetAddr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7267	// TODO: Implement this properly once stack serialization is solved
7268	return cg.finishAir(inst, switch (cg.ptr_size) {
7269	.wasm32 => .{ .imm32 = 0 },
7270	.wasm64 => .{ .imm64 = 0 },
7271	}, &.{});
7272	}
7273
7274	fn airErrorName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7275	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7276	const operand = try cg.resolveInst(un_op);
7277	// Each entry to this table is a slice (ptr+len).
7278	// The operand in this instruction represents the index within this table.
7279	// This means to get the final name, we emit the base pointer and then perform
7280	// pointer arithmetic to find the pointer to this slice and return that.
7281	//
7282	// As the names are global and the slice elements are constant, we do not have
7283	// to make a copy of the ptr+value but can point towards them directly.
7284	const pt = cg.pt;
7285	const name_ty = Type.slice_const_u8_sentinel_0;
7286	const abi_size = name_ty.abiSize(pt.zcu);
7287
7288	// Lowers to a i32.const or i64.const with the error table memory address.
7289	cg.error_name_table_ref_count += 1;
7290	try cg.addTag(.error_name_table_ref);
7291	try cg.emitWValue(operand);
7292	switch (cg.ptr_size) {
7293	.wasm32 => {
7294	try cg.addImm32(@intCast(abi_size));
7295	try cg.addTag(.i32_mul);
7296	try cg.addTag(.i32_add);
7297	},
7298	.wasm64 => {
7299	try cg.addImm64(abi_size);
7300	try cg.addTag(.i64_mul);
7301	try cg.addTag(.i64_add);
7302	},
7303	}
7304
7305	return cg.finishAir(inst, .stack, &.{un_op});
7306	}
7307
7308	fn airPtrSliceFieldPtr(cg: *CodeGen, inst: Air.Inst.Index, offset: u32) InnerError!void {
7309	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7310	const slice_ptr = try cg.resolveInst(ty_op.operand);
7311	const result = try cg.buildPointerOffset(slice_ptr, offset, .new);
7312	return cg.finishAir(inst, result, &.{ty_op.operand});
7313	}
7314
7315	fn airDbgStmt(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7316	const dbg_stmt = cg.air.instructions.items(.data)[@backingInt(inst)].dbg_stmt;
7317	try cg.addInst(.{ .tag = .dbg_line, .data = .{
7318	.payload = try cg.addExtra(Mir.DbgLineColumn{
7319	.line = dbg_stmt.line,
7320	.column = dbg_stmt.column,
7321	}),
7322	} });
7323	return cg.finishAir(inst, .none, &.{});
7324	}
7325
7326	fn airDbgInlineBlock(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7327	const block = cg.air.unwrapDbgBlock(inst);
7328	// TODO
7329	try cg.lowerBlock(inst, block.ty, block.body);
7330	}
7331
7332	fn airDbgVar(
7333	cg: *CodeGen,
7334	inst: Air.Inst.Index,
7335	local_tag: link.File.Dwarf.WipNav.LocalVarTag,
7336	is_ptr: bool,
7337	) InnerError!void {
7338	_ = is_ptr;
7339	_ = local_tag;
7340	return cg.finishAir(inst, .none, &.{});
7341	}
7342
7343	fn airTry(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7344	const unwrapped_try = cg.air.unwrapTry(inst);
7345	const body = unwrapped_try.else_body;
7346	const err_union = try cg.resolveInst(unwrapped_try.error_union);
7347	const err_union_ty = cg.typeOf(unwrapped_try.error_union);
7348	const result = try lowerTry(cg, inst, err_union, body, err_union_ty, false);
7349	return cg.finishAir(inst, result, &.{unwrapped_try.error_union});
7350	}
7351
7352	fn airTryPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7353	const zcu = cg.pt.zcu;
7354	const unwrapped_try = cg.air.unwrapTryPtr(inst);
7355	const err_union_ptr = try cg.resolveInst(unwrapped_try.error_union_ptr);
7356	const body = unwrapped_try.else_body;
7357	const err_union_ty = cg.typeOf(unwrapped_try.error_union_ptr).childType(zcu);
7358	const result = try lowerTry(cg, inst, err_union_ptr, body, err_union_ty, true);
7359	return cg.finishAir(inst, result, &.{unwrapped_try.error_union_ptr});
7360	}
7361
7362	fn lowerTry(
7363	cg: *CodeGen,
7364	inst: Air.Inst.Index,
7365	err_union: WValue,
7366	body: []const Air.Inst.Index,
7367	err_union_ty: Type,
7368	operand_is_ptr: bool,
7369	) InnerError!WValue {
7370	_ = inst;
7371	const zcu = cg.pt.zcu;
7372
7373	const pl_ty = err_union_ty.errorUnionPayload(zcu);
7374	const pl_has_bits = pl_ty.hasRuntimeBits(zcu);
7375
7376	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7377	// Block we can jump out of when error is not set
7378	try cg.startBlock(.block, .empty);
7379
7380	// check if the error tag is set for the error union.
7381	try cg.emitWValue(err_union);
7382	if (pl_has_bits or operand_is_ptr) {
7383	const err_offset: u32 = @intCast(errUnionErrorOffset(pl_ty, zcu));
7384	try cg.addMemArg(.i32_load16_u, .{
7385	.offset = err_union.offset() + err_offset,
7386	.alignment = @intCast(Type.anyerror.abiAlignment(zcu).toByteUnits().?),
7387	});
7388	}
7389	try cg.addTag(.i32_eqz);
7390	try cg.addLabel(.br_if, 0); // jump out of block when error is '0'
7391
7392	try cg.branches.append(cg.gpa, .{});
7393	defer {
7394	var branch = cg.branches.pop().?;
7395	branch.deinit(cg.gpa);
7396	}
7397	try cg.genBody(body);
7398	try cg.endBlock();
7399	}
7400
7401	// if we reach here it means error was not set, and we want the payload
7402	if (!pl_has_bits and !operand_is_ptr) {
7403	return .none;
7404	}
7405
7406	const pl_offset: u32 = @intCast(errUnionPayloadOffset(pl_ty, zcu));
7407	if (operand_is_ptr or isByRef(pl_ty, zcu, cg.target)) {
7408	return buildPointerOffset(cg, err_union, pl_offset, .new);
7409	}
7410	const payload = try cg.load(err_union, pl_ty, pl_offset);
7411	return payload.toLocal(cg, pl_ty);
7412	}
7413
7414	/// Calls a compiler-rt intrinsic by creating an undefined symbol,
7415	/// then lowering the arguments and calling the symbol as a function call.
7416	/// This function call assumes the C-ABI.
7417	/// Asserts arguments are not stack values when the return value is
7418	/// passed as the first parameter.
7419	/// May leave the return value on the stack.
7420	fn callIntrinsic(
7421	cg: *CodeGen,
7422	intrinsic: Mir.Intrinsic,
7423	param_types: []const InternPool.Index,
7424	return_type: Type,
7425	args: []const WValue,
7426	) InnerError!WValue {
7427	assert(param_types.len == args.len);
7428	const zcu = cg.pt.zcu;
7429
7430	// Always pass over C-ABI
7431
7432	const want_sret_param = firstParamSRet(.{ .wasm_mvp = .{} }, return_type, zcu, cg.target);
7433	// if we want return as first param, we allocate a pointer to stack,
7434	// and emit it as our first argument
7435	const sret = if (want_sret_param) blk: {
7436	const sret_local = try cg.allocStack(return_type);
7437	try cg.lowerToStack(sret_local);
7438	break :blk sret_local;
7439	} else .none;
7440
7441	// Lower all arguments to the stack before we call our function
7442	for (args, 0..) |arg, arg_i| {
7443	assert(!(want_sret_param and arg == .stack));
7444	assert(Type.fromInterned(param_types[arg_i]).hasRuntimeBits(zcu));
7445	try cg.lowerArg(.{ .wasm_mvp = .{} }, Type.fromInterned(param_types[arg_i]), arg);
7446	}
7447
7448	try cg.addInst(.{ .tag = .call_intrinsic, .data = .{ .intrinsic = intrinsic } });
7449
7450	if (!return_type.hasRuntimeBits(zcu)) {
7451	return .none;
7452	} else if (want_sret_param) {
7453	return sret;
7454	} else {
7455	return .stack;
7456	}
7457	}
7458
7459	fn airTagName(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7460	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7461	const operand = try cg.resolveInst(un_op);
7462	const enum_ty = cg.typeOf(un_op);
7463
7464	try cg.addInst(.{ .tag = .enum_tag_name_table_ref, .data = .{ .ip_index = enum_ty.toIntern() } });
7465	try cg.lowerToStack(operand);
7466	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7467
7468	switch (cg.ptr_size) {
7469	.wasm32 => {
7470	try cg.addImm32(@intCast(8));
7471	try cg.addTag(.i32_mul);
7472	try cg.addTag(.i32_add);
7473	},
7474	.wasm64 => {
7475	try cg.addImm64(8);
7476	try cg.addTag(.i64_mul);
7477	try cg.addTag(.i64_add);
7478	},
7479	}
7480
7481	return cg.finishAir(inst, .stack, &.{un_op});
7482	}
7483
7484	fn airIsNamedEnumValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7485	const un_op = cg.air.instructions.items(.data)[@backingInt(inst)].un_op;
7486	const operand = try cg.resolveInst(un_op);
7487	const enum_ty = cg.typeOf(un_op);
7488
7489	try cg.lowerToStack(operand);
7490	try cg.addInst(.{ .tag = .call_tag_index, .data = .{ .ip_index = enum_ty.toIntern() } });
7491	try cg.addImm32(~@as(u32, 0));
7492	try cg.addTag(.i32_ne);
7493
7494	return cg.finishAir(inst, .stack, &.{un_op});
7495	}
7496
7497	fn airErrorSetHasValue(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7498	const zcu = cg.pt.zcu;
7499	const ip = &zcu.intern_pool;
7500	const ty_op = cg.air.instructions.items(.data)[@backingInt(inst)].ty_op;
7501
7502	const operand = try cg.resolveInst(ty_op.operand);
7503	const error_set_ty = ty_op.ty;
7504	const result = try cg.allocLocal(Type.bool);
7505
7506	const names = error_set_ty.errorSetNames(zcu);
7507	var values = try std.array_list.Managed(u32).initCapacity(cg.gpa, names.len);
7508	defer values.deinit();
7509
7510	var lowest: ?u32 = null;
7511	var highest: ?u32 = null;
7512	for (0..names.len) |name_index| {
7513	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
7514	if (lowest) |*l| {
7515	if (err_int < l.*) {
7516	l.* = err_int;
7517	}
7518	} else {
7519	lowest = err_int;
7520	}
7521	if (highest) |*h| {
7522	if (err_int > h.*) {
7523	h.* = err_int;
7524	}
7525	} else {
7526	highest = err_int;
7527	}
7528
7529	values.appendAssumeCapacity(err_int);
7530	}
7531
7532	// start block for 'true' branch
7533	try cg.startBlock(.block, .empty);
7534	// start block for 'false' branch
7535	try cg.startBlock(.block, .empty);
7536	// block for the jump table itself
7537	try cg.startBlock(.block, .empty);
7538
7539	// lower operand to determine jump table target
7540	try cg.emitWValue(operand);
7541	try cg.addImm32(lowest.?);
7542	try cg.addTag(.i32_sub);
7543
7544	// Account for default branch so always add '1'
7545	const depth = @as(u32, @intCast(highest.? - lowest.? + 1));
7546	const jump_table: Mir.JumpTable = .{ .length = depth + 1 };
7547	const table_extra_index = try cg.addExtra(jump_table);
7548	try cg.addInst(.{ .tag = .br_table, .data = .{ .payload = table_extra_index } });
7549	try cg.mir_extra.ensureUnusedCapacity(cg.gpa, depth + 1);
7550
7551	var value: u32 = lowest.?;
7552	while (value <= highest.?) : (value += 1) {
7553	const idx: u32 = blk: {
7554	for (values.items) |val| {
7555	if (val == value) break :blk 1;
7556	}
7557	break :blk 0;
7558	};
7559	cg.mir_extra.appendAssumeCapacity(idx);
7560	}
7561	cg.mir_extra.appendAssumeCapacity(0); // outside lowest...highest
7562	try cg.endBlock();
7563
7564	// 'false' branch (i.e. error set does not have value
7565	// ensure we set local to 0 in case the local was re-used.
7566	try cg.addImm32(0);
7567	try cg.addLocal(.local_set, result.local.value);
7568	try cg.addLabel(.br, 1);
7569	try cg.endBlock();
7570
7571	// 'true' branch
7572	try cg.addImm32(1);
7573	try cg.addLocal(.local_set, result.local.value);
7574	try cg.addLabel(.br, 0);
7575	try cg.endBlock();
7576
7577	return cg.finishAir(inst, result, &.{ty_op.operand});
7578	}
7579
7580	inline fn useAtomicFeature(cg: *const CodeGen) bool {
7581	return cg.target.cpu.has(.wasm, .atomics);
7582	}
7583
7584	fn airCmpxchg(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7585	const zcu = cg.pt.zcu;
7586	const ty_pl = cg.air.instructions.items(.data)[@backingInt(inst)].ty_pl;
7587	const extra = cg.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
7588
7589	const ptr_ty = cg.typeOf(extra.ptr);
7590	const ty = ptr_ty.childType(zcu);
7591	const result_ty = cg.typeOfIndex(inst);
7592
7593	const int_ty: IntType = .fromType(cg, ty);
7594
7595	const ptr_operand = try cg.resolveInst(extra.ptr);
7596	const expected_val = try cg.resolveInst(extra.expected_value);
7597	const new_val = try cg.resolveInst(extra.new_value);
7598
7599	const cmp_result = try cg.allocLocal(Type.bool);
7600
7601	const ptr_val = if (cg.useAtomicFeature()) val: {
7602	const val_local = try cg.allocLocal(ty);
7603	try cg.emitWValue(ptr_operand);
7604	try cg.lowerToStack(expected_val);
7605	try cg.lowerToStack(new_val);
7606	try cg.addAtomicMemArg(switch (ty.abiSize(zcu)) {
7607	1 => .i32_atomic_rmw8_cmpxchg_u,
7608	2 => .i32_atomic_rmw16_cmpxchg_u,
7609	4 => .i32_atomic_rmw_cmpxchg,
7610	8 => .i32_atomic_rmw_cmpxchg,
7611	else => |size| return cg.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}),
7612	}, .{
7613	.offset = ptr_operand.offset(),
7614	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7615	});
7616	try cg.addLocal(.local_tee, val_local.local.value);
7617	_ = try cg.intCmp(int_ty, .eq, .stack, expected_val);
7618	try cg.addLocal(.local_set, cmp_result.local.value);
7619	break :val val_local;
7620	} else val: {
7621	if (ty.abiSize(zcu) > 8) {
7622	return cg.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{});
7623	}
7624	const ptr_val = try WValue.toLocal(try cg.load(ptr_operand, ty, 0), cg, ty);
7625
7626	try cg.lowerToStack(ptr_operand);
7627	try cg.lowerToStack(new_val);
7628	try cg.emitWValue(ptr_val);
7629	_ = try cg.intCmp(int_ty, .eq, ptr_val, expected_val);
7630	try cg.addLocal(.local_tee, cmp_result.local.value);
7631	try cg.addTag(.select);
7632	try cg.store(.stack, .stack, ty, 0);
7633
7634	break :val ptr_val;
7635	};
7636
7637	const result = if (isByRef(result_ty, zcu, cg.target)) val: {
7638	try cg.emitWValue(cmp_result);
7639	try cg.addImm32(~@as(u32, 0));
7640	try cg.addTag(.i32_xor);
7641	try cg.addImm32(1);
7642	try cg.addTag(.i32_and);
7643	const and_result = try WValue.toLocal(.stack, cg, Type.bool);
7644	const result_ptr = try cg.allocStack(result_ty);
7645	try cg.store(result_ptr, and_result, Type.bool, @as(u32, @intCast(ty.abiSize(zcu))));
7646	try cg.store(result_ptr, ptr_val, ty, 0);
7647	break :val result_ptr;
7648	} else val: {
7649	try cg.addImm32(0);
7650	try cg.emitWValue(ptr_val);
7651	try cg.emitWValue(cmp_result);
7652	try cg.addTag(.select);
7653	break :val .stack;
7654	};
7655
7656	return cg.finishAir(inst, result, &.{ extra.ptr, extra.expected_value, extra.new_value });
7657	}
7658
7659	fn airAtomicLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7660	const zcu = cg.pt.zcu;
7661	const atomic_load = cg.air.instructions.items(.data)[@backingInt(inst)].atomic_load;
7662	const ptr = try cg.resolveInst(atomic_load.ptr);
7663	const ty = cg.typeOfIndex(inst);
7664
7665	if (cg.useAtomicFeature()) {
7666	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7667	1 => .i32_atomic_load8_u,
7668	2 => .i32_atomic_load16_u,
7669	4 => .i32_atomic_load,
7670	8 => .i64_atomic_load,
7671	else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7672	};
7673	try cg.emitWValue(ptr);
7674	try cg.addAtomicMemArg(tag, .{
7675	.offset = ptr.offset(),
7676	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7677	});
7678	} else {
7679	_ = try cg.load(ptr, ty, 0);
7680	}
7681
7682	return cg.finishAir(inst, .stack, &.{atomic_load.ptr});
7683	}
7684
7685	fn airAtomicRmw(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7686	const zcu = cg.pt.zcu;
7687	const pl_op = cg.air.instructions.items(.data)[@backingInt(inst)].pl_op;
7688	const extra = cg.air.extraData(Air.AtomicRmw, pl_op.payload).data;
7689
7690	const ptr = try cg.resolveInst(pl_op.operand);
7691	const operand = try cg.resolveInst(extra.operand);
7692	const ty = cg.typeOfIndex(inst);
7693	const op: std.lang.AtomicRmwOp = extra.op();
7694
7695	if (cg.useAtomicFeature()) {
7696	const int_ty: IntType = .fromType(cg, ty);
7697	switch (op) {
7698	.Max,
7699	.Min,
7700	.Nand,
7701	=> {
7702	const tmp = try cg.load(ptr, ty, 0);
7703	const value = try tmp.toLocal(cg, ty);
7704
7705	// create a loop to cmpxchg the new value
7706	try cg.startBlock(.loop, .empty);
7707
7708	try cg.emitWValue(ptr);
7709	try cg.emitWValue(value);
7710	if (op == .Nand) {
7711	const and_res = try cg.intAnd(int_ty, value, operand);
7712	if (int_ty.bits <= 32) {
7713	try cg.addImm32(~@as(u32, 0));
7714	} else if (int_ty.bits <= 64) {
7715	try cg.addImm64(~@as(u64, 0));
7716	} else {
7717	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7718	}
7719	_ = try cg.intXor(int_ty, and_res, .stack);
7720	} else {
7721	try cg.emitWValue(value);
7722	try cg.emitWValue(operand);
7723	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, value, operand);
7724	try cg.addTag(.select);
7725	}
7726	try cg.addAtomicMemArg(
7727	switch (ty.abiSize(zcu)) {
7728	1 => .i32_atomic_rmw8_cmpxchg_u,
7729	2 => .i32_atomic_rmw16_cmpxchg_u,
7730	4 => .i32_atomic_rmw_cmpxchg,
7731	8 => .i64_atomic_rmw_cmpxchg,
7732	else => return cg.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}),
7733	},
7734	.{
7735	.offset = ptr.offset(),
7736	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7737	},
7738	);
7739	const select_res = try cg.allocLocal(ty);
7740	try cg.addLocal(.local_tee, select_res.local.value);
7741	_ = try cg.intCmp(int_ty, .neq, .stack, value); // leave on stack so we can use it for br_if
7742
7743	try cg.emitWValue(select_res);
7744	try cg.addLocal(.local_set, value.local.value);
7745
7746	try cg.addLabel(.br_if, 0);
7747	try cg.endBlock();
7748	return cg.finishAir(inst, value, &.{ pl_op.operand, extra.operand });
7749	},
7750
7751	// the other operations have their own instructions for Wasm.
7752	else => {
7753	try cg.emitWValue(ptr);
7754	try cg.emitWValue(operand);
7755	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7756	1 => switch (op) {
7757	.Xchg => .i32_atomic_rmw8_xchg_u,
7758	.Add => .i32_atomic_rmw8_add_u,
7759	.Sub => .i32_atomic_rmw8_sub_u,
7760	.And => .i32_atomic_rmw8_and_u,
7761	.Or => .i32_atomic_rmw8_or_u,
7762	.Xor => .i32_atomic_rmw8_xor_u,
7763	else => unreachable,
7764	},
7765	2 => switch (op) {
7766	.Xchg => .i32_atomic_rmw16_xchg_u,
7767	.Add => .i32_atomic_rmw16_add_u,
7768	.Sub => .i32_atomic_rmw16_sub_u,
7769	.And => .i32_atomic_rmw16_and_u,
7770	.Or => .i32_atomic_rmw16_or_u,
7771	.Xor => .i32_atomic_rmw16_xor_u,
7772	else => unreachable,
7773	},
7774	4 => switch (op) {
7775	.Xchg => .i32_atomic_rmw_xchg,
7776	.Add => .i32_atomic_rmw_add,
7777	.Sub => .i32_atomic_rmw_sub,
7778	.And => .i32_atomic_rmw_and,
7779	.Or => .i32_atomic_rmw_or,
7780	.Xor => .i32_atomic_rmw_xor,
7781	else => unreachable,
7782	},
7783	8 => switch (op) {
7784	.Xchg => .i64_atomic_rmw_xchg,
7785	.Add => .i64_atomic_rmw_add,
7786	.Sub => .i64_atomic_rmw_sub,
7787	.And => .i64_atomic_rmw_and,
7788	.Or => .i64_atomic_rmw_or,
7789	.Xor => .i64_atomic_rmw_xor,
7790	else => unreachable,
7791	},
7792	else => |size| return cg.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}),
7793	};
7794	try cg.addAtomicMemArg(tag, .{
7795	.offset = ptr.offset(),
7796	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7797	});
7798	return cg.finishAir(inst, .stack, &.{ pl_op.operand, extra.operand });
7799	},
7800	}
7801	} else {
7802	const loaded = try cg.load(ptr, ty, 0);
7803	const result = try loaded.toLocal(cg, ty);
7804
7805	switch (op) {
7806	.Xchg => {
7807	try cg.store(ptr, operand, ty, 0);
7808	},
7809	.Add,
7810	.Sub,
7811	=> {
7812	if (ty.isAnyFloat()) {
7813	const float_ty: FloatType = .fromType(cg, ty);
7814	try cg.emitWValue(ptr);
7815	_ = switch (op) {
7816	.Add => try cg.floatAdd(float_ty, result, operand),
7817	.Sub => try cg.floatSub(float_ty, result, operand),
7818	else => unreachable,
7819	};
7820	try cg.store(.stack, .stack, ty, ptr.offset());
7821	} else {
7822	const int_ty: IntType = .fromType(cg, ty);
7823	try cg.emitWValue(ptr);
7824	_ = switch (op) {
7825	.Add => try cg.intAdd(int_ty, result, operand),
7826	.Sub => try cg.intSub(int_ty, result, operand),
7827	else => unreachable,
7828	};
7829	_ = try cg.intWrap(int_ty, .stack);
7830	try cg.store(.stack, .stack, ty, ptr.offset());
7831	}
7832	},
7833	.And,
7834	.Or,
7835	.Xor,
7836	=> {
7837	const int_ty: IntType = .fromType(cg, ty);
7838	try cg.emitWValue(ptr);
7839	_ = switch (op) {
7840	.And => try cg.intAnd(int_ty, result, operand),
7841	.Or => try cg.intOr(int_ty, result, operand),
7842	.Xor => try cg.intXor(int_ty, result, operand),
7843	else => unreachable,
7844	};
7845	try cg.store(.stack, .stack, ty, ptr.offset());
7846	},
7847	.Max,
7848	.Min,
7849	=> {
7850	if (ty.isAnyFloat()) {
7851	const float_ty: FloatType = .fromType(cg, ty);
7852	try cg.emitWValue(ptr);
7853	try cg.emitWValue(result);
7854	try cg.emitWValue(operand);
7855	_ = try cg.floatCmp(float_ty, if (op == .Max) .gt else .lt, result, operand);
7856	try cg.addTag(.select);
7857	try cg.store(.stack, .stack, ty, ptr.offset());
7858	} else {
7859	const int_ty: IntType = .fromType(cg, ty);
7860	try cg.emitWValue(ptr);
7861	try cg.emitWValue(result);
7862	try cg.emitWValue(operand);
7863	_ = try cg.intCmp(int_ty, if (op == .Max) .gt else .lt, result, operand);
7864	try cg.addTag(.select);
7865	try cg.store(.stack, .stack, ty, ptr.offset());
7866	}
7867	},
7868	.Nand => {
7869	const int_ty: IntType = .fromType(cg, ty);
7870	try cg.emitWValue(ptr);
7871	const and_res = try cg.intAnd(int_ty, result, operand);
7872	if (int_ty.bits <= 32) {
7873	try cg.addImm32(~@as(u32, 0));
7874	} else if (int_ty.bits <= 64) {
7875	try cg.addImm64(~@as(u64, 0));
7876	} else {
7877	return cg.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{});
7878	}
7879	_ = try cg.intXor(int_ty, and_res, .stack);
7880	try cg.store(.stack, .stack, ty, ptr.offset());
7881	},
7882	}
7883
7884	return cg.finishAir(inst, result, &.{ pl_op.operand, extra.operand });
7885	}
7886	}
7887
7888	fn airAtomicStore(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7889	const zcu = cg.pt.zcu;
7890	const bin_op = cg.air.instructions.items(.data)[@backingInt(inst)].bin_op;
7891
7892	const ptr = try cg.resolveInst(bin_op.lhs);
7893	const operand = try cg.resolveInst(bin_op.rhs);
7894	const ptr_ty = cg.typeOf(bin_op.lhs);
7895	const ty = ptr_ty.childType(zcu);
7896
7897	if (cg.useAtomicFeature()) {
7898	const tag: std.wasm.AtomicsOpcode = switch (ty.abiSize(zcu)) {
7899	1 => .i32_atomic_store8,
7900	2 => .i32_atomic_store16,
7901	4 => .i32_atomic_store,
7902	8 => .i64_atomic_store,
7903	else => |size| return cg.fail("TODO: @atomicLoad for types with abi size {d}", .{size}),
7904	};
7905	try cg.emitWValue(ptr);
7906	try cg.lowerToStack(operand);
7907	try cg.addAtomicMemArg(tag, .{
7908	.offset = ptr.offset(),
7909	.alignment = @intCast(ty.abiAlignment(zcu).toByteUnits().?),
7910	});
7911	} else {
7912	try cg.store(ptr, operand, ty, 0);
7913	}
7914
7915	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
7916	}
7917
7918	fn airFrameAddress(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7919	if (cg.initial_stack_value == .none) {
7920	try cg.initializeStack();
7921	}
7922	try cg.emitWValue(cg.bottom_stack_value);
7923	return cg.finishAir(inst, .stack, &.{});
7924	}
7925
7926	fn airRuntimeNavPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7927	const ty_nav = cg.air.instructions.items(.data)[@backingInt(inst)].ty_nav;
7928	const mod = cg.pt.zcu.navFileScope(cg.owner_nav).mod.?;
7929	if (mod.single_threaded) {
7930	const result: WValue = .{ .nav_ref = .{
7931	.nav_index = ty_nav.nav,
7932	.offset = 0,
7933	} };
7934	return cg.finishAir(inst, result, &.{});
7935	}
7936	return cg.fail("TODO: thread-local variables", .{});
7937	}
7938
7939	fn airAsm(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
7940	const unwrapped_asm = cg.air.unwrapAsm(inst);
7941	const outputs = unwrapped_asm.outputs;
7942	const inputs = unwrapped_asm.inputs;
7943
7944	const zcu = cg.pt.zcu;
7945	const output_ty = cg.typeOfIndex(inst);
7946
7947	const result: WValue = if (output_ty.hasRuntimeBits(zcu))
7948	try cg.allocLocal(output_ty)
7949	else
7950	.none;
7951
7952	if (unwrapped_asm.source.len != 0) {
7953	var local_map: assembly.LocalMap = .empty;
7954	defer local_map.deinit(cg.gpa);
7955
7956	{
7957	var it = unwrapped_asm.iterateOutputs();
7958	if (it.next()) |output| {
7959	const constraint = output.constraint;
7960	assert(output.operand == .none);
7961	const name = output.name;
7962
7963	if (!mem.eql(u8, constraint, "=r")) {
7964	return cg.fail("Self-hosted wasm backend requires output constraint to be equal \"=r\"", .{});
7965	}
7966
7967	const gop = try local_map.getOrPutValue(cg.gpa, name, result.local.value);
7968	assert(!gop.found_existing); // first value
7969
7970	assert(it.next() == null);
7971	}
7972	}
7973
7974	{
7975	var it = unwrapped_asm.iterateInputs();
7976	while (it.next()) |input| {
7977	const constraint = input.constraint;
7978	const operand = try cg.resolveInst(input.operand);
7979	const name = input.name;
7980
7981	if (!mem.eql(u8, constraint, "r")) {
7982	return cg.fail("Self-hosted wasm backend requires input constraint to be equal \"r\"", .{});
7983	}
7984
7985	try cg.lowerToStack(operand);
7986	const op_local = try WValue.toLocal(.stack, cg, cg.typeOf(input.operand));
7987
7988	const gop = try local_map.getOrPutValue(cg.gpa, name, op_local.local.value);
7989	if (gop.found_existing) {
7990	return cg.fail("Duplicate asm variable name \"{s}\"", .{name});
7991	}
7992	}
7993	}
7994
7995	try assembly.assemble(cg, unwrapped_asm.source, &local_map);
7996	}
7997
7998	var bt = cg.liveness.iterateBigTomb(inst);
7999	for (outputs) |output| if (output != .none) cg.feed(&bt, output);
8000	for (inputs) |input| cg.feed(&bt, input);
8001	return cg.finishAirResult(inst, result);
8002	}
8003
8004	fn typeOf(cg: *CodeGen, inst: Air.Inst.Ref) Type {
8005	const zcu = cg.pt.zcu;
8006	return cg.air.typeOf(inst, &zcu.intern_pool);
8007	}
8008
8009	fn typeOfIndex(cg: *CodeGen, inst: Air.Inst.Index) Type {
8010	const zcu = cg.pt.zcu;
8011	return cg.air.typeOfIndex(inst, &zcu.intern_pool);
8012	}