Commit 66f55a1cb49a7310cfa694c34a12140dc68d7d3b | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-03-29 13:41:10+02:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-03-29 13:41:10+02:00
log	66f55a1cb49a7310cfa694c34a12140dc68d7d3b
tree	9f90bfbd615dbaeee77cdb4e1dd1b00fdf99381f
parent	3dc5f13989676ae0bfb9389e2b162e3945b38241
parent	abaf3dfbe9b79dcdaf742fa1c08ce6f54061e237

Merge pull request 'llvm: simplifications, refactors, and make incremental work' (#31678) from llvm-enhancements into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31678

64 files changed, 9352 insertions(+), 9804 deletions(-)

lib
- compiler_rt
  - mulo.zig
- std
  - zig
    llvm
    Builder.zig
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - print.zig
- Compilation.zig
- Sema.zig
- Sema
  - bitcast.zig
- Type.zig
- Value.zig
- Zcu.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
  - c.zig
  - llvm.zig
  - llvm
    FuncGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig
- link
  - MachO
    Object.zig
  - Wasm
    Object.zig
- main.zig
test
- incremental
- tests.zig

lib/compiler_rt/mulo.zig+6-1

...	...	@@ -19,7 +19,12 @@ comptime {
19	19	inline fn muloXi4_genericSmall(comptime ST: type, a: ST, b: ST, overflow: *c_int) ST {
20	20	overflow.* = 0;
21	21	const min = math.minInt(ST);
22		const res: ST = a *% b;
	22	const res: ST = if (ST == i128 and builtin.target.cpu.arch.isWasm()) res: {
	23	// Despite compiler-rt being built with `-fno-builtin`, LLVM still converts this function to
	24	// a call to `__muloti4` on WASM. This is an upstream bug: circumvent it by directly calling
	25	// the "lower-level" compiler-rt routine for this wrapping multiplication.
	26	break :res @import("mulXi3.zig").__multi3(a, b);
	27	} else a *% b;
23	28	// Hacker's Delight section Overflow subsection Multiplication
24	29	// case a=-2^{31}, b=-1 problem, because
25	30	// on some machines a*b = -2^{31} with overflow

lib/std/zig/llvm/Builder.zig+107-48

...	...	@@ -2343,12 +2343,13 @@ pub const Global = struct {
2343	2343	none = maxInt(u32),
2344	2344	_,
2345	2345
2346		pub fn unwrap(self: Index, builder: *const Builder) Index {
2347		var cur = self;
	2346	pub fn unwrap(orig_index: Index, builder: *const Builder) Index {
	2347	var cur = orig_index;
2348	2348	while (true) {
2349		const replacement = cur.getReplacement(builder);
2350		if (replacement == .none) return cur;
2351		cur = replacement;
	2349	switch (builder.globals.values()[@intFromEnum(cur)].kind) {
	2350	.replaced => \|replacement\| cur = replacement,
	2351	else => return cur,
	2352	}
2352	2353	}
2353	2354	}
2354	2355
...	...	@@ -2388,8 +2389,12 @@ pub const Global = struct {
2388	2389	return self.ptrConst(builder).type;
2389	2390	}
2390	2391
2391		pub fn toConst(self: Index) Constant {
2392		return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(self));
	2392	pub fn toConst(global: Index) Constant {
	2393	return @enumFromInt(@intFromEnum(Constant.first_global) + @intFromEnum(global));
	2394	}
	2395
	2396	pub fn toValue(global: Index) Value {
	2397	return global.toConst().toValue();
2393	2398	}
2394	2399
2395	2400	pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void {
...	...	@@ -2450,6 +2455,42 @@ pub const Global = struct {
2450	2455	self.ptr(builder).kind = .{ .replaced = .none };
2451	2456	}
2452	2457
	2458	/// Replaces whatever this `Global` currently contains with a new `Function`. Similar to
	2459	/// `Builder.addFunction`, but the same `Global` is reused.
	2460	pub fn toNewFunction(global: Index, builder: *Builder) Allocator.Error!Function.Index {
	2461	try builder.functions.ensureUnusedCapacity(builder.gpa, 1);
	2462	errdefer comptime unreachable;
	2463	const function: Function.Index = @enumFromInt(builder.functions.items.len);
	2464	builder.functions.appendAssumeCapacity(.{
	2465	.global = global,
	2466	.strip = undefined,
	2467	});
	2468	global.ptr(builder).kind = .{ .function = function };
	2469	return function;
	2470	}
	2471
	2472	/// Replaces whatever this `Global` currently contains with a new `Variable`. Similar to
	2473	/// `Builder.addVariable`, but the same `Global` is reused.
	2474	pub fn toNewVariable(global: Index, builder: *Builder) Allocator.Error!Variable.Index {
	2475	try builder.variables.ensureUnusedCapacity(builder.gpa, 1);
	2476	errdefer comptime unreachable;
	2477	const variable: Variable.Index = @enumFromInt(builder.variables.items.len);
	2478	builder.variables.appendAssumeCapacity(.{ .global = global });
	2479	global.ptr(builder).kind = .{ .variable = variable };
	2480	return variable;
	2481	}
	2482
	2483	/// Replaces whatever this `Global` currently contains with a new `Alias`. Similar to
	2484	/// `Builder.addAlias`, but the same `Global` is reused.
	2485	pub fn toNewAlias(global: Index, builder: *Builder) Allocator.Error!Alias.Index {
	2486	try builder.aliases.ensureUnusedCapacity(builder.gpa, 1);
	2487	errdefer comptime unreachable;
	2488	const alias: Alias.Index = @enumFromInt(builder.aliases.items.len);
	2489	builder.aliass.appendAssumeCapacity(.{ .global = global, .aliasee = .none });
	2490	global.ptr(builder).kind = .{ .alias = alias };
	2491	return alias;
	2492	}
	2493
2453	2494	fn updateDsoLocal(self: Index, builder: *Builder) void {
2454	2495	const self_ptr = self.ptr(builder);
2455	2496	switch (self_ptr.linkage) {
...	...	@@ -2494,13 +2535,6 @@ pub const Global = struct {
2494	2535	self.renameAssumeCapacity(builder.next_replaced_global, builder);
2495	2536	self.ptr(builder).kind = .{ .replaced = other.unwrap(builder) };
2496	2537	}
2497
2498		fn getReplacement(self: Index, builder: *const Builder) Index {
2499		return switch (builder.globals.values()[@intFromEnum(self)].kind) {
2500		.replaced => \|replacement\| replacement,
2501		else => .none,
2502		};
2503		}
2504	2538	};
2505	2539	};
2506	2540
...	...	@@ -2593,22 +2627,6 @@ pub const Variable = struct {
2593	2627	return self.toConst(builder).toValue();
2594	2628	}
2595	2629
2596		pub fn setLinkage(self: Index, linkage: Linkage, builder: *Builder) void {
2597		return self.ptrConst(builder).global.setLinkage(linkage, builder);
2598		}
2599
2600		pub fn setVisibility(self: Index, visibility: Visibility, builder: *Builder) void {
2601		return self.ptrConst(builder).global.setVisibility(visibility, builder);
2602		}
2603
2604		pub fn setDllStorageClass(self: Index, class: DllStorageClass, builder: *Builder) void {
2605		return self.ptrConst(builder).global.setDllStorageClass(class, builder);
2606		}
2607
2608		pub fn setUnnamedAddr(self: Index, unnamed_addr: UnnamedAddr, builder: *Builder) void {
2609		return self.ptrConst(builder).global.setUnnamedAddr(unnamed_addr, builder);
2610		}
2611
2612	2630	pub fn setThreadLocal(self: Index, thread_local: ThreadLocal, builder: *Builder) void {
2613	2631	self.ptr(builder).thread_local = thread_local;
2614	2632	}
...	...	@@ -9692,8 +9710,12 @@ pub fn print(self: Builder, w: Writer) (Writer.Error \|\| Allocator.Error)!void
9692	9710
9693	9711	if (self.variables.items.len > 0) {
9694	9712	if (need_newline) try w.writeByte('\n') else need_newline = true;
9695		for (self.variables.items) \|variable\| {
9696		if (variable.global.getReplacement(self) != .none) continue;
	9713	for (self.variables.items, 0..) \|variable, variable_i\| {
	9714	// Skip the variable if its global has been repurposed for something else.
	9715	switch (variable.global.ptrConst(self).kind) {
	9716	.variable => \|v\| if (@intFromEnum(v) != variable_i) continue,
	9717	else => continue,
	9718	}
9697	9719	const global = variable.global.ptrConst(self);
9698	9720	metadata_formatter.need_comma = true;
9699	9721	defer metadata_formatter.need_comma = undefined;
...	...	@@ -9723,8 +9745,12 @@ pub fn print(self: Builder, w: Writer) (Writer.Error \|\| Allocator.Error)!void
9723	9745
9724	9746	if (self.aliases.items.len > 0) {
9725	9747	if (need_newline) try w.writeByte('\n') else need_newline = true;
9726		for (self.aliases.items) \|alias\| {
9727		if (alias.global.getReplacement(self) != .none) continue;
	9748	for (self.aliases.items, 0..) \|alias, alias_i\| {
	9749	// Skip the alias if its global has been repurposed for something else.
	9750	switch (alias.global.ptrConst(self).kind) {
	9751	.alias => \|a\| if (@intFromEnum(a) != alias_i) continue,
	9752	else => continue,
	9753	}
9728	9754	const global = alias.global.ptrConst(self);
9729	9755	metadata_formatter.need_comma = true;
9730	9756	defer metadata_formatter.need_comma = undefined;
...	...	@@ -9750,7 +9776,11 @@ pub fn print(self: Builder, w: Writer) (Writer.Error \|\| Allocator.Error)!void
9750	9776	defer attribute_groups.deinit(self.gpa);
9751	9777
9752	9778	for (0.., self.functions.items) \|function_i, function\| {
9753		if (function.global.getReplacement(self) != .none) continue;
	9779	// Skip the function if its global has been repurposed for something else.
	9780	switch (function.global.ptrConst(self).kind) {
	9781	.function => \|f\| if (@intFromEnum(f) != function_i) continue,
	9782	else => continue,
	9783	}
9754	9784	if (need_newline) try w.writeByte('\n') else need_newline = true;
9755	9785	const function_index: Function.Index = @enumFromInt(function_i);
9756	9786	const global = function.global.ptrConst(self);
...	...	@@ -13687,20 +13717,32 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
13687	13717	self.aliases.items.len,
13688	13718	);
13689	13719
13690		for (self.variables.items) \|variable\| {
13691		if (variable.global.getReplacement(self) != .none) continue;
	13720	for (self.variables.items, 0..) \|variable, variable_i\| {
	13721	// Skip the variable if its global has been repurposed for something else.
	13722	switch (variable.global.ptrConst(self).kind) {
	13723	.variable => \|v\| if (@intFromEnum(v) != variable_i) continue,
	13724	else => continue,
	13725	}
13692	13726
13693	13727	globals.putAssumeCapacity(variable.global, {});
13694	13728	}
13695	13729
13696		for (self.functions.items) \|function\| {
13697		if (function.global.getReplacement(self) != .none) continue;
	13730	for (self.functions.items, 0..) \|function, function_i\| {
	13731	// Skip the function if its global has been repurposed for something else.
	13732	switch (function.global.ptrConst(self).kind) {
	13733	.function => \|f\| if (@intFromEnum(f) != function_i) continue,
	13734	else => continue,
	13735	}
13698	13736
13699	13737	globals.putAssumeCapacity(function.global, {});
13700	13738	}
13701	13739
13702		for (self.aliases.items) \|alias\| {
13703		if (alias.global.getReplacement(self) != .none) continue;
	13740	for (self.aliases.items, 0..) \|alias, alias_i\| {
	13741	// Skip the alias if its global has been repurposed for something else.
	13742	switch (alias.global.ptrConst(self).kind) {
	13743	.alias => \|a\| if (@intFromEnum(a) != alias_i) continue,
	13744	else => continue,
	13745	}
13704	13746
13705	13747	globals.putAssumeCapacity(alias.global, {});
13706	13748	}
...	...	@@ -13742,8 +13784,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
13742	13784	defer section_map.deinit(self.gpa);
13743	13785	try section_map.ensureUnusedCapacity(self.gpa, globals.count());
13744	13786
13745		for (self.variables.items) \|variable\| {
13746		if (variable.global.getReplacement(self) != .none) continue;
	13787	for (self.variables.items, 0..) \|variable, variable_i\| {
	13788	// Skip the variable if its global has been repurposed for something else.
	13789	switch (variable.global.ptrConst(self).kind) {
	13790	.variable => \|v\| if (@intFromEnum(v) != variable_i) continue,
	13791	else => continue,
	13792	}
13747	13793
13748	13794	const section = blk: {
13749	13795	if (variable.section == .none) break :blk 0;
...	...	@@ -13789,8 +13835,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
13789	13835	});
13790	13836	}
13791	13837
13792		for (self.functions.items) \|func\| {
13793		if (func.global.getReplacement(self) != .none) continue;
	13838	for (self.functions.items, 0..) \|func, func_i\| {
	13839	// Skip the function if its global has been repurposed for something else.
	13840	switch (func.global.ptrConst(self).kind) {
	13841	.function => \|f\| if (@intFromEnum(f) != func_i) continue,
	13842	else => continue,
	13843	}
13794	13844
13795	13845	const section = blk: {
13796	13846	if (func.section == .none) break :blk 0;
...	...	@@ -13830,8 +13880,12 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
13830	13880	});
13831	13881	}
13832	13882
13833		for (self.aliases.items) \|alias\| {
13834		if (alias.global.getReplacement(self) != .none) continue;
	13883	for (self.aliases.items, 0..) \|alias, alias_i\| {
	13884	// Skip the alias if its global has been repurposed for something else.
	13885	switch (alias.global.ptrConst(self).kind) {
	13886	.alias => \|a\| if (@intFromEnum(a) != alias_i) continue,
	13887	else => continue,
	13888	}
13835	13889
13836	13890	const strtab = alias.global.strtab(self);
13837	13891
...	...	@@ -14635,8 +14689,13 @@ pub fn toBitcode(self: *Builder, allocator: Allocator, producer: Producer) bitco
14635	14689	};
14636	14690
14637	14691	for (self.functions.items, 0..) \|func, func_index\| {
	14692	// Skip the function if its global has been repurposed for something else.
	14693	switch (func.global.ptrConst(self).kind) {
	14694	.function => \|f\| if (@intFromEnum(f) != func_index) continue,
	14695	else => continue,
	14696	}
	14697
14638	14698	const FunctionBlock = ir.ModuleBlock.FunctionBlock;
14639		if (func.global.getReplacement(self) != .none) continue;
14640	14699
14641	14700	if (func.instructions.len == 0) continue;
14642	14701

src/Air.zig+14-17

...	...	@@ -870,14 +870,20 @@ pub const Inst = struct {
870	870	/// Uses the `pl_op` field, payload represents the index of the target memory.
871	871	wasm_memory_grow,
872	872
873		/// Returns `true` if and only if the operand, an integer with
874		/// the same size as the error integer type, is less than the
875		/// total number of errors in the Module.
	873	/// Returns `true` if and only if the operand, an integer with the same
	874	/// size as the error integer type, is less than or equal to the total
	875	/// number of errors in the Zcu. The "or equal to" is a consequence of
	876	/// value 0 being reserved for the "non-error" status in error unions.
	877	///
	878	/// This instruction exists (as opposed to just using `cmp_lte` against
	879	/// a constant) because the number of errors in the Zcu is not known
	880	/// until `Compilation.flush`. Before then, semantic analysis could
	881	/// discover new errors at any time.
	882	///
876	883	/// Result type is always `bool`.
	884	///
877	885	/// Uses the `un_op` field.
878		/// Note that the number of errors in the Module cannot be considered stable until
879		/// flush().
880		cmp_lt_errors_len,
	886	cmp_lte_errors_len,
881	887
882	888	/// Returns pointer to current error return trace.
883	889	err_return_trace,
...	...	@@ -1616,7 +1622,7 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1616	1622	.cmp_gte_optimized,
1617	1623	.cmp_gt_optimized,
1618	1624	.cmp_neq_optimized,
1619		.cmp_lt_errors_len,
	1625	.cmp_lte_errors_len,
1620	1626	.is_null,
1621	1627	.is_non_null,
1622	1628	.is_null_ptr,
...	...	@@ -1836,15 +1842,6 @@ pub fn internedToRef(ip_index: InternPool.Index) Inst.Ref {
1836	1842	return .fromIntern(ip_index);
1837	1843	}
1838	1844
1839		/// Returns `null` if runtime-known.
1840		pub fn value(air: Air, inst: Inst.Ref, pt: Zcu.PerThread) !?Value {
1841		if (inst.toInterned()) \|ip_index\| {
1842		return .fromInterned(ip_index);
1843		}
1844		const index = inst.toIndex().?;
1845		return air.typeOfIndex(index, &pt.zcu.intern_pool).onePossibleValue(pt);
1846		}
1847
1848	1845	pub const NullTerminatedString = enum(u32) {
1849	1846	none = std.math.maxInt(u32),
1850	1847	_,
...	...	@@ -2061,7 +2058,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2061	2058	.mul_add,
2062	2059	.field_parent_ptr,
2063	2060	.wasm_memory_size,
2064		.cmp_lt_errors_len,
	2061	.cmp_lte_errors_len,
2065	2062	.err_return_trace,
2066	2063	.addrspace_cast,
2067	2064	.save_err_return_trace_index,

src/Air/Legalize.zig+1-1

...	...	@@ -884,7 +884,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
884	884	.field_parent_ptr,
885	885	.wasm_memory_size,
886	886	.wasm_memory_grow,
887		.cmp_lt_errors_len,
	887	.cmp_lte_errors_len,
888	888	.err_return_trace,
889	889	.set_err_return_trace,
890	890	.addrspace_cast,

src/Air/Liveness.zig+1-1

...	...	@@ -565,7 +565,7 @@ fn analyzeInst(
565	565	.trunc_float,
566	566	.neg,
567	567	.neg_optimized,
568		.cmp_lt_errors_len,
	568	.cmp_lte_errors_len,
569	569	.set_err_return_trace,
570	570	.c_va_end,
571	571	=> {

src/Air/Liveness/Verify.zig+1-1

...	...	@@ -152,7 +152,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
152	152	.trunc_float,
153	153	.neg,
154	154	.neg_optimized,
155		.cmp_lt_errors_len,
	155	.cmp_lte_errors_len,
156	156	.set_err_return_trace,
157	157	.c_va_end,
158	158	=> {

src/Air/print.zig+1-1

...	...	@@ -211,7 +211,7 @@ const Writer = struct {
211	211	.trunc_float,
212	212	.neg,
213	213	.neg_optimized,
214		.cmp_lt_errors_len,
	214	.cmp_lte_errors_len,
215	215	.set_err_return_trace,
216	216	.c_va_end,
217	217	=> try w.writeUnOp(s, inst),

src/Compilation.zig+1-1

...	...	@@ -2485,7 +2485,7 @@ pub fn create(gpa: Allocator, arena: Allocator, io: Io, diag: *CreateDiagnostic,
2485	2485
2486	2486	if (use_llvm) {
2487	2487	if (opt_zcu) \|zcu\| {
2488		zcu.llvm_object = try LlvmObject.create(arena, comp);
	2488	zcu.llvm_object = try LlvmObject.create(arena, zcu);
2489	2489	}
2490	2490	}
2491	2491

src/Sema.zig+3-4

...	...	@@ -5751,7 +5751,6 @@ fn zirExport(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!void
5751	5751	if (ptr_info.byte_offset != 0) {
5752	5752	return sema.fail(block, ptr_src, "TODO: export pointer in middle of value", .{});
5753	5753	}
5754		if (zcu.llvm_object != null and options.linkage == .internal) return;
5755	5754	try sema.exports.append(zcu.gpa, .{
5756	5755	.opts = options,
5757	5756	.src = src,
...	...	@@ -7832,10 +7831,10 @@ fn zirErrorFromInt(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstD
7832	7831	}
7833	7832	try sema.requireRuntimeBlock(block, src, operand_src);
7834	7833	if (block.wantSafety()) {
7835		const is_lt_len = try block.addUnOp(.cmp_lt_errors_len, operand);
	7834	const is_lte_len = try block.addUnOp(.cmp_lte_errors_len, operand);
7836	7835	const zero_val = Air.internedToRef((try pt.intValue(err_int_ty, 0)).toIntern());
7837	7836	const is_non_zero = try block.addBinOp(.cmp_neq, operand, zero_val);
7838		const ok = try block.addBinOp(.bool_and, is_lt_len, is_non_zero);
	7837	const ok = try block.addBinOp(.bool_and, is_lte_len, is_non_zero);
7839	7838	try sema.addSafetyCheck(block, src, ok, .invalid_error_code);
7840	7839	}
7841	7840	return block.addInst(.{
...	...	@@ -18896,7 +18895,7 @@ fn finishStructInit(
18896	18895	var bit_offset: u16 = 0;
18897	18896	for (field_inits) \|field_init\| {
18898	18897	const field_val = sema.resolveValue(field_init).?;
18899		field_val.writeToPackedMemory(pt, buf, bit_offset) catch \|err\| switch (err) {
	18898	field_val.writeToPackedMemory(zcu, buf, bit_offset) catch \|err\| switch (err) {
18900	18899	error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
18901	18900	error.OutOfMemory => \|e\| return e,
18902	18901	};

src/Sema/bitcast.zig+2-2

...	...	@@ -443,7 +443,7 @@ const UnpackValueBits = struct {
443	443	// This @intCast is okay because no primitive can exceed the size of a u16.
444	444	const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
445	445	const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
446		try val.writeToPackedMemory(unpack.pt, buf, 0);
	446	try val.writeToPackedMemory(zcu, buf, 0);
447	447	const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
448	448	try unpack.primitive(sub_val);
449	449	},
...	...	@@ -722,7 +722,7 @@ const PackValueBits = struct {
722	722	const val = Value.fromInterned(ip_val);
723	723	const ty = val.typeOf(zcu);
724	724	if (!val.isUndef(zcu)) {
725		try val.writeToPackedMemory(pt, buf, cur_bit_off);
	725	try val.writeToPackedMemory(zcu, buf, cur_bit_off);
726	726	}
727	727	cur_bit_off += @intCast(ty.bitSize(zcu));
728	728	}

src/Type.zig+1-1

...	...	@@ -1594,7 +1594,7 @@ pub fn unionTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 {
1594	1594	return zcu.unionTagFieldIndex(union_obj, enum_tag);
1595	1595	}
1596	1596
1597		pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *Zcu) bool {
	1597	pub fn unionHasAllZeroBitFieldTypes(ty: Type, zcu: *const Zcu) bool {
1598	1598	assertHasLayout(ty, zcu);
1599	1599	const ip = &zcu.intern_pool;
1600	1600	const union_obj = zcu.typeToUnion(ty).?;

src/Value.zig+32-36

...	...	@@ -245,13 +245,12 @@ pub fn toBool(val: Value) bool {
245	245	///
246	246	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
247	247	/// the end of the value in memory.
248		pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{
	248	pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
249	249	ReinterpretDeclRef,
250	250	IllDefinedMemoryLayout,
251	251	Unimplemented,
252	252	OutOfMemory,
253	253	}!void {
254		const zcu = pt.zcu;
255	254	const target = zcu.getTarget();
256	255	const endian = target.cpu.arch.endian();
257	256	const ip = &zcu.intern_pool;
...	...	@@ -289,14 +288,18 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{
289	288	else => unreachable,
290	289	},
291	290	.array => {
	291	const aggregate = ip.indexToKey(val.toIntern()).aggregate;
292	292	const len = ty.arrayLen(zcu);
293	293	const elem_ty = ty.childType(zcu);
294	294	const elem_size: usize = @intCast(elem_ty.abiSize(zcu));
295	295	var elem_i: usize = 0;
296	296	var buf_off: usize = 0;
297	297	while (elem_i < len) : (elem_i += 1) {
298		const elem_val = try val.elemValue(pt, elem_i);
299		try elem_val.writeToMemory(pt, buffer[buf_off..]);
	298	switch (aggregate.storage) {
	299	.bytes => \|bytes\| buffer[buf_off] = bytes.at(elem_i, ip),
	300	.elems => \|elems\| try Value.fromInterned(elems[elem_i]).writeToMemory(zcu, buffer[buf_off..]),
	301	.repeated_elem => \|elem\| try Value.fromInterned(elem).writeToMemory(zcu, buffer[buf_off..]),
	302	}
300	303	buf_off += elem_size;
301	304	}
302	305	},
...	...	@@ -304,7 +307,7 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{
304	307	// We use byte_count instead of abi_size here, so that any padding bytes
305	308	// follow the data bytes, on both big- and little-endian systems.
306	309	const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
307		return writeToPackedMemory(val, pt, buffer[0..byte_count], 0);
	310	return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
308	311	},
309	312	.@"struct" => {
310	313	const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
...	...	@@ -320,42 +323,33 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{
320	323	.elems => \|elems\| elems[field_index],
321	324	.repeated_elem => \|elem\| elem,
322	325	});
323		try writeToMemory(field_val, pt, buffer[off..]);
	326	try writeToMemory(field_val, zcu, buffer[off..]);
324	327	},
325	328	.@"packed" => {
326	329	const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
327		return Value.fromInterned(int_index).writeToMemory(pt, buffer);
	330	return Value.fromInterned(int_index).writeToMemory(zcu, buffer);
328	331	},
329	332	}
330	333	},
331	334	.@"union" => switch (ty.containerLayout(zcu)) {
332	335	.auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
333	336	.@"extern" => {
334		if (val.unionTag(zcu)) \|union_tag\| {
335		const union_obj = zcu.typeToUnion(ty).?;
336		const field_index = zcu.unionTagFieldIndex(union_obj, union_tag).?;
337		const field_type = Type.fromInterned(union_obj.field_types.get(ip)[field_index]);
338		const field_val = try val.fieldValue(pt, field_index);
339		const byte_count: usize = @intCast(field_type.abiSize(zcu));
340		return writeToMemory(field_val, pt, buffer[0..byte_count]);
341		} else {
342		const backing_ty = try ty.externUnionBackingType(pt);
343		const byte_count: usize = @intCast(backing_ty.abiSize(zcu));
344		return writeToMemory(val.unionPayload(zcu), pt, buffer[0..byte_count]);
345		}
	337	const payload_val = val.unionPayload(zcu);
	338	return writeToMemory(payload_val, zcu, buffer);
346	339	},
347	340	.@"packed" => {
348	341	const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
349		return writeToMemory(int_val, pt, buffer);
	342	return writeToMemory(int_val, zcu, buffer);
350	343	},
351	344	},
352	345	.optional => {
353	346	if (!ty.isPtrLikeOptional(zcu)) return error.IllDefinedMemoryLayout;
354	347	const opt_val = val.optionalValue(zcu);
355	348	if (opt_val) \|some\| {
356		return some.writeToMemory(pt, buffer);
	349	return some.writeToMemory(zcu, buffer);
357	350	} else {
358		return writeToMemory(try pt.intValue(Type.usize, 0), pt, buffer);
	351	const byte_count = Type.usize.abiSize(zcu);
	352	@memset(buffer[0..@intCast(byte_count)], 0); // null pointer
359	353	}
360	354	},
361	355	else => return error.Unimplemented,
...	...	@@ -368,11 +362,10 @@ pub fn writeToMemory(val: Value, pt: Zcu.PerThread, buffer: []u8) error{
368	362	/// big-endian packed memory layouts start at the end of the buffer.
369	363	pub fn writeToPackedMemory(
370	364	val: Value,
371		pt: Zcu.PerThread,
	365	zcu: *const Zcu,
372	366	buffer: []u8,
373	367	bit_offset: usize,
374	368	) error{ ReinterpretDeclRef, OutOfMemory }!void {
375		const zcu = pt.zcu;
376	369	const ip = &zcu.intern_pool;
377	370	const target = zcu.getTarget();
378	371	const endian = target.cpu.arch.endian();
...	...	@@ -399,7 +392,7 @@ pub fn writeToPackedMemory(
399	392	},
400	393	.@"enum" => {
401	394	const int_val = val.intFromEnum(zcu);
402		return int_val.writeToPackedMemory(pt, buffer, bit_offset);
	395	return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
403	396	},
404	397	.pointer => {
405	398	assert(!ty.isSlice(zcu)); // No well defined layout.
...	...	@@ -430,25 +423,29 @@ pub fn writeToPackedMemory(
430	423
431	424	var bits: u16 = 0;
432	425	var elem_i: usize = 0;
	426	const aggregate = ip.indexToKey(val.toIntern()).aggregate;
433	427	while (elem_i < len) : (elem_i += 1) {
434	428	// On big-endian systems, LLVM reverses the element order of vectors by default
435	429	const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
436		const elem_val = try val.elemValue(pt, tgt_elem_i);
437		try elem_val.writeToPackedMemory(pt, buffer, bit_offset + bits);
	430	switch (aggregate.storage) {
	431	.bytes => \|bytes\| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
	432	.elems => \|elems\| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
	433	.repeated_elem => \|elem\| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
	434	}
438	435	bits += elem_bit_size;
439	436	}
440	437	},
441	438	.@"struct", .@"union" => {
442	439	assert(ty.containerLayout(zcu) == .@"packed");
443	440	const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
444		return int_val.writeToPackedMemory(pt, buffer, bit_offset);
	441	return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
445	442	},
446	443	.optional => {
447	444	assert(ty.isPtrLikeOptional(zcu));
448	445	if (val.optionalValue(zcu)) \|ptr_val\| {
449		return ptr_val.writeToPackedMemory(pt, buffer, bit_offset);
	446	return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);
450	447	} else {
451		return Value.zero_usize.writeToPackedMemory(pt, buffer, bit_offset);
	448	return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);
452	449	}
453	450	},
454	451	else => @panic("TODO implement writeToPackedMemory for more types"),
...	...	@@ -889,7 +886,7 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
889	886	const sfba = sfba_state.get();
890	887	const buf = try sfba.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
891	888	defer sfba.free(buf);
892		int_val.writeToPackedMemory(pt, buf, 0) catch \|err\| switch (err) {
	889	int_val.writeToPackedMemory(zcu, buf, 0) catch \|err\| switch (err) {
893	890	error.ReinterpretDeclRef => unreachable, // it's an integer
894	891	error.OutOfMemory => \|e\| return e,
895	892	};
...	...	@@ -902,7 +899,7 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
902	899	};
903	900	}
904	901
905		pub fn unionTag(val: Value, zcu: *Zcu) ?Value {
	902	pub fn unionTag(val: Value, zcu: *const Zcu) ?Value {
906	903	return switch (zcu.intern_pool.indexToKey(val.toIntern())) {
907	904	.undef, .enum_tag => val,
908	905	.un => \|un\| if (un.tag != .none) Value.fromInterned(un.tag) else return null,
...	...	@@ -910,7 +907,7 @@ pub fn unionTag(val: Value, zcu: *Zcu) ?Value {
910	907	};
911	908	}
912	909
913		pub fn unionPayload(val: Value, zcu: *Zcu) Value {
	910	pub fn unionPayload(val: Value, zcu: *const Zcu) Value {
914	911	return switch (zcu.intern_pool.indexToKey(val.toIntern())) {
915	912	.un => \|un\| Value.fromInterned(un.val),
916	913	else => unreachable,
...	...	@@ -1605,15 +1602,14 @@ pub fn mulAddScalar(
1605	1602
1606	1603	/// If the value is represented in-memory as a series of bytes that all
1607	1604	/// have the same value, return that byte value, otherwise null.
1608		pub fn hasRepeatedByteRepr(val: Value, pt: Zcu.PerThread) !?u8 {
1609		const zcu = pt.zcu;
	1605	pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
1610	1606	const ty = val.typeOf(zcu);
1611	1607	const abi_size = std.math.cast(usize, ty.abiSize(zcu)) orelse return null;
1612	1608	assert(abi_size >= 1);
1613	1609	const byte_buffer = try zcu.gpa.alloc(u8, abi_size);
1614	1610	defer zcu.gpa.free(byte_buffer);
1615	1611
1616		writeToMemory(val, pt, byte_buffer) catch \|err\| switch (err) {
	1612	writeToMemory(val, zcu, byte_buffer) catch \|err\| switch (err) {
1617	1613	error.OutOfMemory => return error.OutOfMemory,
1618	1614	error.ReinterpretDeclRef => return null,
1619	1615	// TODO: The writeToMemory function was originally created for the purpose

src/Zcu.zig+3-1

...	...	@@ -3731,7 +3731,9 @@ pub fn resetUnit(zcu: *Zcu, unit: AnalUnit) void {
3731	3731	};
3732	3732	for (zcu.all_exports.items[base..][0..len], base..) \|exp, exp_index_usize\| {
3733	3733	const exp_index: Export.Index = @enumFromInt(exp_index_usize);
3734		if (zcu.comp.bin_file) \|lf\| {
	3734	if (zcu.llvm_object) \|llvm_object\| {
	3735	_ = llvm_object; // TODO: delete exports from LLVM
	3736	} else if (zcu.comp.bin_file) \|lf\| {
3735	3737	lf.deleteExport(exp.exported, exp.opts.name);
3736	3738	}
3737	3739	if (zcu.failed_exports.fetchSwapRemove(exp_index)) \|failed_kv\| {

src/Zcu/PerThread.zig+2-9

...	...	@@ -1910,14 +1910,7 @@ fn analyzeNavVal(
1910	1910
1911	1911	try sema.flushExports();
1912	1912
1913		queue_codegen: {
1914		if (!queue_linker_work) break :queue_codegen;
1915
1916		if (!nav_ty.hasRuntimeBits(zcu)) {
1917		if (comp.config.use_llvm) break :queue_codegen;
1918		if (file.mod.?.strip) break :queue_codegen;
1919		}
1920
	1913	if (queue_linker_work) {
1921	1914	comp.link_prog_node.increaseEstimatedTotalItems(1);
1922	1915	try comp.link_queue.enqueueZcu(comp, pt.tid, .{ .link_nav = nav_id });
1923	1916	}
...	...	@@ -3751,7 +3744,7 @@ fn processExportsInner(
3751	3744	if (skip_linker_work) return;
3752	3745
3753	3746	if (zcu.llvm_object) \|llvm_object\| {
3754		try zcu.handleUpdateExports(export_indices, llvm_object.updateExports(pt, exported, export_indices));
	3747	try zcu.handleUpdateExports(export_indices, llvm_object.updateExports(exported, export_indices));
3755	3748	} else if (zcu.comp.bin_file) \|lf\| {
3756	3749	try zcu.handleUpdateExports(export_indices, lf.updateExports(pt, exported, export_indices));
3757	3750	}

src/codegen/aarch64/Select.zig+3-3

...	...	@@ -522,7 +522,7 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
522	522	.is_named_enum_value,
523	523	.tag_name,
524	524	.error_name,
525		.cmp_lt_errors_len,
	525	.cmp_lte_errors_len,
526	526	=> {
527	527	const un_op = air_data[@intFromEnum(air_inst_index)].un_op;
528	528
...	...	@@ -7175,7 +7175,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
7175	7175	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
7176	7176	},
7177	7177	.wasm_memory_size, .wasm_memory_grow => unreachable,
7178		.cmp_lt_errors_len => {
	7178	.cmp_lte_errors_len => {
7179	7179	if (isel.live_values.fetchRemove(air.inst_index)) \|is_vi\| unused: {
7180	7180	defer is_vi.value.deref(isel);
7181	7181	const is_ra = try is_vi.value.defReg(isel) orelse break :unused;
...	...	@@ -11364,7 +11364,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
11364	11364	const zcu = isel.pt.zcu;
11365	11365	const ip = &zcu.intern_pool;
11366	11366	if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11367		constant.writeToMemory(isel.pt, buffer) catch \|err\| switch (err) {
	11367	constant.writeToMemory(zcu, buffer) catch \|err\| switch (err) {
11368	11368	error.OutOfMemory => return error.OutOfMemory,
11369	11369	error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
11370	11370	};

src/codegen/c.zig+14-14

...	...	@@ -435,8 +435,7 @@ pub const Function = struct {
435	435	fn resolveInst(f: *Function, ref: Air.Inst.Ref) !CValue {
436	436	const gop = try f.value_map.getOrPut(ref);
437	437	if (!gop.found_existing) {
438		const val = try f.air.value(ref, f.dg.pt);
439		gop.value_ptr.* = .{ .constant = val.? };
	438	gop.value_ptr.* = .{ .constant = .fromInterned(ref.toInterned().?) };
440	439	}
441	440	return gop.value_ptr.*;
442	441	}
...	...	@@ -2723,7 +2722,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2723	2722	const extra = f.air.extraData(Air.VectorCmp, ty_pl.payload).data;
2724	2723	break :blk try airCmpOp(f, inst, extra, extra.compareOperator());
2725	2724	},
2726		.cmp_lt_errors_len => try airCmpLtErrorsLen(f, inst),
	2725	.cmp_lte_errors_len => try airCmpLteErrorsLen(f, inst),
2727	2726
2728	2727	// bool_and and bool_or are non-short-circuit operations
2729	2728	.bool_and, .bit_and => try airBinOp(f, inst, "&", "and", .none),
...	...	@@ -3389,7 +3388,7 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3389	3388	const ptr_val = try f.resolveInst(bin_op.lhs);
3390	3389	const src_ty = f.typeOf(bin_op.rhs);
3391	3390
3392		const val_is_undef = if (try f.air.value(bin_op.rhs, pt)) \|v\| v.isUndef(zcu) else false;
	3391	const val_is_undef = if (bin_op.rhs.toInterned()) \|ip_index\| Value.fromInterned(ip_index).isUndef(zcu) else false;
3393	3392
3394	3393	const w = &f.code.writer;
3395	3394	if (val_is_undef) {
...	...	@@ -3729,7 +3728,7 @@ fn airEquality(
3729	3728	return local;
3730	3729	}
3731	3730
3732		fn airCmpLtErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue {
	3731	fn airCmpLteErrorsLen(f: *Function, inst: Air.Inst.Index) !CValue {
3733	3732	const un_op = f.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
3734	3733
3735	3734	const operand = try f.resolveInst(un_op);
...	...	@@ -3922,8 +3921,8 @@ fn airCall(
3922	3921
3923	3922	callee: {
3924	3923	known: {
3925		const callee_val = (try f.air.value(call.callee, pt)) orelse break :known;
3926		const fn_nav, const need_cast = switch (ip.indexToKey(callee_val.toIntern())) {
	3924	const callee_ip_index = call.callee.toInterned() orelse break :known;
	3925	const fn_nav, const need_cast = switch (ip.indexToKey(callee_ip_index)) {
3927	3926	.@"extern" => \|@"extern"\| .{ @"extern".owner_nav, false },
3928	3927	.func => \|func\| .{ func.owner_nav, Type.fromInterned(func.ty).fnCallingConvention(zcu) != .naked and
3929	3928	Type.fromInterned(func.uncoerced_ty).fnCallingConvention(zcu) == .naked },
...	...	@@ -4027,7 +4026,7 @@ fn airDbgVar(f: *Function, inst: Air.Inst.Index) !CValue {
4027	4026	const tag = f.air.instructions.items(.tag)[@intFromEnum(inst)];
4028	4027	const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
4029	4028	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
4030		const operand_is_undef = if (try f.air.value(pl_op.operand, pt)) \|v\| v.isUndef(zcu) else false;
	4029	const operand_is_undef = if (pl_op.operand.toInterned()) \|ip_index\| Value.fromInterned(ip_index).isUndef(zcu) else false;
4031	4030	if (!operand_is_undef) _ = try f.resolveInst(pl_op.operand);
4032	4031
4033	4032	try reap(f, inst, &.{pl_op.operand});
...	...	@@ -4204,7 +4203,8 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
4204	4203	const br = f.air.instructions.items(.data)[@intFromEnum(inst)].br;
4205	4204	const w = &f.code.writer;
4206	4205
4207		if (try f.air.value(br.operand, pt)) \|cond_val\| {
	4206	if (br.operand.toInterned()) \|cond_ip_index\| {
	4207	const cond_val: Value = .fromInterned(cond_ip_index);
4208	4208	// Comptime-known dispatch. Iterate the cases to find the correct
4209	4209	// one, and branch directly to the corresponding case.
4210	4210	const switch_br = f.air.unwrapSwitch(br.block_inst);
...	...	@@ -4539,12 +4539,12 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void
4539	4539	try f.writeCValue(w, cond_val, .other);
4540	4540	try w.writeAll(", ");
4541	4541	}
4542		const item_value = try f.air.value(item, pt);
	4542	const item_value: Value = .fromInterned(item.toInterned().?);
4543	4543	// If `item_value` is a pointer with a known integer address, print the address
4544	4544	// with no cast to avoid a warning.
4545	4545	write_val: {
4546	4546	if (cond_ty.zigTypeTag(zcu) == .pointer) {
4547		if (item_value.?.getUnsignedInt(zcu)) \|item_int\| {
	4547	if (item_value.getUnsignedInt(zcu)) \|item_int\| {
4548	4548	try w.print("{f}", .{try f.fmtIntLiteralDec(try pt.intValue(lowered_cond_ty, item_int))});
4549	4549	break :write_val;
4550	4550	}
...	...	@@ -4552,7 +4552,7 @@ fn airSwitchBr(f: *Function, inst: Air.Inst.Index, is_dispatch_loop: bool) !void
4552	4552	try f.renderType(w, .usize);
4553	4553	try w.writeByte(')');
4554	4554	}
4555		try f.dg.renderValue(w, (try f.air.value(item, pt)).?, .other);
	4555	try f.dg.renderValue(w, .fromInterned(item.toInterned().?), .other);
4556	4556	}
4557	4557	switch (cond_cint) {
4558	4558	.zig_u128, .zig_i128 => try w.writeByte(')'),
...	...	@@ -4710,7 +4710,7 @@ fn lowerSwitchCmp(
4710	4710	try f.writeCValue(w, cond_val, .other);
4711	4711	try w.writeAll(if (use_builtin) ", " else compareOperatorC(operator));
4712	4712	if (class == .big) try w.writeByte('&');
4713		try f.dg.renderValue(w, (try f.air.value(case_inst, pt)).?, .other);
	4713	try f.dg.renderValue(w, .fromInterned(case_inst.toInterned().?), .other);
4714	4714	if (use_builtin) {
4715	4715	try f.dg.renderBuiltinInfo(w, ty, if (class == .big) .bits else .none);
4716	4716	try w.writeByte(')');
...	...	@@ -6100,7 +6100,7 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
6100	6100	const value = try f.resolveInst(bin_op.rhs);
6101	6101	const elem_ty = f.typeOf(bin_op.rhs);
6102	6102	const elem_abi_size = elem_ty.abiSize(zcu);
6103		const val_is_undef = if (try f.air.value(bin_op.rhs, pt)) \|val\| val.isUndef(zcu) else false;
	6103	const val_is_undef = if (bin_op.rhs.toInterned()) \|ip_index\| Value.fromInterned(ip_index).isUndef(zcu) else false;
6104	6104	const w = &f.code.writer;
6105	6105
6106	6106	if (val_is_undef) {

src/codegen/llvm.zig+1302-9592

...	...	@@ -1,16 +1,24 @@
1	1	const builtin = @import("builtin");
2	2
	3	const FuncGen = @import("llvm/FuncGen.zig");
	4	const buildAllocaInner = FuncGen.buildAllocaInner;
	5	const isByRef = FuncGen.isByRef;
	6	const firstParamSRet = FuncGen.firstParamSRet;
	7	const lowerFnRetTy = FuncGen.lowerFnRetTy;
	8	const iterateParamTypes = FuncGen.iterateParamTypes;
	9	const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
	10	const aarch64_c_abi = @import("aarch64/abi.zig");
	11
3	12	const std = @import("std");
4	13	const Io = std.Io;
5	14	const assert = std.debug.assert;
6	15	const Allocator = std.mem.Allocator;
7	16	const log = std.log.scoped(.codegen);
8		const math = std.math;
9	17	const DW = std.dwarf;
10	18	const Builder = std.zig.llvm.Builder;
11	19
12	20	const build_options = @import("build_options");
13		const llvm = if (build_options.have_llvm)
	21	const bindings = if (build_options.have_llvm)
14	22	@import("llvm/bindings.zig")
15	23	else
16	24	@compileError("LLVM unavailable");
...	...	@@ -24,21 +32,9 @@ const Air = @import("../Air.zig");
24	32	const Value = @import("../Value.zig");
25	33	const Type = @import("../Type.zig");
26	34	const codegen = @import("../codegen.zig");
27		const x86_64_abi = @import("x86_64/abi.zig");
28		const wasm_c_abi = @import("wasm/abi.zig");
29		const aarch64_c_abi = @import("aarch64/abi.zig");
30		const arm_c_abi = @import("arm/abi.zig");
31		const riscv_c_abi = @import("riscv64/abi.zig");
32		const mips_c_abi = @import("mips/abi.zig");
33	35	const dev = @import("../dev.zig");
34	36
35	37	const target_util = @import("../target.zig");
36		const libcFloatPrefix = target_util.libcFloatPrefix;
37		const libcFloatSuffix = target_util.libcFloatSuffix;
38		const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
39		const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
40
41		const Error = error{ OutOfMemory, CodegenFail };
42	38
43	39	pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
44	40	return comptime &.initMany(&.{
...	...	@@ -493,7 +489,7 @@ pub fn dataLayout(target: *const std.Target) []const u8 {
493	489	};
494	490	}
495	491
496		// Avoid depending on `llvm.CodeModel` in the bitcode-only case.
	492	// Avoid depending on `bindings.CodeModel` in the bitcode-only case.
497	493	const CodeModel = enum {
498	494	default,
499	495	tiny,
...	...	@@ -553,20 +549,16 @@ pub const Object = struct {
553	549	/// type from the global error set.
554	550	debug_anyerror_fwd_ref: Builder.Metadata.Optional,
555	551
556		target: *const std.Target,
557		/// Ideally we would use `llvm_module.getNamedFunction` to go from *Decl to LLVM function,
558		/// but that has some downsides:
559		/// * we have to compute the fully qualified name every time we want to do the lookup
560		/// * for externally linked functions, the name is not fully qualified, but when
561		/// a Decl goes from exported to not exported and vice-versa, we would use the wrong
562		/// version of the name and incorrectly get function not found in the llvm module.
563		/// * it works for functions not all globals.
564		/// Therefore, this table keeps track of the mapping.
	552	zcu: *Zcu,
	553	/// Maps a `Nav` to the corresponding LLVM global.
565	554	nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index),
566		/// Same deal as `decl_map` but for anonymous declarations, which are always global constants.
567		uav_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index),
	555	/// Same as `nav_map` but for UAVs (which are always global constants).
	556	uav_map: std.AutoHashMapUnmanaged(struct {
	557	val: InternPool.Index,
	558	@"addrspace": std.builtin.AddressSpace,
	559	}, Builder.Variable.Index),
568	560	/// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction.
569		enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Global.Index),
	561	enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index),
570	562	/// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction.
571	563	named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index),
572	564	/// Maps Zig types to LLVM types. The table memory is backed by the GPA of
...	...	@@ -578,36 +570,25 @@ pub const Object = struct {
578	570	/// Note that the values are not added until `emit`, when all errors in
579	571	/// the compilation are known.
580	572	error_name_table: Builder.Variable.Index,
581
582		/// Memoizes a null `?usize` value.
583		null_opt_usize: Builder.Constant,
584
585		/// When an LLVM struct type is created, an entry is inserted into this
586		/// table for every zig source field of the struct that has a corresponding
587		/// LLVM struct field. comptime fields are not included. Zero-bit fields are
588		/// mapped to a field at the correct byte, which may be a padding field, or
589		/// are not mapped, in which case they are semantically at the end of the
590		/// struct.
591		/// The value is the LLVM struct field index.
592		/// This is denormalized data.
593		struct_field_map: std.AutoHashMapUnmanaged(ZigStructField, c_uint),
	573	/// Constant variable whose value is the number of errors in the Zcu.
	574	///
	575	/// Initially `.none`---populated lazily by `getErrorsLen`.
	576	///
	577	/// If this is not `.none`, the variable's initializer is set in `emit`.
	578	errors_len_variable: Builder.Variable.Index,
594	579
595	580	/// Values for `@llvm.used`.
596	581	used: std.ArrayList(Builder.Constant),
597	582
598		const ZigStructField = struct {
599		struct_ty: InternPool.Index,
600		field_index: u32,
601		};
602
603	583	pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
604	584
605		pub const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
	585	const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
606	586
607		pub fn create(arena: Allocator, comp: *Compilation) !Ptr {
	587	pub fn create(arena: Allocator, zcu: *Zcu) !Ptr {
608	588	dev.check(.llvm_backend);
	589	const comp = zcu.comp;
609	590	const gpa = comp.gpa;
610		const target = &comp.root_mod.resolved_target.result;
	591	const target = zcu.getTarget();
611	592	const llvm_target_triple = try targetTriple(arena, target);
612	593
613	594	var builder = try Builder.init(.{
...	...	@@ -635,10 +616,7 @@ pub const Object = struct {
635	616	// way already, but here we throw all that sweet information
636	617	// into the garbage can by converting into absolute paths. What
637	618	// a terrible tragedy.
638		const compile_unit_dir = blk: {
639		const zcu = comp.zcu orelse break :blk comp.dirs.cwd;
640		break :blk try zcu.main_mod.root.toAbsolute(comp.dirs, arena);
641		};
	619	const compile_unit_dir = try zcu.main_mod.root.toAbsolute(comp.dirs, arena);
642	620
643	621	const debug_file = try builder.debugFile(
644	622	try builder.metadataString(comp.root_name),
...	...	@@ -684,15 +662,14 @@ pub const Object = struct {
684	662	.debug_file_map = .empty,
685	663	.debug_types = .empty,
686	664	.debug_anyerror_fwd_ref = .none,
687		.target = target,
	665	.zcu = zcu,
688	666	.nav_map = .empty,
689	667	.uav_map = .empty,
690	668	.enum_tag_name_map = .empty,
691	669	.named_enum_map = .empty,
692	670	.type_map = .empty,
693	671	.error_name_table = .none,
694		.null_opt_usize = .no_init,
695		.struct_field_map = .empty,
	672	.errors_len_variable = .none,
696	673	.used = .empty,
697	674	};
698	675	return obj;
...	...	@@ -712,15 +689,14 @@ pub const Object = struct {
712	689	self.named_enum_map.deinit(gpa);
713	690	self.type_map.deinit(gpa);
714	691	self.builder.deinit();
715		self.struct_field_map.deinit(gpa);
716	692	self.* = undefined;
717	693	}
718	694
719		fn genErrorNameTable(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {
	695	fn genErrorNameTable(o: *Object) Allocator.Error!void {
720	696	// If o.error_name_table is null, then it was not referenced by any instructions.
721	697	if (o.error_name_table == .none) return;
722	698
723		const zcu = pt.zcu;
	699	const zcu = o.zcu;
724	700	const ip = &zcu.intern_pool;
725	701
726	702	const error_name_list = ip.global_error_set.getNamesFromMainThread();
...	...	@@ -729,21 +705,21 @@ pub const Object = struct {
729	705
730	706	// TODO: Address space
731	707	const slice_ty = Type.slice_const_u8_sentinel_0;
732		const llvm_usize_ty = try o.lowerType(pt, Type.usize);
733		const llvm_slice_ty = try o.lowerType(pt, slice_ty);
	708	const llvm_usize_ty = try o.lowerType(.usize);
	709	const llvm_slice_ty = try o.lowerType(slice_ty);
734	710	const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
735	711
736	712	llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
737	713	for (llvm_errors[1..], error_name_list) \|*llvm_error, name\| {
738	714	const name_string = try o.builder.stringNull(name.toSlice(ip));
739	715	const name_init = try o.builder.stringConst(name_string);
740		const name_variable_index =
741		try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
	716	const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
742	717	try name_variable_index.setInitializer(name_init, &o.builder);
743		name_variable_index.setLinkage(.private, &o.builder);
744	718	name_variable_index.setMutability(.constant, &o.builder);
745		name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
746		name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
	719	name_variable_index.setAlignment(comptime .fromByteUnits(1), &o.builder);
	720	const global_index = name_variable_index.ptrConst(&o.builder).global;
	721	global_index.setLinkage(.private, &o.builder);
	722	global_index.setUnnamedAddr(.unnamed_addr, &o.builder);
747	723
748	724	llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{
749	725	name_variable_index.toConst(&o.builder),
...	...	@@ -751,52 +727,17 @@ pub const Object = struct {
751	727	});
752	728	}
753	729
754		const table_variable_index = try o.builder.addVariable(.empty, llvm_table_ty, .default);
755		try table_variable_index.setInitializer(
	730	try o.error_name_table.setInitializer(
756	731	try o.builder.arrayConst(llvm_table_ty, llvm_errors),
757	732	&o.builder,
758	733	);
759		table_variable_index.setLinkage(.private, &o.builder);
760		table_variable_index.setMutability(.constant, &o.builder);
761		table_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
762		table_variable_index.setAlignment(
763		slice_ty.abiAlignment(zcu).toLlvm(),
764		&o.builder,
765		);
766
767		try o.error_name_table.setInitializer(table_variable_index.toConst(&o.builder), &o.builder);
768		}
769
770		fn genCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !void {
771		// If there is no such function in the module, it means the source code does not need it.
772		const name = o.builder.strtabStringIfExists(lt_errors_fn_name) orelse return;
773		const llvm_fn = o.builder.getGlobal(name) orelse return;
774		const errors_len = pt.zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
775
776		var wip = try Builder.WipFunction.init(&o.builder, .{
777		.function = llvm_fn.ptrConst(&o.builder).kind.function,
778		.strip = true,
779		});
780		defer wip.deinit();
781		wip.cursor = .{ .block = try wip.block(0, "Entry") };
782
783		// Example source of the following LLVM IR:
784		// fn __zig_lt_errors_len(index: u16) bool {
785		// return index <= total_errors_len;
786		// }
787
788		const lhs = wip.arg(0);
789		const rhs = try o.builder.intValue(try o.errorIntType(pt), errors_len);
790		const is_lt = try wip.icmp(.ule, lhs, rhs, "");
791		_ = try wip.ret(is_lt);
792		try wip.finish();
793	734	}
794	735
795		fn genModuleLevelAssembly(object: *Object, pt: Zcu.PerThread) Allocator.Error!void {
	736	fn genModuleLevelAssembly(object: *Object) Allocator.Error!void {
796	737	const b = &object.builder;
797	738	const gpa = b.gpa;
798	739	b.module_asm.clearRetainingCapacity();
799		for (pt.zcu.global_assembly.values()) \|assembly\| {
	740	for (object.zcu.global_assembly.values()) \|assembly\| {
800	741	try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1);
801	742	b.module_asm.appendSliceAssumeCapacity(assembly);
802	743	b.module_asm.appendAssumeCapacity('\n');
...	...	@@ -823,15 +764,19 @@ pub const Object = struct {
823	764	};
824	765
825	766	pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ LinkFailure, OutOfMemory }!void {
826		const zcu = pt.zcu;
	767	const zcu = o.zcu;
827	768	const comp = zcu.comp;
828	769	const io = comp.io;
829	770	const diags = &comp.link_diags;
830	771
831	772	{
832		try o.genErrorNameTable(pt);
833		try o.genCmpLtErrorsLenFunction(pt);
834		try o.genModuleLevelAssembly(pt);
	773	if (o.errors_len_variable != .none) {
	774	const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
	775	const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);
	776	try o.errors_len_variable.setInitializer(init_val, &o.builder);
	777	}
	778	try o.genErrorNameTable();
	779	try o.genModuleLevelAssembly();
835	780
836	781	if (o.used.items.len > 0) {
837	782	const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr);
...	...	@@ -841,14 +786,14 @@ pub const Object = struct {
841	786	array_llvm_ty,
842	787	.default,
843	788	);
844		compiler_used_variable.setLinkage(.appending, &o.builder);
845		compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder);
846	789	try compiler_used_variable.setInitializer(init_val, &o.builder);
	790	compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder);
	791	compiler_used_variable.ptrConst(&o.builder).global.setLinkage(.appending, &o.builder);
847	792	}
848	793
849	794	if (!o.builder.strip) {
850	795	if (o.debug_anyerror_fwd_ref.unwrap()) \|fwd_ref\| {
851		const debug_anyerror_type = try o.lowerDebugAnyerrorType(pt);
	796	const debug_anyerror_type = try o.lowerDebugAnyerrorType();
852	797	o.builder.resolveDebugForwardReference(fwd_ref, debug_anyerror_type);
853	798	}
854	799
...	...	@@ -995,7 +940,6 @@ pub const Object = struct {
995	940	.version = build_options.semver,
996	941	});
997	942	defer o.gpa.free(bitcode);
998		o.builder.clearAndFree();
999	943
1000	944	if (options.pre_bc_path) \|path\| {
1001	945	var file = Io.Dir.cwd().createFile(io, path, .{}) catch \|err\|
...	...	@@ -1026,20 +970,20 @@ pub const Object = struct {
1026	970
1027	971	initializeLLVMTarget(comp.root_mod.resolved_target.result.cpu.arch);
1028	972
1029		const context: *llvm.Context = llvm.Context.create();
	973	const context: *bindings.Context = .create();
1030	974	errdefer context.dispose();
1031	975
1032		const bitcode_memory_buffer = llvm.MemoryBuffer.createMemoryBufferWithMemoryRange(
	976	const bitcode_memory_buffer = bindings.MemoryBuffer.createMemoryBufferWithMemoryRange(
1033	977	@ptrCast(bitcode.ptr),
1034	978	bitcode.len * 4,
1035	979	"BitcodeBuffer",
1036		llvm.Bool.False,
	980	bindings.Bool.False,
1037	981	);
1038	982	defer bitcode_memory_buffer.dispose();
1039	983
1040	984	context.enableBrokenDebugInfoCheck();
1041	985
1042		var module: *llvm.Module = undefined;
	986	var module: *bindings.Module = undefined;
1043	987	if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) {
1044	988	return diags.fail("Failed to parse bitcode", .{});
1045	989	}
...	...	@@ -1047,28 +991,28 @@ pub const Object = struct {
1047	991	};
1048	992	defer context.dispose();
1049	993
1050		var target: *llvm.Target = undefined;
	994	var target: *bindings.Target = undefined;
1051	995	var error_message: [*:0]const u8 = undefined;
1052		if (llvm.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {
1053		defer llvm.disposeMessage(error_message);
	996	if (bindings.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {
	997	defer bindings.disposeMessage(error_message);
1054	998	return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message });
1055	999	}
1056	1000
1057	1001	const optimize_mode = comp.root_mod.optimize_mode;
1058	1002
1059		const opt_level: llvm.CodeGenOptLevel = if (optimize_mode == .Debug)
	1003	const opt_level: bindings.CodeGenOptLevel = if (optimize_mode == .Debug)
1060	1004	.None
1061	1005	else
1062	1006	.Aggressive;
1063	1007
1064		const reloc_mode: llvm.RelocMode = if (comp.root_mod.pic)
	1008	const reloc_mode: bindings.RelocMode = if (comp.root_mod.pic)
1065	1009	.PIC
1066	1010	else if (comp.config.link_mode == .dynamic)
1067		llvm.RelocMode.DynamicNoPIC
	1011	bindings.RelocMode.DynamicNoPIC
1068	1012	else
1069	1013	.Static;
1070	1014
1071		const code_model: llvm.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
	1015	const code_model: bindings.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
1072	1016	.default => .Default,
1073	1017	.tiny => .Tiny,
1074	1018	.small => .Small,
...	...	@@ -1077,12 +1021,12 @@ pub const Object = struct {
1077	1021	.large => .Large,
1078	1022	};
1079	1023
1080		const float_abi: llvm.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)
	1024	const float_abi: bindings.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)
1081	1025	.Hard
1082	1026	else
1083	1027	.Soft;
1084	1028
1085		var target_machine = llvm.TargetMachine.create(
	1029	var target_machine = bindings.TargetMachine.create(
1086	1030	target,
1087	1031	target_triple_sentinel,
1088	1032	if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) \|s\| s.ptr else null,
...	...	@@ -1105,7 +1049,7 @@ pub const Object = struct {
1105	1049	// Unfortunately, LLVM shits the bed when we ask for both binary and assembly.
1106	1050	// So we call the entire pipeline multiple times if this is requested.
1107	1051	// var error_message: [*:0]const u8 = undefined;
1108		var lowered_options: llvm.TargetMachine.EmitOptions = .{
	1052	var lowered_options: bindings.TargetMachine.EmitOptions = .{
1109	1053	.is_debug = options.is_debug,
1110	1054	.is_small = options.is_small,
1111	1055	.time_report_out = null, // set below to make sure it's only set for a single `emitToFile`
...	...	@@ -1154,7 +1098,7 @@ pub const Object = struct {
1154	1098	};
1155	1099	if (options.asm_path != null and options.bin_path != null) {
1156	1100	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1157		defer llvm.disposeMessage(error_message);
	1101	defer bindings.disposeMessage(error_message);
1158	1102	return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{
1159	1103	emit_bin_msg, post_llvm_ir_msg, error_message,
1160	1104	});
...	...	@@ -1170,7 +1114,7 @@ pub const Object = struct {
1170	1114
1171	1115	lowered_options.asm_filename = if (options.asm_path) \|x\| x.ptr else null;
1172	1116	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
1173		defer llvm.disposeMessage(error_message);
	1117	defer bindings.disposeMessage(error_message);
1174	1118	return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{
1175	1119	emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message,
1176	1120	});
...	...	@@ -1189,9 +1133,10 @@ pub const Object = struct {
1189	1133	func_index: InternPool.Index,
1190	1134	air: *const Air,
1191	1135	liveness: *const ?Air.Liveness,
1192		) !void {
1193		const zcu = pt.zcu;
	1136	) Zcu.CodegenFailError!void {
	1137	const zcu = o.zcu;
1194	1138	const comp = zcu.comp;
	1139	const gpa = comp.gpa;
1195	1140	const ip = &zcu.intern_pool;
1196	1141	const func = zcu.funcInfo(func_index);
1197	1142	const nav = ip.getNav(func.owner_nav);
...	...	@@ -1201,16 +1146,42 @@ pub const Object = struct {
1201	1146	const fn_info = zcu.typeToFunc(fn_ty).?;
1202	1147	const target = &owner_mod.resolved_target.result;
1203	1148
1204		var ng: NavGen = .{
1205		.object = o,
1206		.nav_index = func.owner_nav,
1207		.pt = pt,
1208		.err_msg = null,
1209		};
	1149	const gop = try o.nav_map.getOrPut(gpa, func.owner_nav);
	1150	if (!gop.found_existing) {
	1151	errdefer assert(o.nav_map.remove(func.owner_nav));
	1152	// First time lowering this NAV! Create a fresh global.
	1153	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
	1154	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
	1155	.type = .void, // placeholder; populated below
	1156	.kind = .{ .alias = .none }, // placeholder; populated below
	1157	});
	1158	}
	1159	const llvm_global = gop.value_ptr.*;
1210	1160
1211		const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
	1161	const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
	1162	.function => \|function\| function, // re-use existing `Builder.Function`
	1163	.replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder),
	1164	};
	1165	{
	1166	const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
	1167	global.type = try o.lowerType(fn_ty);
	1168	global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
	1169	global.linkage = if (o.builder.strip) .private else .internal;
	1170	global.visibility = .default;
	1171	global.dll_storage_class = .default;
	1172	global.unnamed_addr = .unnamed_addr;
	1173	}
	1174	llvm_function.setAlignment(switch (nav.resolved.?.@"align") {
	1175	.none => fn_ty.abiAlignment(zcu).toLlvm(),
	1176	else => \|a\| a.toLlvm(),
	1177	}, &o.builder);
	1178	llvm_function.setSection(s: {
	1179	const section = nav.resolved.?.@"linksection".toSlice(ip) orelse break :s .none;
	1180	break :s try o.builder.string(section);
	1181	}, &o.builder);
	1182	try o.addLlvmFunctionAttributes(pt, func.owner_nav, llvm_function);
1212	1183
1213		var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
	1184	var attributes = try llvm_function.ptrConst(&o.builder).attributes.toWip(&o.builder);
1214	1185	defer attributes.deinit(&o.builder);
1215	1186
1216	1187	const func_analysis = func.analysisUnordered(ip);
...	...	@@ -1280,49 +1251,41 @@ pub const Object = struct {
1280	1251	} }, &o.builder);
1281	1252	}
1282	1253
1283		if (nav.resolved.?.@"linksection".toSlice(ip)) \|section\|
1284		function_index.setSection(try o.builder.string(section), &o.builder);
1285
1286	1254	var deinit_wip = true;
1287	1255	var wip = try Builder.WipFunction.init(&o.builder, .{
1288		.function = function_index,
	1256	.function = llvm_function,
1289	1257	.strip = owner_mod.strip,
1290	1258	});
1291	1259	defer if (deinit_wip) wip.deinit();
1292	1260	wip.cursor = .{ .block = try wip.block(0, "Entry") };
1293	1261
1294		var llvm_arg_i: u32 = 0;
1295
1296		// This gets the LLVM values from the function and stores them in `ng.args`.
1297		const sret = firstParamSRet(fn_info, zcu, target);
1298		const ret_ptr: Builder.Value = if (sret) param: {
1299		const param = wip.arg(llvm_arg_i);
1300		llvm_arg_i += 1;
1301		break :param param;
1302		} else .none;
1303
1304	1262	if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) \|s\| switch (s) {
1305	1263	.signed => try attributes.addRetAttr(.signext, &o.builder),
1306	1264	.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
1307	1265	};
1308	1266
1309		const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
1310
1311		const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
1312		const param = wip.arg(llvm_arg_i);
1313		llvm_arg_i += 1;
1314		break :param param;
1315		} else .none;
1316
1317	1267	// This is the list of args we will use that correspond directly to the AIR arg
1318	1268	// instructions. Depending on the calling convention, this list is not necessarily
1319	1269	// a bijection with the actual LLVM parameters of the function.
1320		const gpa = o.gpa;
1321	1270	var args: std.ArrayList(Builder.Value) = .empty;
1322	1271	defer args.deinit(gpa);
1323	1272
1324		{
1325		var it = iterateParamTypes(o, pt, fn_info);
	1273	const ret_ptr: Builder.Value, const err_ret_trace: Builder.Value = implicit_args: {
	1274	var it = iterateParamTypes(o, fn_info);
	1275
	1276	const ret_ptr: Builder.Value = if (firstParamSRet(fn_info, zcu, target)) param: {
	1277	const param = wip.arg(it.llvm_index);
	1278	it.llvm_index += 1;
	1279	break :param param;
	1280	} else .none;
	1281
	1282	const err_return_tracing = fn_info.cc == .auto and comp.config.any_error_tracing;
	1283	const err_ret_trace: Builder.Value = if (err_return_tracing) param: {
	1284	const param = wip.arg(it.llvm_index);
	1285	it.llvm_index += 1;
	1286	break :param param;
	1287	} else .none;
	1288
1326	1289	while (try it.next()) \|lowering\| {
1327	1290	try args.ensureUnusedCapacity(gpa, 1);
1328	1291
...	...	@@ -1332,7 +1295,7 @@ pub const Object = struct {
1332	1295	assert(!it.byval_attr);
1333	1296	const param_index = it.zig_index - 1;
1334	1297	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
1335		const param = wip.arg(llvm_arg_i);
	1298	const param = wip.arg(it.llvm_index - 1);
1336	1299
1337	1300	if (isByRef(param_ty, zcu)) {
1338	1301	const alignment = param_ty.abiAlignment(zcu).toLlvm();
...	...	@@ -1342,149 +1305,119 @@ pub const Object = struct {
1342	1305	args.appendAssumeCapacity(arg_ptr);
1343	1306	} else {
1344	1307	args.appendAssumeCapacity(param);
1345
1346		try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, llvm_arg_i);
1347	1308	}
1348		llvm_arg_i += 1;
1349	1309	},
1350	1310	.byref => {
1351		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1352		const param_llvm_ty = try o.lowerType(pt, param_ty);
1353		const param = wip.arg(llvm_arg_i);
1354		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1355
1356		try o.addByRefParamAttrs(&attributes, llvm_arg_i, alignment, it.byval_attr, param_llvm_ty);
1357		llvm_arg_i += 1;
	1311	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1312	const param = wip.arg(it.llvm_index - 1);
1358	1313
1359	1314	if (isByRef(param_ty, zcu)) {
1360	1315	args.appendAssumeCapacity(param);
1361	1316	} else {
	1317	const param_llvm_ty = try o.lowerType(param_ty);
	1318	const alignment = param_ty.abiAlignment(zcu).toLlvm();
1362	1319	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1363	1320	}
1364	1321	},
1365	1322	.byref_mut => {
1366		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1367		const param_llvm_ty = try o.lowerType(pt, param_ty);
1368		const param = wip.arg(llvm_arg_i);
1369		const alignment = param_ty.abiAlignment(zcu).toLlvm();
1370
1371		try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder);
1372		llvm_arg_i += 1;
	1323	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1324	const param = wip.arg(it.llvm_index - 1);
1373	1325
1374	1326	if (isByRef(param_ty, zcu)) {
1375	1327	args.appendAssumeCapacity(param);
1376	1328	} else {
	1329	const param_llvm_ty = try o.lowerType(param_ty);
	1330	const alignment = param_ty.abiAlignment(zcu).toLlvm();
1377	1331	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, param, alignment, ""));
1378	1332	}
1379	1333	},
1380	1334	.abi_sized_int => {
1381	1335	assert(!it.byval_attr);
1382		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1383		const param = wip.arg(llvm_arg_i);
1384		llvm_arg_i += 1;
	1336	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1337	const param = wip.arg(it.llvm_index - 1);
1385	1338
1386		const param_llvm_ty = try o.lowerType(pt, param_ty);
	1339	const param_llvm_ty = try o.lowerType(param_ty);
1387	1340	const alignment = param_ty.abiAlignment(zcu).toLlvm();
1388	1341	const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1389	1342	_ = try wip.store(.normal, param, arg_ptr, alignment);
1390	1343
1391		args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1392		arg_ptr
1393		else
1394		try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1344	if (isByRef(param_ty, zcu)) {
	1345	args.appendAssumeCapacity(arg_ptr);
	1346	} else {
	1347	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1348	}
1395	1349	},
1396	1350	.slice => {
1397	1351	assert(!it.byval_attr);
1398		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1399		const ptr_info = param_ty.ptrInfo(zcu);
1400
1401		if (math.cast(u5, it.zig_index - 1)) \|i\| {
1402		if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
1403		try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
1404		}
1405		}
1406		if (param_ty.zigTypeTag(zcu) != .optional and
1407		!ptr_info.flags.is_allowzero and
1408		ptr_info.flags.address_space == .generic)
1409		{
1410		try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
1411		}
1412		if (ptr_info.flags.is_const) {
1413		try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
1414		}
1415		const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
1416		else => \|a\| .wrap(a.toLlvm()),
1417		.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
1418		};
1419		try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
1420		const ptr_param = wip.arg(llvm_arg_i);
1421		llvm_arg_i += 1;
1422		const len_param = wip.arg(llvm_arg_i);
1423		llvm_arg_i += 1;
1424
1425		const slice_llvm_ty = try o.lowerType(pt, param_ty);
1426		args.appendAssumeCapacity(
1427		try wip.buildAggregate(slice_llvm_ty, &.{ ptr_param, len_param }, ""),
	1352	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1353	assert(!isByRef(param_ty, zcu));
	1354	const slice_val = try wip.buildAggregate(
	1355	try o.lowerType(param_ty),
	1356	&.{ wip.arg(it.llvm_index - 2), wip.arg(it.llvm_index - 1) },
	1357	"",
1428	1358	);
	1359	args.appendAssumeCapacity(slice_val);
1429	1360	},
1430	1361	.multiple_llvm_types => {
1431	1362	assert(!it.byval_attr);
1432	1363	const field_types = it.types_buffer[0..it.types_len];
1433		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1434		const param_llvm_ty = try o.lowerType(pt, param_ty);
	1364	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1365	const param_llvm_ty = try o.lowerType(param_ty);
1435	1366	const param_alignment = param_ty.abiAlignment(zcu).toLlvm();
1436	1367	const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, param_alignment, target);
1437	1368	const llvm_ty = try o.builder.structType(.normal, field_types);
1438		for (0..field_types.len) \|field_i\| {
1439		const param = wip.arg(llvm_arg_i);
1440		llvm_arg_i += 1;
	1369	const llvm_args_start = it.llvm_index - field_types.len;
	1370	for (0..field_types.len, llvm_args_start..) \|field_i, llvm_arg_index\| {
	1371	const param = wip.arg(@intCast(llvm_arg_index));
1441	1372	const field_ptr = try wip.gepStruct(llvm_ty, arg_ptr, field_i, "");
1442		const alignment = Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
	1373	const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8));
1443	1374	_ = try wip.store(.normal, param, field_ptr, alignment);
1444	1375	}
1445	1376
1446		const is_by_ref = isByRef(param_ty, zcu);
1447		args.appendAssumeCapacity(if (is_by_ref)
1448		arg_ptr
1449		else
1450		try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, ""));
	1377	if (isByRef(param_ty, zcu)) {
	1378	args.appendAssumeCapacity(arg_ptr);
	1379	} else {
	1380	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, param_alignment, ""));
	1381	}
1451	1382	},
1452	1383	.float_array => {
1453		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1454		const param_llvm_ty = try o.lowerType(pt, param_ty);
1455		const param = wip.arg(llvm_arg_i);
1456		llvm_arg_i += 1;
	1384	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1385	const param_llvm_ty = try o.lowerType(param_ty);
	1386	const param = wip.arg(it.llvm_index - 1);
1457	1387
1458	1388	const alignment = param_ty.abiAlignment(zcu).toLlvm();
1459	1389	const arg_ptr = try buildAllocaInner(&wip, param_llvm_ty, alignment, target);
1460	1390	_ = try wip.store(.normal, param, arg_ptr, alignment);
1461	1391
1462		args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1463		arg_ptr
1464		else
1465		try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1392	if (isByRef(param_ty, zcu)) {
	1393	args.appendAssumeCapacity(arg_ptr);
	1394	} else {
	1395	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1396	}
1466	1397	},
1467	1398	.i32_array, .i64_array => {
1468		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
1469		const param_llvm_ty = try o.lowerType(pt, param_ty);
1470		const param = wip.arg(llvm_arg_i);
1471		llvm_arg_i += 1;
	1399	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	1400	const param_llvm_ty = try o.lowerType(param_ty);
	1401	const param = wip.arg(it.llvm_index - 1);
1472	1402
1473	1403	const alignment = param_ty.abiAlignment(zcu).toLlvm();
1474	1404	const arg_ptr = try buildAllocaInner(&wip, param.typeOfWip(&wip), alignment, target);
1475	1405	_ = try wip.store(.normal, param, arg_ptr, alignment);
1476	1406
1477		args.appendAssumeCapacity(if (isByRef(param_ty, zcu))
1478		arg_ptr
1479		else
1480		try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1407	if (isByRef(param_ty, zcu)) {
	1408	args.appendAssumeCapacity(arg_ptr);
	1409	} else {
	1410	args.appendAssumeCapacity(try wip.load(.normal, param_llvm_ty, arg_ptr, alignment, ""));
	1411	}
1481	1412	},
1482	1413	}
1483	1414	}
1484		}
	1415
	1416	break :implicit_args .{ ret_ptr, err_ret_trace };
	1417	};
1485	1418
1486	1419	const file, const subprogram = if (!wip.strip) debug_info: {
1487		const file = try o.getDebugFile(pt, file_scope);
	1420	const file = try o.getDebugFile(file_scope);
1488	1421
1489	1422	const line_number = zcu.navSrcLine(func.owner_nav) + 1;
1490	1423	const is_internal_linkage = ip.indexToKey(nav.resolved.?.value) != .@"extern";
...	...	@@ -1493,7 +1426,7 @@ pub const Object = struct {
1493	1426	const subprogram = try o.builder.debugSubprogram(
1494	1427	file,
1495	1428	try o.builder.metadataString(nav.name.toSlice(ip)),
1496		try o.builder.metadataStringFromStrtabString(function_index.name(&o.builder)),
	1429	try o.builder.metadataString(nav.fqn.toSlice(ip)),
1497	1430	line_number,
1498	1431	line_number + func.lbrace_line,
1499	1432	debug_decl_type,
...	...	@@ -1510,7 +1443,7 @@ pub const Object = struct {
1510	1443	},
1511	1444	o.debug_compile_unit.unwrap().?,
1512	1445	);
1513		function_index.setSubprogram(subprogram, &o.builder);
	1446	llvm_function.setSubprogram(subprogram, &o.builder);
1514	1447	break :debug_info .{ file, subprogram };
1515	1448	} else .{undefined} ** 2;
1516	1449
...	...	@@ -1527,7 +1460,7 @@ pub const Object = struct {
1527	1460	const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
1528	1461	const counters_variable = try o.builder.addVariable(anon_name, .void, .default);
1529	1462	try o.used.append(gpa, counters_variable.toConst(&o.builder));
1530		counters_variable.setLinkage(.private, &o.builder);
	1463	counters_variable.ptrConst(&o.builder).global.setLinkage(.private, &o.builder);
1531	1464	counters_variable.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
1532	1465
1533	1466	if (target.ofmt == .macho) {
...	...	@@ -1543,10 +1476,12 @@ pub const Object = struct {
1543	1476	};
1544	1477
1545	1478	var fg: FuncGen = .{
	1479	.object = o,
	1480	.nav_index = func.owner_nav,
	1481	.pt = pt,
1546	1482	.gpa = gpa,
1547	1483	.air = air.*,
1548	1484	.liveness = liveness.*.?,
1549		.ng = &ng,
1550	1485	.wip = wip,
1551	1486	.is_naked = fn_info.cc == .naked,
1552	1487	.fuzz = fuzz,
...	...	@@ -1561,22 +1496,18 @@ pub const Object = struct {
1561	1496	.sync_scope = if (owner_mod.single_threaded) .singlethread else .system,
1562	1497	.file = file,
1563	1498	.scope = subprogram,
	1499	.inlined_at = .none,
1564	1500	.base_line = zcu.navSrcLine(func.owner_nav),
1565	1501	.prev_dbg_line = 0,
1566	1502	.prev_dbg_column = 0,
1567	1503	.err_ret_trace = err_ret_trace,
1568	1504	.disable_intrinsics = disable_intrinsics,
	1505	.allowzero_access = false,
1569	1506	};
1570	1507	defer fg.deinit();
1571	1508	deinit_wip = false;
1572	1509
1573		fg.genBody(air.getMainBody(), .poi) catch \|err\| switch (err) {
1574		error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) {
1575		error.CodegenFail => return,
1576		error.OutOfMemory => \|e\| return e,
1577		},
1578		else => \|e\| return e,
1579		};
	1510	try fg.genBody(air.getMainBody(), .poi);
1580	1511
1581	1512	// If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
1582	1513	// function as considering null pointers valid so that LLVM's optimizers don't remove these
...	...	@@ -1587,7 +1518,7 @@ pub const Object = struct {
1587	1518	_ = try attributes.removeFnAttr(.null_pointer_is_valid);
1588	1519	}
1589	1520
1590		function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
	1521	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
1591	1522
1592	1523	if (fg.fuzz) \|*f\| {
1593	1524	{
...	...	@@ -1602,31 +1533,162 @@ pub const Object = struct {
1602	1533	// Due to error "members of llvm.compiler.used must be named", this global needs a name.
1603	1534	const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
1604	1535	const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default);
1605		try o.used.append(gpa, pcs_variable.toConst(&o.builder));
1606		pcs_variable.setLinkage(.private, &o.builder);
	1536	try pcs_variable.setInitializer(init_val, &o.builder);
1607	1537	pcs_variable.setMutability(.constant, &o.builder);
	1538	pcs_variable.setSection(switch (target.ofmt) {
	1539	.macho => try o.builder.string("__DATA,__sancov_pcs1"),
	1540	else => try o.builder.string("__sancov_pcs1"),
	1541	}, &o.builder);
1608	1542	pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder);
1609		if (target.ofmt == .macho) {
1610		pcs_variable.setSection(try o.builder.string("__DATA,__sancov_pcs1"), &o.builder);
1611		} else {
1612		pcs_variable.setSection(try o.builder.string("__sancov_pcs1"), &o.builder);
1613		}
1614		try pcs_variable.setInitializer(init_val, &o.builder);
	1543	const pcs_global = pcs_variable.ptrConst(&o.builder).global;
	1544	pcs_global.setLinkage(.private, &o.builder);
	1545	try o.used.append(gpa, pcs_global.toConst());
1615	1546	}
1616	1547
1617	1548	try fg.wip.finish();
1618	1549	try o.flushTypePool(pt);
1619	1550	}
1620	1551
1621		pub fn updateNav(self: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {
1622		var ng: NavGen = .{
1623		.object = self,
1624		.nav_index = nav_index,
1625		.pt = pt,
1626		.err_msg = null,
	1552	pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) !void {
	1553	const zcu = o.zcu;
	1554	const ip = &zcu.intern_pool;
	1555	const comp = zcu.comp;
	1556	const gpa = comp.gpa;
	1557
	1558	const nav = ip.getNav(nav_id);
	1559	const resolved = nav.resolved.?;
	1560
	1561	const opt_extern: ?InternPool.Key.Extern = switch (ip.indexToKey(resolved.value)) {
	1562	.@"extern" => \|@"extern"\| @"extern",
	1563	else => null,
	1564	};
	1565	const nav_ty: Type = .fromInterned(resolved.type);
	1566	const llvm_ty: Builder.Type = if (opt_extern != null) ty: {
	1567	// We must lower this declaration no matter what. If it has a type we can't actually
	1568	// represent (because it doesn't have runtime bits), we instead lower as the zero-size
	1569	// type `[0 x i8]`. I don't think the type on an extern declaration actually does much
	1570	// anyway.
	1571	if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);
	1572	break :ty try o.builder.arrayType(0, .i8);
	1573	} else if (nav_ty.hasRuntimeBits(zcu)) ty: {
	1574	break :ty try o.lowerType(nav_ty);
	1575	} else {
	1576	// This is a non-extern zero-bit `Nav`---we're not interested in it.
	1577	// TODO: we might need to rethink this a little under incremental compilation. If a
	1578	// declaration becomes zero-bit, we can't just leave its old value there, because it
	1579	// might now be ill-formed.
	1580	return;
	1581	};
	1582
	1583	const gop = try o.nav_map.getOrPut(gpa, nav_id);
	1584	if (!gop.found_existing) {
	1585	errdefer assert(o.nav_map.remove(nav_id));
	1586	// First time lowering this NAV! Create a fresh global.
	1587	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
	1588	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
	1589	.type = .void, // placeholder; populated below
	1590	.kind = .{ .alias = .none }, // placeholder; populated below
	1591	});
	1592	}
	1593	const llvm_global = gop.value_ptr.*;
	1594
	1595	llvm_global.ptr(&o.builder).type = llvm_ty;
	1596	llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(resolved.@"addrspace", zcu.getTarget());
	1597
	1598	if (opt_extern) \|@"extern"\| {
	1599	const name = name: {
	1600	const name_slice = nav.name.toSlice(ip);
	1601	if (zcu.getTarget().cpu.arch.isWasm() and nav_ty.zigTypeTag(zcu) == .@"fn") {
	1602	if (@"extern".lib_name.toSlice(ip)) \|lib_name_slice\| {
	1603	if (!std.mem.eql(u8, lib_name_slice, "c")) {
	1604	break :name try o.builder.strtabStringFmt("{s}\|{s}", .{ name_slice, lib_name_slice });
	1605	}
	1606	}
	1607	}
	1608	break :name try o.builder.strtabString(name_slice);
	1609	};
	1610	if (o.builder.getGlobal(name)) \|other_global\| {
	1611	if (other_global != llvm_global) {
	1612	// Another global already has this name; just use it in place of this global.
	1613	try llvm_global.replace(other_global, &o.builder);
	1614	return;
	1615	}
	1616	}
	1617	try llvm_global.rename(name, &o.builder);
	1618	llvm_global.ptr(&o.builder).unnamed_addr = .default;
	1619	llvm_global.ptr(&o.builder).dll_storage_class = switch (@"extern".is_dll_import) {
	1620	true => .dllimport,
	1621	false => .default,
	1622	};
	1623	llvm_global.ptr(&o.builder).linkage = switch (@"extern".linkage) {
	1624	.internal => if (o.builder.strip) .private else .internal,
	1625	.strong => .external,
	1626	.weak => .extern_weak,
	1627	.link_once => unreachable,
	1628	};
	1629	llvm_global.ptr(&o.builder).visibility = .fromSymbolVisibility(@"extern".visibility);
	1630	} else {
	1631	llvm_global.ptr(&o.builder).linkage = if (o.builder.strip) .private else .internal;
	1632	llvm_global.ptr(&o.builder).visibility = .default;
	1633	llvm_global.ptr(&o.builder).dll_storage_class = .default;
	1634	llvm_global.ptr(&o.builder).unnamed_addr = .unnamed_addr;
	1635	}
	1636
	1637	const llvm_align = switch (resolved.@"align") {
	1638	.none => nav_ty.abiAlignment(zcu).toLlvm(),
	1639	else => \|a\| a.toLlvm(),
1627	1640	};
1628		try ng.genDecl();
1629		try self.flushTypePool(pt);
	1641	const llvm_section: Builder.String = if (resolved.@"linksection".toSlice(ip)) \|section\| s: {
	1642	break :s try o.builder.string(section);
	1643	} else .none;
	1644
	1645	// Actual function bodies with AIR go through `updateFunc` instead, so the only functions we
	1646	// can see are extern functions or other comptime function body values (e.g. undefined). Of
	1647	// these, only extern functions need to be lowered to LLVM functions.
	1648	if (opt_extern != null and nav_ty.zigTypeTag(zcu) == .@"fn" and nav_ty.fnHasRuntimeBits(zcu)) {
	1649	const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
	1650	.function => \|function\| function, // re-use existing `Builder.Function`
	1651	.replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder),
	1652	};
	1653	llvm_function.setAlignment(llvm_align, &o.builder);
	1654	llvm_function.setSection(llvm_section, &o.builder);
	1655	try o.addLlvmFunctionAttributes(pt, nav_id, llvm_function);
	1656	} else {
	1657	const file_scope = nav.srcInst(ip).resolveFile(ip);
	1658	const mod = zcu.fileByIndex(file_scope).mod.?;
	1659
	1660	const llvm_variable: Builder.Variable.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
	1661	.variable => \|variable\| variable, // re-use existing `Builder.Variable`
	1662	.replaced, .alias, .function => try llvm_global.toNewVariable(&o.builder),
	1663	};
	1664	llvm_variable.setAlignment(llvm_align, &o.builder);
	1665	llvm_variable.setSection(llvm_section, &o.builder);
	1666	llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
	1667	try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);
	1668	llvm_variable.setThreadLocal(tl: {
	1669	if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
	1670	break :tl .default;
	1671	}, &o.builder);
	1672
	1673	if (!mod.strip) {
	1674	const debug_file = try o.getDebugFile(file_scope);
	1675	const debug_global_var_expr = try o.builder.debugGlobalVarExpression(
	1676	try o.builder.debugGlobalVar(
	1677	try o.builder.metadataString(nav.name.toSlice(ip)), // Name
	1678	try o.builder.metadataString(nav.fqn.toSlice(ip)), // Linkage name
	1679	debug_file, // File
	1680	debug_file, // Scope
	1681	zcu.navSrcLine(nav_id) + 1,
	1682	try o.getDebugType(pt, nav_ty),
	1683	llvm_variable,
	1684	.{ .local = llvm_global.ptrConst(&o.builder).linkage == .internal },
	1685	),
	1686	try o.builder.debugExpression(&.{}),
	1687	);
	1688	llvm_variable.setGlobalVariableExpression(debug_global_var_expr, &o.builder);
	1689	try o.debug_globals.append(o.gpa, debug_global_var_expr);
	1690	}
	1691	}
1630	1692	}
1631	1693
1632	1694	fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {
...	...	@@ -1634,19 +1696,43 @@ pub const Object = struct {
1634	1696	}
1635	1697
1636	1698	pub fn updateExports(
1637		self: *Object,
1638		pt: Zcu.PerThread,
	1699	o: *Object,
1639	1700	exported: Zcu.Exported,
1640	1701	export_indices: []const Zcu.Export.Index,
1641	1702	) link.File.UpdateExportsError!void {
1642		const zcu = pt.zcu;
1643		const nav_index = switch (exported) {
1644		.nav => \|nav\| nav,
1645		.uav => \|uav\| return updateExportedValue(self, pt, uav, export_indices),
1646		};
	1703	const zcu = o.zcu;
1647	1704	const ip = &zcu.intern_pool;
1648		const global_index = self.nav_map.get(nav_index).?;
	1705	const ty: Type, const llvm_ptr: Builder.Constant = switch (exported) {
	1706	.nav => \|nav\| exp: {
	1707	const nav_ty: Type = .fromInterned(ip.getNav(nav).resolved.?.type);
	1708	const nav_ref = try o.lowerNavRef(nav);
	1709	break :exp .{ nav_ty, nav_ref };
	1710	},
	1711	.uav => \|uav\| exp: {
	1712	const uav_ty = Value.fromInterned(uav).typeOf(zcu);
	1713	const uav_ref = try o.lowerUavRef(
	1714	uav,
	1715	uav_ty.abiAlignment(zcu),
	1716	target_util.defaultAddressSpace(zcu.getTarget(), .global_constant),
	1717	);
	1718	break :exp .{ uav_ty, uav_ref };
	1719	},
	1720	};
	1721	switch (llvm_ptr.unwrap()) {
	1722	.global => \|global\| return o.updateExportedGlobal(global, ty, export_indices),
	1723	.constant => @panic("LLVM TODO: export zero-bit value"),
	1724	}
	1725	}
	1726
	1727	fn updateExportedGlobal(
	1728	o: *Object,
	1729	global_index: Builder.Global.Index,
	1730	ty: Type,
	1731	export_indices: []const Zcu.Export.Index,
	1732	) link.File.UpdateExportsError!void {
	1733	const zcu = o.zcu;
1649	1734	const comp = zcu.comp;
	1735	const ip = &zcu.intern_pool;
1650	1736
1651	1737	// If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use.
1652	1738	coff_export_flags: {
...	...	@@ -1656,7 +1742,7 @@ pub const Object = struct {
1656	1742	.elf, .wasm => break :coff_export_flags,
1657	1743	.coff => \|*coff\| coff,
1658	1744	};
1659		if (!ip.isFunctionType(ip.getNav(nav_index).resolved.?.type)) break :coff_export_flags;
	1745	if (ty.zigTypeTag(zcu) != .@"fn") break :coff_export_flags;
1660	1746	const flags = &coff.lld_export_flags;
1661	1747	for (export_indices) \|export_index\| {
1662	1748	const name = export_index.ptr(zcu).opts.name;
...	...	@@ -1669,153 +1755,88 @@ pub const Object = struct {
1669	1755	}
1670	1756	}
1671	1757
1672		if (export_indices.len != 0) {
1673		return updateExportedGlobal(self, zcu, global_index, export_indices);
1674		} else {
1675		const fqn = try self.builder.strtabString(ip.getNav(nav_index).fqn.toSlice(ip));
1676		try global_index.rename(fqn, &self.builder);
1677		global_index.setLinkage(if (self.builder.strip) .private else .internal, &self.builder);
1678		if (comp.config.dll_export_fns)
1679		global_index.setDllStorageClass(.default, &self.builder);
1680		global_index.setUnnamedAddr(.unnamed_addr, &self.builder);
	1758	// If the first export specifies a linksection, set the exported variable's section to that
	1759	// one. This is kind of a hack because `std.builtin.ExportOptions.section` doesn't actually
	1760	// make much sense: the linksection should be associated with the declaration itself rather
	1761	// than some particular symbol it is exported as!
	1762	if (export_indices[0].ptr(zcu).opts.section.toSlice(ip)) \|section_slice\| {
	1763	const variable = &global_index.ptrConst(&o.builder).kind.variable;
	1764	variable.setSection(try o.builder.string(section_slice), &o.builder);
1681	1765	}
1682		}
1683	1766
1684		fn updateExportedValue(
1685		o: *Object,
1686		pt: Zcu.PerThread,
1687		exported_value: InternPool.Index,
1688		export_indices: []const Zcu.Export.Index,
1689		) link.File.UpdateExportsError!void {
1690		const zcu = pt.zcu;
1691		const gpa = zcu.gpa;
1692		const ip = &zcu.intern_pool;
1693		const main_exp_name = try o.builder.strtabString(export_indices[0].ptr(zcu).opts.name.toSlice(ip));
1694		const global_index = i: {
1695		const gop = try o.uav_map.getOrPut(gpa, exported_value);
1696		if (gop.found_existing) {
1697		const global_index = gop.value_ptr.*;
1698		try global_index.rename(main_exp_name, &o.builder);
1699		break :i global_index;
1700		}
1701		const llvm_addr_space = toLlvmAddressSpace(.generic, o.target);
1702		const variable_index = try o.builder.addVariable(
1703		main_exp_name,
1704		try o.lowerType(pt, Type.fromInterned(ip.typeOf(exported_value))),
1705		llvm_addr_space,
1706		);
1707		const global_index = variable_index.ptrConst(&o.builder).global;
1708		gop.value_ptr.* = global_index;
1709		// This line invalidates `gop`.
1710		const init_val = try o.lowerValue(pt, exported_value);
1711		try variable_index.setInitializer(init_val, &o.builder);
1712		break :i global_index;
1713		};
1714		return updateExportedGlobal(o, zcu, global_index, export_indices);
1715		}
	1767	const llvm_global_ty = global_index.typeOf(&o.builder);
1716	1768
1717		fn updateExportedGlobal(
1718		o: *Object,
1719		zcu: *Zcu,
1720		global_index: Builder.Global.Index,
1721		export_indices: []const Zcu.Export.Index,
1722		) link.File.UpdateExportsError!void {
1723		const comp = zcu.comp;
1724		const ip = &zcu.intern_pool;
1725		const first_export = export_indices[0].ptr(zcu);
1726
1727		// We will rename this global to have a name matching `first_export`.
1728		// Successive exports become aliases.
1729		// If the first export name already exists, then there is a corresponding
1730		// extern global - we replace it with this global.
1731		const first_exp_name = try o.builder.strtabString(first_export.opts.name.toSlice(ip));
1732		if (o.builder.getGlobal(first_exp_name)) \|other_global\| replace: {
1733		if (other_global.toConst().getBase(&o.builder) == global_index.toConst().getBase(&o.builder)) {
1734		break :replace; // this global already has the name we want
1735		}
1736		try global_index.takeName(other_global, &o.builder);
1737		try other_global.replace(global_index, &o.builder);
1738		// Problem: now we need to replace in the decl_map that
1739		// the extern decl index points to this new global. However we don't
1740		// know the decl index.
1741		// Even if we did, a future incremental update to the extern would then
1742		// treat the LLVM global as an extern rather than an export, so it would
1743		// need a way to check that.
1744		// This is a TODO that needs to be solved when making
1745		// the LLVM backend support incremental compilation.
1746		} else {
1747		try global_index.rename(first_exp_name, &o.builder);
1748		}
	1769	// All exports are represented as aliases to the original global.
1749	1770
1750		global_index.setUnnamedAddr(.default, &o.builder);
1751		if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden)
1752		global_index.setDllStorageClass(.dllexport, &o.builder);
1753		global_index.setLinkage(switch (first_export.opts.linkage) {
1754		.internal => unreachable,
1755		.strong => .external,
1756		.weak => .weak_odr,
1757		.link_once => .linkonce_odr,
1758		}, &o.builder);
1759		global_index.setVisibility(switch (first_export.opts.visibility) {
1760		.default => .default,
1761		.hidden => .hidden,
1762		.protected => .protected,
1763		}, &o.builder);
1764		if (first_export.opts.section.toSlice(ip)) \|section\|
1765		switch (global_index.ptrConst(&o.builder).kind) {
1766		.variable => \|impl_index\| impl_index.setSection(
1767		try o.builder.string(section),
1768		&o.builder,
1769		),
1770		.function => unreachable,
1771		.alias => unreachable,
1772		.replaced => unreachable,
1773		};
	1771	// TODO: we currently do not delete old exports. To do that we'll need to track which
	1772	// globals actually are exports.
1774	1773
1775		// If a Decl is exported more than one time (which is rare),
1776		// we add aliases for all but the first export.
1777		// TODO LLVM C API does not support deleting aliases.
1778		// The planned solution to this is https://github.com/ziglang/zig/issues/13265
1779		// Until then we iterate over existing aliases and make them point
1780		// to the correct decl, or otherwise add a new alias. Old aliases are leaked.
1781		for (export_indices[1..]) \|export_idx\| {
	1774	for (export_indices) \|export_idx\| {
1782	1775	const exp = export_idx.ptr(zcu);
1783	1776	const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip));
1784		if (o.builder.getGlobal(exp_name)) \|global\| {
1785		switch (global.ptrConst(&o.builder).kind) {
	1777
	1778	// Our goal is to make an alias with the name `exp_name`, but if that name is already
	1779	// taken by some existing global, we need to figure out what to do with that existing
	1780	// global.
	1781	//
	1782	// The name, aliasee, and type will be set within this block. Other properties of the
	1783	// alias will be set below.
	1784	const alias_global: Builder.Global.Index = global: {
	1785	const existing_global = o.builder.getGlobal(exp_name) orelse {
	1786	// There is no existing global with this name, so make a new alias.
	1787	const alias = try o.builder.addAlias(
	1788	exp_name,
	1789	llvm_global_ty,
	1790	.default,
	1791	global_index.toConst(),
	1792	);
	1793	break :global alias.ptrConst(&o.builder).global;
	1794	};
	1795	// There is an existing global with this name, so we can't just create an alias. We
	1796	// need to figure out what to do with the existing global instead.
	1797	switch (existing_global.ptrConst(&o.builder).kind) {
1786	1798	.alias => \|alias\| {
	1799	// We can just repurpose the existing alias.
1787	1800	alias.setAliasee(global_index.toConst(), &o.builder);
1788		continue;
	1801	alias.ptrConst(&o.builder).global.ptr(&o.builder).type = global_index.typeOf(&o.builder);
	1802	break :global existing_global;
1789	1803	},
1790	1804	.variable, .function => {
1791		// This existing global is an `extern` corresponding to this export.
1792		// Replace it with the global being exported.
1793		// This existing global must be replaced with the alias.
1794		try global.rename(.empty, &o.builder);
1795		try global.replace(global_index, &o.builder);
	1805	// This must be an extern, which is no good to us---we need an alias. The
	1806	// extern should refer to the value we're exporting, so replace it with the
	1807	// exported value. That will free up the name for us to create a new alias.
	1808	// We need to make a new global which is an alias. Replace this existing one
	1809	// with the target global, making the name available and fixing references
	1810	// to this global to point to the target.
	1811	try existing_global.replace(global_index, &o.builder);
	1812	// The name is now free, so create an alias.
	1813	const alias = try o.builder.addAlias(
	1814	exp_name,
	1815	llvm_global_ty,
	1816	.default,
	1817	global_index.toConst(),
	1818	);
	1819	break :global alias.ptrConst(&o.builder).global;
1796	1820	},
1797		.replaced => unreachable,
	1821	.replaced => unreachable, // a replaced global would have lost the name `exp_name`
1798	1822	}
1799		}
1800		const alias_index = try o.builder.addAlias(
1801		.empty,
1802		global_index.typeOf(&o.builder),
1803		.default,
1804		global_index.toConst(),
1805		);
1806		try alias_index.rename(exp_name, &o.builder);
1807
1808		const alias_global_index = alias_index.ptrConst(&o.builder).global;
1809		alias_global_index.setUnnamedAddr(.default, &o.builder);
1810		if (comp.config.dll_export_fns and first_export.opts.visibility != .hidden)
1811		alias_global_index.setDllStorageClass(.dllexport, &o.builder);
1812		alias_global_index.setLinkage(switch (first_export.opts.linkage) {
1813		.internal => unreachable,
	1823	};
	1824
	1825	// Now for a bit of setup which
	1826
	1827	// We need the alias to not be `unnamed_addr` to ensure that the alias address equals
	1828	// the address of the original global.
	1829	alias_global.setUnnamedAddr(.default, &o.builder);
	1830
	1831	if (comp.config.dll_export_fns and exp.opts.visibility != .hidden)
	1832	alias_global.setDllStorageClass(.dllexport, &o.builder);
	1833	alias_global.setLinkage(switch (exp.opts.linkage) {
	1834	.internal => if (o.builder.strip) .private else .internal, // we still did useful work in replacing an existing symbol if there was one
1814	1835	.strong => .external,
1815	1836	.weak => .weak_odr,
1816	1837	.link_once => .linkonce_odr,
1817	1838	}, &o.builder);
1818		alias_global_index.setVisibility(switch (first_export.opts.visibility) {
	1839	alias_global.setVisibility(switch (exp.opts.visibility) {
1819	1840	.default => .default,
1820	1841	.hidden => .hidden,
1821	1842	.protected => .protected,
...	...	@@ -1826,7 +1847,10 @@ pub const Object = struct {
1826	1847	pub fn updateContainerType(o: *Object, pt: Zcu.PerThread, ty: InternPool.Index, success: bool) Allocator.Error!void {
1827	1848	try o.type_pool.updateContainerType(pt, .{ .llvm = o }, ty, success);
1828	1849	if (o.named_enum_map.get(ty)) \|function_index\| {
1829		try o.updateIsNamedEnumValueFunction(pt, .fromInterned(ty), function_index);
	1850	try o.updateIsNamedEnumValueFunction(.fromInterned(ty), function_index);
	1851	}
	1852	if (o.enum_tag_name_map.get(ty)) \|function_index\| {
	1853	try o.updateEnumTagNameFunction(.fromInterned(ty), function_index);
1830	1854	}
1831	1855	}
1832	1856
...	...	@@ -1834,7 +1858,8 @@ pub const Object = struct {
1834	1858	///
1835	1859	/// `val` is always a type because `o.type_pool` only contains types.
1836	1860	pub fn addConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1837		const zcu = pt.zcu;
	1861	_ = pt;
	1862	const zcu = o.zcu;
1838	1863	const gpa = zcu.comp.gpa;
1839	1864	assert(zcu.intern_pool.typeOf(val) == .type_type);
1840	1865
...	...	@@ -1860,7 +1885,7 @@ pub const Object = struct {
1860	1885	///
1861	1886	/// `val` is always a type because `o.type_pool` only contains types.
1862	1887	pub fn updateConstIncomplete(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1863		const zcu = pt.zcu;
	1888	const zcu = o.zcu;
1864	1889	assert(zcu.intern_pool.typeOf(val) == .type_type);
1865	1890
1866	1891	const ty: Type = .fromInterned(val);
...	...	@@ -1874,7 +1899,12 @@ pub const Object = struct {
1874	1899	assert(val != .anyerror_type);
1875	1900	const fwd_ref = o.debug_types.items[@intFromEnum(index)];
1876	1901	const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)});
1877		const debug_incomplete_type = try o.builder.debugSignedType(name_str, 0);
	1902	// If `ty` is a function, use a dummy function type to prevent existing debug
	1903	// subprograms from becoming ill-formed.
	1904	const debug_incomplete_type = switch (ty.zigTypeTag(zcu)) {
	1905	.@"fn" => try o.builder.debugSubroutineType(null),
	1906	else => try o.builder.debugSignedType(name_str, 0),
	1907	};
1878	1908	o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type);
1879	1909	}
1880	1910	}
...	...	@@ -1882,7 +1912,7 @@ pub const Object = struct {
1882	1912	///
1883	1913	/// `val` is always a type because `o.type_pool` only contains types.
1884	1914	pub fn updateConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1885		const zcu = pt.zcu;
	1915	const zcu = o.zcu;
1886	1916	assert(zcu.intern_pool.typeOf(val) == .type_type);
1887	1917
1888	1918	const ty: Type = .fromInterned(val);
...	...	@@ -1904,13 +1934,13 @@ pub const Object = struct {
1904	1934	}
1905	1935	}
1906	1936
1907		fn getDebugFile(o: *Object, pt: Zcu.PerThread, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {
	1937	pub fn getDebugFile(o: *Object, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {
1908	1938	const gpa = o.gpa;
1909	1939	const gop = try o.debug_file_map.getOrPut(gpa, file_index);
1910	1940	errdefer assert(o.debug_file_map.remove(file_index));
1911	1941	if (gop.found_existing) return gop.value_ptr.*;
1912		const path = pt.zcu.fileByIndex(file_index).path;
1913		const abs_path = try path.toAbsolute(pt.zcu.comp.dirs, gpa);
	1942	const path = o.zcu.fileByIndex(file_index).path;
	1943	const abs_path = try path.toAbsolute(o.zcu.comp.dirs, gpa);
1914	1944	defer gpa.free(abs_path);
1915	1945
1916	1946	gop.value_ptr.* = try o.builder.debugFile(
...	...	@@ -1920,7 +1950,7 @@ pub const Object = struct {
1920	1950	return gop.value_ptr.*;
1921	1951	}
1922	1952
1923		fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata {
	1953	pub fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata {
1924	1954	assert(!o.builder.strip);
1925	1955	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
1926	1956	return o.debug_types.items[@intFromEnum(index)];
...	...	@@ -1937,8 +1967,8 @@ pub const Object = struct {
1937	1967	assert(!o.builder.strip);
1938	1968
1939	1969	const gpa = o.gpa;
1940		const target = o.target;
1941		const zcu = pt.zcu;
	1970	const zcu = o.zcu;
	1971	const target = zcu.getTarget();
1942	1972	const ip = &zcu.intern_pool;
1943	1973
1944	1974	const name = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)});
...	...	@@ -2203,7 +2233,9 @@ pub const Object = struct {
2203	2233	},
2204	2234	.@"fn" => {
2205	2235	if (!ty.fnHasRuntimeBits(zcu)) {
2206		return o.builder.debugSignedType(name, 0);
	2236	// Use a dummy function type to prevent existing debug subprograms from
	2237	// becoming ill-formed.
	2238	return o.builder.debugSubroutineType(null);
2207	2239	}
2208	2240
2209	2241	const fn_info = zcu.typeToFunc(ty).?;
...	...	@@ -2212,13 +2244,14 @@ pub const Object = struct {
2212	2244	defer debug_param_types.deinit(gpa);
2213	2245
2214	2246	// Return type goes first.
2215		const sret = firstParamSRet(fn_info, zcu, target);
2216		const ret_ty: Type = if (sret) .void else .fromInterned(fn_info.return_type);
2217		debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty));
2218
2219		if (sret) {
2220		const ptr_ty = try pt.singleMutPtrType(Type.fromInterned(fn_info.return_type));
	2247	if (firstParamSRet(fn_info, zcu, target)) {
	2248	// Actual return type is void, then first arg is the sret pointer.
	2249	const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
	2250	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
2221	2251	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty));
	2252	} else {
	2253	const ret_ty: Type = .fromInterned(fn_info.return_type);
	2254	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty));
2222	2255	}
2223	2256
2224	2257	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
...	...	@@ -2287,7 +2320,7 @@ pub const Object = struct {
2287	2320
2288	2321	const struct_type = zcu.typeToStruct(ty).?;
2289	2322
2290		const file = try o.getDebugFile(pt, struct_type.zir_index.resolveFile(ip));
	2323	const file = try o.getDebugFile(struct_type.zir_index.resolveFile(ip));
2291	2324	const scope = if (ty.getParentNamespace(zcu).unwrap()) \|parent_namespace\|
2292	2325	try o.namespaceToDebugScope(pt, parent_namespace)
2293	2326	else
...	...	@@ -2354,7 +2387,7 @@ pub const Object = struct {
2354	2387	.@"union" => {
2355	2388	const union_type = ip.loadUnionType(ty.toIntern());
2356	2389
2357		const file = try o.getDebugFile(pt, union_type.zir_index.resolveFile(ip));
	2390	const file = try o.getDebugFile(union_type.zir_index.resolveFile(ip));
2358	2391	const scope = if (ty.getParentNamespace(zcu).unwrap()) \|parent_namespace\|
2359	2392	try o.namespaceToDebugScope(pt, parent_namespace)
2360	2393	else
...	...	@@ -2512,7 +2545,7 @@ pub const Object = struct {
2512	2545	);
2513	2546	},
2514	2547	.@"enum" => {
2515		const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
	2548	const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2516	2549	const scope = if (ty.getParentNamespace(zcu).unwrap()) \|parent_namespace\|
2517	2550	try o.namespaceToDebugScope(pt, parent_namespace)
2518	2551	else
...	...	@@ -2573,7 +2606,7 @@ pub const Object = struct {
2573	2606	return o.builder.debugSignedType(name, 0);
2574	2607	}
2575	2608
2576		const file = try o.getDebugFile(pt, ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
	2609	const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2577	2610	const scope = if (ty.getParentNamespace(zcu).unwrap()) \|parent_namespace\|
2578	2611	try o.namespaceToDebugScope(pt, parent_namespace)
2579	2612	else
...	...	@@ -2598,8 +2631,8 @@ pub const Object = struct {
2598	2631	}
2599	2632
2600	2633	/// Called in `emit` so that the global error set is fully populated.
2601		fn lowerDebugAnyerrorType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Metadata {
2602		const zcu = pt.zcu;
	2634	fn lowerDebugAnyerrorType(o: *Object) Allocator.Error!Builder.Metadata {
	2635	const zcu = o.zcu;
2603	2636	const ip = &zcu.intern_pool;
2604	2637	const gpa = zcu.comp.gpa;
2605	2638
...	...	@@ -2633,7 +2666,7 @@ pub const Object = struct {
2633	2666	null, // file
2634	2667	o.debug_compile_unit.unwrap().?, // scope
2635	2668	0, // line
2636		try o.getDebugType(pt, try pt.intType(.unsigned, error_set_bits)),
	2669	try o.builder.debugUnsignedType(null, error_set_bits),
2637	2670	Type.anyerror.abiSize(zcu) * 8,
2638	2671	Type.anyerror.abiAlignment(zcu).toByteUnits().? * 8,
2639	2672	try o.builder.metadataTuple(enumerators),
...	...	@@ -2643,92 +2676,44 @@ pub const Object = struct {
2643	2676	}
2644	2677
2645	2678	fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata {
2646		const zcu = pt.zcu;
	2679	const zcu = o.zcu;
2647	2680	const namespace = zcu.namespacePtr(namespace_index);
2648		if (namespace.parent == .none) return try o.getDebugFile(pt, namespace.file_scope);
	2681	if (namespace.parent == .none) return try o.getDebugFile(namespace.file_scope);
2649	2682	return o.getDebugType(pt, .fromInterned(namespace.owner_type));
2650	2683	}
2651	2684
2652		fn allocTypeName(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error![:0]const u8 {
2653		var aw: Io.Writer.Allocating = .init(o.gpa);
2654		defer aw.deinit();
2655		ty.print(&aw.writer, pt, null) catch \|err\| switch (err) {
2656		error.WriteFailed => return error.OutOfMemory,
2657		};
2658		return aw.toOwnedSliceSentinel(0);
2659		}
2660
2661		/// If the llvm function does not exist, create it.
2662		/// Note that this can be called before the function's semantic analysis has
2663		/// completed, so if any attributes rely on that, they must be done in updateFunc, not here.
2664		fn resolveLlvmFunction(
	2685	/// Sets the attributes and callconv of the given `Builder.Function`, which corresponds to the
	2686	/// given `Nav` (which is a function).
	2687	fn addLlvmFunctionAttributes(
2665	2688	o: *Object,
2666	2689	pt: Zcu.PerThread,
2667		nav_index: InternPool.Nav.Index,
2668		) Allocator.Error!Builder.Function.Index {
2669		const zcu = pt.zcu;
	2690	nav_id: InternPool.Nav.Index,
	2691	function_index: Builder.Function.Index,
	2692	) Allocator.Error!void {
	2693	const zcu = o.zcu;
2670	2694	const ip = &zcu.intern_pool;
2671		const gpa = o.gpa;
2672		const nav = ip.getNav(nav_index);
2673		const owner_mod = zcu.navFileScope(nav_index).mod.?;
	2695	const nav = ip.getNav(nav_id);
	2696	const owner_mod = zcu.navFileScope(nav_id).mod.?;
2674	2697	const ty: Type = .fromInterned(nav.resolved.?.type);
2675		const gop = try o.nav_map.getOrPut(gpa, nav_index);
2676		if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.function;
2677	2698
2678	2699	const fn_info = zcu.typeToFunc(ty).?;
2679	2700	const target = &owner_mod.resolved_target.result;
2680		const sret = firstParamSRet(fn_info, zcu, target);
2681
2682		const is_extern, const lib_name = if (nav.getExtern(ip)) \|@"extern"\|
2683		.{ true, @"extern".lib_name }
2684		else
2685		.{ false, .none };
2686		const function_index = try o.builder.addFunction(
2687		try o.lowerType(pt, ty),
2688		try o.builder.strtabString((if (is_extern) nav.name else nav.fqn).toSlice(ip)),
2689		toLlvmAddressSpace(nav.resolved.?.@"addrspace", target),
2690		);
2691		gop.value_ptr.* = function_index.ptrConst(&o.builder).global;
2692	2701
2693	2702	var attributes: Builder.FunctionAttributes.Wip = .{};
2694	2703	defer attributes.deinit(&o.builder);
2695	2704
2696		if (!is_extern) {
2697		function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
2698		function_index.setUnnamedAddr(.unnamed_addr, &o.builder);
2699		} else {
2700		if (target.cpu.arch.isWasm()) {
2701		try attributes.addFnAttr(.{ .string = .{
2702		.kind = try o.builder.string("wasm-import-name"),
2703		.value = try o.builder.string(nav.name.toSlice(ip)),
	2705	if (target.cpu.arch.isWasm()) if (nav.getExtern(ip)) \|@"extern"\| {
	2706	try attributes.addFnAttr(.{ .string = .{
	2707	.kind = try o.builder.string("wasm-import-name"),
	2708	.value = try o.builder.string(nav.name.toSlice(ip)),
	2709	} }, &o.builder);
	2710	if (@"extern".lib_name.toSlice(ip)) \|lib_name_slice\| {
	2711	if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{
	2712	.kind = try o.builder.string("wasm-import-module"),
	2713	.value = try o.builder.string(lib_name_slice),
2704	2714	} }, &o.builder);
2705		if (lib_name.toSlice(ip)) \|lib_name_slice\| {
2706		if (!std.mem.eql(u8, lib_name_slice, "c")) try attributes.addFnAttr(.{ .string = .{
2707		.kind = try o.builder.string("wasm-import-module"),
2708		.value = try o.builder.string(lib_name_slice),
2709		} }, &o.builder);
2710		}
2711	2715	}
2712		}
2713
2714		var llvm_arg_i: u32 = 0;
2715		if (sret) {
2716		// Sret pointers must not be address 0
2717		try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
2718		try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
2719
2720		const raw_llvm_ret_ty = try o.lowerType(pt, Type.fromInterned(fn_info.return_type));
2721		try attributes.addParamAttr(llvm_arg_i, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2722
2723		llvm_arg_i += 1;
2724		}
2725
2726		const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
2727
2728		if (err_return_tracing) {
2729		try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
2730		llvm_arg_i += 1;
2731		}
	2716	};
2732	2717
2733	2718	if (fn_info.cc == .async) {
2734	2719	@panic("TODO: LLVM backend lower async function");
...	...	@@ -2803,9 +2788,6 @@ pub const Object = struct {
2803	2788	}
2804	2789	}
2805	2790
2806		if (nav.resolved.?.@"align" != .none)
2807		function_index.setAlignment(nav.resolved.?.@"align".toLlvm(), &o.builder);
2808
2809	2791	// Function attributes that are independent of analysis results of the function body.
2810	2792	try o.addCommonFnAttributes(
2811	2793	&attributes,
...	...	@@ -2821,18 +2803,81 @@ pub const Object = struct {
2821	2803
2822	2804	if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
2823	2805
2824		function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
2825		return function_index;
2826		}
	2806	var it = iterateParamTypes(o, fn_info);
	2807	if (firstParamSRet(fn_info, zcu, target)) {
	2808	// Sret pointers must not be address 0
	2809	try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
	2810	try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
2827	2811
2828		fn addCommonFnAttributes(
2829		o: *Object,
2830		attributes: *Builder.FunctionAttributes.Wip,
2831		owner_mod: *Package.Module,
2832		omit_frame_pointer: bool,
2833		) Allocator.Error!void {
2834		if (!owner_mod.red_zone) {
2835		try attributes.addFnAttr(.noredzone, &o.builder);
	2812	const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));
	2813	try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
	2814	it.llvm_index += 1;
	2815	}
	2816	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
	2817	if (err_return_tracing) {
	2818	try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
	2819	it.llvm_index += 1;
	2820	}
	2821	while (try it.next()) \|lowering\| switch (lowering) {
	2822	.byval => {
	2823	const param_index = it.zig_index - 1;
	2824	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]);
	2825	if (!isByRef(param_ty, zcu)) {
	2826	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
	2827	}
	2828	},
	2829	.byref => {
	2830	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	2831	const param_llvm_ty = try o.lowerType(param_ty);
	2832	const alignment = param_ty.abiAlignment(zcu);
	2833	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
	2834	},
	2835	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
	2836	.slice => {
	2837	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	2838	const ptr_info = param_ty.ptrInfo(zcu);
	2839	const llvm_ptr_index = it.llvm_index - 2;
	2840	if (std.math.cast(u5, it.zig_index - 1)) \|i\| {
	2841	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
	2842	try attributes.addParamAttr(llvm_ptr_index, .@"noalias", &o.builder);
	2843	}
	2844	}
	2845	if (param_ty.zigTypeTag(zcu) != .optional and
	2846	!ptr_info.flags.is_allowzero and
	2847	ptr_info.flags.address_space == .generic)
	2848	{
	2849	try attributes.addParamAttr(llvm_ptr_index, .nonnull, &o.builder);
	2850	}
	2851	if (ptr_info.flags.is_const) {
	2852	try attributes.addParamAttr(llvm_ptr_index, .readonly, &o.builder);
	2853	}
	2854	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
	2855	else => \|a\| .wrap(a.toLlvm()),
	2856	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
	2857	};
	2858	try attributes.addParamAttr(llvm_ptr_index, .{ .@"align" = elem_align }, &o.builder);
	2859	},
	2860	// No attributes needed for these.
	2861	.no_bits,
	2862	.abi_sized_int,
	2863	.multiple_llvm_types,
	2864	.float_array,
	2865	.i32_array,
	2866	.i64_array,
	2867	=> continue,
	2868	};
	2869
	2870	function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
	2871	}
	2872
	2873	fn addCommonFnAttributes(
	2874	o: *Object,
	2875	attributes: *Builder.FunctionAttributes.Wip,
	2876	owner_mod: *Package.Module,
	2877	omit_frame_pointer: bool,
	2878	) Allocator.Error!void {
	2879	if (!owner_mod.red_zone) {
	2880	try attributes.addFnAttr(.noredzone, &o.builder);
2836	2881	}
2837	2882	if (omit_frame_pointer) {
2838	2883	try attributes.addFnAttr(.{ .string = .{
...	...	@@ -2895,109 +2940,16 @@ pub const Object = struct {
2895	2940	}
2896	2941	}
2897	2942
2898		fn resolveGlobalUav(
2899		o: *Object,
2900		pt: Zcu.PerThread,
2901		uav: InternPool.Index,
2902		llvm_addr_space: Builder.AddrSpace,
2903		alignment: InternPool.Alignment,
2904		) Allocator.Error!Builder.Variable.Index {
2905		assert(alignment != .none);
2906		// TODO: Add address space to the anon_decl_map
2907		const gop = try o.uav_map.getOrPut(o.gpa, uav);
2908		if (gop.found_existing) {
2909		// Keep the greater of the two alignments.
2910		const variable_index = gop.value_ptr.ptr(&o.builder).kind.variable;
2911		const old_alignment = InternPool.Alignment.fromLlvm(variable_index.getAlignment(&o.builder));
2912		const max_alignment = old_alignment.maxStrict(alignment);
2913		variable_index.setAlignment(max_alignment.toLlvm(), &o.builder);
2914		return variable_index;
2915		}
2916		errdefer assert(o.uav_map.remove(uav));
2917
2918		const zcu = pt.zcu;
2919		const decl_ty = zcu.intern_pool.typeOf(uav);
2920
2921		const variable_index = try o.builder.addVariable(
2922		try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav)}),
2923		try o.lowerType(pt, Type.fromInterned(decl_ty)),
2924		llvm_addr_space,
2925		);
2926		gop.value_ptr.* = variable_index.ptrConst(&o.builder).global;
2927
2928		try variable_index.setInitializer(try o.lowerValue(pt, uav), &o.builder);
2929		variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
2930		variable_index.setMutability(.constant, &o.builder);
2931		variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
2932		variable_index.setAlignment(alignment.toLlvm(), &o.builder);
2933		return variable_index;
2934		}
2935
2936		fn resolveGlobalNav(
2937		o: *Object,
2938		pt: Zcu.PerThread,
2939		nav_index: InternPool.Nav.Index,
2940		) Allocator.Error!Builder.Variable.Index {
2941		const gop = try o.nav_map.getOrPut(o.gpa, nav_index);
2942		if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable;
2943		errdefer assert(o.nav_map.remove(nav_index));
2944
2945		const zcu = pt.zcu;
2946		const ip = &zcu.intern_pool;
2947		const nav = ip.getNav(nav_index);
2948		const linkage: std.builtin.GlobalLinkage, const visibility: Builder.Visibility, const is_dll_import: bool = switch (nav.resolved.?.value) {
2949		.none => .{ .internal, .default, false }, // this is a source declaration which is not marked `extern`
2950		else => \|val\| switch (ip.indexToKey(val)) {
2951		else => .{ .internal, .default, false },
2952		.@"extern" => \|e\| .{ e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import },
2953		},
2954		};
2955
2956		const variable_index = try o.builder.addVariable(
2957		try o.builder.strtabString(switch (linkage) {
2958		.internal => nav.fqn,
2959		.strong, .weak => nav.name,
2960		.link_once => unreachable,
2961		}.toSlice(ip)),
2962		try o.lowerType(pt, .fromInterned(nav.resolved.?.type)),
2963		toLlvmGlobalAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget()),
2964		);
2965		gop.value_ptr.* = variable_index.ptrConst(&o.builder).global;
2966
2967		// This is needed for declarations created by `@extern`.
2968		switch (linkage) {
2969		.internal => {
2970		variable_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
2971		variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
2972		},
2973		.strong, .weak => {
2974		variable_index.setLinkage(switch (linkage) {
2975		.internal => unreachable,
2976		.strong => .external,
2977		.weak => .extern_weak,
2978		.link_once => unreachable,
2979		}, &o.builder);
2980		variable_index.setUnnamedAddr(.default, &o.builder);
2981		if (nav.resolved.?.@"threadlocal" and !zcu.navFileScope(nav_index).mod.?.single_threaded)
2982		variable_index.setThreadLocal(.generaldynamic, &o.builder);
2983		if (is_dll_import) variable_index.setDllStorageClass(.dllimport, &o.builder);
2984		},
2985		.link_once => unreachable,
2986		}
2987		variable_index.setVisibility(visibility, &o.builder);
2988		return variable_index;
2989		}
2990
2991		fn errorIntType(o: *Object, pt: Zcu.PerThread) Allocator.Error!Builder.Type {
2992		return o.builder.intType(pt.zcu.errorSetBits());
	2943	pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {
	2944	return o.builder.intType(o.zcu.errorSetBits());
2993	2945	}
2994	2946
2995		fn lowerType(o: *Object, pt: Zcu.PerThread, t: Type) Allocator.Error!Builder.Type {
2996		const zcu = pt.zcu;
	2947	pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
	2948	const zcu = o.zcu;
2997	2949	const target = zcu.getTarget();
2998	2950	const ip = &zcu.intern_pool;
2999	2951	return switch (t.toIntern()) {
3000		.u0_type, .i0_type => unreachable,
	2952	.u0_type, .i0_type => unreachable, // no runtime bits
3001	2953	inline .u1_type,
3002	2954	.u8_type,
3003	2955	.i8_type,
...	...	@@ -3046,18 +2998,18 @@ pub const Object = struct {
3046	2998	return .i8;
3047	2999	},
3048	3000	.bool_type => .i1,
3049		.void_type => .void,
3050		.type_type => unreachable,
3051		.anyerror_type => try o.errorIntType(pt),
3052		.comptime_int_type,
3053		.comptime_float_type,
3054		.noreturn_type,
3055		=> unreachable,
	3001	.anyerror_type => try o.errorIntType(),
	3002	.void_type => unreachable, // no runtime bits
	3003	.type_type => unreachable, // no runtime bits
	3004	.comptime_int_type => unreachable, // no runtime bits
	3005	.comptime_float_type => unreachable, // no runtime bits
	3006	.noreturn_type => unreachable, // no runtime bits
	3007	.null_type => unreachable, // no runtime bits
	3008	.undefined_type => unreachable, // no runtime bits
	3009	.enum_literal_type => unreachable, // no runtime bits
	3010	.optional_noreturn_type => unreachable, // no runtime bits
	3011	.empty_tuple_type => unreachable, // no runtime bits
3056	3012	.anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
3057		.null_type,
3058		.undefined_type,
3059		.enum_literal_type,
3060		=> unreachable,
3061	3013	.ptr_usize_type,
3062	3014	.ptr_const_comptime_int_type,
3063	3015	.manyptr_u8_type,
...	...	@@ -3066,14 +3018,11 @@ pub const Object = struct {
3066	3018	=> .ptr,
3067	3019	.slice_const_u8_type,
3068	3020	.slice_const_u8_sentinel_0_type,
3069		=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(pt, Type.usize) }),
3070		.optional_noreturn_type => unreachable,
	3021	=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),
3071	3022	.anyerror_void_error_union_type,
3072	3023	.adhoc_inferred_error_set_type,
3073		=> try o.errorIntType(pt),
3074		.generic_poison_type,
3075		.empty_tuple_type,
3076		=> unreachable,
	3024	=> try o.errorIntType(),
	3025	.generic_poison_type => unreachable,
3077	3026	// values, not types
3078	3027	.undef,
3079	3028	.undef_bool,
...	...	@@ -3107,24 +3056,28 @@ pub const Object = struct {
3107	3056	.one, .many, .c => ptr_ty,
3108	3057	.slice => try o.builder.structType(.normal, &.{
3109	3058	ptr_ty,
3110		try o.lowerType(pt, Type.usize),
	3059	try o.lowerType(.usize),
3111	3060	}),
3112	3061	};
3113	3062	},
3114	3063	.array_type => \|array_type\| o.builder.arrayType(
3115	3064	array_type.lenIncludingSentinel(),
3116		try o.lowerType(pt, Type.fromInterned(array_type.child)),
	3065	try o.lowerType(.fromInterned(array_type.child)),
3117	3066	),
3118	3067	.vector_type => \|vector_type\| o.builder.vectorType(
3119	3068	.normal,
3120	3069	vector_type.len,
3121		try o.lowerType(pt, Type.fromInterned(vector_type.child)),
	3070	try o.lowerType(.fromInterned(vector_type.child)),
3122	3071	),
3123	3072	.opt_type => \|child_ty\| {
3124	3073	// Must stay in sync with `opt_payload` logic in `lowerPtr`.
3125		if (!Type.fromInterned(child_ty).hasRuntimeBits(zcu)) return .i8;
	3074	switch (Type.fromInterned(child_ty).classify(zcu)) {
	3075	.no_possible_value, .fully_comptime => unreachable,
	3076	.one_possible_value => return .i8,
	3077	.runtime, .partially_comptime => {},
	3078	}
3126	3079
3127		const payload_ty = try o.lowerType(pt, Type.fromInterned(child_ty));
	3080	const payload_ty = try o.lowerType(.fromInterned(child_ty));
3128	3081	if (t.optionalReprIsPayload(zcu)) return payload_ty;
3129	3082
3130	3083	comptime assert(optional_layout_version == 3);
...	...	@@ -3143,10 +3096,15 @@ pub const Object = struct {
3143	3096	.error_union_type => \|error_union_type\| {
3144	3097	// Must stay in sync with `codegen.errUnionPayloadOffset`.
3145	3098	// See logic in `lowerPtr`.
3146		const error_type = try o.errorIntType(pt);
3147		if (!Type.fromInterned(error_union_type.payload_type).hasRuntimeBits(zcu))
3148		return error_type;
3149		const payload_type = try o.lowerType(pt, Type.fromInterned(error_union_type.payload_type));
	3099	const error_type = try o.errorIntType();
	3100
	3101	switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
	3102	.fully_comptime => unreachable,
	3103	.no_possible_value, .one_possible_value => return error_type,
	3104	.runtime, .partially_comptime => {},
	3105	}
	3106
	3107	const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));
3150	3108
3151	3109	const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3152	3110	const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
...	...	@@ -3186,17 +3144,18 @@ pub const Object = struct {
3186	3144	const struct_type = ip.loadStructType(t.toIntern());
3187	3145
3188	3146	if (struct_type.layout == .@"packed") {
3189		const int_ty = try o.lowerType(pt, .fromInterned(struct_type.packed_backing_int_type));
	3147	const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));
3190	3148	try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3191	3149	return int_ty;
3192	3150	}
3193	3151
	3152	assert(struct_type.size > 0);
	3153
3194	3154	var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
3195	3155	defer llvm_field_types.deinit(o.gpa);
3196	3156	// Although we can estimate how much capacity to add, these cannot be
3197	3157	// relied upon because of the recursive calls to lowerType below.
3198	3158	try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3199		try o.struct_field_map.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
3200	3159
3201	3160	comptime assert(struct_layout_version == 2);
3202	3161	var offset: u64 = 0;
...	...	@@ -3221,24 +3180,9 @@ pub const Object = struct {
3221	3180	try o.builder.arrayType(padding_len, .i8),
3222	3181	);
3223	3182
3224		if (!field_ty.hasRuntimeBits(zcu)) {
3225		// This is a zero-bit field. If there are runtime bits after this field,
3226		// map to the next LLVM field (which we know exists): otherwise, don't
3227		// map the field, indicating it's at the end of the struct.
3228		if (offset != struct_type.size) {
3229		try o.struct_field_map.put(o.gpa, .{
3230		.struct_ty = t.toIntern(),
3231		.field_index = field_index,
3232		}, @intCast(llvm_field_types.items.len));
3233		}
3234		continue;
3235		}
	3183	if (!field_ty.hasRuntimeBits(zcu)) continue;
3236	3184
3237		try o.struct_field_map.put(o.gpa, .{
3238		.struct_ty = t.toIntern(),
3239		.field_index = field_index,
3240		}, @intCast(llvm_field_types.items.len));
3241		try llvm_field_types.append(o.gpa, try o.lowerType(pt, field_ty));
	3185	try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));
3242	3186
3243	3187	offset += field_ty.abiSize(zcu);
3244	3188	}
...	...	@@ -3270,19 +3214,15 @@ pub const Object = struct {
3270	3214	// Although we can estimate how much capacity to add, these cannot be
3271	3215	// relied upon because of the recursive calls to lowerType below.
3272	3216	try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3273		try o.struct_field_map.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
3274	3217
3275	3218	comptime assert(struct_layout_version == 2);
3276	3219	var offset: u64 = 0;
3277		var big_align: InternPool.Alignment = .none;
3278
3279		const struct_size = t.abiSize(zcu);
	3220	var big_align: InternPool.Alignment = .@"1";
3280	3221
3281	3222	for (
3282	3223	tuple_type.types.get(ip),
3283	3224	tuple_type.values.get(ip),
3284		0..,
3285		) \|field_ty, field_val, field_index\| {
	3225	) \|field_ty, field_val\| {
3286	3226	if (field_val != .none) continue;
3287	3227
3288	3228	const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
...	...	@@ -3296,22 +3236,9 @@ pub const Object = struct {
3296	3236	try o.builder.arrayType(padding_len, .i8),
3297	3237	);
3298	3238	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3299		// This is a zero-bit field. If there are runtime bits after this field,
3300		// map to the next LLVM field (which we know exists): otherwise, don't
3301		// map the field, indicating it's at the end of the struct.
3302		if (offset != struct_size) {
3303		try o.struct_field_map.put(o.gpa, .{
3304		.struct_ty = t.toIntern(),
3305		.field_index = @intCast(field_index),
3306		}, @intCast(llvm_field_types.items.len));
3307		}
3308	3239	continue;
3309	3240	}
3310		try o.struct_field_map.put(o.gpa, .{
3311		.struct_ty = t.toIntern(),
3312		.field_index = @intCast(field_index),
3313		}, @intCast(llvm_field_types.items.len));
3314		try llvm_field_types.append(o.gpa, try o.lowerType(pt, Type.fromInterned(field_ty)));
	3241	try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));
3315	3242
3316	3243	offset += Type.fromInterned(field_ty).abiSize(zcu);
3317	3244	}
...	...	@@ -3324,6 +3251,7 @@ pub const Object = struct {
3324	3251	try o.builder.arrayType(padding_len, .i8),
3325	3252	);
3326	3253	}
	3254	assert(offset > 0);
3327	3255	return o.builder.structType(.normal, llvm_field_types.items);
3328	3256	},
3329	3257	.union_type => {
...	...	@@ -3332,21 +3260,23 @@ pub const Object = struct {
3332	3260	const union_obj = ip.loadUnionType(t.toIntern());
3333	3261
3334	3262	if (union_obj.layout == .@"packed") {
3335		const int_ty = try o.lowerType(pt, .fromInterned(union_obj.packed_backing_int_type));
	3263	const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));
3336	3264	try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3337	3265	return int_ty;
3338	3266	}
3339	3267
	3268	assert(union_obj.size > 0);
	3269
3340	3270	const layout = Type.getUnionLayout(union_obj, zcu);
3341	3271
3342	3272	if (layout.payload_size == 0) {
3343		const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
	3273	const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
3344	3274	try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3345	3275	return enum_tag_ty;
3346	3276	}
3347	3277
3348	3278	const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3349		const aligned_field_llvm_ty = try o.lowerType(pt, aligned_field_ty);
	3279	const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);
3350	3280
3351	3281	const payload_ty = ty: {
3352	3282	if (layout.most_aligned_field_size == layout.payload_size) {
...	...	@@ -3372,7 +3302,7 @@ pub const Object = struct {
3372	3302	);
3373	3303	return ty;
3374	3304	}
3375		const enum_tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
	3305	const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
3376	3306
3377	3307	// Put the tag before or after the payload depending on which one's
3378	3308	// alignment is greater.
...	...	@@ -3400,16 +3330,10 @@ pub const Object = struct {
3400	3330	);
3401	3331	return ty;
3402	3332	},
3403		.opaque_type => {
3404		const gop = try o.type_map.getOrPut(o.gpa, t.toIntern());
3405		if (!gop.found_existing) {
3406		gop.value_ptr.* = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3407		}
3408		return gop.value_ptr.*;
3409		},
3410		.enum_type => try o.lowerType(pt, t.intTagType(zcu)),
3411		.func_type => \|func_type\| try o.lowerTypeFn(pt, func_type),
3412		.error_set_type, .inferred_error_set_type => try o.errorIntType(pt),
	3333	.opaque_type => unreachable, // no runtime bits
	3334	.enum_type => try o.lowerType(t.intTagType(zcu)),
	3335	.func_type => \|func_type\| try o.lowerFnType(t, func_type),
	3336	.error_set_type, .inferred_error_set_type => try o.errorIntType(),
3413	3337	// values, not types
3414	3338	.undef,
3415	3339	.simple_value,
...	...	@@ -3434,11 +3358,14 @@ pub const Object = struct {
3434	3358	};
3435	3359	}
3436	3360
3437		fn lowerTypeFn(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
3438		const zcu = pt.zcu;
	3361	fn lowerFnType(o: *Object, fn_ty: Type, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	3362	const zcu = o.zcu;
3439	3363	const ip = &zcu.intern_pool;
3440	3364	const target = zcu.getTarget();
3441		const ret_ty = try lowerFnRetTy(o, pt, fn_info);
	3365
	3366	assert(fn_ty.fnHasRuntimeBits(zcu));
	3367
	3368	const ret_ty = try lowerFnRetTy(o, fn_info);
3442	3369
3443	3370	var llvm_params: std.ArrayList(Builder.Type) = .empty;
3444	3371	defer llvm_params.deinit(o.gpa);
...	...	@@ -3453,12 +3380,12 @@ pub const Object = struct {
3453	3380	try llvm_params.append(o.gpa, llvm_ptr_ty);
3454	3381	}
3455	3382
3456		var it = iterateParamTypes(o, pt, fn_info);
	3383	var it = iterateParamTypes(o, fn_info);
3457	3384	while (try it.next()) \|lowering\| switch (lowering) {
3458	3385	.no_bits => continue,
3459	3386	.byval => {
3460	3387	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3461		try llvm_params.append(o.gpa, try o.lowerType(pt, param_ty));
	3388	try llvm_params.append(o.gpa, try o.lowerType(param_ty));
3462	3389	},
3463	3390	.byref, .byref_mut => {
3464	3391	try llvm_params.append(o.gpa, .ptr);
...	...	@@ -3473,7 +3400,7 @@ pub const Object = struct {
3473	3400	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3474	3401	try llvm_params.appendSlice(o.gpa, &.{
3475	3402	try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3476		try o.lowerType(pt, Type.usize),
	3403	try o.lowerType(.usize),
3477	3404	});
3478	3405	},
3479	3406	.multiple_llvm_types => {
...	...	@@ -3481,7 +3408,7 @@ pub const Object = struct {
3481	3408	},
3482	3409	.float_array => \|count\| {
3483	3410	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3484		const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
	3411	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
3485	3412	try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3486	3413	},
3487	3414	.i32_array, .i64_array => \|arr_len\| {
...	...	@@ -3500,20 +3427,17 @@ pub const Object = struct {
3500	3427	);
3501	3428	}
3502	3429
3503		fn lowerValue(o: *Object, pt: Zcu.PerThread, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
3504		const zcu = pt.zcu;
	3430	pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
	3431	const zcu = o.zcu;
3505	3432	const ip = &zcu.intern_pool;
3506	3433	const target = zcu.getTarget();
3507	3434
3508		const val = Value.fromInterned(arg_val);
	3435	const val: Value = .fromInterned(arg_val);
3509	3436	const val_key = ip.indexToKey(val.toIntern());
3510	3437
3511		if (val.isUndef(zcu)) {
3512		return o.builder.undefConst(try o.lowerType(pt, Type.fromInterned(val_key.typeOf())));
3513		}
3514
3515	3438	const ty: Type = .fromInterned(val_key.typeOf());
3516	3439	ty.assertHasLayout(zcu);
	3440	assert(ty.hasRuntimeBits(zcu));
3517	3441
3518	3442	return switch (val_key) {
3519	3443	.int_type,
...	...	@@ -3534,7 +3458,7 @@ pub const Object = struct {
3534	3458	.inferred_error_set_type,
3535	3459	=> unreachable, // types, not values
3536	3460
3537		.undef => unreachable, // handled above
	3461	.undef => return o.builder.undefConst(try o.lowerType(ty)),
3538	3462	.simple_value => \|simple_value\| switch (simple_value) {
3539	3463	.void => unreachable, // non-runtime value
3540	3464	.null => unreachable, // non-runtime value
...	...	@@ -3544,46 +3468,40 @@ pub const Object = struct {
3544	3468	.true => .true,
3545	3469	},
3546	3470	.enum_literal => unreachable, // non-runtime value
3547		.@"extern" => \|@"extern"\| {
3548		const function_index = try o.resolveLlvmFunction(pt, @"extern".owner_nav);
3549		return function_index.ptrConst(&o.builder).global.toConst();
3550		},
3551		.func => \|func\| {
3552		const function_index = try o.resolveLlvmFunction(pt, func.owner_nav);
3553		return function_index.ptrConst(&o.builder).global.toConst();
3554		},
	3471	.@"extern" => unreachable, // non-runtime value
	3472	.func => unreachable, // non-runtime value
3555	3473	.int => {
3556	3474	var bigint_space: Value.BigIntSpace = undefined;
3557	3475	const bigint = val.toBigInt(&bigint_space, zcu);
3558		return lowerBigInt(o, pt, ty, bigint);
	3476	const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);
	3477	return o.builder.bigIntConst(llvm_int_ty, bigint);
3559	3478	},
3560	3479	.err => \|err\| {
3561		const int = try pt.getErrorValue(err.name);
3562		const llvm_int = try o.builder.intConst(try o.errorIntType(pt), int);
3563		return llvm_int;
	3480	const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
	3481	return o.builder.intConst(try o.errorIntType(), int);
3564	3482	},
3565	3483	.error_union => \|error_union\| {
3566		const err_val = switch (error_union.val) {
3567		.err_name => \|err_name\| try pt.intern(.{ .err = .{
3568		.ty = ty.errorUnionSet(zcu).toIntern(),
3569		.name = err_name,
3570		} }),
3571		.payload => (try pt.intValue(try pt.errorIntType(), 0)).toIntern(),
	3484	const llvm_error_ty = try o.errorIntType();
	3485	const llvm_error_value = switch (error_union.val) {
	3486	.err_name => \|name\| try o.builder.intConst(
	3487	llvm_error_ty,
	3488	zcu.intern_pool.getErrorValueIfExists(name).?,
	3489	),
	3490	.payload => try o.builder.intConst(llvm_error_ty, 0),
3572	3491	};
3573		const err_int_ty = try pt.errorIntType();
	3492
3574	3493	const payload_type = ty.errorUnionPayload(zcu);
3575	3494	if (!payload_type.hasRuntimeBits(zcu)) {
3576	3495	// We use the error type directly as the type.
3577		return o.lowerValue(pt, err_val);
	3496	return llvm_error_value;
3578	3497	}
3579	3498
3580	3499	const payload_align = payload_type.abiAlignment(zcu);
3581		const error_align = err_int_ty.abiAlignment(zcu);
3582		const llvm_error_value = try o.lowerValue(pt, err_val);
3583		const llvm_payload_value = try o.lowerValue(pt, switch (error_union.val) {
3584		.err_name => try pt.intern(.{ .undef = payload_type.toIntern() }),
3585		.payload => \|payload\| payload,
3586		});
	3500	const error_align = Type.errorAbiAlignment(zcu);
	3501	const llvm_payload_value = switch (error_union.val) {
	3502	.err_name => try o.builder.undefConst(try o.lowerType(payload_type)),
	3503	.payload => \|payload\| try o.lowerValue(payload),
	3504	};
3587	3505
3588	3506	var fields: [3]Builder.Type = undefined;
3589	3507	var vals: [3]Builder.Constant = undefined;
...	...	@@ -3597,7 +3515,7 @@ pub const Object = struct {
3597	3515	fields[0] = vals[0].typeOf(&o.builder);
3598	3516	fields[1] = vals[1].typeOf(&o.builder);
3599	3517
3600		const llvm_ty = try o.lowerType(pt, ty);
	3518	const llvm_ty = try o.lowerType(ty);
3601	3519	const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3602	3520	if (llvm_ty_fields.len > 2) {
3603	3521	assert(llvm_ty_fields.len == 3);
...	...	@@ -3609,7 +3527,7 @@ pub const Object = struct {
3609	3527	fields[0..llvm_ty_fields.len],
3610	3528	), vals[0..llvm_ty_fields.len]);
3611	3529	},
3612		.enum_tag => \|enum_tag\| o.lowerValue(pt, enum_tag.int),
	3530	.enum_tag => \|enum_tag\| o.lowerValue(enum_tag.int),
3613	3531	.float => switch (ty.floatBits(target)) {
3614	3532	16 => if (backendSupportsF16(target))
3615	3533	try o.builder.halfConst(val.toFloat(f16, zcu))
...	...	@@ -3624,10 +3542,10 @@ pub const Object = struct {
3624	3542	128 => try o.builder.fp128Const(val.toFloat(f128, zcu)),
3625	3543	else => unreachable,
3626	3544	},
3627		.ptr => try o.lowerPtr(pt, arg_val, 0),
3628		.slice => \|slice\| return o.builder.structConst(try o.lowerType(pt, ty), &.{
3629		try o.lowerValue(pt, slice.ptr),
3630		try o.lowerValue(pt, slice.len),
	3545	.ptr => try o.lowerPtr(arg_val, 0),
	3546	.slice => \|slice\| return o.builder.structConst(try o.lowerType(ty), &.{
	3547	try o.lowerValue(slice.ptr),
	3548	try o.lowerValue(slice.len),
3631	3549	}),
3632	3550	.opt => \|opt\| {
3633	3551	comptime assert(optional_layout_version == 3);
...	...	@@ -3637,7 +3555,7 @@ pub const Object = struct {
3637	3555	if (!payload_ty.hasRuntimeBits(zcu)) {
3638	3556	return non_null_bit;
3639	3557	}
3640		const llvm_ty = try o.lowerType(pt, ty);
	3558	const llvm_ty = try o.lowerType(ty);
3641	3559	if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3642	3560	.none => switch (llvm_ty.tag(&o.builder)) {
3643	3561	.integer => try o.builder.intConst(llvm_ty, 0),
...	...	@@ -3645,16 +3563,16 @@ pub const Object = struct {
3645	3563	.structure => try o.builder.zeroInitConst(llvm_ty),
3646	3564	else => unreachable,
3647	3565	},
3648		else => \|payload\| try o.lowerValue(pt, payload),
	3566	else => \|payload\| try o.lowerValue(payload),
3649	3567	};
3650	3568	assert(payload_ty.zigTypeTag(zcu) != .@"fn");
3651	3569
3652	3570	var fields: [3]Builder.Type = undefined;
3653	3571	var vals: [3]Builder.Constant = undefined;
3654		vals[0] = try o.lowerValue(pt, switch (opt.val) {
3655		.none => try pt.intern(.{ .undef = payload_ty.toIntern() }),
3656		else => \|payload\| payload,
3657		});
	3572	vals[0] = switch (opt.val) {
	3573	.none => try o.builder.undefConst(try o.lowerType(payload_ty)),
	3574	else => \|payload\| try o.lowerValue(payload),
	3575	};
3658	3576	vals[1] = non_null_bit;
3659	3577	fields[0] = vals[0].typeOf(&o.builder);
3660	3578	fields[1] = vals[1].typeOf(&o.builder);
...	...	@@ -3670,14 +3588,14 @@ pub const Object = struct {
3670	3588	fields[0..llvm_ty_fields.len],
3671	3589	), vals[0..llvm_ty_fields.len]);
3672	3590	},
3673		.bitpack => \|bitpack\| return o.lowerValue(pt, bitpack.backing_int_val),
	3591	.bitpack => \|bitpack\| return o.lowerValue(bitpack.backing_int_val),
3674	3592	.aggregate => \|aggregate\| switch (ip.indexToKey(ty.toIntern())) {
3675	3593	.array_type => \|array_type\| switch (aggregate.storage) {
3676	3594	.bytes => \|bytes\| try o.builder.stringConst(try o.builder.string(
3677	3595	bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3678	3596	)),
3679	3597	.elems => \|elems\| {
3680		const array_ty = try o.lowerType(pt, ty);
	3598	const array_ty = try o.lowerType(ty);
3681	3599	const elem_ty = array_ty.childType(&o.builder);
3682	3600	assert(elems.len == array_ty.aggregateLen(&o.builder));
3683	3601
...	...	@@ -3697,7 +3615,7 @@ pub const Object = struct {
3697	3615
3698	3616	var need_unnamed = false;
3699	3617	for (vals, fields, elems) \|result_val, result_field, elem\| {
3700		result_val.* = try o.lowerValue(pt, elem);
	3618	result_val.* = try o.lowerValue(elem);
3701	3619	result_field.* = result_val.typeOf(&o.builder);
3702	3620	if (result_field.* != elem_ty) need_unnamed = true;
3703	3621	}
...	...	@@ -3709,7 +3627,7 @@ pub const Object = struct {
3709	3627	.repeated_elem => \|elem\| {
3710	3628	const len: usize = @intCast(array_type.len);
3711	3629	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3712		const array_ty = try o.lowerType(pt, ty);
	3630	const array_ty = try o.lowerType(ty);
3713	3631	const elem_ty = array_ty.childType(&o.builder);
3714	3632
3715	3633	const ExpectedContents = extern struct {
...	...	@@ -3727,12 +3645,12 @@ pub const Object = struct {
3727	3645	defer allocator.free(fields);
3728	3646
3729	3647	var need_unnamed = false;
3730		@memset(vals[0..len], try o.lowerValue(pt, elem));
	3648	@memset(vals[0..len], try o.lowerValue(elem));
3731	3649	@memset(fields[0..len], vals[0].typeOf(&o.builder));
3732	3650	if (fields[0] != elem_ty) need_unnamed = true;
3733	3651
3734	3652	if (array_type.sentinel != .none) {
3735		vals[len] = try o.lowerValue(pt, array_type.sentinel);
	3653	vals[len] = try o.lowerValue(array_type.sentinel);
3736	3654	fields[len] = vals[len].typeOf(&o.builder);
3737	3655	if (fields[len] != elem_ty) need_unnamed = true;
3738	3656	}
...	...	@@ -3744,7 +3662,7 @@ pub const Object = struct {
3744	3662	},
3745	3663	},
3746	3664	.vector_type => \|vector_type\| {
3747		const vector_ty = try o.lowerType(pt, ty);
	3665	const vector_ty = try o.lowerType(ty);
3748	3666	switch (aggregate.storage) {
3749	3667	.bytes, .elems => {
3750	3668	const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
...	...	@@ -3761,7 +3679,7 @@ pub const Object = struct {
3761	3679	result_val.* = try o.builder.intConst(.i8, byte);
3762	3680	},
3763	3681	.elems => \|elems\| for (vals, elems) \|*result_val, elem\| {
3764		result_val.* = try o.lowerValue(pt, elem);
	3682	result_val.* = try o.lowerValue(elem);
3765	3683	},
3766	3684	.repeated_elem => unreachable,
3767	3685	}
...	...	@@ -3769,12 +3687,12 @@ pub const Object = struct {
3769	3687	},
3770	3688	.repeated_elem => \|elem\| return o.builder.splatConst(
3771	3689	vector_ty,
3772		try o.lowerValue(pt, elem),
	3690	try o.lowerValue(elem),
3773	3691	),
3774	3692	}
3775	3693	},
3776	3694	.tuple_type => \|tuple\| {
3777		const struct_ty = try o.lowerType(pt, ty);
	3695	const struct_ty = try o.lowerType(ty);
3778	3696	const llvm_len = struct_ty.aggregateLen(&o.builder);
3779	3697
3780	3698	const ExpectedContents = extern struct {
...	...	@@ -3794,14 +3712,14 @@ pub const Object = struct {
3794	3712	comptime assert(struct_layout_version == 2);
3795	3713	var llvm_index: usize = 0;
3796	3714	var offset: u64 = 0;
3797		var big_align: InternPool.Alignment = .none;
	3715	var big_align: InternPool.Alignment = .@"1";
3798	3716	var need_unnamed = false;
3799	3717	for (
3800	3718	tuple.types.get(ip),
3801	3719	tuple.values.get(ip),
3802	3720	0..,
3803		) \|field_ty, field_val, field_index\| {
3804		if (field_val != .none) continue;
	3721	) \|field_ty, field_comptime_val, field_index\| {
	3722	if (field_comptime_val != .none) continue;
3805	3723	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
3806	3724
3807	3725	const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
...	...	@@ -3819,8 +3737,11 @@ pub const Object = struct {
3819	3737	llvm_index += 1;
3820	3738	}
3821	3739
3822		vals[llvm_index] =
3823		try o.lowerValue(pt, (try val.fieldValue(pt, field_index)).toIntern());
	3740	vals[llvm_index] = switch (aggregate.storage) {
	3741	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
	3742	.elems => \|elems\| try o.lowerValue(elems[field_index]),
	3743	.repeated_elem => \|elem\| try o.lowerValue(elem),
	3744	};
3824	3745	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3825	3746	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
3826	3747	need_unnamed = true;
...	...	@@ -3848,7 +3769,7 @@ pub const Object = struct {
3848	3769	},
3849	3770	.struct_type => {
3850	3771	const struct_type = ip.loadStructType(ty.toIntern());
3851		const struct_ty = try o.lowerType(pt, ty);
	3772	const struct_ty = try o.lowerType(ty);
3852	3773	assert(struct_type.layout != .@"packed");
3853	3774	const llvm_len = struct_ty.aggregateLen(&o.builder);
3854	3775
...	...	@@ -3892,10 +3813,11 @@ pub const Object = struct {
3892	3813	continue;
3893	3814	}
3894	3815
3895		vals[llvm_index] = try o.lowerValue(
3896		pt,
3897		(try val.fieldValue(pt, field_index)).toIntern(),
3898		);
	3816	vals[llvm_index] = switch (aggregate.storage) {
	3817	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
	3818	.elems => \|elems\| try o.lowerValue(elems[field_index]),
	3819	.repeated_elem => \|elem\| try o.lowerValue(elem),
	3820	};
3899	3821	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3900	3822	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
3901	3823	need_unnamed = true;
...	...	@@ -3924,9 +3846,9 @@ pub const Object = struct {
3924	3846	else => unreachable,
3925	3847	},
3926	3848	.un => \|un\| {
3927		const union_ty = try o.lowerType(pt, ty);
	3849	const union_ty = try o.lowerType(ty);
3928	3850	const layout = ty.unionGetLayout(zcu);
3929		if (layout.payload_size == 0) return o.lowerValue(pt, un.tag);
	3851	if (layout.payload_size == 0) return o.lowerValue(un.tag);
3930	3852
3931	3853	const union_obj = zcu.typeToUnion(ty).?;
3932	3854	const container_layout = union_obj.layout;
...	...	@@ -3947,7 +3869,7 @@ pub const Object = struct {
3947	3869	const padding_len = layout.payload_size;
3948	3870	break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3949	3871	}
3950		const payload = try o.lowerValue(pt, un.val);
	3872	const payload = try o.lowerValue(un.val);
3951	3873	const payload_ty = payload.typeOf(&o.builder);
3952	3874	if (payload_ty != union_ty.structFields(&o.builder)[
3953	3875	@intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
...	...	@@ -3962,7 +3884,7 @@ pub const Object = struct {
3962	3884	);
3963	3885	} else p: {
3964	3886	assert(layout.tag_size == 0);
3965		const union_val = try o.lowerValue(pt, un.val);
	3887	const union_val = try o.lowerValue(un.val);
3966	3888	need_unnamed = true;
3967	3889	break :p union_val;
3968	3890	};
...	...	@@ -3972,7 +3894,7 @@ pub const Object = struct {
3972	3894	try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3973	3895	else
3974	3896	union_ty, &.{payload});
3975		const tag = try o.lowerValue(pt, un.tag);
	3897	const tag = try o.lowerValue(un.tag);
3976	3898	const tag_ty = tag.typeOf(&o.builder);
3977	3899	var fields: [3]Builder.Type = undefined;
3978	3900	var vals: [3]Builder.Constant = undefined;
...	...	@@ -3998,52 +3920,45 @@ pub const Object = struct {
3998	3920	};
3999	3921	}
4000	3922
4001		fn lowerBigInt(
4002		o: *Object,
4003		pt: Zcu.PerThread,
4004		ty: Type,
4005		bigint: std.math.big.int.Const,
4006		) Allocator.Error!Builder.Constant {
4007		const zcu = pt.zcu;
4008		return o.builder.bigIntConst(try o.builder.intType(ty.intInfo(zcu).bits), bigint);
4009		}
4010
4011	3923	fn lowerPtr(
4012	3924	o: *Object,
4013		pt: Zcu.PerThread,
4014	3925	ptr_val: InternPool.Index,
4015	3926	prev_offset: u64,
4016	3927	) Allocator.Error!Builder.Constant {
4017		const zcu = pt.zcu;
	3928	const zcu = o.zcu;
4018	3929	const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
4019	3930	const offset: u64 = prev_offset + ptr.byte_offset;
4020	3931	return switch (ptr.base_addr) {
4021	3932	.nav => \|nav\| {
4022		const base_ptr = try o.lowerNavRefValue(pt, nav);
	3933	const base_ptr = try o.lowerNavRef(nav);
4023	3934	return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
4024	3935	try o.builder.intConst(.i64, offset),
4025	3936	});
4026	3937	},
4027	3938	.uav => \|uav\| {
4028		const base_ptr = try o.lowerUavRef(pt, uav);
	3939	const orig_ptr_ty: Type = .fromInterned(uav.orig_ty);
	3940	const base_ptr = try o.lowerUavRef(
	3941	uav.val,
	3942	orig_ptr_ty.ptrAlignment(zcu),
	3943	orig_ptr_ty.ptrAddressSpace(zcu),
	3944	);
4029	3945	return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
4030	3946	try o.builder.intConst(.i64, offset),
4031	3947	});
4032	3948	},
4033	3949	.int => try o.builder.castConst(
4034	3950	.inttoptr,
4035		try o.builder.intConst(try o.lowerType(pt, Type.usize), offset),
4036		try o.lowerType(pt, Type.fromInterned(ptr.ty)),
	3951	try o.builder.intConst(try o.lowerType(.usize), offset),
	3952	try o.lowerType(.fromInterned(ptr.ty)),
4037	3953	),
4038	3954	.eu_payload => \|eu_ptr\| try o.lowerPtr(
4039		pt,
4040	3955	eu_ptr,
4041	3956	offset + codegen.errUnionPayloadOffset(
4042	3957	Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
4043	3958	zcu,
4044	3959	),
4045	3960	),
4046		.opt_payload => \|opt_ptr\| try o.lowerPtr(pt, opt_ptr, offset),
	3961	.opt_payload => \|opt_ptr\| try o.lowerPtr(opt_ptr, offset),
4047	3962	.field => \|field\| {
4048	3963	const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu);
4049	3964	const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) {
...	...	@@ -4061,132 +3976,118 @@ pub const Object = struct {
4061	3976	},
4062	3977	else => unreachable,
4063	3978	};
4064		return o.lowerPtr(pt, field.base, offset + field_off);
	3979	return o.lowerPtr(field.base, offset + field_off);
4065	3980	},
4066	3981	.arr_elem => \|arr_elem\| {
4067	3982	const base_ptr_ty = Value.fromInterned(arr_elem.base).typeOf(zcu);
4068	3983	assert(base_ptr_ty.ptrSize(zcu) == .many);
4069	3984	const elem_size = base_ptr_ty.childType(zcu).abiSize(zcu);
4070		return o.lowerPtr(pt, arr_elem.base, offset + elem_size * arr_elem.index);
	3985	return o.lowerPtr(arr_elem.base, offset + elem_size * arr_elem.index);
4071	3986	},
4072	3987	.comptime_field => unreachable,
4073	3988	.comptime_alloc => unreachable,
4074	3989	};
4075	3990	}
4076	3991
4077		/// This logic is very similar to `lowerNavRefValue` but for anonymous declarations.
4078		/// Maybe the logic could be unified.
4079		fn lowerUavRef(
	3992	pub fn lowerPtrToVoid(
4080	3993	o: *Object,
4081		pt: Zcu.PerThread,
4082		uav: InternPool.Key.Ptr.BaseAddr.Uav,
	3994	/// Must not be `.none`.
	3995	@"align": InternPool.Alignment,
	3996	@"addrspace": std.builtin.AddressSpace,
4083	3997	) Allocator.Error!Builder.Constant {
4084		const zcu = pt.zcu;
	3998	const addr: u64 = @"align".toByteUnits().?;
	3999	const llvm_usize = try o.lowerType(.usize);
	4000	const llvm_addr = try o.builder.intConst(llvm_usize, addr);
	4001	const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
	4002	return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
	4003	}
	4004
	4005	pub fn lowerUavRef(
	4006	o: *Object,
	4007	uav_val: InternPool.Index,
	4008	/// Must not be `.none`.
	4009	@"align": InternPool.Alignment,
	4010	@"addrspace": std.builtin.AddressSpace,
	4011	) Allocator.Error!Builder.Constant {
	4012	assert(@"align" != .none);
	4013
	4014	const zcu = o.zcu;
4085	4015	const ip = &zcu.intern_pool;
4086		const uav_val = uav.val;
4087		const uav_ty = Type.fromInterned(ip.typeOf(uav_val));
4088		const target = zcu.getTarget();
	4016	const gpa = zcu.comp.gpa;
	4017
	4018	const uav_ty: Type = .fromInterned(ip.typeOf(uav_val));
4089	4019
4090	4020	switch (ip.indexToKey(uav_val)) {
4091		.func => @panic("TODO"),
4092		.@"extern" => @panic("TODO"),
	4021	.func => unreachable, // should be using a Nav ref
	4022	.@"extern" => unreachable, // should be using a Nav ref
4093	4023	else => {},
4094	4024	}
4095	4025
4096		const ptr_ty = Type.fromInterned(uav.orig_ty);
4097
4098		if (!uav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) {
4099		return o.lowerPtrToVoid(pt, ptr_ty);
	4026	if (!uav_ty.hasRuntimeBits(zcu)) {
	4027	return o.lowerPtrToVoid(@"align", @"addrspace");
4100	4028	}
4101	4029
4102		assert(uav_ty.zigTypeTag(zcu) != .@"fn"); // should be using a Nav ref
4103
4104		const llvm_addr_space = toLlvmAddressSpace(ptr_ty.ptrAddressSpace(zcu), target);
4105		const alignment = ptr_ty.ptrAlignment(zcu);
4106		const llvm_global = (try o.resolveGlobalUav(pt, uav.val, llvm_addr_space, alignment)).ptrConst(&o.builder).global;
4107
4108		const llvm_val = try o.builder.convConst(
4109		llvm_global.toConst(),
4110		try o.builder.ptrType(llvm_addr_space),
4111		);
4112
4113		return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4114		}
	4030	const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget());
4115	4031
4116		fn lowerNavRefValue(o: *Object, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) Allocator.Error!Builder.Constant {
4117		const zcu = pt.zcu;
	4032	const gop = try o.uav_map.getOrPut(gpa, .{ .val = uav_val, .@"addrspace" = @"addrspace" });
	4033	if (gop.found_existing) {
	4034	// Keep the greater of the two alignments.
	4035	const llvm_variable = gop.value_ptr.*;
	4036	const old_align: InternPool.Alignment = .fromLlvm(llvm_variable.getAlignment(&o.builder));
	4037	llvm_variable.setAlignment(old_align.maxStrict(@"align").toLlvm(), &o.builder);
	4038	return llvm_variable.ptrConst(&o.builder).global.toConst();
	4039	}
	4040	errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
	4041
	4042	const llvm_ty = try o.lowerType(uav_ty);
	4043	const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
	4044	const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
	4045	gop.value_ptr.* = llvm_variable;
	4046	try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);
	4047	llvm_variable.setMutability(.constant, &o.builder);
	4048	llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
	4049	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
	4050	llvm_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
	4051	llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
	4052	return llvm_global.toConst();
	4053	}
	4054
	4055	pub fn lowerNavRef(o: *Object, nav_id: InternPool.Nav.Index) Allocator.Error!Builder.Constant {
	4056	const zcu = o.zcu;
4118	4057	const ip = &zcu.intern_pool;
	4058	const gpa = zcu.comp.gpa;
4119	4059
4120		const nav = ip.getNav(nav_index);
4121
	4060	const nav = ip.getNav(nav_id);
4122	4061	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
4123		const ptr_ty = try pt.navPtrType(nav_index);
4124
4125		if (nav.getExtern(ip) == null and !nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) {
4126		return o.lowerPtrToVoid(pt, ptr_ty);
4127		}
4128
4129		const llvm_global = if (nav_ty.zigTypeTag(zcu) == .@"fn")
4130		(try o.resolveLlvmFunction(pt, nav_index)).ptrConst(&o.builder).global
4131		else
4132		(try o.resolveGlobalNav(pt, nav_index)).ptrConst(&o.builder).global;
4133
4134		const llvm_val = try o.builder.convConst(
4135		llvm_global.toConst(),
4136		try o.builder.ptrType(toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget())),
4137		);
4138
4139		return o.builder.convConst(llvm_val, try o.lowerType(pt, ptr_ty));
4140		}
4141
4142		fn lowerPtrToVoid(o: *Object, pt: Zcu.PerThread, ptr_ty: Type) Allocator.Error!Builder.Constant {
4143		const zcu = pt.zcu;
4144		// Even though we are pointing at something which has zero bits (e.g. `void`),
4145		// Pointers are defined to have bits. So we must return something here.
4146		// The value cannot be undefined, because we use the `nonnull` annotation
4147		// for non-optional pointers. We also need to respect the alignment, even though
4148		// the address will never be dereferenced.
4149		const int: u64 = ptr_ty.ptrInfo(zcu).flags.alignment.toByteUnits() orelse
4150		// Note that these 0xaa values are appropriate even in release-optimized builds
4151		// because we need a well-defined value that is not null, and LLVM does not
4152		// have an "undef_but_not_null" attribute. As an example, if this `alloc` AIR
4153		// instruction is followed by a `wrap_optional`, it will return this value
4154		// verbatim, and the result should test as non-null.
4155		switch (zcu.getTarget().ptrBitWidth()) {
4156		16 => 0xaaaa,
4157		32 => 0xaaaaaaaa,
4158		64 => 0xaaaaaaaa_aaaaaaaa,
4159		else => unreachable,
	4062	if (!nav_ty.isRuntimeFnOrHasRuntimeBits(zcu) and nav.getExtern(ip) == null) {
	4063	const nav_align = switch (nav.resolved.?.@"align") {
	4064	.none => nav_ty.abiAlignment(zcu),
	4065	else => \|a\| a,
4160	4066	};
4161		const llvm_usize = try o.lowerType(pt, Type.usize);
4162		const llvm_ptr_ty = try o.lowerType(pt, ptr_ty);
4163		return o.builder.castConst(.inttoptr, try o.builder.intConst(llvm_usize, int), llvm_ptr_ty);
4164		}
4165
4166		/// If the operand type of an atomic operation is not byte sized we need to
4167		/// widen it before using it and then truncate the result.
4168		/// RMW exchange of floating-point values is bitcasted to same-sized integer
4169		/// types to work around a LLVM deficiency when targeting ARM/AArch64.
4170		fn getAtomicAbiType(o: *Object, pt: Zcu.PerThread, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
4171		const zcu = pt.zcu;
4172		switch (ty.zigTypeTag(zcu)) {
4173		.int, .@"enum", .@"struct", .@"union" => {},
4174		.float => {
4175		if (!is_rmw_xchg) return .none;
4176		return o.builder.intType(@intCast(ty.abiSize(zcu) * 8));
4177		},
4178		.bool => return .i8,
4179		else => return .none,
4180		}
4181		const bit_count = ty.bitSize(zcu);
4182		if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
4183		return o.builder.intType(@intCast(ty.abiSize(zcu) * 8));
4184		} else {
4185		return .none;
	4067	return o.lowerPtrToVoid(nav_align, nav.resolved.?.@"addrspace");
	4068	}
	4069
	4070	const gop = try o.nav_map.getOrPut(gpa, nav_id);
	4071	if (!gop.found_existing) {
	4072	errdefer assert(o.nav_map.remove(nav_id));
	4073	// The NAV hasn't been lowered yet, so generate a placeholder global whose details will
	4074	// be filled in later.
	4075	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
	4076	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
	4077	.type = .void, // placeholder; populated by `updateNav`/`updateFunc`
	4078	.kind = .{ .alias = .none }, // placeholder; populated by `updateNav`/`updateFunc`
	4079	});
4186	4080	}
	4081	const llvm_global = gop.value_ptr.*;
	4082
	4083	// We need to make sure the global's address space is up to date, because that affects the
	4084	// type of a pointer to this global. But everything else about the global will be populated
	4085	// by `updateNav` or `updateFunc`.
	4086	llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget());
	4087	return llvm_global.toConst();
4187	4088	}
4188	4089
4189		fn addByValParamAttrs(
	4090	pub fn addByValParamAttrs(
4190	4091	o: *Object,
4191	4092	pt: Zcu.PerThread,
4192	4093	attributes: *Builder.FunctionAttributes.Wip,
...	...	@@ -4195,10 +4096,10 @@ pub const Object = struct {
4195	4096	fn_info: InternPool.Key.FuncType,
4196	4097	llvm_arg_i: u32,
4197	4098	) Allocator.Error!void {
4198		const zcu = pt.zcu;
	4099	const zcu = o.zcu;
4199	4100	if (param_ty.isPtrAtRuntime(zcu)) {
4200	4101	const ptr_info = param_ty.ptrInfo(zcu);
4201		if (math.cast(u5, param_index)) \|i\| {
	4102	if (std.math.cast(u5, param_index)) \|i\| {
4202	4103	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
4203	4104	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
4204	4105	}
...	...	@@ -4214,7 +4115,7 @@ pub const Object = struct {
4214	4115	.x86_64_interrupt,
4215	4116	.x86_interrupt,
4216	4117	=> {
4217		const child_type = try lowerType(o, pt, Type.fromInterned(ptr_info.child));
	4118	const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));
4218	4119	try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4219	4120	},
4220	4121	}
...	...	@@ -4232,7 +4133,7 @@ pub const Object = struct {
4232	4133	};
4233	4134	}
4234	4135
4235		fn addByRefParamAttrs(
	4136	pub fn addByRefParamAttrs(
4236	4137	o: *Object,
4237	4138	attributes: *Builder.FunctionAttributes.Wip,
4238	4139	llvm_arg_i: u32,
...	...	@@ -4246,52 +4147,74 @@ pub const Object = struct {
4246	4147	if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
4247	4148	}
4248	4149
4249		fn llvmFieldIndex(o: *Object, struct_ty: Type, field_index: usize) ?c_uint {
4250		return o.struct_field_map.get(.{
4251		.struct_ty = struct_ty.toIntern(),
4252		.field_index = @intCast(field_index),
4253		});
4254		}
	4150	pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
	4151	if (o.error_name_table != .none) return o.error_name_table;
4255	4152
4256		fn getCmpLtErrorsLenFunction(o: *Object, pt: Zcu.PerThread) !Builder.Function.Index {
4257		const name = try o.builder.strtabString(lt_errors_fn_name);
4258		if (o.builder.getGlobal(name)) \|llvm_fn\| return llvm_fn.ptrConst(&o.builder).kind.function;
4259
4260		const zcu = pt.zcu;
4261		const target = &zcu.root_mod.resolved_target.result;
4262		const function_index = try o.builder.addFunction(
4263		try o.builder.fnType(.i1, &.{try o.errorIntType(pt)}, .normal),
4264		name,
4265		toLlvmAddressSpace(.generic, target),
	4153	const name = try o.builder.strtabString("__zig_error_name_table");
	4154	// TODO: Address space
	4155	const variable_index = try o.builder.addVariable(name, .ptr, .default);
	4156	variable_index.setMutability(.constant, &o.builder);
	4157	variable_index.setAlignment(
	4158	Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(),
	4159	&o.builder,
4266	4160	);
	4161	const global_index = variable_index.ptrConst(&o.builder).global;
	4162	global_index.setLinkage(.private, &o.builder);
	4163	global_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4267	4164
4268		var attributes: Builder.FunctionAttributes.Wip = .{};
4269		defer attributes.deinit(&o.builder);
4270		try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
	4165	o.error_name_table = variable_index;
	4166	return variable_index;
	4167	}
4271	4168
4272		function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
4273		function_index.setCallConv(.fastcc, &o.builder);
4274		function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4275		return function_index;
	4169	pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
	4170	const builder = &o.builder;
	4171	if (o.errors_len_variable == .none) {
	4172	const llvm_err_int_ty = try o.errorIntType();
	4173	const name = try builder.strtabString("__zig_errors_len");
	4174	const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
	4175	variable_index.setMutability(.constant, builder);
	4176	variable_index.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder);
	4177	const global_index = variable_index.ptrConst(&o.builder).global;
	4178	global_index.setLinkage(.private, builder);
	4179	global_index.setUnnamedAddr(.unnamed_addr, builder);
	4180	o.errors_len_variable = variable_index;
	4181	}
	4182	return o.errors_len_variable;
4276	4183	}
4277	4184
4278		fn getEnumTagNameFunction(o: *Object, pt: Zcu.PerThread, enum_ty: Type) !Builder.Function.Index {
4279		const zcu = pt.zcu;
	4185	pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index {
	4186	const zcu = o.zcu;
4280	4187	const ip = &zcu.intern_pool;
4281		const enum_type = ip.loadEnumType(enum_ty.toIntern());
4282	4188
4283	4189	const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern());
4284		if (gop.found_existing) return gop.value_ptr.ptrConst(&o.builder).kind.function;
	4190	if (gop.found_existing) return gop.value_ptr.*;
4285	4191	errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern()));
4286
4287		const usize_ty = try o.lowerType(pt, Type.usize);
4288		const ret_ty = try o.lowerType(pt, Type.slice_const_u8_sentinel_0);
4289		const target = &zcu.root_mod.resolved_target.result;
4290	4192	const function_index = try o.builder.addFunction(
4291		try o.builder.fnType(ret_ty, &.{try o.lowerType(pt, Type.fromInterned(enum_type.int_tag_type))}, .normal),
4292		try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_type.name.fmt(ip)}),
4293		toLlvmAddressSpace(.generic, target),
	4193	// Dummy function type; `updateEnumTagNameFunction` will replace it with the correct type.
	4194	// TODO: change the builder API so we don't need to do this.
	4195	try o.builder.fnType(.void, &.{}, .normal),
	4196	try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
	4197	toLlvmAddressSpace(.generic, zcu.getTarget()),
4294	4198	);
	4199	gop.value_ptr.* = function_index;
	4200	try o.updateEnumTagNameFunction(enum_ty, function_index);
	4201	return function_index;
	4202	}
	4203	fn updateEnumTagNameFunction(
	4204	o: *Object,
	4205	enum_ty: Type,
	4206	function_index: Builder.Function.Index,
	4207	) Allocator.Error!void {
	4208	const zcu = o.zcu;
	4209	const ip = &zcu.intern_pool;
	4210	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
	4211
	4212	const llvm_usize_ty = try o.lowerType(.usize);
	4213	const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);
	4214	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4215
	4216	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
	4217	try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
4295	4218
4296	4219	var attributes: Builder.FunctionAttributes.Wip = .{};
4297	4220	defer attributes.deinit(&o.builder);
...	...	@@ -4300,7 +4223,6 @@ pub const Object = struct {
4300	4223	function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
4301	4224	function_index.setCallConv(.fastcc, &o.builder);
4302	4225	function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4303		gop.value_ptr.* = function_index.ptrConst(&o.builder).global;
4304	4226
4305	4227	var wip = try Builder.WipFunction.init(&o.builder, .{
4306	4228	.function = function_index,
...	...	@@ -4314,33 +4236,33 @@ pub const Object = struct {
4314	4236	var wip_switch = try wip.@"switch"(
4315	4237	tag_int_value,
4316	4238	bad_value_block,
4317		@intCast(enum_type.field_names.len),
	4239	@intCast(loaded_enum.field_names.len),
4318	4240	.none,
4319	4241	);
4320	4242	defer wip_switch.finish(&wip);
4321	4243
4322		for (0..enum_type.field_names.len) \|field_index\| {
4323		const name = try o.builder.stringNull(enum_type.field_names.get(ip)[field_index].toSlice(ip));
	4244	for (0..loaded_enum.field_names.len) \|field_index\| {
	4245	const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip));
4324	4246	const name_init = try o.builder.stringConst(name);
4325		const name_variable_index =
4326		try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
	4247	const name_variable_index = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
4327	4248	try name_variable_index.setInitializer(name_init, &o.builder);
4328		name_variable_index.setLinkage(.private, &o.builder);
4329	4249	name_variable_index.setMutability(.constant, &o.builder);
4330		name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4331	4250	name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder);
	4251	const name_global_index = name_variable_index.ptrConst(&o.builder).global;
	4252	name_global_index.setLinkage(.private, &o.builder);
	4253	name_global_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4332	4254
4333		const name_val = try o.builder.structValue(ret_ty, &.{
4334		name_variable_index.toConst(&o.builder),
4335		try o.builder.intConst(usize_ty, name.slice(&o.builder).?.len - 1),
	4255	const name_val = try o.builder.structValue(llvm_ret_ty, &.{
	4256	name_global_index.toConst(),
	4257	try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1),
4336	4258	});
4337	4259
4338	4260	const return_block = try wip.block(1, "Name");
4339		const this_tag_int_value = try o.lowerValue(
4340		pt,
4341		(try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(),
4342		);
4343		try wip_switch.addCase(this_tag_int_value, return_block, &wip);
	4261	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
	4262	.none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
	4263	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	4264	};
	4265	try wip_switch.addCase(llvm_tag_val, return_block, &wip);
4344	4266
4345	4267	wip.cursor = .{ .block = return_block };
4346	4268	_ = try wip.ret(name_val);
...	...	@@ -4350,38 +4272,53 @@ pub const Object = struct {
4350	4272	_ = try wip.@"unreachable"();
4351	4273
4352	4274	try wip.finish();
4353		return function_index;
4354	4275	}
4355	4276
4356		fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy {
	4277	pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy {
4357	4278	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
4358	4279	return o.lazy_abi_aligns.items[@intFromEnum(index)];
4359	4280	}
4360	4281
	4282	pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index {
	4283	const zcu = o.zcu;
	4284	const ip = &zcu.intern_pool;
	4285
	4286	const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
	4287	if (gop.found_existing) return gop.value_ptr.*;
	4288	errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
	4289	const function_index = try o.builder.addFunction(
	4290	// Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type.
	4291	// TODO: change the builder API so we don't need to do this.
	4292	try o.builder.fnType(.void, &.{}, .normal),
	4293	try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
	4294	toLlvmAddressSpace(.generic, zcu.getTarget()),
	4295	);
	4296	gop.value_ptr.* = function_index;
	4297	try o.updateIsNamedEnumValueFunction(enum_ty, function_index);
	4298	return function_index;
	4299	}
4361	4300	fn updateIsNamedEnumValueFunction(
4362	4301	o: *Object,
4363		pt: Zcu.PerThread,
4364	4302	enum_ty: Type,
4365	4303	function_index: Builder.Function.Index,
4366	4304	) Allocator.Error!void {
4367		const zcu = pt.zcu;
4368		const builder = &o.builder;
4369		const loaded_enum = zcu.intern_pool.loadEnumType(enum_ty.toIntern());
4370		function_index.ptrConst(builder).global.ptr(builder).type = try builder.fnType(
4371		.i1,
4372		&.{try o.lowerType(pt, .fromInterned(loaded_enum.int_tag_type))},
4373		.normal,
4374		);
	4305	const zcu = o.zcu;
	4306	const ip = &zcu.intern_pool;
	4307	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
	4308
	4309	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4310	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
	4311	try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
4375	4312
4376	4313	var attributes: Builder.FunctionAttributes.Wip = .{};
4377		defer attributes.deinit(builder);
	4314	defer attributes.deinit(&o.builder);
4378	4315	try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
4379	4316
4380		function_index.setLinkage(if (o.builder.strip) .private else .internal, builder);
4381		function_index.setCallConv(.fastcc, builder);
4382		function_index.setAttributes(try attributes.finish(builder), builder);
	4317	function_index.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
	4318	function_index.setCallConv(.fastcc, &o.builder);
	4319	function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4383	4320
4384		var wip: Builder.WipFunction = try .init(builder, .{
	4321	var wip: Builder.WipFunction = try .init(&o.builder, .{
4385	4322	.function = function_index,
4386	4323	.strip = true,
4387	4324	});
...	...	@@ -4394,13 +4331,19 @@ pub const Object = struct {
4394	4331	var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(loaded_enum.field_names.len), .none);
4395	4332	defer wip_switch.finish(&wip);
4396	4333
4397		for (0..loaded_enum.field_names.len) \|field_index\| {
4398		const this_tag_int_value = try o.lowerValue(
4399		pt,
4400		(try pt.enumValueFieldIndex(enum_ty, @intCast(field_index))).toIntern(),
4401		);
4402		try wip_switch.addCase(this_tag_int_value, named_block, &wip);
	4334	if (loaded_enum.field_values.len > 0) {
	4335	for (loaded_enum.field_values.get(ip)) \|tag_val_ip\| {
	4336	const llvm_tag_val = try o.lowerValue(tag_val_ip);
	4337	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
	4338	}
	4339	} else {
	4340	// Auto-numbered.
	4341	for (0..loaded_enum.field_names.len) \|field_index\| {
	4342	const llvm_tag_val = try o.builder.intConst(llvm_int_ty, field_index);
	4343	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
	4344	}
4403	4345	}
	4346
4404	4347	wip.cursor = .{ .block = named_block };
4405	4348	_ = try wip.ret(.true);
4406	4349
...	...	@@ -4409,8167 +4352,306 @@ pub const Object = struct {
4409	4352
4410	4353	try wip.finish();
4411	4354	}
4412		};
4413
4414		pub const NavGen = struct {
4415		object: *Object,
4416		nav_index: InternPool.Nav.Index,
4417		pt: Zcu.PerThread,
4418		err_msg: ?*Zcu.ErrorMsg,
4419
4420		fn ownerModule(ng: NavGen) *Package.Module {
4421		return ng.pt.zcu.navFileScope(ng.nav_index).mod.?;
4422		}
4423
4424		fn todo(ng: *NavGen, comptime format: []const u8, args: anytype) Error {
4425		@branchHint(.cold);
4426		assert(ng.err_msg == null);
4427		const o = ng.object;
4428		const gpa = o.gpa;
4429		const src_loc = ng.pt.zcu.navSrcLoc(ng.nav_index);
4430		ng.err_msg = try Zcu.ErrorMsg.create(gpa, src_loc, "TODO (LLVM): " ++ format, args);
4431		return error.CodegenFail;
4432		}
4433
4434		fn genDecl(ng: *NavGen) !void {
4435		const o = ng.object;
4436		const pt = ng.pt;
4437		const zcu = pt.zcu;
4438		const ip = &zcu.intern_pool;
4439		const nav_index = ng.nav_index;
4440		const nav = ip.getNav(nav_index);
4441		const resolved = nav.resolved.?;
4442	4355
4443		const lib_name, const linkage, const visibility: Builder.Visibility, const is_dll_import, const init_val, const owner_nav = switch (ip.indexToKey(resolved.value)) {
4444		else => .{ .none, .internal, .default, false, resolved.value, nav_index },
4445		.@"extern" => \|e\| .{ e.lib_name, e.linkage, .fromSymbolVisibility(e.visibility), e.is_dll_import, .none, e.owner_nav },
	4356	pub fn getLibcFunction(
	4357	o: *Object,
	4358	fn_name: Builder.StrtabString,
	4359	param_types: []const Builder.Type,
	4360	return_type: Builder.Type,
	4361	) Allocator.Error!Builder.Function.Index {
	4362	if (o.builder.getGlobal(fn_name)) \|global\| return switch (global.ptrConst(&o.builder).kind) {
	4363	.alias => \|alias\| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
	4364	.function => \|function\| function,
	4365	.variable, .replaced => unreachable,
4446	4366	};
4447		const ty: Type = .fromInterned(nav.resolved.?.type);
4448
4449		if (linkage != .internal and ip.isFunctionType(ty.toIntern())) {
4450		const function_index = try o.resolveLlvmFunction(pt, owner_nav);
4451		// Add parameter attributes which weren't set by `resolveLlvmFunction`
4452		const fn_info = zcu.typeToFunc(ty).?;
4453		var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder);
4454		defer attributes.deinit(&o.builder);
4455		var it = iterateParamTypes(o, pt, fn_info);
4456		if (firstParamSRet(fn_info, zcu, zcu.getTarget())) it.llvm_index += 1;
4457		if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) it.llvm_index += 1;
4458		while (try it.next()) \|lowering\| switch (lowering) {
4459		.byval => {
4460		const param_index = it.zig_index - 1;
4461		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
4462		if (!isByRef(param_ty, zcu)) {
4463		try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
4464		}
4465		},
4466		.byref => {
4467		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
4468		const param_llvm_ty = try o.lowerType(pt, param_ty);
4469		const alignment = param_ty.abiAlignment(zcu);
4470		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
4471		},
4472		.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
4473		// No attributes needed for these.
4474		.no_bits,
4475		.abi_sized_int,
4476		.multiple_llvm_types,
4477		.float_array,
4478		.i32_array,
4479		.i64_array,
4480		=> continue,
4481
4482		.slice => unreachable, // extern functions do not support slice types.
4483		};
4484		function_index.setAttributes(try attributes.finish(&o.builder), &o.builder);
4485		} else {
4486		const variable_index = try o.resolveGlobalNav(pt, nav_index);
4487		variable_index.setAlignment(zcu.navAlignment(nav_index).toLlvm(), &o.builder);
4488		if (resolved.@"linksection".toSlice(ip)) \|section\|
4489		variable_index.setSection(try o.builder.string(section), &o.builder);
4490		if (resolved.@"const") variable_index.setMutability(.constant, &o.builder);
4491		try variable_index.setInitializer(switch (init_val) {
4492		.none => .no_init,
4493		else => try o.lowerValue(pt, init_val),
4494		}, &o.builder);
4495		variable_index.setVisibility(visibility, &o.builder);
4496
4497		const file_scope = zcu.navFileScopeIndex(nav_index);
4498		const mod = zcu.fileByIndex(file_scope).mod.?;
4499		if (resolved.@"threadlocal" and !mod.single_threaded)
4500		variable_index.setThreadLocal(.generaldynamic, &o.builder);
	4367	return o.builder.addFunction(
	4368	try o.builder.fnType(return_type, param_types, .normal),
	4369	fn_name,
	4370	toLlvmAddressSpace(.generic, o.zcu.getTarget()),
	4371	);
	4372	}
	4373	};
4501	4374
4502		const line_number = zcu.navSrcLine(nav_index) + 1;
	4375	const CallingConventionInfo = struct {
	4376	/// The LLVM calling convention to use.
	4377	llvm_cc: Builder.CallConv,
	4378	/// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
	4379	align_stack: bool,
	4380	/// Whether the function needs a `naked` attribute.
	4381	naked: bool,
	4382	/// How many leading parameters to apply the `inreg` attribute to.
	4383	inreg_param_count: u2 = 0,
	4384	};
4503	4385
4504		if (!mod.strip) {
4505		const debug_file = try o.getDebugFile(pt, file_scope);
4506
4507		const debug_global_var = try o.builder.debugGlobalVar(
4508		try o.builder.metadataString(nav.name.toSlice(ip)), // Name
4509		try o.builder.metadataStringFromStrtabString(variable_index.name(&o.builder)), // Linkage name
4510		debug_file, // File
4511		debug_file, // Scope
4512		line_number,
4513		try o.getDebugType(pt, ty),
4514		variable_index,
4515		.{ .local = linkage == .internal },
4516		);
4517
4518		const debug_expression = try o.builder.debugExpression(&.{});
4519
4520		const debug_global_var_expression = try o.builder.debugGlobalVarExpression(
4521		debug_global_var,
4522		debug_expression,
4523		);
4524
4525		variable_index.setGlobalVariableExpression(debug_global_var_expression, &o.builder);
4526		try o.debug_globals.append(o.gpa, debug_global_var_expression);
4527		}
4528		}
4529
4530		switch (linkage) {
4531		.internal => {},
4532		.strong, .weak => {
4533		const global_index = o.nav_map.get(nav_index).?;
4534
4535		const decl_name = decl_name: {
4536		if (zcu.getTarget().cpu.arch.isWasm() and ty.zigTypeTag(zcu) == .@"fn") {
4537		if (lib_name.toSlice(ip)) \|lib_name_slice\| {
4538		if (!std.mem.eql(u8, lib_name_slice, "c")) {
4539		break :decl_name try o.builder.strtabStringFmt("{f}\|{s}", .{ nav.name.fmt(ip), lib_name_slice });
4540		}
4541		}
4542		}
4543		break :decl_name try o.builder.strtabString(nav.name.toSlice(ip));
4544		};
4545
4546		if (o.builder.getGlobal(decl_name)) \|other_global\| {
4547		if (other_global != global_index) {
4548		// Another global already has this name; just use it in place of this global.
4549		try global_index.replace(other_global, &o.builder);
4550		return;
4551		}
4552		}
4553
4554		try global_index.rename(decl_name, &o.builder);
4555		global_index.setUnnamedAddr(.default, &o.builder);
4556		if (is_dll_import) {
4557		global_index.setDllStorageClass(.dllimport, &o.builder);
4558		} else if (zcu.comp.config.dll_export_fns) {
4559		global_index.setDllStorageClass(.default, &o.builder);
4560		}
4561
4562		global_index.setLinkage(switch (linkage) {
4563		.internal => unreachable,
4564		.strong => .external,
4565		.weak => .extern_weak,
4566		.link_once => unreachable,
4567		}, &o.builder);
4568		global_index.setVisibility(visibility, &o.builder);
4569		},
4570		.link_once => unreachable,
4571		}
4572		}
4573		};
4574
4575		pub const FuncGen = struct {
4576		gpa: Allocator,
4577		ng: *NavGen,
4578		air: Air,
4579		liveness: Air.Liveness,
4580		wip: Builder.WipFunction,
4581		is_naked: bool,
4582		fuzz: ?Fuzz,
4583
4584		file: Builder.Metadata,
4585		scope: Builder.Metadata,
4586
4587		inlined_at: Builder.Metadata.Optional = .none,
4588
4589		base_line: u32,
4590		prev_dbg_line: c_uint,
4591		prev_dbg_column: c_uint,
4592
4593		/// This stores the LLVM values used in a function, such that they can be referred to
4594		/// in other instructions. This table is cleared before every function is generated.
4595		func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
4596
4597		/// If the return type is sret, this is the result pointer. Otherwise null.
4598		/// Note that this can disagree with isByRef for the return type in the case
4599		/// of C ABI functions.
4600		ret_ptr: Builder.Value,
4601		/// Any function that needs to perform Valgrind client requests needs an array alloca
4602		/// instruction, however a maximum of one per function is needed.
4603		valgrind_client_request_array: Builder.Value = .none,
4604		/// These fields are used to refer to the LLVM value of the function parameters
4605		/// in an Arg instruction.
4606		/// This list may be shorter than the list according to the zig type system;
4607		/// it omits 0-bit types. If the function uses sret as the first parameter,
4608		/// this slice does not include it.
4609		args: []const Builder.Value,
4610		arg_index: u32,
4611		arg_inline_index: u32,
4612
4613		err_ret_trace: Builder.Value = .none,
4614
4615		/// This data structure is used to implement breaking to blocks.
4616		blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
4617		parent_bb: Builder.Function.Block.Index,
4618		breaks: *BreakList,
4619		}),
4620
4621		/// Maps `loop` instructions to the bb to branch to to repeat the loop.
4622		loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
4623
4624		/// Maps `loop_switch_br` instructions to the information required to lower
4625		/// dispatches (`switch_dispatch` instructions).
4626		switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
4627
4628		sync_scope: Builder.SyncScope,
4629
4630		disable_intrinsics: bool,
4631
4632		/// Have we seen loads or stores involving `allowzero` pointers?
4633		allowzero_access: bool = false,
4634
4635		fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
4636		// LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
4637		// pessimizing optimization for functions with accesses to such pointers.
4638		if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
4639		}
4640
4641		const Fuzz = struct {
4642		counters_variable: Builder.Variable.Index,
4643		pcs: std.ArrayList(Builder.Constant),
4644
4645		fn deinit(f: *Fuzz, gpa: Allocator) void {
4646		f.pcs.deinit(gpa);
4647		f.* = undefined;
4648		}
4649		};
4650
4651		const SwitchDispatchInfo = struct {
4652		/// These are the blocks corresponding to each switch case.
4653		/// The final element corresponds to the `else` case.
4654		/// Slices allocated into `gpa`.
4655		case_blocks: []Builder.Function.Block.Index,
4656		/// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
4657		switch_weights: Builder.Function.Instruction.BrCond.Weights,
4658		/// If not `null`, we have manually constructed a jump table to reach the desired block.
4659		/// `table` can be used if the value is between `min` and `max` inclusive.
4660		/// We perform this lowering manually to avoid some questionable behavior from LLVM.
4661		/// See `airSwitchBr` for details.
4662		jmp_table: ?JmpTable,
4663
4664		const JmpTable = struct {
4665		min: Builder.Constant,
4666		max: Builder.Constant,
4667		in_bounds_hint: enum { none, unpredictable, likely, unlikely },
4668		/// Pointer to the jump table itself, to be used with `indirectbr`.
4669		/// The index into the jump table is the dispatch condition minus `min`.
4670		/// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
4671		table: Builder.Constant,
4672		/// `true` if `table` conatins a reference to the `else` block.
4673		/// In this case, the `indirectbr` must include the `else` block in its target list.
4674		table_includes_else: bool,
4675		};
4676		};
4677
4678		const BreakList = union {
4679		list: std.MultiArrayList(struct {
4680		bb: Builder.Function.Block.Index,
4681		val: Builder.Value,
4682		}),
4683		len: usize,
4684		};
4685
4686		fn deinit(self: *FuncGen) void {
4687		const gpa = self.gpa;
4688		if (self.fuzz) \|*f\| f.deinit(self.gpa);
4689		self.wip.deinit();
4690		self.func_inst_table.deinit(gpa);
4691		self.blocks.deinit(gpa);
4692		self.loops.deinit(gpa);
4693		var it = self.switch_dispatch_info.valueIterator();
4694		while (it.next()) \|info\| {
4695		self.gpa.free(info.case_blocks);
4696		}
4697		self.switch_dispatch_info.deinit(gpa);
4698		}
4699
4700		fn todo(self: *FuncGen, comptime format: []const u8, args: anytype) Error {
4701		@branchHint(.cold);
4702		return self.ng.todo(format, args);
4703		}
4704
4705		fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) !Builder.Value {
4706		const gpa = self.gpa;
4707		const gop = try self.func_inst_table.getOrPut(gpa, inst);
4708		if (gop.found_existing) return gop.value_ptr.*;
4709
4710		const llvm_val = try self.resolveValue((try self.air.value(inst, self.ng.pt)).?);
4711		gop.value_ptr.* = llvm_val.toValue();
4712		return llvm_val.toValue();
4713		}
4714
4715		fn resolveValue(self: *FuncGen, val: Value) Error!Builder.Constant {
4716		const o = self.ng.object;
4717		const pt = self.ng.pt;
4718		const zcu = pt.zcu;
4719		const ty = val.typeOf(zcu);
4720		const llvm_val = try o.lowerValue(pt, val.toIntern());
4721		if (!isByRef(ty, zcu)) return llvm_val;
4722
4723		// We have an LLVM value but we need to create a global constant and
4724		// set the value as its initializer, and then return a pointer to the global.
4725		const target = zcu.getTarget();
4726		const variable_index = try o.builder.addVariable(
4727		.empty,
4728		llvm_val.typeOf(&o.builder),
4729		toLlvmGlobalAddressSpace(.generic, target),
4730		);
4731		try variable_index.setInitializer(llvm_val, &o.builder);
4732		variable_index.setLinkage(.private, &o.builder);
4733		variable_index.setMutability(.constant, &o.builder);
4734		variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
4735		variable_index.setAlignment(ty.abiAlignment(zcu).toLlvm(), &o.builder);
4736		return o.builder.convConst(
4737		variable_index.toConst(&o.builder),
4738		try o.builder.ptrType(toLlvmAddressSpace(.generic, target)),
4739		);
4740		}
4741
4742		fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) Error!void {
4743		const o = self.ng.object;
4744		const zcu = self.ng.pt.zcu;
4745		const ip = &zcu.intern_pool;
4746		const air_tags = self.air.instructions.items(.tag);
4747		switch (coverage_point) {
4748		.none => {},
4749		.poi => if (self.fuzz) \|*fuzz\| {
4750		const poi_index = fuzz.pcs.items.len;
4751		const base_ptr = fuzz.counters_variable.toValue(&o.builder);
4752		const ptr = if (poi_index == 0) base_ptr else try self.wip.gep(.inbounds, .i8, base_ptr, &.{
4753		try o.builder.intValue(.i32, poi_index),
4754		}, "");
4755		const one = try o.builder.intValue(.i8, 1);
4756		_ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
4757
4758		// LLVM does not allow blockaddress on the entry block.
4759		const pc = if (self.wip.cursor.block == .entry)
4760		self.wip.function.toConst(&o.builder)
4761		else
4762		try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
4763		const gpa = self.gpa;
4764		try fuzz.pcs.append(gpa, pc);
4765		},
4766		}
4767		for (body, 0..) \|inst, i\| {
4768		if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
4769
4770		const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
4771		// zig fmt: off
4772
4773		// No "scalarize" legalizations are enabled, so these instructions never appear.
4774		.legalize_vec_elem_val => unreachable,
4775		.legalize_vec_store_elem => unreachable,
4776		// No soft float legalizations are enabled.
4777		.legalize_compiler_rt_call => unreachable,
4778
4779		.add => try self.airAdd(inst, .normal),
4780		.add_optimized => try self.airAdd(inst, .fast),
4781		.add_wrap => try self.airAddWrap(inst),
4782		.add_sat => try self.airAddSat(inst),
4783
4784		.sub => try self.airSub(inst, .normal),
4785		.sub_optimized => try self.airSub(inst, .fast),
4786		.sub_wrap => try self.airSubWrap(inst),
4787		.sub_sat => try self.airSubSat(inst),
4788
4789		.mul => try self.airMul(inst, .normal),
4790		.mul_optimized => try self.airMul(inst, .fast),
4791		.mul_wrap => try self.airMulWrap(inst),
4792		.mul_sat => try self.airMulSat(inst),
4793
4794		.add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4795		.sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4796		.mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4797
4798		.div_float => try self.airDivFloat(inst, .normal),
4799		.div_trunc => try self.airDivTrunc(inst, .normal),
4800		.div_floor => try self.airDivFloor(inst, .normal),
4801		.div_exact => try self.airDivExact(inst, .normal),
4802		.rem => try self.airRem(inst, .normal),
4803		.mod => try self.airMod(inst, .normal),
4804		.abs => try self.airAbs(inst),
4805		.ptr_add => try self.airPtrAdd(inst),
4806		.ptr_sub => try self.airPtrSub(inst),
4807		.shl => try self.airShl(inst),
4808		.shl_sat => try self.airShlSat(inst),
4809		.shl_exact => try self.airShlExact(inst),
4810		.min => try self.airMin(inst),
4811		.max => try self.airMax(inst),
4812		.slice => try self.airSlice(inst),
4813		.mul_add => try self.airMulAdd(inst),
4814
4815		.div_float_optimized => try self.airDivFloat(inst, .fast),
4816		.div_trunc_optimized => try self.airDivTrunc(inst, .fast),
4817		.div_floor_optimized => try self.airDivFloor(inst, .fast),
4818		.div_exact_optimized => try self.airDivExact(inst, .fast),
4819		.rem_optimized => try self.airRem(inst, .fast),
4820		.mod_optimized => try self.airMod(inst, .fast),
4821
4822		.add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
4823		.sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
4824		.mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
4825		.shl_with_overflow => try self.airShlWithOverflow(inst),
4826
4827		.bit_and, .bool_and => try self.airAnd(inst),
4828		.bit_or, .bool_or => try self.airOr(inst),
4829		.xor => try self.airXor(inst),
4830		.shr => try self.airShr(inst, false),
4831		.shr_exact => try self.airShr(inst, true),
4832
4833		.sqrt => try self.airUnaryOp(inst, .sqrt),
4834		.sin => try self.airUnaryOp(inst, .sin),
4835		.cos => try self.airUnaryOp(inst, .cos),
4836		.tan => try self.airUnaryOp(inst, .tan),
4837		.exp => try self.airUnaryOp(inst, .exp),
4838		.exp2 => try self.airUnaryOp(inst, .exp2),
4839		.log => try self.airUnaryOp(inst, .log),
4840		.log2 => try self.airUnaryOp(inst, .log2),
4841		.log10 => try self.airUnaryOp(inst, .log10),
4842		.floor => try self.airUnaryOp(inst, .floor),
4843		.ceil => try self.airUnaryOp(inst, .ceil),
4844		.round => try self.airUnaryOp(inst, .round),
4845		.trunc_float => try self.airUnaryOp(inst, .trunc),
4846
4847		.neg => try self.airNeg(inst, .normal),
4848		.neg_optimized => try self.airNeg(inst, .fast),
4849
4850		.cmp_eq => try self.airCmp(inst, .eq, .normal),
4851		.cmp_gt => try self.airCmp(inst, .gt, .normal),
4852		.cmp_gte => try self.airCmp(inst, .gte, .normal),
4853		.cmp_lt => try self.airCmp(inst, .lt, .normal),
4854		.cmp_lte => try self.airCmp(inst, .lte, .normal),
4855		.cmp_neq => try self.airCmp(inst, .neq, .normal),
4856
4857		.cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
4858		.cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
4859		.cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
4860		.cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
4861		.cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
4862		.cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
4863
4864		.cmp_vector => try self.airCmpVector(inst, .normal),
4865		.cmp_vector_optimized => try self.airCmpVector(inst, .fast),
4866		.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
4867
4868		.is_non_null => try self.airIsNonNull(inst, false, .ne),
4869		.is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
4870		.is_null => try self.airIsNonNull(inst, false, .eq),
4871		.is_null_ptr => try self.airIsNonNull(inst, true , .eq),
4872
4873		.is_non_err => try self.airIsErr(inst, .eq, false),
4874		.is_non_err_ptr => try self.airIsErr(inst, .eq, true),
4875		.is_err => try self.airIsErr(inst, .ne, false),
4876		.is_err_ptr => try self.airIsErr(inst, .ne, true),
4877
4878		.alloc => try self.airAlloc(inst),
4879		.ret_ptr => try self.airRetPtr(inst),
4880		.arg => try self.airArg(inst),
4881		.bitcast => try self.airBitCast(inst),
4882		.breakpoint => try self.airBreakpoint(inst),
4883		.ret_addr => try self.airRetAddr(inst),
4884		.frame_addr => try self.airFrameAddress(inst),
4885		.@"try" => try self.airTry(inst, false),
4886		.try_cold => try self.airTry(inst, true),
4887		.try_ptr => try self.airTryPtr(inst, false),
4888		.try_ptr_cold => try self.airTryPtr(inst, true),
4889		.intcast => try self.airIntCast(inst, false),
4890		.intcast_safe => try self.airIntCast(inst, true),
4891		.trunc => try self.airTrunc(inst),
4892		.fptrunc => try self.airFptrunc(inst),
4893		.fpext => try self.airFpext(inst),
4894		.load => try self.airLoad(inst),
4895		.not => try self.airNot(inst),
4896		.store => try self.airStore(inst, false),
4897		.store_safe => try self.airStore(inst, true),
4898		.assembly => try self.airAssembly(inst),
4899		.slice_ptr => try self.airSliceField(inst, 0),
4900		.slice_len => try self.airSliceField(inst, 1),
4901
4902		.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
4903		.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
4904
4905		.int_from_float => try self.airIntFromFloat(inst, .normal),
4906		.int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
4907		.int_from_float_safe => unreachable, // handled by `legalizeFeatures`
4908		.int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
4909
4910		.array_to_slice => try self.airArrayToSlice(inst),
4911		.float_from_int => try self.airFloatFromInt(inst),
4912		.cmpxchg_weak => try self.airCmpxchg(inst, .weak),
4913		.cmpxchg_strong => try self.airCmpxchg(inst, .strong),
4914		.atomic_rmw => try self.airAtomicRmw(inst),
4915		.atomic_load => try self.airAtomicLoad(inst),
4916		.memset => try self.airMemset(inst, false),
4917		.memset_safe => try self.airMemset(inst, true),
4918		.memcpy => try self.airMemcpy(inst),
4919		.memmove => try self.airMemmove(inst),
4920		.set_union_tag => try self.airSetUnionTag(inst),
4921		.get_union_tag => try self.airGetUnionTag(inst),
4922		.clz => try self.airClzCtz(inst, .ctlz),
4923		.ctz => try self.airClzCtz(inst, .cttz),
4924		.popcount => try self.airBitOp(inst, .ctpop),
4925		.byte_swap => try self.airByteSwap(inst),
4926		.bit_reverse => try self.airBitOp(inst, .bitreverse),
4927		.tag_name => try self.airTagName(inst),
4928		.error_name => try self.airErrorName(inst),
4929		.splat => try self.airSplat(inst),
4930		.select => try self.airSelect(inst),
4931		.shuffle_one => try self.airShuffleOne(inst),
4932		.shuffle_two => try self.airShuffleTwo(inst),
4933		.aggregate_init => try self.airAggregateInit(inst),
4934		.union_init => try self.airUnionInit(inst),
4935		.prefetch => try self.airPrefetch(inst),
4936		.addrspace_cast => try self.airAddrSpaceCast(inst),
4937
4938		.is_named_enum_value => try self.airIsNamedEnumValue(inst),
4939		.error_set_has_value => try self.airErrorSetHasValue(inst),
4940
4941		.reduce => try self.airReduce(inst, .normal),
4942		.reduce_optimized => try self.airReduce(inst, .fast),
4943
4944		.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
4945		.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
4946		.atomic_store_release => try self.airAtomicStore(inst, .release),
4947		.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
4948
4949		.struct_field_ptr => try self.airStructFieldPtr(inst),
4950		.struct_field_val => try self.airStructFieldVal(inst),
4951
4952		.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
4953		.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
4954		.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
4955		.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
4956
4957		.field_parent_ptr => try self.airFieldParentPtr(inst),
4958
4959		.array_elem_val => try self.airArrayElemVal(inst),
4960		.slice_elem_val => try self.airSliceElemVal(inst),
4961		.slice_elem_ptr => try self.airSliceElemPtr(inst),
4962		.ptr_elem_val => try self.airPtrElemVal(inst),
4963		.ptr_elem_ptr => try self.airPtrElemPtr(inst),
4964
4965		.optional_payload => try self.airOptionalPayload(inst),
4966		.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
4967		.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
4968
4969		.unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
4970		.unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
4971		.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
4972		.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
4973		.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
4974		.err_return_trace => try self.airErrReturnTrace(inst),
4975		.set_err_return_trace => try self.airSetErrReturnTrace(inst),
4976		.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
4977
4978		.wrap_optional => try self.airWrapOptional(body[i..]),
4979		.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
4980		.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
4981
4982		.wasm_memory_size => try self.airWasmMemorySize(inst),
4983		.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
4984
4985		.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
4986
4987		.inferred_alloc, .inferred_alloc_comptime => unreachable,
4988
4989		.dbg_stmt => try self.airDbgStmt(inst),
4990		.dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
4991		.dbg_var_ptr => try self.airDbgVarPtr(inst),
4992		.dbg_var_val => try self.airDbgVarVal(inst, false),
4993		.dbg_arg_inline => try self.airDbgVarVal(inst, true),
4994
4995		.c_va_arg => try self.airCVaArg(inst),
4996		.c_va_copy => try self.airCVaCopy(inst),
4997		.c_va_end => try self.airCVaEnd(inst),
4998		.c_va_start => try self.airCVaStart(inst),
4999
5000		.work_item_id => try self.airWorkItemId(inst),
5001		.work_group_size => try self.airWorkGroupSize(inst),
5002		.work_group_id => try self.airWorkGroupId(inst),
5003
5004		// Instructions that are known to always be `noreturn` based on their tag.
5005		.br => return self.airBr(inst),
5006		.repeat => return self.airRepeat(inst),
5007		.switch_dispatch => return self.airSwitchDispatch(inst),
5008		.cond_br => return self.airCondBr(inst),
5009		.switch_br => return self.airSwitchBr(inst, false),
5010		.loop_switch_br => return self.airSwitchBr(inst, true),
5011		.loop => return self.airLoop(inst),
5012		.ret => return self.airRet(inst, false),
5013		.ret_safe => return self.airRet(inst, true),
5014		.ret_load => return self.airRetLoad(inst),
5015		.trap => return self.airTrap(inst),
5016		.unreach => return self.airUnreach(inst),
5017
5018		// Instructions which may be `noreturn`.
5019		.block => res: {
5020		const res = try self.airBlock(inst);
5021		if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5022		break :res res;
5023		},
5024		.dbg_inline_block => res: {
5025		const res = try self.airDbgInlineBlock(inst);
5026		if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
5027		break :res res;
5028		},
5029		.call, .call_always_tail, .call_never_tail, .call_never_inline => \|tag\| res: {
5030		const res = try self.airCall(inst, switch (tag) {
5031		.call => .auto,
5032		.call_always_tail => .always_tail,
5033		.call_never_tail => .never_tail,
5034		.call_never_inline => .never_inline,
5035		else => unreachable,
5036		});
5037		// TODO: the AIR we emit for calls is a bit weird - the instruction has
5038		// type `noreturn`, but there are instructions (and maybe a safety check) following
5039		// nonetheless. The `unreachable` or safety check should be emitted by backends instead.
5040		//if (self.typeOfIndex(inst).isNoReturn(mod)) return;
5041		break :res res;
5042		},
5043
5044		// zig fmt: on
5045		};
5046		if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
5047		}
5048		unreachable;
5049		}
5050
5051		fn genBodyDebugScope(
5052		self: *FuncGen,
5053		maybe_inline_func: ?InternPool.Index,
5054		body: []const Air.Inst.Index,
5055		coverage_point: Air.CoveragePoint,
5056		) Error!void {
5057		if (self.wip.strip) return self.genBody(body, coverage_point);
5058
5059		const old_debug_location = self.wip.debug_location;
5060		const old_file = self.file;
5061		const old_inlined_at = self.inlined_at;
5062		const old_base_line = self.base_line;
5063		defer if (maybe_inline_func) \|_\| {
5064		self.wip.debug_location = old_debug_location;
5065		self.file = old_file;
5066		self.inlined_at = old_inlined_at;
5067		self.base_line = old_base_line;
5068		};
5069
5070		const old_scope = self.scope;
5071		defer self.scope = old_scope;
5072
5073		if (maybe_inline_func) \|inline_func\| {
5074		const o = self.ng.object;
5075		const pt = self.ng.pt;
5076		const zcu = pt.zcu;
5077		const ip = &zcu.intern_pool;
5078
5079		const func = zcu.funcInfo(inline_func);
5080		const nav = ip.getNav(func.owner_nav);
5081		const file_scope = zcu.navFileScopeIndex(func.owner_nav);
5082		const mod = zcu.fileByIndex(file_scope).mod.?;
5083
5084		self.file = try o.getDebugFile(pt, file_scope);
5085
5086		self.base_line = zcu.navSrcLine(func.owner_nav);
5087		const line_number = self.base_line + 1;
5088		self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
5089
5090		const fn_ty = try pt.funcType(.{
5091		.param_types = &.{},
5092		.return_type = .void_type,
5093		});
5094
5095		self.scope = try o.builder.debugSubprogram(
5096		self.file,
5097		try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
5098		try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
5099		line_number,
5100		line_number + func.lbrace_line,
5101		try o.getDebugType(pt, fn_ty),
5102		.{
5103		.di_flags = .{ .StaticMember = true },
5104		.sp_flags = .{
5105		.Optimized = mod.optimize_mode != .Debug,
5106		.Definition = true,
5107		.LocalToUnit = true, // inline functions cannot be exported
5108		},
5109		},
5110		o.debug_compile_unit.unwrap().?,
5111		);
5112		}
5113
5114		self.scope = try self.ng.object.builder.debugLexicalBlock(
5115		self.scope,
5116		self.file,
5117		self.prev_dbg_line,
5118		self.prev_dbg_column,
5119		);
5120		self.wip.debug_location = .{ .location = .{
5121		.line = self.prev_dbg_line,
5122		.column = self.prev_dbg_column,
5123		.scope = self.scope.toOptional(),
5124		.inlined_at = self.inlined_at,
5125		} };
5126
5127		try self.genBody(body, coverage_point);
5128		}
5129
5130		pub const CallAttr = enum {
5131		Auto,
5132		NeverTail,
5133		NeverInline,
5134		AlwaysTail,
5135		AlwaysInline,
5136		};
5137
5138		fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) !Builder.Value {
5139		const air_call = self.air.unwrapCall(inst);
5140		const args = air_call.args;
5141		const o = self.ng.object;
5142		const pt = self.ng.pt;
5143		const zcu = pt.zcu;
5144		const ip = &zcu.intern_pool;
5145		const callee_ty = self.typeOf(air_call.callee);
5146		const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
5147		.@"fn" => callee_ty,
5148		.pointer => callee_ty.childType(zcu),
5149		else => unreachable,
5150		};
5151		const fn_info = zcu.typeToFunc(zig_fn_ty).?;
5152		const return_type = Type.fromInterned(fn_info.return_type);
5153		const llvm_fn = try self.resolveInst(air_call.callee);
5154		const target = zcu.getTarget();
5155		const sret = firstParamSRet(fn_info, zcu, target);
5156
5157		var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
5158		defer llvm_args.deinit();
5159
5160		var attributes: Builder.FunctionAttributes.Wip = .{};
5161		defer attributes.deinit(&o.builder);
5162
5163		if (self.disable_intrinsics) {
5164		try attributes.addFnAttr(.nobuiltin, &o.builder);
5165		}
5166
5167		switch (modifier) {
5168		.auto, .always_tail => {},
5169		.never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
5170		.no_suspend, .always_inline, .compile_time => unreachable,
5171		}
5172
5173		const ret_ptr = if (!sret) null else blk: {
5174		const llvm_ret_ty = try o.lowerType(pt, return_type);
5175		try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
5176
5177		const alignment = return_type.abiAlignment(zcu).toLlvm();
5178		const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
5179		try llvm_args.append(ret_ptr);
5180		break :blk ret_ptr;
5181		};
5182
5183		const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
5184		if (err_return_tracing) {
5185		assert(self.err_ret_trace != .none);
5186		try llvm_args.append(self.err_ret_trace);
5187		}
5188
5189		var it = iterateParamTypes(o, pt, fn_info);
5190		while (try it.nextCall(self, args)) \|lowering\| switch (lowering) {
5191		.no_bits => continue,
5192		.byval => {
5193		const arg = args[it.zig_index - 1];
5194		const param_ty = self.typeOf(arg);
5195		const llvm_arg = try self.resolveInst(arg);
5196		const llvm_param_ty = try o.lowerType(pt, param_ty);
5197		if (isByRef(param_ty, zcu)) {
5198		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5199		const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
5200		try llvm_args.append(loaded);
5201		} else {
5202		try llvm_args.append(llvm_arg);
5203		}
5204		},
5205		.byref => {
5206		const arg = args[it.zig_index - 1];
5207		const param_ty = self.typeOf(arg);
5208		const llvm_arg = try self.resolveInst(arg);
5209		if (isByRef(param_ty, zcu)) {
5210		try llvm_args.append(llvm_arg);
5211		} else {
5212		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5213		const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
5214		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5215		_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5216		try llvm_args.append(arg_ptr);
5217		}
5218		},
5219		.byref_mut => {
5220		const arg = args[it.zig_index - 1];
5221		const param_ty = self.typeOf(arg);
5222		const llvm_arg = try self.resolveInst(arg);
5223
5224		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5225		const param_llvm_ty = try o.lowerType(pt, param_ty);
5226		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
5227		if (isByRef(param_ty, zcu)) {
5228		const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
5229		_ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
5230		} else {
5231		_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
5232		}
5233		try llvm_args.append(arg_ptr);
5234		},
5235		.abi_sized_int => {
5236		const arg = args[it.zig_index - 1];
5237		const param_ty = self.typeOf(arg);
5238		const llvm_arg = try self.resolveInst(arg);
5239		const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8));
5240
5241		if (isByRef(param_ty, zcu)) {
5242		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5243		const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, "");
5244		try llvm_args.append(loaded);
5245		} else {
5246		// LLVM does not allow bitcasting structs so we must allocate
5247		// a local, store as one type, and then load as another type.
5248		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5249		const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
5250		_ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
5251		const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
5252		try llvm_args.append(loaded);
5253		}
5254		},
5255		.slice => {
5256		const arg = args[it.zig_index - 1];
5257		const llvm_arg = try self.resolveInst(arg);
5258		const ptr = try self.wip.extractValue(llvm_arg, &.{0}, "");
5259		const len = try self.wip.extractValue(llvm_arg, &.{1}, "");
5260		try llvm_args.appendSlice(&.{ ptr, len });
5261		},
5262		.multiple_llvm_types => {
5263		const arg = args[it.zig_index - 1];
5264		const param_ty = self.typeOf(arg);
5265		const llvm_types = it.types_buffer[0..it.types_len];
5266		const llvm_arg = try self.resolveInst(arg);
5267		const is_by_ref = isByRef(param_ty, zcu);
5268		const arg_ptr = if (is_by_ref) llvm_arg else ptr: {
5269		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5270		const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5271		_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5272		break :ptr ptr;
5273		};
5274
5275		const llvm_ty = try o.builder.structType(.normal, llvm_types);
5276		try llvm_args.ensureUnusedCapacity(it.types_len);
5277		for (llvm_types, 0..) \|field_ty, i\| {
5278		const alignment =
5279		Builder.Alignment.fromByteUnits(@divExact(target.ptrBitWidth(), 8));
5280		const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
5281		const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
5282		llvm_args.appendAssumeCapacity(loaded);
5283		}
5284		},
5285		.float_array => \|count\| {
5286		const arg = args[it.zig_index - 1];
5287		const arg_ty = self.typeOf(arg);
5288		var llvm_arg = try self.resolveInst(arg);
5289		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5290		if (!isByRef(arg_ty, zcu)) {
5291		const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5292		_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5293		llvm_arg = ptr;
5294		}
5295
5296		const float_ty = try o.lowerType(pt, aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
5297		const array_ty = try o.builder.arrayType(count, float_ty);
5298
5299		const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5300		try llvm_args.append(loaded);
5301		},
5302		.i32_array, .i64_array => \|arr_len\| {
5303		const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
5304		const arg = args[it.zig_index - 1];
5305		const arg_ty = self.typeOf(arg);
5306		var llvm_arg = try self.resolveInst(arg);
5307		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
5308		if (!isByRef(arg_ty, zcu)) {
5309		const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
5310		_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
5311		llvm_arg = ptr;
5312		}
5313
5314		const array_ty =
5315		try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
5316		const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
5317		try llvm_args.append(loaded);
5318		},
5319		};
5320
5321		{
5322		// Add argument attributes.
5323		it = iterateParamTypes(o, pt, fn_info);
5324		it.llvm_index += @intFromBool(sret);
5325		it.llvm_index += @intFromBool(err_return_tracing);
5326		while (try it.next()) \|lowering\| switch (lowering) {
5327		.byval => {
5328		const param_index = it.zig_index - 1;
5329		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5330		if (!isByRef(param_ty, zcu)) {
5331		try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
5332		}
5333		},
5334		.byref => {
5335		const param_index = it.zig_index - 1;
5336		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
5337		const param_llvm_ty = try o.lowerType(pt, param_ty);
5338		const alignment = param_ty.abiAlignment(zcu).toLlvm();
5339		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
5340		},
5341		.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
5342		// No attributes needed for these.
5343		.no_bits,
5344		.abi_sized_int,
5345		.multiple_llvm_types,
5346		.float_array,
5347		.i32_array,
5348		.i64_array,
5349		=> continue,
5350
5351		.slice => {
5352		assert(!it.byval_attr);
5353		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
5354		const ptr_info = param_ty.ptrInfo(zcu);
5355		const llvm_arg_i = it.llvm_index - 2;
5356
5357		if (math.cast(u5, it.zig_index - 1)) \|i\| {
5358		if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
5359		try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
5360		}
5361		}
5362		if (param_ty.zigTypeTag(zcu) != .optional and
5363		!ptr_info.flags.is_allowzero and
5364		ptr_info.flags.address_space == .generic)
5365		{
5366		try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
5367		}
5368		if (ptr_info.flags.is_const) {
5369		try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
5370		}
5371		const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
5372		else => \|a\| .wrap(a.toLlvm()),
5373		.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
5374		};
5375		try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
5376		},
5377		};
5378		}
5379
5380		const call = try self.wip.call(
5381		switch (modifier) {
5382		.auto, .never_inline => .normal,
5383		.never_tail => .notail,
5384		.always_tail => .musttail,
5385		.no_suspend, .always_inline, .compile_time => unreachable,
5386		},
5387		toLlvmCallConvTag(fn_info.cc, target).?,
5388		try attributes.finish(&o.builder),
5389		try o.lowerType(pt, zig_fn_ty),
5390		llvm_fn,
5391		llvm_args.items,
5392		"",
5393		);
5394
5395		if (fn_info.return_type == .noreturn_type and modifier != .always_tail) {
5396		return .none;
5397		}
5398
5399		if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
5400		return .none;
5401		}
5402
5403		const llvm_ret_ty = try o.lowerType(pt, return_type);
5404		if (ret_ptr) \|rp\| {
5405		if (isByRef(return_type, zcu)) {
5406		return rp;
5407		} else {
5408		// our by-ref status disagrees with sret so we must load.
5409		const return_alignment = return_type.abiAlignment(zcu).toLlvm();
5410		return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
5411		}
5412		}
5413
5414		const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5415
5416		if (abi_ret_ty != llvm_ret_ty) {
5417		// In this case the function return type is honoring the calling convention by having
5418		// a different LLVM type than the usual one. We solve this here at the callsite
5419		// by using our canonical type, then loading it if necessary.
5420		const alignment = return_type.abiAlignment(zcu).toLlvm();
5421		const rp = try self.buildAlloca(abi_ret_ty, alignment);
5422		_ = try self.wip.store(.normal, call, rp, alignment);
5423		return if (isByRef(return_type, zcu))
5424		rp
5425		else
5426		try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
5427		}
5428
5429		if (isByRef(return_type, zcu)) {
5430		// our by-ref status disagrees with sret so we must allocate, store,
5431		// and return the allocation pointer.
5432		const alignment = return_type.abiAlignment(zcu).toLlvm();
5433		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5434		_ = try self.wip.store(.normal, call, rp, alignment);
5435		return rp;
5436		} else {
5437		return call;
5438		}
5439		}
5440
5441		fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) !void {
5442		const o = fg.ng.object;
5443		const pt = fg.ng.pt;
5444		const zcu = pt.zcu;
5445		const target = zcu.getTarget();
5446		const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin()));
5447		const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
5448		const panic_global = try o.resolveLlvmFunction(pt, panic_func.owner_nav);
5449
5450		const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
5451		if (has_err_trace) assert(fg.err_ret_trace != .none);
5452		_ = try fg.wip.callIntrinsicAssumeCold();
5453		_ = try fg.wip.call(
5454		.normal,
5455		toLlvmCallConvTag(fn_info.cc, target).?,
5456		.none,
5457		panic_global.typeOf(&o.builder),
5458		panic_global.toValue(&o.builder),
5459		if (has_err_trace) &.{fg.err_ret_trace} else &.{},
5460		"",
5461		);
5462		_ = try fg.wip.@"unreachable"();
5463		}
5464
5465		fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !void {
5466		const o = self.ng.object;
5467		const pt = self.ng.pt;
5468		const zcu = pt.zcu;
5469		const ip = &zcu.intern_pool;
5470		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5471		const ret_ty = self.typeOf(un_op);
5472
5473		if (self.ret_ptr != .none) {
5474		const ptr_ty = try pt.singleMutPtrType(ret_ty);
5475
5476		const operand = try self.resolveInst(un_op);
5477		const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
5478		if (val_is_undef and safety) undef: {
5479		const ptr_info = ptr_ty.ptrInfo(zcu);
5480		const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
5481		if (needs_bitmask) {
5482		// TODO: only some bits are to be undef, we cannot write with a simple memset.
5483		// meanwhile, ignore the write rather than stomping over valid bits.
5484		// https://github.com/ziglang/zig/issues/15337
5485		break :undef;
5486		}
5487		const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5488		_ = try self.wip.callMemSet(
5489		self.ret_ptr,
5490		ptr_ty.ptrAlignment(zcu).toLlvm(),
5491		try o.builder.intValue(.i8, 0xaa),
5492		len,
5493		.normal,
5494		self.disable_intrinsics,
5495		);
5496		const owner_mod = self.ng.ownerModule();
5497		if (owner_mod.valgrind) {
5498		try self.valgrindMarkUndef(self.ret_ptr, len);
5499		}
5500		_ = try self.wip.retVoid();
5501		return;
5502		}
5503
5504		const unwrapped_operand = operand.unwrap();
5505		const unwrapped_ret = self.ret_ptr.unwrap();
5506
5507		// Return value was stored previously
5508		if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
5509		_ = try self.wip.retVoid();
5510		return;
5511		}
5512
5513		try self.store(self.ret_ptr, ptr_ty, operand, .none);
5514		_ = try self.wip.retVoid();
5515		return;
5516		}
5517		const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?;
5518		if (!ret_ty.hasRuntimeBits(zcu)) {
5519		if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5520		// Functions with an empty error set are emitted with an error code
5521		// return type and return zero so they can be function pointers coerced
5522		// to functions that return anyerror.
5523		_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5524		} else {
5525		_ = try self.wip.retVoid();
5526		}
5527		return;
5528		}
5529
5530		const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5531		const operand = try self.resolveInst(un_op);
5532		const val_is_undef = if (try self.air.value(un_op, pt)) \|val\| val.isUndef(zcu) else false;
5533		const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5534
5535		if (val_is_undef and safety) {
5536		const llvm_ret_ty = operand.typeOfWip(&self.wip);
5537		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5538		const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), ret_ty.abiSize(zcu));
5539		_ = try self.wip.callMemSet(
5540		rp,
5541		alignment,
5542		try o.builder.intValue(.i8, 0xaa),
5543		len,
5544		.normal,
5545		self.disable_intrinsics,
5546		);
5547		const owner_mod = self.ng.ownerModule();
5548		if (owner_mod.valgrind) {
5549		try self.valgrindMarkUndef(rp, len);
5550		}
5551		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5552		return;
5553		}
5554
5555		if (isByRef(ret_ty, zcu)) {
5556		// operand is a pointer however self.ret_ptr is null so that means
5557		// we need to return a value.
5558		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
5559		return;
5560		}
5561
5562		const llvm_ret_ty = operand.typeOfWip(&self.wip);
5563		if (abi_ret_ty == llvm_ret_ty) {
5564		_ = try self.wip.ret(operand);
5565		return;
5566		}
5567
5568		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
5569		_ = try self.wip.store(.normal, operand, rp, alignment);
5570		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
5571		return;
5572		}
5573
5574		fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) !void {
5575		const o = self.ng.object;
5576		const pt = self.ng.pt;
5577		const zcu = pt.zcu;
5578		const ip = &zcu.intern_pool;
5579		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5580		const ptr_ty = self.typeOf(un_op);
5581		const ret_ty = ptr_ty.childType(zcu);
5582		const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.ng.nav_index).resolved.?.type)).?;
5583		if (!ret_ty.hasRuntimeBits(zcu)) {
5584		if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
5585		// Functions with an empty error set are emitted with an error code
5586		// return type and return zero so they can be function pointers coerced
5587		// to functions that return anyerror.
5588		_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(pt), 0));
5589		} else {
5590		_ = try self.wip.retVoid();
5591		}
5592		return;
5593		}
5594		if (self.ret_ptr != .none) {
5595		_ = try self.wip.retVoid();
5596		return;
5597		}
5598		const ptr = try self.resolveInst(un_op);
5599		const abi_ret_ty = try lowerFnRetTy(o, pt, fn_info);
5600		const alignment = ret_ty.abiAlignment(zcu).toLlvm();
5601		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
5602		return;
5603		}
5604
5605		fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5606		const o = self.ng.object;
5607		const pt = self.ng.pt;
5608		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5609		const list = try self.resolveInst(ty_op.operand);
5610		const arg_ty = ty_op.ty.toType();
5611		const llvm_arg_ty = try o.lowerType(pt, arg_ty);
5612
5613		return self.wip.vaArg(list, llvm_arg_ty, "");
5614		}
5615
5616		fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5617		const o = self.ng.object;
5618		const pt = self.ng.pt;
5619		const zcu = pt.zcu;
5620		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5621		const src_list = try self.resolveInst(ty_op.operand);
5622		const va_list_ty = ty_op.ty.toType();
5623		const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5624
5625		const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5626		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5627
5628		_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
5629		return if (isByRef(va_list_ty, zcu))
5630		dest_list
5631		else
5632		try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5633		}
5634
5635		fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5636		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5637		const src_list = try self.resolveInst(un_op);
5638
5639		_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
5640		return .none;
5641		}
5642
5643		fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5644		const o = self.ng.object;
5645		const pt = self.ng.pt;
5646		const zcu = pt.zcu;
5647		const va_list_ty = self.typeOfIndex(inst);
5648		const llvm_va_list_ty = try o.lowerType(pt, va_list_ty);
5649
5650		const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
5651		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
5652
5653		_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
5654		return if (isByRef(va_list_ty, zcu))
5655		dest_list
5656		else
5657		try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
5658		}
5659
5660		fn airCmp(
5661		self: *FuncGen,
5662		inst: Air.Inst.Index,
5663		op: math.CompareOperator,
5664		fast: Builder.FastMathKind,
5665		) !Builder.Value {
5666		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
5667		const lhs = try self.resolveInst(bin_op.lhs);
5668		const rhs = try self.resolveInst(bin_op.rhs);
5669		const operand_ty = self.typeOf(bin_op.lhs);
5670
5671		return self.cmp(fast, op, operand_ty, lhs, rhs);
5672		}
5673
5674		fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
5675		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
5676		const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
5677
5678		const lhs = try self.resolveInst(extra.lhs);
5679		const rhs = try self.resolveInst(extra.rhs);
5680		const vec_ty = self.typeOf(extra.lhs);
5681		const cmp_op = extra.compareOperator();
5682
5683		return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
5684		}
5685
5686		fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5687		const o = self.ng.object;
5688		const pt = self.ng.pt;
5689		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5690		const operand = try self.resolveInst(un_op);
5691		const llvm_fn = try o.getCmpLtErrorsLenFunction(pt);
5692		return self.wip.call(
5693		.normal,
5694		.fastcc,
5695		.none,
5696		llvm_fn.typeOf(&o.builder),
5697		llvm_fn.toValue(&o.builder),
5698		&.{operand},
5699		"",
5700		);
5701		}
5702
5703		fn cmp(
5704		self: *FuncGen,
5705		fast: Builder.FastMathKind,
5706		op: math.CompareOperator,
5707		operand_ty: Type,
5708		lhs: Builder.Value,
5709		rhs: Builder.Value,
5710		) Allocator.Error!Builder.Value {
5711		const o = self.ng.object;
5712		const pt = self.ng.pt;
5713		const zcu = pt.zcu;
5714		const scalar_ty = operand_ty.scalarType(zcu);
5715		const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
5716		.@"enum" => scalar_ty.intTagType(zcu),
5717		.int, .bool, .pointer, .error_set => scalar_ty,
5718		.optional => blk: {
5719		const payload_ty = operand_ty.optionalChild(zcu);
5720		if (!payload_ty.hasRuntimeBits(zcu) or
5721		operand_ty.optionalReprIsPayload(zcu))
5722		{
5723		break :blk operand_ty;
5724		}
5725		// We need to emit instructions to check for equality/inequality
5726		// of optionals that are not pointers.
5727		const is_by_ref = isByRef(scalar_ty, zcu);
5728		const opt_llvm_ty = try o.lowerType(pt, scalar_ty);
5729		const lhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, lhs, is_by_ref, .normal);
5730		const rhs_non_null = try self.optCmpNull(.ne, opt_llvm_ty, rhs, is_by_ref, .normal);
5731		const llvm_i2 = try o.builder.intType(2);
5732		const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
5733		const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
5734		const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
5735		const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
5736		const both_null_block = try self.wip.block(1, "BothNull");
5737		const mixed_block = try self.wip.block(1, "Mixed");
5738		const both_pl_block = try self.wip.block(1, "BothNonNull");
5739		const end_block = try self.wip.block(3, "End");
5740		var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
5741		defer wip_switch.finish(&self.wip);
5742		try wip_switch.addCase(
5743		try o.builder.intConst(llvm_i2, 0b00),
5744		both_null_block,
5745		&self.wip,
5746		);
5747		try wip_switch.addCase(
5748		try o.builder.intConst(llvm_i2, 0b11),
5749		both_pl_block,
5750		&self.wip,
5751		);
5752
5753		self.wip.cursor = .{ .block = both_null_block };
5754		_ = try self.wip.br(end_block);
5755
5756		self.wip.cursor = .{ .block = mixed_block };
5757		_ = try self.wip.br(end_block);
5758
5759		self.wip.cursor = .{ .block = both_pl_block };
5760		const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true);
5761		const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true);
5762		const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
5763		_ = try self.wip.br(end_block);
5764		const both_pl_block_end = self.wip.cursor.block;
5765
5766		self.wip.cursor = .{ .block = end_block };
5767		const llvm_i1_0 = Builder.Value.false;
5768		const llvm_i1_1 = Builder.Value.true;
5769		const incoming_values: [3]Builder.Value = .{
5770		switch (op) {
5771		.eq => llvm_i1_1,
5772		.neq => llvm_i1_0,
5773		else => unreachable,
5774		},
5775		switch (op) {
5776		.eq => llvm_i1_0,
5777		.neq => llvm_i1_1,
5778		else => unreachable,
5779		},
5780		payload_cmp,
5781		};
5782
5783		const phi = try self.wip.phi(.i1, "");
5784		phi.finish(
5785		&incoming_values,
5786		&.{ both_null_block, mixed_block, both_pl_block_end },
5787		&self.wip,
5788		);
5789		return phi.toValue();
5790		},
5791		.float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
5792		.@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu),
5793		else => unreachable,
5794		};
5795		const is_signed = int_ty.isSignedInt(zcu);
5796		const cond: Builder.IntegerCondition = switch (op) {
5797		.eq => .eq,
5798		.neq => .ne,
5799		.lt => if (is_signed) .slt else .ult,
5800		.lte => if (is_signed) .sle else .ule,
5801		.gt => if (is_signed) .sgt else .ugt,
5802		.gte => if (is_signed) .sge else .uge,
5803		};
5804		return self.wip.icmp(cond, lhs, rhs, "");
5805		}
5806
5807		fn airBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
5808		const block = self.air.unwrapBlock(inst);
5809		return self.lowerBlock(inst, null, block.body);
5810		}
5811
5812		fn lowerBlock(
5813		self: *FuncGen,
5814		inst: Air.Inst.Index,
5815		maybe_inline_func: ?InternPool.Index,
5816		body: []const Air.Inst.Index,
5817		) !Builder.Value {
5818		const o = self.ng.object;
5819		const pt = self.ng.pt;
5820		const zcu = pt.zcu;
5821		const inst_ty = self.typeOfIndex(inst);
5822
5823		if (inst_ty.isNoReturn(zcu)) {
5824		try self.genBodyDebugScope(maybe_inline_func, body, .none);
5825		return .none;
5826		}
5827
5828		const have_block_result = inst_ty.hasRuntimeBits(zcu);
5829
5830		var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
5831		defer if (have_block_result) breaks.list.deinit(self.gpa);
5832
5833		const parent_bb = try self.wip.block(0, "Block");
5834		try self.blocks.putNoClobber(self.gpa, inst, .{
5835		.parent_bb = parent_bb,
5836		.breaks = &breaks,
5837		});
5838		defer assert(self.blocks.remove(inst));
5839
5840		try self.genBodyDebugScope(maybe_inline_func, body, .none);
5841
5842		self.wip.cursor = .{ .block = parent_bb };
5843
5844		// Create a phi node only if the block returns a value.
5845		if (have_block_result) {
5846		const raw_llvm_ty = try o.lowerType(pt, inst_ty);
5847		const llvm_ty: Builder.Type = ty: {
5848		// If the zig tag type is a function, this represents an actual function body; not
5849		// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
5850		// of function pointers, however the phi makes it a runtime value and therefore
5851		// the LLVM type has to be wrapped in a pointer.
5852		if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
5853		break :ty .ptr;
5854		}
5855		break :ty raw_llvm_ty;
5856		};
5857
5858		parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
5859		const phi = try self.wip.phi(llvm_ty, "");
5860		phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
5861		return phi.toValue();
5862		} else {
5863		parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
5864		return .none;
5865		}
5866		}
5867
5868		fn airBr(self: *FuncGen, inst: Air.Inst.Index) !void {
5869		const zcu = self.ng.pt.zcu;
5870		const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
5871		const block = self.blocks.get(branch.block_inst).?;
5872
5873		// Add the values to the lists only if the break provides a value.
5874		const operand_ty = self.typeOf(branch.operand);
5875		if (operand_ty.hasRuntimeBits(zcu)) {
5876		const val = try self.resolveInst(branch.operand);
5877
5878		// For the phi node, we need the basic blocks and the values of the
5879		// break instructions.
5880		try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
5881		} else block.breaks.len += 1;
5882		_ = try self.wip.br(block.parent_bb);
5883		}
5884
5885		fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) !void {
5886		const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
5887		const loop_bb = self.loops.get(repeat.loop_inst).?;
5888		loop_bb.ptr(&self.wip).incoming += 1;
5889		_ = try self.wip.br(loop_bb);
5890		}
5891
5892		fn lowerSwitchDispatch(
5893		self: *FuncGen,
5894		switch_inst: Air.Inst.Index,
5895		cond_ref: Air.Inst.Ref,
5896		dispatch_info: SwitchDispatchInfo,
5897		) !void {
5898		const o = self.ng.object;
5899		const pt = self.ng.pt;
5900		const zcu = pt.zcu;
5901		const cond_ty = self.typeOf(cond_ref);
5902		const switch_br = self.air.unwrapSwitch(switch_inst);
5903
5904		if (try self.air.value(cond_ref, pt)) \|cond_val\| {
5905		// Comptime-known dispatch. Iterate the cases to find the correct
5906		// one, and branch to the corresponding element of `case_blocks`.
5907		var it = switch_br.iterateCases();
5908		const target_case_idx = target: while (it.next()) \|case\| {
5909		for (case.items) \|item\| {
5910		const val = Value.fromInterned(item.toInterned().?);
5911		if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
5912		}
5913		for (case.ranges) \|range\| {
5914		const low = Value.fromInterned(range[0].toInterned().?);
5915		const high = Value.fromInterned(range[1].toInterned().?);
5916		if (cond_val.compareHetero(.gte, low, zcu) and
5917		cond_val.compareHetero(.lte, high, zcu))
5918		{
5919		break :target case.idx;
5920		}
5921		}
5922		} else dispatch_info.case_blocks.len - 1;
5923		const target_block = dispatch_info.case_blocks[target_case_idx];
5924		target_block.ptr(&self.wip).incoming += 1;
5925		_ = try self.wip.br(target_block);
5926		return;
5927		}
5928
5929		// Runtime-known dispatch.
5930		const cond = try self.resolveInst(cond_ref);
5931
5932		if (dispatch_info.jmp_table) \|jmp_table\| {
5933		// We should use the constructed jump table.
5934		// First, check the bounds to branch to the `else` case if needed.
5935		const inbounds = try self.wip.bin(
5936		.@"and",
5937		try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
5938		try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
5939		"",
5940		);
5941		const jmp_table_block = try self.wip.block(1, "Then");
5942		const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
5943		else_block.ptr(&self.wip).incoming += 1;
5944		_ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
5945		.none => .none,
5946		.unpredictable => .unpredictable,
5947		.likely => .then_likely,
5948		.unlikely => .else_likely,
5949		});
5950
5951		self.wip.cursor = .{ .block = jmp_table_block };
5952
5953		// Figure out the list of blocks we might branch to.
5954		// This includes all case blocks, but it might not include the `else` block if
5955		// the table is dense.
5956		const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
5957		const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
5958
5959		// Make sure to cast the index to a usize so it's not treated as negative!
5960		const table_index = try self.wip.conv(
5961		.unsigned,
5962		try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
5963		try o.lowerType(pt, .usize),
5964		"",
5965		);
5966		const target_ptr_ptr = try self.wip.gep(
5967		.inbounds,
5968		.ptr,
5969		jmp_table.table.toValue(),
5970		&.{table_index},
5971		"",
5972		);
5973		const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
5974
5975		// Do the branch!
5976		_ = try self.wip.indirectbr(target_ptr, target_blocks);
5977
5978		// Mark all target blocks as having one more incoming branch.
5979		for (target_blocks) \|case_block\| {
5980		case_block.ptr(&self.wip).incoming += 1;
5981		}
5982
5983		return;
5984		}
5985
5986		// We must lower to an actual LLVM `switch` instruction.
5987		// The switch prongs will correspond to our scalar cases. Ranges will
5988		// be handled by conditional branches in the `else` prong.
5989
5990		const llvm_usize = try o.lowerType(pt, Type.usize);
5991		const cond_int = if (cond.typeOfWip(&self.wip).isPointer(&o.builder))
5992		try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
5993		else
5994		cond;
5995
5996		const llvm_cases_len, const last_range_case = info: {
5997		var llvm_cases_len: u32 = 0;
5998		var last_range_case: ?u32 = null;
5999		var it = switch_br.iterateCases();
6000		while (it.next()) \|case\| {
6001		if (case.ranges.len > 0) last_range_case = case.idx;
6002		llvm_cases_len += @intCast(case.items.len);
6003		}
6004		break :info .{ llvm_cases_len, last_range_case };
6005		};
6006
6007		// The `else` of the LLVM `switch` is the actual `else` prong only
6008		// if there are no ranges. Otherwise, the `else` will have a
6009		// conditional chain before the "true" `else` prong.
6010		const llvm_else_block = if (last_range_case == null)
6011		dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6012		else
6013		try self.wip.block(0, "RangeTest");
6014
6015		llvm_else_block.ptr(&self.wip).incoming += 1;
6016
6017		var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
6018		defer wip_switch.finish(&self.wip);
6019
6020		// Construct the actual cases. Set the cursor to the `else` block so
6021		// we can construct ranges at the same time as scalar cases.
6022		self.wip.cursor = .{ .block = llvm_else_block };
6023
6024		var it = switch_br.iterateCases();
6025		while (it.next()) \|case\| {
6026		const case_block = dispatch_info.case_blocks[case.idx];
6027
6028		for (case.items) \|item\| {
6029		const llvm_item = (try self.resolveInst(item)).toConst().?;
6030		const llvm_int_item = if (llvm_item.typeOf(&o.builder).isPointer(&o.builder))
6031		try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
6032		else
6033		llvm_item;
6034		try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
6035		}
6036		case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
6037
6038		if (case.ranges.len == 0) continue;
6039
6040		// Add a conditional for the ranges, directing to the relevant bb.
6041		// We don't need to consider `cold` branch hints since that information is stored
6042		// in the target bb body, but we do care about likely/unlikely/unpredictable.
6043
6044		const hint = switch_br.getHint(case.idx);
6045
6046		var range_cond: ?Builder.Value = null;
6047		for (case.ranges) \|range\| {
6048		const llvm_min = try self.resolveInst(range[0]);
6049		const llvm_max = try self.resolveInst(range[1]);
6050		const cond_part = try self.wip.bin(
6051		.@"and",
6052		try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
6053		try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
6054		"",
6055		);
6056		if (range_cond) \|prev\| {
6057		range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
6058		} else range_cond = cond_part;
6059		}
6060
6061		// If the check fails, we either branch to the "true" `else` case,
6062		// or to the next range condition.
6063		const range_else_block = if (case.idx == last_range_case.?)
6064		dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
6065		else
6066		try self.wip.block(0, "RangeTest");
6067
6068		_ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
6069		.none, .cold => .none,
6070		.unpredictable => .unpredictable,
6071		.likely => .then_likely,
6072		.unlikely => .else_likely,
6073		});
6074		case_block.ptr(&self.wip).incoming += 1;
6075		range_else_block.ptr(&self.wip).incoming += 1;
6076
6077		// Construct the next range conditional (if any) in the false branch.
6078		self.wip.cursor = .{ .block = range_else_block };
6079		}
6080		}
6081
6082		fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) !void {
6083		const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
6084		const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
6085		return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
6086		}
6087
6088		fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) !void {
6089		const cond_br = self.air.unwrapCondBr(inst);
6090		const cond = try self.resolveInst(cond_br.condition);
6091		const then_body = cond_br.then_body;
6092		const else_body = cond_br.else_body;
6093
6094		const Hint = enum {
6095		none,
6096		unpredictable,
6097		then_likely,
6098		else_likely,
6099		then_cold,
6100		else_cold,
6101		};
6102		const hint: Hint = switch (cond_br.branch_hints.true) {
6103		.none => switch (cond_br.branch_hints.false) {
6104		.none => .none,
6105		.likely => .else_likely,
6106		.unlikely => .then_likely,
6107		.cold => .else_cold,
6108		.unpredictable => .unpredictable,
6109		},
6110		.likely => switch (cond_br.branch_hints.false) {
6111		.none => .then_likely,
6112		.likely => .unpredictable,
6113		.unlikely => .then_likely,
6114		.cold => .else_cold,
6115		.unpredictable => .unpredictable,
6116		},
6117		.unlikely => switch (cond_br.branch_hints.false) {
6118		.none => .else_likely,
6119		.likely => .else_likely,
6120		.unlikely => .unpredictable,
6121		.cold => .else_cold,
6122		.unpredictable => .unpredictable,
6123		},
6124		.cold => .then_cold,
6125		.unpredictable => .unpredictable,
6126		};
6127
6128		const then_block = try self.wip.block(1, "Then");
6129		const else_block = try self.wip.block(1, "Else");
6130		_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
6131		.none, .then_cold, .else_cold => .none,
6132		.unpredictable => .unpredictable,
6133		.then_likely => .then_likely,
6134		.else_likely => .else_likely,
6135		});
6136
6137		self.wip.cursor = .{ .block = then_block };
6138		if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
6139		try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
6140
6141		self.wip.cursor = .{ .block = else_block };
6142		if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
6143		try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
6144
6145		// No need to reset the insert cursor since this instruction is noreturn.
6146		}
6147
6148		fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6149		const unwrapped_try = self.air.unwrapTry(inst);
6150		const err_union = try self.resolveInst(unwrapped_try.error_union);
6151		const body = unwrapped_try.else_body;
6152		const err_union_ty = self.typeOf(unwrapped_try.error_union);
6153		const is_unused = self.liveness.isUnused(inst);
6154		return lowerTry(self, err_union, body, err_union_ty, false, .none, false, is_unused, err_cold);
6155		}
6156
6157		fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) !Builder.Value {
6158		const zcu = self.ng.pt.zcu;
6159		const unwrapped_try = self.air.unwrapTryPtr(inst);
6160		const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
6161		const body = unwrapped_try.else_body;
6162		const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr);
6163		const err_union_ty = err_union_ptr_ty.childType(zcu);
6164		const is_unused = self.liveness.isUnused(inst);
6165
6166		self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
6167
6168		return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), true, is_unused, err_cold);
6169		}
6170
6171		fn lowerTry(
6172		fg: *FuncGen,
6173		err_union: Builder.Value,
6174		body: []const Air.Inst.Index,
6175		err_union_ty: Type,
6176		operand_is_ptr: bool,
6177		operand_ptr_align: InternPool.Alignment,
6178		can_elide_load: bool,
6179		is_unused: bool,
6180		err_cold: bool,
6181		) !Builder.Value {
6182		const o = fg.ng.object;
6183		const pt = fg.ng.pt;
6184		const zcu = pt.zcu;
6185		const payload_ty = err_union_ty.errorUnionPayload(zcu);
6186		const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
6187		const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
6188		const error_type = try o.errorIntType(pt);
6189
6190		const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
6191		operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
6192		operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
6193		} else .{ .none, .none };
6194
6195		if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
6196		const loaded = loaded: {
6197		const access_kind: Builder.MemoryAccessKind =
6198		if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
6199
6200		if (!payload_has_bits) {
6201		break :loaded if (operand_is_ptr)
6202		try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
6203		else
6204		err_union;
6205		}
6206		const err_field_index = try errUnionErrorOffset(payload_ty, pt);
6207		if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
6208		const err_field_ptr =
6209		try fg.wip.gepStruct(err_union_llvm_ty, err_union, err_field_index, "");
6210		break :loaded try fg.wip.load(
6211		if (operand_is_ptr) access_kind else .normal,
6212		error_type,
6213		err_field_ptr,
6214		err_set_align.toLlvm(),
6215		"",
6216		);
6217		}
6218		break :loaded try fg.wip.extractValue(err_union, &.{err_field_index}, "");
6219		};
6220		const zero = try o.builder.intValue(error_type, 0);
6221		const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
6222
6223		const return_block = try fg.wip.block(1, "TryRet");
6224		const continue_block = try fg.wip.block(1, "TryCont");
6225		_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
6226
6227		fg.wip.cursor = .{ .block = return_block };
6228		if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
6229		try fg.genBodyDebugScope(null, body, .poi);
6230
6231		fg.wip.cursor = .{ .block = continue_block };
6232		}
6233		if (is_unused) return .none;
6234		if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
6235		const offset = try errUnionPayloadOffset(payload_ty, pt);
6236		if (operand_is_ptr) {
6237		return fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6238		} else if (isByRef(err_union_ty, zcu)) {
6239		const payload_ptr = try fg.wip.gepStruct(err_union_llvm_ty, err_union, offset, "");
6240		if (isByRef(payload_ty, zcu)) {
6241		if (can_elide_load)
6242		return payload_ptr;
6243
6244		return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
6245		}
6246		const load_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
6247		return fg.wip.load(.normal, load_ty, payload_ptr, payload_align.toLlvm(), "");
6248		}
6249		return fg.wip.extractValue(err_union, &.{offset}, "");
6250		}
6251
6252		fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) !void {
6253		const o = self.ng.object;
6254		const pt = self.ng.pt;
6255		const zcu = pt.zcu;
6256
6257		const switch_br = self.air.unwrapSwitch(inst);
6258
6259		// For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
6260		// For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
6261		// scalar and range cases in the same prong.
6262		// +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
6263		// conditionals to handle ranges.
6264		const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
6265		defer self.gpa.free(case_blocks);
6266		// We set incoming as 0 for now, and increment it as we construct dispatches.
6267		for (case_blocks[0 .. case_blocks.len - 1]) \|b\| b. = try self.wip.block(0, "Case");
6268		case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
6269
6270		// There's a special case here to manually generate a jump table in some cases.
6271		//
6272		// Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
6273		// table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
6274		// use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
6275		// bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
6276		// destroying the cache -- but it also actually generates slightly different jump tables for each case,
6277		// and a separate conditional branch beforehand to handle dispatching back to the case we're currently
6278		// within(!!).
6279		//
6280		// This asm is really, really, not what we want. As such, we will construct the jump table manually where
6281		// appropriate (the values are dense and relatively few), and use it when lowering dispatches.
6282
6283		const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
6284		if (!is_dispatch_loop) break :jmp_table null;
6285
6286		// Workaround for:
6287		// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
6288		// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
6289		if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
6290
6291		// On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
6292		// about acceptable - it won't fill L1d cache on most CPUs.
6293		const max_table_len = 1024;
6294
6295		const cond_ty = self.typeOf(switch_br.operand);
6296		switch (cond_ty.zigTypeTag(zcu)) {
6297		.bool, .pointer => break :jmp_table null,
6298		.@"enum", .int, .error_set, .@"struct", .@"union" => {},
6299		else => unreachable,
6300		}
6301
6302		if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
6303
6304		// Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
6305		// If they are, then we will construct a jump table.
6306		const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null;
6307		const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
6308		const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
6309		const table_len = max_int - min_int + 1;
6310		if (table_len > max_table_len) break :jmp_table null;
6311
6312		const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
6313		defer self.gpa.free(table_elems);
6314
6315		// Set them all to the `else` branch, then iterate over the AIR switch
6316		// and replace all values which correspond to other prongs.
6317		@memset(table_elems, try o.builder.blockAddrConst(
6318		self.wip.function,
6319		case_blocks[case_blocks.len - 1],
6320		));
6321		var item_count: u32 = 0;
6322		var it = switch_br.iterateCases();
6323		while (it.next()) \|case\| {
6324		const case_block = case_blocks[case.idx];
6325		const case_block_addr = try o.builder.blockAddrConst(
6326		self.wip.function,
6327		case_block,
6328		);
6329		for (case.items) \|item\| {
6330		const val = Value.fromInterned(item.toInterned().?);
6331		const table_idx = val.toUnsignedInt(zcu) - min_int;
6332		table_elems[@intCast(table_idx)] = case_block_addr;
6333		item_count += 1;
6334		}
6335		for (case.ranges) \|range\| {
6336		const low = Value.fromInterned(range[0].toInterned().?);
6337		const high = Value.fromInterned(range[1].toInterned().?);
6338		const low_idx = low.toUnsignedInt(zcu) - min_int;
6339		const high_idx = high.toUnsignedInt(zcu) - min_int;
6340		@memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
6341		item_count += @intCast(high_idx + 1 - low_idx);
6342		}
6343		}
6344
6345		const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
6346		const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
6347
6348		const table_variable = try o.builder.addVariable(
6349		try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}),
6350		table_llvm_ty,
6351		.default,
6352		);
6353		try table_variable.setInitializer(table_val, &o.builder);
6354		table_variable.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
6355		table_variable.setUnnamedAddr(.unnamed_addr, &o.builder);
6356
6357		const table_includes_else = item_count != table_len;
6358
6359		break :jmp_table .{
6360		.min = try o.lowerValue(pt, min.toIntern()),
6361		.max = try o.lowerValue(pt, max.toIntern()),
6362		.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
6363		.none, .cold => .none,
6364		.unpredictable => .unpredictable,
6365		.likely => .likely,
6366		.unlikely => .unlikely,
6367		},
6368		.table = table_variable.toConst(&o.builder),
6369		.table_includes_else = table_includes_else,
6370		};
6371		};
6372
6373		const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
6374		if (jmp_table != null) break :weights .none; // not used
6375
6376		// First pass. If any weights are `.unpredictable`, unpredictable.
6377		// If all are `.none` or `.cold`, none.
6378		var any_likely = false;
6379		for (0..switch_br.cases_len) \|case_idx\| {
6380		switch (switch_br.getHint(@intCast(case_idx))) {
6381		.none, .cold => {},
6382		.likely, .unlikely => any_likely = true,
6383		.unpredictable => break :weights .unpredictable,
6384		}
6385		}
6386		switch (switch_br.getElseHint()) {
6387		.none, .cold => {},
6388		.likely, .unlikely => any_likely = true,
6389		.unpredictable => break :weights .unpredictable,
6390		}
6391		if (!any_likely) break :weights .none;
6392
6393		const llvm_cases_len = llvm_cases_len: {
6394		var len: u32 = 0;
6395		var it = switch_br.iterateCases();
6396		while (it.next()) \|case\| len += @intCast(case.items.len);
6397		break :llvm_cases_len len;
6398		};
6399
6400		var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
6401		defer self.gpa.free(weights);
6402		var weight_idx: usize = 0;
6403
6404		const branch_weights_str = try o.builder.metadataString("branch_weights");
6405		weights[weight_idx] = branch_weights_str.toMetadata();
6406		weight_idx += 1;
6407
6408		const else_weight: u32 = switch (switch_br.getElseHint()) {
6409		.unpredictable => unreachable,
6410		.none, .cold => 1000,
6411		.likely => 2000,
6412		.unlikely => 1,
6413		};
6414		weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
6415		weight_idx += 1;
6416
6417		var it = switch_br.iterateCases();
6418		while (it.next()) \|case\| {
6419		const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
6420		.unpredictable => unreachable,
6421		.none, .cold => 1000,
6422		.likely => 2000,
6423		.unlikely => 1,
6424		};
6425		const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
6426		@memset(weights[weight_idx..][0..case.items.len], weight_meta);
6427		weight_idx += case.items.len;
6428		}
6429
6430		assert(weight_idx == weights.len);
6431		break :weights .fromMetadata(try o.builder.metadataTuple(weights));
6432		};
6433
6434		const dispatch_info: SwitchDispatchInfo = .{
6435		.case_blocks = case_blocks,
6436		.switch_weights = weights,
6437		.jmp_table = jmp_table,
6438		};
6439
6440		if (is_dispatch_loop) {
6441		try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
6442		}
6443		defer if (is_dispatch_loop) {
6444		assert(self.switch_dispatch_info.remove(inst));
6445		};
6446
6447		// Generate the initial dispatch.
6448		// If this is a simple `switch_br`, this is the only dispatch.
6449		try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
6450
6451		// Iterate the cases and generate their bodies.
6452		var it = switch_br.iterateCases();
6453		while (it.next()) \|case\| {
6454		const case_block = case_blocks[case.idx];
6455		self.wip.cursor = .{ .block = case_block };
6456		if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6457		try self.genBodyDebugScope(null, case.body, .none);
6458		}
6459		self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
6460		const else_body = it.elseBody();
6461		if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
6462		if (else_body.len > 0) {
6463		try self.genBodyDebugScope(null, it.elseBody(), .none);
6464		} else {
6465		_ = try self.wip.@"unreachable"();
6466		}
6467		}
6468
6469		fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value {
6470		const zcu = self.ng.pt.zcu;
6471		var it = switch_br.iterateCases();
6472		var min: ?Value = null;
6473		var max: ?Value = null;
6474		while (it.next()) \|case\| {
6475		for (case.items) \|item\| {
6476		const val = Value.fromInterned(item.toInterned().?);
6477		const low = if (min) \|m\| val.compareHetero(.lt, m, zcu) else true;
6478		const high = if (max) \|m\| val.compareHetero(.gt, m, zcu) else true;
6479		if (low) min = val;
6480		if (high) max = val;
6481		}
6482		for (case.ranges) \|range\| {
6483		const vals: [2]Value = .{
6484		Value.fromInterned(range[0].toInterned().?),
6485		Value.fromInterned(range[1].toInterned().?),
6486		};
6487		const low = if (min) \|m\| vals[0].compareHetero(.lt, m, zcu) else true;
6488		const high = if (max) \|m\| vals[1].compareHetero(.gt, m, zcu) else true;
6489		if (low) min = vals[0];
6490		if (high) max = vals[1];
6491		}
6492		}
6493		if (min == null) {
6494		assert(max == null);
6495		return null;
6496		}
6497		return .{ min.?, max.? };
6498		}
6499
6500		fn airLoop(self: *FuncGen, inst: Air.Inst.Index) !void {
6501		const block = self.air.unwrapBlock(inst);
6502		const body = block.body;
6503		const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
6504		_ = try self.wip.br(loop_block);
6505
6506		try self.loops.putNoClobber(self.gpa, inst, loop_block);
6507		defer assert(self.loops.remove(inst));
6508
6509		self.wip.cursor = .{ .block = loop_block };
6510		try self.genBodyDebugScope(null, body, .none);
6511		}
6512
6513		fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6514		const o = self.ng.object;
6515		const pt = self.ng.pt;
6516		const zcu = pt.zcu;
6517		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6518		const operand_ty = self.typeOf(ty_op.operand);
6519		const array_ty = operand_ty.childType(zcu);
6520		const llvm_usize = try o.lowerType(pt, Type.usize);
6521		const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
6522		const slice_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
6523		const operand = try self.resolveInst(ty_op.operand);
6524		if (!array_ty.hasRuntimeBits(zcu))
6525		return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
6526		const ptr = try self.wip.gep(.inbounds, try o.lowerType(pt, array_ty), operand, &.{
6527		try o.builder.intValue(llvm_usize, 0), try o.builder.intValue(llvm_usize, 0),
6528		}, "");
6529		return self.wip.buildAggregate(slice_llvm_ty, &.{ ptr, len }, "");
6530		}
6531
6532		fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6533		const o = self.ng.object;
6534		const pt = self.ng.pt;
6535		const zcu = pt.zcu;
6536		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6537
6538		const operand = try self.resolveInst(ty_op.operand);
6539		const operand_ty = self.typeOf(ty_op.operand);
6540		const operand_scalar_ty = operand_ty.scalarType(zcu);
6541		const is_signed_int = operand_scalar_ty.isSignedInt(zcu);
6542
6543		const dest_ty = self.typeOfIndex(inst);
6544		const dest_scalar_ty = dest_ty.scalarType(zcu);
6545		const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6546		const target = zcu.getTarget();
6547
6548		if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
6549		if (is_signed_int) .signed else .unsigned,
6550		operand,
6551		dest_llvm_ty,
6552		"",
6553		);
6554
6555		const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
6556		return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)});
6557		};
6558		const rt_int_ty = try o.builder.intType(rt_int_bits);
6559		var extended = try self.wip.conv(
6560		if (is_signed_int) .signed else .unsigned,
6561		operand,
6562		rt_int_ty,
6563		"",
6564		);
6565		const dest_bits = dest_scalar_ty.floatBits(target);
6566		const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits);
6567		const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits);
6568		const sign_prefix = if (is_signed_int) "" else "un";
6569		const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
6570		sign_prefix,
6571		compiler_rt_operand_abbrev,
6572		compiler_rt_dest_abbrev,
6573		});
6574
6575		var param_type = rt_int_ty;
6576		if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) {
6577		// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6578		// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6579		param_type = try o.builder.vectorType(.normal, 2, .i64);
6580		extended = try self.wip.cast(.bitcast, extended, param_type, "");
6581		}
6582
6583		const libc_fn = try self.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty);
6584		return self.wip.call(
6585		.normal,
6586		.ccc,
6587		.none,
6588		libc_fn.typeOf(&o.builder),
6589		libc_fn.toValue(&o.builder),
6590		&.{extended},
6591		"",
6592		);
6593		}
6594
6595		fn airIntFromFloat(
6596		self: *FuncGen,
6597		inst: Air.Inst.Index,
6598		fast: Builder.FastMathKind,
6599		) !Builder.Value {
6600		_ = fast;
6601
6602		const o = self.ng.object;
6603		const pt = self.ng.pt;
6604		const zcu = pt.zcu;
6605		const target = zcu.getTarget();
6606		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6607
6608		const operand = try self.resolveInst(ty_op.operand);
6609		const operand_ty = self.typeOf(ty_op.operand);
6610		const operand_scalar_ty = operand_ty.scalarType(zcu);
6611
6612		const dest_ty = self.typeOfIndex(inst);
6613		const dest_scalar_ty = dest_ty.scalarType(zcu);
6614		const dest_llvm_ty = try o.lowerType(pt, dest_ty);
6615
6616		if (intrinsicsAllowed(operand_scalar_ty, target)) {
6617		// TODO set fast math flag
6618		return self.wip.conv(
6619		if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned,
6620		operand,
6621		dest_llvm_ty,
6622		"",
6623		);
6624		}
6625
6626		const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
6627		return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)});
6628		};
6629		const ret_ty = try o.builder.intType(rt_int_bits);
6630		const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
6631		// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
6632		// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
6633		break :b try o.builder.vectorType(.normal, 2, .i64);
6634		} else ret_ty;
6635
6636		const operand_bits = operand_scalar_ty.floatBits(target);
6637		const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits);
6638
6639		const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits);
6640		const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns";
6641
6642		const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
6643		sign_prefix,
6644		compiler_rt_operand_abbrev,
6645		compiler_rt_dest_abbrev,
6646		});
6647
6648		const operand_llvm_ty = try o.lowerType(pt, operand_ty);
6649		const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
6650		var result = try self.wip.call(
6651		.normal,
6652		.ccc,
6653		.none,
6654		libc_fn.typeOf(&o.builder),
6655		libc_fn.toValue(&o.builder),
6656		&.{operand},
6657		"",
6658		);
6659
6660		if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, "");
6661		if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, "");
6662		return result;
6663		}
6664
6665		fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6666		const zcu = fg.ng.pt.zcu;
6667		return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
6668		}
6669
6670		fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
6671		const o = fg.ng.object;
6672		const pt = fg.ng.pt;
6673		const zcu = pt.zcu;
6674		const llvm_usize = try o.lowerType(pt, Type.usize);
6675		switch (ty.ptrSize(zcu)) {
6676		.slice => {
6677		const len = try fg.wip.extractValue(ptr, &.{1}, "");
6678		const elem_ty = ty.childType(zcu);
6679		const abi_size = elem_ty.abiSize(zcu);
6680		if (abi_size == 1) return len;
6681		const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
6682		return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
6683		},
6684		.one => {
6685		const array_ty = ty.childType(zcu);
6686		const elem_ty = array_ty.childType(zcu);
6687		const abi_size = elem_ty.abiSize(zcu);
6688		return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
6689		},
6690		.many, .c => unreachable,
6691		}
6692		}
6693
6694		fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) !Builder.Value {
6695		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6696		const operand = try self.resolveInst(ty_op.operand);
6697		return self.wip.extractValue(operand, &.{index}, "");
6698		}
6699
6700		fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: c_uint) !Builder.Value {
6701		const o = self.ng.object;
6702		const pt = self.ng.pt;
6703		const zcu = pt.zcu;
6704		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6705		const slice_ptr = try self.resolveInst(ty_op.operand);
6706		const slice_ptr_ty = self.typeOf(ty_op.operand);
6707		const slice_llvm_ty = try o.lowerType(pt, slice_ptr_ty.childType(zcu));
6708
6709		return self.wip.gepStruct(slice_llvm_ty, slice_ptr, index, "");
6710		}
6711
6712		fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6713		const o = self.ng.object;
6714		const pt = self.ng.pt;
6715		const zcu = pt.zcu;
6716		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6717		const slice_ty = self.typeOf(bin_op.lhs);
6718		const slice = try self.resolveInst(bin_op.lhs);
6719		const index = try self.resolveInst(bin_op.rhs);
6720		const slice_info = slice_ty.ptrInfo(zcu);
6721		assert(slice_info.flags.size == .slice);
6722		const elem_ty: Type = .fromInterned(slice_info.child);
6723		const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6724		const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6725		const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6726		const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
6727		const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
6728		self.maybeMarkAllowZeroAccess(slice_info);
6729		if (isByRef(elem_ty, zcu)) {
6730		return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
6731		} else {
6732		return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
6733		}
6734		}
6735
6736		fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6737		const o = self.ng.object;
6738		const pt = self.ng.pt;
6739		const zcu = pt.zcu;
6740		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6741		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6742		const slice_ty = self.typeOf(bin_op.lhs);
6743
6744		const slice = try self.resolveInst(bin_op.lhs);
6745		const index = try self.resolveInst(bin_op.rhs);
6746		const llvm_elem_ty = try o.lowerType(pt, slice_ty.childType(zcu));
6747		const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
6748		return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{index}, "");
6749		}
6750
6751		fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6752		const o = self.ng.object;
6753		const pt = self.ng.pt;
6754		const zcu = pt.zcu;
6755
6756		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6757		const array_ty = self.typeOf(bin_op.lhs);
6758		const array_llvm_val = try self.resolveInst(bin_op.lhs);
6759		const rhs = try self.resolveInst(bin_op.rhs);
6760		const array_llvm_ty = try o.lowerType(pt, array_ty);
6761		const elem_ty = array_ty.childType(zcu);
6762		if (isByRef(array_ty, zcu)) {
6763		const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{
6764		try o.builder.intValue(try o.lowerType(pt, Type.usize), 0),
6765		rhs,
6766		}, "");
6767		if (isByRef(elem_ty, zcu)) {
6768		const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm();
6769		return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal);
6770		} else {
6771		return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
6772		}
6773		}
6774
6775		// This branch can be reached for vectors, which are always by-value.
6776		return self.wip.extractElement(array_llvm_val, rhs, "");
6777		}
6778
6779		fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6780		const o = self.ng.object;
6781		const pt = self.ng.pt;
6782		const zcu = pt.zcu;
6783		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6784		const ptr_ty = self.typeOf(bin_op.lhs);
6785		const elem_ty = ptr_ty.indexableElem(zcu);
6786		const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6787		const base_ptr = try self.resolveInst(bin_op.lhs);
6788		const rhs = try self.resolveInst(bin_op.rhs);
6789		const ptr = try self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
6790		if (isByRef(elem_ty, zcu)) {
6791		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6792		const ptr_align = (ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu))).toLlvm();
6793		return self.loadByRef(ptr, elem_ty, ptr_align, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal);
6794		}
6795
6796		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
6797
6798		return self.load(ptr, ptr_ty);
6799		}
6800
6801		fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6802		const o = self.ng.object;
6803		const pt = self.ng.pt;
6804		const zcu = pt.zcu;
6805		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6806		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
6807		const ptr_ty = self.typeOf(bin_op.lhs);
6808		const elem_ty = ptr_ty.indexableElem(zcu);
6809		assert(elem_ty.hasRuntimeBits(zcu));
6810
6811		const base_ptr = try self.resolveInst(bin_op.lhs);
6812		const rhs = try self.resolveInst(bin_op.rhs);
6813
6814		const elem_ptr = ty_pl.ty.toType();
6815		if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
6816
6817		const llvm_elem_ty = try o.lowerType(pt, elem_ty);
6818		return self.wip.gep(.inbounds, llvm_elem_ty, base_ptr, &.{rhs}, "");
6819		}
6820
6821		fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6822		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6823		const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6824		const struct_ptr = try self.resolveInst(struct_field.struct_operand);
6825		const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
6826		return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index);
6827		}
6828
6829		fn airStructFieldPtrIndex(
6830		self: *FuncGen,
6831		inst: Air.Inst.Index,
6832		field_index: u32,
6833		) !Builder.Value {
6834		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6835		const struct_ptr = try self.resolveInst(ty_op.operand);
6836		const struct_ptr_ty = self.typeOf(ty_op.operand);
6837		return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index);
6838		}
6839
6840		fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6841		const o = self.ng.object;
6842		const pt = self.ng.pt;
6843		const zcu = pt.zcu;
6844		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6845		const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
6846		const struct_ty = self.typeOf(struct_field.struct_operand);
6847		const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
6848		const field_index = struct_field.field_index;
6849		const field_ty = struct_ty.fieldType(field_index, zcu);
6850		if (!field_ty.hasRuntimeBits(zcu)) return .none;
6851
6852		if (!isByRef(struct_ty, zcu)) {
6853		assert(!isByRef(field_ty, zcu));
6854		switch (struct_ty.zigTypeTag(zcu)) {
6855		.@"struct" => switch (struct_ty.containerLayout(zcu)) {
6856		.@"packed" => {
6857		const struct_type = zcu.typeToStruct(struct_ty).?;
6858		const bit_offset = zcu.structPackedFieldBitOffset(struct_type, field_index);
6859		const containing_int = struct_llvm_val;
6860		const shift_amt =
6861		try o.builder.intValue(containing_int.typeOfWip(&self.wip), bit_offset);
6862		const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
6863		const elem_llvm_ty = try o.lowerType(pt, field_ty);
6864		if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
6865		const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6866		const truncated_int =
6867		try self.wip.cast(.trunc, shifted_value, same_size_int, "");
6868		return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6869		}
6870		return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
6871		},
6872		else => {
6873		const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6874		return self.wip.extractValue(struct_llvm_val, &.{llvm_field_index}, "");
6875		},
6876		},
6877		.@"union" => {
6878		assert(struct_ty.containerLayout(zcu) == .@"packed");
6879		const containing_int = struct_llvm_val;
6880		const elem_llvm_ty = try o.lowerType(pt, field_ty);
6881		if (field_ty.zigTypeTag(zcu) == .float or field_ty.zigTypeTag(zcu) == .vector) {
6882		const same_size_int = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
6883		const truncated_int =
6884		try self.wip.cast(.trunc, containing_int, same_size_int, "");
6885		return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
6886		}
6887		return self.wip.cast(.trunc, containing_int, elem_llvm_ty, "");
6888		},
6889		else => unreachable,
6890		}
6891		}
6892
6893		switch (struct_ty.zigTypeTag(zcu)) {
6894		.@"struct" => {
6895		const layout = struct_ty.containerLayout(zcu);
6896		assert(layout != .@"packed");
6897		const struct_llvm_ty = try o.lowerType(pt, struct_ty);
6898		const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
6899		const field_ptr =
6900		try self.wip.gepStruct(struct_llvm_ty, struct_llvm_val, llvm_field_index, "");
6901		const explicit_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
6902		const field_ptr_ty = try pt.ptrType(.{
6903		.child = field_ty.toIntern(),
6904		.flags = .{ .alignment = explicit_alignment },
6905		});
6906		if (isByRef(field_ty, zcu)) {
6907		const alignment = switch (explicit_alignment) {
6908		.none => field_ty.abiAlignment(zcu),
6909		else => \|a\| a,
6910		};
6911		return self.loadByRef(field_ptr, field_ty, alignment.toLlvm(), .normal);
6912		} else {
6913		return self.load(field_ptr, field_ptr_ty);
6914		}
6915		},
6916		.@"union" => {
6917		const union_llvm_ty = try o.lowerType(pt, struct_ty);
6918		const layout = struct_ty.unionGetLayout(zcu);
6919		const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
6920		const field_ptr =
6921		try self.wip.gepStruct(union_llvm_ty, struct_llvm_val, payload_index, "");
6922		const payload_alignment = layout.payload_align.toLlvm();
6923		if (isByRef(field_ty, zcu)) {
6924		return self.loadByRef(field_ptr, field_ty, payload_alignment, .normal);
6925		} else {
6926		return self.loadTruncate(.normal, field_ty, field_ptr, payload_alignment);
6927		}
6928		},
6929		else => unreachable,
6930		}
6931		}
6932
6933		fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6934		const o = self.ng.object;
6935		const pt = self.ng.pt;
6936		const zcu = pt.zcu;
6937		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6938		const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
6939
6940		const field_ptr = try self.resolveInst(extra.field_ptr);
6941
6942		const parent_ty = ty_pl.ty.toType().childType(zcu);
6943		const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
6944		if (field_offset == 0) return field_ptr;
6945
6946		const res_ty = try o.lowerType(pt, ty_pl.ty.toType());
6947		const llvm_usize = try o.lowerType(pt, Type.usize);
6948
6949		const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
6950		const base_ptr_int = try self.wip.bin(
6951		.@"sub nuw",
6952		field_ptr_int,
6953		try o.builder.intValue(llvm_usize, field_offset),
6954		"",
6955		);
6956		return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
6957		}
6958
6959		fn airNot(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6960		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6961		const operand = try self.resolveInst(ty_op.operand);
6962
6963		return self.wip.not(operand, "");
6964		}
6965
6966		fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) !void {
6967		_ = inst;
6968		_ = try self.wip.@"unreachable"();
6969		}
6970
6971		fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6972		const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
6973		self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
6974		self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
6975
6976		self.wip.debug_location = .{ .location = .{
6977		.line = self.prev_dbg_line,
6978		.column = self.prev_dbg_column,
6979		.scope = self.scope.toOptional(),
6980		.inlined_at = self.inlined_at,
6981		} };
6982
6983		return .none;
6984		}
6985
6986		fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6987		_ = self;
6988		_ = inst;
6989		return .none;
6990		}
6991
6992		fn airDbgInlineBlock(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6993		const block = self.air.unwrapDbgBlock(inst);
6994		self.arg_inline_index = 0;
6995		return self.lowerBlock(inst, block.func, block.body);
6996		}
6997
6998		fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
6999		const o = self.ng.object;
7000		const pt = self.ng.pt;
7001		const zcu = pt.zcu;
7002		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7003		const operand = try self.resolveInst(pl_op.operand);
7004		const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7005		const ptr_ty = self.typeOf(pl_op.operand);
7006
7007		const debug_local_var = try o.builder.debugLocalVar(
7008		try o.builder.metadataString(name.toSlice(self.air)),
7009		self.file,
7010		self.scope,
7011		self.prev_dbg_line,
7012		try o.getDebugType(pt, ptr_ty.childType(zcu)),
7013		);
7014
7015		_ = try self.wip.callIntrinsic(
7016		.normal,
7017		.none,
7018		.@"dbg.declare",
7019		&.{},
7020		&.{
7021		(try self.wip.debugValue(operand)).toValue(),
7022		debug_local_var.toValue(),
7023		(try o.builder.debugExpression(&.{})).toValue(),
7024		},
7025		"",
7026		);
7027
7028		return .none;
7029		}
7030
7031		fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) !Builder.Value {
7032		const o = self.ng.object;
7033		const pt = self.ng.pt;
7034		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7035		const operand = try self.resolveInst(pl_op.operand);
7036		const operand_ty = self.typeOf(pl_op.operand);
7037		const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
7038		const name_slice = name.toSlice(self.air);
7039		const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
7040		const debug_local_var = if (is_arg) try o.builder.debugParameter(
7041		metadata_name,
7042		self.file,
7043		self.scope,
7044		self.prev_dbg_line,
7045		try o.getDebugType(pt, operand_ty),
7046		arg_no: {
7047		self.arg_inline_index += 1;
7048		break :arg_no self.arg_inline_index;
7049		},
7050		) else try o.builder.debugLocalVar(
7051		metadata_name,
7052		self.file,
7053		self.scope,
7054		self.prev_dbg_line,
7055		try o.getDebugType(pt, operand_ty),
7056		);
7057
7058		const zcu = pt.zcu;
7059		const owner_mod = self.ng.ownerModule();
7060		if (isByRef(operand_ty, zcu)) {
7061		_ = try self.wip.callIntrinsic(
7062		.normal,
7063		.none,
7064		.@"dbg.declare",
7065		&.{},
7066		&.{
7067		(try self.wip.debugValue(operand)).toValue(),
7068		debug_local_var.toValue(),
7069		(try o.builder.debugExpression(&.{})).toValue(),
7070		},
7071		"",
7072		);
7073		} else if (owner_mod.optimize_mode == .Debug and !self.is_naked) {
7074		// We avoid taking this path for naked functions because there's no guarantee that such
7075		// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
7076
7077		const alignment = operand_ty.abiAlignment(zcu).toLlvm();
7078		const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
7079		_ = try self.wip.store(.normal, operand, alloca, alignment);
7080		_ = try self.wip.callIntrinsic(
7081		.normal,
7082		.none,
7083		.@"dbg.declare",
7084		&.{},
7085		&.{
7086		(try self.wip.debugValue(alloca)).toValue(),
7087		debug_local_var.toValue(),
7088		(try o.builder.debugExpression(&.{})).toValue(),
7089		},
7090		"",
7091		);
7092		} else {
7093		_ = try self.wip.callIntrinsic(
7094		.normal,
7095		.none,
7096		.@"dbg.value",
7097		&.{},
7098		&.{
7099		(try self.wip.debugValue(operand)).toValue(),
7100		debug_local_var.toValue(),
7101		(try o.builder.debugExpression(&.{})).toValue(),
7102		},
7103		"",
7104		);
7105		}
7106		return .none;
7107		}
7108
7109		fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7110		// Eventually, the Zig compiler needs to be reworked to have inline
7111		// assembly go through the same parsing code regardless of backend, and
7112		// have LLVM-flavored inline assembly be output from that assembler.
7113		// We don't have such an assembler implemented yet though. For now,
7114		// this implementation feeds the inline assembly code directly to LLVM.
7115
7116		const o = self.ng.object;
7117		const unwrapped_asm = self.air.unwrapAsm(inst);
7118		const is_volatile = unwrapped_asm.is_volatile;
7119		const gpa = self.gpa;
7120
7121		const outputs = unwrapped_asm.outputs;
7122		const inputs = unwrapped_asm.inputs;
7123
7124		var llvm_constraints: std.ArrayList(u8) = .empty;
7125		defer llvm_constraints.deinit(gpa);
7126
7127		var arena_allocator = std.heap.ArenaAllocator.init(gpa);
7128		defer arena_allocator.deinit();
7129		const arena = arena_allocator.allocator();
7130
7131		// The exact number of return / parameter values depends on which output values
7132		// are passed by reference as indirect outputs (determined below).
7133		const max_return_count = outputs.len;
7134		const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
7135		const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
7136		const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
7137
7138		const max_param_count = max_return_count + inputs.len + outputs.len;
7139		const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
7140		const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
7141		// This stores whether we need to add an elementtype attribute and
7142		// if so, the element type itself.
7143		const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
7144		const pt = self.ng.pt;
7145		const zcu = pt.zcu;
7146		const ip = &zcu.intern_pool;
7147		const target = zcu.getTarget();
7148
7149		var llvm_ret_i: usize = 0;
7150		var llvm_param_i: usize = 0;
7151		var total_i: usize = 0;
7152
7153		var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
7154		try name_map.ensureUnusedCapacity(arena, max_param_count);
7155
7156		var it = unwrapped_asm.iterateOutputs();
7157		while (it.next()) \|output\| {
7158		const constraint = output.constraint;
7159		const name = output.name;
7160
7161		try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
7162		if (total_i != 0) {
7163		llvm_constraints.appendAssumeCapacity(',');
7164		}
7165		llvm_constraints.appendAssumeCapacity('=');
7166
7167		if (output.operand != .none) {
7168		const output_inst = try self.resolveInst(output.operand);
7169		const output_ty = self.typeOf(output.operand);
7170		assert(output_ty.zigTypeTag(zcu) == .pointer);
7171		const elem_llvm_ty = try o.lowerType(pt, output_ty.childType(zcu));
7172
7173		switch (constraint[0]) {
7174		'=' => {},
7175		'+' => llvm_rw_vals[output.index] = output_inst,
7176		else => return self.todo("unsupported output constraint on output type '{c}'", .{
7177		constraint[0],
7178		}),
7179		}
7180
7181		self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
7182
7183		// Pass any non-return outputs indirectly, if the constraint accepts a memory location
7184		llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
7185		if (llvm_ret_indirect[output.index]) {
7186		// Pass the result by reference as an indirect output (e.g. "=*m")
7187		llvm_constraints.appendAssumeCapacity('*');
7188
7189		llvm_param_values[llvm_param_i] = output_inst;
7190		llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
7191		llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
7192		llvm_param_i += 1;
7193		} else {
7194		// Pass the result directly (e.g. "=r")
7195		llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
7196		llvm_ret_i += 1;
7197		}
7198		} else {
7199		switch (constraint[0]) {
7200		'=' => {},
7201		else => return self.todo("unsupported output constraint on result type '{s}'", .{
7202		constraint,
7203		}),
7204		}
7205
7206		llvm_ret_indirect[output.index] = false;
7207
7208		const ret_ty = self.typeOfIndex(inst);
7209		llvm_ret_types[llvm_ret_i] = try o.lowerType(pt, ret_ty);
7210		llvm_ret_i += 1;
7211		}
7212
7213		// LLVM uses commas internally to separate different constraints,
7214		// alternative constraints are achieved with pipes.
7215		// We still allow the user to use commas in a way that is similar
7216		// to GCC's inline assembly.
7217		// http://llvm.org/docs/LangRef.html#constraint-codes
7218		for (constraint[1..]) \|byte\| {
7219		switch (byte) {
7220		',' => llvm_constraints.appendAssumeCapacity('\|'),
7221		'*' => {}, // Indirect outputs are handled above
7222		else => llvm_constraints.appendAssumeCapacity(byte),
7223		}
7224		}
7225
7226		if (!std.mem.eql(u8, name, "_")) {
7227		const gop = name_map.getOrPutAssumeCapacity(name);
7228		if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
7229		gop.value_ptr.* = @intCast(total_i);
7230		}
7231		total_i += 1;
7232		}
7233
7234		it = unwrapped_asm.iterateInputs();
7235		while (it.next()) \|input\| {
7236		const constraint = input.constraint;
7237		const name = input.name;
7238
7239		const arg_llvm_value = try self.resolveInst(input.operand);
7240		const arg_ty = self.typeOf(input.operand);
7241		const is_by_ref = isByRef(arg_ty, zcu);
7242		if (is_by_ref) {
7243		if (constraintAllowsMemory(constraint)) {
7244		llvm_param_values[llvm_param_i] = arg_llvm_value;
7245		llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7246		} else {
7247		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7248		const arg_llvm_ty = try o.lowerType(pt, arg_ty);
7249		const load_inst =
7250		try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
7251		llvm_param_values[llvm_param_i] = load_inst;
7252		llvm_param_types[llvm_param_i] = arg_llvm_ty;
7253		}
7254		} else {
7255		if (constraintAllowsRegister(constraint)) {
7256		llvm_param_values[llvm_param_i] = arg_llvm_value;
7257		llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
7258		} else {
7259		const alignment = arg_ty.abiAlignment(zcu).toLlvm();
7260		const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
7261		_ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
7262		llvm_param_values[llvm_param_i] = arg_ptr;
7263		llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
7264		}
7265		}
7266
7267		try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
7268		if (total_i != 0) {
7269		llvm_constraints.appendAssumeCapacity(',');
7270		}
7271		for (constraint) \|byte\| {
7272		llvm_constraints.appendAssumeCapacity(switch (byte) {
7273		',' => '\|',
7274		else => byte,
7275		});
7276		}
7277
7278		if (!std.mem.eql(u8, name, "_")) {
7279		const gop = name_map.getOrPutAssumeCapacity(name);
7280		if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
7281		gop.value_ptr.* = @intCast(total_i);
7282		}
7283
7284		// In the case of indirect inputs, LLVM requires the callsite to have
7285		// an elementtype(<ty>) attribute.
7286		llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
7287		if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
7288
7289		break :blk try o.lowerType(pt, if (is_by_ref) arg_ty else arg_ty.childType(zcu));
7290		} else .none;
7291
7292		llvm_param_i += 1;
7293		total_i += 1;
7294		}
7295
7296		it = unwrapped_asm.iterateOutputs();
7297		while (it.next()) \|output\| {
7298		const constraint = output.constraint;
7299
7300		if (constraint[0] != '+') continue;
7301
7302		const rw_ty = self.typeOf(output.operand);
7303		const llvm_elem_ty = try o.lowerType(pt, rw_ty.childType(zcu));
7304		if (llvm_ret_indirect[output.index]) {
7305		llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
7306		llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
7307		} else {
7308		const alignment = rw_ty.abiAlignment(zcu).toLlvm();
7309		const loaded = try self.wip.load(
7310		if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7311		llvm_elem_ty,
7312		llvm_rw_vals[output.index],
7313		alignment,
7314		"",
7315		);
7316		llvm_param_values[llvm_param_i] = loaded;
7317		llvm_param_types[llvm_param_i] = llvm_elem_ty;
7318		}
7319
7320		try llvm_constraints.print(gpa, ",{d}", .{output.index});
7321
7322		// In the case of indirect inputs, LLVM requires the callsite to have
7323		// an elementtype(<ty>) attribute.
7324		llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
7325
7326		llvm_param_i += 1;
7327		total_i += 1;
7328		}
7329
7330		if (total_i != 0) try llvm_constraints.append(gpa, ',');
7331		const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
7332		const clobbers_ty = clobbers_val.typeOf(zcu);
7333		var clobbers_bigint_buf: Value.BigIntSpace = undefined;
7334		const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
7335		for (0..clobbers_ty.structFieldCount(zcu)) \|field_index\| {
7336		assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
7337		const limb_bits = @bitSizeOf(std.math.big.Limb);
7338		if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
7339		switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
7340		0 => continue, // field is false
7341		1 => {}, // field is true
7342		}
7343		const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
7344		total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
7345		}
7346
7347		// We have finished scanning through all inputs/outputs, so the number of
7348		// parameters and return values is known.
7349		const param_count = llvm_param_i;
7350		const return_count = llvm_ret_i;
7351
7352		// For some targets, Clang unconditionally adds some clobbers to all inline assembly.
7353		// While this is probably not strictly necessary, if we don't follow Clang's lead
7354		// here then we may risk tripping LLVM bugs since anything not used by Clang tends
7355		// to be buggy and regress often.
7356		switch (target.cpu.arch) {
7357		.x86_64, .x86 => {
7358		try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
7359		total_i += 3;
7360		},
7361		.mips, .mipsel, .mips64, .mips64el => {
7362		try llvm_constraints.appendSlice(gpa, "~{$1},");
7363		total_i += 1;
7364		},
7365		else => {},
7366		}
7367
7368		if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
7369
7370		const asm_source = unwrapped_asm.source;
7371
7372		// hackety hacks until stage2 has proper inline asm in the frontend.
7373		var rendered_template = std.array_list.Managed(u8).init(gpa);
7374		defer rendered_template.deinit();
7375
7376		const State = enum { start, percent, input, modifier };
7377
7378		var state: State = .start;
7379
7380		var name_start: usize = undefined;
7381		var modifier_start: usize = undefined;
7382		for (asm_source, 0..) \|byte, i\| {
7383		switch (state) {
7384		.start => switch (byte) {
7385		'%' => state = .percent,
7386		'$' => try rendered_template.appendSlice("$$"),
7387		else => try rendered_template.append(byte),
7388		},
7389		.percent => switch (byte) {
7390		'%' => {
7391		try rendered_template.append('%');
7392		state = .start;
7393		},
7394		'[' => {
7395		try rendered_template.append('$');
7396		try rendered_template.append('{');
7397		name_start = i + 1;
7398		state = .input;
7399		},
7400		'=' => {
7401		try rendered_template.appendSlice("${:uid}");
7402		state = .start;
7403		},
7404		else => {
7405		try rendered_template.append('%');
7406		try rendered_template.append(byte);
7407		state = .start;
7408		},
7409		},
7410		.input => switch (byte) {
7411		']', ':' => {
7412		const name = asm_source[name_start..i];
7413
7414		const index = name_map.get(name) orelse {
7415		// we should validate the assembly in Sema; by now it is too late
7416		return self.todo("unknown input or output name: '{s}'", .{name});
7417		};
7418		try rendered_template.print("{d}", .{index});
7419		if (byte == ':') {
7420		try rendered_template.append(':');
7421		modifier_start = i + 1;
7422		state = .modifier;
7423		} else {
7424		try rendered_template.append('}');
7425		state = .start;
7426		}
7427		},
7428		else => {},
7429		},
7430		.modifier => switch (byte) {
7431		']' => {
7432		try rendered_template.appendSlice(asm_source[modifier_start..i]);
7433		try rendered_template.append('}');
7434		state = .start;
7435		},
7436		else => {},
7437		},
7438		}
7439		}
7440
7441		var attributes: Builder.FunctionAttributes.Wip = .{};
7442		defer attributes.deinit(&o.builder);
7443		for (llvm_param_attrs[0..param_count], 0..) \|llvm_elem_ty, i\| if (llvm_elem_ty != .none)
7444		try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
7445
7446		const ret_llvm_ty = switch (return_count) {
7447		0 => .void,
7448		1 => llvm_ret_types[0],
7449		else => try o.builder.structType(.normal, llvm_ret_types),
7450		};
7451		const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
7452		const call = try self.wip.callAsm(
7453		try attributes.finish(&o.builder),
7454		llvm_fn_ty,
7455		.{ .sideeffect = is_volatile },
7456		try o.builder.string(rendered_template.items),
7457		try o.builder.string(llvm_constraints.items),
7458		llvm_param_values[0..param_count],
7459		"",
7460		);
7461
7462		var ret_val = call;
7463		llvm_ret_i = 0;
7464		for (outputs, 0..) \|output, i\| {
7465		if (llvm_ret_indirect[i]) continue;
7466
7467		const output_value = if (return_count > 1)
7468		try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
7469		else
7470		call;
7471
7472		if (output != .none) {
7473		const output_ptr = try self.resolveInst(output);
7474		const output_ptr_ty = self.typeOf(output);
7475		const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
7476		_ = try self.wip.store(
7477		if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
7478		output_value,
7479		output_ptr,
7480		alignment,
7481		);
7482		} else {
7483		ret_val = output_value;
7484		}
7485		llvm_ret_i += 1;
7486		}
7487
7488		return ret_val;
7489		}
7490
7491		fn airIsNonNull(
7492		self: *FuncGen,
7493		inst: Air.Inst.Index,
7494		operand_is_ptr: bool,
7495		cond: Builder.IntegerCondition,
7496		) !Builder.Value {
7497		const o = self.ng.object;
7498		const pt = self.ng.pt;
7499		const zcu = pt.zcu;
7500		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7501		const operand = try self.resolveInst(un_op);
7502		const operand_ty = self.typeOf(un_op);
7503		const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7504		const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7505		const payload_ty = optional_ty.optionalChild(zcu);
7506
7507		const access_kind: Builder.MemoryAccessKind =
7508		if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7509
7510		if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7511
7512		if (optional_ty.optionalReprIsPayload(zcu)) {
7513		const loaded = if (operand_is_ptr)
7514		try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7515		else
7516		operand;
7517		if (payload_ty.isSlice(zcu)) {
7518		const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
7519		const ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(
7520		payload_ty.ptrAddressSpace(zcu),
7521		zcu.getTarget(),
7522		));
7523		return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
7524		}
7525		return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
7526		}
7527
7528		comptime assert(optional_layout_version == 3);
7529
7530		if (!payload_ty.hasRuntimeBits(zcu)) {
7531		const loaded = if (operand_is_ptr)
7532		try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7533		else
7534		operand;
7535		return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
7536		}
7537
7538		const is_by_ref = operand_is_ptr or isByRef(optional_ty, zcu);
7539		return self.optCmpNull(cond, optional_llvm_ty, operand, is_by_ref, access_kind);
7540		}
7541
7542		fn airIsErr(
7543		self: *FuncGen,
7544		inst: Air.Inst.Index,
7545		cond: Builder.IntegerCondition,
7546		operand_is_ptr: bool,
7547		) !Builder.Value {
7548		const o = self.ng.object;
7549		const pt = self.ng.pt;
7550		const zcu = pt.zcu;
7551		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7552		const operand = try self.resolveInst(un_op);
7553		const operand_ty = self.typeOf(un_op);
7554		const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7555		const payload_ty = err_union_ty.errorUnionPayload(zcu);
7556		const error_type = try o.errorIntType(pt);
7557		const zero = try o.builder.intValue(error_type, 0);
7558
7559		const access_kind: Builder.MemoryAccessKind =
7560		if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7561
7562		if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7563		const val: Builder.Constant = switch (cond) {
7564		.eq => .true, // 0 == 0
7565		.ne => .false, // 0 != 0
7566		else => unreachable,
7567		};
7568		return val.toValue();
7569		}
7570
7571		if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7572
7573		if (!payload_ty.hasRuntimeBits(zcu)) {
7574		const loaded = if (operand_is_ptr)
7575		try self.wip.load(access_kind, try o.lowerType(pt, err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
7576		else
7577		operand;
7578		return self.wip.icmp(cond, loaded, zero, "");
7579		}
7580
7581		const err_field_index = try errUnionErrorOffset(payload_ty, pt);
7582
7583		const loaded = if (operand_is_ptr or isByRef(err_union_ty, zcu)) loaded: {
7584		const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7585		const err_alignment = if (operand_is_ptr)
7586		operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
7587		else
7588		.none;
7589		const err_field_ptr =
7590		try self.wip.gepStruct(err_union_llvm_ty, operand, err_field_index, "");
7591		break :loaded try self.wip.load(access_kind, error_type, err_field_ptr, err_alignment.toLlvm(), "");
7592		} else try self.wip.extractValue(operand, &.{err_field_index}, "");
7593		return self.wip.icmp(cond, loaded, zero, "");
7594		}
7595
7596		fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7597		const o = self.ng.object;
7598		const pt = self.ng.pt;
7599		const zcu = pt.zcu;
7600		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7601		const operand = try self.resolveInst(ty_op.operand);
7602		const optional_ty = self.typeOf(ty_op.operand).childType(zcu);
7603		const payload_ty = optional_ty.optionalChild(zcu);
7604		if (!payload_ty.hasRuntimeBits(zcu)) {
7605		// We have a pointer to a zero-bit value and we need to return
7606		// a pointer to a zero-bit value.
7607		return operand;
7608		}
7609		if (optional_ty.optionalReprIsPayload(zcu)) {
7610		// The payload and the optional are the same value.
7611		return operand;
7612		}
7613		return self.wip.gepStruct(try o.lowerType(pt, optional_ty), operand, 0, "");
7614		}
7615
7616		fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7617		comptime assert(optional_layout_version == 3);
7618
7619		const o = self.ng.object;
7620		const pt = self.ng.pt;
7621		const zcu = pt.zcu;
7622		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7623		const operand = try self.resolveInst(ty_op.operand);
7624		const optional_ptr_ty = self.typeOf(ty_op.operand);
7625		const optional_ty = optional_ptr_ty.childType(zcu);
7626		const payload_ty = optional_ty.optionalChild(zcu);
7627		const non_null_bit = try o.builder.intValue(.i8, 1);
7628
7629		const access_kind: Builder.MemoryAccessKind =
7630		if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7631
7632		if (!payload_ty.hasRuntimeBits(zcu)) {
7633		self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7634
7635		// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
7636		// Default alignment store because align of the non null bit is 1 anyway.
7637		_ = try self.wip.store(access_kind, non_null_bit, operand, .default);
7638		return operand;
7639		}
7640		if (optional_ty.optionalReprIsPayload(zcu)) {
7641		// The payload and the optional are the same value.
7642		// Setting to non-null will be done when the payload is set.
7643		return operand;
7644		}
7645
7646		// First set the non-null bit.
7647		const optional_llvm_ty = try o.lowerType(pt, optional_ty);
7648		const non_null_ptr = try self.wip.gepStruct(optional_llvm_ty, operand, 1, "");
7649
7650		self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
7651
7652		// Default alignment store because align of the non null bit is 1 anyway.
7653		_ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
7654
7655		// Then return the payload pointer (only if it's used).
7656		if (self.liveness.isUnused(inst)) return .none;
7657
7658		return self.wip.gepStruct(optional_llvm_ty, operand, 0, "");
7659		}
7660
7661		fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7662		const o = self.ng.object;
7663		const pt = self.ng.pt;
7664		const zcu = pt.zcu;
7665		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7666		const operand = try self.resolveInst(ty_op.operand);
7667		const optional_ty = self.typeOf(ty_op.operand);
7668		const payload_ty = self.typeOfIndex(inst);
7669		if (!payload_ty.hasRuntimeBits(zcu)) return .none;
7670
7671		if (optional_ty.optionalReprIsPayload(zcu)) {
7672		// Payload value is the same as the optional value.
7673		return operand;
7674		}
7675
7676		const opt_llvm_ty = try o.lowerType(pt, optional_ty);
7677		return self.optPayloadHandle(opt_llvm_ty, operand, optional_ty, false);
7678		}
7679
7680		fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) !Builder.Value {
7681		const o = self.ng.object;
7682		const pt = self.ng.pt;
7683		const zcu = pt.zcu;
7684		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7685		const operand = try self.resolveInst(ty_op.operand);
7686		const operand_ty = self.typeOf(ty_op.operand);
7687		const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7688		const result_ty = self.typeOfIndex(inst);
7689		const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
7690
7691		if (!payload_ty.hasRuntimeBits(zcu)) {
7692		return if (operand_is_ptr) operand else .none;
7693		}
7694		const offset = try errUnionPayloadOffset(payload_ty, pt);
7695		const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7696		if (operand_is_ptr) {
7697		return self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7698		} else if (isByRef(err_union_ty, zcu)) {
7699		const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
7700		const payload_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7701		if (isByRef(payload_ty, zcu)) {
7702		return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
7703		}
7704		const payload_llvm_ty = err_union_llvm_ty.structFields(&o.builder)[offset];
7705		return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
7706		}
7707		return self.wip.extractValue(operand, &.{offset}, "");
7708		}
7709
7710		fn airErrUnionErr(
7711		self: *FuncGen,
7712		inst: Air.Inst.Index,
7713		operand_is_ptr: bool,
7714		) !Builder.Value {
7715		const o = self.ng.object;
7716		const pt = self.ng.pt;
7717		const zcu = pt.zcu;
7718		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7719		const operand = try self.resolveInst(ty_op.operand);
7720		const operand_ty = self.typeOf(ty_op.operand);
7721		const error_type = try o.errorIntType(pt);
7722		const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
7723		if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
7724		if (operand_is_ptr) {
7725		return operand;
7726		} else {
7727		return o.builder.intValue(error_type, 0);
7728		}
7729		}
7730
7731		const access_kind: Builder.MemoryAccessKind =
7732		if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7733
7734		const payload_ty = err_union_ty.errorUnionPayload(zcu);
7735		if (!payload_ty.hasRuntimeBits(zcu)) {
7736		if (!operand_is_ptr) return operand;
7737
7738		self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7739
7740		return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
7741		}
7742
7743		const offset = try errUnionErrorOffset(payload_ty, pt);
7744
7745		if (operand_is_ptr or isByRef(err_union_ty, zcu)) {
7746		if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
7747
7748		const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7749		const err_field_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, offset, "");
7750		return self.wip.load(access_kind, error_type, err_field_ptr, .default, "");
7751		}
7752
7753		return self.wip.extractValue(operand, &.{offset}, "");
7754		}
7755
7756		fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7757		const o = self.ng.object;
7758		const pt = self.ng.pt;
7759		const zcu = pt.zcu;
7760		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7761		const operand = try self.resolveInst(ty_op.operand);
7762		const err_union_ptr_ty = self.typeOf(ty_op.operand);
7763		const err_union_ty = err_union_ptr_ty.childType(zcu);
7764		const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
7765
7766		const payload_ty = err_union_ty.errorUnionPayload(zcu);
7767		const non_error_val = try o.builder.intValue(try o.errorIntType(pt), 0);
7768
7769		const access_kind: Builder.MemoryAccessKind =
7770		if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
7771
7772		if (!payload_ty.hasRuntimeBits(zcu)) {
7773		self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7774		_ = try self.wip.store(access_kind, non_error_val, operand, err_union_ptr_align.toLlvm());
7775		return operand;
7776		}
7777		const err_union_llvm_ty = try o.lowerType(pt, err_union_ty);
7778		{
7779		self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
7780
7781		const err_int_ty = try pt.errorIntType();
7782		const error_alignment = err_int_ty.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
7783		const error_offset = try errUnionErrorOffset(payload_ty, pt);
7784		// First set the non-error value.
7785		const non_null_ptr = try self.wip.gepStruct(err_union_llvm_ty, operand, error_offset, "");
7786		_ = try self.wip.store(access_kind, non_error_val, non_null_ptr, error_alignment);
7787		}
7788		// Then return the payload pointer (only if it is used).
7789		if (self.liveness.isUnused(inst)) return .none;
7790
7791		const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7792		return self.wip.gepStruct(err_union_llvm_ty, operand, payload_offset, "");
7793		}
7794
7795		fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !Builder.Value {
7796		assert(self.err_ret_trace != .none);
7797		return self.err_ret_trace;
7798		}
7799
7800		fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7801		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
7802		self.err_ret_trace = try self.resolveInst(un_op);
7803		return .none;
7804		}
7805
7806		fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7807		const o = self.ng.object;
7808		const pt = self.ng.pt;
7809		const zcu = pt.zcu;
7810
7811		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
7812		const struct_ty = ty_pl.ty.toType();
7813		const field_index = ty_pl.payload;
7814
7815		const struct_llvm_ty = try o.lowerType(pt, struct_ty);
7816		const llvm_field_index = o.llvmFieldIndex(struct_ty, field_index).?;
7817		assert(self.err_ret_trace != .none);
7818		const field_ptr = try self.wip.gepStruct(struct_llvm_ty, self.err_ret_trace, llvm_field_index, "");
7819		const field_alignment = struct_ty.explicitFieldAlignment(field_index, zcu);
7820		const field_ty = struct_ty.fieldType(field_index, zcu);
7821		const field_ptr_ty = try pt.ptrType(.{
7822		.child = field_ty.toIntern(),
7823		.flags = .{ .alignment = field_alignment },
7824		});
7825		return self.load(field_ptr, field_ptr_ty);
7826		}
7827
7828		/// As an optimization, we want to avoid unnecessary copies of
7829		/// error union/optional types when returning from a function.
7830		/// Here, we scan forward in the current block, looking to see
7831		/// if the next instruction is a return (ignoring debug instructions).
7832		///
7833		/// The first instruction of `body_tail` is a wrap instruction.
7834		fn isNextRet(
7835		self: *FuncGen,
7836		body_tail: []const Air.Inst.Index,
7837		) bool {
7838		const air_tags = self.air.instructions.items(.tag);
7839		for (body_tail[1..]) \|body_inst\| {
7840		switch (air_tags[@intFromEnum(body_inst)]) {
7841		.ret => return true,
7842		.dbg_stmt => continue,
7843		else => return false,
7844		}
7845		}
7846		// The only way to get here is to hit the end of a loop instruction
7847		// (implicit repeat).
7848		return false;
7849		}
7850
7851		fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7852		const o = self.ng.object;
7853		const pt = self.ng.pt;
7854		const zcu = pt.zcu;
7855		const inst = body_tail[0];
7856		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7857		const payload_ty = self.typeOf(ty_op.operand);
7858		const non_null_bit = try o.builder.intValue(.i8, 1);
7859		comptime assert(optional_layout_version == 3);
7860		assert(payload_ty.hasRuntimeBits(zcu));
7861		const operand = try self.resolveInst(ty_op.operand);
7862		const optional_ty = self.typeOfIndex(inst);
7863		if (optional_ty.optionalReprIsPayload(zcu)) return operand;
7864		const llvm_optional_ty = try o.lowerType(pt, optional_ty);
7865		if (isByRef(optional_ty, zcu)) {
7866		const directReturn = self.isNextRet(body_tail);
7867		const optional_ptr = if (directReturn)
7868		self.ret_ptr
7869		else brk: {
7870		const alignment = optional_ty.abiAlignment(zcu).toLlvm();
7871		const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
7872		break :brk optional_ptr;
7873		};
7874
7875		const payload_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 0, "");
7876		const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7877		try self.store(payload_ptr, payload_ptr_ty, operand, .none);
7878		const non_null_ptr = try self.wip.gepStruct(llvm_optional_ty, optional_ptr, 1, "");
7879		_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
7880		return optional_ptr;
7881		}
7882		return self.wip.buildAggregate(llvm_optional_ty, &.{ operand, non_null_bit }, "");
7883		}
7884
7885		fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7886		const o = self.ng.object;
7887		const pt = self.ng.pt;
7888		const zcu = pt.zcu;
7889		const inst = body_tail[0];
7890		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7891		const err_un_ty = self.typeOfIndex(inst);
7892		const operand = try self.resolveInst(ty_op.operand);
7893		const payload_ty = self.typeOf(ty_op.operand);
7894		assert(payload_ty.hasRuntimeBits(zcu));
7895		const ok_err_code = try o.builder.intValue(try o.errorIntType(pt), 0);
7896		const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
7897
7898		const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7899		const error_offset = try errUnionErrorOffset(payload_ty, pt);
7900		if (isByRef(err_un_ty, zcu)) {
7901		const directReturn = self.isNextRet(body_tail);
7902		const result_ptr = if (directReturn)
7903		self.ret_ptr
7904		else brk: {
7905		const alignment = err_un_ty.abiAlignment(pt.zcu).toLlvm();
7906		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7907		break :brk result_ptr;
7908		};
7909
7910		const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
7911		const err_int_ty = try pt.errorIntType();
7912		const error_alignment = err_int_ty.abiAlignment(pt.zcu).toLlvm();
7913		_ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
7914		const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
7915		const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7916		try self.store(payload_ptr, payload_ptr_ty, operand, .none);
7917		return result_ptr;
7918		}
7919		var fields: [2]Builder.Value = undefined;
7920		fields[payload_offset] = operand;
7921		fields[error_offset] = ok_err_code;
7922		return self.wip.buildAggregate(err_un_llvm_ty, &fields, "");
7923		}
7924
7925		fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) !Builder.Value {
7926		const o = self.ng.object;
7927		const pt = self.ng.pt;
7928		const zcu = pt.zcu;
7929		const inst = body_tail[0];
7930		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
7931		const err_un_ty = self.typeOfIndex(inst);
7932		const payload_ty = err_un_ty.errorUnionPayload(zcu);
7933		const operand = try self.resolveInst(ty_op.operand);
7934		if (!payload_ty.hasRuntimeBits(zcu)) return operand;
7935		const err_un_llvm_ty = try o.lowerType(pt, err_un_ty);
7936
7937		const payload_offset = try errUnionPayloadOffset(payload_ty, pt);
7938		const error_offset = try errUnionErrorOffset(payload_ty, pt);
7939		if (isByRef(err_un_ty, zcu)) {
7940		const directReturn = self.isNextRet(body_tail);
7941		const result_ptr = if (directReturn)
7942		self.ret_ptr
7943		else brk: {
7944		const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
7945		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
7946		break :brk result_ptr;
7947		};
7948
7949		const err_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, error_offset, "");
7950		const err_int_ty = try pt.errorIntType();
7951		const error_alignment = err_int_ty.abiAlignment(zcu).toLlvm();
7952		_ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
7953		const payload_ptr = try self.wip.gepStruct(err_un_llvm_ty, result_ptr, payload_offset, "");
7954		const payload_ptr_ty = try pt.singleMutPtrType(payload_ty);
7955		// TODO store undef to payload_ptr
7956		_ = payload_ptr;
7957		_ = payload_ptr_ty;
7958		return result_ptr;
7959		}
7960
7961		// TODO set payload bytes to undef
7962		const undef = try o.builder.undefValue(err_un_llvm_ty);
7963		return self.wip.insertValue(undef, operand, &.{error_offset}, "");
7964		}
7965
7966		fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7967		const o = self.ng.object;
7968		const pt = self.ng.pt;
7969		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7970		const index = pl_op.payload;
7971		const llvm_usize = try o.lowerType(pt, Type.usize);
7972		return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
7973		try o.builder.intValue(.i32, index),
7974		}, "");
7975		}
7976
7977		fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7978		const o = self.ng.object;
7979		const pt = self.ng.pt;
7980		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
7981		const index = pl_op.payload;
7982		const llvm_isize = try o.lowerType(pt, Type.isize);
7983		return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
7984		try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
7985		}, "");
7986		}
7987
7988		fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7989		const o = fg.ng.object;
7990		const pt = fg.ng.pt;
7991		const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
7992		const llvm_ptr_const = try o.lowerNavRefValue(pt, ty_nav.nav);
7993		return llvm_ptr_const.toValue();
7994		}
7995
7996		fn airMin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
7997		const o = self.ng.object;
7998		const pt = self.ng.pt;
7999		const zcu = pt.zcu;
8000		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8001		const lhs = try self.resolveInst(bin_op.lhs);
8002		const rhs = try self.resolveInst(bin_op.rhs);
8003		const inst_ty = self.typeOfIndex(inst);
8004		const scalar_ty = inst_ty.scalarType(zcu);
8005
8006		if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
8007		return self.wip.callIntrinsic(
8008		.normal,
8009		.none,
8010		if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
8011		&.{try o.lowerType(pt, inst_ty)},
8012		&.{ lhs, rhs },
8013		"",
8014		);
8015		}
8016
8017		fn airMax(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8018		const o = self.ng.object;
8019		const pt = self.ng.pt;
8020		const zcu = pt.zcu;
8021		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8022		const lhs = try self.resolveInst(bin_op.lhs);
8023		const rhs = try self.resolveInst(bin_op.rhs);
8024		const inst_ty = self.typeOfIndex(inst);
8025		const scalar_ty = inst_ty.scalarType(zcu);
8026
8027		if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
8028		return self.wip.callIntrinsic(
8029		.normal,
8030		.none,
8031		if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
8032		&.{try o.lowerType(pt, inst_ty)},
8033		&.{ lhs, rhs },
8034		"",
8035		);
8036		}
8037
8038		fn airSlice(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8039		const o = self.ng.object;
8040		const pt = self.ng.pt;
8041		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8042		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8043		const ptr = try self.resolveInst(bin_op.lhs);
8044		const len = try self.resolveInst(bin_op.rhs);
8045		const inst_ty = self.typeOfIndex(inst);
8046		return self.wip.buildAggregate(try o.lowerType(pt, inst_ty), &.{ ptr, len }, "");
8047		}
8048
8049		fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8050		const zcu = self.ng.pt.zcu;
8051		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8052		const lhs = try self.resolveInst(bin_op.lhs);
8053		const rhs = try self.resolveInst(bin_op.rhs);
8054		const inst_ty = self.typeOfIndex(inst);
8055		const scalar_ty = inst_ty.scalarType(zcu);
8056
8057		if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
8058		return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
8059		}
8060
8061		fn airSafeArithmetic(
8062		fg: *FuncGen,
8063		inst: Air.Inst.Index,
8064		signed_intrinsic: Builder.Intrinsic,
8065		unsigned_intrinsic: Builder.Intrinsic,
8066		) !Builder.Value {
8067		const o = fg.ng.object;
8068		const pt = fg.ng.pt;
8069		const zcu = pt.zcu;
8070
8071		const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8072		const lhs = try fg.resolveInst(bin_op.lhs);
8073		const rhs = try fg.resolveInst(bin_op.rhs);
8074		const inst_ty = fg.typeOfIndex(inst);
8075		const scalar_ty = inst_ty.scalarType(zcu);
8076
8077		const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8078		const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8079		const results =
8080		try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
8081
8082		const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
8083		const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
8084		const overflow_bit = if (overflow_bits_ty.isVector(&o.builder))
8085		try fg.wip.callIntrinsic(
8086		.normal,
8087		.none,
8088		.@"vector.reduce.or",
8089		&.{overflow_bits_ty},
8090		&.{overflow_bits},
8091		"",
8092		)
8093		else
8094		overflow_bits;
8095
8096		const fail_block = try fg.wip.block(1, "OverflowFail");
8097		const ok_block = try fg.wip.block(1, "OverflowOk");
8098		_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
8099
8100		fg.wip.cursor = .{ .block = fail_block };
8101		try fg.buildSimplePanic(.integer_overflow);
8102
8103		fg.wip.cursor = .{ .block = ok_block };
8104		return fg.wip.extractValue(results, &.{0}, "");
8105		}
8106
8107		fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8108		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8109		const lhs = try self.resolveInst(bin_op.lhs);
8110		const rhs = try self.resolveInst(bin_op.rhs);
8111
8112		return self.wip.bin(.add, lhs, rhs, "");
8113		}
8114
8115		fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8116		const o = self.ng.object;
8117		const pt = self.ng.pt;
8118		const zcu = pt.zcu;
8119		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8120		const lhs = try self.resolveInst(bin_op.lhs);
8121		const rhs = try self.resolveInst(bin_op.rhs);
8122		const inst_ty = self.typeOfIndex(inst);
8123		const scalar_ty = inst_ty.scalarType(zcu);
8124		assert(scalar_ty.zigTypeTag(zcu) == .int);
8125		return self.wip.callIntrinsic(
8126		.normal,
8127		.none,
8128		if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
8129		&.{try o.lowerType(pt, inst_ty)},
8130		&.{ lhs, rhs },
8131		"",
8132		);
8133		}
8134
8135		fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8136		const zcu = self.ng.pt.zcu;
8137		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8138		const lhs = try self.resolveInst(bin_op.lhs);
8139		const rhs = try self.resolveInst(bin_op.rhs);
8140		const inst_ty = self.typeOfIndex(inst);
8141		const scalar_ty = inst_ty.scalarType(zcu);
8142
8143		if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
8144		return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
8145		}
8146
8147		fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8148		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8149		const lhs = try self.resolveInst(bin_op.lhs);
8150		const rhs = try self.resolveInst(bin_op.rhs);
8151
8152		return self.wip.bin(.sub, lhs, rhs, "");
8153		}
8154
8155		fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8156		const o = self.ng.object;
8157		const pt = self.ng.pt;
8158		const zcu = pt.zcu;
8159		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8160		const lhs = try self.resolveInst(bin_op.lhs);
8161		const rhs = try self.resolveInst(bin_op.rhs);
8162		const inst_ty = self.typeOfIndex(inst);
8163		const scalar_ty = inst_ty.scalarType(zcu);
8164		assert(scalar_ty.zigTypeTag(zcu) == .int);
8165		return self.wip.callIntrinsic(
8166		.normal,
8167		.none,
8168		if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
8169		&.{try o.lowerType(pt, inst_ty)},
8170		&.{ lhs, rhs },
8171		"",
8172		);
8173		}
8174
8175		fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8176		const zcu = self.ng.pt.zcu;
8177		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8178		const lhs = try self.resolveInst(bin_op.lhs);
8179		const rhs = try self.resolveInst(bin_op.rhs);
8180		const inst_ty = self.typeOfIndex(inst);
8181		const scalar_ty = inst_ty.scalarType(zcu);
8182
8183		if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
8184		return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
8185		}
8186
8187		fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8188		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8189		const lhs = try self.resolveInst(bin_op.lhs);
8190		const rhs = try self.resolveInst(bin_op.rhs);
8191
8192		return self.wip.bin(.mul, lhs, rhs, "");
8193		}
8194
8195		fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8196		const o = self.ng.object;
8197		const pt = self.ng.pt;
8198		const zcu = pt.zcu;
8199		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8200		const lhs = try self.resolveInst(bin_op.lhs);
8201		const rhs = try self.resolveInst(bin_op.rhs);
8202		const inst_ty = self.typeOfIndex(inst);
8203		const scalar_ty = inst_ty.scalarType(zcu);
8204		assert(scalar_ty.zigTypeTag(zcu) == .int);
8205		return self.wip.callIntrinsic(
8206		.normal,
8207		.none,
8208		if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
8209		&.{try o.lowerType(pt, inst_ty)},
8210		&.{ lhs, rhs, .@"0" },
8211		"",
8212		);
8213		}
8214
8215		fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8216		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8217		const lhs = try self.resolveInst(bin_op.lhs);
8218		const rhs = try self.resolveInst(bin_op.rhs);
8219		const inst_ty = self.typeOfIndex(inst);
8220
8221		return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8222		}
8223
8224		fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8225		const zcu = self.ng.pt.zcu;
8226		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8227		const lhs = try self.resolveInst(bin_op.lhs);
8228		const rhs = try self.resolveInst(bin_op.rhs);
8229		const inst_ty = self.typeOfIndex(inst);
8230		const scalar_ty = inst_ty.scalarType(zcu);
8231
8232		if (scalar_ty.isRuntimeFloat()) {
8233		const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8234		return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
8235		}
8236		return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
8237		}
8238
8239		fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8240		const o = self.ng.object;
8241		const pt = self.ng.pt;
8242		const zcu = pt.zcu;
8243		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8244		const lhs = try self.resolveInst(bin_op.lhs);
8245		const rhs = try self.resolveInst(bin_op.rhs);
8246		const inst_ty = self.typeOfIndex(inst);
8247		const scalar_ty = inst_ty.scalarType(zcu);
8248
8249		if (scalar_ty.isRuntimeFloat()) {
8250		const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8251		return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
8252		}
8253		if (scalar_ty.isSignedInt(zcu)) {
8254		const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8255
8256		const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8257		var stack align(@max(
8258		@alignOf(std.heap.StackFallbackAllocator(0)),
8259		@alignOf(ExpectedContents),
8260		)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8261		const allocator = stack.get();
8262
8263		const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8264		var smin_big_int: std.math.big.int.Mutable = .{
8265		.limbs = try allocator.alloc(
8266		std.math.big.Limb,
8267		std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8268		),
8269		.len = undefined,
8270		.positive = undefined,
8271		};
8272		defer allocator.free(smin_big_int.limbs);
8273		smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8274		const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8275		inst_llvm_ty.scalarType(&o.builder),
8276		smin_big_int.toConst(),
8277		));
8278
8279		const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
8280		const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
8281		const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
8282		const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
8283		const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
8284		const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
8285		return self.wip.bin(.@"add nsw", div, correction, "divFloor");
8286		}
8287		return self.wip.bin(.udiv, lhs, rhs, "");
8288		}
8289
8290		fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8291		const zcu = self.ng.pt.zcu;
8292		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8293		const lhs = try self.resolveInst(bin_op.lhs);
8294		const rhs = try self.resolveInst(bin_op.rhs);
8295		const inst_ty = self.typeOfIndex(inst);
8296		const scalar_ty = inst_ty.scalarType(zcu);
8297
8298		if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
8299		return self.wip.bin(
8300		if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
8301		lhs,
8302		rhs,
8303		"",
8304		);
8305		}
8306
8307		fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8308		const zcu = self.ng.pt.zcu;
8309		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8310		const lhs = try self.resolveInst(bin_op.lhs);
8311		const rhs = try self.resolveInst(bin_op.rhs);
8312		const inst_ty = self.typeOfIndex(inst);
8313		const scalar_ty = inst_ty.scalarType(zcu);
8314
8315		if (scalar_ty.isRuntimeFloat())
8316		return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8317		return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
8318		.srem
8319		else
8320		.urem, lhs, rhs, "");
8321		}
8322
8323		fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
8324		const o = self.ng.object;
8325		const pt = self.ng.pt;
8326		const zcu = pt.zcu;
8327		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8328		const lhs = try self.resolveInst(bin_op.lhs);
8329		const rhs = try self.resolveInst(bin_op.rhs);
8330		const inst_ty = self.typeOfIndex(inst);
8331		const inst_llvm_ty = try o.lowerType(pt, inst_ty);
8332		const scalar_ty = inst_ty.scalarType(zcu);
8333
8334		if (scalar_ty.isRuntimeFloat()) {
8335		const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
8336		const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
8337		const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
8338		const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8339		const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero });
8340		return self.wip.select(fast, ltz, c, a, "");
8341		}
8342		if (scalar_ty.isSignedInt(zcu)) {
8343		const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
8344		var stack align(@max(
8345		@alignOf(std.heap.StackFallbackAllocator(0)),
8346		@alignOf(ExpectedContents),
8347		)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
8348		const allocator = stack.get();
8349
8350		const scalar_bits = inst_llvm_ty.scalarBits(&o.builder);
8351		var smin_big_int: std.math.big.int.Mutable = .{
8352		.limbs = try allocator.alloc(
8353		std.math.big.Limb,
8354		std.math.big.int.calcTwosCompLimbCount(scalar_bits),
8355		),
8356		.len = undefined,
8357		.positive = undefined,
8358		};
8359		defer allocator.free(smin_big_int.limbs);
8360		smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
8361		const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
8362		inst_llvm_ty.scalarType(&o.builder),
8363		smin_big_int.toConst(),
8364		));
8365
8366		const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
8367		const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
8368		const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
8369		const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
8370		const zero = try o.builder.zeroInitValue(inst_llvm_ty);
8371		const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
8372		return self.wip.bin(.@"add nsw", correction, rem, "mod");
8373		}
8374		return self.wip.bin(.urem, lhs, rhs, "");
8375		}
8376
8377		fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8378		const o = self.ng.object;
8379		const pt = self.ng.pt;
8380		const zcu = pt.zcu;
8381		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8382		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8383		const ptr = try self.resolveInst(bin_op.lhs);
8384		const offset = try self.resolveInst(bin_op.rhs);
8385		const ptr_ty = self.typeOf(bin_op.lhs);
8386		const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
8387		switch (ptr_ty.ptrSize(zcu)) {
8388		// It's a pointer to an array, so according to LLVM we need an extra GEP index.
8389		.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8390		try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), offset,
8391		}, ""),
8392		.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{offset}, ""),
8393		.slice => {
8394		const base = try self.wip.extractValue(ptr, &.{0}, "");
8395		return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{offset}, "");
8396		},
8397		}
8398		}
8399
8400		fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8401		const o = self.ng.object;
8402		const pt = self.ng.pt;
8403		const zcu = pt.zcu;
8404		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8405		const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
8406		const ptr = try self.resolveInst(bin_op.lhs);
8407		const offset = try self.resolveInst(bin_op.rhs);
8408		const negative_offset = try self.wip.neg(offset, "");
8409		const ptr_ty = self.typeOf(bin_op.lhs);
8410		const llvm_elem_ty = try o.lowerType(pt, ptr_ty.childType(zcu));
8411		switch (ptr_ty.ptrSize(zcu)) {
8412		// It's a pointer to an array, so according to LLVM we need an extra GEP index.
8413		.one => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{
8414		try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), negative_offset,
8415		}, ""),
8416		.c, .many => return self.wip.gep(.inbounds, llvm_elem_ty, ptr, &.{negative_offset}, ""),
8417		.slice => {
8418		const base = try self.wip.extractValue(ptr, &.{0}, "");
8419		return self.wip.gep(.inbounds, llvm_elem_ty, base, &.{negative_offset}, "");
8420		},
8421		}
8422		}
8423
8424		fn airOverflow(
8425		self: *FuncGen,
8426		inst: Air.Inst.Index,
8427		signed_intrinsic: Builder.Intrinsic,
8428		unsigned_intrinsic: Builder.Intrinsic,
8429		) !Builder.Value {
8430		const o = self.ng.object;
8431		const pt = self.ng.pt;
8432		const zcu = pt.zcu;
8433		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8434		const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8435
8436		const lhs = try self.resolveInst(extra.lhs);
8437		const rhs = try self.resolveInst(extra.rhs);
8438
8439		const lhs_ty = self.typeOf(extra.lhs);
8440		const scalar_ty = lhs_ty.scalarType(zcu);
8441		const inst_ty = self.typeOfIndex(inst);
8442
8443		const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
8444		const llvm_inst_ty = try o.lowerType(pt, inst_ty);
8445		const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
8446		const results =
8447		try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
8448
8449		const result_val = try self.wip.extractValue(results, &.{0}, "");
8450		const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
8451
8452		const result_index = o.llvmFieldIndex(inst_ty, 0).?;
8453		const overflow_index = o.llvmFieldIndex(inst_ty, 1).?;
8454
8455		if (isByRef(inst_ty, zcu)) {
8456		const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
8457		const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
8458		{
8459		const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, result_index, "");
8460		_ = try self.wip.store(.normal, result_val, field_ptr, result_alignment);
8461		}
8462		{
8463		const field_ptr = try self.wip.gepStruct(llvm_inst_ty, alloca_inst, overflow_index, "");
8464		_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
8465		}
8466
8467		return alloca_inst;
8468		}
8469
8470		var fields: [2]Builder.Value = undefined;
8471		fields[result_index] = result_val;
8472		fields[overflow_index] = overflow_bit;
8473		return self.wip.buildAggregate(llvm_inst_ty, &fields, "");
8474		}
8475
8476		fn buildElementwiseCall(
8477		self: *FuncGen,
8478		llvm_fn: Builder.Function.Index,
8479		args_vectors: []const Builder.Value,
8480		result_vector: Builder.Value,
8481		vector_len: usize,
8482		) !Builder.Value {
8483		const o = self.ng.object;
8484		assert(args_vectors.len <= 3);
8485
8486		var i: usize = 0;
8487		var result = result_vector;
8488		while (i < vector_len) : (i += 1) {
8489		const index_i32 = try o.builder.intValue(.i32, i);
8490
8491		var args: [3]Builder.Value = undefined;
8492		for (args[0..args_vectors.len], args_vectors) \|*arg_elem, arg_vector\| {
8493		arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, "");
8494		}
8495		const result_elem = try self.wip.call(
8496		.normal,
8497		.ccc,
8498		.none,
8499		llvm_fn.typeOf(&o.builder),
8500		llvm_fn.toValue(&o.builder),
8501		args[0..args_vectors.len],
8502		"",
8503		);
8504		result = try self.wip.insertElement(result, result_elem, index_i32, "");
8505		}
8506		return result;
8507		}
8508
8509		fn getLibcFunction(
8510		self: *FuncGen,
8511		fn_name: Builder.StrtabString,
8512		param_types: []const Builder.Type,
8513		return_type: Builder.Type,
8514		) Allocator.Error!Builder.Function.Index {
8515		const o = self.ng.object;
8516		if (o.builder.getGlobal(fn_name)) \|global\| return switch (global.ptrConst(&o.builder).kind) {
8517		.alias => \|alias\| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
8518		.function => \|function\| function,
8519		.variable, .replaced => unreachable,
8520		};
8521		return o.builder.addFunction(
8522		try o.builder.fnType(return_type, param_types, .normal),
8523		fn_name,
8524		toLlvmAddressSpace(.generic, self.ng.pt.zcu.getTarget()),
8525		);
8526		}
8527
8528		/// Creates a floating point comparison by lowering to the appropriate
8529		/// hardware instruction or softfloat routine for the target
8530		fn buildFloatCmp(
8531		self: *FuncGen,
8532		fast: Builder.FastMathKind,
8533		pred: math.CompareOperator,
8534		ty: Type,
8535		params: [2]Builder.Value,
8536		) !Builder.Value {
8537		const o = self.ng.object;
8538		const pt = self.ng.pt;
8539		const zcu = pt.zcu;
8540		const target = zcu.getTarget();
8541		const scalar_ty = ty.scalarType(zcu);
8542		const scalar_llvm_ty = try o.lowerType(pt, scalar_ty);
8543
8544		if (intrinsicsAllowed(scalar_ty, target)) {
8545		const cond: Builder.FloatCondition = switch (pred) {
8546		.eq => .oeq,
8547		.neq => .une,
8548		.lt => .olt,
8549		.lte => .ole,
8550		.gt => .ogt,
8551		.gte => .oge,
8552		};
8553		return self.wip.fcmp(fast, cond, params[0], params[1], "");
8554		}
8555
8556		const float_bits = scalar_ty.floatBits(target);
8557		const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits);
8558		const fn_base_name = switch (pred) {
8559		.neq => "ne",
8560		.eq => "eq",
8561		.lt => "lt",
8562		.lte => "le",
8563		.gt => "gt",
8564		.gte => "ge",
8565		};
8566		const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev });
8567
8568		const libc_fn = try self.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32);
8569
8570		const int_cond: Builder.IntegerCondition = switch (pred) {
8571		.eq => .eq,
8572		.neq => .ne,
8573		.lt => .slt,
8574		.lte => .sle,
8575		.gt => .sgt,
8576		.gte => .sge,
8577		};
8578
8579		if (ty.zigTypeTag(zcu) == .vector) {
8580		const vec_len = ty.vectorLen(zcu);
8581		const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32);
8582
8583		const init = try o.builder.poisonValue(vector_result_ty);
8584		const result = try self.buildElementwiseCall(libc_fn, &params, init, vec_len);
8585
8586		const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0");
8587		return self.wip.icmp(int_cond, result, zero_vector, "");
8588		}
8589
8590		const result = try self.wip.call(
8591		.normal,
8592		.ccc,
8593		.none,
8594		libc_fn.typeOf(&o.builder),
8595		libc_fn.toValue(&o.builder),
8596		&params,
8597		"",
8598		);
8599		return self.wip.icmp(int_cond, result, .@"0", "");
8600		}
8601
8602		const FloatOp = enum {
8603		add,
8604		ceil,
8605		cos,
8606		div,
8607		exp,
8608		exp2,
8609		fabs,
8610		floor,
8611		fma,
8612		fmax,
8613		fmin,
8614		fmod,
8615		log,
8616		log10,
8617		log2,
8618		mul,
8619		neg,
8620		round,
8621		sin,
8622		sqrt,
8623		sub,
8624		tan,
8625		trunc,
8626		};
8627
8628		const FloatOpStrat = union(enum) {
8629		intrinsic: []const u8,
8630		libc: Builder.String,
8631		};
8632
8633		/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
8634		/// by lowering to the appropriate hardware instruction or softfloat
8635		/// routine for the target
8636		fn buildFloatOp(
8637		self: *FuncGen,
8638		comptime op: FloatOp,
8639		fast: Builder.FastMathKind,
8640		ty: Type,
8641		comptime params_len: usize,
8642		params: [params_len]Builder.Value,
8643		) !Builder.Value {
8644		const o = self.ng.object;
8645		const pt = self.ng.pt;
8646		const zcu = pt.zcu;
8647		const target = zcu.getTarget();
8648		const scalar_ty = ty.scalarType(zcu);
8649		const llvm_ty = try o.lowerType(pt, ty);
8650
8651		if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
8652		// Some operations are dedicated LLVM instructions, not available as intrinsics
8653		.neg => return self.wip.un(.fneg, params[0], ""),
8654		.add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) {
8655		.normal => switch (op) {
8656		.add => .fadd,
8657		.sub => .fsub,
8658		.mul => .fmul,
8659		.div => .fdiv,
8660		.fmod => .frem,
8661		else => unreachable,
8662		},
8663		.fast => switch (op) {
8664		.add => .@"fadd fast",
8665		.sub => .@"fsub fast",
8666		.mul => .@"fmul fast",
8667		.div => .@"fdiv fast",
8668		.fmod => .@"frem fast",
8669		else => unreachable,
8670		},
8671		}, params[0], params[1], ""),
8672		.fmax,
8673		.fmin,
8674		.ceil,
8675		.cos,
8676		.exp,
8677		.exp2,
8678		.fabs,
8679		.floor,
8680		.log,
8681		.log10,
8682		.log2,
8683		.round,
8684		.sin,
8685		.sqrt,
8686		.trunc,
8687		.fma,
8688		=> return self.wip.callIntrinsic(fast, .none, switch (op) {
8689		.fmax => .maxnum,
8690		.fmin => .minnum,
8691		.ceil => .ceil,
8692		.cos => .cos,
8693		.exp => .exp,
8694		.exp2 => .exp2,
8695		.fabs => .fabs,
8696		.floor => .floor,
8697		.log => .log,
8698		.log10 => .log10,
8699		.log2 => .log2,
8700		.round => .round,
8701		.sin => .sin,
8702		.sqrt => .sqrt,
8703		.trunc => .trunc,
8704		.fma => .fma,
8705		else => unreachable,
8706		}, &.{llvm_ty}, &params, ""),
8707		.tan => unreachable,
8708		};
8709
8710		const float_bits = scalar_ty.floatBits(target);
8711		const fn_name = switch (op) {
8712		.neg => {
8713		// In this case we can generate a softfloat negation by XORing the
8714		// bits with a constant.
8715		const int_ty = try o.builder.intType(@intCast(float_bits));
8716		const cast_ty = try llvm_ty.changeScalar(int_ty, &o.builder);
8717		const sign_mask = try o.builder.splatValue(
8718		cast_ty,
8719		try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)),
8720		);
8721		const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, "");
8722		const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, "");
8723		return self.wip.cast(.bitcast, result, llvm_ty, "");
8724		},
8725		.add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
8726		@tagName(op), compilerRtFloatAbbrev(float_bits),
8727		}),
8728		.ceil,
8729		.cos,
8730		.exp,
8731		.exp2,
8732		.fabs,
8733		.floor,
8734		.fma,
8735		.fmax,
8736		.fmin,
8737		.fmod,
8738		.log,
8739		.log10,
8740		.log2,
8741		.round,
8742		.sin,
8743		.sqrt,
8744		.tan,
8745		.trunc,
8746		=> try o.builder.strtabStringFmt("{s}{s}{s}", .{
8747		libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
8748		}),
8749		};
8750
8751		const scalar_llvm_ty = llvm_ty.scalarType(&o.builder);
8752		const libc_fn = try self.getLibcFunction(
8753		fn_name,
8754		([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len],
8755		scalar_llvm_ty,
8756		);
8757		if (ty.zigTypeTag(zcu) == .vector) {
8758		const result = try o.builder.poisonValue(llvm_ty);
8759		return self.buildElementwiseCall(libc_fn, &params, result, ty.vectorLen(zcu));
8760		}
8761
8762		return self.wip.call(
8763		fast.toCallKind(),
8764		.ccc,
8765		.none,
8766		libc_fn.typeOf(&o.builder),
8767		libc_fn.toValue(&o.builder),
8768		&params,
8769		"",
8770		);
8771		}
8772
8773		fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8774		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
8775		const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
8776
8777		const mulend1 = try self.resolveInst(extra.lhs);
8778		const mulend2 = try self.resolveInst(extra.rhs);
8779		const addend = try self.resolveInst(pl_op.operand);
8780
8781		const ty = self.typeOfIndex(inst);
8782		return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
8783		}
8784
8785		fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8786		const o = self.ng.object;
8787		const pt = self.ng.pt;
8788		const zcu = pt.zcu;
8789		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
8790		const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
8791
8792		const lhs = try self.resolveInst(extra.lhs);
8793		const rhs = try self.resolveInst(extra.rhs);
8794
8795		const lhs_ty = self.typeOf(extra.lhs);
8796		if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu))
8797		return self.ng.todo("implement vector shifts with scalar rhs", .{});
8798		const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8799
8800		const dest_ty = self.typeOfIndex(inst);
8801		const llvm_dest_ty = try o.lowerType(pt, dest_ty);
8802
8803		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8804
8805		const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
8806		const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
8807		.ashr
8808		else
8809		.lshr, result, casted_rhs, "");
8810
8811		const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
8812
8813		const result_index = o.llvmFieldIndex(dest_ty, 0).?;
8814		const overflow_index = o.llvmFieldIndex(dest_ty, 1).?;
8815
8816		if (isByRef(dest_ty, zcu)) {
8817		const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
8818		const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
8819		{
8820		const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, result_index, "");
8821		_ = try self.wip.store(.normal, result, field_ptr, result_alignment);
8822		}
8823		{
8824		const field_ptr = try self.wip.gepStruct(llvm_dest_ty, alloca_inst, overflow_index, "");
8825		_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
8826		}
8827		return alloca_inst;
8828		}
8829
8830		var fields: [2]Builder.Value = undefined;
8831		fields[result_index] = result;
8832		fields[overflow_index] = overflow_bit;
8833		return self.wip.buildAggregate(llvm_dest_ty, &fields, "");
8834		}
8835
8836		fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8837		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8838		const lhs = try self.resolveInst(bin_op.lhs);
8839		const rhs = try self.resolveInst(bin_op.rhs);
8840		return self.wip.bin(.@"and", lhs, rhs, "");
8841		}
8842
8843		fn airOr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8844		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8845		const lhs = try self.resolveInst(bin_op.lhs);
8846		const rhs = try self.resolveInst(bin_op.rhs);
8847		return self.wip.bin(.@"or", lhs, rhs, "");
8848		}
8849
8850		fn airXor(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8851		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8852		const lhs = try self.resolveInst(bin_op.lhs);
8853		const rhs = try self.resolveInst(bin_op.rhs);
8854		return self.wip.bin(.xor, lhs, rhs, "");
8855		}
8856
8857		fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8858		const o = self.ng.object;
8859		const pt = self.ng.pt;
8860		const zcu = pt.zcu;
8861		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8862
8863		const lhs = try self.resolveInst(bin_op.lhs);
8864		const rhs = try self.resolveInst(bin_op.rhs);
8865
8866		const lhs_ty = self.typeOf(bin_op.lhs);
8867		if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8868		return self.ng.todo("implement vector shifts with scalar rhs", .{});
8869		const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8870
8871		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8872		return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
8873		.@"shl nsw"
8874		else
8875		.@"shl nuw", lhs, casted_rhs, "");
8876		}
8877
8878		fn airShl(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8879		const o = self.ng.object;
8880		const pt = self.ng.pt;
8881		const zcu = pt.zcu;
8882		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8883
8884		const lhs = try self.resolveInst(bin_op.lhs);
8885		const rhs = try self.resolveInst(bin_op.rhs);
8886
8887		const lhs_ty = self.typeOf(bin_op.lhs);
8888		if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8889		return self.ng.todo("implement vector shifts with scalar rhs", .{});
8890
8891		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8892		return self.wip.bin(.shl, lhs, casted_rhs, "");
8893		}
8894
8895		fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8896		const o = self.ng.object;
8897		const pt = self.ng.pt;
8898		const zcu = pt.zcu;
8899		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8900
8901		const lhs = try self.resolveInst(bin_op.lhs);
8902		const rhs = try self.resolveInst(bin_op.rhs);
8903
8904		const lhs_ty = self.typeOf(bin_op.lhs);
8905		const lhs_info = lhs_ty.intInfo(zcu);
8906		const llvm_lhs_ty = try o.lowerType(pt, lhs_ty);
8907		const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder);
8908
8909		const rhs_ty = self.typeOf(bin_op.rhs);
8910		if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu))
8911		return self.ng.todo("implement vector shifts with scalar rhs", .{});
8912		const rhs_info = rhs_ty.intInfo(zcu);
8913		assert(rhs_info.signedness == .unsigned);
8914		const llvm_rhs_ty = try o.lowerType(pt, rhs_ty);
8915		const llvm_rhs_scalar_ty = llvm_rhs_ty.scalarType(&o.builder);
8916
8917		const result = try self.wip.callIntrinsic(
8918		.normal,
8919		.none,
8920		switch (lhs_info.signedness) {
8921		.signed => .@"sshl.sat",
8922		.unsigned => .@"ushl.sat",
8923		},
8924		&.{llvm_lhs_ty},
8925		&.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
8926		"",
8927		);
8928
8929		// LLVM langref says "If b is (statically or dynamically) equal to or
8930		// larger than the integer bit width of the arguments, the result is a
8931		// poison value."
8932		// However Zig semantics says that saturating shift left can never produce
8933		// undefined; instead it saturates.
8934		if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
8935		const bits = try o.builder.splatValue(
8936		llvm_rhs_ty,
8937		try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
8938		);
8939		const in_range = try self.wip.icmp(.ult, rhs, bits, "");
8940		const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
8941		.signed => {
8942		const zero = try o.builder.splatValue(
8943		llvm_lhs_ty,
8944		try o.builder.intConst(llvm_lhs_scalar_ty, 0),
8945		);
8946		const smin = try o.builder.splatValue(
8947		llvm_lhs_ty,
8948		try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
8949		);
8950		const smax = try o.builder.splatValue(
8951		llvm_lhs_ty,
8952		try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
8953		);
8954		const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
8955		const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
8956		const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
8957		break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
8958		},
8959		.unsigned => {
8960		const zero = try o.builder.splatValue(
8961		llvm_lhs_ty,
8962		try o.builder.intConst(llvm_lhs_scalar_ty, 0),
8963		);
8964		const umax = try o.builder.splatValue(
8965		llvm_lhs_ty,
8966		try o.builder.intConst(llvm_lhs_scalar_ty, -1),
8967		);
8968		const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
8969		break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
8970		},
8971		};
8972		return self.wip.select(.normal, in_range, result, lhs_sat, "");
8973		}
8974
8975		fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value {
8976		const o = self.ng.object;
8977		const pt = self.ng.pt;
8978		const zcu = pt.zcu;
8979		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
8980
8981		const lhs = try self.resolveInst(bin_op.lhs);
8982		const rhs = try self.resolveInst(bin_op.rhs);
8983
8984		const lhs_ty = self.typeOf(bin_op.lhs);
8985		if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu))
8986		return self.ng.todo("implement vector shifts with scalar rhs", .{});
8987		const lhs_scalar_ty = lhs_ty.scalarType(zcu);
8988
8989		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(pt, lhs_ty), "");
8990		const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
8991
8992		return self.wip.bin(if (is_exact)
8993		if (is_signed_int) .@"ashr exact" else .@"lshr exact"
8994		else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
8995		}
8996
8997		fn airAbs(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
8998		const o = self.ng.object;
8999		const pt = self.ng.pt;
9000		const zcu = pt.zcu;
9001		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9002		const operand = try self.resolveInst(ty_op.operand);
9003		const operand_ty = self.typeOf(ty_op.operand);
9004		const scalar_ty = operand_ty.scalarType(zcu);
9005
9006		switch (scalar_ty.zigTypeTag(zcu)) {
9007		.int => return self.wip.callIntrinsic(
9008		.normal,
9009		.none,
9010		.abs,
9011		&.{try o.lowerType(pt, operand_ty)},
9012		&.{ operand, try o.builder.intValue(.i1, 0) },
9013		"",
9014		),
9015		.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
9016		else => unreachable,
9017		}
9018		}
9019
9020		fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9021		const o = fg.ng.object;
9022		const pt = fg.ng.pt;
9023		const zcu = pt.zcu;
9024		const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9025		const dest_ty = fg.typeOfIndex(inst);
9026		const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9027		const operand = try fg.resolveInst(ty_op.operand);
9028		const operand_ty = fg.typeOf(ty_op.operand);
9029		const operand_info = operand_ty.intInfo(zcu);
9030
9031		const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
9032
9033		bounds_check: {
9034		const dest_scalar = dest_ty.scalarType(zcu);
9035		const operand_scalar = operand_ty.scalarType(zcu);
9036
9037		const dest_info = dest_ty.intInfo(zcu);
9038
9039		const have_min_check, const have_max_check = c: {
9040		const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
9041		const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
9042
9043		const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
9044		const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
9045
9046		break :c .{
9047		operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
9048		dest_pos_bits < operand_pos_bits,
9049		};
9050		};
9051
9052		if (!have_min_check and !have_max_check) break :bounds_check;
9053
9054		const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9055		const operand_scalar_llvm_ty = try o.lowerType(pt, operand_scalar);
9056
9057		const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
9058		assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
9059
9060		const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
9061
9062		if (have_min_check) {
9063		const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9064		const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
9065		const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
9066		const ok = if (is_vector) ok: {
9067		const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9068		break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9069		} else ok_maybe_vec;
9070		if (safety) {
9071		const fail_block = try fg.wip.block(1, "IntMinFail");
9072		const ok_block = try fg.wip.block(1, "IntMinOk");
9073		_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9074		fg.wip.cursor = .{ .block = fail_block };
9075		try fg.buildSimplePanic(panic_id);
9076		fg.wip.cursor = .{ .block = ok_block };
9077		} else {
9078		_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9079		}
9080		}
9081
9082		if (have_max_check) {
9083		const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
9084		const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
9085		const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
9086		const ok = if (is_vector) ok: {
9087		const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
9088		break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
9089		} else ok_maybe_vec;
9090		if (safety) {
9091		const fail_block = try fg.wip.block(1, "IntMaxFail");
9092		const ok_block = try fg.wip.block(1, "IntMaxOk");
9093		_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
9094		fg.wip.cursor = .{ .block = fail_block };
9095		try fg.buildSimplePanic(panic_id);
9096		fg.wip.cursor = .{ .block = ok_block };
9097		} else {
9098		_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
9099		}
9100		}
9101		}
9102
9103		const result = try fg.wip.conv(switch (operand_info.signedness) {
9104		.signed => .signed,
9105		.unsigned => .unsigned,
9106		}, operand, dest_llvm_ty, "");
9107
9108		if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
9109		const llvm_fn = try fg.getIsNamedEnumValueFunction(dest_ty);
9110		const is_valid_enum_val = try fg.wip.call(
9111		.normal,
9112		.fastcc,
9113		.none,
9114		llvm_fn.typeOf(&o.builder),
9115		llvm_fn.toValue(&o.builder),
9116		&.{result},
9117		"",
9118		);
9119		const fail_block = try fg.wip.block(1, "ValidEnumFail");
9120		const ok_block = try fg.wip.block(1, "ValidEnumOk");
9121		_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
9122		fg.wip.cursor = .{ .block = fail_block };
9123		try fg.buildSimplePanic(.invalid_enum_value);
9124		fg.wip.cursor = .{ .block = ok_block };
9125		}
9126
9127		return result;
9128		}
9129
9130		fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9131		const o = self.ng.object;
9132		const pt = self.ng.pt;
9133		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9134		const operand = try self.resolveInst(ty_op.operand);
9135		const dest_llvm_ty = try o.lowerType(pt, self.typeOfIndex(inst));
9136		return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
9137		}
9138
9139		fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9140		const o = self.ng.object;
9141		const pt = self.ng.pt;
9142		const zcu = pt.zcu;
9143		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9144		const operand = try self.resolveInst(ty_op.operand);
9145		const operand_ty = self.typeOf(ty_op.operand);
9146		const dest_ty = self.typeOfIndex(inst);
9147		const target = zcu.getTarget();
9148
9149		if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9150		return self.wip.cast(.fptrunc, operand, try o.lowerType(pt, dest_ty), "");
9151		} else {
9152		const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9153		const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9154
9155		const dest_bits = dest_ty.floatBits(target);
9156		const src_bits = operand_ty.floatBits(target);
9157		const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
9158		compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9159		});
9160
9161		const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9162		return self.wip.call(
9163		.normal,
9164		.ccc,
9165		.none,
9166		libc_fn.typeOf(&o.builder),
9167		libc_fn.toValue(&o.builder),
9168		&.{operand},
9169		"",
9170		);
9171		}
9172		}
9173
9174		fn airFpext(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9175		const o = self.ng.object;
9176		const pt = self.ng.pt;
9177		const zcu = pt.zcu;
9178		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9179		const operand = try self.resolveInst(ty_op.operand);
9180		const operand_ty = self.typeOf(ty_op.operand);
9181		const dest_ty = self.typeOfIndex(inst);
9182		const target = zcu.getTarget();
9183
9184		if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
9185		return self.wip.cast(.fpext, operand, try o.lowerType(pt, dest_ty), "");
9186		} else {
9187		const operand_llvm_ty = try o.lowerType(pt, operand_ty);
9188		const dest_llvm_ty = try o.lowerType(pt, dest_ty);
9189
9190		const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
9191		const src_bits = operand_ty.scalarType(zcu).floatBits(target);
9192		const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
9193		compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
9194		});
9195
9196		const libc_fn = try self.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
9197		if (dest_ty.isVector(zcu)) return self.buildElementwiseCall(
9198		libc_fn,
9199		&.{operand},
9200		try o.builder.poisonValue(dest_llvm_ty),
9201		dest_ty.vectorLen(zcu),
9202		);
9203		return self.wip.call(
9204		.normal,
9205		.ccc,
9206		.none,
9207		libc_fn.typeOf(&o.builder),
9208		libc_fn.toValue(&o.builder),
9209		&.{operand},
9210		"",
9211		);
9212		}
9213		}
9214
9215		fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9216		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9217		const operand_ty = self.typeOf(ty_op.operand);
9218		const inst_ty = self.typeOfIndex(inst);
9219		const operand = try self.resolveInst(ty_op.operand);
9220		return self.bitCast(operand, operand_ty, inst_ty);
9221		}
9222
9223		fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) !Builder.Value {
9224		const o = self.ng.object;
9225		const pt = self.ng.pt;
9226		const zcu = pt.zcu;
9227		const operand_is_ref = isByRef(operand_ty, zcu);
9228		const result_is_ref = isByRef(inst_ty, zcu);
9229		const llvm_dest_ty = try o.lowerType(pt, inst_ty);
9230
9231		if (operand_is_ref and result_is_ref) {
9232		// They are both pointers, so just return the same opaque pointer :)
9233		return operand;
9234		}
9235
9236		if (llvm_dest_ty.isInteger(&o.builder) and
9237		operand.typeOfWip(&self.wip).isInteger(&o.builder))
9238		{
9239		return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
9240		}
9241
9242		const operand_scalar_ty = operand_ty.scalarType(zcu);
9243		const inst_scalar_ty = inst_ty.scalarType(zcu);
9244		if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
9245		return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
9246		}
9247		if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
9248		return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
9249		}
9250
9251		if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
9252		const elem_ty = operand_ty.childType(zcu);
9253		if (!result_is_ref) {
9254		return self.ng.todo("implement bitcast vector to non-ref array", .{});
9255		}
9256		const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9257		const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9258		const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9259		if (bitcast_ok) {
9260		_ = try self.wip.store(.normal, operand, array_ptr, alignment);
9261		} else {
9262		// If the ABI size of the element type is not evenly divisible by size in bits;
9263		// a simple bitcast will not work, and we fall back to extractelement.
9264		const llvm_usize = try o.lowerType(pt, Type.usize);
9265		const usize_zero = try o.builder.intValue(llvm_usize, 0);
9266		const vector_len = operand_ty.arrayLen(zcu);
9267		var i: u64 = 0;
9268		while (i < vector_len) : (i += 1) {
9269		const elem_ptr = try self.wip.gep(.inbounds, llvm_dest_ty, array_ptr, &.{
9270		usize_zero, try o.builder.intValue(llvm_usize, i),
9271		}, "");
9272		const elem =
9273		try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
9274		_ = try self.wip.store(.normal, elem, elem_ptr, .default);
9275		}
9276		}
9277		return array_ptr;
9278		} else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
9279		const elem_ty = operand_ty.childType(zcu);
9280		const llvm_vector_ty = try o.lowerType(pt, inst_ty);
9281		if (!operand_is_ref) return self.ng.todo("implement bitcast non-ref array to vector", .{});
9282
9283		const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
9284		if (bitcast_ok) {
9285		// The array is aligned to the element's alignment, while the vector might have a completely
9286		// different alignment. This means we need to enforce the alignment of this load.
9287		const alignment = elem_ty.abiAlignment(zcu).toLlvm();
9288		return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
9289		} else {
9290		// If the ABI size of the element type is not evenly divisible by size in bits;
9291		// a simple bitcast will not work, and we fall back to extractelement.
9292		const array_llvm_ty = try o.lowerType(pt, operand_ty);
9293		const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9294		const llvm_usize = try o.lowerType(pt, Type.usize);
9295		const usize_zero = try o.builder.intValue(llvm_usize, 0);
9296		const vector_len = operand_ty.arrayLen(zcu);
9297		var vector = try o.builder.poisonValue(llvm_vector_ty);
9298		var i: u64 = 0;
9299		while (i < vector_len) : (i += 1) {
9300		const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, operand, &.{
9301		usize_zero, try o.builder.intValue(llvm_usize, i),
9302		}, "");
9303		const elem = try self.wip.load(.normal, elem_llvm_ty, elem_ptr, .default, "");
9304		vector =
9305		try self.wip.insertElement(vector, elem, try o.builder.intValue(.i32, i), "");
9306		}
9307		return vector;
9308		}
9309		}
9310
9311		if (operand_is_ref) {
9312		const alignment = operand_ty.abiAlignment(zcu).toLlvm();
9313		return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
9314		}
9315
9316		if (result_is_ref) {
9317		const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9318		const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9319		_ = try self.wip.store(.normal, operand, result_ptr, alignment);
9320		return result_ptr;
9321		}
9322
9323		if (llvm_dest_ty.isStruct(&o.builder) or
9324		((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
9325		operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
9326		{
9327		// Both our operand and our result are values, not pointers,
9328		// but LLVM won't let us bitcast struct values or vectors with padding bits.
9329		// Therefore, we store operand to alloca, then load for result.
9330		const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
9331		const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
9332		_ = try self.wip.store(.normal, operand, result_ptr, alignment);
9333		return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
9334		}
9335
9336		return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
9337		}
9338
9339		fn airArg(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9340		const o = self.ng.object;
9341		const pt = self.ng.pt;
9342		const zcu = pt.zcu;
9343		const arg_val = self.args[self.arg_index];
9344		self.arg_index += 1;
9345
9346		// llvm does not support debug info for naked function arguments
9347		if (self.is_naked) return arg_val;
9348
9349		const inst_ty = self.typeOfIndex(inst);
9350
9351		const func = zcu.funcInfo(zcu.navValue(self.ng.nav_index).toIntern());
9352		const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
9353		const file = zcu.fileByIndex(func_zir.file);
9354
9355		const mod = file.mod.?;
9356		if (mod.strip) return arg_val;
9357		const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
9358		const zir = &file.zir.?;
9359		const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
9360
9361		const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
9362		const lbrace_col = func.lbrace_column + 1;
9363
9364		const debug_parameter = try o.builder.debugParameter(
9365		if (name.len > 0) try o.builder.metadataString(name) else null,
9366		self.file,
9367		self.scope,
9368		lbrace_line,
9369		try o.getDebugType(pt, inst_ty),
9370		self.arg_index,
9371		);
9372
9373		const old_location = self.wip.debug_location;
9374		self.wip.debug_location = .{ .location = .{
9375		.line = lbrace_line,
9376		.column = lbrace_col,
9377		.scope = self.scope.toOptional(),
9378		.inlined_at = .none,
9379		} };
9380
9381		if (isByRef(inst_ty, zcu)) {
9382		_ = try self.wip.callIntrinsic(
9383		.normal,
9384		.none,
9385		.@"dbg.declare",
9386		&.{},
9387		&.{
9388		(try self.wip.debugValue(arg_val)).toValue(),
9389		debug_parameter.toValue(),
9390		(try o.builder.debugExpression(&.{})).toValue(),
9391		},
9392		"",
9393		);
9394		} else if (mod.optimize_mode == .Debug) {
9395		const alignment = inst_ty.abiAlignment(zcu).toLlvm();
9396		const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
9397		_ = try self.wip.store(.normal, arg_val, alloca, alignment);
9398		_ = try self.wip.callIntrinsic(
9399		.normal,
9400		.none,
9401		.@"dbg.declare",
9402		&.{},
9403		&.{
9404		(try self.wip.debugValue(alloca)).toValue(),
9405		debug_parameter.toValue(),
9406		(try o.builder.debugExpression(&.{})).toValue(),
9407		},
9408		"",
9409		);
9410		} else {
9411		_ = try self.wip.callIntrinsic(
9412		.normal,
9413		.none,
9414		.@"dbg.value",
9415		&.{},
9416		&.{
9417		(try self.wip.debugValue(arg_val)).toValue(),
9418		debug_parameter.toValue(),
9419		(try o.builder.debugExpression(&.{})).toValue(),
9420		},
9421		"",
9422		);
9423		}
9424
9425		self.wip.debug_location = old_location;
9426		return arg_val;
9427		}
9428
9429		fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9430		const o = self.ng.object;
9431		const pt = self.ng.pt;
9432		const zcu = pt.zcu;
9433		const ptr_ty = self.typeOfIndex(inst);
9434		const pointee_type = ptr_ty.childType(zcu);
9435		if (!pointee_type.hasRuntimeBits(zcu))
9436		return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9437
9438		const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
9439		const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9440		return self.buildAlloca(pointee_llvm_ty, alignment);
9441		}
9442
9443		fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9444		const o = self.ng.object;
9445		const pt = self.ng.pt;
9446		const zcu = pt.zcu;
9447		const ptr_ty = self.typeOfIndex(inst);
9448		const ret_ty = ptr_ty.childType(zcu);
9449		if (!ret_ty.hasRuntimeBits(zcu))
9450		return (try o.lowerPtrToVoid(pt, ptr_ty)).toValue();
9451		if (self.ret_ptr != .none) return self.ret_ptr;
9452		const ret_llvm_ty = try o.lowerType(pt, ret_ty);
9453		const alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9454		return self.buildAlloca(ret_llvm_ty, alignment);
9455		}
9456
9457		/// Use this instead of builder.buildAlloca, because this function makes sure to
9458		/// put the alloca instruction at the top of the function!
9459		fn buildAlloca(
9460		self: *FuncGen,
9461		llvm_ty: Builder.Type,
9462		alignment: Builder.Alignment,
9463		) Allocator.Error!Builder.Value {
9464		const target = self.ng.pt.zcu.getTarget();
9465		return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
9466		}
9467
9468		fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9469		const o = self.ng.object;
9470		const pt = self.ng.pt;
9471		const zcu = pt.zcu;
9472		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9473		const dest_ptr = try self.resolveInst(bin_op.lhs);
9474		const ptr_ty = self.typeOf(bin_op.lhs);
9475		const operand_ty = ptr_ty.childType(zcu);
9476
9477		const val_is_undef = if (try self.air.value(bin_op.rhs, pt)) \|val\| val.isUndef(zcu) else false;
9478		if (val_is_undef) {
9479		const owner_mod = self.ng.ownerModule();
9480
9481		// Even if safety is disabled, we still emit a memset to undefined since it conveys
9482		// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
9483		// between using 0xaa or actual undefined for the fill byte.
9484		//
9485		// However, for Debug builds specifically, we avoid emitting the memset because LLVM
9486		// will neither use the information nor get rid of the memset, thus leaving an
9487		// unexpected call in the user's code. This is problematic if the code in question is
9488		// not ready to correctly make calls yet, such as in our early PIE startup code, or in
9489		// the early stages of a dynamic linker, etc.
9490		if (!safety and owner_mod.optimize_mode == .Debug) {
9491		return .none;
9492		}
9493
9494		const ptr_info = ptr_ty.ptrInfo(zcu);
9495		const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
9496		if (needs_bitmask) {
9497		// TODO: only some bits are to be undef, we cannot write with a simple memset.
9498		// meanwhile, ignore the write rather than stomping over valid bits.
9499		// https://github.com/ziglang/zig/issues/15337
9500		return .none;
9501		}
9502
9503		self.maybeMarkAllowZeroAccess(ptr_info);
9504
9505		const len = try o.builder.intValue(try o.lowerType(pt, Type.usize), operand_ty.abiSize(zcu));
9506		_ = try self.wip.callMemSet(
9507		dest_ptr,
9508		ptr_ty.ptrAlignment(zcu).toLlvm(),
9509		if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
9510		len,
9511		if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9512		self.disable_intrinsics,
9513		);
9514		if (safety and owner_mod.valgrind) {
9515		try self.valgrindMarkUndef(dest_ptr, len);
9516		}
9517		return .none;
9518		}
9519
9520		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9521
9522		const src_operand = try self.resolveInst(bin_op.rhs);
9523		try self.store(dest_ptr, ptr_ty, src_operand, .none);
9524		return .none;
9525		}
9526
9527		fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9528		const pt = fg.ng.pt;
9529		const zcu = pt.zcu;
9530		const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
9531		const ptr_ty = fg.typeOf(ty_op.operand);
9532		const ptr_info = ptr_ty.ptrInfo(zcu);
9533		const ptr = try fg.resolveInst(ty_op.operand);
9534		fg.maybeMarkAllowZeroAccess(ptr_info);
9535		return fg.load(ptr, ptr_ty);
9536		}
9537
9538		fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !void {
9539		_ = inst;
9540		const target = self.ng.object.target;
9541		if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and
9542		target.cpu.has(.mips, .notraps))
9543		{
9544		// Emit a MIPS `break` instruction followed by an infinite loop (to fulfill the noreturn)
9545		// since this CPU does not support trap instructions.
9546		const o = self.ng.object;
9547		_ = try self.wip.callAsm(
9548		.none,
9549		try o.builder.fnType(.void, &.{}, .normal),
9550		.{ .sideeffect = true },
9551		try o.builder.string("break\n0:\nj 0b\nnop"),
9552		try o.builder.string("~{memory}"),
9553		&.{},
9554		"",
9555		);
9556		} else {
9557		_ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
9558		}
9559		_ = try self.wip.@"unreachable"();
9560		}
9561
9562		fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9563		_ = inst;
9564		_ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
9565		return .none;
9566		}
9567
9568		fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9569		_ = inst;
9570		const o = self.ng.object;
9571		const pt = self.ng.pt;
9572		const llvm_usize = try o.lowerType(pt, Type.usize);
9573		if (!target_util.supportsReturnAddress(self.ng.pt.zcu.getTarget(), self.ng.ownerModule().optimize_mode)) {
9574		// https://github.com/ziglang/zig/issues/11946
9575		return o.builder.intValue(llvm_usize, 0);
9576		}
9577		const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
9578		return self.wip.cast(.ptrtoint, result, llvm_usize, "");
9579		}
9580
9581		fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9582		_ = inst;
9583		const o = self.ng.object;
9584		const pt = self.ng.pt;
9585		const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
9586		return self.wip.cast(.ptrtoint, result, try o.lowerType(pt, Type.usize), "");
9587		}
9588
9589		fn airCmpxchg(
9590		self: *FuncGen,
9591		inst: Air.Inst.Index,
9592		kind: Builder.Function.Instruction.CmpXchg.Kind,
9593		) !Builder.Value {
9594		const o = self.ng.object;
9595		const pt = self.ng.pt;
9596		const zcu = pt.zcu;
9597		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
9598		const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
9599		const ptr = try self.resolveInst(extra.ptr);
9600		const ptr_ty = self.typeOf(extra.ptr);
9601		var expected_value = try self.resolveInst(extra.expected_value);
9602		var new_value = try self.resolveInst(extra.new_value);
9603		const operand_ty = ptr_ty.childType(zcu);
9604		const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9605		const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9606		if (llvm_abi_ty != .none) {
9607		// operand needs widening and truncating
9608		const signedness: Builder.Function.Instruction.Cast.Signedness =
9609		if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
9610		expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
9611		new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
9612		}
9613
9614		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9615
9616		const result = try self.wip.cmpxchg(
9617		kind,
9618		if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
9619		ptr,
9620		expected_value,
9621		new_value,
9622		self.sync_scope,
9623		toLlvmAtomicOrdering(extra.successOrder()),
9624		toLlvmAtomicOrdering(extra.failureOrder()),
9625		ptr_ty.ptrAlignment(zcu).toLlvm(),
9626		"",
9627		);
9628
9629		const optional_ty = self.typeOfIndex(inst);
9630
9631		var payload = try self.wip.extractValue(result, &.{0}, "");
9632		if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
9633		const success_bit = try self.wip.extractValue(result, &.{1}, "");
9634
9635		if (optional_ty.optionalReprIsPayload(zcu)) {
9636		const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
9637		return self.wip.select(.normal, success_bit, zero, payload, "");
9638		}
9639
9640		comptime assert(optional_layout_version == 3);
9641
9642		const non_null_bit = try self.wip.not(success_bit, "");
9643		return buildOptional(self, optional_ty, payload, non_null_bit);
9644		}
9645
9646		fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9647		const o = self.ng.object;
9648		const pt = self.ng.pt;
9649		const zcu = pt.zcu;
9650		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
9651		const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
9652		const ptr = try self.resolveInst(pl_op.operand);
9653		const ptr_ty = self.typeOf(pl_op.operand);
9654		const operand_ty = ptr_ty.childType(zcu);
9655		const operand = try self.resolveInst(extra.operand);
9656		const is_signed_int = operand_ty.isSignedInt(zcu);
9657		const is_float = operand_ty.isRuntimeFloat();
9658		const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
9659		const ordering = toLlvmAtomicOrdering(extra.ordering());
9660		const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, op == .xchg);
9661		const llvm_operand_ty = try o.lowerType(pt, operand_ty);
9662
9663		const access_kind: Builder.MemoryAccessKind =
9664		if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9665		const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
9666
9667		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9668
9669		if (llvm_abi_ty != .none) {
9670		// operand needs widening and truncating or bitcasting.
9671		return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
9672		access_kind,
9673		op,
9674		ptr,
9675		try self.wip.cast(
9676		if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
9677		operand,
9678		llvm_abi_ty,
9679		"",
9680		),
9681		self.sync_scope,
9682		ordering,
9683		ptr_alignment,
9684		"",
9685		), llvm_operand_ty, "");
9686		}
9687
9688		if (!llvm_operand_ty.isPointer(&o.builder)) return self.wip.atomicrmw(
9689		access_kind,
9690		op,
9691		ptr,
9692		operand,
9693		self.sync_scope,
9694		ordering,
9695		ptr_alignment,
9696		"",
9697		);
9698
9699		// It's a pointer but we need to treat it as an int.
9700		return self.wip.cast(.inttoptr, try self.wip.atomicrmw(
9701		access_kind,
9702		op,
9703		ptr,
9704		try self.wip.cast(.ptrtoint, operand, try o.lowerType(pt, Type.usize), ""),
9705		self.sync_scope,
9706		ordering,
9707		ptr_alignment,
9708		"",
9709		), llvm_operand_ty, "");
9710		}
9711
9712		fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9713		const o = self.ng.object;
9714		const pt = self.ng.pt;
9715		const zcu = pt.zcu;
9716		const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
9717		const ptr = try self.resolveInst(atomic_load.ptr);
9718		const ptr_ty = self.typeOf(atomic_load.ptr);
9719		const info = ptr_ty.ptrInfo(zcu);
9720		const elem_ty = Type.fromInterned(info.child);
9721		if (!elem_ty.hasRuntimeBits(zcu)) return .none;
9722		const ordering = toLlvmAtomicOrdering(atomic_load.order);
9723		const llvm_abi_ty = try o.getAtomicAbiType(pt, elem_ty, false);
9724		const ptr_alignment = (if (info.flags.alignment != .none)
9725		@as(InternPool.Alignment, info.flags.alignment)
9726		else
9727		Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
9728		const access_kind: Builder.MemoryAccessKind =
9729		if (info.flags.is_volatile) .@"volatile" else .normal;
9730		const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9731
9732		self.maybeMarkAllowZeroAccess(info);
9733
9734		if (llvm_abi_ty != .none) {
9735		// operand needs widening and truncating
9736		const loaded = try self.wip.loadAtomic(
9737		access_kind,
9738		llvm_abi_ty,
9739		ptr,
9740		self.sync_scope,
9741		ordering,
9742		ptr_alignment,
9743		"",
9744		);
9745		return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
9746		}
9747		return self.wip.loadAtomic(
9748		access_kind,
9749		elem_llvm_ty,
9750		ptr,
9751		self.sync_scope,
9752		ordering,
9753		ptr_alignment,
9754		"",
9755		);
9756		}
9757
9758		fn airAtomicStore(
9759		self: *FuncGen,
9760		inst: Air.Inst.Index,
9761		ordering: Builder.AtomicOrdering,
9762		) !Builder.Value {
9763		const o = self.ng.object;
9764		const pt = self.ng.pt;
9765		const zcu = pt.zcu;
9766		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9767		const ptr_ty = self.typeOf(bin_op.lhs);
9768		const operand_ty = ptr_ty.childType(zcu);
9769		if (!operand_ty.hasRuntimeBits(zcu)) return .none;
9770		const ptr = try self.resolveInst(bin_op.lhs);
9771		var element = try self.resolveInst(bin_op.rhs);
9772		const llvm_abi_ty = try o.getAtomicAbiType(pt, operand_ty, false);
9773
9774		if (llvm_abi_ty != .none) {
9775		// operand needs widening
9776		element = try self.wip.conv(
9777		if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
9778		element,
9779		llvm_abi_ty,
9780		"",
9781		);
9782		}
9783
9784		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9785
9786		try self.store(ptr, ptr_ty, element, ordering);
9787		return .none;
9788		}
9789
9790		fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) !Builder.Value {
9791		const o = self.ng.object;
9792		const pt = self.ng.pt;
9793		const zcu = pt.zcu;
9794		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9795		const dest_slice = try self.resolveInst(bin_op.lhs);
9796		const ptr_ty = self.typeOf(bin_op.lhs);
9797		const elem_ty = self.typeOf(bin_op.rhs);
9798		const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
9799		const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
9800		const access_kind: Builder.MemoryAccessKind =
9801		if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9802
9803		self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
9804
9805		if (try self.air.value(bin_op.rhs, pt)) \|elem_val\| {
9806		if (elem_val.isUndef(zcu)) {
9807		// Even if safety is disabled, we still emit a memset to undefined since it conveys
9808		// extra information to LLVM. However, safety makes the difference between using
9809		// 0xaa or actual undefined for the fill byte.
9810		const fill_byte = if (safety)
9811		try o.builder.intValue(.i8, 0xaa)
9812		else
9813		try o.builder.undefValue(.i8);
9814		const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9815		_ = try self.wip.callMemSet(
9816		dest_ptr,
9817		dest_ptr_align,
9818		fill_byte,
9819		len,
9820		access_kind,
9821		self.disable_intrinsics,
9822		);
9823		const owner_mod = self.ng.ownerModule();
9824		if (safety and owner_mod.valgrind) {
9825		try self.valgrindMarkUndef(dest_ptr, len);
9826		}
9827		return .none;
9828		}
9829
9830		// Test if the element value is compile-time known to be a
9831		// repeating byte pattern, for example, `@as(u64, 0)` has a
9832		// repeating byte pattern of 0 bytes. In such case, the memset
9833		// intrinsic can be used.
9834		if (try elem_val.hasRepeatedByteRepr(pt)) \|byte_val\| {
9835		const fill_byte = try o.builder.intValue(.i8, byte_val);
9836		const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9837		_ = try self.wip.callMemSet(
9838		dest_ptr,
9839		dest_ptr_align,
9840		fill_byte,
9841		len,
9842		access_kind,
9843		self.disable_intrinsics,
9844		);
9845		return .none;
9846		}
9847		}
9848
9849		const value = try self.resolveInst(bin_op.rhs);
9850		const elem_abi_size = elem_ty.abiSize(zcu);
9851
9852		if (elem_abi_size == 1) {
9853		// In this case we can take advantage of LLVM's intrinsic.
9854		const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
9855		const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
9856
9857		_ = try self.wip.callMemSet(
9858		dest_ptr,
9859		dest_ptr_align,
9860		fill_byte,
9861		len,
9862		access_kind,
9863		self.disable_intrinsics,
9864		);
9865		return .none;
9866		}
9867
9868		// non-byte-sized element. lower with a loop. something like this:
9869
9870		// entry:
9871		// ...
9872		// %end_ptr = getelementptr %ptr, %len
9873		// br %loop
9874		// loop:
9875		// %it_ptr = phi body %next_ptr, entry %ptr
9876		// %end = cmp eq %it_ptr, %end_ptr
9877		// br %end, %body, %end
9878		// body:
9879		// store %it_ptr, %value
9880		// %next_ptr = getelementptr %it_ptr, 1
9881		// br %loop
9882		// end:
9883		// ...
9884		const entry_block = self.wip.cursor.block;
9885		const loop_block = try self.wip.block(2, "InlineMemsetLoop");
9886		const body_block = try self.wip.block(1, "InlineMemsetBody");
9887		const end_block = try self.wip.block(1, "InlineMemsetEnd");
9888
9889		const llvm_usize_ty = try o.lowerType(pt, Type.usize);
9890		const len = switch (ptr_ty.ptrSize(zcu)) {
9891		.slice => try self.wip.extractValue(dest_slice, &.{1}, ""),
9892		.one => try o.builder.intValue(llvm_usize_ty, ptr_ty.childType(zcu).arrayLen(zcu)),
9893		.many, .c => unreachable,
9894		};
9895		const elem_llvm_ty = try o.lowerType(pt, elem_ty);
9896		const end_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, dest_ptr, &.{len}, "");
9897		_ = try self.wip.br(loop_block);
9898
9899		self.wip.cursor = .{ .block = loop_block };
9900		const it_ptr = try self.wip.phi(.ptr, "");
9901		const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
9902		_ = try self.wip.brCond(end, body_block, end_block, .none);
9903
9904		self.wip.cursor = .{ .block = body_block };
9905		const elem_abi_align = elem_ty.abiAlignment(zcu);
9906		const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
9907		if (isByRef(elem_ty, zcu)) {
9908		_ = try self.wip.callMemCpy(
9909		it_ptr.toValue(),
9910		it_ptr_align,
9911		value,
9912		elem_abi_align.toLlvm(),
9913		try o.builder.intValue(llvm_usize_ty, elem_abi_size),
9914		access_kind,
9915		self.disable_intrinsics,
9916		);
9917		} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
9918		const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
9919		try o.builder.intValue(llvm_usize_ty, 1),
9920		}, "");
9921		_ = try self.wip.br(loop_block);
9922
9923		self.wip.cursor = .{ .block = end_block };
9924		it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
9925		return .none;
9926		}
9927
9928		fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9929		const pt = self.ng.pt;
9930		const zcu = pt.zcu;
9931		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9932		const dest_slice = try self.resolveInst(bin_op.lhs);
9933		const dest_ptr_ty = self.typeOf(bin_op.lhs);
9934		const src_slice = try self.resolveInst(bin_op.rhs);
9935		const src_ptr_ty = self.typeOf(bin_op.rhs);
9936		const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
9937		const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
9938		const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
9939		const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
9940		dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9941
9942		self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
9943		self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
9944
9945		_ = try self.wip.callMemCpy(
9946		dest_ptr,
9947		dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
9948		src_ptr,
9949		src_ptr_ty.ptrAlignment(zcu).toLlvm(),
9950		len,
9951		access_kind,
9952		self.disable_intrinsics,
9953		);
9954		return .none;
9955		}
9956
9957		fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9958		const pt = self.ng.pt;
9959		const zcu = pt.zcu;
9960		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9961		const dest_slice = try self.resolveInst(bin_op.lhs);
9962		const dest_ptr_ty = self.typeOf(bin_op.lhs);
9963		const src_slice = try self.resolveInst(bin_op.rhs);
9964		const src_ptr_ty = self.typeOf(bin_op.rhs);
9965		const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
9966		const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
9967		const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
9968		const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
9969		dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9970
9971		_ = try self.wip.callMemMove(
9972		dest_ptr,
9973		dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
9974		src_ptr,
9975		src_ptr_ty.ptrAlignment(zcu).toLlvm(),
9976		len,
9977		access_kind,
9978		);
9979		return .none;
9980		}
9981
9982		fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
9983		const o = self.ng.object;
9984		const pt = self.ng.pt;
9985		const zcu = pt.zcu;
9986		const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
9987		const un_ptr_ty = self.typeOf(bin_op.lhs);
9988		const un_ty = un_ptr_ty.childType(zcu);
9989		const layout = un_ty.unionGetLayout(zcu);
9990		if (layout.tag_size == 0) return .none;
9991
9992		const access_kind: Builder.MemoryAccessKind =
9993		if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
9994
9995		self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
9996
9997		const union_ptr = try self.resolveInst(bin_op.lhs);
9998		const new_tag = try self.resolveInst(bin_op.rhs);
9999		const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
10000		if (layout.payload_size == 0) {
10001		_ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
10002		return .none;
10003		}
10004		const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10005		const tag_field_ptr = try self.wip.gepStruct(try o.lowerType(pt, un_ty), union_ptr, tag_index, "");
10006		const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
10007		0 => union_ptr_align,
10008		else => \|off\| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
10009		};
10010		_ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
10011		return .none;
10012		}
10013
10014		fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10015		const o = self.ng.object;
10016		const pt = self.ng.pt;
10017		const zcu = pt.zcu;
10018		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10019		const un_ty = self.typeOf(ty_op.operand);
10020		const layout = un_ty.unionGetLayout(zcu);
10021		if (layout.tag_size == 0) return .none;
10022		const union_handle = try self.resolveInst(ty_op.operand);
10023		if (isByRef(un_ty, zcu)) {
10024		const llvm_un_ty = try o.lowerType(pt, un_ty);
10025		if (layout.payload_size == 0)
10026		return self.wip.load(.normal, llvm_un_ty, union_handle, .default, "");
10027		const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10028		const tag_field_ptr = try self.wip.gepStruct(llvm_un_ty, union_handle, tag_index, "");
10029		const llvm_tag_ty = llvm_un_ty.structFields(&o.builder)[tag_index];
10030		return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
10031		} else {
10032		if (layout.payload_size == 0) return union_handle;
10033		const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10034		return self.wip.extractValue(union_handle, &.{tag_index}, "");
10035		}
10036		}
10037
10038		fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) !Builder.Value {
10039		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10040		const operand = try self.resolveInst(un_op);
10041		const operand_ty = self.typeOf(un_op);
10042
10043		return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
10044		}
10045
10046		fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10047		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10048		const operand = try self.resolveInst(un_op);
10049		const operand_ty = self.typeOf(un_op);
10050
10051		return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
10052		}
10053
10054		fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10055		const o = self.ng.object;
10056		const pt = self.ng.pt;
10057		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10058		const inst_ty = self.typeOfIndex(inst);
10059		const operand_ty = self.typeOf(ty_op.operand);
10060		const operand = try self.resolveInst(ty_op.operand);
10061
10062		const result = try self.wip.callIntrinsic(
10063		.normal,
10064		.none,
10065		intrinsic,
10066		&.{try o.lowerType(pt, operand_ty)},
10067		&.{ operand, .false },
10068		"",
10069		);
10070		return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10071		}
10072
10073		fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value {
10074		const o = self.ng.object;
10075		const pt = self.ng.pt;
10076		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10077		const inst_ty = self.typeOfIndex(inst);
10078		const operand_ty = self.typeOf(ty_op.operand);
10079		const operand = try self.resolveInst(ty_op.operand);
10080
10081		const result = try self.wip.callIntrinsic(
10082		.normal,
10083		.none,
10084		intrinsic,
10085		&.{try o.lowerType(pt, operand_ty)},
10086		&.{operand},
10087		"",
10088		);
10089		return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10090		}
10091
10092		fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10093		const o = self.ng.object;
10094		const pt = self.ng.pt;
10095		const zcu = pt.zcu;
10096		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10097		const operand_ty = self.typeOf(ty_op.operand);
10098		var bits = operand_ty.intInfo(zcu).bits;
10099		assert(bits % 8 == 0);
10100
10101		const inst_ty = self.typeOfIndex(inst);
10102		var operand = try self.resolveInst(ty_op.operand);
10103		var llvm_operand_ty = try o.lowerType(pt, operand_ty);
10104
10105		if (bits % 16 == 8) {
10106		// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
10107		// The truncated result at the end will be the correct bswap
10108		const scalar_ty = try o.builder.intType(@intCast(bits + 8));
10109		if (operand_ty.zigTypeTag(zcu) == .vector) {
10110		const vec_len = operand_ty.vectorLen(zcu);
10111		llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
10112		} else llvm_operand_ty = scalar_ty;
10113
10114		const shift_amt =
10115		try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
10116		const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
10117		operand = try self.wip.bin(.shl, extended, shift_amt, "");
10118
10119		bits = bits + 8;
10120		}
10121
10122		const result =
10123		try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
10124		return self.wip.conv(.unsigned, result, try o.lowerType(pt, inst_ty), "");
10125		}
10126
10127		fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10128		const o = self.ng.object;
10129		const pt = self.ng.pt;
10130		const zcu = pt.zcu;
10131		const ip = &zcu.intern_pool;
10132		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10133		const operand = try self.resolveInst(ty_op.operand);
10134		const error_set_ty = ty_op.ty.toType();
10135
10136		const names = error_set_ty.errorSetNames(zcu);
10137		const valid_block = try self.wip.block(@intCast(names.len), "Valid");
10138		const invalid_block = try self.wip.block(1, "Invalid");
10139		const end_block = try self.wip.block(2, "End");
10140		var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
10141		defer wip_switch.finish(&self.wip);
10142
10143		for (0..names.len) \|name_index\| {
10144		const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
10145		const this_tag_int_value = try o.builder.intConst(try o.errorIntType(pt), err_int);
10146		try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
10147		}
10148		self.wip.cursor = .{ .block = valid_block };
10149		_ = try self.wip.br(end_block);
10150
10151		self.wip.cursor = .{ .block = invalid_block };
10152		_ = try self.wip.br(end_block);
10153
10154		self.wip.cursor = .{ .block = end_block };
10155		const phi = try self.wip.phi(.i1, "");
10156		phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
10157		return phi.toValue();
10158		}
10159
10160		fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10161		const o = self.ng.object;
10162		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10163		const operand = try self.resolveInst(un_op);
10164		const enum_ty = self.typeOf(un_op);
10165
10166		const llvm_fn = try self.getIsNamedEnumValueFunction(enum_ty);
10167		return self.wip.call(
10168		.normal,
10169		.fastcc,
10170		.none,
10171		llvm_fn.typeOf(&o.builder),
10172		llvm_fn.toValue(&o.builder),
10173		&.{operand},
10174		"",
10175		);
10176		}
10177
10178		fn getIsNamedEnumValueFunction(self: *FuncGen, enum_ty: Type) !Builder.Function.Index {
10179		const o = self.ng.object;
10180		const pt = self.ng.pt;
10181		const zcu = pt.zcu;
10182		const ip = &zcu.intern_pool;
10183
10184		const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
10185		if (gop.found_existing) return gop.value_ptr.*;
10186		errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
10187		const function_index = try o.builder.addFunction(
10188		// Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type.
10189		// TODO: change the builder API so we don't need to do this.
10190		try o.builder.fnType(.void, &.{}, .normal),
10191		try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
10192		toLlvmAddressSpace(.generic, zcu.getTarget()),
10193		);
10194		gop.value_ptr.* = function_index;
10195		try o.updateIsNamedEnumValueFunction(pt, enum_ty, function_index);
10196		return function_index;
10197		}
10198
10199		fn airTagName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10200		const o = self.ng.object;
10201		const pt = self.ng.pt;
10202		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10203		const operand = try self.resolveInst(un_op);
10204		const enum_ty = self.typeOf(un_op);
10205
10206		const llvm_fn = try o.getEnumTagNameFunction(pt, enum_ty);
10207		return self.wip.call(
10208		.normal,
10209		.fastcc,
10210		.none,
10211		llvm_fn.typeOf(&o.builder),
10212		llvm_fn.toValue(&o.builder),
10213		&.{operand},
10214		"",
10215		);
10216		}
10217
10218		fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10219		const o = self.ng.object;
10220		const pt = self.ng.pt;
10221		const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
10222		const operand = try self.resolveInst(un_op);
10223		const slice_ty = self.typeOfIndex(inst);
10224		const slice_llvm_ty = try o.lowerType(pt, slice_ty);
10225
10226		// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
10227		const extended_operand = try self.wip.conv(.unsigned, operand, try o.lowerType(pt, .usize), "");
10228
10229		const error_name_table_ptr = try self.getErrorNameTable();
10230		const error_name_table =
10231		try self.wip.load(.normal, .ptr, error_name_table_ptr.toValue(&o.builder), .default, "");
10232		const error_name_ptr =
10233		try self.wip.gep(.inbounds, slice_llvm_ty, error_name_table, &.{extended_operand}, "");
10234		return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
10235		}
10236
10237		fn airSplat(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10238		const o = self.ng.object;
10239		const pt = self.ng.pt;
10240		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10241		const scalar = try self.resolveInst(ty_op.operand);
10242		const vector_ty = self.typeOfIndex(inst);
10243		return self.wip.splatVector(try o.lowerType(pt, vector_ty), scalar, "");
10244		}
10245
10246		fn airSelect(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10247		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10248		const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
10249		const pred = try self.resolveInst(pl_op.operand);
10250		const a = try self.resolveInst(extra.lhs);
10251		const b = try self.resolveInst(extra.rhs);
10252
10253		return self.wip.select(.normal, pred, a, b, "");
10254		}
10255
10256		fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10257		const o = fg.ng.object;
10258		const pt = fg.ng.pt;
10259		const zcu = pt.zcu;
10260		const gpa = zcu.gpa;
10261
10262		const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
10263
10264		const operand = try fg.resolveInst(unwrapped.operand);
10265		const mask = unwrapped.mask;
10266		const operand_ty = fg.typeOf(unwrapped.operand);
10267		const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10268		const llvm_result_ty = try o.lowerType(pt, unwrapped.result_ty);
10269		const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10270		const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
10271		const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10272		const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10273
10274		// LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
10275		// And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
10276		// least a bit". So, there are two cases here.
10277		//
10278		// If the operand length equals the mask length, we do just the one `shufflevector`, where
10279		// the second operand is a constant vector with comptime-known elements at the right indices
10280		// and poison values elsewhere (in the indices which won't be selected).
10281		//
10282		// Otherwise, we lower to two `shufflevector` instructions. The first shuffles the runtime
10283		// operand with an all-poison vector to extract and correctly position all of the runtime
10284		// elements. We also make a constant vector with all of the comptime elements correctly
10285		// positioned. Then, our second instruction selects elements from those "runtime-or-poison"
10286		// and "comptime-or-poison" vectors to compute the result.
10287
10288		// This buffer is used primarily for the mask constants.
10289		const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
10290		defer gpa.free(llvm_elem_buf);
10291
10292		// ...but first, we'll collect all of the comptime-known values.
10293		var any_defined_comptime_value = false;
10294		for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
10295		llvm_elem.* = switch (mask_elem.unwrap()) {
10296		.elem => llvm_poison_elem,
10297		.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
10298		any_defined_comptime_value = true;
10299		break :elem try o.lowerValue(pt, val);
10300		} else llvm_poison_elem,
10301		};
10302		}
10303		// This vector is like the result, but runtime elements are replaced with poison.
10304		const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
10305		break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
10306		} else try o.builder.poisonValue(llvm_result_ty);
10307
10308		if (operand_ty.vectorLen(zcu) == mask.len) {
10309		// input length equals mask/output length, so we lower to one instruction
10310		for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
10311		llvm_elem.* = switch (mask_elem.unwrap()) {
10312		.elem => \|idx\| try o.builder.intConst(.i32, idx),
10313		.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10314		break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10315		} else llvm_poison_mask_elem,
10316		};
10317		}
10318		return fg.wip.shuffleVector(
10319		operand,
10320		comptime_and_poison,
10321		try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10322		"",
10323		);
10324		}
10325
10326		for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
10327		llvm_elem.* = switch (mask_elem.unwrap()) {
10328		.elem => \|idx\| try o.builder.intConst(.i32, idx),
10329		.value => llvm_poison_mask_elem,
10330		};
10331		}
10332		// This vector is like our result, but all comptime-known elements are poison.
10333		const runtime_and_poison = try fg.wip.shuffleVector(
10334		operand,
10335		try o.builder.poisonValue(llvm_operand_ty),
10336		try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10337		"",
10338		);
10339
10340		if (!any_defined_comptime_value) {
10341		// `comptime_and_poison` is just poison; a second shuffle would be a nop.
10342		return runtime_and_poison;
10343		}
10344
10345		// In this second shuffle, the inputs, the mask, and the output all have the same length.
10346		for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
10347		llvm_elem.* = switch (mask_elem.unwrap()) {
10348		.elem => try o.builder.intConst(.i32, elem_idx),
10349		.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
10350		break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
10351		} else llvm_poison_mask_elem,
10352		};
10353		}
10354		// Merge the runtime and comptime elements with the mask we just built.
10355		return fg.wip.shuffleVector(
10356		runtime_and_poison,
10357		comptime_and_poison,
10358		try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
10359		"",
10360		);
10361		}
10362
10363		fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10364		const o = fg.ng.object;
10365		const pt = fg.ng.pt;
10366		const zcu = pt.zcu;
10367		const gpa = zcu.gpa;
10368
10369		const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
10370
10371		const mask = unwrapped.mask;
10372		const llvm_elem_ty = try o.lowerType(pt, unwrapped.result_ty.childType(zcu));
10373		const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
10374		const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
10375
10376		// This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
10377		// length of the longer one with an initial `shufflevector` if necessary, and then do the
10378		// actual computation with a second `shufflevector`.
10379
10380		const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
10381		const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
10382		const operand_len: u32 = @max(operand_a_len, operand_b_len);
10383
10384		// If we need to extend an operand, this is the type that mask will have.
10385		const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
10386
10387		const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
10388		defer gpa.free(llvm_elem_buf);
10389
10390		const operand_a: Builder.Value = extend: {
10391		const raw = try fg.resolveInst(unwrapped.operand_a);
10392		if (operand_a_len == operand_len) break :extend raw;
10393		// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10394		const mask_elems = llvm_elem_buf[0..operand_len];
10395		for (mask_elems[0..operand_a_len], 0..) \|*llvm_elem, elem_idx\| {
10396		llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10397		}
10398		@memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
10399		const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
10400		break :extend try fg.wip.shuffleVector(
10401		raw,
10402		try o.builder.poisonValue(llvm_this_operand_ty),
10403		try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10404		"",
10405		);
10406		};
10407		const operand_b: Builder.Value = extend: {
10408		const raw = try fg.resolveInst(unwrapped.operand_b);
10409		if (operand_b_len == operand_len) break :extend raw;
10410		// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
10411		const mask_elems = llvm_elem_buf[0..operand_len];
10412		for (mask_elems[0..operand_b_len], 0..) \|*llvm_elem, elem_idx\| {
10413		llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
10414		}
10415		@memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
10416		const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
10417		break :extend try fg.wip.shuffleVector(
10418		raw,
10419		try o.builder.poisonValue(llvm_this_operand_ty),
10420		try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
10421		"",
10422		);
10423		};
10424
10425		// `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
10426		// an initial shuffle if necessary). Now for the easy bit.
10427
10428		const mask_elems = llvm_elem_buf[0..mask.len];
10429		for (mask, mask_elems) \|mask_elem, *llvm_mask_elem\| {
10430		llvm_mask_elem.* = switch (mask_elem.unwrap()) {
10431		.a_elem => \|idx\| try o.builder.intConst(.i32, idx),
10432		.b_elem => \|idx\| try o.builder.intConst(.i32, operand_len + idx),
10433		.undef => llvm_poison_mask_elem,
10434		};
10435		}
10436		return fg.wip.shuffleVector(
10437		operand_a,
10438		operand_b,
10439		try o.builder.vectorValue(llvm_mask_ty, mask_elems),
10440		"",
10441		);
10442		}
10443
10444		/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
10445		///
10446		/// Equivalent to:
10447		/// reduce: {
10448		/// var i: usize = 0;
10449		/// var accum: T = init;
10450		/// while (i < vec.len) : (i += 1) {
10451		/// accum = llvm_fn(accum, vec[i]);
10452		/// }
10453		/// break :reduce accum;
10454		/// }
10455		///
10456		fn buildReducedCall(
10457		self: *FuncGen,
10458		llvm_fn: Builder.Function.Index,
10459		operand_vector: Builder.Value,
10460		vector_len: usize,
10461		accum_init: Builder.Value,
10462		) !Builder.Value {
10463		const o = self.ng.object;
10464		const pt = self.ng.pt;
10465		const usize_ty = try o.lowerType(pt, Type.usize);
10466		const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
10467		const llvm_result_ty = accum_init.typeOfWip(&self.wip);
10468
10469		// Allocate and initialize our mutable variables
10470		const i_ptr = try self.buildAlloca(usize_ty, .default);
10471		_ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
10472		const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
10473		_ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
10474
10475		// Setup the loop
10476		const loop = try self.wip.block(2, "ReduceLoop");
10477		const loop_exit = try self.wip.block(1, "AfterReduce");
10478		_ = try self.wip.br(loop);
10479		{
10480		self.wip.cursor = .{ .block = loop };
10481
10482		// while (i < vec.len)
10483		const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
10484		const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
10485		const loop_then = try self.wip.block(1, "ReduceLoopThen");
10486
10487		_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
10488
10489		{
10490		self.wip.cursor = .{ .block = loop_then };
10491
10492		// accum = f(accum, vec[i]);
10493		const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10494		const element = try self.wip.extractElement(operand_vector, i, "");
10495		const new_accum = try self.wip.call(
10496		.normal,
10497		.ccc,
10498		.none,
10499		llvm_fn.typeOf(&o.builder),
10500		llvm_fn.toValue(&o.builder),
10501		&.{ accum, element },
10502		"",
10503		);
10504		_ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
10505
10506		// i += 1
10507		const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
10508		_ = try self.wip.store(.normal, new_i, i_ptr, .default);
10509		_ = try self.wip.br(loop);
10510		}
10511		}
10512
10513		self.wip.cursor = .{ .block = loop_exit };
10514		return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
10515		}
10516
10517		fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value {
10518		const o = self.ng.object;
10519		const pt = self.ng.pt;
10520		const zcu = pt.zcu;
10521		const target = zcu.getTarget();
10522
10523		const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
10524		const operand = try self.resolveInst(reduce.operand);
10525		const operand_ty = self.typeOf(reduce.operand);
10526		const llvm_operand_ty = try o.lowerType(pt, operand_ty);
10527		const scalar_ty = self.typeOfIndex(inst);
10528		const llvm_scalar_ty = try o.lowerType(pt, scalar_ty);
10529
10530		switch (reduce.operation) {
10531		.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10532		.And => .@"vector.reduce.and",
10533		.Or => .@"vector.reduce.or",
10534		.Xor => .@"vector.reduce.xor",
10535		else => unreachable,
10536		}, &.{llvm_operand_ty}, &.{operand}, ""),
10537		.Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
10538		.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10539		.Min => if (scalar_ty.isSignedInt(zcu))
10540		.@"vector.reduce.smin"
10541		else
10542		.@"vector.reduce.umin",
10543		.Max => if (scalar_ty.isSignedInt(zcu))
10544		.@"vector.reduce.smax"
10545		else
10546		.@"vector.reduce.umax",
10547		else => unreachable,
10548		}, &.{llvm_operand_ty}, &.{operand}, ""),
10549		.float => if (intrinsicsAllowed(scalar_ty, target))
10550		return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10551		.Min => .@"vector.reduce.fmin",
10552		.Max => .@"vector.reduce.fmax",
10553		else => unreachable,
10554		}, &.{llvm_operand_ty}, &.{operand}, ""),
10555		else => unreachable,
10556		},
10557		.Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
10558		.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
10559		.Add => .@"vector.reduce.add",
10560		.Mul => .@"vector.reduce.mul",
10561		else => unreachable,
10562		}, &.{llvm_operand_ty}, &.{operand}, ""),
10563		.float => if (intrinsicsAllowed(scalar_ty, target))
10564		return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
10565		.Add => .@"vector.reduce.fadd",
10566		.Mul => .@"vector.reduce.fmul",
10567		else => unreachable,
10568		}, &.{llvm_operand_ty}, &.{ switch (reduce.operation) {
10569		.Add => try o.builder.fpValue(llvm_scalar_ty, -0.0),
10570		.Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0),
10571		else => unreachable,
10572		}, operand }, ""),
10573		else => unreachable,
10574		},
10575		}
10576
10577		// Reduction could not be performed with intrinsics.
10578		// Use a manual loop over a softfloat call instead.
10579		const float_bits = scalar_ty.floatBits(target);
10580		const fn_name = switch (reduce.operation) {
10581		.Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
10582		libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10583		}),
10584		.Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
10585		libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
10586		}),
10587		.Add => try o.builder.strtabStringFmt("__add{s}f3", .{
10588		compilerRtFloatAbbrev(float_bits),
10589		}),
10590		.Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
10591		compilerRtFloatAbbrev(float_bits),
10592		}),
10593		else => unreachable,
10594		};
10595
10596		const libc_fn =
10597		try self.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty);
10598		const init_val = switch (llvm_scalar_ty) {
10599		.i16 => try o.builder.intValue(.i16, @as(i16, @bitCast(
10600		@as(f16, switch (reduce.operation) {
10601		.Min, .Max => std.math.nan(f16),
10602		.Add => -0.0,
10603		.Mul => 1.0,
10604		else => unreachable,
10605		}),
10606		))),
10607		.i80 => try o.builder.intValue(.i80, @as(i80, @bitCast(
10608		@as(f80, switch (reduce.operation) {
10609		.Min, .Max => std.math.nan(f80),
10610		.Add => -0.0,
10611		.Mul => 1.0,
10612		else => unreachable,
10613		}),
10614		))),
10615		.i128 => try o.builder.intValue(.i128, @as(i128, @bitCast(
10616		@as(f128, switch (reduce.operation) {
10617		.Min, .Max => std.math.nan(f128),
10618		.Add => -0.0,
10619		.Mul => 1.0,
10620		else => unreachable,
10621		}),
10622		))),
10623		else => unreachable,
10624		};
10625		return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val);
10626		}
10627
10628		fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10629		const o = self.ng.object;
10630		const pt = self.ng.pt;
10631		const zcu = pt.zcu;
10632		const ip = &zcu.intern_pool;
10633		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10634		const result_ty = self.typeOfIndex(inst);
10635		const len: usize = @intCast(result_ty.arrayLen(zcu));
10636		const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
10637		const llvm_result_ty = try o.lowerType(pt, result_ty);
10638
10639		switch (result_ty.zigTypeTag(zcu)) {
10640		.vector => {
10641		var vector = try o.builder.poisonValue(llvm_result_ty);
10642		for (elements, 0..) \|elem, i\| {
10643		const index_u32 = try o.builder.intValue(.i32, i);
10644		const llvm_elem = try self.resolveInst(elem);
10645		vector = try self.wip.insertElement(vector, llvm_elem, index_u32, "");
10646		}
10647		return vector;
10648		},
10649		.@"struct" => {
10650		if (zcu.typeToPackedStruct(result_ty)) \|struct_type\| {
10651		const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
10652		const big_bits = backing_int_ty.bitSize(zcu);
10653		const int_ty = try o.builder.intType(@intCast(big_bits));
10654		comptime assert(Type.packed_struct_layout_version == 2);
10655		var running_int = try o.builder.intValue(int_ty, 0);
10656		var running_bits: u16 = 0;
10657		for (elements, struct_type.field_types.get(ip)) \|elem, field_ty\| {
10658		if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
10659
10660		const non_int_val = try self.resolveInst(elem);
10661		const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
10662		const small_int_ty = try o.builder.intType(ty_bit_size);
10663		const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
10664		try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
10665		else
10666		try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
10667		const shift_rhs = try o.builder.intValue(int_ty, running_bits);
10668		const extended_int_val =
10669		try self.wip.conv(.unsigned, small_int_val, int_ty, "");
10670		const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, "");
10671		running_int = try self.wip.bin(.@"or", running_int, shifted, "");
10672		running_bits += ty_bit_size;
10673		}
10674		return running_int;
10675		}
10676
10677		assert(result_ty.containerLayout(zcu) != .@"packed");
10678
10679		if (isByRef(result_ty, zcu)) {
10680		// TODO in debug builds init to undef so that the padding will be 0xaa
10681		// even if we fully populate the fields.
10682		const alignment = result_ty.abiAlignment(zcu).toLlvm();
10683		const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10684
10685		for (elements, 0..) \|elem, i\| {
10686		if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10687
10688		const llvm_elem = try self.resolveInst(elem);
10689		const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10690		const field_ptr = try self.wip.gepStruct(llvm_result_ty, alloca_inst, llvm_i, "");
10691
10692		const field_ptr_ty = try pt.ptrType(.{
10693		.child = self.typeOf(elem).toIntern(),
10694		.flags = .{
10695		.alignment = result_ty.explicitFieldAlignment(i, zcu),
10696		},
10697		});
10698		try self.store(field_ptr, field_ptr_ty, llvm_elem, .none);
10699		}
10700
10701		return alloca_inst;
10702		} else {
10703		var result = try o.builder.poisonValue(llvm_result_ty);
10704		for (elements, 0..) \|elem, i\| {
10705		if ((try result_ty.structFieldValueComptime(pt, i)) != null) continue;
10706
10707		const llvm_elem = try self.resolveInst(elem);
10708		const llvm_i = o.llvmFieldIndex(result_ty, i).?;
10709		result = try self.wip.insertValue(result, llvm_elem, &.{llvm_i}, "");
10710		}
10711		return result;
10712		}
10713		},
10714		.array => {
10715		assert(isByRef(result_ty, zcu));
10716
10717		const llvm_usize = try o.lowerType(pt, Type.usize);
10718		const usize_zero = try o.builder.intValue(llvm_usize, 0);
10719		const alignment = result_ty.abiAlignment(zcu).toLlvm();
10720		const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
10721
10722		const array_info = result_ty.arrayInfo(zcu);
10723		const elem_ptr_ty = try pt.ptrType(.{
10724		.child = array_info.elem_type.toIntern(),
10725		});
10726
10727		for (elements, 0..) \|elem, i\| {
10728		const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10729		usize_zero, try o.builder.intValue(llvm_usize, i),
10730		}, "");
10731		const llvm_elem = try self.resolveInst(elem);
10732		try self.store(elem_ptr, elem_ptr_ty, llvm_elem, .none);
10733		}
10734		if (array_info.sentinel) \|sent_val\| {
10735		const elem_ptr = try self.wip.gep(.inbounds, llvm_result_ty, alloca_inst, &.{
10736		usize_zero, try o.builder.intValue(llvm_usize, array_info.len),
10737		}, "");
10738		const llvm_elem = try self.resolveValue(sent_val);
10739		try self.store(elem_ptr, elem_ptr_ty, llvm_elem.toValue(), .none);
10740		}
10741
10742		return alloca_inst;
10743		},
10744		else => unreachable,
10745		}
10746		}
10747
10748		fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10749		const o = self.ng.object;
10750		const pt = self.ng.pt;
10751		const zcu = pt.zcu;
10752		const ip = &zcu.intern_pool;
10753		const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
10754		const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
10755		const union_ty = self.typeOfIndex(inst);
10756		const union_llvm_ty = try o.lowerType(pt, union_ty);
10757		const union_obj = zcu.typeToUnion(union_ty).?;
10758
10759		assert(union_obj.layout != .@"packed");
10760
10761		const layout = Type.getUnionLayout(union_obj, zcu);
10762
10763		const tag_int_val = blk: {
10764		const tag_ty = union_ty.unionTagTypeHypothetical(zcu);
10765		const tag_val = try pt.enumValueFieldIndex(tag_ty, extra.field_index);
10766		break :blk tag_val.intFromEnum(zcu);
10767		};
10768		if (layout.payload_size == 0) {
10769		if (layout.tag_size == 0) {
10770		return .none;
10771		}
10772		assert(!isByRef(union_ty, zcu));
10773		var big_int_space: Value.BigIntSpace = undefined;
10774		const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
10775		return try o.builder.bigIntValue(union_llvm_ty, tag_big_int);
10776		}
10777		assert(isByRef(union_ty, zcu));
10778		// The llvm type of the alloca will be the named LLVM union type, and will not
10779		// necessarily match the format that we need, depending on which tag is active.
10780		// We must construct the correct unnamed struct type here, in order to then set
10781		// the fields appropriately.
10782		const alignment = layout.abi_align.toLlvm();
10783		const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
10784		const llvm_payload = try self.resolveInst(extra.init);
10785		const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
10786		const field_llvm_ty = try o.lowerType(pt, field_ty);
10787		const field_size = field_ty.abiSize(zcu);
10788		const field_align = union_ty.explicitFieldAlignment(extra.field_index, zcu);
10789		const llvm_usize = try o.lowerType(pt, Type.usize);
10790		const usize_zero = try o.builder.intValue(llvm_usize, 0);
10791
10792		assert(field_ty.hasRuntimeBits(zcu));
10793
10794		const llvm_union_ty = t: {
10795		const payload_ty = p: {
10796		if (field_size == layout.payload_size) {
10797		break :p field_llvm_ty;
10798		}
10799		const padding_len = layout.payload_size - field_size;
10800		break :p try o.builder.structType(.@"packed", &.{
10801		field_llvm_ty, try o.builder.arrayType(padding_len, .i8),
10802		});
10803		};
10804		if (layout.tag_size == 0) break :t try o.builder.structType(.normal, &.{payload_ty});
10805		const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
10806		var fields: [3]Builder.Type = undefined;
10807		var fields_len: usize = 2;
10808		if (layout.tag_align.compare(.gte, layout.payload_align)) {
10809		fields = .{ tag_ty, payload_ty, undefined };
10810		} else {
10811		fields = .{ payload_ty, tag_ty, undefined };
10812		}
10813		if (layout.padding != 0) {
10814		fields[fields_len] = try o.builder.arrayType(layout.padding, .i8);
10815		fields_len += 1;
10816		}
10817		break :t try o.builder.structType(.normal, fields[0..fields_len]);
10818		};
10819
10820		// Now we follow the layout as expressed above with GEP instructions to set the
10821		// tag and the payload.
10822		const field_ptr_ty = try pt.ptrType(.{
10823		.child = field_ty.toIntern(),
10824		.flags = .{ .alignment = field_align },
10825		});
10826		if (layout.tag_size == 0) {
10827		const indices = [3]Builder.Value{ usize_zero, .@"0", .@"0" };
10828		const len: usize = if (field_size == layout.payload_size) 2 else 3;
10829		const field_ptr =
10830		try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
10831		try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
10832		return result_ptr;
10833		}
10834
10835		{
10836		const payload_index = @intFromBool(layout.tag_align.compare(.gte, layout.payload_align));
10837		const indices: [3]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, payload_index), .@"0" };
10838		const len: usize = if (field_size == layout.payload_size) 2 else 3;
10839		const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, indices[0..len], "");
10840		try self.store(field_ptr, field_ptr_ty, llvm_payload, .none);
10841		}
10842		{
10843		const tag_index = @intFromBool(layout.tag_align.compare(.lt, layout.payload_align));
10844		const indices: [2]Builder.Value = .{ usize_zero, try o.builder.intValue(.i32, tag_index) };
10845		const field_ptr = try self.wip.gep(.inbounds, llvm_union_ty, result_ptr, &indices, "");
10846		const tag_ty = try o.lowerType(pt, .fromInterned(union_obj.enum_tag_type));
10847		var big_int_space: Value.BigIntSpace = undefined;
10848		const tag_big_int = tag_int_val.toBigInt(&big_int_space, zcu);
10849		const llvm_tag = try o.builder.bigIntValue(tag_ty, tag_big_int);
10850		const tag_alignment = Type.fromInterned(union_obj.enum_tag_type).abiAlignment(zcu).toLlvm();
10851		_ = try self.wip.store(.normal, llvm_tag, field_ptr, tag_alignment);
10852		}
10853
10854		return result_ptr;
10855		}
10856
10857		fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10858		const o = self.ng.object;
10859		const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
10860
10861		comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0);
10862		comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1);
10863
10864		comptime assert(prefetch.locality >= 0);
10865		comptime assert(prefetch.locality <= 3);
10866
10867		comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0);
10868		comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1);
10869
10870		// LLVM fails during codegen of instruction cache prefetchs for these architectures.
10871		// This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
10872		// by the target.
10873		// To work around this, don't emit llvm.prefetch in this case.
10874		// See https://bugs.llvm.org/show_bug.cgi?id=21037
10875		const zcu = self.ng.pt.zcu;
10876		const target = zcu.getTarget();
10877		switch (prefetch.cache) {
10878		.instruction => switch (target.cpu.arch) {
10879		.x86_64,
10880		.x86,
10881		.powerpc,
10882		.powerpcle,
10883		.powerpc64,
10884		.powerpc64le,
10885		=> return .none,
10886		.arm, .armeb, .thumb, .thumbeb => {
10887		switch (prefetch.rw) {
10888		.write => return .none,
10889		else => {},
10890		}
10891		},
10892		else => {},
10893		},
10894		.data => {},
10895		}
10896
10897		_ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
10898		try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
10899		try o.builder.intValue(.i32, prefetch.rw),
10900		try o.builder.intValue(.i32, prefetch.locality),
10901		try o.builder.intValue(.i32, prefetch.cache),
10902		}, "");
10903		return .none;
10904		}
10905
10906		fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10907		const o = self.ng.object;
10908		const pt = self.ng.pt;
10909		const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
10910		const inst_ty = self.typeOfIndex(inst);
10911		const operand = try self.resolveInst(ty_op.operand);
10912
10913		return self.wip.cast(.addrspacecast, operand, try o.lowerType(pt, inst_ty), "");
10914		}
10915
10916		fn workIntrinsic(
10917		self: *FuncGen,
10918		dimension: u32,
10919		default: u32,
10920		comptime basename: []const u8,
10921		) !Builder.Value {
10922		return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
10923		0 => @field(Builder.Intrinsic, basename ++ ".x"),
10924		1 => @field(Builder.Intrinsic, basename ++ ".y"),
10925		2 => @field(Builder.Intrinsic, basename ++ ".z"),
10926		else => return self.ng.object.builder.intValue(.i32, default),
10927		}, &.{}, &.{}, "");
10928		}
10929
10930		fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10931		const target = self.ng.pt.zcu.getTarget();
10932
10933		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10934		const dimension = pl_op.payload;
10935
10936		return switch (target.cpu.arch) {
10937		.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
10938		.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
10939		else => unreachable,
10940		};
10941		}
10942
10943		fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10944		const o = self.ng.object;
10945		const pt = self.ng.pt;
10946		const target = pt.zcu.getTarget();
10947
10948		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10949		const dimension = pl_op.payload;
10950
10951		switch (target.cpu.arch) {
10952		.amdgcn => {
10953		if (dimension >= 3) return .@"1";
10954
10955		// Fetch the dispatch pointer, which points to this structure:
10956		// https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
10957		const dispatch_ptr =
10958		try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
10959
10960		// Load the work_group_* member from the struct as u16.
10961		// Just treat the dispatch pointer as an array of u16 to keep things simple.
10962		const workgroup_size_ptr = try self.wip.gep(.inbounds, .i16, dispatch_ptr, &.{
10963		try o.builder.intValue(try o.lowerType(pt, Type.usize), 2 + dimension),
10964		}, "");
10965		const workgroup_size_alignment = comptime Builder.Alignment.fromByteUnits(2);
10966		return self.wip.load(.normal, .i16, workgroup_size_ptr, workgroup_size_alignment, "");
10967		},
10968		.nvptx, .nvptx64 => {
10969		return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
10970		},
10971		else => unreachable,
10972		}
10973		}
10974
10975		fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value {
10976		const target = self.ng.pt.zcu.getTarget();
10977
10978		const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
10979		const dimension = pl_op.payload;
10980
10981		return switch (target.cpu.arch) {
10982		.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
10983		.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
10984		else => unreachable,
10985		};
10986		}
10987
10988		fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index {
10989		const o = self.ng.object;
10990		const pt = self.ng.pt;
10991
10992		const table = o.error_name_table;
10993		if (table != .none) return table;
10994
10995		// TODO: Address space
10996		const variable_index =
10997		try o.builder.addVariable(try o.builder.strtabString("__zig_err_name_table"), .ptr, .default);
10998		variable_index.setLinkage(.private, &o.builder);
10999		variable_index.setMutability(.constant, &o.builder);
11000		variable_index.setUnnamedAddr(.unnamed_addr, &o.builder);
11001		variable_index.setAlignment(
11002		Type.slice_const_u8_sentinel_0.abiAlignment(pt.zcu).toLlvm(),
11003		&o.builder,
11004		);
11005
11006		o.error_name_table = variable_index;
11007		return variable_index;
11008		}
11009
11010		/// Assumes the optional is not pointer-like and payload has bits.
11011		fn optCmpNull(
11012		self: *FuncGen,
11013		cond: Builder.IntegerCondition,
11014		opt_llvm_ty: Builder.Type,
11015		opt_handle: Builder.Value,
11016		is_by_ref: bool,
11017		access_kind: Builder.MemoryAccessKind,
11018		) Allocator.Error!Builder.Value {
11019		const o = self.ng.object;
11020		const field = b: {
11021		if (is_by_ref) {
11022		const field_ptr = try self.wip.gepStruct(opt_llvm_ty, opt_handle, 1, "");
11023		break :b try self.wip.load(access_kind, .i8, field_ptr, .default, "");
11024		}
11025		break :b try self.wip.extractValue(opt_handle, &.{1}, "");
11026		};
11027		comptime assert(optional_layout_version == 3);
11028
11029		return self.wip.icmp(cond, field, try o.builder.intValue(.i8, 0), "");
11030		}
11031
11032		/// Assumes the optional is not pointer-like and payload has bits.
11033		fn optPayloadHandle(
11034		fg: *FuncGen,
11035		opt_llvm_ty: Builder.Type,
11036		opt_handle: Builder.Value,
11037		opt_ty: Type,
11038		can_elide_load: bool,
11039		) !Builder.Value {
11040		const pt = fg.ng.pt;
11041		const zcu = pt.zcu;
11042		const payload_ty = opt_ty.optionalChild(zcu);
11043
11044		if (isByRef(opt_ty, zcu)) {
11045		// We have a pointer and we need to return a pointer to the first field.
11046		const payload_ptr = try fg.wip.gepStruct(opt_llvm_ty, opt_handle, 0, "");
11047
11048		const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
11049		if (isByRef(payload_ty, zcu)) {
11050		if (can_elide_load)
11051		return payload_ptr;
11052
11053		return fg.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
11054		}
11055		return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_alignment);
11056		}
11057
11058		assert(!isByRef(payload_ty, zcu));
11059		return fg.wip.extractValue(opt_handle, &.{0}, "");
11060		}
11061
11062		fn buildOptional(
11063		self: *FuncGen,
11064		optional_ty: Type,
11065		payload: Builder.Value,
11066		non_null_bit: Builder.Value,
11067		) !Builder.Value {
11068		const o = self.ng.object;
11069		const pt = self.ng.pt;
11070		const zcu = pt.zcu;
11071		const optional_llvm_ty = try o.lowerType(pt, optional_ty);
11072		const non_null_field = try self.wip.cast(.zext, non_null_bit, .i8, "");
11073
11074		if (isByRef(optional_ty, zcu)) {
11075		const payload_alignment = optional_ty.abiAlignment(pt.zcu).toLlvm();
11076		const alloca_inst = try self.buildAlloca(optional_llvm_ty, payload_alignment);
11077
11078		{
11079		const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 0, "");
11080		_ = try self.wip.store(.normal, payload, field_ptr, payload_alignment);
11081		}
11082		{
11083		const non_null_alignment = comptime Builder.Alignment.fromByteUnits(1);
11084		const field_ptr = try self.wip.gepStruct(optional_llvm_ty, alloca_inst, 1, "");
11085		_ = try self.wip.store(.normal, non_null_field, field_ptr, non_null_alignment);
11086		}
11087
11088		return alloca_inst;
11089		}
11090
11091		return self.wip.buildAggregate(optional_llvm_ty, &.{ payload, non_null_field }, "");
11092		}
11093
11094		fn fieldPtr(
11095		self: *FuncGen,
11096		aggregate_ptr: Builder.Value,
11097		aggregate_ptr_ty: Type,
11098		field_index: u32,
11099		) !Builder.Value {
11100		const o = self.ng.object;
11101		const pt = self.ng.pt;
11102		const zcu = pt.zcu;
11103		const aggregate_ty = aggregate_ptr_ty.childType(zcu);
11104		if (aggregate_ty.containerLayout(zcu) == .@"packed") {
11105		// A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the
11106		// bit offset is represented in the pointer type.
11107		return aggregate_ptr;
11108		}
11109		switch (aggregate_ty.zigTypeTag(zcu)) {
11110		.@"struct" => {
11111		if (!aggregate_ty.hasRuntimeBits(zcu)) {
11112		return aggregate_ptr;
11113		}
11114		const struct_llvm_ty = try o.lowerType(pt, aggregate_ty);
11115		if (o.llvmFieldIndex(aggregate_ty, field_index)) \|llvm_field_index\| {
11116		return self.wip.gepStruct(struct_llvm_ty, aggregate_ptr, llvm_field_index, "");
11117		} else {
11118		// If we found no index then this means this is a zero sized field at the
11119		// end of the struct. Treat our struct pointer as an array of two and get
11120		// the index to the element at index `1` to get a pointer to the end of
11121		// the struct.
11122		const llvm_index = try o.builder.intValue(
11123		try o.lowerType(pt, Type.usize),
11124		@intFromBool(aggregate_ty.hasRuntimeBits(zcu)),
11125		);
11126		return self.wip.gep(.inbounds, struct_llvm_ty, aggregate_ptr, &.{llvm_index}, "");
11127		}
11128		},
11129		.@"union" => {
11130		const layout = aggregate_ty.unionGetLayout(zcu);
11131		if (layout.payload_size == 0) return aggregate_ptr;
11132		const payload_index = @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align));
11133		const union_llvm_ty = try o.lowerType(pt, aggregate_ty);
11134		return self.wip.gepStruct(union_llvm_ty, aggregate_ptr, payload_index, "");
11135		},
11136		else => unreachable,
11137		}
11138		}
11139
11140		/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
11141		fn loadTruncate(
11142		fg: *FuncGen,
11143		access_kind: Builder.MemoryAccessKind,
11144		payload_ty: Type,
11145		payload_ptr: Builder.Value,
11146		payload_alignment: Builder.Alignment,
11147		) !Builder.Value {
11148		// from https://llvm.org/docs/LangRef.html#load-instruction :
11149		// "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
11150		// => so load the byte aligned value and trunc the unwanted bits.
11151
11152		const o = fg.ng.object;
11153		const pt = fg.ng.pt;
11154		const zcu = pt.zcu;
11155		const payload_llvm_ty = try o.lowerType(pt, payload_ty);
11156		const abi_size = payload_ty.abiSize(zcu);
11157
11158		const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
11159		try o.builder.intType(@intCast(abi_size * 8))
11160		else
11161		payload_llvm_ty;
11162		const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
11163		const shifted = if (payload_llvm_ty != load_llvm_ty and o.target.cpu.arch.endian() == .big)
11164		try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
11165		load_llvm_ty,
11166		(payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
11167		), "")
11168		else
11169		loaded;
11170
11171		return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
11172		}
11173
11174		/// Load a by-ref type by constructing a new alloca and performing a memcpy.
11175		fn loadByRef(
11176		fg: *FuncGen,
11177		ptr: Builder.Value,
11178		pointee_type: Type,
11179		ptr_alignment: Builder.Alignment,
11180		access_kind: Builder.MemoryAccessKind,
11181		) !Builder.Value {
11182		const o = fg.ng.object;
11183		const pt = fg.ng.pt;
11184		const pointee_llvm_ty = try o.lowerType(pt, pointee_type);
11185		const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
11186		.max(pointee_type.abiAlignment(pt.zcu)).toLlvm();
11187		const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
11188		const size_bytes = pointee_type.abiSize(pt.zcu);
11189		_ = try fg.wip.callMemCpy(
11190		result_ptr,
11191		result_align,
11192		ptr,
11193		ptr_alignment,
11194		try o.builder.intValue(try o.lowerType(pt, Type.usize), size_bytes),
11195		access_kind,
11196		fg.disable_intrinsics,
11197		);
11198		return result_ptr;
11199		}
11200
11201		/// This function always performs a copy. For isByRef=true types, it creates a new
11202		/// alloca and copies the value into it, then returns the alloca instruction.
11203		/// For isByRef=false types, it creates a load instruction and returns it.
11204		fn load(self: *FuncGen, ptr: Builder.Value, ptr_ty: Type) !Builder.Value {
11205		const o = self.ng.object;
11206		const pt = self.ng.pt;
11207		const zcu = pt.zcu;
11208		const info = ptr_ty.ptrInfo(zcu);
11209		const elem_ty = Type.fromInterned(info.child);
11210		if (!elem_ty.hasRuntimeBits(zcu)) return .none;
11211
11212		const ptr_alignment = (if (info.flags.alignment != .none)
11213		@as(InternPool.Alignment, info.flags.alignment)
11214		else
11215		elem_ty.abiAlignment(zcu)).toLlvm();
11216
11217		const access_kind: Builder.MemoryAccessKind =
11218		if (info.flags.is_volatile) .@"volatile" else .normal;
11219
11220		if (info.flags.vector_index != .none) {
11221		const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11222		const vec_elem_ty = try o.lowerType(pt, elem_ty);
11223		const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11224
11225		const loaded_vector = try self.wip.load(access_kind, vec_ty, ptr, ptr_alignment, "");
11226		return self.wip.extractElement(loaded_vector, index_u32, "");
11227		}
11228
11229		if (info.packed_offset.host_size == 0) {
11230		if (isByRef(elem_ty, zcu)) {
11231		return self.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
11232		}
11233		return self.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
11234		}
11235
11236		const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11237		const containing_int =
11238		try self.wip.load(access_kind, containing_int_ty, ptr, ptr_alignment, "");
11239
11240		const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11241		const shift_amt = try o.builder.intValue(containing_int_ty, info.packed_offset.bit_offset);
11242		const shifted_value = try self.wip.bin(.lshr, containing_int, shift_amt, "");
11243		const elem_llvm_ty = try o.lowerType(pt, elem_ty);
11244
11245		if (isByRef(elem_ty, zcu)) {
11246		const result_align = elem_ty.abiAlignment(zcu).toLlvm();
11247		const result_ptr = try self.buildAlloca(elem_llvm_ty, result_align);
11248
11249		const same_size_int = try o.builder.intType(@intCast(elem_bits));
11250		const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11251		_ = try self.wip.store(.normal, truncated_int, result_ptr, result_align);
11252		return result_ptr;
11253		}
11254
11255		if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
11256		const same_size_int = try o.builder.intType(@intCast(elem_bits));
11257		const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11258		return self.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
11259		}
11260
11261		if (elem_ty.isPtrAtRuntime(zcu)) {
11262		const same_size_int = try o.builder.intType(@intCast(elem_bits));
11263		const truncated_int = try self.wip.cast(.trunc, shifted_value, same_size_int, "");
11264		return self.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
11265		}
11266
11267		return self.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
11268		}
11269
11270		fn store(
11271		self: *FuncGen,
11272		ptr: Builder.Value,
11273		ptr_ty: Type,
11274		elem: Builder.Value,
11275		ordering: Builder.AtomicOrdering,
11276		) !void {
11277		const o = self.ng.object;
11278		const pt = self.ng.pt;
11279		const zcu = pt.zcu;
11280		const info = ptr_ty.ptrInfo(zcu);
11281		const elem_ty = Type.fromInterned(info.child);
11282		if (!elem_ty.hasRuntimeBits(zcu)) {
11283		return;
11284		}
11285		const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
11286		const access_kind: Builder.MemoryAccessKind =
11287		if (info.flags.is_volatile) .@"volatile" else .normal;
11288
11289		if (info.flags.vector_index != .none) {
11290		const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
11291		const vec_elem_ty = try o.lowerType(pt, elem_ty);
11292		const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
11293
11294		const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
11295
11296		const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
11297
11298		assert(ordering == .none);
11299		_ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
11300		return;
11301		}
11302
11303		if (info.packed_offset.host_size != 0) {
11304		const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
11305		assert(ordering == .none);
11306		const containing_int =
11307		try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
11308		const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
11309		const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
11310		// Convert to equally-sized integer type in order to perform the bit
11311		// operations on the value to store
11312		const value_bits_type = try o.builder.intType(@intCast(elem_bits));
11313		const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
11314		try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
11315		else
11316		try self.wip.cast(.bitcast, elem, value_bits_type, "");
11317
11318		const mask_val = blk: {
11319		const zext = try self.wip.cast(
11320		.zext,
11321		try o.builder.intValue(value_bits_type, -1),
11322		containing_int_ty,
11323		"",
11324		);
11325		const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
11326		break :blk try self.wip.bin(
11327		.xor,
11328		shl,
11329		try o.builder.intValue(containing_int_ty, -1),
11330		"",
11331		);
11332		};
11333
11334		const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
11335		const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
11336		const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
11337		const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
11338
11339		assert(ordering == .none);
11340		_ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
11341		return;
11342		}
11343		if (!isByRef(elem_ty, zcu)) {
11344		_ = try self.wip.storeAtomic(
11345		access_kind,
11346		elem,
11347		ptr,
11348		self.sync_scope,
11349		ordering,
11350		ptr_alignment,
11351		);
11352		return;
11353		}
11354		assert(ordering == .none);
11355		_ = try self.wip.callMemCpy(
11356		ptr,
11357		ptr_alignment,
11358		elem,
11359		elem_ty.abiAlignment(zcu).toLlvm(),
11360		try o.builder.intValue(try o.lowerType(pt, Type.usize), elem_ty.abiSize(zcu)),
11361		access_kind,
11362		self.disable_intrinsics,
11363		);
11364		}
11365
11366		fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
11367		const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
11368		const o = fg.ng.object;
11369		const pt = fg.ng.pt;
11370		const usize_ty = try o.lowerType(pt, Type.usize);
11371		const zero = try o.builder.intValue(usize_ty, 0);
11372		const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
11373		const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
11374		_ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
11375		}
11376
11377		fn valgrindClientRequest(
11378		fg: *FuncGen,
11379		default_value: Builder.Value,
11380		request: Builder.Value,
11381		a1: Builder.Value,
11382		a2: Builder.Value,
11383		a3: Builder.Value,
11384		a4: Builder.Value,
11385		a5: Builder.Value,
11386		) Allocator.Error!Builder.Value {
11387		const o = fg.ng.object;
11388		const pt = fg.ng.pt;
11389		const zcu = pt.zcu;
11390		const target = zcu.getTarget();
11391		if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
11392
11393		const llvm_usize = try o.lowerType(pt, Type.usize);
11394		const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
11395
11396		const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
11397		const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
11398		const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
11399		fg.valgrind_client_request_array = array_ptr;
11400		break :a array_ptr;
11401		} else fg.valgrind_client_request_array;
11402		const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
11403		const zero = try o.builder.intValue(llvm_usize, 0);
11404		for (array_elements, 0..) \|elem, i\| {
11405		const elem_ptr = try fg.wip.gep(.inbounds, array_llvm_ty, array_ptr, &.{
11406		zero, try o.builder.intValue(llvm_usize, i),
11407		}, "");
11408		_ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
11409		}
11410
11411		const arch_specific: struct {
11412		template: [:0]const u8,
11413		constraints: [:0]const u8,
11414		} = switch (target.cpu.arch) {
11415		.arm, .armeb, .thumb, .thumbeb => .{
11416		.template =
11417		\\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
11418		\\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
11419		\\ orr r10, r10, r10
11420		,
11421		.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11422		},
11423		.aarch64, .aarch64_be => .{
11424		.template =
11425		\\ ror x12, x12, #3 ; ror x12, x12, #13
11426		\\ ror x12, x12, #51 ; ror x12, x12, #61
11427		\\ orr x10, x10, x10
11428		,
11429		.constraints = "={x3},{x4},{x3},~{cc},~{memory}",
11430		},
11431		.mips, .mipsel => .{
11432		.template =
11433		\\ srl $$0, $$0, 13
11434		\\ srl $$0, $$0, 29
11435		\\ srl $$0, $$0, 3
11436		\\ srl $$0, $$0, 19
11437		\\ or $$13, $$13, $$13
11438		,
11439		.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11440		},
11441		.mips64, .mips64el => .{
11442		.template =
11443		\\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
11444		\\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
11445		\\ or $$13, $$13, $$13
11446		,
11447		.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
11448		},
11449		.powerpc, .powerpcle => .{
11450		.template =
11451		\\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
11452		\\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
11453		\\ or 1, 1, 1
11454		,
11455		.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11456		},
11457		.powerpc64, .powerpc64le => .{
11458		.template =
11459		\\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
11460		\\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
11461		\\ or 1, 1, 1
11462		,
11463		.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
11464		},
11465		.riscv64 => .{
11466		.template =
11467		\\ .option push
11468		\\ .option norvc
11469		\\ srli zero, zero, 3
11470		\\ srli zero, zero, 13
11471		\\ srli zero, zero, 51
11472		\\ srli zero, zero, 61
11473		\\ or a0, a0, a0
11474		\\ .option pop
11475		,
11476		.constraints = "={a3},{a4},{a3},~{cc},~{memory}",
11477		},
11478		.s390x => .{
11479		.template =
11480		\\ lr %r15, %r15
11481		\\ lr %r1, %r1
11482		\\ lr %r2, %r2
11483		\\ lr %r3, %r3
11484		\\ lr %r2, %r2
11485		,
11486		.constraints = "={r3},{r2},{r3},~{cc},~{memory}",
11487		},
11488		.x86 => .{
11489		.template =
11490		\\ roll $$3, %edi ; roll $$13, %edi
11491		\\ roll $$61, %edi ; roll $$51, %edi
11492		\\ xchgl %ebx, %ebx
11493		,
11494		.constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11495		},
11496		.x86_64 => .{
11497		.template =
11498		\\ rolq $$3, %rdi ; rolq $$13, %rdi
11499		\\ rolq $$61, %rdi ; rolq $$51, %rdi
11500		\\ xchgq %rbx, %rbx
11501		,
11502		.constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
11503		},
11504		else => unreachable,
11505		};
11506
11507		return fg.wip.callAsm(
11508		.none,
11509		try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
11510		.{ .sideeffect = true },
11511		try o.builder.string(arch_specific.template),
11512		try o.builder.string(arch_specific.constraints),
11513		&.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
11514		"",
11515		);
11516		}
11517
11518		fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
11519		const zcu = fg.ng.pt.zcu;
11520		return fg.air.typeOf(inst, &zcu.intern_pool);
11521		}
11522
11523		fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
11524		const zcu = fg.ng.pt.zcu;
11525		return fg.air.typeOfIndex(inst, &zcu.intern_pool);
11526		}
11527		};
11528
11529		fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering {
11530		return switch (atomic_order) {
11531		.unordered => .unordered,
11532		.monotonic => .monotonic,
11533		.acquire => .acquire,
11534		.release => .release,
11535		.acq_rel => .acq_rel,
11536		.seq_cst => .seq_cst,
11537		};
11538		}
11539
11540		fn toLlvmAtomicRmwBinOp(
11541		op: std.builtin.AtomicRmwOp,
11542		is_signed: bool,
11543		is_float: bool,
11544		) Builder.Function.Instruction.AtomicRmw.Operation {
11545		return switch (op) {
11546		.Xchg => .xchg,
11547		.Add => if (is_float) .fadd else return .add,
11548		.Sub => if (is_float) .fsub else return .sub,
11549		.And => .@"and",
11550		.Nand => .nand,
11551		.Or => .@"or",
11552		.Xor => .xor,
11553		.Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
11554		.Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
11555		};
11556		}
11557
11558		const CallingConventionInfo = struct {
11559		/// The LLVM calling convention to use.
11560		llvm_cc: Builder.CallConv,
11561		/// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
11562		align_stack: bool,
11563		/// Whether the function needs a `naked` attribute.
11564		naked: bool,
11565		/// How many leading parameters to apply the `inreg` attribute to.
11566		inreg_param_count: u2 = 0,
11567		};
11568
11569		pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
11570		const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
11571		const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
11572		inline else => \|pl\| switch (@TypeOf(pl)) {
11573		void => .{ null, 0 },
11574		std.builtin.CallingConvention.ArcInterruptOptions,
11575		std.builtin.CallingConvention.ArmInterruptOptions,
11576		std.builtin.CallingConvention.RiscvInterruptOptions,
11577		std.builtin.CallingConvention.ShInterruptOptions,
11578		std.builtin.CallingConvention.MicroblazeInterruptOptions,
11579		std.builtin.CallingConvention.MipsInterruptOptions,
11580		std.builtin.CallingConvention.CommonOptions,
11581		=> .{ pl.incoming_stack_alignment, 0 },
11582		std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
11583		else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
11584		},
11585		};
11586		return .{
11587		.llvm_cc = llvm_cc,
11588		.align_stack = if (incoming_stack_alignment) \|a\| need_align: {
11589		const normal_stack_align = target.stackAlignment();
11590		break :need_align a < normal_stack_align;
11591		} else false,
11592		.naked = cc == .naked,
11593		.inreg_param_count = register_params,
11594		};
11595		}
11596		fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
11597		if (target.cCallingConvention()) \|default_c\| {
11598		if (cc_tag == default_c) {
11599		return .ccc;
11600		}
11601		}
11602		return switch (cc_tag) {
11603		.@"inline" => unreachable,
11604		.auto, .async => .fastcc,
11605		.naked => .ccc,
11606		.x86_64_sysv => .x86_64_sysvcc,
11607		.x86_64_win => .win64cc,
11608		.x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
11609		.x86_regcallcc
11610		else
11611		null,
11612		.x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
11613		.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11614		else
11615		null,
11616		.x86_64_vectorcall => .x86_vectorcallcc,
11617		.x86_64_interrupt => .x86_intrcc,
11618		.x86_stdcall => .x86_stdcallcc,
11619		.x86_fastcall => .x86_fastcallcc,
11620		.x86_thiscall => .x86_thiscallcc,
11621		.x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
11622		.x86_regcallcc
11623		else
11624		null,
11625		.x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
11626		.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
11627		else
11628		null,
11629		.x86_vectorcall => .x86_vectorcallcc,
11630		.x86_interrupt => .x86_intrcc,
11631		.aarch64_vfabi => .aarch64_vector_pcs,
11632		.aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
11633		.arm_aapcs => .arm_aapcscc,
11634		.arm_aapcs_vfp => .arm_aapcs_vfpcc,
11635		.riscv64_lp64_v => .riscv_vectorcallcc,
11636		.riscv32_ilp32_v => .riscv_vectorcallcc,
11637		.avr_builtin => .avr_builtincc,
11638		.avr_signal => .avr_signalcc,
11639		.avr_interrupt => .avr_intrcc,
11640		.m68k_rtd => .m68k_rtdcc,
11641		.m68k_interrupt => .m68k_intrcc,
11642		.msp430_interrupt => .msp430_intrcc,
11643		.amdgcn_kernel => .amdgpu_kernel,
11644		.amdgcn_cs => .amdgpu_cs,
11645		.nvptx_device => .ptx_device,
11646		.nvptx_kernel => .ptx_kernel,
11647
11648		// Calling conventions which LLVM uses function attributes for.
11649		.riscv64_interrupt,
11650		.riscv32_interrupt,
11651		.arm_interrupt,
11652		.mips64_interrupt,
11653		.mips_interrupt,
11654		.csky_interrupt,
11655		=> .ccc,
11656
11657		// All the calling conventions which LLVM does not have a general representation for.
11658		// Note that these are often still supported through the `cCallingConvention` path above via `ccc`.
11659		.x86_16_cdecl,
11660		.x86_16_stdcall,
11661		.x86_16_regparmcall,
11662		.x86_16_interrupt,
11663		.x86_sysv,
11664		.x86_win,
11665		.x86_thiscall_mingw,
11666		.x86_64_x32,
11667		.aarch64_aapcs,
11668		.aarch64_aapcs_darwin,
11669		.aarch64_aapcs_win,
11670		.alpha_osf,
11671		.microblaze_std,
11672		.microblaze_interrupt,
11673		.mips64_n64,
11674		.mips64_n32,
11675		.mips_o32,
11676		.riscv64_lp64,
11677		.riscv32_ilp32,
11678		.sparc64_sysv,
11679		.sparc_sysv,
11680		.powerpc64_elf,
11681		.powerpc64_elf_altivec,
11682		.powerpc64_elf_v2,
11683		.powerpc_sysv,
11684		.powerpc_sysv_altivec,
11685		.powerpc_aix,
11686		.powerpc_aix_altivec,
11687		.wasm_mvp,
11688		.arc_sysv,
11689		.arc_interrupt,
11690		.avr_gnu,
11691		.bpf_std,
11692		.csky_sysv,
11693		.hexagon_sysv,
11694		.hexagon_sysv_hvx,
11695		.hppa_elf,
11696		.hppa64_elf,
11697		.kvx_lp64,
11698		.kvx_ilp32,
11699		.lanai_sysv,
11700		.loongarch64_lp64,
11701		.loongarch32_ilp32,
11702		.m68k_sysv,
11703		.m68k_gnu,
11704		.msp430_eabi,
11705		.or1k_sysv,
11706		.propeller_sysv,
11707		.s390x_sysv,
11708		.s390x_sysv_vx,
11709		.sh_gnu,
11710		.sh_renesas,
11711		.sh_interrupt,
11712		.ve_sysv,
11713		.xcore_xs1,
11714		.xcore_xs2,
11715		.xtensa_call0,
11716		.xtensa_windowed,
11717		.amdgcn_device,
11718		.spirv_device,
11719		.spirv_kernel,
11720		.spirv_fragment,
11721		.spirv_vertex,
11722		=> null,
11723		};
11724		}
11725
11726		/// Convert a zig-address space to an llvm address space.
11727		fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
11728		for (llvmAddrSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;
11729		unreachable;
11730		}
11731
11732		const AddrSpaceInfo = struct {
11733		zig: ?std.builtin.AddressSpace,
11734		llvm: Builder.AddrSpace,
11735		non_integral: bool = false,
11736		size: ?u16 = null,
11737		abi: ?u16 = null,
11738		pref: ?u16 = null,
11739		idx: ?u16 = null,
11740		force_in_data_layout: bool = false,
11741		};
11742		fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {
11743		return switch (target.cpu.arch) {
11744		.x86, .x86_64 => &.{
11745		.{ .zig = .generic, .llvm = .default },
11746		.{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs },
11747		.{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs },
11748		.{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss },
11749		.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
11750		.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
11751		.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
11752		},
11753		.nvptx, .nvptx64 => &.{
11754		.{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic },
11755		.{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global },
11756		.{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant },
11757		.{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param },
11758		.{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared },
11759		.{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local },
11760		},
11761		.amdgcn => &.{
11762		.{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true },
11763		.{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true },
11764		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 },
11765		.{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 },
11766		.{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true },
11767		.{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 },
11768		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 },
11769		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 },
11770		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 },
11771		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 },
11772		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 },
11773		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 },
11774		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 },
11775		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 },
11776		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 },
11777		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 },
11778		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 },
11779		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 },
11780		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 },
11781		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 },
11782		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 },
11783		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 },
11784		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 },
11785		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 },
11786		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 },
11787		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 },
11788		.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register },
11789		},
11790		.avr => &.{
11791		.{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 },
11792		.{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 },
11793		.{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 },
11794		.{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 },
11795		.{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 },
11796		.{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 },
11797		.{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 },
11798		},
11799		.wasm32, .wasm64 => &.{
11800		.{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true },
11801		.{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true },
11802		.{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
11803		.{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
11804		},
11805		.m68k => &.{
11806		.{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 },
11807		},
11808		else => &.{
11809		.{ .zig = .generic, .llvm = .default },
	4386	pub fn toLlvmCallConv(cc: std.builtin.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
	4387	const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
	4388	const incoming_stack_alignment: ?u64, const register_params: u2 = switch (cc) {
	4389	inline else => \|pl\| switch (@TypeOf(pl)) {
	4390	void => .{ null, 0 },
	4391	std.builtin.CallingConvention.ArcInterruptOptions,
	4392	std.builtin.CallingConvention.ArmInterruptOptions,
	4393	std.builtin.CallingConvention.RiscvInterruptOptions,
	4394	std.builtin.CallingConvention.ShInterruptOptions,
	4395	std.builtin.CallingConvention.MicroblazeInterruptOptions,
	4396	std.builtin.CallingConvention.MipsInterruptOptions,
	4397	std.builtin.CallingConvention.CommonOptions,
	4398	=> .{ pl.incoming_stack_alignment, 0 },
	4399	std.builtin.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params },
	4400	else => @compileError("TODO: toLlvmCallConv" ++ @tagName(pl)),
11810	4401	},
11811	4402	};
11812		}
11813
11814		/// On some targets, local values that are in the generic address space must be generated into a
11815		/// different address, space and then cast back to the generic address space.
11816		/// For example, on GPUs local variable declarations must be generated into the local address space.
11817		/// This function returns the address space local values should be generated into.
11818		fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
11819		return switch (target.cpu.arch) {
11820		// On amdgcn, locals should be generated into the private address space.
11821		// To make Zig not impossible to use, these are then converted to addresses in the
11822		// generic address space and treates as regular pointers. This is the way that HIP also does it.
11823		.amdgcn => Builder.AddrSpace.amdgpu.private,
11824		else => .default,
11825		};
11826		}
11827
11828		/// On some targets, global values that are in the generic address space must be generated into a
11829		/// different address space, and then cast back to the generic address space.
11830		fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {
11831		return switch (target.cpu.arch) {
11832		// On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
11833		// them.
11834		.amdgcn => Builder.AddrSpace.amdgpu.global,
11835		else => .default,
11836		};
11837		}
11838
11839		/// Return the actual address space that a value should be stored in if its a global address space.
11840		/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
11841		fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
11842		return switch (wanted_address_space) {
11843		.generic => llvmDefaultGlobalAddressSpace(target),
11844		else => \|as\| toLlvmAddressSpace(as, target),
11845		};
11846		}
11847
11848		fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
11849		if (isByRef(ty, zcu)) {
11850		return true;
11851		} else if (target.cpu.arch.isX86() and
11852		!target.cpu.has(.x86, .evex512) and
11853		ty.totalVectorBits(zcu) >= 512)
11854		{
11855		// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
11856		// "512-bit vector arguments require 'evex512' for AVX512"
11857		return true;
11858		} else {
11859		return false;
11860		}
11861		}
11862
11863		fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
11864		const return_type = Type.fromInterned(fn_info.return_type);
11865		if (!return_type.hasRuntimeBits(zcu)) return false;
11866
11867		return switch (fn_info.cc) {
11868		.auto => returnTypeByRef(zcu, target, return_type),
11869		.x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
11870		.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
11871		.x86_sysv, .x86_win => isByRef(return_type, zcu),
11872		.x86_stdcall => !isScalar(zcu, return_type),
11873		.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
11874		.aarch64_aapcs,
11875		.aarch64_aapcs_darwin,
11876		.aarch64_aapcs_win,
11877		=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
11878		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
11879		.memory, .i64_array => true,
11880		.i32_array => \|size\| size != 1,
11881		.byval => false,
11882		},
11883		.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
11884		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
11885		.memory, .i32_array => true,
11886		.byval => false,
11887		},
11888		else => false, // TODO: investigate other targets/callconvs
	4403	return .{
	4404	.llvm_cc = llvm_cc,
	4405	.align_stack = if (incoming_stack_alignment) \|a\| need_align: {
	4406	const normal_stack_align = target.stackAlignment();
	4407	break :need_align a < normal_stack_align;
	4408	} else false,
	4409	.naked = cc == .naked,
	4410	.inreg_param_count = register_params,
11889	4411	};
11890	4412	}
11891
11892		fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
11893		const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
11894		if (class[0] == .memory) return true;
11895		if (class[0] == .x87 and class[2] != .none) return true;
11896		return false;
11897		}
11898
11899		/// In order to support the C calling convention, some return types need to be lowered
11900		/// completely differently in the function prototype to honor the C ABI, and then
11901		/// be effectively bitcasted to the actual return type.
11902		fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11903		const zcu = pt.zcu;
11904		const return_type = Type.fromInterned(fn_info.return_type);
11905		if (!return_type.hasRuntimeBits(zcu)) {
11906		assert(!return_type.isError(zcu));
11907		return .void;
11908		}
11909		const target = zcu.getTarget();
11910		switch (fn_info.cc) {
11911		.@"inline" => unreachable,
11912		.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(pt, return_type),
11913
11914		.x86_64_sysv => return lowerSystemVFnRetTy(o, pt, fn_info),
11915		.x86_64_win => return lowerWin64FnRetTy(o, pt, fn_info),
11916		.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(pt, return_type) else .void,
11917		.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(pt, return_type),
11918		.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
11919		.memory => return .void,
11920		.float_array => return o.lowerType(pt, return_type),
11921		.byval => return o.lowerType(pt, return_type),
11922		.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
11923		.double_integer => return o.builder.arrayType(2, .i64),
11924		},
11925		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
11926		.memory, .i64_array => return .void,
11927		.i32_array => \|len\| return if (len == 1) .i32 else .void,
11928		.byval => return o.lowerType(pt, return_type),
11929		},
11930		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
11931		.memory, .i32_array => return .void,
11932		.byval => return o.lowerType(pt, return_type),
11933		},
11934		.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
11935		.memory => return .void,
11936		.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
11937		.double_integer => {
11938		const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
11939		.riscv64, .riscv64be => .i64,
11940		.riscv32, .riscv32be => .i32,
11941		else => unreachable,
11942		};
11943		return o.builder.structType(.normal, &.{ integer, integer });
11944		},
11945		.byval => return o.lowerType(pt, return_type),
11946		.fields => {
11947		var types_len: usize = 0;
11948		var types: [8]Builder.Type = undefined;
11949		for (0..return_type.structFieldCount(zcu)) \|field_index\| {
11950		const field_ty = return_type.fieldType(field_index, zcu);
11951		if (!field_ty.hasRuntimeBits(zcu)) continue;
11952		types[types_len] = try o.lowerType(pt, field_ty);
11953		types_len += 1;
11954		}
11955		return o.builder.structType(.normal, types[0..types_len]);
11956		},
11957		},
11958		.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
11959		.direct => \|scalar_ty\| return o.lowerType(pt, scalar_ty),
11960		.indirect => return .void,
11961		},
11962		// TODO investigate other callconvs
11963		else => return o.lowerType(pt, return_type),
11964		}
11965		}
11966
11967		fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11968		const zcu = pt.zcu;
11969		const return_type = Type.fromInterned(fn_info.return_type);
11970		switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
11971		.integer => {
11972		if (isScalar(zcu, return_type)) {
11973		return o.lowerType(pt, return_type);
11974		} else {
11975		return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
11976		}
11977		},
11978		.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
11979		.memory => return .void,
11980		.sse => return o.lowerType(pt, return_type),
11981		else => unreachable,
11982		}
11983		}
11984
11985		fn lowerSystemVFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
11986		const zcu = pt.zcu;
11987		const ip = &zcu.intern_pool;
11988		const return_type = Type.fromInterned(fn_info.return_type);
11989		return_type.assertHasLayout(zcu);
11990		if (isScalar(zcu, return_type)) {
11991		return o.lowerType(pt, return_type);
11992		}
11993		const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
11994		var types_index: u32 = 0;
11995		var types_buffer: [8]Builder.Type = undefined;
11996		for (classes) \|class\| {
11997		switch (class) {
11998		.integer => {
11999		types_buffer[types_index] = .i64;
12000		types_index += 1;
12001		},
12002		.sse => {
12003		types_buffer[types_index] = .double;
12004		types_index += 1;
12005		},
12006		.sseup => {
12007		if (types_buffer[types_index - 1] == .double) {
12008		types_buffer[types_index - 1] = .fp128;
12009		} else {
12010		types_buffer[types_index] = .double;
12011		types_index += 1;
12012		}
12013		},
12014		.float => {
12015		types_buffer[types_index] = .float;
12016		types_index += 1;
12017		},
12018		.float_combine => {
12019		types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
12020		types_index += 1;
12021		},
12022		.x87 => {
12023		if (types_index != 0 or classes[2] != .none) return .void;
12024		types_buffer[types_index] = .x86_fp80;
12025		types_index += 1;
12026		},
12027		.x87up => continue,
12028		.none => break,
12029		.memory, .integer_per_element => return .void,
12030		.win_i128 => unreachable, // windows only
12031		}
12032		}
12033		const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12034		if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12035		assert(first_non_integer orelse classes.len == types_index);
12036		switch (ip.indexToKey(return_type.toIntern())) {
12037		.struct_type => {
12038		const size = return_type.abiSize(zcu);
12039		assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12040		if (size % 8 > 0) {
12041		types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
12042		}
12043		},
12044		else => {},
12045		}
12046		if (types_index == 1) return types_buffer[0];
12047		}
12048		return o.builder.structType(.normal, types_buffer[0..types_index]);
12049		}
12050
12051		const ParamTypeIterator = struct {
12052		object: *Object,
12053		pt: Zcu.PerThread,
12054		fn_info: InternPool.Key.FuncType,
12055		zig_index: u32,
12056		llvm_index: u32,
12057		types_len: u32,
12058		types_buffer: [8]Builder.Type,
12059		byval_attr: bool,
12060
12061		const Lowering = union(enum) {
12062		no_bits,
12063		byval,
12064		byref,
12065		byref_mut,
12066		abi_sized_int,
12067		multiple_llvm_types,
12068		slice,
12069		float_array: u8,
12070		i32_array: u8,
12071		i64_array: u8,
12072		};
12073
12074		fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
12075		if (it.zig_index >= it.fn_info.param_types.len) return null;
12076		const ip = &it.pt.zcu.intern_pool;
12077		const ty = it.fn_info.param_types.get(ip)[it.zig_index];
12078		it.byval_attr = false;
12079		return nextInner(it, Type.fromInterned(ty));
12080		}
12081
12082		/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
12083		fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
12084		assert(std.meta.eql(it.pt, fg.ng.pt));
12085		const ip = &it.pt.zcu.intern_pool;
12086		if (it.zig_index >= it.fn_info.param_types.len) {
12087		if (it.zig_index >= args.len) {
12088		return null;
12089		} else {
12090		return nextInner(it, fg.typeOf(args[it.zig_index]));
12091		}
12092		} else {
12093		return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index]));
12094		}
12095		}
12096
12097		fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12098		const pt = it.pt;
12099		const zcu = pt.zcu;
12100		const target = zcu.getTarget();
12101
12102		if (!ty.hasRuntimeBits(zcu)) {
12103		it.zig_index += 1;
12104		return .no_bits;
12105		}
12106		switch (it.fn_info.cc) {
12107		.@"inline" => unreachable,
12108		.auto => {
12109		it.zig_index += 1;
12110		it.llvm_index += 1;
12111		if (ty.isSlice(zcu) or
12112		(ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
12113		{
12114		it.llvm_index += 1;
12115		return .slice;
12116		} else if (isByRef(ty, zcu)) {
12117		return .byref;
12118		} else if (target.cpu.arch.isX86() and
12119		!target.cpu.has(.x86, .evex512) and
12120		ty.totalVectorBits(zcu) >= 512)
12121		{
12122		// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
12123		// "512-bit vector arguments require 'evex512' for AVX512"
12124		return .byref;
12125		} else {
12126		return .byval;
12127		}
12128		},
12129		.async => {
12130		@panic("TODO implement async function lowering in the LLVM backend");
12131		},
12132		.x86_64_sysv => return it.nextSystemV(ty),
12133		.x86_64_win => return it.nextWin64(ty),
12134		.x86_stdcall => {
12135		it.zig_index += 1;
12136		it.llvm_index += 1;
12137
12138		if (isScalar(zcu, ty)) {
12139		return .byval;
12140		} else {
12141		it.byval_attr = true;
12142		return .byref;
12143		}
12144		},
12145		.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
12146		it.zig_index += 1;
12147		it.llvm_index += 1;
12148		switch (aarch64_c_abi.classifyType(ty, zcu)) {
12149		.memory => return .byref_mut,
12150		.float_array => \|len\| return Lowering{ .float_array = len },
12151		.byval => return .byval,
12152		.integer => {
12153		it.types_len = 1;
12154		it.types_buffer[0] = .i64;
12155		return .multiple_llvm_types;
12156		},
12157		.double_integer => return Lowering{ .i64_array = 2 },
12158		}
12159		},
12160		.arm_aapcs, .arm_aapcs_vfp => {
12161		it.zig_index += 1;
12162		it.llvm_index += 1;
12163		switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
12164		.memory => {
12165		it.byval_attr = true;
12166		return .byref;
12167		},
12168		.byval => return .byval,
12169		.i32_array => \|size\| return Lowering{ .i32_array = size },
12170		.i64_array => \|size\| return Lowering{ .i64_array = size },
12171		}
12172		},
12173		.mips_o32 => {
12174		it.zig_index += 1;
12175		it.llvm_index += 1;
12176		switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
12177		.memory => {
12178		it.byval_attr = true;
12179		return .byref;
12180		},
12181		.byval => return .byval,
12182		.i32_array => \|size\| return Lowering{ .i32_array = size },
12183		}
12184		},
12185		.riscv64_lp64, .riscv32_ilp32 => {
12186		it.zig_index += 1;
12187		it.llvm_index += 1;
12188		switch (riscv_c_abi.classifyType(ty, zcu)) {
12189		.memory => return .byref_mut,
12190		.byval => return .byval,
12191		.integer => return .abi_sized_int,
12192		.double_integer => return Lowering{ .i64_array = 2 },
12193		.fields => {
12194		it.types_len = 0;
12195		for (0..ty.structFieldCount(zcu)) \|field_index\| {
12196		const field_ty = ty.fieldType(field_index, zcu);
12197		if (!field_ty.hasRuntimeBits(zcu)) continue;
12198		it.types_buffer[it.types_len] = try it.object.lowerType(pt, field_ty);
12199		it.types_len += 1;
12200		}
12201		it.llvm_index += it.types_len - 1;
12202		return .multiple_llvm_types;
12203		},
12204		}
12205		},
12206		.wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) {
12207		.direct => \|scalar_ty\| {
12208		if (isScalar(zcu, ty)) {
12209		it.zig_index += 1;
12210		it.llvm_index += 1;
12211		return .byval;
12212		} else {
12213		var types_buffer: [8]Builder.Type = undefined;
12214		types_buffer[0] = try it.object.lowerType(pt, scalar_ty);
12215		it.types_buffer = types_buffer;
12216		it.types_len = 1;
12217		it.llvm_index += 1;
12218		it.zig_index += 1;
12219		return .multiple_llvm_types;
12220		}
12221		},
12222		.indirect => {
12223		it.zig_index += 1;
12224		it.llvm_index += 1;
12225		it.byval_attr = true;
12226		return .byref;
12227		},
12228		},
12229		// TODO investigate other callconvs
12230		else => {
12231		it.zig_index += 1;
12232		it.llvm_index += 1;
12233		return .byval;
12234		},
12235		}
12236		}
12237
12238		fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
12239		const zcu = it.pt.zcu;
12240		switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
12241		.integer => {
12242		if (isScalar(zcu, ty)) {
12243		it.zig_index += 1;
12244		it.llvm_index += 1;
12245		return .byval;
12246		} else {
12247		it.zig_index += 1;
12248		it.llvm_index += 1;
12249		return .abi_sized_int;
12250		}
12251		},
12252		.win_i128 => {
12253		it.zig_index += 1;
12254		it.llvm_index += 1;
12255		return .byref;
12256		},
12257		.memory => {
12258		it.zig_index += 1;
12259		it.llvm_index += 1;
12260		return .byref_mut;
12261		},
12262		.sse => {
12263		it.zig_index += 1;
12264		it.llvm_index += 1;
12265		return .byval;
12266		},
12267		else => unreachable,
	4413	pub fn toLlvmCallConvTag(cc_tag: std.builtin.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
	4414	if (target.cCallingConvention()) \|default_c\| {
	4415	if (cc_tag == default_c) {
	4416	return .ccc;
12268	4417	}
12269	4418	}
	4419	return switch (cc_tag) {
	4420	.@"inline" => unreachable,
	4421	.auto, .async => .fastcc,
	4422	.naked => .ccc,
	4423	.x86_64_sysv => .x86_64_sysvcc,
	4424	.x86_64_win => .win64cc,
	4425	.x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
	4426	.x86_regcallcc
	4427	else
	4428	null,
	4429	.x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
	4430	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
	4431	else
	4432	null,
	4433	.x86_64_vectorcall => .x86_vectorcallcc,
	4434	.x86_64_interrupt => .x86_intrcc,
	4435	.x86_stdcall => .x86_stdcallcc,
	4436	.x86_fastcall => .x86_fastcallcc,
	4437	.x86_thiscall => .x86_thiscallcc,
	4438	.x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
	4439	.x86_regcallcc
	4440	else
	4441	null,
	4442	.x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
	4443	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
	4444	else
	4445	null,
	4446	.x86_vectorcall => .x86_vectorcallcc,
	4447	.x86_interrupt => .x86_intrcc,
	4448	.aarch64_vfabi => .aarch64_vector_pcs,
	4449	.aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
	4450	.arm_aapcs => .arm_aapcscc,
	4451	.arm_aapcs_vfp => .arm_aapcs_vfpcc,
	4452	.riscv64_lp64_v => .riscv_vectorcallcc,
	4453	.riscv32_ilp32_v => .riscv_vectorcallcc,
	4454	.avr_builtin => .avr_builtincc,
	4455	.avr_signal => .avr_signalcc,
	4456	.avr_interrupt => .avr_intrcc,
	4457	.m68k_rtd => .m68k_rtdcc,
	4458	.m68k_interrupt => .m68k_intrcc,
	4459	.msp430_interrupt => .msp430_intrcc,
	4460	.amdgcn_kernel => .amdgpu_kernel,
	4461	.amdgcn_cs => .amdgpu_cs,
	4462	.nvptx_device => .ptx_device,
	4463	.nvptx_kernel => .ptx_kernel,
12270	4464
12271		fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
12272		const zcu = it.pt.zcu;
12273		const ip = &zcu.intern_pool;
12274		ty.assertHasLayout(zcu);
12275		const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg);
12276		if (classes[0] == .memory) {
12277		it.zig_index += 1;
12278		it.llvm_index += 1;
12279		it.byval_attr = true;
12280		return .byref;
12281		}
12282		if (isScalar(zcu, ty)) {
12283		it.zig_index += 1;
12284		it.llvm_index += 1;
12285		return .byval;
12286		}
12287		var types_index: u32 = 0;
12288		var types_buffer: [8]Builder.Type = undefined;
12289		for (classes) \|class\| {
12290		switch (class) {
12291		.integer => {
12292		types_buffer[types_index] = .i64;
12293		types_index += 1;
12294		},
12295		.sse => {
12296		types_buffer[types_index] = .double;
12297		types_index += 1;
12298		},
12299		.sseup => {
12300		if (types_buffer[types_index - 1] == .double) {
12301		types_buffer[types_index - 1] = .fp128;
12302		} else {
12303		types_buffer[types_index] = .double;
12304		types_index += 1;
12305		}
12306		},
12307		.float => {
12308		types_buffer[types_index] = .float;
12309		types_index += 1;
12310		},
12311		.float_combine => {
12312		types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
12313		types_index += 1;
12314		},
12315		.x87 => {
12316		it.zig_index += 1;
12317		it.llvm_index += 1;
12318		it.byval_attr = true;
12319		return .byref;
12320		},
12321		.x87up => unreachable,
12322		.none => break,
12323		.memory => unreachable, // handled above
12324		.win_i128 => unreachable, // windows only
12325		.integer_per_element => {
12326		@panic("TODO");
12327		},
12328		}
12329		}
12330		const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
12331		if (first_non_integer == null or classes[first_non_integer.?] == .none) {
12332		assert(first_non_integer orelse classes.len == types_index);
12333		if (types_index == 1) {
12334		it.zig_index += 1;
12335		it.llvm_index += 1;
12336		return .abi_sized_int;
12337		}
12338		if (it.llvm_index + types_index > 6) {
12339		it.zig_index += 1;
12340		it.llvm_index += 1;
12341		it.byval_attr = true;
12342		return .byref;
12343		}
12344		switch (ip.indexToKey(ty.toIntern())) {
12345		.struct_type => {
12346		const size = ty.abiSize(zcu);
12347		assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
12348		if (size % 8 > 0) {
12349		types_buffer[types_index - 1] =
12350		try it.object.builder.intType(@intCast(size % 8 * 8));
12351		}
12352		},
12353		else => {},
12354		}
12355		}
12356		it.types_len = types_index;
12357		it.types_buffer = types_buffer;
12358		it.llvm_index += types_index;
12359		it.zig_index += 1;
12360		return .multiple_llvm_types;
12361		}
12362		};
	4465	// Calling conventions which LLVM uses function attributes for.
	4466	.riscv64_interrupt,
	4467	.riscv32_interrupt,
	4468	.arm_interrupt,
	4469	.mips64_interrupt,
	4470	.mips_interrupt,
	4471	.csky_interrupt,
	4472	=> .ccc,
12363	4473
12364		fn iterateParamTypes(object: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) ParamTypeIterator {
12365		return .{
12366		.object = object,
12367		.pt = pt,
12368		.fn_info = fn_info,
12369		.zig_index = 0,
12370		.llvm_index = 0,
12371		.types_len = 0,
12372		.types_buffer = undefined,
12373		.byval_attr = false,
	4474	// All the calling conventions which LLVM does not have a general representation for.
	4475	// Note that these are often still supported through the `cCallingConvention` path above via `ccc`.
	4476	.x86_16_cdecl,
	4477	.x86_16_stdcall,
	4478	.x86_16_regparmcall,
	4479	.x86_16_interrupt,
	4480	.x86_sysv,
	4481	.x86_win,
	4482	.x86_thiscall_mingw,
	4483	.x86_64_x32,
	4484	.aarch64_aapcs,
	4485	.aarch64_aapcs_darwin,
	4486	.aarch64_aapcs_win,
	4487	.alpha_osf,
	4488	.microblaze_std,
	4489	.microblaze_interrupt,
	4490	.mips64_n64,
	4491	.mips64_n32,
	4492	.mips_o32,
	4493	.riscv64_lp64,
	4494	.riscv32_ilp32,
	4495	.sparc64_sysv,
	4496	.sparc_sysv,
	4497	.powerpc64_elf,
	4498	.powerpc64_elf_altivec,
	4499	.powerpc64_elf_v2,
	4500	.powerpc_sysv,
	4501	.powerpc_sysv_altivec,
	4502	.powerpc_aix,
	4503	.powerpc_aix_altivec,
	4504	.wasm_mvp,
	4505	.arc_sysv,
	4506	.arc_interrupt,
	4507	.avr_gnu,
	4508	.bpf_std,
	4509	.csky_sysv,
	4510	.hexagon_sysv,
	4511	.hexagon_sysv_hvx,
	4512	.hppa_elf,
	4513	.hppa64_elf,
	4514	.kvx_lp64,
	4515	.kvx_ilp32,
	4516	.lanai_sysv,
	4517	.loongarch64_lp64,
	4518	.loongarch32_ilp32,
	4519	.m68k_sysv,
	4520	.m68k_gnu,
	4521	.msp430_eabi,
	4522	.or1k_sysv,
	4523	.propeller_sysv,
	4524	.s390x_sysv,
	4525	.s390x_sysv_vx,
	4526	.sh_gnu,
	4527	.sh_renesas,
	4528	.sh_interrupt,
	4529	.ve_sysv,
	4530	.xcore_xs1,
	4531	.xcore_xs2,
	4532	.xtensa_call0,
	4533	.xtensa_windowed,
	4534	.amdgcn_device,
	4535	.spirv_device,
	4536	.spirv_kernel,
	4537	.spirv_fragment,
	4538	.spirv_vertex,
	4539	=> null,
12374	4540	};
12375	4541	}
12376	4542
12377		/// This function deliberately does not handle `_BitInt` because it typically
12378		/// has different ABI than regular integer types, and there is no currently no
12379		/// way to determine whether a Zig integer type is meant to represent e.g. `int`
12380		/// or `_BitInt(32)`.
12381		fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness {
12382		switch (cc) {
12383		.auto, .@"inline", .async => return null,
12384		else => {},
12385		}
12386
12387		const int_info = switch (ty.zigTypeTag(zcu)) {
12388		.bool => Type.u1.intInfo(zcu),
12389		else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null,
12390		};
12391		assert(int_info.bits >= 0);
	4543	/// Convert a zig-address space to an llvm address space.
	4544	pub fn toLlvmAddressSpace(address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
	4545	for (llvmAddrSpaceInfo(target)) \|info\| if (info.zig == address_space) return info.llvm;
	4546	unreachable;
	4547	}
12392	4548
12393		const target = zcu.getTarget();
	4549	const AddrSpaceInfo = struct {
	4550	zig: ?std.builtin.AddressSpace,
	4551	llvm: Builder.AddrSpace,
	4552	non_integral: bool = false,
	4553	size: ?u16 = null,
	4554	abi: ?u16 = null,
	4555	pref: ?u16 = null,
	4556	idx: ?u16 = null,
	4557	force_in_data_layout: bool = false,
	4558	};
	4559	fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {
12394	4560	return switch (target.cpu.arch) {
12395		.aarch64,
12396		.aarch64_be,
12397		=> switch (target.os.tag) {
12398		.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) {
12399		8, 16 => int_info.signedness,
12400		else => null,
12401		},
12402		else => null,
12403		},
12404
12405		.avr,
12406		=> switch (int_info.bits) {
12407		8 => int_info.signedness,
12408		else => null,
12409		},
12410
12411		.lanai,
12412		=> null,
12413
12414		.loongarch64,
12415		.riscv64,
12416		.riscv64be,
12417		=> switch (int_info.bits) {
12418		8, 16 => int_info.signedness,
12419		32 => .signed,
12420		else => null,
12421		},
12422
12423		.mips,
12424		.mipsel,
12425		.mips64,
12426		.mips64el,
12427		=> switch (int_info.bits) {
12428		8, 16, 64 => int_info.signedness,
12429		// https://github.com/llvm/llvm-project/issues/179088
12430		// 32 => .signed,
12431		else => null,
	4561	.x86, .x86_64 => &.{
	4562	.{ .zig = .generic, .llvm = .default },
	4563	.{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs },
	4564	.{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs },
	4565	.{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss },
	4566	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
	4567	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
	4568	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
12432	4569	},
12433
12434		.powerpc64,
12435		.powerpc64le,
12436		.s390x,
12437		.sparc64,
12438		.ve,
12439		=> switch (int_info.bits) {
12440		8, 16, 32 => int_info.signedness,
12441		else => null,
	4570	.nvptx, .nvptx64 => &.{
	4571	.{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic },
	4572	.{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global },
	4573	.{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant },
	4574	.{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param },
	4575	.{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared },
	4576	.{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local },
12442	4577	},
12443
12444		else => switch (int_info.bits) {
12445		8, 16 => int_info.signedness,
12446		else => null,
	4578	.amdgcn => &.{
	4579	.{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true },
	4580	.{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true },
	4581	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 },
	4582	.{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 },
	4583	.{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true },
	4584	.{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 },
	4585	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 },
	4586	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 },
	4587	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 },
	4588	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 },
	4589	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 },
	4590	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 },
	4591	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 },
	4592	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 },
	4593	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 },
	4594	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 },
	4595	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 },
	4596	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 },
	4597	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 },
	4598	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 },
	4599	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 },
	4600	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 },
	4601	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 },
	4602	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 },
	4603	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 },
	4604	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 },
	4605	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register },
12447	4606	},
12448		};
12449		}
12450
12451		/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
12452		/// or as an LLVM value.
12453		fn isByRef(ty: Type, zcu: *Zcu) bool {
12454		// For tuples and structs, if there are more than this many non-void
12455		// fields, then we make it byref, otherwise byval.
12456		const max_fields_byval = 0;
12457		const ip = &zcu.intern_pool;
12458
12459		switch (ty.zigTypeTag(zcu)) {
12460		.type,
12461		.comptime_int,
12462		.comptime_float,
12463		.enum_literal,
12464		.undefined,
12465		.null,
12466		.@"opaque",
12467		=> unreachable,
12468
12469		.noreturn,
12470		.void,
12471		.bool,
12472		.int,
12473		.float,
12474		.pointer,
12475		.error_set,
12476		.@"fn",
12477		.@"enum",
12478		.vector,
12479		.@"anyframe",
12480		=> return false,
12481
12482		.array, .frame => return ty.hasRuntimeBits(zcu),
12483		.@"struct" => {
12484		const struct_type = switch (ip.indexToKey(ty.toIntern())) {
12485		.tuple_type => \|tuple\| {
12486		var count: usize = 0;
12487		for (tuple.types.get(ip), tuple.values.get(ip)) \|field_ty, field_val\| {
12488		if (field_val != .none or !Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
12489
12490		count += 1;
12491		if (count > max_fields_byval) return true;
12492		if (isByRef(Type.fromInterned(field_ty), zcu)) return true;
12493		}
12494		return false;
12495		},
12496		.struct_type => ip.loadStructType(ty.toIntern()),
12497		else => unreachable,
12498		};
12499
12500		// Packed structs are represented to LLVM as integers.
12501		if (struct_type.layout == .@"packed") return false;
12502
12503		const field_types = struct_type.field_types.get(ip);
12504		var it = struct_type.iterateRuntimeOrder(ip);
12505		var count: usize = 0;
12506		while (it.next()) \|field_index\| {
12507		count += 1;
12508		if (count > max_fields_byval) return true;
12509		const field_ty = Type.fromInterned(field_types[field_index]);
12510		if (isByRef(field_ty, zcu)) return true;
12511		}
12512		return false;
	4607	.avr => &.{
	4608	.{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 },
	4609	.{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 },
	4610	.{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 },
	4611	.{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 },
	4612	.{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 },
	4613	.{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 },
	4614	.{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 },
12513	4615	},
12514		.@"union" => switch (ty.containerLayout(zcu)) {
12515		.@"packed" => return false,
12516		else => return ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
	4616	.wasm32, .wasm64 => &.{
	4617	.{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true },
	4618	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true },
	4619	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
	4620	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
12517	4621	},
12518		.error_union => {
12519		const payload_ty = ty.errorUnionPayload(zcu);
12520		if (!payload_ty.hasRuntimeBits(zcu)) {
12521		return false;
12522		}
12523		return true;
	4622	.m68k => &.{
	4623	.{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 },
12524	4624	},
12525		.optional => {
12526		const payload_ty = ty.optionalChild(zcu);
12527		if (!payload_ty.hasRuntimeBits(zcu)) {
12528		return false;
12529		}
12530		if (ty.optionalReprIsPayload(zcu)) {
12531		return false;
12532		}
12533		return true;
	4625	else => &.{
	4626	.{ .zig = .generic, .llvm = .default },
12534	4627	},
12535		}
	4628	};
12536	4629	}
12537	4630
12538		fn isScalar(zcu: *Zcu, ty: Type) bool {
12539		return switch (ty.zigTypeTag(zcu)) {
12540		.void,
12541		.bool,
12542		.noreturn,
12543		.int,
12544		.float,
12545		.pointer,
12546		.optional,
12547		.error_set,
12548		.@"enum",
12549		.@"anyframe",
12550		.vector,
12551		=> true,
12552
12553		.@"struct" => ty.containerLayout(zcu) == .@"packed",
12554		.@"union" => ty.containerLayout(zcu) == .@"packed",
12555		else => false,
	4631	/// On some targets, global values that are in the generic address space must be generated into a
	4632	/// different address space, and then cast back to the generic address space.
	4633	fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {
	4634	return switch (target.cpu.arch) {
	4635	// On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
	4636	// them.
	4637	.amdgcn => Builder.AddrSpace.amdgpu.global,
	4638	else => .default,
12556	4639	};
12557	4640	}
12558	4641
12559		/// This function returns true if we expect LLVM to lower x86_fp80 correctly
12560		/// and false if we expect LLVM to crash if it encounters an x86_fp80 type,
12561		/// or if it produces miscompilations.
12562		fn backendSupportsF80(target: *const std.Target) bool {
12563		return switch (target.cpu.arch) {
12564		.x86, .x86_64 => !target.cpu.has(.x86, .soft_float),
12565		else => false,
	4642	/// Return the actual address space that a value should be stored in if its a global address space.
	4643	/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
	4644	fn toLlvmGlobalAddressSpace(wanted_address_space: std.builtin.AddressSpace, target: *const std.Target) Builder.AddrSpace {
	4645	return switch (wanted_address_space) {
	4646	.generic => llvmDefaultGlobalAddressSpace(target),
	4647	else => \|as\| toLlvmAddressSpace(as, target),
12566	4648	};
12567	4649	}
12568	4650
12569	4651	/// This function returns true if we expect LLVM to lower f16 correctly
12570	4652	/// and false if we expect LLVM to crash if it encounters an f16 type,
12571	4653	/// or if it produces miscompilations.
12572		fn backendSupportsF16(target: *const std.Target) bool {
	4654	pub fn backendSupportsF16(target: *const std.Target) bool {
12573	4655	return switch (target.cpu.arch) {
12574	4656	// https://github.com/llvm/llvm-project/issues/97981
12575	4657	.csky,
...	...	@@ -12594,10 +4676,20 @@ fn backendSupportsF16(target: *const std.Target) bool {
12594	4676	};
12595	4677	}
12596	4678
	4679	/// This function returns true if we expect LLVM to lower x86_fp80 correctly
	4680	/// and false if we expect LLVM to crash if it encounters an x86_fp80 type,
	4681	/// or if it produces miscompilations.
	4682	pub fn backendSupportsF80(target: *const std.Target) bool {
	4683	return switch (target.cpu.arch) {
	4684	.x86, .x86_64 => !target.cpu.has(.x86, .soft_float),
	4685	else => false,
	4686	};
	4687	}
	4688
12597	4689	/// This function returns true if we expect LLVM to lower f128 correctly,
12598	4690	/// and false if we expect LLVM to crash if it encounters an f128 type,
12599	4691	/// or if it produces miscompilations.
12600		fn backendSupportsF128(target: *const std.Target) bool {
	4692	pub fn backendSupportsF128(target: *const std.Target) bool {
12601	4693	return switch (target.cpu.arch) {
12602	4694	// https://github.com/llvm/llvm-project/issues/121122
12603	4695	.amdgcn,
...	...	@@ -12616,17 +4708,6 @@ fn backendSupportsF128(target: *const std.Target) bool {
12616	4708	};
12617	4709	}
12618	4710
12619		/// LLVM does not support all relevant intrinsics for all targets, so we
12620		/// may need to manually generate a compiler-rt call.
12621		fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool {
12622		return switch (scalar_ty.toIntern()) {
12623		.f16_type => backendSupportsF16(target),
12624		.f80_type => (target.cTypeBitSize(.longdouble) == 80) and backendSupportsF80(target),
12625		.f128_type => (target.cTypeBitSize(.longdouble) == 128) and backendSupportsF128(target),
12626		else => true,
12627		};
12628		}
12629
12630	4711	/// We need to insert extra padding if LLVM's isn't enough.
12631	4712	/// However we don't want to ever call LLVMABIAlignmentOfType or
12632	4713	/// LLVMABISizeOfType because these functions will trip assertions
...	...	@@ -12638,264 +4719,186 @@ const struct_layout_version = 2;
12638	4719
12639	4720	// TODO: Restore the non_null field to i1 once
12640	4721	// https://github.com/llvm/llvm-project/issues/56585/ is fixed
12641		const optional_layout_version = 3;
12642
12643		const lt_errors_fn_name = "__zig_lt_errors_len";
12644
12645		fn compilerRtIntBits(bits: u16) ?u16 {
12646		inline for (.{ 32, 64, 128 }) \|b\| {
12647		if (bits <= b) {
12648		return b;
12649		}
12650		}
12651		return null;
12652		}
12653
12654		fn buildAllocaInner(
12655		wip: *Builder.WipFunction,
12656		llvm_ty: Builder.Type,
12657		alignment: Builder.Alignment,
12658		target: *const std.Target,
12659		) Allocator.Error!Builder.Value {
12660		const address_space = llvmAllocaAddressSpace(target);
12661
12662		const alloca = blk: {
12663		const prev_cursor = wip.cursor;
12664		const prev_debug_location = wip.debug_location;
12665		defer {
12666		wip.cursor = prev_cursor;
12667		if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
12668		wip.debug_location = prev_debug_location;
12669		}
12670
12671		wip.cursor = .{ .block = .entry };
12672		wip.debug_location = .no_location;
12673		break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
12674		};
12675
12676		// The pointer returned from this function should have the generic address space,
12677		// if this isn't the case then cast it to the generic address space.
12678		return wip.conv(.unneeded, alloca, .ptr, "");
12679		}
12680
12681		fn errUnionPayloadOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12682		const zcu = pt.zcu;
12683		const err_int_ty = try pt.errorIntType();
12684		return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.gt, payload_ty.abiAlignment(zcu)));
12685		}
12686
12687		fn errUnionErrorOffset(payload_ty: Type, pt: Zcu.PerThread) !u1 {
12688		const zcu = pt.zcu;
12689		const err_int_ty = try pt.errorIntType();
12690		return @intFromBool(err_int_ty.abiAlignment(zcu).compare(.lte, payload_ty.abiAlignment(zcu)));
12691		}
12692
12693		/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
12694		///
12695		/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12696		fn constraintAllowsMemory(constraint: []const u8) bool {
12697		// TODO: This implementation is woefully incomplete.
12698		for (constraint) \|byte\| {
12699		switch (byte) {
12700		'=', '*', ',', '&' => {},
12701		'm', 'o', 'X', 'g' => return true,
12702		else => {},
12703		}
12704		} else return false;
12705		}
12706
12707		/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
12708		///
12709		/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
12710		fn constraintAllowsRegister(constraint: []const u8) bool {
12711		// TODO: This implementation is woefully incomplete.
12712		for (constraint) \|byte\| {
12713		switch (byte) {
12714		'=', '*', ',', '&' => {},
12715		'm', 'o' => {},
12716		else => return true,
12717		}
12718		} else return false;
12719		}
	4722	pub const optional_layout_version = 3;
12720	4723
12721	4724	pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
12722	4725	switch (arch) {
12723	4726	.aarch64, .aarch64_be => {
12724		llvm.LLVMInitializeAArch64Target();
12725		llvm.LLVMInitializeAArch64TargetInfo();
12726		llvm.LLVMInitializeAArch64TargetMC();
12727		llvm.LLVMInitializeAArch64AsmPrinter();
12728		llvm.LLVMInitializeAArch64AsmParser();
	4727	bindings.LLVMInitializeAArch64Target();
	4728	bindings.LLVMInitializeAArch64TargetInfo();
	4729	bindings.LLVMInitializeAArch64TargetMC();
	4730	bindings.LLVMInitializeAArch64AsmPrinter();
	4731	bindings.LLVMInitializeAArch64AsmParser();
12729	4732	},
12730	4733	.amdgcn => {
12731		llvm.LLVMInitializeAMDGPUTarget();
12732		llvm.LLVMInitializeAMDGPUTargetInfo();
12733		llvm.LLVMInitializeAMDGPUTargetMC();
12734		llvm.LLVMInitializeAMDGPUAsmPrinter();
12735		llvm.LLVMInitializeAMDGPUAsmParser();
	4734	bindings.LLVMInitializeAMDGPUTarget();
	4735	bindings.LLVMInitializeAMDGPUTargetInfo();
	4736	bindings.LLVMInitializeAMDGPUTargetMC();
	4737	bindings.LLVMInitializeAMDGPUAsmPrinter();
	4738	bindings.LLVMInitializeAMDGPUAsmParser();
12736	4739	},
12737	4740	.thumb, .thumbeb, .arm, .armeb => {
12738		llvm.LLVMInitializeARMTarget();
12739		llvm.LLVMInitializeARMTargetInfo();
12740		llvm.LLVMInitializeARMTargetMC();
12741		llvm.LLVMInitializeARMAsmPrinter();
12742		llvm.LLVMInitializeARMAsmParser();
	4741	bindings.LLVMInitializeARMTarget();
	4742	bindings.LLVMInitializeARMTargetInfo();
	4743	bindings.LLVMInitializeARMTargetMC();
	4744	bindings.LLVMInitializeARMAsmPrinter();
	4745	bindings.LLVMInitializeARMAsmParser();
12743	4746	},
12744	4747	.avr => {
12745		llvm.LLVMInitializeAVRTarget();
12746		llvm.LLVMInitializeAVRTargetInfo();
12747		llvm.LLVMInitializeAVRTargetMC();
12748		llvm.LLVMInitializeAVRAsmPrinter();
12749		llvm.LLVMInitializeAVRAsmParser();
	4748	bindings.LLVMInitializeAVRTarget();
	4749	bindings.LLVMInitializeAVRTargetInfo();
	4750	bindings.LLVMInitializeAVRTargetMC();
	4751	bindings.LLVMInitializeAVRAsmPrinter();
	4752	bindings.LLVMInitializeAVRAsmParser();
12750	4753	},
12751	4754	.bpfel, .bpfeb => {
12752		llvm.LLVMInitializeBPFTarget();
12753		llvm.LLVMInitializeBPFTargetInfo();
12754		llvm.LLVMInitializeBPFTargetMC();
12755		llvm.LLVMInitializeBPFAsmPrinter();
12756		llvm.LLVMInitializeBPFAsmParser();
	4755	bindings.LLVMInitializeBPFTarget();
	4756	bindings.LLVMInitializeBPFTargetInfo();
	4757	bindings.LLVMInitializeBPFTargetMC();
	4758	bindings.LLVMInitializeBPFAsmPrinter();
	4759	bindings.LLVMInitializeBPFAsmParser();
12757	4760	},
12758	4761	.hexagon => {
12759		llvm.LLVMInitializeHexagonTarget();
12760		llvm.LLVMInitializeHexagonTargetInfo();
12761		llvm.LLVMInitializeHexagonTargetMC();
12762		llvm.LLVMInitializeHexagonAsmPrinter();
12763		llvm.LLVMInitializeHexagonAsmParser();
	4762	bindings.LLVMInitializeHexagonTarget();
	4763	bindings.LLVMInitializeHexagonTargetInfo();
	4764	bindings.LLVMInitializeHexagonTargetMC();
	4765	bindings.LLVMInitializeHexagonAsmPrinter();
	4766	bindings.LLVMInitializeHexagonAsmParser();
12764	4767	},
12765	4768	.lanai => {
12766		llvm.LLVMInitializeLanaiTarget();
12767		llvm.LLVMInitializeLanaiTargetInfo();
12768		llvm.LLVMInitializeLanaiTargetMC();
12769		llvm.LLVMInitializeLanaiAsmPrinter();
12770		llvm.LLVMInitializeLanaiAsmParser();
	4769	bindings.LLVMInitializeLanaiTarget();
	4770	bindings.LLVMInitializeLanaiTargetInfo();
	4771	bindings.LLVMInitializeLanaiTargetMC();
	4772	bindings.LLVMInitializeLanaiAsmPrinter();
	4773	bindings.LLVMInitializeLanaiAsmParser();
12771	4774	},
12772	4775	.mips, .mipsel, .mips64, .mips64el => {
12773		llvm.LLVMInitializeMipsTarget();
12774		llvm.LLVMInitializeMipsTargetInfo();
12775		llvm.LLVMInitializeMipsTargetMC();
12776		llvm.LLVMInitializeMipsAsmPrinter();
12777		llvm.LLVMInitializeMipsAsmParser();
	4776	bindings.LLVMInitializeMipsTarget();
	4777	bindings.LLVMInitializeMipsTargetInfo();
	4778	bindings.LLVMInitializeMipsTargetMC();
	4779	bindings.LLVMInitializeMipsAsmPrinter();
	4780	bindings.LLVMInitializeMipsAsmParser();
12778	4781	},
12779	4782	.msp430 => {
12780		llvm.LLVMInitializeMSP430Target();
12781		llvm.LLVMInitializeMSP430TargetInfo();
12782		llvm.LLVMInitializeMSP430TargetMC();
12783		llvm.LLVMInitializeMSP430AsmPrinter();
12784		llvm.LLVMInitializeMSP430AsmParser();
	4783	bindings.LLVMInitializeMSP430Target();
	4784	bindings.LLVMInitializeMSP430TargetInfo();
	4785	bindings.LLVMInitializeMSP430TargetMC();
	4786	bindings.LLVMInitializeMSP430AsmPrinter();
	4787	bindings.LLVMInitializeMSP430AsmParser();
12785	4788	},
12786	4789	.nvptx, .nvptx64 => {
12787		llvm.LLVMInitializeNVPTXTarget();
12788		llvm.LLVMInitializeNVPTXTargetInfo();
12789		llvm.LLVMInitializeNVPTXTargetMC();
12790		llvm.LLVMInitializeNVPTXAsmPrinter();
	4790	bindings.LLVMInitializeNVPTXTarget();
	4791	bindings.LLVMInitializeNVPTXTargetInfo();
	4792	bindings.LLVMInitializeNVPTXTargetMC();
	4793	bindings.LLVMInitializeNVPTXAsmPrinter();
12791	4794	// There is no LLVMInitializeNVPTXAsmParser function available.
12792	4795	},
12793	4796	.powerpc, .powerpcle, .powerpc64, .powerpc64le => {
12794		llvm.LLVMInitializePowerPCTarget();
12795		llvm.LLVMInitializePowerPCTargetInfo();
12796		llvm.LLVMInitializePowerPCTargetMC();
12797		llvm.LLVMInitializePowerPCAsmPrinter();
12798		llvm.LLVMInitializePowerPCAsmParser();
	4797	bindings.LLVMInitializePowerPCTarget();
	4798	bindings.LLVMInitializePowerPCTargetInfo();
	4799	bindings.LLVMInitializePowerPCTargetMC();
	4800	bindings.LLVMInitializePowerPCAsmPrinter();
	4801	bindings.LLVMInitializePowerPCAsmParser();
12799	4802	},
12800	4803	.riscv32, .riscv32be, .riscv64, .riscv64be => {
12801		llvm.LLVMInitializeRISCVTarget();
12802		llvm.LLVMInitializeRISCVTargetInfo();
12803		llvm.LLVMInitializeRISCVTargetMC();
12804		llvm.LLVMInitializeRISCVAsmPrinter();
12805		llvm.LLVMInitializeRISCVAsmParser();
	4804	bindings.LLVMInitializeRISCVTarget();
	4805	bindings.LLVMInitializeRISCVTargetInfo();
	4806	bindings.LLVMInitializeRISCVTargetMC();
	4807	bindings.LLVMInitializeRISCVAsmPrinter();
	4808	bindings.LLVMInitializeRISCVAsmParser();
12806	4809	},
12807	4810	.sparc, .sparc64 => {
12808		llvm.LLVMInitializeSparcTarget();
12809		llvm.LLVMInitializeSparcTargetInfo();
12810		llvm.LLVMInitializeSparcTargetMC();
12811		llvm.LLVMInitializeSparcAsmPrinter();
12812		llvm.LLVMInitializeSparcAsmParser();
	4811	bindings.LLVMInitializeSparcTarget();
	4812	bindings.LLVMInitializeSparcTargetInfo();
	4813	bindings.LLVMInitializeSparcTargetMC();
	4814	bindings.LLVMInitializeSparcAsmPrinter();
	4815	bindings.LLVMInitializeSparcAsmParser();
12813	4816	},
12814	4817	.s390x => {
12815		llvm.LLVMInitializeSystemZTarget();
12816		llvm.LLVMInitializeSystemZTargetInfo();
12817		llvm.LLVMInitializeSystemZTargetMC();
12818		llvm.LLVMInitializeSystemZAsmPrinter();
12819		llvm.LLVMInitializeSystemZAsmParser();
	4818	bindings.LLVMInitializeSystemZTarget();
	4819	bindings.LLVMInitializeSystemZTargetInfo();
	4820	bindings.LLVMInitializeSystemZTargetMC();
	4821	bindings.LLVMInitializeSystemZAsmPrinter();
	4822	bindings.LLVMInitializeSystemZAsmParser();
12820	4823	},
12821	4824	.wasm32, .wasm64 => {
12822		llvm.LLVMInitializeWebAssemblyTarget();
12823		llvm.LLVMInitializeWebAssemblyTargetInfo();
12824		llvm.LLVMInitializeWebAssemblyTargetMC();
12825		llvm.LLVMInitializeWebAssemblyAsmPrinter();
12826		llvm.LLVMInitializeWebAssemblyAsmParser();
	4825	bindings.LLVMInitializeWebAssemblyTarget();
	4826	bindings.LLVMInitializeWebAssemblyTargetInfo();
	4827	bindings.LLVMInitializeWebAssemblyTargetMC();
	4828	bindings.LLVMInitializeWebAssemblyAsmPrinter();
	4829	bindings.LLVMInitializeWebAssemblyAsmParser();
12827	4830	},
12828	4831	.x86, .x86_64 => {
12829		llvm.LLVMInitializeX86Target();
12830		llvm.LLVMInitializeX86TargetInfo();
12831		llvm.LLVMInitializeX86TargetMC();
12832		llvm.LLVMInitializeX86AsmPrinter();
12833		llvm.LLVMInitializeX86AsmParser();
	4832	bindings.LLVMInitializeX86Target();
	4833	bindings.LLVMInitializeX86TargetInfo();
	4834	bindings.LLVMInitializeX86TargetMC();
	4835	bindings.LLVMInitializeX86AsmPrinter();
	4836	bindings.LLVMInitializeX86AsmParser();
12834	4837	},
12835	4838	.xtensa => {
12836	4839	if (build_options.llvm_has_xtensa) {
12837		llvm.LLVMInitializeXtensaTarget();
12838		llvm.LLVMInitializeXtensaTargetInfo();
12839		llvm.LLVMInitializeXtensaTargetMC();
	4840	bindings.LLVMInitializeXtensaTarget();
	4841	bindings.LLVMInitializeXtensaTargetInfo();
	4842	bindings.LLVMInitializeXtensaTargetMC();
12840	4843	// There is no LLVMInitializeXtensaAsmPrinter function.
12841		llvm.LLVMInitializeXtensaAsmParser();
	4844	bindings.LLVMInitializeXtensaAsmParser();
12842	4845	}
12843	4846	},
12844	4847	.xcore => {
12845		llvm.LLVMInitializeXCoreTarget();
12846		llvm.LLVMInitializeXCoreTargetInfo();
12847		llvm.LLVMInitializeXCoreTargetMC();
12848		llvm.LLVMInitializeXCoreAsmPrinter();
	4848	bindings.LLVMInitializeXCoreTarget();
	4849	bindings.LLVMInitializeXCoreTargetInfo();
	4850	bindings.LLVMInitializeXCoreTargetMC();
	4851	bindings.LLVMInitializeXCoreAsmPrinter();
12849	4852	// There is no LLVMInitializeXCoreAsmParser function.
12850	4853	},
12851	4854	.m68k => {
12852	4855	if (build_options.llvm_has_m68k) {
12853		llvm.LLVMInitializeM68kTarget();
12854		llvm.LLVMInitializeM68kTargetInfo();
12855		llvm.LLVMInitializeM68kTargetMC();
12856		llvm.LLVMInitializeM68kAsmPrinter();
12857		llvm.LLVMInitializeM68kAsmParser();
	4856	bindings.LLVMInitializeM68kTarget();
	4857	bindings.LLVMInitializeM68kTargetInfo();
	4858	bindings.LLVMInitializeM68kTargetMC();
	4859	bindings.LLVMInitializeM68kAsmPrinter();
	4860	bindings.LLVMInitializeM68kAsmParser();
12858	4861	}
12859	4862	},
12860	4863	.csky => {
12861	4864	if (build_options.llvm_has_csky) {
12862		llvm.LLVMInitializeCSKYTarget();
12863		llvm.LLVMInitializeCSKYTargetInfo();
12864		llvm.LLVMInitializeCSKYTargetMC();
	4865	bindings.LLVMInitializeCSKYTarget();
	4866	bindings.LLVMInitializeCSKYTargetInfo();
	4867	bindings.LLVMInitializeCSKYTargetMC();
12865	4868	// There is no LLVMInitializeCSKYAsmPrinter function.
12866		llvm.LLVMInitializeCSKYAsmParser();
	4869	bindings.LLVMInitializeCSKYAsmParser();
12867	4870	}
12868	4871	},
12869	4872	.ve => {
12870		llvm.LLVMInitializeVETarget();
12871		llvm.LLVMInitializeVETargetInfo();
12872		llvm.LLVMInitializeVETargetMC();
12873		llvm.LLVMInitializeVEAsmPrinter();
12874		llvm.LLVMInitializeVEAsmParser();
	4873	bindings.LLVMInitializeVETarget();
	4874	bindings.LLVMInitializeVETargetInfo();
	4875	bindings.LLVMInitializeVETargetMC();
	4876	bindings.LLVMInitializeVEAsmPrinter();
	4877	bindings.LLVMInitializeVEAsmParser();
12875	4878	},
12876	4879	.arc => {
12877	4880	if (build_options.llvm_has_arc) {
12878		llvm.LLVMInitializeARCTarget();
12879		llvm.LLVMInitializeARCTargetInfo();
12880		llvm.LLVMInitializeARCTargetMC();
12881		llvm.LLVMInitializeARCAsmPrinter();
	4881	bindings.LLVMInitializeARCTarget();
	4882	bindings.LLVMInitializeARCTargetInfo();
	4883	bindings.LLVMInitializeARCTargetMC();
	4884	bindings.LLVMInitializeARCAsmPrinter();
12882	4885	// There is no LLVMInitializeARCAsmParser function.
12883	4886	}
12884	4887	},
12885	4888	.loongarch32, .loongarch64 => {
12886		llvm.LLVMInitializeLoongArchTarget();
12887		llvm.LLVMInitializeLoongArchTargetInfo();
12888		llvm.LLVMInitializeLoongArchTargetMC();
12889		llvm.LLVMInitializeLoongArchAsmPrinter();
12890		llvm.LLVMInitializeLoongArchAsmParser();
	4889	bindings.LLVMInitializeLoongArchTarget();
	4890	bindings.LLVMInitializeLoongArchTargetInfo();
	4891	bindings.LLVMInitializeLoongArchTargetMC();
	4892	bindings.LLVMInitializeLoongArchAsmPrinter();
	4893	bindings.LLVMInitializeLoongArchAsmParser();
12891	4894	},
12892	4895	.spirv32,
12893	4896	.spirv64,
12894	4897	=> {
12895		llvm.LLVMInitializeSPIRVTarget();
12896		llvm.LLVMInitializeSPIRVTargetInfo();
12897		llvm.LLVMInitializeSPIRVTargetMC();
12898		llvm.LLVMInitializeSPIRVAsmPrinter();
	4898	bindings.LLVMInitializeSPIRVTarget();
	4899	bindings.LLVMInitializeSPIRVTargetInfo();
	4900	bindings.LLVMInitializeSPIRVTargetMC();
	4901	bindings.LLVMInitializeSPIRVAsmPrinter();
12899	4902	},
12900	4903
12901	4904	// LLVM does does not have a backend for these.
...	...	@@ -12916,296 +4919,3 @@ pub fn initializeLLVMTarget(arch: std.Target.Cpu.Arch) void {
12916	4919	=> unreachable,
12917	4920	}
12918	4921	}
12919
12920		fn minIntConst(b: Builder, min_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
12921		const info = min_ty.intInfo(zcu);
12922		if (info.signedness == .unsigned or info.bits == 0) {
12923		return b.intConst(as_ty, 0);
12924		}
12925		if (std.math.cast(u6, info.bits - 1)) \|shift\| {
12926		const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
12927		return b.intConst(as_ty, min_val);
12928		}
12929		var res: std.math.big.int.Managed = try .init(zcu.gpa);
12930		defer res.deinit();
12931		try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
12932		return b.bigIntConst(as_ty, res.toConst());
12933		}
12934
12935		fn maxIntConst(b: Builder, max_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
12936		const info = max_ty.intInfo(zcu);
12937		switch (info.bits) {
12938		0 => return b.intConst(as_ty, 0),
12939		1 => switch (info.signedness) {
12940		.signed => return b.intConst(as_ty, 0),
12941		.unsigned => return b.intConst(as_ty, 1),
12942		},
12943		else => {},
12944		}
12945		const unsigned_bits = switch (info.signedness) {
12946		.unsigned => info.bits,
12947		.signed => info.bits - 1,
12948		};
12949		if (std.math.cast(u6, unsigned_bits)) \|shift\| {
12950		const max_val: u64 = (@as(u64, 1) << shift) - 1;
12951		return b.intConst(as_ty, max_val);
12952		}
12953		var res: std.math.big.int.Managed = try .init(zcu.gpa);
12954		defer res.deinit();
12955		try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
12956		return b.bigIntConst(as_ty, res.toConst());
12957		}
12958
12959		/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
12960		fn appendConstraints(
12961		gpa: Allocator,
12962		llvm_constraints: *std.ArrayList(u8),
12963		zig_name: []const u8,
12964		target: *const std.Target,
12965		) error{OutOfMemory}!usize {
12966		switch (target.cpu.arch) {
12967		.mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) \|llvm_tag\| {
12968		const llvm_name = @tagName(llvm_tag);
12969		try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
12970		llvm_constraints.appendSliceAssumeCapacity("~{");
12971		llvm_constraints.appendSliceAssumeCapacity(llvm_name);
12972		llvm_constraints.appendSliceAssumeCapacity("},");
12973		return 1;
12974		},
12975		else => {},
12976		}
12977
12978		try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
12979		llvm_constraints.appendSliceAssumeCapacity("~{");
12980		llvm_constraints.appendSliceAssumeCapacity(zig_name);
12981		llvm_constraints.appendSliceAssumeCapacity("},");
12982		return 1;
12983		}
12984
12985		const mips_clobber_overrides = std.StaticStringMap(enum {
12986		@"$msair",
12987		@"$msacsr",
12988		@"$msaaccess",
12989		@"$msasave",
12990		@"$msamodify",
12991		@"$msarequest",
12992		@"$msamap",
12993		@"$msaunmap",
12994		@"$f0",
12995		@"$f1",
12996		@"$f2",
12997		@"$f3",
12998		@"$f4",
12999		@"$f5",
13000		@"$f6",
13001		@"$f7",
13002		@"$f8",
13003		@"$f9",
13004		@"$f10",
13005		@"$f11",
13006		@"$f12",
13007		@"$f13",
13008		@"$f14",
13009		@"$f15",
13010		@"$f16",
13011		@"$f17",
13012		@"$f18",
13013		@"$f19",
13014		@"$f20",
13015		@"$f21",
13016		@"$f22",
13017		@"$f23",
13018		@"$f24",
13019		@"$f25",
13020		@"$f26",
13021		@"$f27",
13022		@"$f28",
13023		@"$f29",
13024		@"$f30",
13025		@"$f31",
13026		@"$fcc0",
13027		@"$fcc1",
13028		@"$fcc2",
13029		@"$fcc3",
13030		@"$fcc4",
13031		@"$fcc5",
13032		@"$fcc6",
13033		@"$fcc7",
13034		@"$w0",
13035		@"$w1",
13036		@"$w2",
13037		@"$w3",
13038		@"$w4",
13039		@"$w5",
13040		@"$w6",
13041		@"$w7",
13042		@"$w8",
13043		@"$w9",
13044		@"$w10",
13045		@"$w11",
13046		@"$w12",
13047		@"$w13",
13048		@"$w14",
13049		@"$w15",
13050		@"$w16",
13051		@"$w17",
13052		@"$w18",
13053		@"$w19",
13054		@"$w20",
13055		@"$w21",
13056		@"$w22",
13057		@"$w23",
13058		@"$w24",
13059		@"$w25",
13060		@"$w26",
13061		@"$w27",
13062		@"$w28",
13063		@"$w29",
13064		@"$w30",
13065		@"$w31",
13066		@"$0",
13067		@"$1",
13068		@"$2",
13069		@"$3",
13070		@"$4",
13071		@"$5",
13072		@"$6",
13073		@"$7",
13074		@"$8",
13075		@"$9",
13076		@"$10",
13077		@"$11",
13078		@"$12",
13079		@"$13",
13080		@"$14",
13081		@"$15",
13082		@"$16",
13083		@"$17",
13084		@"$18",
13085		@"$19",
13086		@"$20",
13087		@"$21",
13088		@"$22",
13089		@"$23",
13090		@"$24",
13091		@"$25",
13092		@"$26",
13093		@"$27",
13094		@"$28",
13095		@"$29",
13096		@"$30",
13097		@"$31",
13098		}).initComptime(.{
13099		.{ "msa_ir", .@"$msair" },
13100		.{ "msa_csr", .@"$msacsr" },
13101		.{ "msa_access", .@"$msaaccess" },
13102		.{ "msa_save", .@"$msasave" },
13103		.{ "msa_modify", .@"$msamodify" },
13104		.{ "msa_request", .@"$msarequest" },
13105		.{ "msa_map", .@"$msamap" },
13106		.{ "msa_unmap", .@"$msaunmap" },
13107		.{ "f0", .@"$f0" },
13108		.{ "f1", .@"$f1" },
13109		.{ "f2", .@"$f2" },
13110		.{ "f3", .@"$f3" },
13111		.{ "f4", .@"$f4" },
13112		.{ "f5", .@"$f5" },
13113		.{ "f6", .@"$f6" },
13114		.{ "f7", .@"$f7" },
13115		.{ "f8", .@"$f8" },
13116		.{ "f9", .@"$f9" },
13117		.{ "f10", .@"$f10" },
13118		.{ "f11", .@"$f11" },
13119		.{ "f12", .@"$f12" },
13120		.{ "f13", .@"$f13" },
13121		.{ "f14", .@"$f14" },
13122		.{ "f15", .@"$f15" },
13123		.{ "f16", .@"$f16" },
13124		.{ "f17", .@"$f17" },
13125		.{ "f18", .@"$f18" },
13126		.{ "f19", .@"$f19" },
13127		.{ "f20", .@"$f20" },
13128		.{ "f21", .@"$f21" },
13129		.{ "f22", .@"$f22" },
13130		.{ "f23", .@"$f23" },
13131		.{ "f24", .@"$f24" },
13132		.{ "f25", .@"$f25" },
13133		.{ "f26", .@"$f26" },
13134		.{ "f27", .@"$f27" },
13135		.{ "f28", .@"$f28" },
13136		.{ "f29", .@"$f29" },
13137		.{ "f30", .@"$f30" },
13138		.{ "f31", .@"$f31" },
13139		.{ "fcc0", .@"$fcc0" },
13140		.{ "fcc1", .@"$fcc1" },
13141		.{ "fcc2", .@"$fcc2" },
13142		.{ "fcc3", .@"$fcc3" },
13143		.{ "fcc4", .@"$fcc4" },
13144		.{ "fcc5", .@"$fcc5" },
13145		.{ "fcc6", .@"$fcc6" },
13146		.{ "fcc7", .@"$fcc7" },
13147		.{ "w0", .@"$w0" },
13148		.{ "w1", .@"$w1" },
13149		.{ "w2", .@"$w2" },
13150		.{ "w3", .@"$w3" },
13151		.{ "w4", .@"$w4" },
13152		.{ "w5", .@"$w5" },
13153		.{ "w6", .@"$w6" },
13154		.{ "w7", .@"$w7" },
13155		.{ "w8", .@"$w8" },
13156		.{ "w9", .@"$w9" },
13157		.{ "w10", .@"$w10" },
13158		.{ "w11", .@"$w11" },
13159		.{ "w12", .@"$w12" },
13160		.{ "w13", .@"$w13" },
13161		.{ "w14", .@"$w14" },
13162		.{ "w15", .@"$w15" },
13163		.{ "w16", .@"$w16" },
13164		.{ "w17", .@"$w17" },
13165		.{ "w18", .@"$w18" },
13166		.{ "w19", .@"$w19" },
13167		.{ "w20", .@"$w20" },
13168		.{ "w21", .@"$w21" },
13169		.{ "w22", .@"$w22" },
13170		.{ "w23", .@"$w23" },
13171		.{ "w24", .@"$w24" },
13172		.{ "w25", .@"$w25" },
13173		.{ "w26", .@"$w26" },
13174		.{ "w27", .@"$w27" },
13175		.{ "w28", .@"$w28" },
13176		.{ "w29", .@"$w29" },
13177		.{ "w30", .@"$w30" },
13178		.{ "w31", .@"$w31" },
13179		.{ "r0", .@"$0" },
13180		.{ "r1", .@"$1" },
13181		.{ "r2", .@"$2" },
13182		.{ "r3", .@"$3" },
13183		.{ "r4", .@"$4" },
13184		.{ "r5", .@"$5" },
13185		.{ "r6", .@"$6" },
13186		.{ "r7", .@"$7" },
13187		.{ "r8", .@"$8" },
13188		.{ "r9", .@"$9" },
13189		.{ "r10", .@"$10" },
13190		.{ "r11", .@"$11" },
13191		.{ "r12", .@"$12" },
13192		.{ "r13", .@"$13" },
13193		.{ "r14", .@"$14" },
13194		.{ "r15", .@"$15" },
13195		.{ "r16", .@"$16" },
13196		.{ "r17", .@"$17" },
13197		.{ "r18", .@"$18" },
13198		.{ "r19", .@"$19" },
13199		.{ "r20", .@"$20" },
13200		.{ "r21", .@"$21" },
13201		.{ "r22", .@"$22" },
13202		.{ "r23", .@"$23" },
13203		.{ "r24", .@"$24" },
13204		.{ "r25", .@"$25" },
13205		.{ "r26", .@"$26" },
13206		.{ "r27", .@"$27" },
13207		.{ "r28", .@"$28" },
13208		.{ "r29", .@"$29" },
13209		.{ "r30", .@"$30" },
13210		.{ "r31", .@"$31" },
13211		});

src/codegen/llvm/FuncGen.zig created+7714

...	...	@@ -0,0 +1,7714 @@
	1	const FuncGen = @This();
	2
	3	object: *Object,
	4	nav_index: InternPool.Nav.Index,
	5	pt: Zcu.PerThread,
	6	gpa: Allocator,
	7	air: Air,
	8	liveness: Air.Liveness,
	9	wip: Builder.WipFunction,
	10	is_naked: bool,
	11	fuzz: ?Fuzz,
	12
	13	file: Builder.Metadata,
	14	scope: Builder.Metadata,
	15
	16	inlined_at: Builder.Metadata.Optional,
	17
	18	base_line: u32,
	19	prev_dbg_line: u32,
	20	prev_dbg_column: u32,
	21
	22	/// This stores the LLVM values used in a function, such that they can be referred to
	23	/// in other instructions. This table is cleared before every function is generated.
	24	func_inst_table: std.AutoHashMapUnmanaged(Air.Inst.Ref, Builder.Value),
	25
	26	/// If the return type is sret, this is the result pointer. Otherwise null.
	27	/// Note that this can disagree with isByRef for the return type in the case
	28	/// of C ABI functions.
	29	ret_ptr: Builder.Value,
	30	/// Any function that needs to perform Valgrind client requests needs an array alloca
	31	/// instruction, however a maximum of one per function is needed.
	32	valgrind_client_request_array: Builder.Value = .none,
	33	/// These fields are used to refer to the LLVM value of the function parameters
	34	/// in an Arg instruction.
	35	/// This list may be shorter than the list according to the zig type system;
	36	/// it omits 0-bit types. If the function uses sret as the first parameter,
	37	/// this slice does not include it.
	38	args: []const Builder.Value,
	39	arg_index: u32,
	40	arg_inline_index: u32,
	41
	42	err_ret_trace: Builder.Value,
	43
	44	/// This data structure is used to implement breaking to blocks.
	45	blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, struct {
	46	parent_bb: Builder.Function.Block.Index,
	47	breaks: *BreakList,
	48	}),
	49
	50	/// Maps `loop` instructions to the bb to branch to to repeat the loop.
	51	loops: std.AutoHashMapUnmanaged(Air.Inst.Index, Builder.Function.Block.Index),
	52
	53	/// Maps `loop_switch_br` instructions to the information required to lower
	54	/// dispatches (`switch_dispatch` instructions).
	55	switch_dispatch_info: std.AutoHashMapUnmanaged(Air.Inst.Index, SwitchDispatchInfo),
	56
	57	sync_scope: Builder.SyncScope,
	58
	59	disable_intrinsics: bool,
	60
	61	/// Have we seen loads or stores involving `allowzero` pointers?
	62	allowzero_access: bool,
	63
	64	/// In general, codegen should never emit errors; we cannot report useful source locations for them
	65	/// and they don't really play nicely with incremental compilation. The LLVM backend mostly obeys
	66	/// this rule. Where it does not, it calls `todo` to emit an error, and results in this error set
	67	/// being used for the function
	68	///
	69	/// Please avoid using this error set in new code. Ideally, every fallible function in this file
	70	/// should have the error set `Allocator.Error`.
	71	const TodoError = Zcu.CodegenFailError;
	72
	73	/// Avoid introducing new calls to this function---see documentation comment on `TodoError`.
	74	fn todo(fg: *FuncGen, comptime format: []const u8, args: anytype) TodoError {
	75	@branchHint(.cold);
	76	return fg.object.zcu.codegenFail(
	77	fg.nav_index,
	78	"TODO (LLVM): " ++ format,
	79	args,
	80	);
	81	}
	82
	83	fn ownerModule(fg: const FuncGen) Package.Module {
	84	return fg.object.zcu.navFileScope(fg.nav_index).mod.?;
	85	}
	86
	87	fn maybeMarkAllowZeroAccess(self: *FuncGen, info: InternPool.Key.PtrType) void {
	88	// LLVM already considers null pointers to be valid in non-generic address spaces, so avoid
	89	// pessimizing optimization for functions with accesses to such pointers.
	90	if (info.flags.address_space == .generic and info.flags.is_allowzero) self.allowzero_access = true;
	91	}
	92
	93	pub const Fuzz = struct {
	94	counters_variable: Builder.Variable.Index,
	95	pcs: std.ArrayList(Builder.Constant),
	96
	97	fn deinit(f: *Fuzz, gpa: Allocator) void {
	98	f.pcs.deinit(gpa);
	99	f.* = undefined;
	100	}
	101	};
	102
	103	const SwitchDispatchInfo = struct {
	104	/// These are the blocks corresponding to each switch case.
	105	/// The final element corresponds to the `else` case.
	106	/// Slices allocated into `gpa`.
	107	case_blocks: []Builder.Function.Block.Index,
	108	/// This is `.none` if `jmp_table` is set, since we won't use a `switch` instruction to dispatch.
	109	switch_weights: Builder.Function.Instruction.BrCond.Weights,
	110	/// If not `null`, we have manually constructed a jump table to reach the desired block.
	111	/// `table` can be used if the value is between `min` and `max` inclusive.
	112	/// We perform this lowering manually to avoid some questionable behavior from LLVM.
	113	/// See `airSwitchBr` for details.
	114	jmp_table: ?JmpTable,
	115
	116	const JmpTable = struct {
	117	min: Builder.Constant,
	118	max: Builder.Constant,
	119	in_bounds_hint: enum { none, unpredictable, likely, unlikely },
	120	/// Pointer to the jump table itself, to be used with `indirectbr`.
	121	/// The index into the jump table is the dispatch condition minus `min`.
	122	/// The table values are `blockaddress` constants corresponding to blocks in `case_blocks`.
	123	table: Builder.Constant,
	124	/// `true` if `table` conatins a reference to the `else` block.
	125	/// In this case, the `indirectbr` must include the `else` block in its target list.
	126	table_includes_else: bool,
	127	};
	128	};
	129
	130	const BreakList = union {
	131	list: std.MultiArrayList(struct {
	132	bb: Builder.Function.Block.Index,
	133	val: Builder.Value,
	134	}),
	135	len: usize,
	136	};
	137
	138	pub fn deinit(self: *FuncGen) void {
	139	const gpa = self.gpa;
	140	if (self.fuzz) \|*f\| f.deinit(self.gpa);
	141	self.wip.deinit();
	142	self.func_inst_table.deinit(gpa);
	143	self.blocks.deinit(gpa);
	144	self.loops.deinit(gpa);
	145	var it = self.switch_dispatch_info.valueIterator();
	146	while (it.next()) \|info\| {
	147	self.gpa.free(info.case_blocks);
	148	}
	149	self.switch_dispatch_info.deinit(gpa);
	150	}
	151
	152	fn resolveInst(self: *FuncGen, inst: Air.Inst.Ref) Allocator.Error!Builder.Value {
	153	const gpa = self.gpa;
	154	const gop = try self.func_inst_table.getOrPut(gpa, inst);
	155	if (gop.found_existing) return gop.value_ptr.*;
	156
	157	const llvm_val = try self.resolveValue(.fromInterned(inst.toInterned().?));
	158	gop.value_ptr.* = llvm_val.toValue();
	159	return llvm_val.toValue();
	160	}
	161
	162	fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
	163	const o = self.object;
	164	const zcu = o.zcu;
	165	const ty = val.typeOf(zcu);
	166	if (!isByRef(ty, zcu)) {
	167	return o.lowerValue(val.toIntern());
	168	} else {
	169	// We need a pointer to a global constant, i.e. a UAV.
	170	return o.lowerUavRef(
	171	val.toIntern(),
	172	ty.abiAlignment(zcu),
	173	target_util.defaultAddressSpace(zcu.getTarget(), .global_constant),
	174	);
	175	}
	176	}
	177
	178	pub fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.CoveragePoint) TodoError!void {
	179	const o = self.object;
	180	const zcu = self.object.zcu;
	181	const ip = &zcu.intern_pool;
	182	const air_tags = self.air.instructions.items(.tag);
	183	switch (coverage_point) {
	184	.none => {},
	185	.poi => if (self.fuzz) \|*fuzz\| {
	186	const poi_index = fuzz.pcs.items.len;
	187	const base_ptr = fuzz.counters_variable.toValue(&o.builder);
	188	const ptr = try self.ptraddConst(base_ptr, poi_index);
	189	const one = try o.builder.intValue(.i8, 1);
	190	_ = try self.wip.atomicrmw(.normal, .add, ptr, one, self.sync_scope, .monotonic, .default, "");
	191
	192	// LLVM does not allow blockaddress on the entry block.
	193	const pc = if (self.wip.cursor.block == .entry)
	194	self.wip.function.toConst(&o.builder)
	195	else
	196	try o.builder.blockAddrConst(self.wip.function, self.wip.cursor.block);
	197	const gpa = self.gpa;
	198	try fuzz.pcs.append(gpa, pc);
	199	},
	200	}
	201	for (body, 0..) \|inst, i\| {
	202	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
	203
	204	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
	205	// zig fmt: off
	206
	207	// No "scalarize" legalizations are enabled, so these instructions never appear.
	208	.legalize_vec_elem_val => unreachable,
	209	.legalize_vec_store_elem => unreachable,
	210	// No soft float legalizations are enabled.
	211	.legalize_compiler_rt_call => unreachable,
	212
	213	.add => try self.airAdd(inst, .normal),
	214	.add_optimized => try self.airAdd(inst, .fast),
	215	.add_wrap => try self.airAddWrap(inst),
	216	.add_sat => try self.airAddSat(inst),
	217
	218	.sub => try self.airSub(inst, .normal),
	219	.sub_optimized => try self.airSub(inst, .fast),
	220	.sub_wrap => try self.airSubWrap(inst),
	221	.sub_sat => try self.airSubSat(inst),
	222
	223	.mul => try self.airMul(inst, .normal),
	224	.mul_optimized => try self.airMul(inst, .fast),
	225	.mul_wrap => try self.airMulWrap(inst),
	226	.mul_sat => try self.airMulSat(inst),
	227
	228	.add_safe => try self.airSafeArithmetic(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
	229	.sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
	230	.mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
	231
	232	.div_float => try self.airDivFloat(inst, .normal),
	233	.div_trunc => try self.airDivTrunc(inst, .normal),
	234	.div_floor => try self.airDivFloor(inst, .normal),
	235	.div_exact => try self.airDivExact(inst, .normal),
	236	.rem => try self.airRem(inst, .normal),
	237	.mod => try self.airMod(inst, .normal),
	238	.abs => try self.airAbs(inst),
	239	.ptr_add => try self.airPtrAdd(inst),
	240	.ptr_sub => try self.airPtrSub(inst),
	241	.shl => try self.airShl(inst),
	242	.shl_sat => try self.airShlSat(inst),
	243	.shl_exact => try self.airShlExact(inst),
	244	.min => try self.airMin(inst),
	245	.max => try self.airMax(inst),
	246	.slice => try self.airSlice(inst),
	247	.mul_add => try self.airMulAdd(inst),
	248
	249	.div_float_optimized => try self.airDivFloat(inst, .fast),
	250	.div_trunc_optimized => try self.airDivTrunc(inst, .fast),
	251	.div_floor_optimized => try self.airDivFloor(inst, .fast),
	252	.div_exact_optimized => try self.airDivExact(inst, .fast),
	253	.rem_optimized => try self.airRem(inst, .fast),
	254	.mod_optimized => try self.airMod(inst, .fast),
	255
	256	.add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"),
	257	.sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"),
	258	.mul_with_overflow => try self.airOverflow(inst, .@"smul.with.overflow", .@"umul.with.overflow"),
	259	.shl_with_overflow => try self.airShlWithOverflow(inst),
	260
	261	.bit_and, .bool_and => try self.airAnd(inst),
	262	.bit_or, .bool_or => try self.airOr(inst),
	263	.xor => try self.airXor(inst),
	264	.shr => try self.airShr(inst, false),
	265	.shr_exact => try self.airShr(inst, true),
	266
	267	.sqrt => try self.airUnaryOp(inst, .sqrt),
	268	.sin => try self.airUnaryOp(inst, .sin),
	269	.cos => try self.airUnaryOp(inst, .cos),
	270	.tan => try self.airUnaryOp(inst, .tan),
	271	.exp => try self.airUnaryOp(inst, .exp),
	272	.exp2 => try self.airUnaryOp(inst, .exp2),
	273	.log => try self.airUnaryOp(inst, .log),
	274	.log2 => try self.airUnaryOp(inst, .log2),
	275	.log10 => try self.airUnaryOp(inst, .log10),
	276	.floor => try self.airUnaryOp(inst, .floor),
	277	.ceil => try self.airUnaryOp(inst, .ceil),
	278	.round => try self.airUnaryOp(inst, .round),
	279	.trunc_float => try self.airUnaryOp(inst, .trunc),
	280
	281	.neg => try self.airNeg(inst, .normal),
	282	.neg_optimized => try self.airNeg(inst, .fast),
	283
	284	.cmp_eq => try self.airCmp(inst, .eq, .normal),
	285	.cmp_gt => try self.airCmp(inst, .gt, .normal),
	286	.cmp_gte => try self.airCmp(inst, .gte, .normal),
	287	.cmp_lt => try self.airCmp(inst, .lt, .normal),
	288	.cmp_lte => try self.airCmp(inst, .lte, .normal),
	289	.cmp_neq => try self.airCmp(inst, .neq, .normal),
	290
	291	.cmp_eq_optimized => try self.airCmp(inst, .eq, .fast),
	292	.cmp_gt_optimized => try self.airCmp(inst, .gt, .fast),
	293	.cmp_gte_optimized => try self.airCmp(inst, .gte, .fast),
	294	.cmp_lt_optimized => try self.airCmp(inst, .lt, .fast),
	295	.cmp_lte_optimized => try self.airCmp(inst, .lte, .fast),
	296	.cmp_neq_optimized => try self.airCmp(inst, .neq, .fast),
	297
	298	.cmp_vector => try self.airCmpVector(inst, .normal),
	299	.cmp_vector_optimized => try self.airCmpVector(inst, .fast),
	300	.cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst),
	301
	302	.is_non_null => try self.airIsNonNull(inst, false, .ne),
	303	.is_non_null_ptr => try self.airIsNonNull(inst, true , .ne),
	304	.is_null => try self.airIsNonNull(inst, false, .eq),
	305	.is_null_ptr => try self.airIsNonNull(inst, true , .eq),
	306
	307	.is_non_err => try self.airIsErr(inst, .eq, false),
	308	.is_non_err_ptr => try self.airIsErr(inst, .eq, true),
	309	.is_err => try self.airIsErr(inst, .ne, false),
	310	.is_err_ptr => try self.airIsErr(inst, .ne, true),
	311
	312	.alloc => try self.airAlloc(inst),
	313	.ret_ptr => try self.airRetPtr(inst),
	314	.arg => try self.airArg(inst),
	315	.bitcast => try self.airBitCast(inst),
	316	.breakpoint => try self.airBreakpoint(inst),
	317	.ret_addr => try self.airRetAddr(inst),
	318	.frame_addr => try self.airFrameAddress(inst),
	319	.@"try" => try self.airTry(inst, false),
	320	.try_cold => try self.airTry(inst, true),
	321	.try_ptr => try self.airTryPtr(inst, false),
	322	.try_ptr_cold => try self.airTryPtr(inst, true),
	323	.intcast => try self.airIntCast(inst, false),
	324	.intcast_safe => try self.airIntCast(inst, true),
	325	.trunc => try self.airTrunc(inst),
	326	.fptrunc => try self.airFptrunc(inst),
	327	.fpext => try self.airFpext(inst),
	328	.load => try self.airLoad(inst),
	329	.not => try self.airNot(inst),
	330	.store => try self.airStore(inst, false),
	331	.store_safe => try self.airStore(inst, true),
	332	.assembly => try self.airAssembly(inst),
	333	.slice_ptr => try self.airSliceField(inst, 0),
	334	.slice_len => try self.airSliceField(inst, 1),
	335
	336	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
	337	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
	338
	339	.int_from_float => try self.airIntFromFloat(inst, .normal),
	340	.int_from_float_optimized => try self.airIntFromFloat(inst, .fast),
	341	.int_from_float_safe => unreachable, // handled by `legalizeFeatures`
	342	.int_from_float_optimized_safe => unreachable, // handled by `legalizeFeatures`
	343
	344	.array_to_slice => try self.airArrayToSlice(inst),
	345	.float_from_int => try self.airFloatFromInt(inst),
	346	.cmpxchg_weak => try self.airCmpxchg(inst, .weak),
	347	.cmpxchg_strong => try self.airCmpxchg(inst, .strong),
	348	.atomic_rmw => try self.airAtomicRmw(inst),
	349	.atomic_load => try self.airAtomicLoad(inst),
	350	.memset => try self.airMemset(inst, false),
	351	.memset_safe => try self.airMemset(inst, true),
	352	.memcpy => try self.airMemcpy(inst),
	353	.memmove => try self.airMemmove(inst),
	354	.set_union_tag => try self.airSetUnionTag(inst),
	355	.get_union_tag => try self.airGetUnionTag(inst),
	356	.clz => try self.airClzCtz(inst, .ctlz),
	357	.ctz => try self.airClzCtz(inst, .cttz),
	358	.popcount => try self.airBitOp(inst, .ctpop),
	359	.byte_swap => try self.airByteSwap(inst),
	360	.bit_reverse => try self.airBitOp(inst, .bitreverse),
	361	.tag_name => try self.airTagName(inst),
	362	.error_name => try self.airErrorName(inst),
	363	.splat => try self.airSplat(inst),
	364	.select => try self.airSelect(inst),
	365	.shuffle_one => try self.airShuffleOne(inst),
	366	.shuffle_two => try self.airShuffleTwo(inst),
	367	.aggregate_init => try self.airAggregateInit(inst),
	368	.union_init => try self.airUnionInit(inst),
	369	.prefetch => try self.airPrefetch(inst),
	370	.addrspace_cast => try self.airAddrSpaceCast(inst),
	371
	372	.is_named_enum_value => try self.airIsNamedEnumValue(inst),
	373	.error_set_has_value => try self.airErrorSetHasValue(inst),
	374
	375	.reduce => try self.airReduce(inst, .normal),
	376	.reduce_optimized => try self.airReduce(inst, .fast),
	377
	378	.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
	379	.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),
	380	.atomic_store_release => try self.airAtomicStore(inst, .release),
	381	.atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst),
	382
	383	.struct_field_ptr => try self.airStructFieldPtr(inst),
	384	.struct_field_val => try self.airStructFieldVal(inst),
	385
	386	.struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0),
	387	.struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1),
	388	.struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2),
	389	.struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3),
	390
	391	.field_parent_ptr => try self.airFieldParentPtr(inst),
	392
	393	.array_elem_val => try self.airArrayElemVal(inst),
	394	.slice_elem_val => try self.airSliceElemVal(inst),
	395	.slice_elem_ptr => try self.airSliceElemPtr(inst),
	396	.ptr_elem_val => try self.airPtrElemVal(inst),
	397	.ptr_elem_ptr => try self.airPtrElemPtr(inst),
	398
	399	.optional_payload => try self.airOptionalPayload(inst),
	400	.optional_payload_ptr => try self.airOptionalPayloadPtr(inst),
	401	.optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst),
	402
	403	.unwrap_errunion_payload => try self.airErrUnionPayload(inst, false),
	404	.unwrap_errunion_payload_ptr => try self.airErrUnionPayload(inst, true),
	405	.unwrap_errunion_err => try self.airErrUnionErr(inst, false),
	406	.unwrap_errunion_err_ptr => try self.airErrUnionErr(inst, true),
	407	.errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst),
	408	.err_return_trace => try self.airErrReturnTrace(inst),
	409	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
	410	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
	411
	412	.wrap_optional => try self.airWrapOptional(body[i..]),
	413	.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
	414	.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
	415
	416	.wasm_memory_size => try self.airWasmMemorySize(inst),
	417	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
	418
	419	.runtime_nav_ptr => try self.airRuntimeNavPtr(inst),
	420
	421	.inferred_alloc, .inferred_alloc_comptime => unreachable,
	422
	423	.dbg_stmt => try self.airDbgStmt(inst),
	424	.dbg_empty_stmt => try self.airDbgEmptyStmt(inst),
	425	.dbg_var_ptr => try self.airDbgVarPtr(inst),
	426	.dbg_var_val => try self.airDbgVarVal(inst, false),
	427	.dbg_arg_inline => try self.airDbgVarVal(inst, true),
	428
	429	.c_va_arg => try self.airCVaArg(inst),
	430	.c_va_copy => try self.airCVaCopy(inst),
	431	.c_va_end => try self.airCVaEnd(inst),
	432	.c_va_start => try self.airCVaStart(inst),
	433
	434	.work_item_id => try self.airWorkItemId(inst),
	435	.work_group_size => try self.airWorkGroupSize(inst),
	436	.work_group_id => try self.airWorkGroupId(inst),
	437
	438	// Instructions that are known to always be `noreturn` based on their tag.
	439	.br => return self.airBr(inst),
	440	.repeat => return self.airRepeat(inst),
	441	.switch_dispatch => return self.airSwitchDispatch(inst),
	442	.cond_br => return self.airCondBr(inst),
	443	.switch_br => return self.airSwitchBr(inst, false),
	444	.loop_switch_br => return self.airSwitchBr(inst, true),
	445	.loop => return self.airLoop(inst),
	446	.ret => return self.airRet(inst, false),
	447	.ret_safe => return self.airRet(inst, true),
	448	.ret_load => return self.airRetLoad(inst),
	449	.trap => return self.airTrap(inst),
	450	.unreach => return self.airUnreach(inst),
	451
	452	// Instructions which may be `noreturn`.
	453	.block => res: {
	454	const block = self.air.unwrapBlock(inst);
	455	const res = try self.lowerBlock(inst, null, block.body);
	456	if (block.ty.isNoReturn(zcu)) return;
	457	break :res res;
	458	},
	459	.dbg_inline_block => res: {
	460	const block = self.air.unwrapDbgBlock(inst);
	461	self.arg_inline_index = 0;
	462	const res = try self.lowerBlock(inst, block.func, block.body);
	463	if (block.ty.isNoReturn(zcu)) return;
	464	break :res res;
	465	},
	466	.call, .call_always_tail, .call_never_tail, .call_never_inline => \|tag\| res: {
	467	const res = try self.airCall(inst, switch (tag) {
	468	.call => .auto,
	469	.call_always_tail => .always_tail,
	470	.call_never_tail => .never_tail,
	471	.call_never_inline => .never_inline,
	472	else => unreachable,
	473	});
	474	// TODO: the AIR we emit for calls is a bit weird - the instruction has
	475	// type `noreturn`, but there are instructions (and maybe a safety check) following
	476	// nonetheless. The `unreachable` or safety check should be emitted by backends instead.
	477	//if (self.typeOfIndex(inst).isNoReturn(mod)) return;
	478	break :res res;
	479	},
	480
	481	// zig fmt: on
	482	};
	483	if (val != .none) try self.func_inst_table.putNoClobber(self.gpa, inst.toRef(), val);
	484	}
	485	unreachable;
	486	}
	487
	488	fn genBodyDebugScope(
	489	self: *FuncGen,
	490	maybe_inline_func: ?InternPool.Index,
	491	body: []const Air.Inst.Index,
	492	coverage_point: Air.CoveragePoint,
	493	) TodoError!void {
	494	const o = self.object;
	495
	496	if (self.wip.strip) return self.genBody(body, coverage_point);
	497
	498	const old_debug_location = self.wip.debug_location;
	499	const old_file = self.file;
	500	const old_inlined_at = self.inlined_at;
	501	const old_base_line = self.base_line;
	502	defer if (maybe_inline_func) \|_\| {
	503	self.wip.debug_location = old_debug_location;
	504	self.file = old_file;
	505	self.inlined_at = old_inlined_at;
	506	self.base_line = old_base_line;
	507	};
	508
	509	const old_scope = self.scope;
	510	defer self.scope = old_scope;
	511
	512	if (maybe_inline_func) \|inline_func\| {
	513	const zcu = o.zcu;
	514	const ip = &zcu.intern_pool;
	515
	516	const func = zcu.funcInfo(inline_func);
	517	const nav = ip.getNav(func.owner_nav);
	518	const file_scope = zcu.navFileScopeIndex(func.owner_nav);
	519	const mod = zcu.fileByIndex(file_scope).mod.?;
	520
	521	self.file = try o.getDebugFile(file_scope);
	522
	523	self.base_line = zcu.navSrcLine(func.owner_nav);
	524	const line_number = self.base_line + 1;
	525	self.inlined_at = try self.wip.debug_location.toMetadata(&o.builder);
	526
	527	self.scope = try o.builder.debugSubprogram(
	528	self.file,
	529	try o.builder.metadataString(nav.name.toSlice(&zcu.intern_pool)),
	530	try o.builder.metadataString(nav.fqn.toSlice(&zcu.intern_pool)),
	531	line_number,
	532	line_number + func.lbrace_line,
	533	try o.builder.debugSubroutineType(null),
	534	.{
	535	.di_flags = .{ .StaticMember = true },
	536	.sp_flags = .{
	537	.Optimized = mod.optimize_mode != .Debug,
	538	.Definition = true,
	539	.LocalToUnit = true, // inline functions cannot be exported
	540	},
	541	},
	542	o.debug_compile_unit.unwrap().?,
	543	);
	544	}
	545
	546	self.scope = try o.builder.debugLexicalBlock(
	547	self.scope,
	548	self.file,
	549	self.prev_dbg_line,
	550	self.prev_dbg_column,
	551	);
	552	self.wip.debug_location = .{ .location = .{
	553	.line = self.prev_dbg_line,
	554	.column = self.prev_dbg_column,
	555	.scope = self.scope.toOptional(),
	556	.inlined_at = self.inlined_at,
	557	} };
	558
	559	try self.genBody(body, coverage_point);
	560	}
	561
	562	const CallAttr = enum {
	563	Auto,
	564	NeverTail,
	565	NeverInline,
	566	AlwaysTail,
	567	AlwaysInline,
	568	};
	569
	570	fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifier) Allocator.Error!Builder.Value {
	571	const air_call = self.air.unwrapCall(inst);
	572	const args = air_call.args;
	573	const o = self.object;
	574	const pt = self.pt;
	575	const zcu = o.zcu;
	576	const ip = &zcu.intern_pool;
	577	const callee_ty = self.typeOf(air_call.callee);
	578	const zig_fn_ty = switch (callee_ty.zigTypeTag(zcu)) {
	579	.@"fn" => callee_ty,
	580	.pointer => callee_ty.childType(zcu),
	581	else => unreachable,
	582	};
	583	const fn_info = zcu.typeToFunc(zig_fn_ty).?;
	584	const return_type: Type = .fromInterned(fn_info.return_type);
	585	const llvm_fn = llvm_fn: {
	586	// If the callee is a function body, we need to use a pointer to the global.
	587	if (air_call.callee.toInterned()) \|ip_index\| switch (ip.indexToKey(ip_index)) {
	588	.@"extern" => \|e\| break :llvm_fn (try o.lowerNavRef(e.owner_nav)).toValue(),
	589	.func => \|f\| break :llvm_fn (try o.lowerNavRef(f.owner_nav)).toValue(),
	590	else => {},
	591	};
	592	// Otherwise, the operand is already a function pointer (possibly runtime-known).
	593	break :llvm_fn try self.resolveInst(air_call.callee);
	594	};
	595	const target = zcu.getTarget();
	596	const sret = firstParamSRet(fn_info, zcu, target);
	597
	598	var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
	599	defer llvm_args.deinit();
	600
	601	var attributes: Builder.FunctionAttributes.Wip = .{};
	602	defer attributes.deinit(&o.builder);
	603
	604	if (self.disable_intrinsics) {
	605	try attributes.addFnAttr(.nobuiltin, &o.builder);
	606	}
	607
	608	switch (modifier) {
	609	.auto, .always_tail => {},
	610	.never_tail, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
	611	.no_suspend, .always_inline, .compile_time => unreachable,
	612	}
	613
	614	const ret_ptr = if (!sret) null else blk: {
	615	const llvm_ret_ty = try o.lowerType(return_type);
	616	try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
	617
	618	const alignment = return_type.abiAlignment(zcu).toLlvm();
	619	const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
	620	try llvm_args.append(ret_ptr);
	621	break :blk ret_ptr;
	622	};
	623
	624	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
	625	if (err_return_tracing) {
	626	assert(self.err_ret_trace != .none);
	627	try llvm_args.append(self.err_ret_trace);
	628	}
	629
	630	var it = iterateParamTypes(o, fn_info);
	631	while (try it.nextCall(self, args)) \|lowering\| switch (lowering) {
	632	.no_bits => continue,
	633	.byval => {
	634	const arg = args[it.zig_index - 1];
	635	const param_ty = self.typeOf(arg);
	636	const llvm_arg = try self.resolveInst(arg);
	637	const llvm_param_ty = try o.lowerType(param_ty);
	638	if (isByRef(param_ty, zcu)) {
	639	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	640	const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
	641	try llvm_args.append(loaded);
	642	} else {
	643	try llvm_args.append(llvm_arg);
	644	}
	645	},
	646	.byref => {
	647	const arg = args[it.zig_index - 1];
	648	const param_ty = self.typeOf(arg);
	649	const llvm_arg = try self.resolveInst(arg);
	650	if (isByRef(param_ty, zcu)) {
	651	try llvm_args.append(llvm_arg);
	652	} else {
	653	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	654	const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
	655	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
	656	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
	657	try llvm_args.append(arg_ptr);
	658	}
	659	},
	660	.byref_mut => {
	661	const arg = args[it.zig_index - 1];
	662	const param_ty = self.typeOf(arg);
	663	const llvm_arg = try self.resolveInst(arg);
	664
	665	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	666	const param_llvm_ty = try o.lowerType(param_ty);
	667	const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
	668	if (isByRef(param_ty, zcu)) {
	669	const loaded = try self.wip.load(.normal, param_llvm_ty, llvm_arg, alignment, "");
	670	_ = try self.wip.store(.normal, loaded, arg_ptr, alignment);
	671	} else {
	672	_ = try self.wip.store(.normal, llvm_arg, arg_ptr, alignment);
	673	}
	674	try llvm_args.append(arg_ptr);
	675	},
	676	.abi_sized_int => {
	677	const arg = args[it.zig_index - 1];
	678	const param_ty = self.typeOf(arg);
	679	const llvm_arg = try self.resolveInst(arg);
	680	const int_llvm_ty = try o.builder.intType(@intCast(param_ty.abiSize(zcu) * 8));
	681
	682	if (isByRef(param_ty, zcu)) {
	683	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	684	const loaded = try self.wip.load(.normal, int_llvm_ty, llvm_arg, alignment, "");
	685	try llvm_args.append(loaded);
	686	} else {
	687	// LLVM does not allow bitcasting structs so we must allocate
	688	// a local, store as one type, and then load as another type.
	689	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	690	const int_ptr = try self.buildAlloca(int_llvm_ty, alignment);
	691	_ = try self.wip.store(.normal, llvm_arg, int_ptr, alignment);
	692	const loaded = try self.wip.load(.normal, int_llvm_ty, int_ptr, alignment, "");
	693	try llvm_args.append(loaded);
	694	}
	695	},
	696	.slice => {
	697	const arg = args[it.zig_index - 1];
	698	const llvm_arg = try self.resolveInst(arg);
	699	const ptr = try self.wip.extractValue(llvm_arg, &.{0}, "");
	700	const len = try self.wip.extractValue(llvm_arg, &.{1}, "");
	701	try llvm_args.appendSlice(&.{ ptr, len });
	702	},
	703	.multiple_llvm_types => {
	704	const arg = args[it.zig_index - 1];
	705	const param_ty = self.typeOf(arg);
	706	const llvm_types = it.types_buffer[0..it.types_len];
	707	const llvm_arg = try self.resolveInst(arg);
	708	const is_by_ref = isByRef(param_ty, zcu);
	709	const arg_ptr = if (is_by_ref) llvm_arg else ptr: {
	710	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	711	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
	712	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
	713	break :ptr ptr;
	714	};
	715
	716	const llvm_ty = try o.builder.structType(.normal, llvm_types);
	717	try llvm_args.ensureUnusedCapacity(it.types_len);
	718	for (llvm_types, 0..) \|field_ty, i\| {
	719	const alignment: Builder.Alignment = .fromByteUnits(@divExact(target.ptrBitWidth(), 8));
	720	const field_ptr = try self.wip.gepStruct(llvm_ty, arg_ptr, i, "");
	721	const loaded = try self.wip.load(.normal, field_ty, field_ptr, alignment, "");
	722	llvm_args.appendAssumeCapacity(loaded);
	723	}
	724	},
	725	.float_array => \|count\| {
	726	const arg = args[it.zig_index - 1];
	727	const arg_ty = self.typeOf(arg);
	728	var llvm_arg = try self.resolveInst(arg);
	729	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
	730	if (!isByRef(arg_ty, zcu)) {
	731	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
	732	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
	733	llvm_arg = ptr;
	734	}
	735
	736	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
	737	const array_ty = try o.builder.arrayType(count, float_ty);
	738
	739	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
	740	try llvm_args.append(loaded);
	741	},
	742	.i32_array, .i64_array => \|arr_len\| {
	743	const elem_size: u8 = if (lowering == .i32_array) 32 else 64;
	744	const arg = args[it.zig_index - 1];
	745	const arg_ty = self.typeOf(arg);
	746	var llvm_arg = try self.resolveInst(arg);
	747	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
	748	if (!isByRef(arg_ty, zcu)) {
	749	const ptr = try self.buildAlloca(llvm_arg.typeOfWip(&self.wip), alignment);
	750	_ = try self.wip.store(.normal, llvm_arg, ptr, alignment);
	751	llvm_arg = ptr;
	752	}
	753
	754	const array_ty =
	755	try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
	756	const loaded = try self.wip.load(.normal, array_ty, llvm_arg, alignment, "");
	757	try llvm_args.append(loaded);
	758	},
	759	};
	760
	761	{
	762	// Add argument attributes.
	763	it = iterateParamTypes(o, fn_info);
	764	it.llvm_index += @intFromBool(sret);
	765	it.llvm_index += @intFromBool(err_return_tracing);
	766	while (try it.next()) \|lowering\| switch (lowering) {
	767	.byval => {
	768	const param_index = it.zig_index - 1;
	769	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
	770	if (!isByRef(param_ty, zcu)) {
	771	try o.addByValParamAttrs(pt, &attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
	772	}
	773	},
	774	.byref => {
	775	const param_index = it.zig_index - 1;
	776	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
	777	const param_llvm_ty = try o.lowerType(param_ty);
	778	const alignment = param_ty.abiAlignment(zcu).toLlvm();
	779	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
	780	},
	781	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
	782	// No attributes needed for these.
	783	.no_bits,
	784	.abi_sized_int,
	785	.multiple_llvm_types,
	786	.float_array,
	787	.i32_array,
	788	.i64_array,
	789	=> continue,
	790
	791	.slice => {
	792	assert(!it.byval_attr);
	793	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
	794	const ptr_info = param_ty.ptrInfo(zcu);
	795	const llvm_arg_i = it.llvm_index - 2;
	796
	797	if (math.cast(u5, it.zig_index - 1)) \|i\| {
	798	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
	799	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
	800	}
	801	}
	802	if (param_ty.zigTypeTag(zcu) != .optional and
	803	!ptr_info.flags.is_allowzero and
	804	ptr_info.flags.address_space == .generic)
	805	{
	806	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
	807	}
	808	if (ptr_info.flags.is_const) {
	809	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
	810	}
	811	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
	812	else => \|a\| .wrap(a.toLlvm()),
	813	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
	814	};
	815	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
	816	},
	817	};
	818	}
	819
	820	const call = try self.wip.call(
	821	switch (modifier) {
	822	.auto, .never_inline => .normal,
	823	.never_tail => .notail,
	824	.always_tail => .musttail,
	825	.no_suspend, .always_inline, .compile_time => unreachable,
	826	},
	827	llvm.toLlvmCallConvTag(fn_info.cc, target).?,
	828	try attributes.finish(&o.builder),
	829	try o.lowerType(zig_fn_ty),
	830	llvm_fn,
	831	llvm_args.items,
	832	"",
	833	);
	834
	835	if (fn_info.return_type == .noreturn_type and modifier != .always_tail) {
	836	return .none;
	837	}
	838
	839	if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
	840	return .none;
	841	}
	842
	843	const llvm_ret_ty = try o.lowerType(return_type);
	844	if (ret_ptr) \|rp\| {
	845	if (isByRef(return_type, zcu)) {
	846	return rp;
	847	} else {
	848	// our by-ref status disagrees with sret so we must load.
	849	const return_alignment = return_type.abiAlignment(zcu).toLlvm();
	850	return self.wip.load(.normal, llvm_ret_ty, rp, return_alignment, "");
	851	}
	852	}
	853
	854	const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	855
	856	if (abi_ret_ty != llvm_ret_ty) {
	857	// In this case the function return type is honoring the calling convention by having
	858	// a different LLVM type than the usual one. We solve this here at the callsite
	859	// by using our canonical type, then loading it if necessary.
	860	const alignment = return_type.abiAlignment(zcu).toLlvm();
	861	const rp = try self.buildAlloca(abi_ret_ty, alignment);
	862	_ = try self.wip.store(.normal, call, rp, alignment);
	863	return if (isByRef(return_type, zcu))
	864	rp
	865	else
	866	try self.wip.load(.normal, llvm_ret_ty, rp, alignment, "");
	867	}
	868
	869	if (isByRef(return_type, zcu)) {
	870	// our by-ref status disagrees with sret so we must allocate, store,
	871	// and return the allocation pointer.
	872	const alignment = return_type.abiAlignment(zcu).toLlvm();
	873	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
	874	_ = try self.wip.store(.normal, call, rp, alignment);
	875	return rp;
	876	} else {
	877	return call;
	878	}
	879	}
	880
	881	fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!void {
	882	const o = fg.object;
	883	const zcu = o.zcu;
	884	const target = zcu.getTarget();
	885	const panic_func = zcu.funcInfo(zcu.builtin_decl_values.get(panic_id.toBuiltin()));
	886	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
	887	const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty));
	888
	889	const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
	890
	891	const has_err_trace = zcu.comp.config.any_error_tracing and fn_info.cc == .auto;
	892	if (has_err_trace) assert(fg.err_ret_trace != .none);
	893	_ = try fg.wip.callIntrinsicAssumeCold();
	894	_ = try fg.wip.call(
	895	.normal,
	896	llvm.toLlvmCallConvTag(fn_info.cc, target).?,
	897	.none,
	898	llvm_panic_fn_ty,
	899	llvm_panic_fn_ref.toValue(),
	900	if (has_err_trace) &.{fg.err_ret_trace} else &.{},
	901	"",
	902	);
	903	_ = try fg.wip.@"unreachable"();
	904	}
	905
	906	fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!void {
	907	const o = self.object;
	908	const zcu = o.zcu;
	909	const ip = &zcu.intern_pool;
	910	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	911	const ret_ty = self.typeOf(un_op);
	912
	913	if (self.ret_ptr != .none) {
	914	const operand = try self.resolveInst(un_op);
	915	const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
	916	if (val_is_undef and safety) {
	917	const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
	918	_ = try self.wip.callMemSet(
	919	self.ret_ptr,
	920	ret_ty.abiAlignment(zcu).toLlvm(),
	921	try o.builder.intValue(.i8, 0xaa),
	922	len,
	923	.normal,
	924	self.disable_intrinsics,
	925	);
	926	const owner_mod = self.ownerModule();
	927	if (owner_mod.valgrind) {
	928	try self.valgrindMarkUndef(self.ret_ptr, len);
	929	}
	930	_ = try self.wip.retVoid();
	931	return;
	932	}
	933
	934	const unwrapped_operand = operand.unwrap();
	935	const unwrapped_ret = self.ret_ptr.unwrap();
	936
	937	// Return value was stored previously
	938	if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
	939	_ = try self.wip.retVoid();
	940	return;
	941	}
	942
	943	try self.store(
	944	self.ret_ptr,
	945	.none,
	946	operand,
	947	ret_ty,
	948	);
	949	_ = try self.wip.retVoid();
	950	return;
	951	}
	952	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
	953	if (!ret_ty.hasRuntimeBits(zcu)) {
	954	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
	955	// Functions with an empty error set are emitted with an error code
	956	// return type and return zero so they can be function pointers coerced
	957	// to functions that return anyerror.
	958	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
	959	} else {
	960	_ = try self.wip.retVoid();
	961	}
	962	return;
	963	}
	964
	965	const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	966	const operand = try self.resolveInst(un_op);
	967	const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
	968	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
	969
	970	if (val_is_undef and safety) {
	971	const llvm_ret_ty = operand.typeOfWip(&self.wip);
	972	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
	973	const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
	974	_ = try self.wip.callMemSet(
	975	rp,
	976	alignment,
	977	try o.builder.intValue(.i8, 0xaa),
	978	len,
	979	.normal,
	980	self.disable_intrinsics,
	981	);
	982	const owner_mod = self.ownerModule();
	983	if (owner_mod.valgrind) {
	984	try self.valgrindMarkUndef(rp, len);
	985	}
	986	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
	987	return;
	988	}
	989
	990	if (isByRef(ret_ty, zcu)) {
	991	// operand is a pointer however self.ret_ptr is null so that means
	992	// we need to return a value.
	993	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, alignment, ""));
	994	return;
	995	}
	996
	997	const llvm_ret_ty = operand.typeOfWip(&self.wip);
	998	if (abi_ret_ty == llvm_ret_ty) {
	999	_ = try self.wip.ret(operand);
	1000	return;
	1001	}
	1002
	1003	const rp = try self.buildAlloca(llvm_ret_ty, alignment);
	1004	_ = try self.wip.store(.normal, operand, rp, alignment);
	1005	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, rp, alignment, ""));
	1006	return;
	1007	}
	1008
	1009	fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	1010	const o = self.object;
	1011	const zcu = o.zcu;
	1012	const ip = &zcu.intern_pool;
	1013	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	1014	const ptr_ty = self.typeOf(un_op);
	1015	const ret_ty = ptr_ty.childType(zcu);
	1016	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
	1017	if (!ret_ty.hasRuntimeBits(zcu)) {
	1018	if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
	1019	// Functions with an empty error set are emitted with an error code
	1020	// return type and return zero so they can be function pointers coerced
	1021	// to functions that return anyerror.
	1022	_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
	1023	} else {
	1024	_ = try self.wip.retVoid();
	1025	}
	1026	return;
	1027	}
	1028	if (self.ret_ptr != .none) {
	1029	_ = try self.wip.retVoid();
	1030	return;
	1031	}
	1032	const ptr = try self.resolveInst(un_op);
	1033	const abi_ret_ty = try lowerFnRetTy(o, fn_info);
	1034	const alignment = ret_ty.abiAlignment(zcu).toLlvm();
	1035	_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, ptr, alignment, ""));
	1036	return;
	1037	}
	1038
	1039	fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1040	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	1041	const list = try self.resolveInst(ty_op.operand);
	1042	const arg_ty = ty_op.ty.toType();
	1043	const llvm_arg_ty = try self.object.lowerType(arg_ty);
	1044
	1045	return self.wip.vaArg(list, llvm_arg_ty, "");
	1046	}
	1047
	1048	fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1049	const o = self.object;
	1050	const zcu = o.zcu;
	1051	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	1052	const src_list = try self.resolveInst(ty_op.operand);
	1053	const va_list_ty = ty_op.ty.toType();
	1054	const llvm_va_list_ty = try o.lowerType(va_list_ty);
	1055
	1056	const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
	1057	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1058
	1059	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
	1060	return if (isByRef(va_list_ty, zcu))
	1061	dest_list
	1062	else
	1063	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
	1064	}
	1065
	1066	fn airCVaEnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1067	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	1068	const src_list = try self.resolveInst(un_op);
	1069
	1070	_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
	1071	return .none;
	1072	}
	1073
	1074	fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1075	const o = self.object;
	1076	const zcu = o.zcu;
	1077	const va_list_ty = self.typeOfIndex(inst);
	1078	const llvm_va_list_ty = try o.lowerType(va_list_ty);
	1079
	1080	const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
	1081	const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1082
	1083	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
	1084	return if (isByRef(va_list_ty, zcu))
	1085	dest_list
	1086	else
	1087	try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, "");
	1088	}
	1089
	1090	fn airCmp(
	1091	self: *FuncGen,
	1092	inst: Air.Inst.Index,
	1093	op: math.CompareOperator,
	1094	fast: Builder.FastMathKind,
	1095	) Allocator.Error!Builder.Value {
	1096	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	1097	const lhs = try self.resolveInst(bin_op.lhs);
	1098	const rhs = try self.resolveInst(bin_op.rhs);
	1099	const operand_ty = self.typeOf(bin_op.lhs);
	1100
	1101	return self.cmp(fast, op, operand_ty, lhs, rhs);
	1102	}
	1103
	1104	fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	1105	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	1106	const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
	1107
	1108	const lhs = try self.resolveInst(extra.lhs);
	1109	const rhs = try self.resolveInst(extra.rhs);
	1110	const vec_ty = self.typeOf(extra.lhs);
	1111	const cmp_op = extra.compareOperator();
	1112
	1113	return self.cmp(fast, cmp_op, vec_ty, lhs, rhs);
	1114	}
	1115
	1116	fn airCmpLteErrorsLen(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1117	const o = self.object;
	1118	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	1119	const operand = try self.resolveInst(un_op);
	1120	const errors_len_ptr = try o.getErrorsLen();
	1121	const errors_len_val = try self.wip.load(
	1122	.normal,
	1123	try o.errorIntType(),
	1124	errors_len_ptr.toValue(&o.builder),
	1125	Type.errorAbiAlignment(o.zcu).toLlvm(),
	1126	"",
	1127	);
	1128	return self.wip.icmp(.ule, operand, errors_len_val, "");
	1129	}
	1130
	1131	fn cmp(
	1132	self: *FuncGen,
	1133	fast: Builder.FastMathKind,
	1134	op: math.CompareOperator,
	1135	operand_ty: Type,
	1136	lhs: Builder.Value,
	1137	rhs: Builder.Value,
	1138	) Allocator.Error!Builder.Value {
	1139	const o = self.object;
	1140	const zcu = o.zcu;
	1141	const scalar_ty = operand_ty.scalarType(zcu);
	1142	const int_ty = switch (scalar_ty.zigTypeTag(zcu)) {
	1143	.@"enum" => scalar_ty.intTagType(zcu),
	1144	.int, .bool, .pointer, .error_set => scalar_ty,
	1145	.optional => blk: {
	1146	const payload_ty = operand_ty.optionalChild(zcu);
	1147	if (!payload_ty.hasRuntimeBits(zcu) or
	1148	operand_ty.optionalReprIsPayload(zcu))
	1149	{
	1150	break :blk operand_ty;
	1151	}
	1152	// We need to emit instructions to check for equality/inequality
	1153	// of optionals that are not pointers.
	1154	const lhs_non_null = try self.optCmpNull(.ne, scalar_ty, lhs, .normal);
	1155	const rhs_non_null = try self.optCmpNull(.ne, scalar_ty, rhs, .normal);
	1156	const llvm_i2 = try o.builder.intType(2);
	1157	const lhs_non_null_i2 = try self.wip.cast(.zext, lhs_non_null, llvm_i2, "");
	1158	const rhs_non_null_i2 = try self.wip.cast(.zext, rhs_non_null, llvm_i2, "");
	1159	const lhs_shifted = try self.wip.bin(.shl, lhs_non_null_i2, try o.builder.intValue(llvm_i2, 1), "");
	1160	const lhs_rhs_ored = try self.wip.bin(.@"or", lhs_shifted, rhs_non_null_i2, "");
	1161	const both_null_block = try self.wip.block(1, "BothNull");
	1162	const mixed_block = try self.wip.block(1, "Mixed");
	1163	const both_pl_block = try self.wip.block(1, "BothNonNull");
	1164	const end_block = try self.wip.block(3, "End");
	1165	var wip_switch = try self.wip.@"switch"(lhs_rhs_ored, mixed_block, 2, .none);
	1166	defer wip_switch.finish(&self.wip);
	1167	try wip_switch.addCase(
	1168	try o.builder.intConst(llvm_i2, 0b00),
	1169	both_null_block,
	1170	&self.wip,
	1171	);
	1172	try wip_switch.addCase(
	1173	try o.builder.intConst(llvm_i2, 0b11),
	1174	both_pl_block,
	1175	&self.wip,
	1176	);
	1177
	1178	self.wip.cursor = .{ .block = both_null_block };
	1179	_ = try self.wip.br(end_block);
	1180
	1181	self.wip.cursor = .{ .block = mixed_block };
	1182	_ = try self.wip.br(end_block);
	1183
	1184	self.wip.cursor = .{ .block = both_pl_block };
	1185	const lhs_payload = try self.optPayloadHandle(lhs, scalar_ty, true);
	1186	const rhs_payload = try self.optPayloadHandle(rhs, scalar_ty, true);
	1187	const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload);
	1188	_ = try self.wip.br(end_block);
	1189	const both_pl_block_end = self.wip.cursor.block;
	1190
	1191	self.wip.cursor = .{ .block = end_block };
	1192	const llvm_i1_0 = Builder.Value.false;
	1193	const llvm_i1_1 = Builder.Value.true;
	1194	const incoming_values: [3]Builder.Value = .{
	1195	switch (op) {
	1196	.eq => llvm_i1_1,
	1197	.neq => llvm_i1_0,
	1198	else => unreachable,
	1199	},
	1200	switch (op) {
	1201	.eq => llvm_i1_0,
	1202	.neq => llvm_i1_1,
	1203	else => unreachable,
	1204	},
	1205	payload_cmp,
	1206	};
	1207
	1208	const phi = try self.wip.phi(.i1, "");
	1209	phi.finish(
	1210	&incoming_values,
	1211	&.{ both_null_block, mixed_block, both_pl_block_end },
	1212	&self.wip,
	1213	);
	1214	return phi.toValue();
	1215	},
	1216	.float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }),
	1217	.@"struct", .@"union" => scalar_ty.bitpackBackingInt(zcu),
	1218	else => unreachable,
	1219	};
	1220	const is_signed = int_ty.isSignedInt(zcu);
	1221	const cond: Builder.IntegerCondition = switch (op) {
	1222	.eq => .eq,
	1223	.neq => .ne,
	1224	.lt => if (is_signed) .slt else .ult,
	1225	.lte => if (is_signed) .sle else .ule,
	1226	.gt => if (is_signed) .sgt else .ugt,
	1227	.gte => if (is_signed) .sge else .uge,
	1228	};
	1229	return self.wip.icmp(cond, lhs, rhs, "");
	1230	}
	1231
	1232	fn lowerBlock(
	1233	self: *FuncGen,
	1234	inst: Air.Inst.Index,
	1235	maybe_inline_func: ?InternPool.Index,
	1236	body: []const Air.Inst.Index,
	1237	) TodoError!Builder.Value {
	1238	const o = self.object;
	1239	const zcu = o.zcu;
	1240	const inst_ty = self.typeOfIndex(inst);
	1241
	1242	if (inst_ty.isNoReturn(zcu)) {
	1243	try self.genBodyDebugScope(maybe_inline_func, body, .none);
	1244	return .none;
	1245	}
	1246
	1247	const have_block_result = inst_ty.hasRuntimeBits(zcu);
	1248
	1249	var breaks: BreakList = if (have_block_result) .{ .list = .{} } else .{ .len = 0 };
	1250	defer if (have_block_result) breaks.list.deinit(self.gpa);
	1251
	1252	const parent_bb = try self.wip.block(0, "Block");
	1253	try self.blocks.putNoClobber(self.gpa, inst, .{
	1254	.parent_bb = parent_bb,
	1255	.breaks = &breaks,
	1256	});
	1257	defer assert(self.blocks.remove(inst));
	1258
	1259	try self.genBodyDebugScope(maybe_inline_func, body, .none);
	1260
	1261	self.wip.cursor = .{ .block = parent_bb };
	1262
	1263	// Create a phi node only if the block returns a value.
	1264	if (have_block_result) {
	1265	const raw_llvm_ty = try o.lowerType(inst_ty);
	1266	const llvm_ty: Builder.Type = ty: {
	1267	// If the zig tag type is a function, this represents an actual function body; not
	1268	// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
	1269	// of function pointers, however the phi makes it a runtime value and therefore
	1270	// the LLVM type has to be wrapped in a pointer.
	1271	if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
	1272	break :ty .ptr;
	1273	}
	1274	break :ty raw_llvm_ty;
	1275	};
	1276
	1277	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
	1278	const phi = try self.wip.phi(llvm_ty, "");
	1279	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
	1280	return phi.toValue();
	1281	} else {
	1282	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.len);
	1283	return .none;
	1284	}
	1285	}
	1286
	1287	fn airBr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	1288	const zcu = self.object.zcu;
	1289	const branch = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
	1290	const block = self.blocks.get(branch.block_inst).?;
	1291
	1292	// Add the values to the lists only if the break provides a value.
	1293	const operand_ty = self.typeOf(branch.operand);
	1294	if (operand_ty.hasRuntimeBits(zcu)) {
	1295	const val = try self.resolveInst(branch.operand);
	1296
	1297	// For the phi node, we need the basic blocks and the values of the
	1298	// break instructions.
	1299	try block.breaks.list.append(self.gpa, .{ .bb = self.wip.cursor.block, .val = val });
	1300	} else block.breaks.len += 1;
	1301	_ = try self.wip.br(block.parent_bb);
	1302	}
	1303
	1304	fn airRepeat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	1305	const repeat = self.air.instructions.items(.data)[@intFromEnum(inst)].repeat;
	1306	const loop_bb = self.loops.get(repeat.loop_inst).?;
	1307	loop_bb.ptr(&self.wip).incoming += 1;
	1308	_ = try self.wip.br(loop_bb);
	1309	}
	1310
	1311	fn lowerSwitchDispatch(
	1312	self: *FuncGen,
	1313	switch_inst: Air.Inst.Index,
	1314	cond_ref: Air.Inst.Ref,
	1315	dispatch_info: SwitchDispatchInfo,
	1316	) Allocator.Error!void {
	1317	const o = self.object;
	1318	const zcu = o.zcu;
	1319	const cond_ty = self.typeOf(cond_ref);
	1320	const switch_br = self.air.unwrapSwitch(switch_inst);
	1321
	1322	if (cond_ref.toInterned()) \|cond_ip_index\| {
	1323	const cond_val: Value = .fromInterned(cond_ip_index);
	1324	// Comptime-known dispatch. Iterate the cases to find the correct
	1325	// one, and branch to the corresponding element of `case_blocks`.
	1326	var it = switch_br.iterateCases();
	1327	const target_case_idx = target: while (it.next()) \|case\| {
	1328	for (case.items) \|item\| {
	1329	const val = Value.fromInterned(item.toInterned().?);
	1330	if (cond_val.compareHetero(.eq, val, zcu)) break :target case.idx;
	1331	}
	1332	for (case.ranges) \|range\| {
	1333	const low = Value.fromInterned(range[0].toInterned().?);
	1334	const high = Value.fromInterned(range[1].toInterned().?);
	1335	if (cond_val.compareHetero(.gte, low, zcu) and
	1336	cond_val.compareHetero(.lte, high, zcu))
	1337	{
	1338	break :target case.idx;
	1339	}
	1340	}
	1341	} else dispatch_info.case_blocks.len - 1;
	1342	const target_block = dispatch_info.case_blocks[target_case_idx];
	1343	target_block.ptr(&self.wip).incoming += 1;
	1344	_ = try self.wip.br(target_block);
	1345	return;
	1346	}
	1347
	1348	// Runtime-known dispatch.
	1349	const cond = try self.resolveInst(cond_ref);
	1350
	1351	if (dispatch_info.jmp_table) \|jmp_table\| {
	1352	// We should use the constructed jump table.
	1353	// First, check the bounds to branch to the `else` case if needed.
	1354	const inbounds = try self.wip.bin(
	1355	.@"and",
	1356	try self.cmp(.normal, .gte, cond_ty, cond, jmp_table.min.toValue()),
	1357	try self.cmp(.normal, .lte, cond_ty, cond, jmp_table.max.toValue()),
	1358	"",
	1359	);
	1360	const jmp_table_block = try self.wip.block(1, "Then");
	1361	const else_block = dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1];
	1362	else_block.ptr(&self.wip).incoming += 1;
	1363	_ = try self.wip.brCond(inbounds, jmp_table_block, else_block, switch (jmp_table.in_bounds_hint) {
	1364	.none => .none,
	1365	.unpredictable => .unpredictable,
	1366	.likely => .then_likely,
	1367	.unlikely => .else_likely,
	1368	});
	1369
	1370	self.wip.cursor = .{ .block = jmp_table_block };
	1371
	1372	// Figure out the list of blocks we might branch to.
	1373	// This includes all case blocks, but it might not include the `else` block if
	1374	// the table is dense.
	1375	const target_blocks_len = dispatch_info.case_blocks.len - @intFromBool(!jmp_table.table_includes_else);
	1376	const target_blocks = dispatch_info.case_blocks[0..target_blocks_len];
	1377
	1378	// Make sure to cast the index to a usize so it's not treated as negative!
	1379	const table_index = try self.wip.conv(
	1380	.unsigned,
	1381	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
	1382	try o.lowerType(.usize),
	1383	"",
	1384	);
	1385	const target_ptr_ptr = try self.ptraddScaled(
	1386	jmp_table.table.toValue(),
	1387	table_index,
	1388	Type.usize.abiSize(zcu),
	1389	);
	1390	const target_ptr = try self.wip.load(.normal, .ptr, target_ptr_ptr, .default, "");
	1391
	1392	// Do the branch!
	1393	_ = try self.wip.indirectbr(target_ptr, target_blocks);
	1394
	1395	// Mark all target blocks as having one more incoming branch.
	1396	for (target_blocks) \|case_block\| {
	1397	case_block.ptr(&self.wip).incoming += 1;
	1398	}
	1399
	1400	return;
	1401	}
	1402
	1403	// We must lower to an actual LLVM `switch` instruction.
	1404	// The switch prongs will correspond to our scalar cases. Ranges will
	1405	// be handled by conditional branches in the `else` prong.
	1406
	1407	const llvm_usize = try o.lowerType(.usize);
	1408	const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
	1409	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
	1410	else
	1411	cond;
	1412
	1413	const llvm_cases_len, const last_range_case = info: {
	1414	var llvm_cases_len: u32 = 0;
	1415	var last_range_case: ?u32 = null;
	1416	var it = switch_br.iterateCases();
	1417	while (it.next()) \|case\| {
	1418	if (case.ranges.len > 0) last_range_case = case.idx;
	1419	llvm_cases_len += @intCast(case.items.len);
	1420	}
	1421	break :info .{ llvm_cases_len, last_range_case };
	1422	};
	1423
	1424	// The `else` of the LLVM `switch` is the actual `else` prong only
	1425	// if there are no ranges. Otherwise, the `else` will have a
	1426	// conditional chain before the "true" `else` prong.
	1427	const llvm_else_block = if (last_range_case == null)
	1428	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
	1429	else
	1430	try self.wip.block(0, "RangeTest");
	1431
	1432	llvm_else_block.ptr(&self.wip).incoming += 1;
	1433
	1434	var wip_switch = try self.wip.@"switch"(cond_int, llvm_else_block, llvm_cases_len, dispatch_info.switch_weights);
	1435	defer wip_switch.finish(&self.wip);
	1436
	1437	// Construct the actual cases. Set the cursor to the `else` block so
	1438	// we can construct ranges at the same time as scalar cases.
	1439	self.wip.cursor = .{ .block = llvm_else_block };
	1440
	1441	var it = switch_br.iterateCases();
	1442	while (it.next()) \|case\| {
	1443	const case_block = dispatch_info.case_blocks[case.idx];
	1444
	1445	for (case.items) \|item\| {
	1446	const llvm_item = (try self.resolveInst(item)).toConst().?;
	1447	const llvm_int_item = if (cond_ty.zigTypeTag(zcu) == .pointer)
	1448	try o.builder.castConst(.ptrtoint, llvm_item, llvm_usize)
	1449	else
	1450	llvm_item;
	1451	try wip_switch.addCase(llvm_int_item, case_block, &self.wip);
	1452	}
	1453	case_block.ptr(&self.wip).incoming += @intCast(case.items.len);
	1454
	1455	if (case.ranges.len == 0) continue;
	1456
	1457	// Add a conditional for the ranges, directing to the relevant bb.
	1458	// We don't need to consider `cold` branch hints since that information is stored
	1459	// in the target bb body, but we do care about likely/unlikely/unpredictable.
	1460
	1461	const hint = switch_br.getHint(case.idx);
	1462
	1463	var range_cond: ?Builder.Value = null;
	1464	for (case.ranges) \|range\| {
	1465	const llvm_min = try self.resolveInst(range[0]);
	1466	const llvm_max = try self.resolveInst(range[1]);
	1467	const cond_part = try self.wip.bin(
	1468	.@"and",
	1469	try self.cmp(.normal, .gte, cond_ty, cond, llvm_min),
	1470	try self.cmp(.normal, .lte, cond_ty, cond, llvm_max),
	1471	"",
	1472	);
	1473	if (range_cond) \|prev\| {
	1474	range_cond = try self.wip.bin(.@"or", prev, cond_part, "");
	1475	} else range_cond = cond_part;
	1476	}
	1477
	1478	// If the check fails, we either branch to the "true" `else` case,
	1479	// or to the next range condition.
	1480	const range_else_block = if (case.idx == last_range_case.?)
	1481	dispatch_info.case_blocks[dispatch_info.case_blocks.len - 1]
	1482	else
	1483	try self.wip.block(0, "RangeTest");
	1484
	1485	_ = try self.wip.brCond(range_cond.?, case_block, range_else_block, switch (hint) {
	1486	.none, .cold => .none,
	1487	.unpredictable => .unpredictable,
	1488	.likely => .then_likely,
	1489	.unlikely => .else_likely,
	1490	});
	1491	case_block.ptr(&self.wip).incoming += 1;
	1492	range_else_block.ptr(&self.wip).incoming += 1;
	1493
	1494	// Construct the next range conditional (if any) in the false branch.
	1495	self.wip.cursor = .{ .block = range_else_block };
	1496	}
	1497	}
	1498
	1499	fn airSwitchDispatch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	1500	const br = self.air.instructions.items(.data)[@intFromEnum(inst)].br;
	1501	const dispatch_info = self.switch_dispatch_info.get(br.block_inst).?;
	1502	return self.lowerSwitchDispatch(br.block_inst, br.operand, dispatch_info);
	1503	}
	1504
	1505	fn airCondBr(self: *FuncGen, inst: Air.Inst.Index) TodoError!void {
	1506	const cond_br = self.air.unwrapCondBr(inst);
	1507	const cond = try self.resolveInst(cond_br.condition);
	1508	const then_body = cond_br.then_body;
	1509	const else_body = cond_br.else_body;
	1510
	1511	const Hint = enum {
	1512	none,
	1513	unpredictable,
	1514	then_likely,
	1515	else_likely,
	1516	then_cold,
	1517	else_cold,
	1518	};
	1519	const hint: Hint = switch (cond_br.branch_hints.true) {
	1520	.none => switch (cond_br.branch_hints.false) {
	1521	.none => .none,
	1522	.likely => .else_likely,
	1523	.unlikely => .then_likely,
	1524	.cold => .else_cold,
	1525	.unpredictable => .unpredictable,
	1526	},
	1527	.likely => switch (cond_br.branch_hints.false) {
	1528	.none => .then_likely,
	1529	.likely => .unpredictable,
	1530	.unlikely => .then_likely,
	1531	.cold => .else_cold,
	1532	.unpredictable => .unpredictable,
	1533	},
	1534	.unlikely => switch (cond_br.branch_hints.false) {
	1535	.none => .else_likely,
	1536	.likely => .else_likely,
	1537	.unlikely => .unpredictable,
	1538	.cold => .else_cold,
	1539	.unpredictable => .unpredictable,
	1540	},
	1541	.cold => .then_cold,
	1542	.unpredictable => .unpredictable,
	1543	};
	1544
	1545	const then_block = try self.wip.block(1, "Then");
	1546	const else_block = try self.wip.block(1, "Else");
	1547	_ = try self.wip.brCond(cond, then_block, else_block, switch (hint) {
	1548	.none, .then_cold, .else_cold => .none,
	1549	.unpredictable => .unpredictable,
	1550	.then_likely => .then_likely,
	1551	.else_likely => .else_likely,
	1552	});
	1553
	1554	self.wip.cursor = .{ .block = then_block };
	1555	if (hint == .then_cold) _ = try self.wip.callIntrinsicAssumeCold();
	1556	try self.genBodyDebugScope(null, then_body, cond_br.branch_hints.then_cov);
	1557
	1558	self.wip.cursor = .{ .block = else_block };
	1559	if (hint == .else_cold) _ = try self.wip.callIntrinsicAssumeCold();
	1560	try self.genBodyDebugScope(null, else_body, cond_br.branch_hints.else_cov);
	1561
	1562	// No need to reset the insert cursor since this instruction is noreturn.
	1563	}
	1564
	1565	fn airTry(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value {
	1566	const unwrapped_try = self.air.unwrapTry(inst);
	1567	const err_union = try self.resolveInst(unwrapped_try.error_union);
	1568	const body = unwrapped_try.else_body;
	1569	const err_union_ty = self.typeOf(unwrapped_try.error_union);
	1570	const is_unused = self.liveness.isUnused(inst);
	1571	return lowerTry(self, err_union, body, err_union_ty, false, .none, is_unused, err_cold);
	1572	}
	1573
	1574	fn airTryPtr(self: *FuncGen, inst: Air.Inst.Index, err_cold: bool) TodoError!Builder.Value {
	1575	const zcu = self.object.zcu;
	1576	const unwrapped_try = self.air.unwrapTryPtr(inst);
	1577	const err_union_ptr = try self.resolveInst(unwrapped_try.error_union_ptr);
	1578	const body = unwrapped_try.else_body;
	1579	const err_union_ptr_ty = self.typeOf(unwrapped_try.error_union_ptr);
	1580	const err_union_ty = err_union_ptr_ty.childType(zcu);
	1581	const is_unused = self.liveness.isUnused(inst);
	1582
	1583	self.maybeMarkAllowZeroAccess(self.typeOf(unwrapped_try.error_union_ptr).ptrInfo(zcu));
	1584
	1585	return lowerTry(self, err_union_ptr, body, err_union_ty, true, err_union_ptr_ty.ptrAlignment(zcu), is_unused, err_cold);
	1586	}
	1587
	1588	fn lowerTry(
	1589	fg: *FuncGen,
	1590	err_union: Builder.Value,
	1591	body: []const Air.Inst.Index,
	1592	err_union_ty: Type,
	1593	operand_is_ptr: bool,
	1594	operand_ptr_align: InternPool.Alignment,
	1595	is_unused: bool,
	1596	err_cold: bool,
	1597	) TodoError!Builder.Value {
	1598	const o = fg.object;
	1599	const zcu = o.zcu;
	1600	const payload_ty = err_union_ty.errorUnionPayload(zcu);
	1601	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
	1602	const error_type = try o.errorIntType();
	1603
	1604	const err_set_align: InternPool.Alignment, const payload_align: InternPool.Alignment = if (operand_is_ptr) .{
	1605	operand_ptr_align.minStrict(Type.anyerror.abiAlignment(zcu)),
	1606	operand_ptr_align.minStrict(payload_ty.abiAlignment(zcu)),
	1607	} else .{ .none, .none };
	1608
	1609	if (!err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
	1610	const loaded = loaded: {
	1611	const access_kind: Builder.MemoryAccessKind =
	1612	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	1613
	1614	if (!payload_has_bits) {
	1615	break :loaded if (operand_is_ptr)
	1616	try fg.wip.load(access_kind, error_type, err_union, err_set_align.toLlvm(), "")
	1617	else
	1618	err_union;
	1619	}
	1620
	1621	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
	1622	const offset = codegen.errUnionErrorOffset(payload_ty, zcu);
	1623	const err_field_ptr = try fg.ptraddConst(err_union, offset);
	1624	break :loaded try fg.wip.load(
	1625	if (operand_is_ptr) access_kind else .normal,
	1626	error_type,
	1627	err_field_ptr,
	1628	err_set_align.toLlvm(),
	1629	"",
	1630	);
	1631	};
	1632	const zero = try o.builder.intValue(error_type, 0);
	1633	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
	1634
	1635	const return_block = try fg.wip.block(1, "TryRet");
	1636	const continue_block = try fg.wip.block(1, "TryCont");
	1637	_ = try fg.wip.brCond(is_err, return_block, continue_block, if (err_cold) .none else .else_likely);
	1638
	1639	fg.wip.cursor = .{ .block = return_block };
	1640	if (err_cold) _ = try fg.wip.callIntrinsicAssumeCold();
	1641	try fg.genBodyDebugScope(null, body, .poi);
	1642
	1643	fg.wip.cursor = .{ .block = continue_block };
	1644	}
	1645	if (is_unused) return .none;
	1646	if (!payload_has_bits) return if (operand_is_ptr) err_union else .none;
	1647	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload has no bits
	1648	const payload_ptr = try fg.ptraddConst(err_union, codegen.errUnionPayloadOffset(payload_ty, zcu));
	1649	if (operand_is_ptr) {
	1650	return payload_ptr;
	1651	} else if (isByRef(payload_ty, zcu)) {
	1652	return fg.loadByRef(payload_ptr, payload_ty, payload_align.toLlvm(), .normal);
	1653	} else {
	1654	return fg.wip.load(.normal, try o.lowerType(payload_ty), payload_ptr, payload_align.toLlvm(), "");
	1655	}
	1656	}
	1657
	1658	fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) TodoError!void {
	1659	const o = self.object;
	1660	const zcu = o.zcu;
	1661
	1662	const switch_br = self.air.unwrapSwitch(inst);
	1663
	1664	// For `loop_switch_br`, we need these BBs prepared ahead of time to generate dispatches.
	1665	// For `switch_br`, they allow us to sometimes generate better IR by sharing a BB between
	1666	// scalar and range cases in the same prong.
	1667	// +1 for `else` case. This is not the same as the LLVM `else` prong, as that may first contain
	1668	// conditionals to handle ranges.
	1669	const case_blocks = try self.gpa.alloc(Builder.Function.Block.Index, switch_br.cases_len + 1);
	1670	defer self.gpa.free(case_blocks);
	1671	// We set incoming as 0 for now, and increment it as we construct dispatches.
	1672	for (case_blocks[0 .. case_blocks.len - 1]) \|b\| b. = try self.wip.block(0, "Case");
	1673	case_blocks[case_blocks.len - 1] = try self.wip.block(0, "Default");
	1674
	1675	// There's a special case here to manually generate a jump table in some cases.
	1676	//
	1677	// Labeled switch in Zig is intended to follow the "direct threading" pattern. We would ideally use a jump
	1678	// table, and each `continue` has its own indirect `jmp`, to allow the branch predictor to more accurately
	1679	// use data patterns to predict future dispatches. The problem, however, is that LLVM emits fascinatingly
	1680	// bad asm for this. Not only does it not share the jump table -- which we really need it to do to prevent
	1681	// destroying the cache -- but it also actually generates slightly different jump tables for each case,
	1682	// and a separate conditional branch beforehand to handle dispatching back to the case we're currently
	1683	// within(!!).
	1684	//
	1685	// This asm is really, really, not what we want. As such, we will construct the jump table manually where
	1686	// appropriate (the values are dense and relatively few), and use it when lowering dispatches.
	1687
	1688	const jmp_table: ?SwitchDispatchInfo.JmpTable = jmp_table: {
	1689	if (!is_dispatch_loop) break :jmp_table null;
	1690
	1691	// Workaround for:
	1692	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/lib/MC/WasmObjectWriter.cpp#L560
	1693	// * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll
	1694	if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null;
	1695
	1696	// On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just
	1697	// about acceptable - it won't fill L1d cache on most CPUs.
	1698	const max_table_len = 1024;
	1699
	1700	const cond_ty = self.typeOf(switch_br.operand);
	1701	switch (cond_ty.zigTypeTag(zcu)) {
	1702	.bool, .pointer => break :jmp_table null,
	1703	.@"enum", .int, .error_set, .@"struct", .@"union" => {},
	1704	else => unreachable,
	1705	}
	1706
	1707	if (cond_ty.intInfo(zcu).signedness == .signed) break :jmp_table null;
	1708
	1709	// Don't worry about the size of the type -- it's irrelevant, because the prong values could be fairly dense.
	1710	// If they are, then we will construct a jump table.
	1711	const min, const max = self.switchCaseItemRange(switch_br) orelse break :jmp_table null;
	1712	const min_int = min.getUnsignedInt(zcu) orelse break :jmp_table null;
	1713	const max_int = max.getUnsignedInt(zcu) orelse break :jmp_table null;
	1714	const table_len = max_int - min_int + 1;
	1715	if (table_len > max_table_len) break :jmp_table null;
	1716
	1717	const table_elems = try self.gpa.alloc(Builder.Constant, @intCast(table_len));
	1718	defer self.gpa.free(table_elems);
	1719
	1720	// Set them all to the `else` branch, then iterate over the AIR switch
	1721	// and replace all values which correspond to other prongs.
	1722	@memset(table_elems, try o.builder.blockAddrConst(
	1723	self.wip.function,
	1724	case_blocks[case_blocks.len - 1],
	1725	));
	1726	var item_count: u32 = 0;
	1727	var it = switch_br.iterateCases();
	1728	while (it.next()) \|case\| {
	1729	const case_block = case_blocks[case.idx];
	1730	const case_block_addr = try o.builder.blockAddrConst(
	1731	self.wip.function,
	1732	case_block,
	1733	);
	1734	for (case.items) \|item\| {
	1735	const val = Value.fromInterned(item.toInterned().?);
	1736	const table_idx = val.toUnsignedInt(zcu) - min_int;
	1737	table_elems[@intCast(table_idx)] = case_block_addr;
	1738	item_count += 1;
	1739	}
	1740	for (case.ranges) \|range\| {
	1741	const low = Value.fromInterned(range[0].toInterned().?);
	1742	const high = Value.fromInterned(range[1].toInterned().?);
	1743	const low_idx = low.toUnsignedInt(zcu) - min_int;
	1744	const high_idx = high.toUnsignedInt(zcu) - min_int;
	1745	@memset(table_elems[@intCast(low_idx)..@intCast(high_idx + 1)], case_block_addr);
	1746	item_count += @intCast(high_idx + 1 - low_idx);
	1747	}
	1748	}
	1749
	1750	const table_llvm_ty = try o.builder.arrayType(table_elems.len, .ptr);
	1751	const table_val = try o.builder.arrayConst(table_llvm_ty, table_elems);
	1752
	1753	const table_variable = try o.builder.addVariable(
	1754	try o.builder.strtabStringFmt("__jmptab_{d}", .{@intFromEnum(inst)}),
	1755	table_llvm_ty,
	1756	.default,
	1757	);
	1758	try table_variable.setInitializer(table_val, &o.builder);
	1759	const table_global = table_variable.ptrConst(&o.builder).global;
	1760	table_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
	1761	table_global.setUnnamedAddr(.unnamed_addr, &o.builder);
	1762
	1763	const table_includes_else = item_count != table_len;
	1764
	1765	break :jmp_table .{
	1766	.min = try o.lowerValue(min.toIntern()),
	1767	.max = try o.lowerValue(max.toIntern()),
	1768	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
	1769	.none, .cold => .none,
	1770	.unpredictable => .unpredictable,
	1771	.likely => .likely,
	1772	.unlikely => .unlikely,
	1773	},
	1774	.table = table_global.toConst(),
	1775	.table_includes_else = table_includes_else,
	1776	};
	1777	};
	1778
	1779	const weights: Builder.Function.Instruction.BrCond.Weights = weights: {
	1780	if (jmp_table != null) break :weights .none; // not used
	1781
	1782	// First pass. If any weights are `.unpredictable`, unpredictable.
	1783	// If all are `.none` or `.cold`, none.
	1784	var any_likely = false;
	1785	for (0..switch_br.cases_len) \|case_idx\| {
	1786	switch (switch_br.getHint(@intCast(case_idx))) {
	1787	.none, .cold => {},
	1788	.likely, .unlikely => any_likely = true,
	1789	.unpredictable => break :weights .unpredictable,
	1790	}
	1791	}
	1792	switch (switch_br.getElseHint()) {
	1793	.none, .cold => {},
	1794	.likely, .unlikely => any_likely = true,
	1795	.unpredictable => break :weights .unpredictable,
	1796	}
	1797	if (!any_likely) break :weights .none;
	1798
	1799	const llvm_cases_len = llvm_cases_len: {
	1800	var len: u32 = 0;
	1801	var it = switch_br.iterateCases();
	1802	while (it.next()) \|case\| len += @intCast(case.items.len);
	1803	break :llvm_cases_len len;
	1804	};
	1805
	1806	var weights = try self.gpa.alloc(Builder.Metadata, 1 + llvm_cases_len + 1);
	1807	defer self.gpa.free(weights);
	1808	var weight_idx: usize = 0;
	1809
	1810	const branch_weights_str = try o.builder.metadataString("branch_weights");
	1811	weights[weight_idx] = branch_weights_str.toMetadata();
	1812	weight_idx += 1;
	1813
	1814	const else_weight: u32 = switch (switch_br.getElseHint()) {
	1815	.unpredictable => unreachable,
	1816	.none, .cold => 1000,
	1817	.likely => 2000,
	1818	.unlikely => 1,
	1819	};
	1820	weights[weight_idx] = try o.builder.metadataConstant(try o.builder.intConst(.i32, else_weight));
	1821	weight_idx += 1;
	1822
	1823	var it = switch_br.iterateCases();
	1824	while (it.next()) \|case\| {
	1825	const weight_val: u32 = switch (switch_br.getHint(case.idx)) {
	1826	.unpredictable => unreachable,
	1827	.none, .cold => 1000,
	1828	.likely => 2000,
	1829	.unlikely => 1,
	1830	};
	1831	const weight_meta = try o.builder.metadataConstant(try o.builder.intConst(.i32, weight_val));
	1832	@memset(weights[weight_idx..][0..case.items.len], weight_meta);
	1833	weight_idx += case.items.len;
	1834	}
	1835
	1836	assert(weight_idx == weights.len);
	1837	break :weights .fromMetadata(try o.builder.metadataTuple(weights));
	1838	};
	1839
	1840	const dispatch_info: SwitchDispatchInfo = .{
	1841	.case_blocks = case_blocks,
	1842	.switch_weights = weights,
	1843	.jmp_table = jmp_table,
	1844	};
	1845
	1846	if (is_dispatch_loop) {
	1847	try self.switch_dispatch_info.putNoClobber(self.gpa, inst, dispatch_info);
	1848	}
	1849	defer if (is_dispatch_loop) {
	1850	assert(self.switch_dispatch_info.remove(inst));
	1851	};
	1852
	1853	// Generate the initial dispatch.
	1854	// If this is a simple `switch_br`, this is the only dispatch.
	1855	try self.lowerSwitchDispatch(inst, switch_br.operand, dispatch_info);
	1856
	1857	// Iterate the cases and generate their bodies.
	1858	var it = switch_br.iterateCases();
	1859	while (it.next()) \|case\| {
	1860	const case_block = case_blocks[case.idx];
	1861	self.wip.cursor = .{ .block = case_block };
	1862	if (switch_br.getHint(case.idx) == .cold) _ = try self.wip.callIntrinsicAssumeCold();
	1863	try self.genBodyDebugScope(null, case.body, .none);
	1864	}
	1865	self.wip.cursor = .{ .block = case_blocks[case_blocks.len - 1] };
	1866	const else_body = it.elseBody();
	1867	if (switch_br.getElseHint() == .cold) _ = try self.wip.callIntrinsicAssumeCold();
	1868	if (else_body.len > 0) {
	1869	try self.genBodyDebugScope(null, it.elseBody(), .none);
	1870	} else {
	1871	_ = try self.wip.@"unreachable"();
	1872	}
	1873	}
	1874
	1875	fn switchCaseItemRange(self: *FuncGen, switch_br: Air.UnwrappedSwitch) ?[2]Value {
	1876	const zcu = self.object.zcu;
	1877	var it = switch_br.iterateCases();
	1878	var min: ?Value = null;
	1879	var max: ?Value = null;
	1880	while (it.next()) \|case\| {
	1881	for (case.items) \|item\| {
	1882	const val = Value.fromInterned(item.toInterned().?);
	1883	const low = if (min) \|m\| val.compareHetero(.lt, m, zcu) else true;
	1884	const high = if (max) \|m\| val.compareHetero(.gt, m, zcu) else true;
	1885	if (low) min = val;
	1886	if (high) max = val;
	1887	}
	1888	for (case.ranges) \|range\| {
	1889	const vals: [2]Value = .{
	1890	Value.fromInterned(range[0].toInterned().?),
	1891	Value.fromInterned(range[1].toInterned().?),
	1892	};
	1893	const low = if (min) \|m\| vals[0].compareHetero(.lt, m, zcu) else true;
	1894	const high = if (max) \|m\| vals[1].compareHetero(.gt, m, zcu) else true;
	1895	if (low) min = vals[0];
	1896	if (high) max = vals[1];
	1897	}
	1898	}
	1899	if (min == null) {
	1900	assert(max == null);
	1901	return null;
	1902	}
	1903	return .{ min.?, max.? };
	1904	}
	1905
	1906	fn airLoop(self: *FuncGen, inst: Air.Inst.Index) TodoError!void {
	1907	const block = self.air.unwrapBlock(inst);
	1908	const body = block.body;
	1909	const loop_block = try self.wip.block(1, "Loop"); // `airRepeat` will increment incoming each time
	1910	_ = try self.wip.br(loop_block);
	1911
	1912	try self.loops.putNoClobber(self.gpa, inst, loop_block);
	1913	defer assert(self.loops.remove(inst));
	1914
	1915	self.wip.cursor = .{ .block = loop_block };
	1916	try self.genBodyDebugScope(null, body, .none);
	1917	}
	1918
	1919	fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	1920	const o = self.object;
	1921	const zcu = o.zcu;
	1922	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	1923	const operand_ty = self.typeOf(ty_op.operand);
	1924	const array_ty = operand_ty.childType(zcu);
	1925	const llvm_usize = try o.lowerType(.usize);
	1926	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
	1927	const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst));
	1928	const operand = try self.resolveInst(ty_op.operand);
	1929	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
	1930	}
	1931
	1932	fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
	1933	const o = self.object;
	1934	const zcu = o.zcu;
	1935	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	1936
	1937	const operand = try self.resolveInst(ty_op.operand);
	1938	const operand_ty = self.typeOf(ty_op.operand);
	1939	const operand_scalar_ty = operand_ty.scalarType(zcu);
	1940	const is_signed_int = operand_scalar_ty.isSignedInt(zcu);
	1941
	1942	const dest_ty = self.typeOfIndex(inst);
	1943	const dest_scalar_ty = dest_ty.scalarType(zcu);
	1944	const dest_llvm_ty = try o.lowerType(dest_ty);
	1945	const target = zcu.getTarget();
	1946
	1947	if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
	1948	if (is_signed_int) .signed else .unsigned,
	1949	operand,
	1950	dest_llvm_ty,
	1951	"",
	1952	);
	1953
	1954	const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
	1955	return self.todo("float_from_int on {d} bit integer", .{operand_scalar_ty.bitSize(zcu)});
	1956	};
	1957	const rt_int_ty = try o.builder.intType(rt_int_bits);
	1958	var extended = try self.wip.conv(
	1959	if (is_signed_int) .signed else .unsigned,
	1960	operand,
	1961	rt_int_ty,
	1962	"",
	1963	);
	1964	const dest_bits = dest_scalar_ty.floatBits(target);
	1965	const compiler_rt_operand_abbrev = compilerRtIntAbbrev(rt_int_bits);
	1966	const compiler_rt_dest_abbrev = compilerRtFloatAbbrev(dest_bits);
	1967	const sign_prefix = if (is_signed_int) "" else "un";
	1968	const fn_name = try o.builder.strtabStringFmt("__float{s}{s}i{s}f", .{
	1969	sign_prefix,
	1970	compiler_rt_operand_abbrev,
	1971	compiler_rt_dest_abbrev,
	1972	});
	1973
	1974	var param_type = rt_int_ty;
	1975	if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) {
	1976	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
	1977	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
	1978	param_type = try o.builder.vectorType(.normal, 2, .i64);
	1979	extended = try self.wip.cast(.bitcast, extended, param_type, "");
	1980	}
	1981
	1982	const libc_fn = try o.getLibcFunction(fn_name, &.{param_type}, dest_llvm_ty);
	1983	return self.wip.call(
	1984	.normal,
	1985	.ccc,
	1986	.none,
	1987	libc_fn.typeOf(&o.builder),
	1988	libc_fn.toValue(&o.builder),
	1989	&.{extended},
	1990	"",
	1991	);
	1992	}
	1993
	1994	fn airIntFromFloat(
	1995	self: *FuncGen,
	1996	inst: Air.Inst.Index,
	1997	fast: Builder.FastMathKind,
	1998	) TodoError!Builder.Value {
	1999	_ = fast;
	2000
	2001	const o = self.object;
	2002	const zcu = o.zcu;
	2003	const target = zcu.getTarget();
	2004	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2005
	2006	const operand = try self.resolveInst(ty_op.operand);
	2007	const operand_ty = self.typeOf(ty_op.operand);
	2008	const operand_scalar_ty = operand_ty.scalarType(zcu);
	2009
	2010	const dest_ty = self.typeOfIndex(inst);
	2011	const dest_scalar_ty = dest_ty.scalarType(zcu);
	2012	const dest_llvm_ty = try o.lowerType(dest_ty);
	2013
	2014	if (intrinsicsAllowed(operand_scalar_ty, target)) {
	2015	// TODO set fast math flag
	2016	return self.wip.conv(
	2017	if (dest_scalar_ty.isSignedInt(zcu)) .signed else .unsigned,
	2018	operand,
	2019	dest_llvm_ty,
	2020	"",
	2021	);
	2022	}
	2023
	2024	const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
	2025	return self.todo("int_from_float to {d} bit integer", .{dest_scalar_ty.bitSize(zcu)});
	2026	};
	2027	const ret_ty = try o.builder.intType(rt_int_bits);
	2028	const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
	2029	// On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
	2030	// i128 calling convention to adhere to the ABI that LLVM expects compiler-rt to have.
	2031	break :b try o.builder.vectorType(.normal, 2, .i64);
	2032	} else ret_ty;
	2033
	2034	const operand_bits = operand_scalar_ty.floatBits(target);
	2035	const compiler_rt_operand_abbrev = compilerRtFloatAbbrev(operand_bits);
	2036
	2037	const compiler_rt_dest_abbrev = compilerRtIntAbbrev(rt_int_bits);
	2038	const sign_prefix = if (dest_scalar_ty.isSignedInt(zcu)) "" else "uns";
	2039
	2040	const fn_name = try o.builder.strtabStringFmt("__fix{s}{s}f{s}i", .{
	2041	sign_prefix,
	2042	compiler_rt_operand_abbrev,
	2043	compiler_rt_dest_abbrev,
	2044	});
	2045
	2046	const operand_llvm_ty = try o.lowerType(operand_ty);
	2047	const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
	2048	var result = try self.wip.call(
	2049	.normal,
	2050	.ccc,
	2051	.none,
	2052	libc_fn.typeOf(&o.builder),
	2053	libc_fn.toValue(&o.builder),
	2054	&.{operand},
	2055	"",
	2056	);
	2057
	2058	if (libc_ret_ty != ret_ty) result = try self.wip.cast(.bitcast, result, ret_ty, "");
	2059	if (ret_ty != dest_llvm_ty) result = try self.wip.cast(.trunc, result, dest_llvm_ty, "");
	2060	return result;
	2061	}
	2062
	2063	fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
	2064	const zcu = fg.object.zcu;
	2065	return if (ty.isSlice(zcu)) fg.wip.extractValue(ptr, &.{0}, "") else ptr;
	2066	}
	2067
	2068	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
	2069	const o = fg.object;
	2070	const zcu = o.zcu;
	2071	const llvm_usize = try o.lowerType(.usize);
	2072	switch (ty.ptrSize(zcu)) {
	2073	.slice => {
	2074	const len = try fg.wip.extractValue(ptr, &.{1}, "");
	2075	const elem_ty = ty.childType(zcu);
	2076	const abi_size = elem_ty.abiSize(zcu);
	2077	if (abi_size == 1) return len;
	2078	const abi_size_llvm_val = try o.builder.intValue(llvm_usize, abi_size);
	2079	return fg.wip.bin(.@"mul nuw", len, abi_size_llvm_val, "");
	2080	},
	2081	.one => {
	2082	const array_ty = ty.childType(zcu);
	2083	const elem_ty = array_ty.childType(zcu);
	2084	const abi_size = elem_ty.abiSize(zcu);
	2085	return o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu) * abi_size);
	2086	},
	2087	.many, .c => unreachable,
	2088	}
	2089	}
	2090
	2091	fn airSliceField(self: *FuncGen, inst: Air.Inst.Index, index: u32) Allocator.Error!Builder.Value {
	2092	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2093	const operand = try self.resolveInst(ty_op.operand);
	2094	return self.wip.extractValue(operand, &.{index}, "");
	2095	}
	2096
	2097	fn airPtrSliceFieldPtr(self: *FuncGen, inst: Air.Inst.Index, index: u1) Allocator.Error!Builder.Value {
	2098	const zcu = self.object.zcu;
	2099	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2100	const slice_ptr = try self.resolveInst(ty_op.operand);
	2101	return self.ptraddConst(slice_ptr, index * Type.usize.abiSize(zcu));
	2102	}
	2103
	2104	fn airSliceElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2105	const zcu = self.object.zcu;
	2106	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	2107	const slice_ty = self.typeOf(bin_op.lhs);
	2108	const slice = try self.resolveInst(bin_op.lhs);
	2109	const index = try self.resolveInst(bin_op.rhs);
	2110	const slice_info = slice_ty.ptrInfo(zcu);
	2111	assert(slice_info.flags.size == .slice);
	2112	const elem_ty: Type = .fromInterned(slice_info.child);
	2113	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
	2114	const ptr = try self.ptraddScaled(base_ptr, index, elem_ty.abiSize(zcu));
	2115	const elem_align = slice_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu));
	2116	const access_kind: Builder.MemoryAccessKind = if (slice_info.flags.is_volatile) .@"volatile" else .normal;
	2117	self.maybeMarkAllowZeroAccess(slice_info);
	2118	if (isByRef(elem_ty, zcu)) {
	2119	return self.loadByRef(ptr, elem_ty, elem_align.toLlvm(), access_kind);
	2120	} else {
	2121	return self.loadTruncate(access_kind, elem_ty, ptr, elem_align.toLlvm());
	2122	}
	2123	}
	2124
	2125	fn airSliceElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2126	const zcu = self.object.zcu;
	2127	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	2128	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	2129	const slice_ty = self.typeOf(bin_op.lhs);
	2130
	2131	const slice = try self.resolveInst(bin_op.lhs);
	2132	const index = try self.resolveInst(bin_op.rhs);
	2133	const base_ptr = try self.wip.extractValue(slice, &.{0}, "");
	2134	return self.ptraddScaled(base_ptr, index, slice_ty.childType(zcu).abiSize(zcu));
	2135	}
	2136
	2137	fn airArrayElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2138	const zcu = self.object.zcu;
	2139
	2140	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	2141	const array_ty = self.typeOf(bin_op.lhs);
	2142	const array_llvm_val = try self.resolveInst(bin_op.lhs);
	2143	const rhs = try self.resolveInst(bin_op.rhs);
	2144	const elem_ty = array_ty.childType(zcu);
	2145	if (isByRef(array_ty, zcu)) {
	2146	const elem_ptr = try self.ptraddScaled(array_llvm_val, rhs, elem_ty.abiSize(zcu));
	2147	if (isByRef(elem_ty, zcu)) {
	2148	const elem_align = elem_ty.abiAlignment(zcu).toLlvm();
	2149	return self.loadByRef(elem_ptr, elem_ty, elem_align, .normal);
	2150	} else {
	2151	return self.loadTruncate(.normal, elem_ty, elem_ptr, .default);
	2152	}
	2153	}
	2154
	2155	// This branch can be reached for vectors, which are always by-value.
	2156	return self.wip.extractElement(array_llvm_val, rhs, "");
	2157	}
	2158
	2159	fn airPtrElemVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2160	const zcu = self.object.zcu;
	2161	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	2162	const ptr_ty = self.typeOf(bin_op.lhs);
	2163	const elem_ty = ptr_ty.indexableElem(zcu);
	2164	const base_ptr = try self.resolveInst(bin_op.lhs);
	2165	const rhs = try self.resolveInst(bin_op.rhs);
	2166
	2167	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	2168
	2169	return self.load(
	2170	try self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu)),
	2171	elem_ty,
	2172	ptr_ty.ptrAlignment(zcu).min(elem_ty.abiAlignment(zcu)).toLlvm(),
	2173	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	2174	);
	2175	}
	2176
	2177	fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2178	const zcu = self.object.zcu;
	2179	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	2180	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	2181	const ptr_ty = self.typeOf(bin_op.lhs);
	2182	const elem_ty = ptr_ty.indexableElem(zcu);
	2183	assert(elem_ty.hasRuntimeBits(zcu));
	2184
	2185	const base_ptr = try self.resolveInst(bin_op.lhs);
	2186	const rhs = try self.resolveInst(bin_op.rhs);
	2187
	2188	const elem_ptr = ty_pl.ty.toType();
	2189	if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
	2190
	2191	return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
	2192	}
	2193
	2194	fn airStructFieldPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2195	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	2196	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
	2197	const struct_ptr = try self.resolveInst(struct_field.struct_operand);
	2198	const struct_ptr_ty = self.typeOf(struct_field.struct_operand);
	2199	return self.fieldPtr(struct_ptr, struct_ptr_ty, struct_field.field_index);
	2200	}
	2201
	2202	fn airStructFieldPtrIndex(
	2203	self: *FuncGen,
	2204	inst: Air.Inst.Index,
	2205	field_index: u32,
	2206	) Allocator.Error!Builder.Value {
	2207	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2208	const struct_ptr = try self.resolveInst(ty_op.operand);
	2209	const struct_ptr_ty = self.typeOf(ty_op.operand);
	2210	return self.fieldPtr(struct_ptr, struct_ptr_ty, field_index);
	2211	}
	2212
	2213	fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2214	const o = self.object;
	2215	const zcu = o.zcu;
	2216	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	2217	const struct_field = self.air.extraData(Air.StructField, ty_pl.payload).data;
	2218	const struct_ty = self.typeOf(struct_field.struct_operand);
	2219	const struct_llvm_val = try self.resolveInst(struct_field.struct_operand);
	2220	const field_index = struct_field.field_index;
	2221	const field_ty = struct_ty.fieldType(field_index, zcu);
	2222	assert(field_ty.hasRuntimeBits(zcu));
	2223
	2224	if (!isByRef(struct_ty, zcu)) {
	2225	// All auto/extern struct/union types are by-ref, unless they have no runtime bits, in which
	2226	// case we shouldn't be seeing this instruction to begin with. Therefore we must be dealing
	2227	// with a `packed struct` or `packed union`.
	2228	assert(struct_ty.containerLayout(zcu) == .@"packed");
	2229	assert(!isByRef(field_ty, zcu));
	2230	const field_int_val: Builder.Value = switch (struct_ty.zigTypeTag(zcu)) {
	2231	.@"struct" => field_int_val: {
	2232	const llvm_field_int_ty = try o.builder.intType(@intCast(field_ty.bitSize(zcu)));
	2233	const bit_offset = zcu.structPackedFieldBitOffset(
	2234	zcu.intern_pool.loadStructType(struct_ty.toIntern()),
	2235	field_index,
	2236	);
	2237	const shift_bits = try o.builder.intValue(struct_llvm_val.typeOfWip(&self.wip), bit_offset);
	2238	const shifted = try self.wip.bin(.lshr, struct_llvm_val, shift_bits, "");
	2239	break :field_int_val try self.wip.cast(.trunc, shifted, llvm_field_int_ty, "");
	2240	},
	2241	.@"union" => struct_llvm_val,
	2242	else => unreachable,
	2243	};
	2244	switch (field_ty.zigTypeTag(zcu)) {
	2245	else => unreachable, // not packable
	2246	.void => unreachable, // opv bug in sema
	2247	.int, .bool, .@"enum", .@"struct", .@"union" => {
	2248	// Represented as integers, so already done
	2249	return field_int_val;
	2250	},
	2251	.float => {
	2252	// bitcast int->float
	2253	return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), "");
	2254	},
	2255	}
	2256	}
	2257
	2258	const offset: u64 = switch (struct_ty.zigTypeTag(zcu)) {
	2259	.@"struct" => struct_ty.structFieldOffset(field_index, zcu),
	2260	.@"union" => struct_ty.unionGetLayout(zcu).payloadOffset(),
	2261	else => unreachable,
	2262	};
	2263
	2264	const struct_ptr_align = struct_ty.abiAlignment(zcu);
	2265	const field_ptr = try self.ptraddConst(struct_llvm_val, offset);
	2266	const field_ptr_align: InternPool.Alignment = switch (offset) {
	2267	0 => struct_ptr_align,
	2268	else => struct_ptr_align.minStrict(.fromLog2Units(@ctz(offset))),
	2269	};
	2270
	2271	if (isByRef(field_ty, zcu)) {
	2272	return self.loadByRef(field_ptr, field_ty, field_ptr_align.toLlvm(), .normal);
	2273	} else {
	2274	return self.loadTruncate(.normal, field_ty, field_ptr, field_ptr_align.toLlvm());
	2275	}
	2276	}
	2277
	2278	fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2279	const o = self.object;
	2280	const zcu = o.zcu;
	2281	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	2282	const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data;
	2283
	2284	const field_ptr = try self.resolveInst(extra.field_ptr);
	2285
	2286	const parent_ty = ty_pl.ty.toType().childType(zcu);
	2287	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
	2288	if (field_offset == 0) return field_ptr;
	2289
	2290	const res_ty = try o.lowerType(ty_pl.ty.toType());
	2291	const llvm_usize = try o.lowerType(.usize);
	2292
	2293	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
	2294	const base_ptr_int = try self.wip.bin(
	2295	.@"sub nuw",
	2296	field_ptr_int,
	2297	try o.builder.intValue(llvm_usize, field_offset),
	2298	"",
	2299	);
	2300	return self.wip.cast(.inttoptr, base_ptr_int, res_ty, "");
	2301	}
	2302
	2303	fn airNot(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2304	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2305	const operand = try self.resolveInst(ty_op.operand);
	2306
	2307	return self.wip.not(operand, "");
	2308	}
	2309
	2310	fn airUnreach(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	2311	_ = inst;
	2312	_ = try self.wip.@"unreachable"();
	2313	}
	2314
	2315	fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2316	const dbg_stmt = self.air.instructions.items(.data)[@intFromEnum(inst)].dbg_stmt;
	2317	self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1);
	2318	self.prev_dbg_column = @intCast(dbg_stmt.column + 1);
	2319
	2320	self.wip.debug_location = .{ .location = .{
	2321	.line = self.prev_dbg_line,
	2322	.column = self.prev_dbg_column,
	2323	.scope = self.scope.toOptional(),
	2324	.inlined_at = self.inlined_at,
	2325	} };
	2326
	2327	return .none;
	2328	}
	2329
	2330	fn airDbgEmptyStmt(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2331	_ = self;
	2332	_ = inst;
	2333	return .none;
	2334	}
	2335
	2336	fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2337	const o = self.object;
	2338	const pt = self.pt;
	2339	const zcu = o.zcu;
	2340	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	2341	const operand = try self.resolveInst(pl_op.operand);
	2342	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
	2343	const ptr_ty = self.typeOf(pl_op.operand);
	2344
	2345	const debug_local_var = try o.builder.debugLocalVar(
	2346	try o.builder.metadataString(name.toSlice(self.air)),
	2347	self.file,
	2348	self.scope,
	2349	self.prev_dbg_line,
	2350	try o.getDebugType(pt, ptr_ty.childType(zcu)),
	2351	);
	2352
	2353	_ = try self.wip.callIntrinsic(
	2354	.normal,
	2355	.none,
	2356	.@"dbg.declare",
	2357	&.{},
	2358	&.{
	2359	(try self.wip.debugValue(operand)).toValue(),
	2360	debug_local_var.toValue(),
	2361	(try o.builder.debugExpression(&.{})).toValue(),
	2362	},
	2363	"",
	2364	);
	2365
	2366	return .none;
	2367	}
	2368
	2369	fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Error!Builder.Value {
	2370	const o = self.object;
	2371	const pt = self.pt;
	2372	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	2373	const operand = try self.resolveInst(pl_op.operand);
	2374	const operand_ty = self.typeOf(pl_op.operand);
	2375	const name: Air.NullTerminatedString = @enumFromInt(pl_op.payload);
	2376	const name_slice = name.toSlice(self.air);
	2377	const metadata_name = if (name_slice.len > 0) try o.builder.metadataString(name_slice) else null;
	2378	const debug_local_var = if (is_arg) try o.builder.debugParameter(
	2379	metadata_name,
	2380	self.file,
	2381	self.scope,
	2382	self.prev_dbg_line,
	2383	try o.getDebugType(pt, operand_ty),
	2384	arg_no: {
	2385	self.arg_inline_index += 1;
	2386	break :arg_no self.arg_inline_index;
	2387	},
	2388	) else try o.builder.debugLocalVar(
	2389	metadata_name,
	2390	self.file,
	2391	self.scope,
	2392	self.prev_dbg_line,
	2393	try o.getDebugType(pt, operand_ty),
	2394	);
	2395
	2396	const zcu = o.zcu;
	2397	const owner_mod = self.ownerModule();
	2398	if (isByRef(operand_ty, zcu)) {
	2399	_ = try self.wip.callIntrinsic(
	2400	.normal,
	2401	.none,
	2402	.@"dbg.declare",
	2403	&.{},
	2404	&.{
	2405	(try self.wip.debugValue(operand)).toValue(),
	2406	debug_local_var.toValue(),
	2407	(try o.builder.debugExpression(&.{})).toValue(),
	2408	},
	2409	"",
	2410	);
	2411	} else if (owner_mod.optimize_mode == .Debug and !self.is_naked) {
	2412	// We avoid taking this path for naked functions because there's no guarantee that such
	2413	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
	2414
	2415	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
	2416	const alloca = try self.buildAlloca(operand.typeOfWip(&self.wip), alignment);
	2417	_ = try self.wip.store(.normal, operand, alloca, alignment);
	2418	_ = try self.wip.callIntrinsic(
	2419	.normal,
	2420	.none,
	2421	.@"dbg.declare",
	2422	&.{},
	2423	&.{
	2424	(try self.wip.debugValue(alloca)).toValue(),
	2425	debug_local_var.toValue(),
	2426	(try o.builder.debugExpression(&.{})).toValue(),
	2427	},
	2428	"",
	2429	);
	2430	} else {
	2431	_ = try self.wip.callIntrinsic(
	2432	.normal,
	2433	.none,
	2434	.@"dbg.value",
	2435	&.{},
	2436	&.{
	2437	(try self.wip.debugValue(operand)).toValue(),
	2438	debug_local_var.toValue(),
	2439	(try o.builder.debugExpression(&.{})).toValue(),
	2440	},
	2441	"",
	2442	);
	2443	}
	2444	return .none;
	2445	}
	2446
	2447	fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
	2448	// Eventually, the Zig compiler needs to be reworked to have inline
	2449	// assembly go through the same parsing code regardless of backend, and
	2450	// have LLVM-flavored inline assembly be output from that assembler.
	2451	// We don't have such an assembler implemented yet though. For now,
	2452	// this implementation feeds the inline assembly code directly to LLVM.
	2453
	2454	const o = self.object;
	2455	const unwrapped_asm = self.air.unwrapAsm(inst);
	2456	const is_volatile = unwrapped_asm.is_volatile;
	2457	const gpa = self.gpa;
	2458
	2459	const outputs = unwrapped_asm.outputs;
	2460	const inputs = unwrapped_asm.inputs;
	2461
	2462	var llvm_constraints: std.ArrayList(u8) = .empty;
	2463	defer llvm_constraints.deinit(gpa);
	2464
	2465	var arena_allocator = std.heap.ArenaAllocator.init(gpa);
	2466	defer arena_allocator.deinit();
	2467	const arena = arena_allocator.allocator();
	2468
	2469	// The exact number of return / parameter values depends on which output values
	2470	// are passed by reference as indirect outputs (determined below).
	2471	const max_return_count = outputs.len;
	2472	const llvm_ret_types = try arena.alloc(Builder.Type, max_return_count);
	2473	const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
	2474	const llvm_rw_vals = try arena.alloc(Builder.Value, max_return_count);
	2475
	2476	const max_param_count = max_return_count + inputs.len + outputs.len;
	2477	const llvm_param_types = try arena.alloc(Builder.Type, max_param_count);
	2478	const llvm_param_values = try arena.alloc(Builder.Value, max_param_count);
	2479	// This stores whether we need to add an elementtype attribute and
	2480	// if so, the element type itself.
	2481	const llvm_param_attrs = try arena.alloc(Builder.Type, max_param_count);
	2482	const zcu = o.zcu;
	2483	const ip = &zcu.intern_pool;
	2484	const target = zcu.getTarget();
	2485
	2486	var llvm_ret_i: usize = 0;
	2487	var llvm_param_i: usize = 0;
	2488	var total_i: usize = 0;
	2489
	2490	var name_map: std.StringArrayHashMapUnmanaged(u16) = .empty;
	2491	try name_map.ensureUnusedCapacity(arena, max_param_count);
	2492
	2493	var it = unwrapped_asm.iterateOutputs();
	2494	while (it.next()) \|output\| {
	2495	const constraint = output.constraint;
	2496	const name = output.name;
	2497
	2498	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 3);
	2499	if (total_i != 0) {
	2500	llvm_constraints.appendAssumeCapacity(',');
	2501	}
	2502	llvm_constraints.appendAssumeCapacity('=');
	2503
	2504	if (output.operand != .none) {
	2505	const output_inst = try self.resolveInst(output.operand);
	2506	const output_ty = self.typeOf(output.operand);
	2507	assert(output_ty.zigTypeTag(zcu) == .pointer);
	2508	const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu));
	2509
	2510	switch (constraint[0]) {
	2511	'=' => {},
	2512	'+' => llvm_rw_vals[output.index] = output_inst,
	2513	else => return self.todo("unsupported output constraint on output type '{c}'", .{
	2514	constraint[0],
	2515	}),
	2516	}
	2517
	2518	self.maybeMarkAllowZeroAccess(output_ty.ptrInfo(zcu));
	2519
	2520	// Pass any non-return outputs indirectly, if the constraint accepts a memory location
	2521	llvm_ret_indirect[output.index] = constraintAllowsMemory(constraint);
	2522	if (llvm_ret_indirect[output.index]) {
	2523	// Pass the result by reference as an indirect output (e.g. "=*m")
	2524	llvm_constraints.appendAssumeCapacity('*');
	2525
	2526	llvm_param_values[llvm_param_i] = output_inst;
	2527	llvm_param_types[llvm_param_i] = output_inst.typeOfWip(&self.wip);
	2528	llvm_param_attrs[llvm_param_i] = elem_llvm_ty;
	2529	llvm_param_i += 1;
	2530	} else {
	2531	// Pass the result directly (e.g. "=r")
	2532	llvm_ret_types[llvm_ret_i] = elem_llvm_ty;
	2533	llvm_ret_i += 1;
	2534	}
	2535	} else {
	2536	switch (constraint[0]) {
	2537	'=' => {},
	2538	else => return self.todo("unsupported output constraint on result type '{s}'", .{
	2539	constraint,
	2540	}),
	2541	}
	2542
	2543	llvm_ret_indirect[output.index] = false;
	2544
	2545	const ret_ty = self.typeOfIndex(inst);
	2546	llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty);
	2547	llvm_ret_i += 1;
	2548	}
	2549
	2550	// LLVM uses commas internally to separate different constraints,
	2551	// alternative constraints are achieved with pipes.
	2552	// We still allow the user to use commas in a way that is similar
	2553	// to GCC's inline assembly.
	2554	// http://llvm.org/docs/LangRef.html#constraint-codes
	2555	for (constraint[1..]) \|byte\| {
	2556	switch (byte) {
	2557	',' => llvm_constraints.appendAssumeCapacity('\|'),
	2558	'*' => {}, // Indirect outputs are handled above
	2559	else => llvm_constraints.appendAssumeCapacity(byte),
	2560	}
	2561	}
	2562
	2563	if (!std.mem.eql(u8, name, "_")) {
	2564	const gop = name_map.getOrPutAssumeCapacity(name);
	2565	if (gop.found_existing) return self.todo("duplicate asm output name '{s}'", .{name});
	2566	gop.value_ptr.* = @intCast(total_i);
	2567	}
	2568	total_i += 1;
	2569	}
	2570
	2571	it = unwrapped_asm.iterateInputs();
	2572	while (it.next()) \|input\| {
	2573	const constraint = input.constraint;
	2574	const name = input.name;
	2575
	2576	const arg_llvm_value = try self.resolveInst(input.operand);
	2577	const arg_ty = self.typeOf(input.operand);
	2578	const is_by_ref = isByRef(arg_ty, zcu);
	2579	if (is_by_ref) {
	2580	if (constraintAllowsMemory(constraint)) {
	2581	llvm_param_values[llvm_param_i] = arg_llvm_value;
	2582	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
	2583	} else {
	2584	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
	2585	const arg_llvm_ty = try o.lowerType(arg_ty);
	2586	const load_inst =
	2587	try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
	2588	llvm_param_values[llvm_param_i] = load_inst;
	2589	llvm_param_types[llvm_param_i] = arg_llvm_ty;
	2590	}
	2591	} else {
	2592	if (constraintAllowsRegister(constraint)) {
	2593	llvm_param_values[llvm_param_i] = arg_llvm_value;
	2594	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
	2595	} else {
	2596	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
	2597	const arg_ptr = try self.buildAlloca(arg_llvm_value.typeOfWip(&self.wip), alignment);
	2598	_ = try self.wip.store(.normal, arg_llvm_value, arg_ptr, alignment);
	2599	llvm_param_values[llvm_param_i] = arg_ptr;
	2600	llvm_param_types[llvm_param_i] = arg_ptr.typeOfWip(&self.wip);
	2601	}
	2602	}
	2603
	2604	try llvm_constraints.ensureUnusedCapacity(gpa, constraint.len + 1);
	2605	if (total_i != 0) {
	2606	llvm_constraints.appendAssumeCapacity(',');
	2607	}
	2608	for (constraint) \|byte\| {
	2609	llvm_constraints.appendAssumeCapacity(switch (byte) {
	2610	',' => '\|',
	2611	else => byte,
	2612	});
	2613	}
	2614
	2615	if (!std.mem.eql(u8, name, "_")) {
	2616	const gop = name_map.getOrPutAssumeCapacity(name);
	2617	if (gop.found_existing) return self.todo("duplicate asm input name '{s}'", .{name});
	2618	gop.value_ptr.* = @intCast(total_i);
	2619	}
	2620
	2621	// In the case of indirect inputs, LLVM requires the callsite to have
	2622	// an elementtype(<ty>) attribute.
	2623	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
	2624	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
	2625
	2626	break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu));
	2627	} else .none;
	2628
	2629	llvm_param_i += 1;
	2630	total_i += 1;
	2631	}
	2632
	2633	it = unwrapped_asm.iterateOutputs();
	2634	while (it.next()) \|output\| {
	2635	const constraint = output.constraint;
	2636
	2637	if (constraint[0] != '+') continue;
	2638
	2639	const rw_ty = self.typeOf(output.operand);
	2640	const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu));
	2641	if (llvm_ret_indirect[output.index]) {
	2642	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
	2643	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
	2644	} else {
	2645	const alignment = rw_ty.abiAlignment(zcu).toLlvm();
	2646	const loaded = try self.wip.load(
	2647	if (rw_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	2648	llvm_elem_ty,
	2649	llvm_rw_vals[output.index],
	2650	alignment,
	2651	"",
	2652	);
	2653	llvm_param_values[llvm_param_i] = loaded;
	2654	llvm_param_types[llvm_param_i] = llvm_elem_ty;
	2655	}
	2656
	2657	try llvm_constraints.print(gpa, ",{d}", .{output.index});
	2658
	2659	// In the case of indirect inputs, LLVM requires the callsite to have
	2660	// an elementtype(<ty>) attribute.
	2661	llvm_param_attrs[llvm_param_i] = if (llvm_ret_indirect[output.index]) llvm_elem_ty else .none;
	2662
	2663	llvm_param_i += 1;
	2664	total_i += 1;
	2665	}
	2666
	2667	if (total_i != 0) try llvm_constraints.append(gpa, ',');
	2668	const clobbers_val: Value = .fromInterned(unwrapped_asm.clobbers);
	2669	const clobbers_ty = clobbers_val.typeOf(zcu);
	2670	var clobbers_bigint_buf: Value.BigIntSpace = undefined;
	2671	const clobbers_bigint = clobbers_val.toBigInt(&clobbers_bigint_buf, zcu);
	2672	for (0..clobbers_ty.structFieldCount(zcu)) \|field_index\| {
	2673	assert(clobbers_ty.fieldType(field_index, zcu).toIntern() == .bool_type);
	2674	const limb_bits = @bitSizeOf(std.math.big.Limb);
	2675	if (field_index / limb_bits >= clobbers_bigint.limbs.len) continue; // field is false
	2676	switch (@as(u1, @truncate(clobbers_bigint.limbs[field_index / limb_bits] >> @intCast(field_index % limb_bits)))) {
	2677	0 => continue, // field is false
	2678	1 => {}, // field is true
	2679	}
	2680	const name = clobbers_ty.structFieldName(field_index, zcu).toSlice(ip).?;
	2681	total_i += try appendConstraints(gpa, &llvm_constraints, name, target);
	2682	}
	2683
	2684	// We have finished scanning through all inputs/outputs, so the number of
	2685	// parameters and return values is known.
	2686	const param_count = llvm_param_i;
	2687	const return_count = llvm_ret_i;
	2688
	2689	// For some targets, Clang unconditionally adds some clobbers to all inline assembly.
	2690	// While this is probably not strictly necessary, if we don't follow Clang's lead
	2691	// here then we may risk tripping LLVM bugs since anything not used by Clang tends
	2692	// to be buggy and regress often.
	2693	switch (target.cpu.arch) {
	2694	.x86_64, .x86 => {
	2695	try llvm_constraints.appendSlice(gpa, "~{dirflag},~{fpsr},~{flags},");
	2696	total_i += 3;
	2697	},
	2698	.mips, .mipsel, .mips64, .mips64el => {
	2699	try llvm_constraints.appendSlice(gpa, "~{$1},");
	2700	total_i += 1;
	2701	},
	2702	else => {},
	2703	}
	2704
	2705	if (std.mem.endsWith(u8, llvm_constraints.items, ",")) llvm_constraints.items.len -= 1;
	2706
	2707	const asm_source = unwrapped_asm.source;
	2708
	2709	// hackety hacks until stage2 has proper inline asm in the frontend.
	2710	var rendered_template = std.array_list.Managed(u8).init(gpa);
	2711	defer rendered_template.deinit();
	2712
	2713	const State = enum { start, percent, input, modifier };
	2714
	2715	var state: State = .start;
	2716
	2717	var name_start: usize = undefined;
	2718	var modifier_start: usize = undefined;
	2719	for (asm_source, 0..) \|byte, i\| {
	2720	switch (state) {
	2721	.start => switch (byte) {
	2722	'%' => state = .percent,
	2723	'$' => try rendered_template.appendSlice("$$"),
	2724	else => try rendered_template.append(byte),
	2725	},
	2726	.percent => switch (byte) {
	2727	'%' => {
	2728	try rendered_template.append('%');
	2729	state = .start;
	2730	},
	2731	'[' => {
	2732	try rendered_template.append('$');
	2733	try rendered_template.append('{');
	2734	name_start = i + 1;
	2735	state = .input;
	2736	},
	2737	'=' => {
	2738	try rendered_template.appendSlice("${:uid}");
	2739	state = .start;
	2740	},
	2741	else => {
	2742	try rendered_template.append('%');
	2743	try rendered_template.append(byte);
	2744	state = .start;
	2745	},
	2746	},
	2747	.input => switch (byte) {
	2748	']', ':' => {
	2749	const name = asm_source[name_start..i];
	2750
	2751	const index = name_map.get(name) orelse {
	2752	// we should validate the assembly in Sema; by now it is too late
	2753	return self.todo("unknown input or output name: '{s}'", .{name});
	2754	};
	2755	try rendered_template.print("{d}", .{index});
	2756	if (byte == ':') {
	2757	try rendered_template.append(':');
	2758	modifier_start = i + 1;
	2759	state = .modifier;
	2760	} else {
	2761	try rendered_template.append('}');
	2762	state = .start;
	2763	}
	2764	},
	2765	else => {},
	2766	},
	2767	.modifier => switch (byte) {
	2768	']' => {
	2769	try rendered_template.appendSlice(asm_source[modifier_start..i]);
	2770	try rendered_template.append('}');
	2771	state = .start;
	2772	},
	2773	else => {},
	2774	},
	2775	}
	2776	}
	2777
	2778	var attributes: Builder.FunctionAttributes.Wip = .{};
	2779	defer attributes.deinit(&o.builder);
	2780	for (llvm_param_attrs[0..param_count], 0..) \|llvm_elem_ty, i\| if (llvm_elem_ty != .none)
	2781	try attributes.addParamAttr(i, .{ .elementtype = llvm_elem_ty }, &o.builder);
	2782
	2783	const ret_llvm_ty = switch (return_count) {
	2784	0 => .void,
	2785	1 => llvm_ret_types[0],
	2786	else => try o.builder.structType(.normal, llvm_ret_types),
	2787	};
	2788	const llvm_fn_ty = try o.builder.fnType(ret_llvm_ty, llvm_param_types[0..param_count], .normal);
	2789	const call = try self.wip.callAsm(
	2790	try attributes.finish(&o.builder),
	2791	llvm_fn_ty,
	2792	.{ .sideeffect = is_volatile },
	2793	try o.builder.string(rendered_template.items),
	2794	try o.builder.string(llvm_constraints.items),
	2795	llvm_param_values[0..param_count],
	2796	"",
	2797	);
	2798
	2799	var ret_val = call;
	2800	llvm_ret_i = 0;
	2801	for (outputs, 0..) \|output, i\| {
	2802	if (llvm_ret_indirect[i]) continue;
	2803
	2804	const output_value = if (return_count > 1)
	2805	try self.wip.extractValue(call, &[_]u32{@intCast(llvm_ret_i)}, "")
	2806	else
	2807	call;
	2808
	2809	if (output != .none) {
	2810	const output_ptr = try self.resolveInst(output);
	2811	const output_ptr_ty = self.typeOf(output);
	2812	const alignment = output_ptr_ty.ptrAlignment(zcu).toLlvm();
	2813	_ = try self.wip.store(
	2814	if (output_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	2815	output_value,
	2816	output_ptr,
	2817	alignment,
	2818	);
	2819	} else {
	2820	ret_val = output_value;
	2821	}
	2822	llvm_ret_i += 1;
	2823	}
	2824
	2825	return ret_val;
	2826	}
	2827
	2828	fn airIsNonNull(
	2829	self: *FuncGen,
	2830	inst: Air.Inst.Index,
	2831	operand_is_ptr: bool,
	2832	cond: Builder.IntegerCondition,
	2833	) Allocator.Error!Builder.Value {
	2834	const o = self.object;
	2835	const zcu = o.zcu;
	2836	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	2837	const operand = try self.resolveInst(un_op);
	2838	const operand_ty = self.typeOf(un_op);
	2839	const optional_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
	2840	const optional_llvm_ty = try o.lowerType(optional_ty);
	2841	const payload_ty = optional_ty.optionalChild(zcu);
	2842
	2843	const access_kind: Builder.MemoryAccessKind =
	2844	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	2845
	2846	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
	2847
	2848	if (optional_ty.optionalReprIsPayload(zcu)) {
	2849	const loaded = if (operand_is_ptr)
	2850	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	2851	else
	2852	operand;
	2853	if (payload_ty.isSlice(zcu)) {
	2854	const slice_ptr = try self.wip.extractValue(loaded, &.{0}, "");
	2855	const ptr_ty = try o.builder.ptrType(llvm.toLlvmAddressSpace(
	2856	payload_ty.ptrAddressSpace(zcu),
	2857	zcu.getTarget(),
	2858	));
	2859	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
	2860	}
	2861	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(optional_llvm_ty), "");
	2862	}
	2863
	2864	comptime assert(optional_layout_version == 3);
	2865
	2866	if (!payload_ty.hasRuntimeBits(zcu)) {
	2867	const loaded = if (operand_is_ptr)
	2868	try self.wip.load(access_kind, optional_llvm_ty, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	2869	else
	2870	operand;
	2871	return self.wip.icmp(cond, loaded, try o.builder.intValue(.i8, 0), "");
	2872	}
	2873
	2874	return self.optCmpNull(cond, optional_ty, operand, access_kind);
	2875	}
	2876
	2877	fn airIsErr(
	2878	self: *FuncGen,
	2879	inst: Air.Inst.Index,
	2880	cond: Builder.IntegerCondition,
	2881	operand_is_ptr: bool,
	2882	) Allocator.Error!Builder.Value {
	2883	const o = self.object;
	2884	const zcu = o.zcu;
	2885	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	2886	const operand = try self.resolveInst(un_op);
	2887	const operand_ty = self.typeOf(un_op);
	2888	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
	2889	const payload_ty = err_union_ty.errorUnionPayload(zcu);
	2890	const error_type = try o.errorIntType();
	2891	const zero = try o.builder.intValue(error_type, 0);
	2892
	2893	const access_kind: Builder.MemoryAccessKind =
	2894	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	2895
	2896	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
	2897	const val: Builder.Constant = switch (cond) {
	2898	.eq => .true, // 0 == 0
	2899	.ne => .false, // 0 != 0
	2900	else => unreachable,
	2901	};
	2902	return val.toValue();
	2903	}
	2904
	2905	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
	2906
	2907	if (!payload_ty.hasRuntimeBits(zcu)) {
	2908	const loaded = if (operand_is_ptr)
	2909	try self.wip.load(access_kind, try o.lowerType(err_union_ty), operand, operand_ty.ptrAlignment(zcu).toLlvm(), "")
	2910	else
	2911	operand;
	2912	return self.wip.icmp(cond, loaded, zero, "");
	2913	}
	2914	assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
	2915
	2916	const err_align = if (operand_is_ptr)
	2917	operand_ty.ptrAlignment(zcu).minStrict(Type.anyerror.abiAlignment(zcu))
	2918	else
	2919	.none;
	2920	const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
	2921	const loaded = try self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
	2922	return self.wip.icmp(cond, loaded, zero, "");
	2923	}
	2924
	2925	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2926	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2927	const operand = try self.resolveInst(ty_op.operand);
	2928	// If `Type.optionalReprIsPayload`, then the address should be the same. Otherwise, optional
	2929	// layouts always put the payload at offset 0, so... the address should still be the same.
	2930	return operand;
	2931	}
	2932
	2933	fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2934	comptime assert(optional_layout_version == 3);
	2935
	2936	const o = self.object;
	2937	const zcu = o.zcu;
	2938	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2939	const operand = try self.resolveInst(ty_op.operand);
	2940	const optional_ptr_ty = self.typeOf(ty_op.operand);
	2941	const optional_ty = optional_ptr_ty.childType(zcu);
	2942	const payload_ty = optional_ty.optionalChild(zcu);
	2943	const non_null_bit = try o.builder.intValue(.i8, 1);
	2944
	2945	const access_kind: Builder.MemoryAccessKind =
	2946	if (optional_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	2947
	2948	if (!payload_ty.hasRuntimeBits(zcu)) {
	2949	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
	2950
	2951	// We have a pointer to a i8. We need to set it to 1 and then return the same pointer.
	2952	// Default alignment store because align of the non null bit is 1 anyway.
	2953	_ = try self.wip.store(access_kind, non_null_bit, operand, .default);
	2954	return operand;
	2955	}
	2956	if (optional_ty.optionalReprIsPayload(zcu)) {
	2957	// The payload and the optional are the same value.
	2958	// Setting to non-null will be done when the payload is set.
	2959	return operand;
	2960	}
	2961
	2962	// First set the non-null bit. It's always immediately after the payload (no padding) because it
	2963	// has alignment 1.
	2964	const non_null_ptr = try self.ptraddConst(operand, payload_ty.abiSize(zcu));
	2965
	2966	self.maybeMarkAllowZeroAccess(optional_ptr_ty.ptrInfo(zcu));
	2967
	2968	// Default alignment store because align of the non null bit is 1 anyway.
	2969	_ = try self.wip.store(access_kind, non_null_bit, non_null_ptr, .default);
	2970
	2971	// Then return the payload pointer (only if it's used).
	2972	if (self.liveness.isUnused(inst)) return .none;
	2973
	2974	return operand; // payload is at offset 0
	2975	}
	2976
	2977	fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	2978	const zcu = self.object.zcu;
	2979	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2980	const operand = try self.resolveInst(ty_op.operand);
	2981	const optional_ty = self.typeOf(ty_op.operand);
	2982	const payload_ty = self.typeOfIndex(inst);
	2983	if (!payload_ty.hasRuntimeBits(zcu)) return .none;
	2984
	2985	if (optional_ty.optionalReprIsPayload(zcu)) {
	2986	// Payload value is the same as the optional value.
	2987	return operand;
	2988	}
	2989
	2990	return self.optPayloadHandle(operand, optional_ty, false);
	2991	}
	2992
	2993	fn airErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index, operand_is_ptr: bool) Allocator.Error!Builder.Value {
	2994	const o = self.object;
	2995	const zcu = o.zcu;
	2996	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	2997	const operand = try self.resolveInst(ty_op.operand);
	2998	const operand_ty = self.typeOf(ty_op.operand);
	2999	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
	3000	const result_ty = self.typeOfIndex(inst);
	3001	const payload_ty = if (operand_is_ptr) result_ty.childType(zcu) else result_ty;
	3002
	3003	if (!payload_ty.hasRuntimeBits(zcu)) {
	3004	return if (operand_is_ptr) operand else .none;
	3005	}
	3006	const payload_ptr = try self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
	3007	if (operand_is_ptr) {
	3008	return payload_ptr;
	3009	}
	3010	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
	3011	const payload_alignment = payload_ty.abiAlignment(zcu).toLlvm();
	3012	if (isByRef(payload_ty, zcu)) {
	3013	return self.loadByRef(payload_ptr, payload_ty, payload_alignment, .normal);
	3014	} else {
	3015	const payload_llvm_ty = try o.lowerType(payload_ty);
	3016	return self.wip.load(.normal, payload_llvm_ty, payload_ptr, payload_alignment, "");
	3017	}
	3018	}
	3019
	3020	fn airErrUnionErr(
	3021	self: *FuncGen,
	3022	inst: Air.Inst.Index,
	3023	operand_is_ptr: bool,
	3024	) Allocator.Error!Builder.Value {
	3025	const o = self.object;
	3026	const zcu = o.zcu;
	3027	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3028	const operand = try self.resolveInst(ty_op.operand);
	3029	const operand_ty = self.typeOf(ty_op.operand);
	3030	const error_type = try o.errorIntType();
	3031	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
	3032	if (err_union_ty.errorUnionSet(zcu).errorSetIsEmpty(zcu)) {
	3033	if (operand_is_ptr) {
	3034	return operand;
	3035	} else {
	3036	return o.builder.intValue(error_type, 0);
	3037	}
	3038	}
	3039
	3040	const access_kind: Builder.MemoryAccessKind =
	3041	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	3042
	3043	const payload_ty = err_union_ty.errorUnionPayload(zcu);
	3044	if (!payload_ty.hasRuntimeBits(zcu)) {
	3045	if (!operand_is_ptr) return operand;
	3046
	3047	self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
	3048
	3049	return self.wip.load(access_kind, error_type, operand, operand_ty.ptrAlignment(zcu).toLlvm(), "");
	3050	}
	3051
	3052	assert(isByRef(err_union_ty, zcu)); // error unions are by-ref unless the payload lacks runtime bits
	3053
	3054	if (operand_is_ptr) self.maybeMarkAllowZeroAccess(operand_ty.ptrInfo(zcu));
	3055
	3056	const err_align: InternPool.Alignment = a: {
	3057	const err_abi_align = Type.anyerror.abiAlignment(zcu);
	3058	if (!operand_is_ptr) break :a err_abi_align;
	3059	break :a err_abi_align.minStrict(operand_ty.ptrAlignment(zcu));
	3060	};
	3061
	3062	const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
	3063	return self.wip.load(access_kind, error_type, err_field_ptr, err_align.toLlvm(), "");
	3064	}
	3065
	3066	fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3067	const o = self.object;
	3068	const zcu = o.zcu;
	3069	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3070	const operand = try self.resolveInst(ty_op.operand);
	3071	const err_union_ptr_ty = self.typeOf(ty_op.operand);
	3072	const err_union_ty = err_union_ptr_ty.childType(zcu);
	3073	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
	3074
	3075	const payload_ty = err_union_ty.errorUnionPayload(zcu);
	3076	const non_error_val = try o.builder.intValue(try o.errorIntType(), 0);
	3077
	3078	const access_kind: Builder.MemoryAccessKind =
	3079	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	3080
	3081	self.maybeMarkAllowZeroAccess(err_union_ptr_ty.ptrInfo(zcu));
	3082
	3083	{
	3084	const error_align = Type.anyerror.abiAlignment(zcu).minStrict(err_union_ptr_align).toLlvm();
	3085	// First set the non-error value.
	3086	const error_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
	3087	_ = try self.wip.store(access_kind, non_error_val, error_ptr, error_align);
	3088	}
	3089
	3090	// Then return the payload pointer (only if it is used).
	3091	if (self.liveness.isUnused(inst)) return .none;
	3092	return self.ptraddConst(operand, codegen.errUnionPayloadOffset(payload_ty, zcu));
	3093	}
	3094
	3095	fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) Allocator.Error!Builder.Value {
	3096	assert(self.err_ret_trace != .none);
	3097	return self.err_ret_trace;
	3098	}
	3099
	3100	fn airSetErrReturnTrace(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3101	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	3102	self.err_ret_trace = try self.resolveInst(un_op);
	3103	return .none;
	3104	}
	3105
	3106	fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3107	const zcu = self.object.zcu;
	3108
	3109	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	3110	const struct_ty = ty_pl.ty.toType();
	3111	const field_index = ty_pl.payload;
	3112
	3113	assert(self.err_ret_trace != .none);
	3114
	3115	const field_ty = struct_ty.fieldType(field_index, zcu);
	3116	const field_offset = struct_ty.structFieldOffset(field_index, zcu);
	3117	const field_align = switch (field_offset) {
	3118	0 => struct_ty.abiAlignment(zcu),
	3119	else => struct_ty.abiAlignment(zcu).minStrict(.fromLog2Units(@ctz(field_offset))),
	3120	};
	3121
	3122	const field_ptr = try self.ptraddConst(self.err_ret_trace, field_offset);
	3123	return self.load(field_ptr, field_ty, field_align.toLlvm(), .normal);
	3124	}
	3125
	3126	/// As an optimization, we want to avoid unnecessary copies of
	3127	/// error union/optional types when returning from a function.
	3128	/// Here, we scan forward in the current block, looking to see
	3129	/// if the next instruction is a return (ignoring debug instructions).
	3130	///
	3131	/// The first instruction of `body_tail` is a wrap instruction.
	3132	fn isNextRet(
	3133	self: *FuncGen,
	3134	body_tail: []const Air.Inst.Index,
	3135	) bool {
	3136	const air_tags = self.air.instructions.items(.tag);
	3137	for (body_tail[1..]) \|body_inst\| {
	3138	switch (air_tags[@intFromEnum(body_inst)]) {
	3139	.ret => return true,
	3140	.dbg_stmt => continue,
	3141	else => return false,
	3142	}
	3143	}
	3144	// The only way to get here is to hit the end of a loop instruction
	3145	// (implicit repeat).
	3146	return false;
	3147	}
	3148
	3149	fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3150	const o = self.object;
	3151	const zcu = o.zcu;
	3152	const inst = body_tail[0];
	3153	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3154	const payload_ty = self.typeOf(ty_op.operand);
	3155	const non_null_bit = try o.builder.intValue(.i8, 1);
	3156	comptime assert(optional_layout_version == 3);
	3157	assert(payload_ty.hasRuntimeBits(zcu));
	3158	const operand = try self.resolveInst(ty_op.operand);
	3159	const optional_ty = self.typeOfIndex(inst);
	3160	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
	3161	assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
	3162	const llvm_optional_ty = try o.lowerType(optional_ty);
	3163	const optional_ptr = if (self.isNextRet(body_tail))
	3164	self.ret_ptr
	3165	else brk: {
	3166	const alignment = optional_ty.abiAlignment(zcu).toLlvm();
	3167	const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
	3168	break :brk optional_ptr;
	3169	};
	3170
	3171	const payload_ptr = optional_ptr; // payload always at offset 0
	3172	try self.store(
	3173	payload_ptr,
	3174	.none,
	3175	operand,
	3176	payload_ty,
	3177	);
	3178	// Non-null bit immediately after payload (no padding because the bit has alignment 1).
	3179	const non_null_ptr = try self.ptraddConst(optional_ptr, payload_ty.abiSize(zcu));
	3180	_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, .default);
	3181	return optional_ptr;
	3182	}
	3183
	3184	fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3185	const o = self.object;
	3186	const zcu = o.zcu;
	3187	const inst = body_tail[0];
	3188	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3189	const err_un_ty = self.typeOfIndex(inst);
	3190	const operand = try self.resolveInst(ty_op.operand);
	3191	const payload_ty = self.typeOf(ty_op.operand);
	3192	assert(payload_ty.hasRuntimeBits(zcu));
	3193	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
	3194	const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0);
	3195	const err_un_llvm_ty = try o.lowerType(err_un_ty);
	3196
	3197	const result_ptr = if (self.isNextRet(body_tail))
	3198	self.ret_ptr
	3199	else brk: {
	3200	const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm();
	3201	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
	3202	break :brk result_ptr;
	3203	};
	3204
	3205	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
	3206	const error_alignment = Type.anyerror.abiAlignment(o.zcu).toLlvm();
	3207	_ = try self.wip.store(.normal, ok_err_code, err_ptr, error_alignment);
	3208	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
	3209	try self.store(
	3210	payload_ptr,
	3211	.none,
	3212	operand,
	3213	payload_ty,
	3214	);
	3215	return result_ptr;
	3216	}
	3217
	3218	fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3219	const o = self.object;
	3220	const zcu = o.zcu;
	3221	const inst = body_tail[0];
	3222	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	3223	const err_un_ty = self.typeOfIndex(inst);
	3224	const payload_ty = err_un_ty.errorUnionPayload(zcu);
	3225	const operand = try self.resolveInst(ty_op.operand);
	3226	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
	3227	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
	3228	const err_un_llvm_ty = try o.lowerType(err_un_ty);
	3229
	3230	const result_ptr = if (self.isNextRet(body_tail))
	3231	self.ret_ptr
	3232	else brk: {
	3233	const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
	3234	const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
	3235	break :brk result_ptr;
	3236	};
	3237
	3238	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
	3239	const error_alignment = Type.anyerror.abiAlignment(zcu).toLlvm();
	3240	_ = try self.wip.store(.normal, operand, err_ptr, error_alignment);
	3241	const payload_ptr = try self.ptraddConst(result_ptr, codegen.errUnionPayloadOffset(payload_ty, zcu));
	3242	// TODO store undef to payload_ptr
	3243	_ = payload_ptr;
	3244	return result_ptr;
	3245	}
	3246
	3247	fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3248	const o = self.object;
	3249	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	3250	const index = pl_op.payload;
	3251	const llvm_usize = try o.lowerType(.usize);
	3252	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
	3253	try o.builder.intValue(.i32, index),
	3254	}, "");
	3255	}
	3256
	3257	fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3258	const o = self.object;
	3259	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	3260	const index = pl_op.payload;
	3261	const llvm_isize = try o.lowerType(.isize);
	3262	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
	3263	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
	3264	}, "");
	3265	}
	3266
	3267	fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3268	const o = fg.object;
	3269	const ty_nav = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav;
	3270	const llvm_ptr = try o.lowerNavRef(ty_nav.nav);
	3271	return llvm_ptr.toValue();
	3272	}
	3273
	3274	fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3275	const o = self.object;
	3276	const zcu = o.zcu;
	3277	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3278	const lhs = try self.resolveInst(bin_op.lhs);
	3279	const rhs = try self.resolveInst(bin_op.rhs);
	3280	const inst_ty = self.typeOfIndex(inst);
	3281	const scalar_ty = inst_ty.scalarType(zcu);
	3282
	3283	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs });
	3284	return self.wip.callIntrinsic(
	3285	.normal,
	3286	.none,
	3287	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
	3288	&.{try o.lowerType(inst_ty)},
	3289	&.{ lhs, rhs },
	3290	"",
	3291	);
	3292	}
	3293
	3294	fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3295	const o = self.object;
	3296	const zcu = o.zcu;
	3297	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3298	const lhs = try self.resolveInst(bin_op.lhs);
	3299	const rhs = try self.resolveInst(bin_op.rhs);
	3300	const inst_ty = self.typeOfIndex(inst);
	3301	const scalar_ty = inst_ty.scalarType(zcu);
	3302
	3303	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs });
	3304	return self.wip.callIntrinsic(
	3305	.normal,
	3306	.none,
	3307	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
	3308	&.{try o.lowerType(inst_ty)},
	3309	&.{ lhs, rhs },
	3310	"",
	3311	);
	3312	}
	3313
	3314	fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3315	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	3316	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	3317	const ptr = try self.resolveInst(bin_op.lhs);
	3318	const len = try self.resolveInst(bin_op.rhs);
	3319	const inst_ty = self.typeOfIndex(inst);
	3320	return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, "");
	3321	}
	3322
	3323	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3324	const zcu = self.object.zcu;
	3325	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3326	const lhs = try self.resolveInst(bin_op.lhs);
	3327	const rhs = try self.resolveInst(bin_op.rhs);
	3328	const inst_ty = self.typeOfIndex(inst);
	3329	const scalar_ty = inst_ty.scalarType(zcu);
	3330
	3331	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs });
	3332	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"add nsw" else .@"add nuw", lhs, rhs, "");
	3333	}
	3334
	3335	fn airSafeArithmetic(
	3336	fg: *FuncGen,
	3337	inst: Air.Inst.Index,
	3338	signed_intrinsic: Builder.Intrinsic,
	3339	unsigned_intrinsic: Builder.Intrinsic,
	3340	) Allocator.Error!Builder.Value {
	3341	const o = fg.object;
	3342	const zcu = o.zcu;
	3343
	3344	const bin_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3345	const lhs = try fg.resolveInst(bin_op.lhs);
	3346	const rhs = try fg.resolveInst(bin_op.rhs);
	3347	const inst_ty = fg.typeOfIndex(inst);
	3348	const scalar_ty = inst_ty.scalarType(zcu);
	3349
	3350	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
	3351	const llvm_inst_ty = try o.lowerType(inst_ty);
	3352	const results =
	3353	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
	3354
	3355	const overflow_bits = try fg.wip.extractValue(results, &.{1}, "");
	3356	const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip);
	3357	const overflow_bit = switch (inst_ty.zigTypeTag(zcu)) {
	3358	.vector => try fg.wip.callIntrinsic(
	3359	.normal,
	3360	.none,
	3361	.@"vector.reduce.or",
	3362	&.{overflow_bits_ty},
	3363	&.{overflow_bits},
	3364	"",
	3365	),
	3366	else => overflow_bits,
	3367	};
	3368
	3369	const fail_block = try fg.wip.block(1, "OverflowFail");
	3370	const ok_block = try fg.wip.block(1, "OverflowOk");
	3371	_ = try fg.wip.brCond(overflow_bit, fail_block, ok_block, .none);
	3372
	3373	fg.wip.cursor = .{ .block = fail_block };
	3374	try fg.buildSimplePanic(.integer_overflow);
	3375
	3376	fg.wip.cursor = .{ .block = ok_block };
	3377	return fg.wip.extractValue(results, &.{0}, "");
	3378	}
	3379
	3380	fn airAddWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3381	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3382	const lhs = try self.resolveInst(bin_op.lhs);
	3383	const rhs = try self.resolveInst(bin_op.rhs);
	3384
	3385	return self.wip.bin(.add, lhs, rhs, "");
	3386	}
	3387
	3388	fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3389	const o = self.object;
	3390	const zcu = o.zcu;
	3391	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3392	const lhs = try self.resolveInst(bin_op.lhs);
	3393	const rhs = try self.resolveInst(bin_op.rhs);
	3394	const inst_ty = self.typeOfIndex(inst);
	3395	const scalar_ty = inst_ty.scalarType(zcu);
	3396	assert(scalar_ty.zigTypeTag(zcu) == .int);
	3397	return self.wip.callIntrinsic(
	3398	.normal,
	3399	.none,
	3400	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
	3401	&.{try o.lowerType(inst_ty)},
	3402	&.{ lhs, rhs },
	3403	"",
	3404	);
	3405	}
	3406
	3407	fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3408	const zcu = self.object.zcu;
	3409	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3410	const lhs = try self.resolveInst(bin_op.lhs);
	3411	const rhs = try self.resolveInst(bin_op.rhs);
	3412	const inst_ty = self.typeOfIndex(inst);
	3413	const scalar_ty = inst_ty.scalarType(zcu);
	3414
	3415	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs });
	3416	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"sub nsw" else .@"sub nuw", lhs, rhs, "");
	3417	}
	3418
	3419	fn airSubWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3420	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3421	const lhs = try self.resolveInst(bin_op.lhs);
	3422	const rhs = try self.resolveInst(bin_op.rhs);
	3423
	3424	return self.wip.bin(.sub, lhs, rhs, "");
	3425	}
	3426
	3427	fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3428	const o = self.object;
	3429	const zcu = o.zcu;
	3430	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3431	const lhs = try self.resolveInst(bin_op.lhs);
	3432	const rhs = try self.resolveInst(bin_op.rhs);
	3433	const inst_ty = self.typeOfIndex(inst);
	3434	const scalar_ty = inst_ty.scalarType(zcu);
	3435	assert(scalar_ty.zigTypeTag(zcu) == .int);
	3436	return self.wip.callIntrinsic(
	3437	.normal,
	3438	.none,
	3439	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
	3440	&.{try o.lowerType(inst_ty)},
	3441	&.{ lhs, rhs },
	3442	"",
	3443	);
	3444	}
	3445
	3446	fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3447	const zcu = self.object.zcu;
	3448	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3449	const lhs = try self.resolveInst(bin_op.lhs);
	3450	const rhs = try self.resolveInst(bin_op.rhs);
	3451	const inst_ty = self.typeOfIndex(inst);
	3452	const scalar_ty = inst_ty.scalarType(zcu);
	3453
	3454	if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs });
	3455	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .@"mul nsw" else .@"mul nuw", lhs, rhs, "");
	3456	}
	3457
	3458	fn airMulWrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3459	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3460	const lhs = try self.resolveInst(bin_op.lhs);
	3461	const rhs = try self.resolveInst(bin_op.rhs);
	3462
	3463	return self.wip.bin(.mul, lhs, rhs, "");
	3464	}
	3465
	3466	fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3467	const o = self.object;
	3468	const zcu = o.zcu;
	3469	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3470	const lhs = try self.resolveInst(bin_op.lhs);
	3471	const rhs = try self.resolveInst(bin_op.rhs);
	3472	const inst_ty = self.typeOfIndex(inst);
	3473	const scalar_ty = inst_ty.scalarType(zcu);
	3474	assert(scalar_ty.zigTypeTag(zcu) == .int);
	3475	return self.wip.callIntrinsic(
	3476	.normal,
	3477	.none,
	3478	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
	3479	&.{try o.lowerType(inst_ty)},
	3480	&.{ lhs, rhs, .@"0" },
	3481	"",
	3482	);
	3483	}
	3484
	3485	fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3486	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3487	const lhs = try self.resolveInst(bin_op.lhs);
	3488	const rhs = try self.resolveInst(bin_op.rhs);
	3489	const inst_ty = self.typeOfIndex(inst);
	3490
	3491	return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
	3492	}
	3493
	3494	fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3495	const zcu = self.object.zcu;
	3496	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3497	const lhs = try self.resolveInst(bin_op.lhs);
	3498	const rhs = try self.resolveInst(bin_op.rhs);
	3499	const inst_ty = self.typeOfIndex(inst);
	3500	const scalar_ty = inst_ty.scalarType(zcu);
	3501
	3502	if (scalar_ty.isRuntimeFloat()) {
	3503	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
	3504	return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result});
	3505	}
	3506	return self.wip.bin(if (scalar_ty.isSignedInt(zcu)) .sdiv else .udiv, lhs, rhs, "");
	3507	}
	3508
	3509	fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3510	const o = self.object;
	3511	const zcu = o.zcu;
	3512	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3513	const lhs = try self.resolveInst(bin_op.lhs);
	3514	const rhs = try self.resolveInst(bin_op.rhs);
	3515	const inst_ty = self.typeOfIndex(inst);
	3516	const scalar_ty = inst_ty.scalarType(zcu);
	3517
	3518	if (scalar_ty.isRuntimeFloat()) {
	3519	const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
	3520	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
	3521	}
	3522	if (scalar_ty.isSignedInt(zcu)) {
	3523	const scalar_llvm_ty = try o.lowerType(scalar_ty);
	3524	const inst_llvm_ty = try o.lowerType(inst_ty);
	3525
	3526	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
	3527	var stack align(@max(
	3528	@alignOf(std.heap.StackFallbackAllocator(0)),
	3529	@alignOf(ExpectedContents),
	3530	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
	3531	const allocator = stack.get();
	3532
	3533	const scalar_bits = scalar_ty.intInfo(zcu).bits;
	3534	var smin_big_int: std.math.big.int.Mutable = .{
	3535	.limbs = try allocator.alloc(
	3536	std.math.big.Limb,
	3537	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
	3538	),
	3539	.len = undefined,
	3540	.positive = undefined,
	3541	};
	3542	defer allocator.free(smin_big_int.limbs);
	3543	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
	3544	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
	3545	scalar_llvm_ty,
	3546	smin_big_int.toConst(),
	3547	));
	3548
	3549	const div = try self.wip.bin(.sdiv, lhs, rhs, "divFloor.div");
	3550	const rem = try self.wip.bin(.srem, lhs, rhs, "divFloor.rem");
	3551	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "divFloor.rhs_sign");
	3552	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "divFloor.rem_xor_rhs_sign");
	3553	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "divFloor.need_correction");
	3554	const correction = try self.wip.cast(.sext, need_correction, inst_llvm_ty, "divFloor.correction");
	3555	return self.wip.bin(.@"add nsw", div, correction, "divFloor");
	3556	}
	3557	return self.wip.bin(.udiv, lhs, rhs, "");
	3558	}
	3559
	3560	fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3561	const zcu = self.object.zcu;
	3562	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3563	const lhs = try self.resolveInst(bin_op.lhs);
	3564	const rhs = try self.resolveInst(bin_op.rhs);
	3565	const inst_ty = self.typeOfIndex(inst);
	3566	const scalar_ty = inst_ty.scalarType(zcu);
	3567
	3568	if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs });
	3569	return self.wip.bin(
	3570	if (scalar_ty.isSignedInt(zcu)) .@"sdiv exact" else .@"udiv exact",
	3571	lhs,
	3572	rhs,
	3573	"",
	3574	);
	3575	}
	3576
	3577	fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3578	const zcu = self.object.zcu;
	3579	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3580	const lhs = try self.resolveInst(bin_op.lhs);
	3581	const rhs = try self.resolveInst(bin_op.rhs);
	3582	const inst_ty = self.typeOfIndex(inst);
	3583	const scalar_ty = inst_ty.scalarType(zcu);
	3584
	3585	if (scalar_ty.isRuntimeFloat())
	3586	return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
	3587	return self.wip.bin(if (scalar_ty.isSignedInt(zcu))
	3588	.srem
	3589	else
	3590	.urem, lhs, rhs, "");
	3591	}
	3592
	3593	fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	3594	const o = self.object;
	3595	const zcu = o.zcu;
	3596	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	3597	const lhs = try self.resolveInst(bin_op.lhs);
	3598	const rhs = try self.resolveInst(bin_op.rhs);
	3599	const inst_ty = self.typeOfIndex(inst);
	3600	const inst_llvm_ty = try o.lowerType(inst_ty);
	3601	const scalar_ty = inst_ty.scalarType(zcu);
	3602
	3603	if (scalar_ty.isRuntimeFloat()) {
	3604	const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs });
	3605	const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs });
	3606	const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs });
	3607	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
	3608	const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero });
	3609	return self.wip.select(fast, ltz, c, a, "");
	3610	}
	3611	if (scalar_ty.isSignedInt(zcu)) {
	3612	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
	3613	var stack align(@max(
	3614	@alignOf(std.heap.StackFallbackAllocator(0)),
	3615	@alignOf(ExpectedContents),
	3616	)) = std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);
	3617	const allocator = stack.get();
	3618
	3619	const scalar_bits = scalar_ty.intInfo(zcu).bits;
	3620	var smin_big_int: std.math.big.int.Mutable = .{
	3621	.limbs = try allocator.alloc(
	3622	std.math.big.Limb,
	3623	std.math.big.int.calcTwosCompLimbCount(scalar_bits),
	3624	),
	3625	.len = undefined,
	3626	.positive = undefined,
	3627	};
	3628	defer allocator.free(smin_big_int.limbs);
	3629	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
	3630	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
	3631	try o.lowerType(scalar_ty),
	3632	smin_big_int.toConst(),
	3633	));
	3634
	3635	const rem = try self.wip.bin(.srem, lhs, rhs, "mod.rem");
	3636	const rhs_sign = try self.wip.bin(.@"and", rhs, smin, "mod.rhs_sign");
	3637	const rem_xor_rhs_sign = try self.wip.bin(.xor, rem, rhs_sign, "mod.rem_xor_rhs_sign");
	3638	const need_correction = try self.wip.icmp(.ugt, rem_xor_rhs_sign, smin, "mod.need_correction");
	3639	const zero = try o.builder.zeroInitValue(inst_llvm_ty);
	3640	const correction = try self.wip.select(.normal, need_correction, rhs, zero, "mod.correction");
	3641	return self.wip.bin(.@"add nsw", correction, rem, "mod");
	3642	}
	3643	return self.wip.bin(.urem, lhs, rhs, "");
	3644	}
	3645
	3646	fn airPtrAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3647	const zcu = self.object.zcu;
	3648	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	3649	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	3650	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
	3651	const index = try self.resolveInst(bin_op.rhs);
	3652	const ptr_ty = self.typeOf(bin_op.lhs);
	3653	const elem_ty = ptr_ty.indexableElem(zcu);
	3654	const ptr = switch (ptr_ty.ptrSize(zcu)) {
	3655	.one, .many, .c => ptr_or_slice,
	3656	.slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
	3657	};
	3658	return self.ptraddScaled(ptr, index, elem_ty.abiSize(zcu));
	3659	}
	3660
	3661	fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	3662	const o = self.object;
	3663	const zcu = o.zcu;
	3664	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	3665	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
	3666	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
	3667	const llvm_usize_ty = try o.lowerType(.usize);
	3668	const ptr_ty = self.typeOf(bin_op.lhs);
	3669	const elem_ty = ptr_ty.indexableElem(zcu);
	3670	const ptr = switch (ptr_ty.ptrSize(zcu)) {
	3671	.one, .many, .c => ptr_or_slice,
	3672	.slice => try self.wip.extractValue(ptr_or_slice, &.{0}, ""),
	3673	};
	3674	const scale_val = try o.builder.intValue(llvm_usize_ty, -@as(i65, elem_ty.abiSize(zcu)));
	3675	const positive_index = try self.resolveInst(bin_op.rhs);
	3676	const negative_offset = try self.wip.bin(.@"mul nsw", positive_index, scale_val, "");
	3677	return self.wip.gep(.inbounds, .i8, ptr, &.{negative_offset}, "");
	3678	}
	3679
	3680	fn airOverflow(
	3681	self: *FuncGen,
	3682	inst: Air.Inst.Index,
	3683	signed_intrinsic: Builder.Intrinsic,
	3684	unsigned_intrinsic: Builder.Intrinsic,
	3685	) Allocator.Error!Builder.Value {
	3686	const o = self.object;
	3687	const zcu = o.zcu;
	3688	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	3689	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
	3690
	3691	const lhs = try self.resolveInst(extra.lhs);
	3692	const rhs = try self.resolveInst(extra.rhs);
	3693
	3694	const lhs_ty = self.typeOf(extra.lhs);
	3695	const scalar_ty = lhs_ty.scalarType(zcu);
	3696	const inst_ty = self.typeOfIndex(inst);
	3697	assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
	3698
	3699	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
	3700	const llvm_inst_ty = try o.lowerType(inst_ty);
	3701	const llvm_lhs_ty = try o.lowerType(lhs_ty);
	3702	const results =
	3703	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
	3704
	3705	const result_val = try self.wip.extractValue(results, &.{0}, "");
	3706	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
	3707
	3708	const result_alignment = inst_ty.abiAlignment(zcu).toLlvm();
	3709	const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment);
	3710
	3711	{
	3712	// Store to 'result: IntType' field
	3713	const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(0, zcu));
	3714	_ = try self.wip.store(.normal, result_val, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
	3715	}
	3716
	3717	{
	3718	// Store to 'overflow: u1' field
	3719	const field_ptr = try self.ptraddConst(alloca_inst, inst_ty.structFieldOffset(1, zcu));
	3720	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
	3721	}
	3722
	3723	return alloca_inst;
	3724	}
	3725
	3726	fn buildElementwiseCall(
	3727	self: *FuncGen,
	3728	llvm_fn: Builder.Function.Index,
	3729	args_vectors: []const Builder.Value,
	3730	result_vector: Builder.Value,
	3731	vector_len: usize,
	3732	) Allocator.Error!Builder.Value {
	3733	const o = self.object;
	3734	assert(args_vectors.len <= 3);
	3735
	3736	var i: usize = 0;
	3737	var result = result_vector;
	3738	while (i < vector_len) : (i += 1) {
	3739	const index_i32 = try o.builder.intValue(.i32, i);
	3740
	3741	var args: [3]Builder.Value = undefined;
	3742	for (args[0..args_vectors.len], args_vectors) \|*arg_elem, arg_vector\| {
	3743	arg_elem.* = try self.wip.extractElement(arg_vector, index_i32, "");
	3744	}
	3745	const result_elem = try self.wip.call(
	3746	.normal,
	3747	.ccc,
	3748	.none,
	3749	llvm_fn.typeOf(&o.builder),
	3750	llvm_fn.toValue(&o.builder),
	3751	args[0..args_vectors.len],
	3752	"",
	3753	);
	3754	result = try self.wip.insertElement(result, result_elem, index_i32, "");
	3755	}
	3756	return result;
	3757	}
	3758
	3759	/// Creates a floating point comparison by lowering to the appropriate
	3760	/// hardware instruction or softfloat routine for the target
	3761	fn buildFloatCmp(
	3762	self: *FuncGen,
	3763	fast: Builder.FastMathKind,
	3764	pred: math.CompareOperator,
	3765	ty: Type,
	3766	params: [2]Builder.Value,
	3767	) Allocator.Error!Builder.Value {
	3768	const o = self.object;
	3769	const zcu = o.zcu;
	3770	const target = zcu.getTarget();
	3771	const scalar_ty = ty.scalarType(zcu);
	3772	const scalar_llvm_ty = try o.lowerType(scalar_ty);
	3773
	3774	if (intrinsicsAllowed(scalar_ty, target)) {
	3775	const cond: Builder.FloatCondition = switch (pred) {
	3776	.eq => .oeq,
	3777	.neq => .une,
	3778	.lt => .olt,
	3779	.lte => .ole,
	3780	.gt => .ogt,
	3781	.gte => .oge,
	3782	};
	3783	return self.wip.fcmp(fast, cond, params[0], params[1], "");
	3784	}
	3785
	3786	const float_bits = scalar_ty.floatBits(target);
	3787	const compiler_rt_float_abbrev = compilerRtFloatAbbrev(float_bits);
	3788	const fn_base_name = switch (pred) {
	3789	.neq => "ne",
	3790	.eq => "eq",
	3791	.lt => "lt",
	3792	.lte => "le",
	3793	.gt => "gt",
	3794	.gte => "ge",
	3795	};
	3796	const fn_name = try o.builder.strtabStringFmt("__{s}{s}f2", .{ fn_base_name, compiler_rt_float_abbrev });
	3797
	3798	const libc_fn = try o.getLibcFunction(fn_name, &.{ scalar_llvm_ty, scalar_llvm_ty }, .i32);
	3799
	3800	const int_cond: Builder.IntegerCondition = switch (pred) {
	3801	.eq => .eq,
	3802	.neq => .ne,
	3803	.lt => .slt,
	3804	.lte => .sle,
	3805	.gt => .sgt,
	3806	.gte => .sge,
	3807	};
	3808
	3809	if (ty.zigTypeTag(zcu) == .vector) {
	3810	const vec_len = ty.vectorLen(zcu);
	3811	const vector_result_ty = try o.builder.vectorType(.normal, vec_len, .i32);
	3812
	3813	const init = try o.builder.poisonValue(vector_result_ty);
	3814	const result = try self.buildElementwiseCall(libc_fn, &params, init, vec_len);
	3815
	3816	const zero_vector = try o.builder.splatValue(vector_result_ty, .@"0");
	3817	return self.wip.icmp(int_cond, result, zero_vector, "");
	3818	}
	3819
	3820	const result = try self.wip.call(
	3821	.normal,
	3822	.ccc,
	3823	.none,
	3824	libc_fn.typeOf(&o.builder),
	3825	libc_fn.toValue(&o.builder),
	3826	&params,
	3827	"",
	3828	);
	3829	return self.wip.icmp(int_cond, result, .@"0", "");
	3830	}
	3831
	3832	const FloatOp = enum {
	3833	add,
	3834	ceil,
	3835	cos,
	3836	div,
	3837	exp,
	3838	exp2,
	3839	fabs,
	3840	floor,
	3841	fma,
	3842	fmax,
	3843	fmin,
	3844	fmod,
	3845	log,
	3846	log10,
	3847	log2,
	3848	mul,
	3849	neg,
	3850	round,
	3851	sin,
	3852	sqrt,
	3853	sub,
	3854	tan,
	3855	trunc,
	3856	};
	3857
	3858	const FloatOpStrat = union(enum) {
	3859	intrinsic: []const u8,
	3860	libc: Builder.String,
	3861	};
	3862
	3863	/// Creates a floating point operation (add, sub, fma, sqrt, exp, etc.)
	3864	/// by lowering to the appropriate hardware instruction or softfloat
	3865	/// routine for the target
	3866	fn buildFloatOp(
	3867	self: *FuncGen,
	3868	comptime op: FloatOp,
	3869	fast: Builder.FastMathKind,
	3870	ty: Type,
	3871	comptime params_len: usize,
	3872	params: [params_len]Builder.Value,
	3873	) Allocator.Error!Builder.Value {
	3874	const o = self.object;
	3875	const zcu = o.zcu;
	3876	const target = zcu.getTarget();
	3877	const scalar_ty = ty.scalarType(zcu);
	3878	const llvm_ty = try o.lowerType(ty);
	3879
	3880	if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
	3881	// Some operations are dedicated LLVM instructions, not available as intrinsics
	3882	.neg => return self.wip.un(.fneg, params[0], ""),
	3883	.add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) {
	3884	.normal => switch (op) {
	3885	.add => .fadd,
	3886	.sub => .fsub,
	3887	.mul => .fmul,
	3888	.div => .fdiv,
	3889	.fmod => .frem,
	3890	else => unreachable,
	3891	},
	3892	.fast => switch (op) {
	3893	.add => .@"fadd fast",
	3894	.sub => .@"fsub fast",
	3895	.mul => .@"fmul fast",
	3896	.div => .@"fdiv fast",
	3897	.fmod => .@"frem fast",
	3898	else => unreachable,
	3899	},
	3900	}, params[0], params[1], ""),
	3901	.fmax,
	3902	.fmin,
	3903	.ceil,
	3904	.cos,
	3905	.exp,
	3906	.exp2,
	3907	.fabs,
	3908	.floor,
	3909	.log,
	3910	.log10,
	3911	.log2,
	3912	.round,
	3913	.sin,
	3914	.sqrt,
	3915	.trunc,
	3916	.fma,
	3917	=> return self.wip.callIntrinsic(fast, .none, switch (op) {
	3918	.fmax => .maxnum,
	3919	.fmin => .minnum,
	3920	.ceil => .ceil,
	3921	.cos => .cos,
	3922	.exp => .exp,
	3923	.exp2 => .exp2,
	3924	.fabs => .fabs,
	3925	.floor => .floor,
	3926	.log => .log,
	3927	.log10 => .log10,
	3928	.log2 => .log2,
	3929	.round => .round,
	3930	.sin => .sin,
	3931	.sqrt => .sqrt,
	3932	.trunc => .trunc,
	3933	.fma => .fma,
	3934	else => unreachable,
	3935	}, &.{llvm_ty}, &params, ""),
	3936	.tan => unreachable,
	3937	};
	3938
	3939	const float_bits = scalar_ty.floatBits(target);
	3940	const fn_name = switch (op) {
	3941	.neg => {
	3942	// In this case we can generate a softfloat negation by XORing the
	3943	// bits with a constant.
	3944	const int_ty = try o.builder.intType(@intCast(float_bits));
	3945	const cast_ty = switch (ty.zigTypeTag(zcu)) {
	3946	.vector => try o.builder.vectorType(.normal, ty.vectorLen(zcu), int_ty),
	3947	else => int_ty,
	3948	};
	3949	const sign_mask = try o.builder.splatValue(
	3950	cast_ty,
	3951	try o.builder.intConst(int_ty, @as(u128, 1) << @intCast(float_bits - 1)),
	3952	);
	3953	const bitcasted_operand = try self.wip.cast(.bitcast, params[0], cast_ty, "");
	3954	const result = try self.wip.bin(.xor, bitcasted_operand, sign_mask, "");
	3955	return self.wip.cast(.bitcast, result, llvm_ty, "");
	3956	},
	3957	.add, .sub, .div, .mul => try o.builder.strtabStringFmt("__{s}{s}f3", .{
	3958	@tagName(op), compilerRtFloatAbbrev(float_bits),
	3959	}),
	3960	.ceil,
	3961	.cos,
	3962	.exp,
	3963	.exp2,
	3964	.fabs,
	3965	.floor,
	3966	.fma,
	3967	.fmax,
	3968	.fmin,
	3969	.fmod,
	3970	.log,
	3971	.log10,
	3972	.log2,
	3973	.round,
	3974	.sin,
	3975	.sqrt,
	3976	.tan,
	3977	.trunc,
	3978	=> try o.builder.strtabStringFmt("{s}{s}{s}", .{
	3979	libcFloatPrefix(float_bits), @tagName(op), libcFloatSuffix(float_bits),
	3980	}),
	3981	};
	3982
	3983	const scalar_llvm_ty = try o.lowerType(scalar_ty);
	3984	const libc_fn = try o.getLibcFunction(
	3985	fn_name,
	3986	([1]Builder.Type{scalar_llvm_ty} ** 3)[0..params.len],
	3987	scalar_llvm_ty,
	3988	);
	3989	if (ty.zigTypeTag(zcu) == .vector) {
	3990	const result = try o.builder.poisonValue(llvm_ty);
	3991	return self.buildElementwiseCall(libc_fn, &params, result, ty.vectorLen(zcu));
	3992	}
	3993
	3994	return self.wip.call(
	3995	fast.toCallKind(),
	3996	.ccc,
	3997	.none,
	3998	libc_fn.typeOf(&o.builder),
	3999	libc_fn.toValue(&o.builder),
	4000	&params,
	4001	"",
	4002	);
	4003	}
	4004
	4005	fn airMulAdd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4006	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	4007	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
	4008
	4009	const mulend1 = try self.resolveInst(extra.lhs);
	4010	const mulend2 = try self.resolveInst(extra.rhs);
	4011	const addend = try self.resolveInst(pl_op.operand);
	4012
	4013	const ty = self.typeOfIndex(inst);
	4014	return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend });
	4015	}
	4016
	4017	fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4018	const o = self.object;
	4019	const zcu = o.zcu;
	4020	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	4021	const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
	4022
	4023	const lhs = try self.resolveInst(extra.lhs);
	4024	const rhs = try self.resolveInst(extra.rhs);
	4025
	4026	const lhs_ty = self.typeOf(extra.lhs);
	4027	if (lhs_ty.isVector(zcu) and !self.typeOf(extra.rhs).isVector(zcu)) {
	4028	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
	4029	// features which we do not use. Therefore this branch is currently impossible.
	4030	unreachable;
	4031	}
	4032
	4033	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
	4034
	4035	const dest_ty = self.typeOfIndex(inst);
	4036	assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
	4037	const llvm_dest_ty = try o.lowerType(dest_ty);
	4038
	4039	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4040
	4041	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
	4042	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
	4043	.ashr
	4044	else
	4045	.lshr, result, casted_rhs, "");
	4046
	4047	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
	4048
	4049	const result_alignment = dest_ty.abiAlignment(zcu).toLlvm();
	4050	const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment);
	4051
	4052	{
	4053	// Store to 'result: IntType' field
	4054	const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(0, zcu));
	4055	_ = try self.wip.store(.normal, result, field_ptr, lhs_ty.abiAlignment(zcu).toLlvm());
	4056	}
	4057
	4058	{
	4059	// Store to 'overflow: u1' field
	4060	const field_ptr = try self.ptraddConst(alloca_inst, dest_ty.structFieldOffset(1, zcu));
	4061	_ = try self.wip.store(.normal, overflow_bit, field_ptr, comptime .fromByteUnits(1));
	4062	}
	4063
	4064	return alloca_inst;
	4065	}
	4066
	4067	fn airAnd(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4068	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4069	const lhs = try self.resolveInst(bin_op.lhs);
	4070	const rhs = try self.resolveInst(bin_op.rhs);
	4071	return self.wip.bin(.@"and", lhs, rhs, "");
	4072	}
	4073
	4074	fn airOr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4075	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4076	const lhs = try self.resolveInst(bin_op.lhs);
	4077	const rhs = try self.resolveInst(bin_op.rhs);
	4078	return self.wip.bin(.@"or", lhs, rhs, "");
	4079	}
	4080
	4081	fn airXor(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4082	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4083	const lhs = try self.resolveInst(bin_op.lhs);
	4084	const rhs = try self.resolveInst(bin_op.rhs);
	4085	return self.wip.bin(.xor, lhs, rhs, "");
	4086	}
	4087
	4088	fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4089	const o = self.object;
	4090	const zcu = o.zcu;
	4091	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4092
	4093	const lhs = try self.resolveInst(bin_op.lhs);
	4094	const rhs = try self.resolveInst(bin_op.rhs);
	4095
	4096	const lhs_ty = self.typeOf(bin_op.lhs);
	4097	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
	4098	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
	4099	// features which we do not use. Therefore this branch is currently impossible.
	4100	unreachable;
	4101	}
	4102	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
	4103
	4104	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4105	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
	4106	.@"shl nsw"
	4107	else
	4108	.@"shl nuw", lhs, casted_rhs, "");
	4109	}
	4110
	4111	fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4112	const o = self.object;
	4113	const zcu = o.zcu;
	4114	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4115
	4116	const lhs = try self.resolveInst(bin_op.lhs);
	4117	const rhs = try self.resolveInst(bin_op.rhs);
	4118
	4119	const lhs_ty = self.typeOf(bin_op.lhs);
	4120	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
	4121	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
	4122	// features which we do not use. Therefore this branch is currently impossible.
	4123	unreachable;
	4124	}
	4125	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4126	return self.wip.bin(.shl, lhs, casted_rhs, "");
	4127	}
	4128
	4129	fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4130	const o = self.object;
	4131	const zcu = o.zcu;
	4132	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4133
	4134	const lhs = try self.resolveInst(bin_op.lhs);
	4135	const rhs = try self.resolveInst(bin_op.rhs);
	4136
	4137	const lhs_ty = self.typeOf(bin_op.lhs);
	4138	const lhs_info = lhs_ty.intInfo(zcu);
	4139	const llvm_lhs_ty = try o.lowerType(lhs_ty);
	4140	const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu));
	4141
	4142	const rhs_ty = self.typeOf(bin_op.rhs);
	4143	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
	4144	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
	4145	// features which we do not use. Therefore this branch is currently impossible.
	4146	unreachable;
	4147	}
	4148	const rhs_info = rhs_ty.intInfo(zcu);
	4149	assert(rhs_info.signedness == .unsigned);
	4150	const llvm_rhs_ty = try o.lowerType(rhs_ty);
	4151	const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu));
	4152
	4153	const result = try self.wip.callIntrinsic(
	4154	.normal,
	4155	.none,
	4156	switch (lhs_info.signedness) {
	4157	.signed => .@"sshl.sat",
	4158	.unsigned => .@"ushl.sat",
	4159	},
	4160	&.{llvm_lhs_ty},
	4161	&.{ lhs, try self.wip.conv(.unsigned, rhs, llvm_lhs_ty, "") },
	4162	"",
	4163	);
	4164
	4165	// LLVM langref says "If b is (statically or dynamically) equal to or
	4166	// larger than the integer bit width of the arguments, the result is a
	4167	// poison value."
	4168	// However Zig semantics says that saturating shift left can never produce
	4169	// undefined; instead it saturates.
	4170	if (rhs_info.bits <= math.log2_int(u16, lhs_info.bits)) return result;
	4171	const bits = try o.builder.splatValue(
	4172	llvm_rhs_ty,
	4173	try o.builder.intConst(llvm_rhs_scalar_ty, lhs_info.bits),
	4174	);
	4175	const in_range = try self.wip.icmp(.ult, rhs, bits, "");
	4176	const lhs_sat = lhs_sat: switch (lhs_info.signedness) {
	4177	.signed => {
	4178	const zero = try o.builder.splatValue(
	4179	llvm_lhs_ty,
	4180	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
	4181	);
	4182	const smin = try o.builder.splatValue(
	4183	llvm_lhs_ty,
	4184	try minIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
	4185	);
	4186	const smax = try o.builder.splatValue(
	4187	llvm_lhs_ty,
	4188	try maxIntConst(&o.builder, lhs_ty, llvm_lhs_ty, zcu),
	4189	);
	4190	const lhs_lt_zero = try self.wip.icmp(.slt, lhs, zero, "");
	4191	const slimit = try self.wip.select(.normal, lhs_lt_zero, smin, smax, "");
	4192	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
	4193	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, slimit, "");
	4194	},
	4195	.unsigned => {
	4196	const zero = try o.builder.splatValue(
	4197	llvm_lhs_ty,
	4198	try o.builder.intConst(llvm_lhs_scalar_ty, 0),
	4199	);
	4200	const umax = try o.builder.splatValue(
	4201	llvm_lhs_ty,
	4202	try o.builder.intConst(llvm_lhs_scalar_ty, -1),
	4203	);
	4204	const lhs_eq_zero = try self.wip.icmp(.eq, lhs, zero, "");
	4205	break :lhs_sat try self.wip.select(.normal, lhs_eq_zero, zero, umax, "");
	4206	},
	4207	};
	4208	return self.wip.select(.normal, in_range, result, lhs_sat, "");
	4209	}
	4210
	4211	fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!Builder.Value {
	4212	const o = self.object;
	4213	const zcu = o.zcu;
	4214	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4215
	4216	const lhs = try self.resolveInst(bin_op.lhs);
	4217	const rhs = try self.resolveInst(bin_op.rhs);
	4218
	4219	const lhs_ty = self.typeOf(bin_op.lhs);
	4220	if (lhs_ty.isVector(zcu) and !self.typeOf(bin_op.rhs).isVector(zcu)) {
	4221	// `Sema` does not currently emit this pattern---instead it is specific to `Air.Legalize`
	4222	// features which we do not use. Therefore this branch is currently impossible.
	4223	unreachable;
	4224	}
	4225	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
	4226
	4227	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4228	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
	4229
	4230	return self.wip.bin(if (is_exact)
	4231	if (is_signed_int) .@"ashr exact" else .@"lshr exact"
	4232	else if (is_signed_int) .ashr else .lshr, lhs, casted_rhs, "");
	4233	}
	4234
	4235	fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4236	const o = self.object;
	4237	const zcu = o.zcu;
	4238	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4239	const operand = try self.resolveInst(ty_op.operand);
	4240	const operand_ty = self.typeOf(ty_op.operand);
	4241	const scalar_ty = operand_ty.scalarType(zcu);
	4242
	4243	switch (scalar_ty.zigTypeTag(zcu)) {
	4244	.int => return self.wip.callIntrinsic(
	4245	.normal,
	4246	.none,
	4247	.abs,
	4248	&.{try o.lowerType(operand_ty)},
	4249	&.{ operand, try o.builder.intValue(.i1, 0) },
	4250	"",
	4251	),
	4252	.float => return self.buildFloatOp(.fabs, .normal, operand_ty, 1, .{operand}),
	4253	else => unreachable,
	4254	}
	4255	}
	4256
	4257	fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
	4258	const o = fg.object;
	4259	const zcu = o.zcu;
	4260	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4261	const dest_ty = fg.typeOfIndex(inst);
	4262	const dest_llvm_ty = try o.lowerType(dest_ty);
	4263	const operand = try fg.resolveInst(ty_op.operand);
	4264	const operand_ty = fg.typeOf(ty_op.operand);
	4265	const operand_info = operand_ty.intInfo(zcu);
	4266
	4267	const dest_is_enum = dest_ty.zigTypeTag(zcu) == .@"enum";
	4268
	4269	bounds_check: {
	4270	const dest_scalar = dest_ty.scalarType(zcu);
	4271	const operand_scalar = operand_ty.scalarType(zcu);
	4272
	4273	const dest_info = dest_ty.intInfo(zcu);
	4274
	4275	const have_min_check, const have_max_check = c: {
	4276	const dest_pos_bits = dest_info.bits - @intFromBool(dest_info.signedness == .signed);
	4277	const operand_pos_bits = operand_info.bits - @intFromBool(operand_info.signedness == .signed);
	4278
	4279	const dest_allows_neg = dest_info.signedness == .signed and dest_info.bits > 0;
	4280	const operand_maybe_neg = operand_info.signedness == .signed and operand_info.bits > 0;
	4281
	4282	break :c .{
	4283	operand_maybe_neg and (!dest_allows_neg or dest_info.bits < operand_info.bits),
	4284	dest_pos_bits < operand_pos_bits,
	4285	};
	4286	};
	4287
	4288	if (!have_min_check and !have_max_check) break :bounds_check;
	4289
	4290	const operand_llvm_ty = try o.lowerType(operand_ty);
	4291	const operand_scalar_llvm_ty = try o.lowerType(operand_scalar);
	4292
	4293	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
	4294	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
	4295
	4296	const panic_id: Zcu.SimplePanicId = if (dest_is_enum) .invalid_enum_value else .integer_out_of_bounds;
	4297
	4298	if (have_min_check) {
	4299	const min_const_scalar = try minIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
	4300	const min_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, min_const_scalar) else min_const_scalar.toValue();
	4301	const ok_maybe_vec = try fg.cmp(.normal, .gte, operand_ty, operand, min_val);
	4302	const ok = if (is_vector) ok: {
	4303	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
	4304	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
	4305	} else ok_maybe_vec;
	4306	if (safety) {
	4307	const fail_block = try fg.wip.block(1, "IntMinFail");
	4308	const ok_block = try fg.wip.block(1, "IntMinOk");
	4309	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
	4310	fg.wip.cursor = .{ .block = fail_block };
	4311	try fg.buildSimplePanic(panic_id);
	4312	fg.wip.cursor = .{ .block = ok_block };
	4313	} else {
	4314	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
	4315	}
	4316	}
	4317
	4318	if (have_max_check) {
	4319	const max_const_scalar = try maxIntConst(&o.builder, dest_scalar, operand_scalar_llvm_ty, zcu);
	4320	const max_val = if (is_vector) try o.builder.splatValue(operand_llvm_ty, max_const_scalar) else max_const_scalar.toValue();
	4321	const ok_maybe_vec = try fg.cmp(.normal, .lte, operand_ty, operand, max_val);
	4322	const ok = if (is_vector) ok: {
	4323	const vec_ty = ok_maybe_vec.typeOfWip(&fg.wip);
	4324	break :ok try fg.wip.callIntrinsic(.normal, .none, .@"vector.reduce.and", &.{vec_ty}, &.{ok_maybe_vec}, "");
	4325	} else ok_maybe_vec;
	4326	if (safety) {
	4327	const fail_block = try fg.wip.block(1, "IntMaxFail");
	4328	const ok_block = try fg.wip.block(1, "IntMaxOk");
	4329	_ = try fg.wip.brCond(ok, ok_block, fail_block, .none);
	4330	fg.wip.cursor = .{ .block = fail_block };
	4331	try fg.buildSimplePanic(panic_id);
	4332	fg.wip.cursor = .{ .block = ok_block };
	4333	} else {
	4334	_ = try fg.wip.callIntrinsic(.normal, .none, .assume, &.{}, &.{ok}, "");
	4335	}
	4336	}
	4337	}
	4338
	4339	const result = try fg.wip.conv(switch (operand_info.signedness) {
	4340	.signed => .signed,
	4341	.unsigned => .unsigned,
	4342	}, operand, dest_llvm_ty, "");
	4343
	4344	if (safety and dest_is_enum and !dest_ty.isNonexhaustiveEnum(zcu)) {
	4345	const llvm_fn = try o.getIsNamedEnumValueFunction(dest_ty);
	4346	const is_valid_enum_val = try fg.wip.call(
	4347	.normal,
	4348	.fastcc,
	4349	.none,
	4350	llvm_fn.typeOf(&o.builder),
	4351	llvm_fn.toValue(&o.builder),
	4352	&.{result},
	4353	"",
	4354	);
	4355	const fail_block = try fg.wip.block(1, "ValidEnumFail");
	4356	const ok_block = try fg.wip.block(1, "ValidEnumOk");
	4357	_ = try fg.wip.brCond(is_valid_enum_val, ok_block, fail_block, .none);
	4358	fg.wip.cursor = .{ .block = fail_block };
	4359	try fg.buildSimplePanic(.invalid_enum_value);
	4360	fg.wip.cursor = .{ .block = ok_block };
	4361	}
	4362
	4363	return result;
	4364	}
	4365
	4366	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4367	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4368	const operand = try self.resolveInst(ty_op.operand);
	4369	const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst));
	4370	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
	4371	}
	4372
	4373	fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4374	const o = self.object;
	4375	const zcu = o.zcu;
	4376	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4377	const operand = try self.resolveInst(ty_op.operand);
	4378	const operand_ty = self.typeOf(ty_op.operand);
	4379	const dest_ty = self.typeOfIndex(inst);
	4380	const target = zcu.getTarget();
	4381
	4382	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
	4383	return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), "");
	4384	} else {
	4385	const operand_llvm_ty = try o.lowerType(operand_ty);
	4386	const dest_llvm_ty = try o.lowerType(dest_ty);
	4387
	4388	const dest_bits = dest_ty.floatBits(target);
	4389	const src_bits = operand_ty.floatBits(target);
	4390	const fn_name = try o.builder.strtabStringFmt("__trunc{s}f{s}f2", .{
	4391	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
	4392	});
	4393
	4394	const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
	4395	return self.wip.call(
	4396	.normal,
	4397	.ccc,
	4398	.none,
	4399	libc_fn.typeOf(&o.builder),
	4400	libc_fn.toValue(&o.builder),
	4401	&.{operand},
	4402	"",
	4403	);
	4404	}
	4405	}
	4406
	4407	fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4408	const o = self.object;
	4409	const zcu = o.zcu;
	4410	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4411	const operand = try self.resolveInst(ty_op.operand);
	4412	const operand_ty = self.typeOf(ty_op.operand);
	4413	const dest_ty = self.typeOfIndex(inst);
	4414	const target = zcu.getTarget();
	4415
	4416	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
	4417	return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
	4418	} else {
	4419	const operand_llvm_ty = try o.lowerType(operand_ty);
	4420	const dest_llvm_ty = try o.lowerType(dest_ty);
	4421
	4422	const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
	4423	const src_bits = operand_ty.scalarType(zcu).floatBits(target);
	4424	const fn_name = try o.builder.strtabStringFmt("__extend{s}f{s}f2", .{
	4425	compilerRtFloatAbbrev(src_bits), compilerRtFloatAbbrev(dest_bits),
	4426	});
	4427
	4428	const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, dest_llvm_ty);
	4429	if (dest_ty.isVector(zcu)) return self.buildElementwiseCall(
	4430	libc_fn,
	4431	&.{operand},
	4432	try o.builder.poisonValue(dest_llvm_ty),
	4433	dest_ty.vectorLen(zcu),
	4434	);
	4435	return self.wip.call(
	4436	.normal,
	4437	.ccc,
	4438	.none,
	4439	libc_fn.typeOf(&o.builder),
	4440	libc_fn.toValue(&o.builder),
	4441	&.{operand},
	4442	"",
	4443	);
	4444	}
	4445	}
	4446
	4447	fn airBitCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4448	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4449	const operand_ty = self.typeOf(ty_op.operand);
	4450	const inst_ty = self.typeOfIndex(inst);
	4451	const operand = try self.resolveInst(ty_op.operand);
	4452	return self.bitCast(operand, operand_ty, inst_ty);
	4453	}
	4454
	4455	fn bitCast(self: *FuncGen, operand: Builder.Value, operand_ty: Type, inst_ty: Type) Allocator.Error!Builder.Value {
	4456	const o = self.object;
	4457	const zcu = o.zcu;
	4458	const operand_is_ref = isByRef(operand_ty, zcu);
	4459	const result_is_ref = isByRef(inst_ty, zcu);
	4460	const llvm_dest_ty = try o.lowerType(inst_ty);
	4461
	4462	if (operand_is_ref and result_is_ref) {
	4463	// They are both pointers, so just return the same opaque pointer :)
	4464	return operand;
	4465	}
	4466
	4467	if (inst_ty.isAbiInt(zcu) and operand_ty.isAbiInt(zcu)) {
	4468	return self.wip.conv(.unsigned, operand, llvm_dest_ty, "");
	4469	}
	4470
	4471	const operand_scalar_ty = operand_ty.scalarType(zcu);
	4472	const inst_scalar_ty = inst_ty.scalarType(zcu);
	4473	if (operand_scalar_ty.zigTypeTag(zcu) == .int and inst_scalar_ty.isPtrAtRuntime(zcu)) {
	4474	return self.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
	4475	}
	4476	if (operand_scalar_ty.isPtrAtRuntime(zcu) and inst_scalar_ty.zigTypeTag(zcu) == .int) {
	4477	return self.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
	4478	}
	4479
	4480	if (operand_ty.zigTypeTag(zcu) == .vector and inst_ty.zigTypeTag(zcu) == .array) {
	4481	const elem_ty = operand_scalar_ty;
	4482	assert(result_is_ref); // arrays are always by-ref provided they have runtime bits
	4483	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
	4484	const array_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
	4485	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
	4486	if (bitcast_ok) {
	4487	_ = try self.wip.store(.normal, operand, array_ptr, alignment);
	4488	} else {
	4489	// If the ABI size of the element type is not evenly divisible by size in bits;
	4490	// a simple bitcast will not work, and we fall back to extractelement.
	4491	const elem_size = elem_ty.abiSize(zcu);
	4492	const vector_len = operand_ty.arrayLen(zcu);
	4493	var i: u64 = 0;
	4494	while (i < vector_len) : (i += 1) {
	4495	const arr_elem_ptr = try self.ptraddConst(array_ptr, i * elem_size);
	4496	const vec_elem = try self.wip.extractElement(operand, try o.builder.intValue(.i32, i), "");
	4497	_ = try self.wip.store(.normal, vec_elem, arr_elem_ptr, .default);
	4498	}
	4499	}
	4500	return array_ptr;
	4501	} else if (operand_ty.zigTypeTag(zcu) == .array and inst_ty.zigTypeTag(zcu) == .vector) {
	4502	const elem_ty = operand_ty.childType(zcu);
	4503	assert(operand_is_ref); // arrays are always by-ref provided they have runtime bits
	4504	const llvm_vector_ty = try o.lowerType(inst_ty);
	4505
	4506	const bitcast_ok = elem_ty.bitSize(zcu) == elem_ty.abiSize(zcu) * 8;
	4507	if (bitcast_ok) {
	4508	// The array is aligned to the element's alignment, while the vector might have a completely
	4509	// different alignment. This means we need to enforce the alignment of this load.
	4510	const alignment = elem_ty.abiAlignment(zcu).toLlvm();
	4511	return self.wip.load(.normal, llvm_vector_ty, operand, alignment, "");
	4512	} else {
	4513	// If the ABI size of the element type is not evenly divisible by size in bits;
	4514	// a simple bitcast will not work, and we fall back to extractelement.
	4515	const elem_llvm_ty = try o.lowerType(elem_ty);
	4516	const elem_size = elem_ty.abiSize(zcu);
	4517	const vector_len = operand_ty.arrayLen(zcu);
	4518	var vector = try o.builder.poisonValue(llvm_vector_ty);
	4519	var i: u64 = 0;
	4520	while (i < vector_len) : (i += 1) {
	4521	const arr_elem_ptr = try self.ptraddConst(operand, i * elem_size);
	4522	const arr_elem = try self.wip.load(.normal, elem_llvm_ty, arr_elem_ptr, .default, "");
	4523	vector = try self.wip.insertElement(vector, arr_elem, try o.builder.intValue(.i32, i), "");
	4524	}
	4525	return vector;
	4526	}
	4527	}
	4528
	4529	if (operand_is_ref) {
	4530	const alignment = operand_ty.abiAlignment(zcu).toLlvm();
	4531	return self.wip.load(.normal, llvm_dest_ty, operand, alignment, "");
	4532	}
	4533
	4534	if (result_is_ref) {
	4535	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
	4536	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
	4537	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
	4538	return result_ptr;
	4539	}
	4540
	4541	if (inst_ty.isSliceAtRuntime(zcu) or
	4542	((operand_ty.zigTypeTag(zcu) == .vector or inst_ty.zigTypeTag(zcu) == .vector) and
	4543	operand_ty.bitSize(zcu) != inst_ty.bitSize(zcu)))
	4544	{
	4545	// Both our operand and our result are values, not pointers,
	4546	// but LLVM won't let us bitcast struct values or vectors with padding bits.
	4547	// Therefore, we store operand to alloca, then load for result.
	4548	const alignment = operand_ty.abiAlignment(zcu).max(inst_ty.abiAlignment(zcu)).toLlvm();
	4549	const result_ptr = try self.buildAlloca(llvm_dest_ty, alignment);
	4550	_ = try self.wip.store(.normal, operand, result_ptr, alignment);
	4551	return self.wip.load(.normal, llvm_dest_ty, result_ptr, alignment, "");
	4552	}
	4553
	4554	return self.wip.cast(.bitcast, operand, llvm_dest_ty, "");
	4555	}
	4556
	4557	fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4558	const o = self.object;
	4559	const pt = self.pt;
	4560	const zcu = o.zcu;
	4561	const arg_val = self.args[self.arg_index];
	4562	self.arg_index += 1;
	4563
	4564	// llvm does not support debug info for naked function arguments
	4565	if (self.is_naked) return arg_val;
	4566
	4567	const inst_ty = self.typeOfIndex(inst);
	4568
	4569	const func = zcu.funcInfo(zcu.navValue(self.nav_index).toIntern());
	4570	const func_zir = func.zir_body_inst.resolveFull(&zcu.intern_pool).?;
	4571	const file = zcu.fileByIndex(func_zir.file);
	4572
	4573	const mod = file.mod.?;
	4574	if (mod.strip) return arg_val;
	4575	const arg = self.air.instructions.items(.data)[@intFromEnum(inst)].arg;
	4576	const zir = &file.zir.?;
	4577	const name = zir.nullTerminatedString(zir.getParamName(zir.getParamBody(func_zir.inst)[arg.zir_param_index]).?);
	4578
	4579	const lbrace_line = zcu.navSrcLine(func.owner_nav) + func.lbrace_line + 1;
	4580	const lbrace_col = func.lbrace_column + 1;
	4581
	4582	const debug_parameter = try o.builder.debugParameter(
	4583	if (name.len > 0) try o.builder.metadataString(name) else null,
	4584	self.file,
	4585	self.scope,
	4586	lbrace_line,
	4587	try o.getDebugType(pt, inst_ty),
	4588	self.arg_index,
	4589	);
	4590
	4591	const old_location = self.wip.debug_location;
	4592	self.wip.debug_location = .{ .location = .{
	4593	.line = lbrace_line,
	4594	.column = lbrace_col,
	4595	.scope = self.scope.toOptional(),
	4596	.inlined_at = .none,
	4597	} };
	4598
	4599	if (isByRef(inst_ty, zcu)) {
	4600	_ = try self.wip.callIntrinsic(
	4601	.normal,
	4602	.none,
	4603	.@"dbg.declare",
	4604	&.{},
	4605	&.{
	4606	(try self.wip.debugValue(arg_val)).toValue(),
	4607	debug_parameter.toValue(),
	4608	(try o.builder.debugExpression(&.{})).toValue(),
	4609	},
	4610	"",
	4611	);
	4612	} else if (mod.optimize_mode == .Debug) {
	4613	const alignment = inst_ty.abiAlignment(zcu).toLlvm();
	4614	const alloca = try self.buildAlloca(arg_val.typeOfWip(&self.wip), alignment);
	4615	_ = try self.wip.store(.normal, arg_val, alloca, alignment);
	4616	_ = try self.wip.callIntrinsic(
	4617	.normal,
	4618	.none,
	4619	.@"dbg.declare",
	4620	&.{},
	4621	&.{
	4622	(try self.wip.debugValue(alloca)).toValue(),
	4623	debug_parameter.toValue(),
	4624	(try o.builder.debugExpression(&.{})).toValue(),
	4625	},
	4626	"",
	4627	);
	4628	} else {
	4629	_ = try self.wip.callIntrinsic(
	4630	.normal,
	4631	.none,
	4632	.@"dbg.value",
	4633	&.{},
	4634	&.{
	4635	(try self.wip.debugValue(arg_val)).toValue(),
	4636	debug_parameter.toValue(),
	4637	(try o.builder.debugExpression(&.{})).toValue(),
	4638	},
	4639	"",
	4640	);
	4641	}
	4642
	4643	self.wip.debug_location = old_location;
	4644	return arg_val;
	4645	}
	4646
	4647	fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4648	const o = self.object;
	4649	const zcu = o.zcu;
	4650	const ptr_ty = self.typeOfIndex(inst);
	4651	const ptr_align = ptr_ty.ptrAlignment(zcu);
	4652	const elem_ty = ptr_ty.childType(zcu);
	4653	if (!elem_ty.hasRuntimeBits(zcu)) {
	4654	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
	4655	}
	4656	const llvm_elem_ty = try o.lowerType(elem_ty);
	4657	return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4658	}
	4659
	4660	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4661	if (self.ret_ptr != .none) return self.ret_ptr;
	4662	const o = self.object;
	4663	const zcu = o.zcu;
	4664	const ptr_ty = self.typeOfIndex(inst);
	4665	const ptr_align = ptr_ty.ptrAlignment(zcu);
	4666	const elem_ty = ptr_ty.childType(zcu);
	4667	if (!elem_ty.hasRuntimeBits(zcu)) {
	4668	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
	4669	}
	4670	const llvm_elem_ty = try o.lowerType(elem_ty);
	4671	return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4672	}
	4673
	4674	/// Use this instead of builder.buildAlloca, because this function makes sure to
	4675	/// put the alloca instruction at the top of the function!
	4676	fn buildAlloca(
	4677	self: *FuncGen,
	4678	llvm_ty: Builder.Type,
	4679	alignment: Builder.Alignment,
	4680	) Allocator.Error!Builder.Value {
	4681	const target = self.object.zcu.getTarget();
	4682	return buildAllocaInner(&self.wip, llvm_ty, alignment, target);
	4683	}
	4684
	4685	fn airStore(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
	4686	const o = self.object;
	4687	const zcu = o.zcu;
	4688	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	4689	const dest_ptr = try self.resolveInst(bin_op.lhs);
	4690	const ptr_ty = self.typeOf(bin_op.lhs);
	4691	const operand_ty = ptr_ty.childType(zcu);
	4692
	4693	const val_is_undef = if (bin_op.rhs.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
	4694	if (val_is_undef) {
	4695	const owner_mod = self.ownerModule();
	4696
	4697	// Even if safety is disabled, we still emit a memset to undefined since it conveys
	4698	// extra information to LLVM, and LLVM will optimize it out. Safety makes the difference
	4699	// between using 0xaa or actual undefined for the fill byte.
	4700	//
	4701	// However, for Debug builds specifically, we avoid emitting the memset because LLVM
	4702	// will neither use the information nor get rid of the memset, thus leaving an
	4703	// unexpected call in the user's code. This is problematic if the code in question is
	4704	// not ready to correctly make calls yet, such as in our early PIE startup code, or in
	4705	// the early stages of a dynamic linker, etc.
	4706	if (!safety and owner_mod.optimize_mode == .Debug) {
	4707	return .none;
	4708	}
	4709
	4710	const ptr_info = ptr_ty.ptrInfo(zcu);
	4711	const needs_bitmask = (ptr_info.packed_offset.host_size != 0);
	4712	if (needs_bitmask) {
	4713	// TODO: only some bits are to be undef, we cannot write with a simple memset.
	4714	// meanwhile, ignore the write rather than stomping over valid bits.
	4715	// https://github.com/ziglang/zig/issues/15337
	4716	return .none;
	4717	}
	4718
	4719	self.maybeMarkAllowZeroAccess(ptr_info);
	4720
	4721	const len = try o.builder.intValue(try o.lowerType(.usize), operand_ty.abiSize(zcu));
	4722	_ = try self.wip.callMemSet(
	4723	dest_ptr,
	4724	ptr_ty.ptrAlignment(zcu).toLlvm(),
	4725	if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
	4726	len,
	4727	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	4728	self.disable_intrinsics,
	4729	);
	4730	if (safety and owner_mod.valgrind) {
	4731	try self.valgrindMarkUndef(dest_ptr, len);
	4732	}
	4733	return .none;
	4734	}
	4735
	4736	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	4737
	4738	const src_operand = try self.resolveInst(bin_op.rhs);
	4739	try self.storeFull(dest_ptr, ptr_ty, src_operand, .none);
	4740	return .none;
	4741	}
	4742
	4743	fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4744	const o = fg.object;
	4745	const zcu = o.zcu;
	4746	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4747	const ptr_ty = fg.typeOf(ty_op.operand);
	4748	const ptr_info = ptr_ty.ptrInfo(zcu);
	4749	const ptr = try fg.resolveInst(ty_op.operand);
	4750	const elem_ty = ptr_ty.childType(zcu);
	4751	const llvm_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
	4752
	4753	fg.maybeMarkAllowZeroAccess(ptr_info);
	4754
	4755	const access_kind: Builder.MemoryAccessKind =
	4756	if (ptr_info.flags.is_volatile) .@"volatile" else .normal;
	4757
	4758	if (ptr_info.flags.vector_index != .none) {
	4759	const index_u32 = try o.builder.intValue(.i32, ptr_info.flags.vector_index);
	4760	const vec_elem_ty = try o.lowerType(elem_ty);
	4761	const vec_ty = try o.builder.vectorType(.normal, ptr_info.packed_offset.host_size, vec_elem_ty);
	4762
	4763	const loaded_vector = try fg.wip.load(access_kind, vec_ty, ptr, llvm_ptr_align, "");
	4764	return fg.wip.extractElement(loaded_vector, index_u32, "");
	4765	}
	4766
	4767	if (ptr_info.packed_offset.host_size == 0) {
	4768	return fg.load(ptr, elem_ty, llvm_ptr_align, access_kind);
	4769	}
	4770
	4771	const containing_int_ty = try o.builder.intType(@intCast(ptr_info.packed_offset.host_size * 8));
	4772	const containing_int =
	4773	try fg.wip.load(access_kind, containing_int_ty, ptr, llvm_ptr_align, "");
	4774
	4775	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
	4776	const shift_amt = try o.builder.intValue(containing_int_ty, ptr_info.packed_offset.bit_offset);
	4777	const shifted_value = try fg.wip.bin(.lshr, containing_int, shift_amt, "");
	4778	const elem_llvm_ty = try o.lowerType(elem_ty);
	4779
	4780	if (isByRef(elem_ty, zcu)) {
	4781	const result_align = elem_ty.abiAlignment(zcu).toLlvm();
	4782	const result_ptr = try fg.buildAlloca(elem_llvm_ty, result_align);
	4783
	4784	const same_size_int = try o.builder.intType(@intCast(elem_bits));
	4785	const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
	4786	_ = try fg.wip.store(.normal, truncated_int, result_ptr, result_align);
	4787	return result_ptr;
	4788	}
	4789
	4790	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
	4791	const same_size_int = try o.builder.intType(@intCast(elem_bits));
	4792	const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
	4793	return fg.wip.cast(.bitcast, truncated_int, elem_llvm_ty, "");
	4794	}
	4795
	4796	if (elem_ty.isPtrAtRuntime(zcu)) {
	4797	const same_size_int = try o.builder.intType(@intCast(elem_bits));
	4798	const truncated_int = try fg.wip.cast(.trunc, shifted_value, same_size_int, "");
	4799	return fg.wip.cast(.inttoptr, truncated_int, elem_llvm_ty, "");
	4800	}
	4801
	4802	return fg.wip.cast(.trunc, shifted_value, elem_llvm_ty, "");
	4803	}
	4804
	4805	fn airTrap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
	4806	_ = inst;
	4807	const target = self.object.zcu.getTarget();
	4808	if ((target.cpu.arch == .mips or target.cpu.arch == .mipsel) and
	4809	target.cpu.has(.mips, .notraps))
	4810	{
	4811	// Emit a MIPS `break` instruction followed by an infinite loop (to fulfil the noreturn)
	4812	// since this CPU does not support trap instructions.
	4813	const o = self.object;
	4814	_ = try self.wip.callAsm(
	4815	.none,
	4816	try o.builder.fnType(.void, &.{}, .normal),
	4817	.{ .sideeffect = true },
	4818	try o.builder.string("break\n0:\nj 0b\nnop"),
	4819	try o.builder.string("~{memory}"),
	4820	&.{},
	4821	"",
	4822	);
	4823	} else {
	4824	_ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, "");
	4825	}
	4826	_ = try self.wip.@"unreachable"();
	4827	}
	4828
	4829	fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4830	_ = inst;
	4831	_ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, "");
	4832	return .none;
	4833	}
	4834
	4835	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4836	_ = inst;
	4837	const o = self.object;
	4838	const llvm_usize = try o.lowerType(.usize);
	4839	if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
	4840	// https://github.com/ziglang/zig/issues/11946
	4841	return o.builder.intValue(llvm_usize, 0);
	4842	}
	4843	const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{.@"0"}, "");
	4844	return self.wip.cast(.ptrtoint, result, llvm_usize, "");
	4845	}
	4846
	4847	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4848	_ = inst;
	4849	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
	4850	return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), "");
	4851	}
	4852
	4853	fn airCmpxchg(
	4854	self: *FuncGen,
	4855	inst: Air.Inst.Index,
	4856	kind: Builder.Function.Instruction.CmpXchg.Kind,
	4857	) Allocator.Error!Builder.Value {
	4858	const o = self.object;
	4859	const zcu = o.zcu;
	4860	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	4861	const extra = self.air.extraData(Air.Cmpxchg, ty_pl.payload).data;
	4862	const ptr = try self.resolveInst(extra.ptr);
	4863	const ptr_ty = self.typeOf(extra.ptr);
	4864	var expected_value = try self.resolveInst(extra.expected_value);
	4865	var new_value = try self.resolveInst(extra.new_value);
	4866	const operand_ty = ptr_ty.childType(zcu);
	4867	const llvm_operand_ty = try o.lowerType(operand_ty);
	4868	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
	4869	if (llvm_abi_ty != .none) {
	4870	// operand needs widening and truncating
	4871	const signedness: Builder.Function.Instruction.Cast.Signedness =
	4872	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned;
	4873	expected_value = try self.wip.conv(signedness, expected_value, llvm_abi_ty, "");
	4874	new_value = try self.wip.conv(signedness, new_value, llvm_abi_ty, "");
	4875	}
	4876
	4877	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	4878
	4879	const result = try self.wip.cmpxchg(
	4880	kind,
	4881	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
	4882	ptr,
	4883	expected_value,
	4884	new_value,
	4885	self.sync_scope,
	4886	toLlvmAtomicOrdering(extra.successOrder()),
	4887	toLlvmAtomicOrdering(extra.failureOrder()),
	4888	ptr_ty.ptrAlignment(zcu).toLlvm(),
	4889	"",
	4890	);
	4891
	4892	const optional_ty = self.typeOfIndex(inst);
	4893
	4894	var payload = try self.wip.extractValue(result, &.{0}, "");
	4895	if (llvm_abi_ty != .none) payload = try self.wip.cast(.trunc, payload, llvm_operand_ty, "");
	4896	const success_bit = try self.wip.extractValue(result, &.{1}, "");
	4897
	4898	if (optional_ty.optionalReprIsPayload(zcu)) {
	4899	const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip));
	4900	return self.wip.select(.normal, success_bit, zero, payload, "");
	4901	}
	4902
	4903	assert(isByRef(optional_ty, zcu));
	4904
	4905	comptime assert(optional_layout_version == 3);
	4906
	4907	const non_null_bit = try self.wip.not(success_bit, "");
	4908
	4909	const payload_align = operand_ty.abiAlignment(zcu).toLlvm();
	4910	const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align);
	4911
	4912	// Payload is always the first field at offset 0, so address is `alloca_inst`
	4913	_ = try self.wip.store(.normal, payload, alloca_inst, payload_align);
	4914
	4915	// Non-null bit is after payload with no padding because it has alignment 1
	4916	const non_null_ptr = try self.ptraddConst(alloca_inst, operand_ty.abiSize(zcu));
	4917	_ = try self.wip.store(.normal, non_null_bit, non_null_ptr, comptime .fromByteUnits(1));
	4918
	4919	return alloca_inst;
	4920	}
	4921
	4922	fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4923	const o = self.object;
	4924	const zcu = o.zcu;
	4925	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	4926	const extra = self.air.extraData(Air.AtomicRmw, pl_op.payload).data;
	4927	const ptr = try self.resolveInst(pl_op.operand);
	4928	const ptr_ty = self.typeOf(pl_op.operand);
	4929	const operand_ty = ptr_ty.childType(zcu);
	4930	const operand = try self.resolveInst(extra.operand);
	4931	const is_signed_int = operand_ty.isSignedInt(zcu);
	4932	const is_float = operand_ty.isRuntimeFloat();
	4933	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
	4934	const ordering = toLlvmAtomicOrdering(extra.ordering());
	4935	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
	4936	const llvm_operand_ty = try o.lowerType(operand_ty);
	4937
	4938	const access_kind: Builder.MemoryAccessKind =
	4939	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	4940	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
	4941
	4942	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	4943
	4944	if (llvm_abi_ty != .none) {
	4945	// operand needs widening and truncating or bitcasting.
	4946	return self.wip.cast(if (is_float) .bitcast else .trunc, try self.wip.atomicrmw(
	4947	access_kind,
	4948	op,
	4949	ptr,
	4950	try self.wip.cast(
	4951	if (is_float) .bitcast else if (is_signed_int) .sext else .zext,
	4952	operand,
	4953	llvm_abi_ty,
	4954	"",
	4955	),
	4956	self.sync_scope,
	4957	ordering,
	4958	ptr_alignment,
	4959	"",
	4960	), llvm_operand_ty, "");
	4961	}
	4962
	4963	// If we are storing a pointer we need to convert to and from a plain old integer.
	4964	const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
	4965	.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""),
	4966	else => operand,
	4967	};
	4968
	4969	const raw_result = try self.wip.atomicrmw(
	4970	access_kind,
	4971	op,
	4972	ptr,
	4973	non_ptr_operand,
	4974	self.sync_scope,
	4975	ordering,
	4976	ptr_alignment,
	4977	"",
	4978	);
	4979
	4980	// ...and then convert the result back.
	4981	switch (operand_ty.zigTypeTag(zcu)) {
	4982	.pointer => return self.wip.cast(.inttoptr, raw_result, llvm_operand_ty, ""),
	4983	else => return raw_result,
	4984	}
	4985	}
	4986
	4987	fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4988	const o = self.object;
	4989	const zcu = o.zcu;
	4990	const atomic_load = self.air.instructions.items(.data)[@intFromEnum(inst)].atomic_load;
	4991	const ptr = try self.resolveInst(atomic_load.ptr);
	4992	const ptr_ty = self.typeOf(atomic_load.ptr);
	4993	const info = ptr_ty.ptrInfo(zcu);
	4994	const elem_ty = Type.fromInterned(info.child);
	4995	if (!elem_ty.hasRuntimeBits(zcu)) return .none;
	4996	const ordering = toLlvmAtomicOrdering(atomic_load.order);
	4997	const llvm_abi_ty = try self.getAtomicAbiType(elem_ty, false);
	4998	const ptr_alignment = (if (info.flags.alignment != .none)
	4999	@as(InternPool.Alignment, info.flags.alignment)
	5000	else
	5001	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
	5002	const access_kind: Builder.MemoryAccessKind =
	5003	if (info.flags.is_volatile) .@"volatile" else .normal;
	5004	const elem_llvm_ty = try o.lowerType(elem_ty);
	5005
	5006	self.maybeMarkAllowZeroAccess(info);
	5007
	5008	if (llvm_abi_ty != .none) {
	5009	// operand needs widening and truncating
	5010	const loaded = try self.wip.loadAtomic(
	5011	access_kind,
	5012	llvm_abi_ty,
	5013	ptr,
	5014	self.sync_scope,
	5015	ordering,
	5016	ptr_alignment,
	5017	"",
	5018	);
	5019	return self.wip.cast(.trunc, loaded, elem_llvm_ty, "");
	5020	}
	5021	return self.wip.loadAtomic(
	5022	access_kind,
	5023	elem_llvm_ty,
	5024	ptr,
	5025	self.sync_scope,
	5026	ordering,
	5027	ptr_alignment,
	5028	"",
	5029	);
	5030	}
	5031
	5032	fn airAtomicStore(
	5033	self: *FuncGen,
	5034	inst: Air.Inst.Index,
	5035	ordering: Builder.AtomicOrdering,
	5036	) Allocator.Error!Builder.Value {
	5037	const zcu = self.object.zcu;
	5038	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	5039	const ptr_ty = self.typeOf(bin_op.lhs);
	5040	const operand_ty = ptr_ty.childType(zcu);
	5041	if (!operand_ty.hasRuntimeBits(zcu)) return .none;
	5042	const ptr = try self.resolveInst(bin_op.lhs);
	5043	var element = try self.resolveInst(bin_op.rhs);
	5044	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
	5045
	5046	if (llvm_abi_ty != .none) {
	5047	// operand needs widening
	5048	element = try self.wip.conv(
	5049	if (operand_ty.isSignedInt(zcu)) .signed else .unsigned,
	5050	element,
	5051	llvm_abi_ty,
	5052	"",
	5053	);
	5054	}
	5055
	5056	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	5057
	5058	try self.storeFull(ptr, ptr_ty, element, ordering);
	5059	return .none;
	5060	}
	5061
	5062	fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Builder.Value {
	5063	const o = self.object;
	5064	const zcu = o.zcu;
	5065	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	5066	const dest_slice = try self.resolveInst(bin_op.lhs);
	5067	const ptr_ty = self.typeOf(bin_op.lhs);
	5068	const elem_ty = self.typeOf(bin_op.rhs);
	5069	const dest_ptr_align = ptr_ty.ptrAlignment(zcu).toLlvm();
	5070	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, ptr_ty);
	5071	const access_kind: Builder.MemoryAccessKind =
	5072	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	5073
	5074	self.maybeMarkAllowZeroAccess(ptr_ty.ptrInfo(zcu));
	5075
	5076	if (bin_op.rhs.toInterned()) \|elem_ip_index\| {
	5077	const elem_val: Value = .fromInterned(elem_ip_index);
	5078	if (elem_val.isUndef(zcu)) {
	5079	// Even if safety is disabled, we still emit a memset to undefined since it conveys
	5080	// extra information to LLVM. However, safety makes the difference between using
	5081	// 0xaa or actual undefined for the fill byte.
	5082	const fill_byte = if (safety)
	5083	try o.builder.intValue(.i8, 0xaa)
	5084	else
	5085	try o.builder.undefValue(.i8);
	5086	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
	5087	_ = try self.wip.callMemSet(
	5088	dest_ptr,
	5089	dest_ptr_align,
	5090	fill_byte,
	5091	len,
	5092	access_kind,
	5093	self.disable_intrinsics,
	5094	);
	5095	const owner_mod = self.ownerModule();
	5096	if (safety and owner_mod.valgrind) {
	5097	try self.valgrindMarkUndef(dest_ptr, len);
	5098	}
	5099	return .none;
	5100	}
	5101
	5102	// Test if the element value is compile-time known to be a
	5103	// repeating byte pattern, for example, `@as(u64, 0)` has a
	5104	// repeating byte pattern of 0 bytes. In such case, the memset
	5105	// intrinsic can be used.
	5106	if (try elem_val.hasRepeatedByteRepr(zcu)) \|byte_val\| {
	5107	const fill_byte = try o.builder.intValue(.i8, byte_val);
	5108	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
	5109	_ = try self.wip.callMemSet(
	5110	dest_ptr,
	5111	dest_ptr_align,
	5112	fill_byte,
	5113	len,
	5114	access_kind,
	5115	self.disable_intrinsics,
	5116	);
	5117	return .none;
	5118	}
	5119	}
	5120
	5121	const value = try self.resolveInst(bin_op.rhs);
	5122	const elem_abi_size = elem_ty.abiSize(zcu);
	5123
	5124	if (elem_abi_size == 1) {
	5125	// In this case we can take advantage of LLVM's intrinsic.
	5126	const fill_byte = try self.bitCast(value, elem_ty, Type.u8);
	5127	const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
	5128
	5129	_ = try self.wip.callMemSet(
	5130	dest_ptr,
	5131	dest_ptr_align,
	5132	fill_byte,
	5133	len,
	5134	access_kind,
	5135	self.disable_intrinsics,
	5136	);
	5137	return .none;
	5138	}
	5139
	5140	// non-byte-sized element. lower with a loop. something like this:
	5141
	5142	// entry:
	5143	// ...
	5144	// %end_ptr = getelementptr %ptr, %len
	5145	// br %loop
	5146	// loop:
	5147	// %it_ptr = phi body %next_ptr, entry %ptr
	5148	// %end = cmp eq %it_ptr, %end_ptr
	5149	// br %end, %body, %end
	5150	// body:
	5151	// store %it_ptr, %value
	5152	// %next_ptr = getelementptr %it_ptr, 1
	5153	// br %loop
	5154	// end:
	5155	// ...
	5156	const entry_block = self.wip.cursor.block;
	5157	const loop_block = try self.wip.block(2, "InlineMemsetLoop");
	5158	const body_block = try self.wip.block(1, "InlineMemsetBody");
	5159	const end_block = try self.wip.block(1, "InlineMemsetEnd");
	5160
	5161	const llvm_usize_ty = try o.lowerType(.usize);
	5162	const end_ptr = switch (ptr_ty.ptrSize(zcu)) {
	5163	.slice => try self.ptraddScaled(
	5164	dest_ptr,
	5165	try self.wip.extractValue(dest_slice, &.{1}, ""),
	5166	elem_abi_size,
	5167	),
	5168	.one => try self.ptraddConst(dest_ptr, ptr_ty.childType(zcu).abiSize(zcu)),
	5169	.many, .c => unreachable,
	5170	};
	5171	_ = try self.wip.br(loop_block);
	5172
	5173	self.wip.cursor = .{ .block = loop_block };
	5174	const it_ptr = try self.wip.phi(.ptr, "");
	5175	const end = try self.wip.icmp(.ne, it_ptr.toValue(), end_ptr, "");
	5176	_ = try self.wip.brCond(end, body_block, end_block, .none);
	5177
	5178	self.wip.cursor = .{ .block = body_block };
	5179	const elem_abi_align = elem_ty.abiAlignment(zcu);
	5180	const it_ptr_align = InternPool.Alignment.fromLlvm(dest_ptr_align).min(elem_abi_align).toLlvm();
	5181	if (isByRef(elem_ty, zcu)) {
	5182	_ = try self.wip.callMemCpy(
	5183	it_ptr.toValue(),
	5184	it_ptr_align,
	5185	value,
	5186	elem_abi_align.toLlvm(),
	5187	try o.builder.intValue(llvm_usize_ty, elem_abi_size),
	5188	access_kind,
	5189	self.disable_intrinsics,
	5190	);
	5191	} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
	5192	const next_ptr = try self.ptraddConst(it_ptr.toValue(), elem_abi_size);
	5193	_ = try self.wip.br(loop_block);
	5194
	5195	self.wip.cursor = .{ .block = end_block };
	5196	it_ptr.finish(&.{ next_ptr, dest_ptr }, &.{ body_block, entry_block }, &self.wip);
	5197	return .none;
	5198	}
	5199
	5200	fn airMemcpy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5201	const zcu = self.object.zcu;
	5202	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	5203	const dest_slice = try self.resolveInst(bin_op.lhs);
	5204	const dest_ptr_ty = self.typeOf(bin_op.lhs);
	5205	const src_slice = try self.resolveInst(bin_op.rhs);
	5206	const src_ptr_ty = self.typeOf(bin_op.rhs);
	5207	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
	5208	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
	5209	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
	5210	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
	5211	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	5212
	5213	self.maybeMarkAllowZeroAccess(dest_ptr_ty.ptrInfo(zcu));
	5214	self.maybeMarkAllowZeroAccess(src_ptr_ty.ptrInfo(zcu));
	5215
	5216	_ = try self.wip.callMemCpy(
	5217	dest_ptr,
	5218	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
	5219	src_ptr,
	5220	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
	5221	len,
	5222	access_kind,
	5223	self.disable_intrinsics,
	5224	);
	5225	return .none;
	5226	}
	5227
	5228	fn airMemmove(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5229	const zcu = self.object.zcu;
	5230	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	5231	const dest_slice = try self.resolveInst(bin_op.lhs);
	5232	const dest_ptr_ty = self.typeOf(bin_op.lhs);
	5233	const src_slice = try self.resolveInst(bin_op.rhs);
	5234	const src_ptr_ty = self.typeOf(bin_op.rhs);
	5235	const src_ptr = try self.sliceOrArrayPtr(src_slice, src_ptr_ty);
	5236	const len = try self.sliceOrArrayLenInBytes(dest_slice, dest_ptr_ty);
	5237	const dest_ptr = try self.sliceOrArrayPtr(dest_slice, dest_ptr_ty);
	5238	const access_kind: Builder.MemoryAccessKind = if (src_ptr_ty.isVolatilePtr(zcu) or
	5239	dest_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	5240
	5241	_ = try self.wip.callMemMove(
	5242	dest_ptr,
	5243	dest_ptr_ty.ptrAlignment(zcu).toLlvm(),
	5244	src_ptr,
	5245	src_ptr_ty.ptrAlignment(zcu).toLlvm(),
	5246	len,
	5247	access_kind,
	5248	);
	5249	return .none;
	5250	}
	5251
	5252	fn airSetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5253	const zcu = self.object.zcu;
	5254	const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
	5255	const un_ptr_ty = self.typeOf(bin_op.lhs);
	5256	const un_ty = un_ptr_ty.childType(zcu);
	5257	const layout = un_ty.unionGetLayout(zcu);
	5258
	5259	if (layout.tag_size == 0) return .none; // TODO: stop Sema emitting this
	5260
	5261	const access_kind: Builder.MemoryAccessKind =
	5262	if (un_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
	5263
	5264	self.maybeMarkAllowZeroAccess(un_ptr_ty.ptrInfo(zcu));
	5265
	5266	const union_ptr = try self.resolveInst(bin_op.lhs);
	5267	const new_tag = try self.resolveInst(bin_op.rhs);
	5268	const union_ptr_align = un_ptr_ty.ptrAlignment(zcu);
	5269	if (layout.payload_size == 0) {
	5270	_ = try self.wip.store(access_kind, new_tag, union_ptr, union_ptr_align.toLlvm());
	5271	return .none;
	5272	}
	5273	const tag_field_ptr = try self.ptraddConst(union_ptr, layout.tagOffset());
	5274	const tag_ptr_align: InternPool.Alignment = switch (layout.tagOffset()) {
	5275	0 => union_ptr_align,
	5276	else => \|off\| .minStrict(union_ptr_align, .fromLog2Units(@ctz(off))),
	5277	};
	5278	_ = try self.wip.store(access_kind, new_tag, tag_field_ptr, tag_ptr_align.toLlvm());
	5279	return .none;
	5280	}
	5281
	5282	fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5283	const o = self.object;
	5284	const zcu = o.zcu;
	5285	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5286	const un_ty = self.typeOf(ty_op.operand);
	5287	const layout = un_ty.unionGetLayout(zcu);
	5288	assert(layout.tag_size != 0);
	5289	const operand = try self.resolveInst(ty_op.operand);
	5290	if (isByRef(un_ty, zcu)) {
	5291	const llvm_tag_ty = try o.lowerType(un_ty.unionTagTypeRuntime(zcu).?);
	5292	const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
	5293	return self.wip.load(.normal, llvm_tag_ty, tag_field_ptr, .default, "");
	5294	} else {
	5295	// This is only possible if all fields are zero-bit, in which case `operand` is already an
	5296	// integer value (the union is lowered as its enum tag).
	5297	assert(layout.payload_size == 0);
	5298	return operand;
	5299	}
	5300	}
	5301
	5302	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
	5303	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	5304	const operand = try self.resolveInst(un_op);
	5305	const operand_ty = self.typeOf(un_op);
	5306
	5307	return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand});
	5308	}
	5309
	5310	fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	5311	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	5312	const operand = try self.resolveInst(un_op);
	5313	const operand_ty = self.typeOf(un_op);
	5314
	5315	return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand});
	5316	}
	5317
	5318	fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
	5319	const o = self.object;
	5320	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5321	const inst_ty = self.typeOfIndex(inst);
	5322	const operand_ty = self.typeOf(ty_op.operand);
	5323	const operand = try self.resolveInst(ty_op.operand);
	5324
	5325	const result = try self.wip.callIntrinsic(
	5326	.normal,
	5327	.none,
	5328	intrinsic,
	5329	&.{try o.lowerType(operand_ty)},
	5330	&.{ operand, .false },
	5331	"",
	5332	);
	5333	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5334	}
	5335
	5336	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
	5337	const o = self.object;
	5338	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5339	const inst_ty = self.typeOfIndex(inst);
	5340	const operand_ty = self.typeOf(ty_op.operand);
	5341	const operand = try self.resolveInst(ty_op.operand);
	5342
	5343	const result = try self.wip.callIntrinsic(
	5344	.normal,
	5345	.none,
	5346	intrinsic,
	5347	&.{try o.lowerType(operand_ty)},
	5348	&.{operand},
	5349	"",
	5350	);
	5351	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5352	}
	5353
	5354	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5355	const o = self.object;
	5356	const zcu = o.zcu;
	5357	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5358	const operand_ty = self.typeOf(ty_op.operand);
	5359	var bits = operand_ty.intInfo(zcu).bits;
	5360	assert(bits % 8 == 0);
	5361
	5362	const inst_ty = self.typeOfIndex(inst);
	5363	var operand = try self.resolveInst(ty_op.operand);
	5364	var llvm_operand_ty = try o.lowerType(operand_ty);
	5365
	5366	if (bits % 16 == 8) {
	5367	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
	5368	// The truncated result at the end will be the correct bswap
	5369	const scalar_ty = try o.builder.intType(@intCast(bits + 8));
	5370	if (operand_ty.zigTypeTag(zcu) == .vector) {
	5371	const vec_len = operand_ty.vectorLen(zcu);
	5372	llvm_operand_ty = try o.builder.vectorType(.normal, vec_len, scalar_ty);
	5373	} else llvm_operand_ty = scalar_ty;
	5374
	5375	const shift_amt =
	5376	try o.builder.splatValue(llvm_operand_ty, try o.builder.intConst(scalar_ty, 8));
	5377	const extended = try self.wip.cast(.zext, operand, llvm_operand_ty, "");
	5378	operand = try self.wip.bin(.shl, extended, shift_amt, "");
	5379
	5380	bits = bits + 8;
	5381	}
	5382
	5383	const result =
	5384	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
	5385	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5386	}
	5387
	5388	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5389	const o = self.object;
	5390	const zcu = o.zcu;
	5391	const ip = &zcu.intern_pool;
	5392	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5393	const operand = try self.resolveInst(ty_op.operand);
	5394	const error_set_ty = ty_op.ty.toType();
	5395
	5396	const names = error_set_ty.errorSetNames(zcu);
	5397	const valid_block = try self.wip.block(@intCast(names.len), "Valid");
	5398	const invalid_block = try self.wip.block(1, "Invalid");
	5399	const end_block = try self.wip.block(2, "End");
	5400	var wip_switch = try self.wip.@"switch"(operand, invalid_block, @intCast(names.len), .none);
	5401	defer wip_switch.finish(&self.wip);
	5402
	5403	for (0..names.len) \|name_index\| {
	5404	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
	5405	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int);
	5406	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
	5407	}
	5408	self.wip.cursor = .{ .block = valid_block };
	5409	_ = try self.wip.br(end_block);
	5410
	5411	self.wip.cursor = .{ .block = invalid_block };
	5412	_ = try self.wip.br(end_block);
	5413
	5414	self.wip.cursor = .{ .block = end_block };
	5415	const phi = try self.wip.phi(.i1, "");
	5416	phi.finish(&.{ .true, .false }, &.{ valid_block, invalid_block }, &self.wip);
	5417	return phi.toValue();
	5418	}
	5419
	5420	fn airIsNamedEnumValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5421	const o = self.object;
	5422	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	5423	const operand = try self.resolveInst(un_op);
	5424	const enum_ty = self.typeOf(un_op);
	5425
	5426	const llvm_fn = try o.getIsNamedEnumValueFunction(enum_ty);
	5427	return self.wip.call(
	5428	.normal,
	5429	.fastcc,
	5430	.none,
	5431	llvm_fn.typeOf(&o.builder),
	5432	llvm_fn.toValue(&o.builder),
	5433	&.{operand},
	5434	"",
	5435	);
	5436	}
	5437
	5438	fn airTagName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5439	const o = self.object;
	5440	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	5441	const operand = try self.resolveInst(un_op);
	5442	const enum_ty = self.typeOf(un_op);
	5443
	5444	const llvm_fn = try o.getEnumTagNameFunction(enum_ty);
	5445	return self.wip.call(
	5446	.normal,
	5447	.fastcc,
	5448	.none,
	5449	llvm_fn.typeOf(&o.builder),
	5450	llvm_fn.toValue(&o.builder),
	5451	&.{operand},
	5452	"",
	5453	);
	5454	}
	5455
	5456	fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5457	const o = self.object;
	5458	const zcu = o.zcu;
	5459	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
	5460	const operand = try self.resolveInst(un_op);
	5461	const slice_ty = self.typeOfIndex(inst);
	5462	const slice_llvm_ty = try o.lowerType(slice_ty);
	5463
	5464	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
	5465	const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");
	5466
	5467	const error_name_table_ptr = try o.getErrorNameTable();
	5468	const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
	5469	return self.wip.load(.normal, slice_llvm_ty, error_name_ptr, .default, "");
	5470	}
	5471
	5472	fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5473	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5474	const scalar = try self.resolveInst(ty_op.operand);
	5475	const vector_ty = self.typeOfIndex(inst);
	5476	return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, "");
	5477	}
	5478
	5479	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5480	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	5481	const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
	5482	const pred = try self.resolveInst(pl_op.operand);
	5483	const a = try self.resolveInst(extra.lhs);
	5484	const b = try self.resolveInst(extra.rhs);
	5485
	5486	return self.wip.select(.normal, pred, a, b, "");
	5487	}
	5488
	5489	fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5490	const o = fg.object;
	5491	const zcu = o.zcu;
	5492	const gpa = zcu.gpa;
	5493
	5494	const unwrapped = fg.air.unwrapShuffleOne(zcu, inst);
	5495
	5496	const operand = try fg.resolveInst(unwrapped.operand);
	5497	const mask = unwrapped.mask;
	5498	const operand_ty = fg.typeOf(unwrapped.operand);
	5499	const llvm_operand_ty = try o.lowerType(operand_ty);
	5500	const llvm_result_ty = try o.lowerType(unwrapped.result_ty);
	5501	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5502	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
	5503	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
	5504	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
	5505
	5506	// LLVM requires that the two input vectors have the same length, so lowering isn't trivial.
	5507	// And, in the words of jacobly0: "llvm sucks at shuffles so we do have to hold its hand at
	5508	// least a bit". So, there are two cases here.
	5509	//
	5510	// If the operand length equals the mask length, we do just the one `shufflevector`, where
	5511	// the second operand is a constant vector with comptime-known elements at the right indices
	5512	// and poison values elsewhere (in the indices which won't be selected).
	5513	//
	5514	// Otherwise, we lower to two `shufflevector` instructions. The first shuffles the runtime
	5515	// operand with an all-poison vector to extract and correctly position all of the runtime
	5516	// elements. We also make a constant vector with all of the comptime elements correctly
	5517	// positioned. Then, our second instruction selects elements from those "runtime-or-poison"
	5518	// and "comptime-or-poison" vectors to compute the result.
	5519
	5520	// This buffer is used primarily for the mask constants.
	5521	const llvm_elem_buf = try gpa.alloc(Builder.Constant, mask.len);
	5522	defer gpa.free(llvm_elem_buf);
	5523
	5524	// ...but first, we'll collect all of the comptime-known values.
	5525	var any_defined_comptime_value = false;
	5526	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
	5527	llvm_elem.* = switch (mask_elem.unwrap()) {
	5528	.elem => llvm_poison_elem,
	5529	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
	5530	any_defined_comptime_value = true;
	5531	break :elem try o.lowerValue(val);
	5532	} else llvm_poison_elem,
	5533	};
	5534	}
	5535	// This vector is like the result, but runtime elements are replaced with poison.
	5536	const comptime_and_poison: Builder.Value = if (any_defined_comptime_value) vec: {
	5537	break :vec try o.builder.vectorValue(llvm_result_ty, llvm_elem_buf);
	5538	} else try o.builder.poisonValue(llvm_result_ty);
	5539
	5540	if (operand_ty.vectorLen(zcu) == mask.len) {
	5541	// input length equals mask/output length, so we lower to one instruction
	5542	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
	5543	llvm_elem.* = switch (mask_elem.unwrap()) {
	5544	.elem => \|idx\| try o.builder.intConst(.i32, idx),
	5545	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
	5546	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
	5547	} else llvm_poison_mask_elem,
	5548	};
	5549	}
	5550	return fg.wip.shuffleVector(
	5551	operand,
	5552	comptime_and_poison,
	5553	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
	5554	"",
	5555	);
	5556	}
	5557
	5558	for (mask, llvm_elem_buf) \|mask_elem, *llvm_elem\| {
	5559	llvm_elem.* = switch (mask_elem.unwrap()) {
	5560	.elem => \|idx\| try o.builder.intConst(.i32, idx),
	5561	.value => llvm_poison_mask_elem,
	5562	};
	5563	}
	5564	// This vector is like our result, but all comptime-known elements are poison.
	5565	const runtime_and_poison = try fg.wip.shuffleVector(
	5566	operand,
	5567	try o.builder.poisonValue(llvm_operand_ty),
	5568	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
	5569	"",
	5570	);
	5571
	5572	if (!any_defined_comptime_value) {
	5573	// `comptime_and_poison` is just poison; a second shuffle would be a nop.
	5574	return runtime_and_poison;
	5575	}
	5576
	5577	// In this second shuffle, the inputs, the mask, and the output all have the same length.
	5578	for (mask, llvm_elem_buf, 0..) \|mask_elem, *llvm_elem, elem_idx\| {
	5579	llvm_elem.* = switch (mask_elem.unwrap()) {
	5580	.elem => try o.builder.intConst(.i32, elem_idx),
	5581	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) mask_val: {
	5582	break :mask_val try o.builder.intConst(.i32, mask.len + elem_idx);
	5583	} else llvm_poison_mask_elem,
	5584	};
	5585	}
	5586	// Merge the runtime and comptime elements with the mask we just built.
	5587	return fg.wip.shuffleVector(
	5588	runtime_and_poison,
	5589	comptime_and_poison,
	5590	try o.builder.vectorValue(llvm_mask_ty, llvm_elem_buf),
	5591	"",
	5592	);
	5593	}
	5594
	5595	fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5596	const o = fg.object;
	5597	const zcu = o.zcu;
	5598	const gpa = zcu.gpa;
	5599
	5600	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
	5601
	5602	const mask = unwrapped.mask;
	5603	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5604	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
	5605	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
	5606
	5607	// This is kind of simpler than in `airShuffleOne`. We extend the shorter vector to the
	5608	// length of the longer one with an initial `shufflevector` if necessary, and then do the
	5609	// actual computation with a second `shufflevector`.
	5610
	5611	const operand_a_len = fg.typeOf(unwrapped.operand_a).vectorLen(zcu);
	5612	const operand_b_len = fg.typeOf(unwrapped.operand_b).vectorLen(zcu);
	5613	const operand_len: u32 = @max(operand_a_len, operand_b_len);
	5614
	5615	// If we need to extend an operand, this is the type that mask will have.
	5616	const llvm_operand_mask_ty = try o.builder.vectorType(.normal, operand_len, .i32);
	5617
	5618	const llvm_elem_buf = try gpa.alloc(Builder.Constant, @max(mask.len, operand_len));
	5619	defer gpa.free(llvm_elem_buf);
	5620
	5621	const operand_a: Builder.Value = extend: {
	5622	const raw = try fg.resolveInst(unwrapped.operand_a);
	5623	if (operand_a_len == operand_len) break :extend raw;
	5624	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
	5625	const mask_elems = llvm_elem_buf[0..operand_len];
	5626	for (mask_elems[0..operand_a_len], 0..) \|*llvm_elem, elem_idx\| {
	5627	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
	5628	}
	5629	@memset(mask_elems[operand_a_len..], llvm_poison_mask_elem);
	5630	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_a_len, llvm_elem_ty);
	5631	break :extend try fg.wip.shuffleVector(
	5632	raw,
	5633	try o.builder.poisonValue(llvm_this_operand_ty),
	5634	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
	5635	"",
	5636	);
	5637	};
	5638	const operand_b: Builder.Value = extend: {
	5639	const raw = try fg.resolveInst(unwrapped.operand_b);
	5640	if (operand_b_len == operand_len) break :extend raw;
	5641	// Extend with a `shufflevector`, with a mask `<0, 1, ..., n, poison, poison, ..., poison>`
	5642	const mask_elems = llvm_elem_buf[0..operand_len];
	5643	for (mask_elems[0..operand_b_len], 0..) \|*llvm_elem, elem_idx\| {
	5644	llvm_elem.* = try o.builder.intConst(.i32, elem_idx);
	5645	}
	5646	@memset(mask_elems[operand_b_len..], llvm_poison_mask_elem);
	5647	const llvm_this_operand_ty = try o.builder.vectorType(.normal, operand_b_len, llvm_elem_ty);
	5648	break :extend try fg.wip.shuffleVector(
	5649	raw,
	5650	try o.builder.poisonValue(llvm_this_operand_ty),
	5651	try o.builder.vectorValue(llvm_operand_mask_ty, mask_elems),
	5652	"",
	5653	);
	5654	};
	5655
	5656	// `operand_a` and `operand_b` now have the same length (we've extended the shorter one with
	5657	// an initial shuffle if necessary). Now for the easy bit.
	5658
	5659	const mask_elems = llvm_elem_buf[0..mask.len];
	5660	for (mask, mask_elems) \|mask_elem, *llvm_mask_elem\| {
	5661	llvm_mask_elem.* = switch (mask_elem.unwrap()) {
	5662	.a_elem => \|idx\| try o.builder.intConst(.i32, idx),
	5663	.b_elem => \|idx\| try o.builder.intConst(.i32, operand_len + idx),
	5664	.undef => llvm_poison_mask_elem,
	5665	};
	5666	}
	5667	return fg.wip.shuffleVector(
	5668	operand_a,
	5669	operand_b,
	5670	try o.builder.vectorValue(llvm_mask_ty, mask_elems),
	5671	"",
	5672	);
	5673	}
	5674
	5675	/// Reduce a vector by repeatedly applying `llvm_fn` to produce an accumulated result.
	5676	///
	5677	/// Equivalent to:
	5678	/// reduce: {
	5679	/// var i: usize = 0;
	5680	/// var accum: T = init;
	5681	/// while (i < vec.len) : (i += 1) {
	5682	/// accum = llvm_fn(accum, vec[i]);
	5683	/// }
	5684	/// break :reduce accum;
	5685	/// }
	5686	///
	5687	fn buildReducedCall(
	5688	self: *FuncGen,
	5689	llvm_fn: Builder.Function.Index,
	5690	operand_vector: Builder.Value,
	5691	vector_len: usize,
	5692	accum_init: Builder.Value,
	5693	) Allocator.Error!Builder.Value {
	5694	const o = self.object;
	5695	const usize_ty = try o.lowerType(.usize);
	5696	const llvm_vector_len = try o.builder.intValue(usize_ty, vector_len);
	5697	const llvm_result_ty = accum_init.typeOfWip(&self.wip);
	5698
	5699	// Allocate and initialize our mutable variables
	5700	const i_ptr = try self.buildAlloca(usize_ty, .default);
	5701	_ = try self.wip.store(.normal, try o.builder.intValue(usize_ty, 0), i_ptr, .default);
	5702	const accum_ptr = try self.buildAlloca(llvm_result_ty, .default);
	5703	_ = try self.wip.store(.normal, accum_init, accum_ptr, .default);
	5704
	5705	// Setup the loop
	5706	const loop = try self.wip.block(2, "ReduceLoop");
	5707	const loop_exit = try self.wip.block(1, "AfterReduce");
	5708	_ = try self.wip.br(loop);
	5709	{
	5710	self.wip.cursor = .{ .block = loop };
	5711
	5712	// while (i < vec.len)
	5713	const i = try self.wip.load(.normal, usize_ty, i_ptr, .default, "");
	5714	const cond = try self.wip.icmp(.ult, i, llvm_vector_len, "");
	5715	const loop_then = try self.wip.block(1, "ReduceLoopThen");
	5716
	5717	_ = try self.wip.brCond(cond, loop_then, loop_exit, .none);
	5718
	5719	{
	5720	self.wip.cursor = .{ .block = loop_then };
	5721
	5722	// accum = f(accum, vec[i]);
	5723	const accum = try self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
	5724	const element = try self.wip.extractElement(operand_vector, i, "");
	5725	const new_accum = try self.wip.call(
	5726	.normal,
	5727	.ccc,
	5728	.none,
	5729	llvm_fn.typeOf(&o.builder),
	5730	llvm_fn.toValue(&o.builder),
	5731	&.{ accum, element },
	5732	"",
	5733	);
	5734	_ = try self.wip.store(.normal, new_accum, accum_ptr, .default);
	5735
	5736	// i += 1
	5737	const new_i = try self.wip.bin(.add, i, try o.builder.intValue(usize_ty, 1), "");
	5738	_ = try self.wip.store(.normal, new_i, i_ptr, .default);
	5739	_ = try self.wip.br(loop);
	5740	}
	5741	}
	5742
	5743	self.wip.cursor = .{ .block = loop_exit };
	5744	return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, "");
	5745	}
	5746
	5747	fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
	5748	const o = self.object;
	5749	const zcu = o.zcu;
	5750	const target = zcu.getTarget();
	5751
	5752	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
	5753	const operand = try self.resolveInst(reduce.operand);
	5754	const operand_ty = self.typeOf(reduce.operand);
	5755	const llvm_operand_ty = try o.lowerType(operand_ty);
	5756	const scalar_ty = self.typeOfIndex(inst);
	5757	const llvm_scalar_ty = try o.lowerType(scalar_ty);
	5758
	5759	switch (reduce.operation) {
	5760	.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
	5761	.And => .@"vector.reduce.and",
	5762	.Or => .@"vector.reduce.or",
	5763	.Xor => .@"vector.reduce.xor",
	5764	else => unreachable,
	5765	}, &.{llvm_operand_ty}, &.{operand}, ""),
	5766	.Min, .Max => switch (scalar_ty.zigTypeTag(zcu)) {
	5767	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
	5768	.Min => if (scalar_ty.isSignedInt(zcu))
	5769	.@"vector.reduce.smin"
	5770	else
	5771	.@"vector.reduce.umin",
	5772	.Max => if (scalar_ty.isSignedInt(zcu))
	5773	.@"vector.reduce.smax"
	5774	else
	5775	.@"vector.reduce.umax",
	5776	else => unreachable,
	5777	}, &.{llvm_operand_ty}, &.{operand}, ""),
	5778	.float => if (intrinsicsAllowed(scalar_ty, target))
	5779	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
	5780	.Min => .@"vector.reduce.fmin",
	5781	.Max => .@"vector.reduce.fmax",
	5782	else => unreachable,
	5783	}, &.{llvm_operand_ty}, &.{operand}, ""),
	5784	else => unreachable,
	5785	},
	5786	.Add, .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
	5787	.int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
	5788	.Add => .@"vector.reduce.add",
	5789	.Mul => .@"vector.reduce.mul",
	5790	else => unreachable,
	5791	}, &.{llvm_operand_ty}, &.{operand}, ""),
	5792	.float => if (intrinsicsAllowed(scalar_ty, target))
	5793	return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) {
	5794	.Add => .@"vector.reduce.fadd",
	5795	.Mul => .@"vector.reduce.fmul",
	5796	else => unreachable,
	5797	}, &.{llvm_operand_ty}, &.{ switch (reduce.operation) {
	5798	.Add => try o.builder.fpValue(llvm_scalar_ty, -0.0),
	5799	.Mul => try o.builder.fpValue(llvm_scalar_ty, 1.0),
	5800	else => unreachable,
	5801	}, operand }, ""),
	5802	else => unreachable,
	5803	},
	5804	}
	5805
	5806	// Reduction could not be performed with intrinsics.
	5807	// Use a manual loop over a softfloat call instead.
	5808	const float_bits = scalar_ty.floatBits(target);
	5809	const fn_name = switch (reduce.operation) {
	5810	.Min => try o.builder.strtabStringFmt("{s}fmin{s}", .{
	5811	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
	5812	}),
	5813	.Max => try o.builder.strtabStringFmt("{s}fmax{s}", .{
	5814	libcFloatPrefix(float_bits), libcFloatSuffix(float_bits),
	5815	}),
	5816	.Add => try o.builder.strtabStringFmt("__add{s}f3", .{
	5817	compilerRtFloatAbbrev(float_bits),
	5818	}),
	5819	.Mul => try o.builder.strtabStringFmt("__mul{s}f3", .{
	5820	compilerRtFloatAbbrev(float_bits),
	5821	}),
	5822	else => unreachable,
	5823	};
	5824
	5825	const libc_fn = try o.getLibcFunction(fn_name, &.{ llvm_scalar_ty, llvm_scalar_ty }, llvm_scalar_ty);
	5826	const init_val = switch (llvm_scalar_ty) {
	5827	.i16 => try o.builder.intValue(.i16, @as(i16, @bitCast(
	5828	@as(f16, switch (reduce.operation) {
	5829	.Min, .Max => std.math.nan(f16),
	5830	.Add => -0.0,
	5831	.Mul => 1.0,
	5832	else => unreachable,
	5833	}),
	5834	))),
	5835	.i80 => try o.builder.intValue(.i80, @as(i80, @bitCast(
	5836	@as(f80, switch (reduce.operation) {
	5837	.Min, .Max => std.math.nan(f80),
	5838	.Add => -0.0,
	5839	.Mul => 1.0,
	5840	else => unreachable,
	5841	}),
	5842	))),
	5843	.i128 => try o.builder.intValue(.i128, @as(i128, @bitCast(
	5844	@as(f128, switch (reduce.operation) {
	5845	.Min, .Max => std.math.nan(f128),
	5846	.Add => -0.0,
	5847	.Mul => 1.0,
	5848	else => unreachable,
	5849	}),
	5850	))),
	5851	else => unreachable,
	5852	};
	5853	return self.buildReducedCall(libc_fn, operand, operand_ty.vectorLen(zcu), init_val);
	5854	}
	5855
	5856	fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5857	const o = self.object;
	5858	const zcu = o.zcu;
	5859	const ip = &zcu.intern_pool;
	5860	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	5861	const result_ty = self.typeOfIndex(inst);
	5862	const len: usize = @intCast(result_ty.arrayLen(zcu));
	5863	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
	5864	const llvm_result_ty = try o.lowerType(result_ty);
	5865
	5866	switch (result_ty.zigTypeTag(zcu)) {
	5867	.vector => {
	5868	var vector = try o.builder.poisonValue(llvm_result_ty);
	5869	for (elements, 0..) \|elem, i\| {
	5870	const index_u32 = try o.builder.intValue(.i32, i);
	5871	const llvm_elem = try self.resolveInst(elem);
	5872	vector = try self.wip.insertElement(vector, llvm_elem, index_u32, "");
	5873	}
	5874	return vector;
	5875	},
	5876	.@"struct" => switch (result_ty.containerLayout(zcu)) {
	5877	.@"packed" => {
	5878	const struct_type = ip.loadStructType(result_ty.toIntern());
	5879	const backing_int_ty: Type = .fromInterned(struct_type.packed_backing_int_type);
	5880	const big_bits = backing_int_ty.bitSize(zcu);
	5881	const int_ty = try o.builder.intType(@intCast(big_bits));
	5882	comptime assert(Type.packed_struct_layout_version == 2);
	5883	var running_int = try o.builder.intValue(int_ty, 0);
	5884	var running_bits: u16 = 0;
	5885	for (elements, struct_type.field_types.get(ip)) \|elem, field_ty\| {
	5886	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
	5887
	5888	const non_int_val = try self.resolveInst(elem);
	5889	const ty_bit_size: u16 = @intCast(Type.fromInterned(field_ty).bitSize(zcu));
	5890	const small_int_ty = try o.builder.intType(ty_bit_size);
	5891	const small_int_val = if (Type.fromInterned(field_ty).isPtrAtRuntime(zcu))
	5892	try self.wip.cast(.ptrtoint, non_int_val, small_int_ty, "")
	5893	else
	5894	try self.wip.cast(.bitcast, non_int_val, small_int_ty, "");
	5895	const shift_rhs = try o.builder.intValue(int_ty, running_bits);
	5896	const extended_int_val =
	5897	try self.wip.conv(.unsigned, small_int_val, int_ty, "");
	5898	const shifted = try self.wip.bin(.shl, extended_int_val, shift_rhs, "");
	5899	running_int = try self.wip.bin(.@"or", running_int, shifted, "");
	5900	running_bits += ty_bit_size;
	5901	}
	5902	return running_int;
	5903	},
	5904	.auto, .@"extern" => {
	5905	assert(isByRef(result_ty, zcu));
	5906	// TODO in debug builds init to undef so that the padding will be 0xaa
	5907	// even if we fully populate the fields.
	5908	const struct_align = result_ty.abiAlignment(zcu);
	5909	const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());
	5910
	5911	for (elements, 0..) \|elem, field_index\| {
	5912	if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
	5913	const field_ty = result_ty.fieldType(field_index, zcu);
	5914	if (!field_ty.hasRuntimeBits(zcu)) continue;
	5915	const offset = result_ty.structFieldOffset(field_index, zcu);
	5916	const field_ptr = try self.ptraddConst(alloca_inst, offset);
	5917	const field_ptr_align: InternPool.Alignment = switch (offset) {
	5918	0 => struct_align,
	5919	else => struct_align.minStrict(.fromLog2Units(@ctz(offset))),
	5920	};
	5921
	5922	const llvm_field_val = try self.resolveInst(elem);
	5923
	5924	if (isByRef(field_ty, zcu)) {
	5925	_ = try self.wip.callMemCpy(
	5926	field_ptr,
	5927	field_ptr_align.toLlvm(),
	5928	llvm_field_val,
	5929	field_ty.abiAlignment(zcu).toLlvm(),
	5930	try o.builder.intValue(try o.lowerType(.usize), field_ty.abiSize(zcu)),
	5931	.normal,
	5932	self.disable_intrinsics,
	5933	);
	5934	} else {
	5935	_ = try self.wip.store(
	5936	.normal,
	5937	llvm_field_val,
	5938	field_ptr,
	5939	field_ptr_align.toLlvm(),
	5940	);
	5941	}
	5942	}
	5943
	5944	return alloca_inst;
	5945	},
	5946	},
	5947	.array => {
	5948	assert(isByRef(result_ty, zcu));
	5949
	5950	const alignment = result_ty.abiAlignment(zcu).toLlvm();
	5951	const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
	5952
	5953	const array_info = result_ty.arrayInfo(zcu);
	5954
	5955	const elem_size = array_info.elem_type.abiSize(zcu);
	5956
	5957	for (elements, 0..) \|elem, i\| {
	5958	const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * i);
	5959	const llvm_elem = try self.resolveInst(elem);
	5960	try self.store(elem_ptr, .none, llvm_elem, array_info.elem_type);
	5961	}
	5962	if (array_info.sentinel) \|sent_val\| {
	5963	const elem_ptr = try self.ptraddConst(alloca_inst, elem_size * array_info.len);
	5964	const llvm_elem = try self.resolveValue(sent_val);
	5965	try self.store(elem_ptr, .none, llvm_elem.toValue(), array_info.elem_type);
	5966	}
	5967
	5968	return alloca_inst;
	5969	},
	5970	else => unreachable,
	5971	}
	5972	}
	5973
	5974	fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	5975	const o = self.object;
	5976	const zcu = o.zcu;
	5977	const ip = &zcu.intern_pool;
	5978	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
	5979	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
	5980	const union_ty = self.typeOfIndex(inst);
	5981	const union_llvm_ty = try o.lowerType(union_ty);
	5982	const union_obj = zcu.typeToUnion(union_ty).?;
	5983
	5984	assert(union_obj.layout != .@"packed");
	5985
	5986	const layout = Type.getUnionLayout(union_obj, zcu);
	5987
	5988	assert(layout.payload_size != 0); // otherwise the value would be comptime-known
	5989	assert(isByRef(union_ty, zcu));
	5990
	5991	const alignment = layout.abi_align.toLlvm();
	5992	const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
	5993	const llvm_payload = try self.resolveInst(extra.init);
	5994	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
	5995	assert(field_ty.hasRuntimeBits(zcu));
	5996
	5997	{
	5998	const payload_ptr = try self.ptraddConst(result_ptr, layout.payloadOffset());
	5999	try self.store(payload_ptr, layout.payload_align, llvm_payload, field_ty);
	6000	}
	6001
	6002	if (layout.tag_size != 0) {
	6003	const loaded_enum = ip.loadEnumType(union_obj.enum_tag_type);
	6004	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
	6005	.none => try o.builder.intConst(
	6006	try o.lowerType(.fromInterned(union_obj.enum_tag_type)),
	6007	extra.field_index, // auto-numbered
	6008	),
	6009	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	6010	};
	6011	const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
	6012	_ = try self.wip.store(.normal, llvm_tag_val.toValue(), tag_ptr, layout.tag_align.toLlvm());
	6013	}
	6014
	6015	return result_ptr;
	6016	}
	6017
	6018	fn airPrefetch(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	6019	const o = self.object;
	6020	const prefetch = self.air.instructions.items(.data)[@intFromEnum(inst)].prefetch;
	6021
	6022	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.read) == 0);
	6023	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Rw.write) == 1);
	6024
	6025	comptime assert(prefetch.locality >= 0);
	6026	comptime assert(prefetch.locality <= 3);
	6027
	6028	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.instruction) == 0);
	6029	comptime assert(@intFromEnum(std.builtin.PrefetchOptions.Cache.data) == 1);
	6030
	6031	// LLVM fails during codegen of instruction cache prefetchs for these architectures.
	6032	// This is an LLVM bug as the prefetch intrinsic should be a noop if not supported
	6033	// by the target.
	6034	// To work around this, don't emit llvm.prefetch in this case.
	6035	// See https://bugs.llvm.org/show_bug.cgi?id=21037
	6036	const zcu = self.object.zcu;
	6037	const target = zcu.getTarget();
	6038	switch (prefetch.cache) {
	6039	.instruction => switch (target.cpu.arch) {
	6040	.x86_64,
	6041	.x86,
	6042	.powerpc,
	6043	.powerpcle,
	6044	.powerpc64,
	6045	.powerpc64le,
	6046	=> return .none,
	6047	.arm, .armeb, .thumb, .thumbeb => {
	6048	switch (prefetch.rw) {
	6049	.write => return .none,
	6050	else => {},
	6051	}
	6052	},
	6053	else => {},
	6054	},
	6055	.data => {},
	6056	}
	6057
	6058	_ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{
	6059	try self.sliceOrArrayPtr(try self.resolveInst(prefetch.ptr), self.typeOf(prefetch.ptr)),
	6060	try o.builder.intValue(.i32, prefetch.rw),
	6061	try o.builder.intValue(.i32, prefetch.locality),
	6062	try o.builder.intValue(.i32, prefetch.cache),
	6063	}, "");
	6064	return .none;
	6065	}
	6066
	6067	fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	6068	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	6069	const inst_ty = self.typeOfIndex(inst);
	6070	const operand = try self.resolveInst(ty_op.operand);
	6071	return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), "");
	6072	}
	6073
	6074	fn workIntrinsic(
	6075	self: *FuncGen,
	6076	dimension: u32,
	6077	default: u32,
	6078	comptime basename: []const u8,
	6079	) Allocator.Error!Builder.Value {
	6080	return self.wip.callIntrinsic(.normal, .none, switch (dimension) {
	6081	0 => @field(Builder.Intrinsic, basename ++ ".x"),
	6082	1 => @field(Builder.Intrinsic, basename ++ ".y"),
	6083	2 => @field(Builder.Intrinsic, basename ++ ".z"),
	6084	else => return self.object.builder.intValue(.i32, default),
	6085	}, &.{}, &.{}, "");
	6086	}
	6087
	6088	fn airWorkItemId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	6089	const target = self.object.zcu.getTarget();
	6090
	6091	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	6092	const dimension = pl_op.payload;
	6093
	6094	return switch (target.cpu.arch) {
	6095	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workitem.id"),
	6096	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.tid"),
	6097	else => unreachable,
	6098	};
	6099	}
	6100
	6101	fn airWorkGroupSize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	6102	const target = self.object.zcu.getTarget();
	6103
	6104	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	6105	const dimension = pl_op.payload;
	6106
	6107	switch (target.cpu.arch) {
	6108	.amdgcn => {
	6109	if (dimension >= 3) return .@"1";
	6110
	6111	// Fetch the dispatch pointer, which points to this structure:
	6112	// https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913
	6113	const dispatch_ptr =
	6114	try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, "");
	6115
	6116	// Load the work_group_* member from the struct as u16.
	6117	// Just treat the dispatch pointer as an array of u16 to keep things simple.
	6118	const workgroup_size_ptr = try self.ptraddConst(dispatch_ptr, (2 + dimension) * 2);
	6119	return self.wip.load(.normal, .i16, workgroup_size_ptr, comptime .fromByteUnits(2), "");
	6120	},
	6121	.nvptx, .nvptx64 => {
	6122	return self.workIntrinsic(dimension, 1, "nvvm.read.ptx.sreg.ntid");
	6123	},
	6124	else => unreachable,
	6125	}
	6126	}
	6127
	6128	fn airWorkGroupId(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	6129	const target = self.object.zcu.getTarget();
	6130
	6131	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	6132	const dimension = pl_op.payload;
	6133
	6134	return switch (target.cpu.arch) {
	6135	.amdgcn => self.workIntrinsic(dimension, 0, "amdgcn.workgroup.id"),
	6136	.nvptx, .nvptx64 => self.workIntrinsic(dimension, 0, "nvvm.read.ptx.sreg.ctaid"),
	6137	else => unreachable,
	6138	};
	6139	}
	6140
	6141	/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
	6142	fn optCmpNull(
	6143	self: *FuncGen,
	6144	cond: Builder.IntegerCondition,
	6145	opt_ty: Type,
	6146	opt_ptr: Builder.Value,
	6147	access_kind: Builder.MemoryAccessKind,
	6148	) Allocator.Error!Builder.Value {
	6149	const zcu = self.object.zcu;
	6150	assert(isByRef(opt_ty, zcu));
	6151	comptime assert(optional_layout_version == 3);
	6152	// Non-null bit is always after the payload, with no padding because it has alignment 1.
	6153	const non_null_ptr = try self.ptraddConst(opt_ptr, opt_ty.optionalChild(zcu).abiSize(zcu));
	6154	const non_null = try self.wip.load(access_kind, .i8, non_null_ptr, .default, "");
	6155	return self.wip.icmp(cond, non_null, try self.object.builder.intValue(.i8, 0), "");
	6156	}
	6157
	6158	/// Assumes that `Type.optionalReprIsPayload` is `false` for `opt_ty` and that the payload has bits.
	6159	fn optPayloadHandle(
	6160	fg: *FuncGen,
	6161	opt_ptr: Builder.Value,
	6162	opt_ty: Type,
	6163	can_elide_load: bool,
	6164	) Allocator.Error!Builder.Value {
	6165	const zcu = fg.object.zcu;
	6166	assert(isByRef(opt_ty, zcu));
	6167	const payload_ty = opt_ty.optionalChild(zcu);
	6168
	6169	// Payload is first field so always at the same address as the optional itself.
	6170	const payload_ptr = opt_ptr;
	6171
	6172	const payload_align = payload_ty.abiAlignment(zcu).toLlvm();
	6173	if (isByRef(payload_ty, zcu)) {
	6174	if (can_elide_load) return payload_ptr;
	6175	return fg.loadByRef(payload_ptr, payload_ty, payload_align, .normal);
	6176	} else {
	6177	return fg.loadTruncate(.normal, payload_ty, payload_ptr, payload_align);
	6178	}
	6179	}
	6180
	6181	fn fieldPtr(
	6182	self: *FuncGen,
	6183	aggregate_ptr: Builder.Value,
	6184	aggregate_ptr_ty: Type,
	6185	field_index: u32,
	6186	) Allocator.Error!Builder.Value {
	6187	const zcu = self.object.zcu;
	6188	const aggregate_ty = aggregate_ptr_ty.childType(zcu);
	6189	if (aggregate_ty.containerLayout(zcu) == .@"packed") {
	6190	// A pointer to a bitpack field is equivalent to a pointer to the whole bitpack; the
	6191	// bit offset is represented in the pointer type.
	6192	return aggregate_ptr;
	6193	}
	6194	const offset: u64 = switch (aggregate_ty.zigTypeTag(zcu)) {
	6195	.@"struct" => aggregate_ty.structFieldOffset(field_index, zcu),
	6196	.@"union" => aggregate_ty.unionGetLayout(zcu).payloadOffset(),
	6197	else => unreachable,
	6198	};
	6199	return self.ptraddConst(aggregate_ptr, offset);
	6200	}
	6201
	6202	/// Load a value and, if needed, mask out padding bits for non byte-sized integer values.
	6203	fn loadTruncate(
	6204	fg: *FuncGen,
	6205	access_kind: Builder.MemoryAccessKind,
	6206	payload_ty: Type,
	6207	payload_ptr: Builder.Value,
	6208	payload_alignment: Builder.Alignment,
	6209	) Allocator.Error!Builder.Value {
	6210	// from https://llvm.org/docs/LangRef.html#load-instruction :
	6211	// "When loading a value of a type like i20 with a size that is not an integral number of bytes, the result is undefined if the value was not originally written using a store of the same type. "
	6212	// => so load the byte aligned value and trunc the unwanted bits.
	6213
	6214	const o = fg.object;
	6215	const zcu = o.zcu;
	6216	const payload_llvm_ty = try o.lowerType(payload_ty);
	6217	const abi_size = payload_ty.abiSize(zcu);
	6218
	6219	const load_llvm_ty = if (payload_ty.isAbiInt(zcu))
	6220	try o.builder.intType(@intCast(abi_size * 8))
	6221	else
	6222	payload_llvm_ty;
	6223	const loaded = try fg.wip.load(access_kind, load_llvm_ty, payload_ptr, payload_alignment, "");
	6224	const shifted = if (payload_llvm_ty != load_llvm_ty and zcu.getTarget().cpu.arch.endian() == .big)
	6225	try fg.wip.bin(.lshr, loaded, try o.builder.intValue(
	6226	load_llvm_ty,
	6227	(payload_ty.abiSize(zcu) - (std.math.divCeil(u64, payload_ty.bitSize(zcu), 8) catch unreachable)) * 8,
	6228	), "")
	6229	else
	6230	loaded;
	6231
	6232	return fg.wip.conv(.unneeded, shifted, payload_llvm_ty, "");
	6233	}
	6234
	6235	/// Load a by-ref type by constructing a new alloca and performing a memcpy.
	6236	fn loadByRef(
	6237	fg: *FuncGen,
	6238	ptr: Builder.Value,
	6239	pointee_type: Type,
	6240	ptr_alignment: Builder.Alignment,
	6241	access_kind: Builder.MemoryAccessKind,
	6242	) Allocator.Error!Builder.Value {
	6243	const o = fg.object;
	6244	const pointee_llvm_ty = try o.lowerType(pointee_type);
	6245	const result_align = InternPool.Alignment.fromLlvm(ptr_alignment)
	6246	.max(pointee_type.abiAlignment(o.zcu)).toLlvm();
	6247	const result_ptr = try fg.buildAlloca(pointee_llvm_ty, result_align);
	6248	const size_bytes = pointee_type.abiSize(o.zcu);
	6249	_ = try fg.wip.callMemCpy(
	6250	result_ptr,
	6251	result_align,
	6252	ptr,
	6253	ptr_alignment,
	6254	try o.builder.intValue(try o.lowerType(.usize), size_bytes),
	6255	access_kind,
	6256	fg.disable_intrinsics,
	6257	);
	6258	return result_ptr;
	6259	}
	6260
	6261	/// If `isByRef` returns `true` for `elem_ty`, this still performs a copy by memcpy'ing the value
	6262	/// into a new alloca.
	6263	fn load(
	6264	fg: *FuncGen,
	6265	ptr: Builder.Value,
	6266	elem_ty: Type,
	6267	ptr_alignment: Builder.Alignment,
	6268	access_kind: Builder.MemoryAccessKind,
	6269	) Allocator.Error!Builder.Value {
	6270	const zcu = fg.object.zcu;
	6271	if (isByRef(elem_ty, zcu)) {
	6272	return fg.loadByRef(ptr, elem_ty, ptr_alignment, access_kind);
	6273	} else {
	6274	return fg.loadTruncate(access_kind, elem_ty, ptr, ptr_alignment);
	6275	}
	6276	}
	6277
	6278	fn storeFull(
	6279	self: *FuncGen,
	6280	ptr: Builder.Value,
	6281	ptr_ty: Type,
	6282	elem: Builder.Value,
	6283	ordering: Builder.AtomicOrdering,
	6284	) Allocator.Error!void {
	6285	const o = self.object;
	6286	const zcu = o.zcu;
	6287	const info = ptr_ty.ptrInfo(zcu);
	6288	const elem_ty = Type.fromInterned(info.child);
	6289	if (!elem_ty.hasRuntimeBits(zcu)) {
	6290	return;
	6291	}
	6292	const ptr_alignment = ptr_ty.ptrAlignment(zcu).toLlvm();
	6293	const access_kind: Builder.MemoryAccessKind =
	6294	if (info.flags.is_volatile) .@"volatile" else .normal;
	6295
	6296	if (info.flags.vector_index != .none) {
	6297	const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index);
	6298	const vec_elem_ty = try o.lowerType(elem_ty);
	6299	const vec_ty = try o.builder.vectorType(.normal, info.packed_offset.host_size, vec_elem_ty);
	6300
	6301	const loaded_vector = try self.wip.load(.normal, vec_ty, ptr, ptr_alignment, "");
	6302
	6303	const modified_vector = try self.wip.insertElement(loaded_vector, elem, index_u32, "");
	6304
	6305	assert(ordering == .none);
	6306	_ = try self.wip.store(access_kind, modified_vector, ptr, ptr_alignment);
	6307	return;
	6308	}
	6309
	6310	if (info.packed_offset.host_size != 0) {
	6311	const containing_int_ty = try o.builder.intType(@intCast(info.packed_offset.host_size * 8));
	6312	assert(ordering == .none);
	6313	const containing_int =
	6314	try self.wip.load(.normal, containing_int_ty, ptr, ptr_alignment, "");
	6315	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
	6316	const shift_amt = try o.builder.intConst(containing_int_ty, info.packed_offset.bit_offset);
	6317	// Convert to equally-sized integer type in order to perform the bit
	6318	// operations on the value to store
	6319	const value_bits_type = try o.builder.intType(@intCast(elem_bits));
	6320	const value_bits = if (elem_ty.isPtrAtRuntime(zcu))
	6321	try self.wip.cast(.ptrtoint, elem, value_bits_type, "")
	6322	else
	6323	try self.wip.cast(.bitcast, elem, value_bits_type, "");
	6324
	6325	const mask_val = blk: {
	6326	const zext = try self.wip.cast(
	6327	.zext,
	6328	try o.builder.intValue(value_bits_type, -1),
	6329	containing_int_ty,
	6330	"",
	6331	);
	6332	const shl = try self.wip.bin(.shl, zext, shift_amt.toValue(), "");
	6333	break :blk try self.wip.bin(
	6334	.xor,
	6335	shl,
	6336	try o.builder.intValue(containing_int_ty, -1),
	6337	"",
	6338	);
	6339	};
	6340
	6341	const anded_containing_int = try self.wip.bin(.@"and", containing_int, mask_val, "");
	6342	const extended_value = try self.wip.cast(.zext, value_bits, containing_int_ty, "");
	6343	const shifted_value = try self.wip.bin(.shl, extended_value, shift_amt.toValue(), "");
	6344	const ored_value = try self.wip.bin(.@"or", shifted_value, anded_containing_int, "");
	6345
	6346	assert(ordering == .none);
	6347	_ = try self.wip.store(access_kind, ored_value, ptr, ptr_alignment);
	6348	return;
	6349	}
	6350	if (!isByRef(elem_ty, zcu)) {
	6351	_ = try self.wip.storeAtomic(
	6352	access_kind,
	6353	elem,
	6354	ptr,
	6355	self.sync_scope,
	6356	ordering,
	6357	ptr_alignment,
	6358	);
	6359	return;
	6360	}
	6361	assert(ordering == .none);
	6362	_ = try self.wip.callMemCpy(
	6363	ptr,
	6364	ptr_alignment,
	6365	elem,
	6366	elem_ty.abiAlignment(zcu).toLlvm(),
	6367	try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu)),
	6368	access_kind,
	6369	self.disable_intrinsics,
	6370	);
	6371	}
	6372
	6373	/// Non-atomic, non-volatile, non-packed store.
	6374	fn store(
	6375	fg: *FuncGen,
	6376	ptr: Builder.Value,
	6377	ptr_align: InternPool.Alignment,
	6378	elem: Builder.Value,
	6379	elem_ty: Type,
	6380	) Allocator.Error!void {
	6381	const o = fg.object;
	6382	const zcu = o.zcu;
	6383	const llvm_ptr_align = switch (ptr_align) {
	6384	.none => elem_ty.abiAlignment(zcu).toLlvm(),
	6385	else => ptr_align.toLlvm(),
	6386	};
	6387	if (isByRef(elem_ty, zcu)) {
	6388	_ = try fg.wip.callMemCpy(
	6389	ptr,
	6390	llvm_ptr_align,
	6391	elem,
	6392	elem_ty.abiAlignment(zcu).toLlvm(),
	6393	try o.builder.intValue(
	6394	try o.lowerType(.usize),
	6395	elem_ty.abiSize(zcu),
	6396	),
	6397	.normal,
	6398	fg.disable_intrinsics,
	6399	);
	6400	} else {
	6401	_ = try fg.wip.storeAtomic(
	6402	.normal,
	6403	elem,
	6404	ptr,
	6405	fg.sync_scope,
	6406	.none,
	6407	llvm_ptr_align,
	6408	);
	6409	}
	6410	}
	6411
	6412	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
	6413	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
	6414	const o = fg.object;
	6415	const usize_ty = try o.lowerType(.usize);
	6416	const zero = try o.builder.intValue(usize_ty, 0);
	6417	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
	6418	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
	6419	_ = try valgrindClientRequest(fg, zero, req, ptr_as_usize, len, zero, zero, zero);
	6420	}
	6421
	6422	fn valgrindClientRequest(
	6423	fg: *FuncGen,
	6424	default_value: Builder.Value,
	6425	request: Builder.Value,
	6426	a1: Builder.Value,
	6427	a2: Builder.Value,
	6428	a3: Builder.Value,
	6429	a4: Builder.Value,
	6430	a5: Builder.Value,
	6431	) Allocator.Error!Builder.Value {
	6432	const o = fg.object;
	6433	const zcu = o.zcu;
	6434	const target = zcu.getTarget();
	6435	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
	6436
	6437	const llvm_usize = try o.lowerType(.usize);
	6438	const usize_alignment = Type.usize.abiAlignment(zcu).toLlvm();
	6439
	6440	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
	6441	const array_ptr = if (fg.valgrind_client_request_array == .none) a: {
	6442	const array_ptr = try fg.buildAlloca(array_llvm_ty, usize_alignment);
	6443	fg.valgrind_client_request_array = array_ptr;
	6444	break :a array_ptr;
	6445	} else fg.valgrind_client_request_array;
	6446	const array_elements = [_]Builder.Value{ request, a1, a2, a3, a4, a5 };
	6447	for (array_elements, 0..) \|elem, i\| {
	6448	const elem_ptr = try fg.ptraddConst(array_ptr, i * Type.usize.abiSize(zcu));
	6449	_ = try fg.wip.store(.normal, elem, elem_ptr, usize_alignment);
	6450	}
	6451
	6452	const arch_specific: struct {
	6453	template: [:0]const u8,
	6454	constraints: [:0]const u8,
	6455	} = switch (target.cpu.arch) {
	6456	.arm, .armeb, .thumb, .thumbeb => .{
	6457	.template =
	6458	\\ mov r12, r12, ror #3 ; mov r12, r12, ror #13
	6459	\\ mov r12, r12, ror #29 ; mov r12, r12, ror #19
	6460	\\ orr r10, r10, r10
	6461	,
	6462	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
	6463	},
	6464	.aarch64, .aarch64_be => .{
	6465	.template =
	6466	\\ ror x12, x12, #3 ; ror x12, x12, #13
	6467	\\ ror x12, x12, #51 ; ror x12, x12, #61
	6468	\\ orr x10, x10, x10
	6469	,
	6470	.constraints = "={x3},{x4},{x3},~{cc},~{memory}",
	6471	},
	6472	.mips, .mipsel => .{
	6473	.template =
	6474	\\ srl $$0, $$0, 13
	6475	\\ srl $$0, $$0, 29
	6476	\\ srl $$0, $$0, 3
	6477	\\ srl $$0, $$0, 19
	6478	\\ or $$13, $$13, $$13
	6479	,
	6480	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
	6481	},
	6482	.mips64, .mips64el => .{
	6483	.template =
	6484	\\ dsll $$0, $$0, 3 ; dsll $$0, $$0, 13
	6485	\\ dsll $$0, $$0, 29 ; dsll $$0, $$0, 19
	6486	\\ or $$13, $$13, $$13
	6487	,
	6488	.constraints = "={$11},{$12},{$11},~{memory},~{$1}",
	6489	},
	6490	.powerpc, .powerpcle => .{
	6491	.template =
	6492	\\ rlwinm 0, 0, 3, 0, 31 ; rlwinm 0, 0, 13, 0, 31
	6493	\\ rlwinm 0, 0, 29, 0, 31 ; rlwinm 0, 0, 19, 0, 31
	6494	\\ or 1, 1, 1
	6495	,
	6496	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
	6497	},
	6498	.powerpc64, .powerpc64le => .{
	6499	.template =
	6500	\\ rotldi 0, 0, 3 ; rotldi 0, 0, 13
	6501	\\ rotldi 0, 0, 61 ; rotldi 0, 0, 51
	6502	\\ or 1, 1, 1
	6503	,
	6504	.constraints = "={r3},{r4},{r3},~{cc},~{memory}",
	6505	},
	6506	.riscv64 => .{
	6507	.template =
	6508	\\ .option push
	6509	\\ .option norvc
	6510	\\ srli zero, zero, 3
	6511	\\ srli zero, zero, 13
	6512	\\ srli zero, zero, 51
	6513	\\ srli zero, zero, 61
	6514	\\ or a0, a0, a0
	6515	\\ .option pop
	6516	,
	6517	.constraints = "={a3},{a4},{a3},~{cc},~{memory}",
	6518	},
	6519	.s390x => .{
	6520	.template =
	6521	\\ lr %r15, %r15
	6522	\\ lr %r1, %r1
	6523	\\ lr %r2, %r2
	6524	\\ lr %r3, %r3
	6525	\\ lr %r2, %r2
	6526	,
	6527	.constraints = "={r3},{r2},{r3},~{cc},~{memory}",
	6528	},
	6529	.x86 => .{
	6530	.template =
	6531	\\ roll $$3, %edi ; roll $$13, %edi
	6532	\\ roll $$61, %edi ; roll $$51, %edi
	6533	\\ xchgl %ebx, %ebx
	6534	,
	6535	.constraints = "={edx},{eax},{edx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
	6536	},
	6537	.x86_64 => .{
	6538	.template =
	6539	\\ rolq $$3, %rdi ; rolq $$13, %rdi
	6540	\\ rolq $$61, %rdi ; rolq $$51, %rdi
	6541	\\ xchgq %rbx, %rbx
	6542	,
	6543	.constraints = "={rdx},{rax},{rdx},~{cc},~{memory},~{dirflag},~{fpsr},~{flags}",
	6544	},
	6545	else => unreachable,
	6546	};
	6547
	6548	return fg.wip.callAsm(
	6549	.none,
	6550	try o.builder.fnType(llvm_usize, &.{ llvm_usize, llvm_usize }, .normal),
	6551	.{ .sideeffect = true },
	6552	try o.builder.string(arch_specific.template),
	6553	try o.builder.string(arch_specific.constraints),
	6554	&.{ try fg.wip.cast(.ptrtoint, array_ptr, llvm_usize, ""), default_value },
	6555	"",
	6556	);
	6557	}
	6558
	6559	fn typeOf(fg: *FuncGen, inst: Air.Inst.Ref) Type {
	6560	const zcu = fg.object.zcu;
	6561	return fg.air.typeOf(inst, &zcu.intern_pool);
	6562	}
	6563
	6564	fn typeOfIndex(fg: *FuncGen, inst: Air.Inst.Index) Type {
	6565	const zcu = fg.object.zcu;
	6566	return fg.air.typeOfIndex(inst, &zcu.intern_pool);
	6567	}
	6568
	6569	const ParamTypeIterator = struct {
	6570	object: *Object,
	6571	fn_info: InternPool.Key.FuncType,
	6572	zig_index: u32,
	6573	llvm_index: u32,
	6574	types_len: u32,
	6575	types_buffer: [8]Builder.Type,
	6576	byval_attr: bool,
	6577
	6578	const Lowering = union(enum) {
	6579	no_bits,
	6580	byval,
	6581	byref,
	6582	byref_mut,
	6583	abi_sized_int,
	6584	multiple_llvm_types,
	6585	slice,
	6586	float_array: u8,
	6587	i32_array: u8,
	6588	i64_array: u8,
	6589	};
	6590
	6591	pub fn next(it: *ParamTypeIterator) Allocator.Error!?Lowering {
	6592	if (it.zig_index >= it.fn_info.param_types.len) return null;
	6593	const ip = &it.object.zcu.intern_pool;
	6594	const ty = it.fn_info.param_types.get(ip)[it.zig_index];
	6595	it.byval_attr = false;
	6596	return nextInner(it, Type.fromInterned(ty));
	6597	}
	6598
	6599	/// `airCall` uses this instead of `next` so that it can take into account variadic functions.
	6600	fn nextCall(it: ParamTypeIterator, fg: FuncGen, args: []const Air.Inst.Ref) Allocator.Error!?Lowering {
	6601	const ip = &it.object.zcu.intern_pool;
	6602	if (it.zig_index >= it.fn_info.param_types.len) {
	6603	if (it.zig_index >= args.len) {
	6604	return null;
	6605	} else {
	6606	return nextInner(it, fg.typeOf(args[it.zig_index]));
	6607	}
	6608	} else {
	6609	return nextInner(it, Type.fromInterned(it.fn_info.param_types.get(ip)[it.zig_index]));
	6610	}
	6611	}
	6612
	6613	fn nextInner(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
	6614	const zcu = it.object.zcu;
	6615	const target = zcu.getTarget();
	6616
	6617	if (!ty.hasRuntimeBits(zcu)) {
	6618	it.zig_index += 1;
	6619	return .no_bits;
	6620	}
	6621	switch (it.fn_info.cc) {
	6622	.@"inline" => unreachable,
	6623	.auto => {
	6624	it.zig_index += 1;
	6625	it.llvm_index += 1;
	6626	if (ty.isSlice(zcu) or
	6627	(ty.zigTypeTag(zcu) == .optional and ty.optionalChild(zcu).isSlice(zcu) and !ty.ptrAllowsZero(zcu)))
	6628	{
	6629	it.llvm_index += 1;
	6630	return .slice;
	6631	} else if (isByRef(ty, zcu)) {
	6632	return .byref;
	6633	} else if (target.cpu.arch.isX86() and
	6634	!target.cpu.has(.x86, .evex512) and
	6635	ty.totalVectorBits(zcu) >= 512)
	6636	{
	6637	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
	6638	// "512-bit vector arguments require 'evex512' for AVX512"
	6639	return .byref;
	6640	} else {
	6641	return .byval;
	6642	}
	6643	},
	6644	.async => {
	6645	@panic("TODO implement async function lowering in the LLVM backend");
	6646	},
	6647	.x86_64_sysv => return it.nextSystemV(ty),
	6648	.x86_64_win => return it.nextWin64(ty),
	6649	.x86_stdcall => {
	6650	it.zig_index += 1;
	6651	it.llvm_index += 1;
	6652
	6653	if (isScalar(zcu, ty)) {
	6654	return .byval;
	6655	} else {
	6656	it.byval_attr = true;
	6657	return .byref;
	6658	}
	6659	},
	6660	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => {
	6661	it.zig_index += 1;
	6662	it.llvm_index += 1;
	6663	switch (aarch64_c_abi.classifyType(ty, zcu)) {
	6664	.memory => return .byref_mut,
	6665	.float_array => \|len\| return Lowering{ .float_array = len },
	6666	.byval => return .byval,
	6667	.integer => {
	6668	it.types_len = 1;
	6669	it.types_buffer[0] = .i64;
	6670	return .multiple_llvm_types;
	6671	},
	6672	.double_integer => return Lowering{ .i64_array = 2 },
	6673	}
	6674	},
	6675	.arm_aapcs, .arm_aapcs_vfp => {
	6676	it.zig_index += 1;
	6677	it.llvm_index += 1;
	6678	switch (arm_c_abi.classifyType(ty, zcu, .arg)) {
	6679	.memory => {
	6680	it.byval_attr = true;
	6681	return .byref;
	6682	},
	6683	.byval => return .byval,
	6684	.i32_array => \|size\| return Lowering{ .i32_array = size },
	6685	.i64_array => \|size\| return Lowering{ .i64_array = size },
	6686	}
	6687	},
	6688	.mips_o32 => {
	6689	it.zig_index += 1;
	6690	it.llvm_index += 1;
	6691	switch (mips_c_abi.classifyType(ty, zcu, .arg)) {
	6692	.memory => {
	6693	it.byval_attr = true;
	6694	return .byref;
	6695	},
	6696	.byval => return .byval,
	6697	.i32_array => \|size\| return Lowering{ .i32_array = size },
	6698	}
	6699	},
	6700	.riscv64_lp64, .riscv32_ilp32 => {
	6701	it.zig_index += 1;
	6702	it.llvm_index += 1;
	6703	switch (riscv_c_abi.classifyType(ty, zcu)) {
	6704	.memory => return .byref_mut,
	6705	.byval => return .byval,
	6706	.integer => return .abi_sized_int,
	6707	.double_integer => return Lowering{ .i64_array = 2 },
	6708	.fields => {
	6709	it.types_len = 0;
	6710	for (0..ty.structFieldCount(zcu)) \|field_index\| {
	6711	const field_ty = ty.fieldType(field_index, zcu);
	6712	if (!field_ty.hasRuntimeBits(zcu)) continue;
	6713	it.types_buffer[it.types_len] = try it.object.lowerType(field_ty);
	6714	it.types_len += 1;
	6715	}
	6716	it.llvm_index += it.types_len - 1;
	6717	return .multiple_llvm_types;
	6718	},
	6719	}
	6720	},
	6721	.wasm_mvp => switch (wasm_c_abi.classifyType(ty, zcu)) {
	6722	.direct => \|scalar_ty\| {
	6723	if (isScalar(zcu, ty)) {
	6724	it.zig_index += 1;
	6725	it.llvm_index += 1;
	6726	return .byval;
	6727	} else {
	6728	var types_buffer: [8]Builder.Type = undefined;
	6729	types_buffer[0] = try it.object.lowerType(scalar_ty);
	6730	it.types_buffer = types_buffer;
	6731	it.types_len = 1;
	6732	it.llvm_index += 1;
	6733	it.zig_index += 1;
	6734	return .multiple_llvm_types;
	6735	}
	6736	},
	6737	.indirect => {
	6738	it.zig_index += 1;
	6739	it.llvm_index += 1;
	6740	it.byval_attr = true;
	6741	return .byref;
	6742	},
	6743	},
	6744	// TODO investigate other callconvs
	6745	else => {
	6746	it.zig_index += 1;
	6747	it.llvm_index += 1;
	6748	return .byval;
	6749	},
	6750	}
	6751	}
	6752
	6753	fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
	6754	const zcu = it.object.zcu;
	6755	switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
	6756	.integer => {
	6757	if (isScalar(zcu, ty)) {
	6758	it.zig_index += 1;
	6759	it.llvm_index += 1;
	6760	return .byval;
	6761	} else {
	6762	it.zig_index += 1;
	6763	it.llvm_index += 1;
	6764	return .abi_sized_int;
	6765	}
	6766	},
	6767	.win_i128 => {
	6768	it.zig_index += 1;
	6769	it.llvm_index += 1;
	6770	return .byref;
	6771	},
	6772	.memory => {
	6773	it.zig_index += 1;
	6774	it.llvm_index += 1;
	6775	return .byref_mut;
	6776	},
	6777	.sse => {
	6778	it.zig_index += 1;
	6779	it.llvm_index += 1;
	6780	return .byval;
	6781	},
	6782	else => unreachable,
	6783	}
	6784	}
	6785
	6786	fn nextSystemV(it: *ParamTypeIterator, ty: Type) Allocator.Error!?Lowering {
	6787	const zcu = it.object.zcu;
	6788	const ip = &zcu.intern_pool;
	6789	ty.assertHasLayout(zcu);
	6790	const classes = x86_64_abi.classifySystemV(ty, zcu, zcu.getTarget(), .arg);
	6791	if (classes[0] == .memory) {
	6792	it.zig_index += 1;
	6793	it.llvm_index += 1;
	6794	it.byval_attr = true;
	6795	return .byref;
	6796	}
	6797	if (isScalar(zcu, ty)) {
	6798	it.zig_index += 1;
	6799	it.llvm_index += 1;
	6800	return .byval;
	6801	}
	6802	var types_index: u32 = 0;
	6803	var types_buffer: [8]Builder.Type = undefined;
	6804	for (classes) \|class\| {
	6805	switch (class) {
	6806	.integer => {
	6807	types_buffer[types_index] = .i64;
	6808	types_index += 1;
	6809	},
	6810	.sse => {
	6811	types_buffer[types_index] = .double;
	6812	types_index += 1;
	6813	},
	6814	.sseup => {
	6815	if (types_buffer[types_index - 1] == .double) {
	6816	types_buffer[types_index - 1] = .fp128;
	6817	} else {
	6818	types_buffer[types_index] = .double;
	6819	types_index += 1;
	6820	}
	6821	},
	6822	.float => {
	6823	types_buffer[types_index] = .float;
	6824	types_index += 1;
	6825	},
	6826	.float_combine => {
	6827	types_buffer[types_index] = try it.object.builder.vectorType(.normal, 2, .float);
	6828	types_index += 1;
	6829	},
	6830	.x87 => {
	6831	it.zig_index += 1;
	6832	it.llvm_index += 1;
	6833	it.byval_attr = true;
	6834	return .byref;
	6835	},
	6836	.x87up => unreachable,
	6837	.none => break,
	6838	.memory => unreachable, // handled above
	6839	.win_i128 => unreachable, // windows only
	6840	.integer_per_element => {
	6841	@panic("TODO");
	6842	},
	6843	}
	6844	}
	6845	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
	6846	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
	6847	assert(first_non_integer orelse classes.len == types_index);
	6848	if (types_index == 1) {
	6849	it.zig_index += 1;
	6850	it.llvm_index += 1;
	6851	return .abi_sized_int;
	6852	}
	6853	if (it.llvm_index + types_index > 6) {
	6854	it.zig_index += 1;
	6855	it.llvm_index += 1;
	6856	it.byval_attr = true;
	6857	return .byref;
	6858	}
	6859	switch (ip.indexToKey(ty.toIntern())) {
	6860	.struct_type => {
	6861	const size = ty.abiSize(zcu);
	6862	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
	6863	if (size % 8 > 0) {
	6864	types_buffer[types_index - 1] =
	6865	try it.object.builder.intType(@intCast(size % 8 * 8));
	6866	}
	6867	},
	6868	else => {},
	6869	}
	6870	}
	6871	it.types_len = types_index;
	6872	it.types_buffer = types_buffer;
	6873	it.llvm_index += types_index;
	6874	it.zig_index += 1;
	6875	return .multiple_llvm_types;
	6876	}
	6877	};
	6878	pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) ParamTypeIterator {
	6879	return .{
	6880	.object = object,
	6881	.fn_info = fn_info,
	6882	.zig_index = 0,
	6883	.llvm_index = 0,
	6884	.types_len = 0,
	6885	.types_buffer = undefined,
	6886	.byval_attr = false,
	6887	};
	6888	}
	6889
	6890	fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
	6891	if (isByRef(ty, zcu)) {
	6892	return true;
	6893	} else if (target.cpu.arch.isX86() and
	6894	!target.cpu.has(.x86, .evex512) and
	6895	ty.totalVectorBits(zcu) >= 512)
	6896	{
	6897	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
	6898	// "512-bit vector arguments require 'evex512' for AVX512"
	6899	return true;
	6900	} else {
	6901	return false;
	6902	}
	6903	}
	6904
	6905	pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
	6906	const return_type = Type.fromInterned(fn_info.return_type);
	6907	if (!return_type.hasRuntimeBits(zcu)) return false;
	6908
	6909	return switch (fn_info.cc) {
	6910	.auto => returnTypeByRef(zcu, target, return_type),
	6911	.x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
	6912	.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
	6913	.x86_sysv, .x86_win => isByRef(return_type, zcu),
	6914	.x86_stdcall => !isScalar(zcu, return_type),
	6915	.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
	6916	.aarch64_aapcs,
	6917	.aarch64_aapcs_darwin,
	6918	.aarch64_aapcs_win,
	6919	=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
	6920	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
	6921	.memory, .i64_array => true,
	6922	.i32_array => \|size\| size != 1,
	6923	.byval => false,
	6924	},
	6925	.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
	6926	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
	6927	.memory, .i32_array => true,
	6928	.byval => false,
	6929	},
	6930	else => false, // TODO: investigate other targets/callconvs
	6931	};
	6932	}
	6933
	6934	fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
	6935	const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
	6936	if (class[0] == .memory) return true;
	6937	if (class[0] == .x87 and class[2] != .none) return true;
	6938	return false;
	6939	}
	6940
	6941	/// In order to support the C calling convention, some return types need to be lowered
	6942	/// completely differently in the function prototype to honor the C ABI, and then
	6943	/// be effectively bitcasted to the actual return type.
	6944	pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	6945	const zcu = o.zcu;
	6946	const return_type = Type.fromInterned(fn_info.return_type);
	6947	if (!return_type.hasRuntimeBits(zcu)) {
	6948	assert(!return_type.isError(zcu));
	6949	return .void;
	6950	}
	6951	const target = zcu.getTarget();
	6952	switch (fn_info.cc) {
	6953	.@"inline" => unreachable,
	6954	.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type),
	6955
	6956	.x86_64_sysv => return lowerSystemVFnRetTy(o, fn_info),
	6957	.x86_64_win => return lowerWin64FnRetTy(o, fn_info),
	6958	.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void,
	6959	.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type),
	6960	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
	6961	.memory => return .void,
	6962	.float_array => return o.lowerType(return_type),
	6963	.byval => return o.lowerType(return_type),
	6964	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
	6965	.double_integer => return o.builder.arrayType(2, .i64),
	6966	},
	6967	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
	6968	.memory, .i64_array => return .void,
	6969	.i32_array => \|len\| return if (len == 1) .i32 else .void,
	6970	.byval => return o.lowerType(return_type),
	6971	},
	6972	.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
	6973	.memory, .i32_array => return .void,
	6974	.byval => return o.lowerType(return_type),
	6975	},
	6976	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
	6977	.memory => return .void,
	6978	.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
	6979	.double_integer => {
	6980	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
	6981	.riscv64, .riscv64be => .i64,
	6982	.riscv32, .riscv32be => .i32,
	6983	else => unreachable,
	6984	};
	6985	return o.builder.structType(.normal, &.{ integer, integer });
	6986	},
	6987	.byval => return o.lowerType(return_type),
	6988	.fields => {
	6989	var types_len: usize = 0;
	6990	var types: [8]Builder.Type = undefined;
	6991	for (0..return_type.structFieldCount(zcu)) \|field_index\| {
	6992	const field_ty = return_type.fieldType(field_index, zcu);
	6993	if (!field_ty.hasRuntimeBits(zcu)) continue;
	6994	types[types_len] = try o.lowerType(field_ty);
	6995	types_len += 1;
	6996	}
	6997	return o.builder.structType(.normal, types[0..types_len]);
	6998	},
	6999	},
	7000	.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
	7001	.direct => \|scalar_ty\| return o.lowerType(scalar_ty),
	7002	.indirect => return .void,
	7003	},
	7004	// TODO investigate other callconvs
	7005	else => return o.lowerType(return_type),
	7006	}
	7007	}
	7008
	7009	fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7010	const zcu = o.zcu;
	7011	const return_type = Type.fromInterned(fn_info.return_type);
	7012	switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
	7013	.integer => {
	7014	if (isScalar(zcu, return_type)) {
	7015	return o.lowerType(return_type);
	7016	} else {
	7017	return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
	7018	}
	7019	},
	7020	.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
	7021	.memory => return .void,
	7022	.sse => return o.lowerType(return_type),
	7023	else => unreachable,
	7024	}
	7025	}
	7026
	7027	fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7028	const zcu = o.zcu;
	7029	const ip = &zcu.intern_pool;
	7030	const return_type = Type.fromInterned(fn_info.return_type);
	7031	return_type.assertHasLayout(zcu);
	7032	if (isScalar(zcu, return_type)) {
	7033	return o.lowerType(return_type);
	7034	}
	7035	const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
	7036	var types_index: u32 = 0;
	7037	var types_buffer: [8]Builder.Type = undefined;
	7038	for (classes) \|class\| {
	7039	switch (class) {
	7040	.integer => {
	7041	types_buffer[types_index] = .i64;
	7042	types_index += 1;
	7043	},
	7044	.sse => {
	7045	types_buffer[types_index] = .double;
	7046	types_index += 1;
	7047	},
	7048	.sseup => {
	7049	if (types_buffer[types_index - 1] == .double) {
	7050	types_buffer[types_index - 1] = .fp128;
	7051	} else {
	7052	types_buffer[types_index] = .double;
	7053	types_index += 1;
	7054	}
	7055	},
	7056	.float => {
	7057	types_buffer[types_index] = .float;
	7058	types_index += 1;
	7059	},
	7060	.float_combine => {
	7061	types_buffer[types_index] = try o.builder.vectorType(.normal, 2, .float);
	7062	types_index += 1;
	7063	},
	7064	.x87 => {
	7065	if (types_index != 0 or classes[2] != .none) return .void;
	7066	types_buffer[types_index] = .x86_fp80;
	7067	types_index += 1;
	7068	},
	7069	.x87up => continue,
	7070	.none => break,
	7071	.memory, .integer_per_element => return .void,
	7072	.win_i128 => unreachable, // windows only
	7073	}
	7074	}
	7075	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
	7076	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
	7077	assert(first_non_integer orelse classes.len == types_index);
	7078	switch (ip.indexToKey(return_type.toIntern())) {
	7079	.struct_type => {
	7080	const size = return_type.abiSize(zcu);
	7081	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
	7082	if (size % 8 > 0) {
	7083	types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
	7084	}
	7085	},
	7086	else => {},
	7087	}
	7088	if (types_index == 1) return types_buffer[0];
	7089	}
	7090	return o.builder.structType(.normal, types_buffer[0..types_index]);
	7091	}
	7092
	7093	/// This function deliberately does not handle `_BitInt` because it typically
	7094	/// has different ABI than regular integer types, and there is no currently no
	7095	/// way to determine whether a Zig integer type is meant to represent e.g. `int`
	7096	/// or `_BitInt(32)`.
	7097	pub fn ccAbiPromoteInt(cc: std.builtin.CallingConvention, zcu: *Zcu, ty: Type) ?std.builtin.Signedness {
	7098	switch (cc) {
	7099	.auto, .@"inline", .async => return null,
	7100	else => {},
	7101	}
	7102
	7103	const int_info = switch (ty.zigTypeTag(zcu)) {
	7104	.bool => Type.u1.intInfo(zcu),
	7105	else => if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else return null,
	7106	};
	7107	assert(int_info.bits >= 0);
	7108
	7109	const target = zcu.getTarget();
	7110	return switch (target.cpu.arch) {
	7111	.aarch64,
	7112	.aarch64_be,
	7113	=> switch (target.os.tag) {
	7114	.driverkit, .ios, .maccatalyst, .macos, .tvos, .visionos, .watchos => switch (int_info.bits) {
	7115	8, 16 => int_info.signedness,
	7116	else => null,
	7117	},
	7118	else => null,
	7119	},
	7120
	7121	.avr,
	7122	=> switch (int_info.bits) {
	7123	8 => int_info.signedness,
	7124	else => null,
	7125	},
	7126
	7127	.lanai,
	7128	=> null,
	7129
	7130	.loongarch64,
	7131	.riscv64,
	7132	.riscv64be,
	7133	=> switch (int_info.bits) {
	7134	8, 16 => int_info.signedness,
	7135	32 => .signed,
	7136	else => null,
	7137	},
	7138
	7139	.mips,
	7140	.mipsel,
	7141	.mips64,
	7142	.mips64el,
	7143	=> switch (int_info.bits) {
	7144	8, 16, 64 => int_info.signedness,
	7145	// https://github.com/llvm/llvm-project/issues/179088
	7146	// 32 => .signed,
	7147	else => null,
	7148	},
	7149
	7150	.powerpc64,
	7151	.powerpc64le,
	7152	.s390x,
	7153	.sparc64,
	7154	.ve,
	7155	=> switch (int_info.bits) {
	7156	8, 16, 32 => int_info.signedness,
	7157	else => null,
	7158	},
	7159
	7160	else => switch (int_info.bits) {
	7161	8, 16 => int_info.signedness,
	7162	else => null,
	7163	},
	7164	};
	7165	}
	7166
	7167	fn isScalar(zcu: *Zcu, ty: Type) bool {
	7168	return switch (ty.zigTypeTag(zcu)) {
	7169	.void,
	7170	.bool,
	7171	.noreturn,
	7172	.int,
	7173	.float,
	7174	.pointer,
	7175	.optional,
	7176	.error_set,
	7177	.@"enum",
	7178	.@"anyframe",
	7179	.vector,
	7180	=> true,
	7181
	7182	.@"struct" => ty.containerLayout(zcu) == .@"packed",
	7183	.@"union" => ty.containerLayout(zcu) == .@"packed",
	7184	else => false,
	7185	};
	7186	}
	7187
	7188	pub fn buildAllocaInner(
	7189	wip: *Builder.WipFunction,
	7190	llvm_ty: Builder.Type,
	7191	alignment: Builder.Alignment,
	7192	target: *const std.Target,
	7193	) Allocator.Error!Builder.Value {
	7194	const address_space = llvmAllocaAddressSpace(target);
	7195
	7196	const alloca = blk: {
	7197	const prev_cursor = wip.cursor;
	7198	const prev_debug_location = wip.debug_location;
	7199	defer {
	7200	wip.cursor = prev_cursor;
	7201	if (wip.cursor.block == .entry) wip.cursor.instruction += 1;
	7202	wip.debug_location = prev_debug_location;
	7203	}
	7204
	7205	wip.cursor = .{ .block = .entry };
	7206	wip.debug_location = .no_location;
	7207	break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, "");
	7208	};
	7209
	7210	// The pointer returned from this function should have the generic address space,
	7211	// if this isn't the case then cast it to the generic address space.
	7212	return wip.conv(.unneeded, alloca, .ptr, "");
	7213	}
	7214
	7215	/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
	7216	/// or as an LLVM value.
	7217	pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
	7218	return switch (ty.zigTypeTag(zcu)) {
	7219	.type,
	7220	.comptime_int,
	7221	.comptime_float,
	7222	.enum_literal,
	7223	.undefined,
	7224	.null,
	7225	.@"opaque",
	7226	=> unreachable,
	7227
	7228	.noreturn,
	7229	.void,
	7230	.bool,
	7231	.int,
	7232	.float,
	7233	.pointer,
	7234	.error_set,
	7235	.@"fn",
	7236	.@"enum",
	7237	.vector,
	7238	.@"anyframe",
	7239	=> false,
	7240
	7241	.array,
	7242	.frame,
	7243	=> ty.hasRuntimeBits(zcu),
	7244
	7245	.error_union => ty.errorUnionPayload(zcu).hasRuntimeBits(zcu),
	7246
	7247	.optional => !ty.optionalReprIsPayload(zcu) and ty.optionalChild(zcu).hasRuntimeBits(zcu),
	7248
	7249	.@"struct" => switch (ty.containerLayout(zcu)) {
	7250	.@"packed" => false,
	7251	.auto, .@"extern" => ty.hasRuntimeBits(zcu),
	7252	},
	7253	.@"union" => switch (ty.containerLayout(zcu)) {
	7254	.@"packed" => false,
	7255	else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
	7256	},
	7257	};
	7258	}
	7259
	7260	/// If the operand type of an atomic operation is not byte sized we need to
	7261	/// widen it before using it and then truncate the result.
	7262	/// RMW exchange of floating-point values is bitcasted to same-sized integer
	7263	/// types to work around a LLVM deficiency when targeting ARM/AArch64.
	7264	fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.Error!Builder.Type {
	7265	const zcu = fg.object.zcu;
	7266	switch (ty.zigTypeTag(zcu)) {
	7267	.int, .@"enum", .@"struct", .@"union" => {},
	7268	.float => {
	7269	if (!is_rmw_xchg) return .none;
	7270	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
	7271	},
	7272	.bool => return .i8,
	7273	else => return .none,
	7274	}
	7275	const bit_count = ty.bitSize(zcu);
	7276	if (!std.math.isPowerOfTwo(bit_count) or (bit_count % 8) != 0) {
	7277	return fg.object.builder.intType(@intCast(ty.abiSize(zcu) * 8));
	7278	} else {
	7279	return .none;
	7280	}
	7281	}
	7282
	7283	fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
	7284	if (offset == 0) return ptr;
	7285	const o = fg.object;
	7286	const llvm_usize_ty = try o.lowerType(.usize);
	7287	const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
	7288	return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
	7289	}
	7290	fn ptraddScaled(fg: *FuncGen, ptr: Builder.Value, index: Builder.Value, scale: u64) Allocator.Error!Builder.Value {
	7291	if (scale == 0) return ptr;
	7292	// Right now LLVM seems to fare a bit worse with an explicit `mul nuw` instruction than it does
	7293	// if we use a bigger type for the GEP, so we'll do that. As I understand it, it has not yet
	7294	// been decided whether the planned `ptradd` instruction will accept a scale or not; if it does
	7295	// not then presumably upstream will improve their handling of explicit `mul nuw` computing the
	7296	// offset.
	7297	const llvm_scale_ty = try fg.object.builder.arrayType(scale, .i8);
	7298	return fg.wip.gep(.inbounds, llvm_scale_ty, ptr, &.{index}, "");
	7299	}
	7300
	7301	fn compilerRtIntBits(bits: u16) ?u16 {
	7302	inline for (.{ 32, 64, 128 }) \|b\| {
	7303	if (bits <= b) {
	7304	return b;
	7305	}
	7306	}
	7307	return null;
	7308	}
	7309
	7310	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
	7311	///
	7312	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
	7313	fn constraintAllowsMemory(constraint: []const u8) bool {
	7314	// TODO: This implementation is woefully incomplete.
	7315	for (constraint) \|byte\| {
	7316	switch (byte) {
	7317	'=', '*', ',', '&' => {},
	7318	'm', 'o', 'X', 'g' => return true,
	7319	else => {},
	7320	}
	7321	} else return false;
	7322	}
	7323
	7324	/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
	7325	///
	7326	/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
	7327	fn constraintAllowsRegister(constraint: []const u8) bool {
	7328	// TODO: This implementation is woefully incomplete.
	7329	for (constraint) \|byte\| {
	7330	switch (byte) {
	7331	'=', '*', ',', '&' => {},
	7332	'm', 'o' => {},
	7333	else => return true,
	7334	}
	7335	} else return false;
	7336	}
	7337
	7338	/// Appends zero or more LLVM constraints to `llvm_constraints`, returning how many were added.
	7339	fn appendConstraints(
	7340	gpa: Allocator,
	7341	llvm_constraints: *std.ArrayList(u8),
	7342	zig_name: []const u8,
	7343	target: *const std.Target,
	7344	) error{OutOfMemory}!usize {
	7345	switch (target.cpu.arch) {
	7346	.mips, .mipsel, .mips64, .mips64el => if (mips_clobber_overrides.get(zig_name)) \|llvm_tag\| {
	7347	const llvm_name = @tagName(llvm_tag);
	7348	try llvm_constraints.ensureUnusedCapacity(gpa, llvm_name.len + 4);
	7349	llvm_constraints.appendSliceAssumeCapacity("~{");
	7350	llvm_constraints.appendSliceAssumeCapacity(llvm_name);
	7351	llvm_constraints.appendSliceAssumeCapacity("},");
	7352	return 1;
	7353	},
	7354	else => {},
	7355	}
	7356
	7357	try llvm_constraints.ensureUnusedCapacity(gpa, zig_name.len + 4);
	7358	llvm_constraints.appendSliceAssumeCapacity("~{");
	7359	llvm_constraints.appendSliceAssumeCapacity(zig_name);
	7360	llvm_constraints.appendSliceAssumeCapacity("},");
	7361	return 1;
	7362	}
	7363
	7364	/// LLVM does not support all relevant intrinsics for all targets, so we
	7365	/// may need to manually generate a compiler-rt call.
	7366	fn intrinsicsAllowed(scalar_ty: Type, target: *const std.Target) bool {
	7367	return switch (scalar_ty.toIntern()) {
	7368	.f16_type => llvm.backendSupportsF16(target),
	7369	.f80_type => (target.cTypeBitSize(.longdouble) == 80) and llvm.backendSupportsF80(target),
	7370	.f128_type => (target.cTypeBitSize(.longdouble) == 128) and llvm.backendSupportsF128(target),
	7371	else => true,
	7372	};
	7373	}
	7374
	7375	fn toLlvmAtomicOrdering(atomic_order: std.builtin.AtomicOrder) Builder.AtomicOrdering {
	7376	return switch (atomic_order) {
	7377	.unordered => .unordered,
	7378	.monotonic => .monotonic,
	7379	.acquire => .acquire,
	7380	.release => .release,
	7381	.acq_rel => .acq_rel,
	7382	.seq_cst => .seq_cst,
	7383	};
	7384	}
	7385
	7386	fn toLlvmAtomicRmwBinOp(
	7387	op: std.builtin.AtomicRmwOp,
	7388	is_signed: bool,
	7389	is_float: bool,
	7390	) Builder.Function.Instruction.AtomicRmw.Operation {
	7391	return switch (op) {
	7392	.Xchg => .xchg,
	7393	.Add => if (is_float) .fadd else return .add,
	7394	.Sub => if (is_float) .fsub else return .sub,
	7395	.And => .@"and",
	7396	.Nand => .nand,
	7397	.Or => .@"or",
	7398	.Xor => .xor,
	7399	.Max => if (is_float) .fmax else if (is_signed) .max else return .umax,
	7400	.Min => if (is_float) .fmin else if (is_signed) .min else return .umin,
	7401	};
	7402	}
	7403
	7404	fn minIntConst(b: Builder, min_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
	7405	const info = min_ty.intInfo(zcu);
	7406	if (info.signedness == .unsigned or info.bits == 0) {
	7407	return b.intConst(as_ty, 0);
	7408	}
	7409	if (std.math.cast(u6, info.bits - 1)) \|shift\| {
	7410	const min_val: i64 = @as(i64, std.math.minInt(i64)) >> (63 - shift);
	7411	return b.intConst(as_ty, min_val);
	7412	}
	7413	var res: std.math.big.int.Managed = try .init(zcu.gpa);
	7414	defer res.deinit();
	7415	try res.setTwosCompIntLimit(.min, info.signedness, info.bits);
	7416	return b.bigIntConst(as_ty, res.toConst());
	7417	}
	7418
	7419	fn maxIntConst(b: Builder, max_ty: Type, as_ty: Builder.Type, zcu: const Zcu) Allocator.Error!Builder.Constant {
	7420	const info = max_ty.intInfo(zcu);
	7421	switch (info.bits) {
	7422	0 => return b.intConst(as_ty, 0),
	7423	1 => switch (info.signedness) {
	7424	.signed => return b.intConst(as_ty, 0),
	7425	.unsigned => return b.intConst(as_ty, 1),
	7426	},
	7427	else => {},
	7428	}
	7429	const unsigned_bits = switch (info.signedness) {
	7430	.unsigned => info.bits,
	7431	.signed => info.bits - 1,
	7432	};
	7433	if (std.math.cast(u6, unsigned_bits)) \|shift\| {
	7434	const max_val: u64 = (@as(u64, 1) << shift) - 1;
	7435	return b.intConst(as_ty, max_val);
	7436	}
	7437	var res: std.math.big.int.Managed = try .init(zcu.gpa);
	7438	defer res.deinit();
	7439	try res.setTwosCompIntLimit(.max, info.signedness, info.bits);
	7440	return b.bigIntConst(as_ty, res.toConst());
	7441	}
	7442
	7443	/// On some targets, local values that are in the generic address space must be generated into a
	7444	/// different address, space and then cast back to the generic address space.
	7445	/// For example, on GPUs local variable declarations must be generated into the local address space.
	7446	/// This function returns the address space local values should be generated into.
	7447	fn llvmAllocaAddressSpace(target: *const std.Target) Builder.AddrSpace {
	7448	return switch (target.cpu.arch) {
	7449	// On amdgcn, locals should be generated into the private address space.
	7450	// To make Zig not impossible to use, these are then converted to addresses in the
	7451	// generic address space and treates as regular pointers. This is the way that HIP also does it.
	7452	.amdgcn => Builder.AddrSpace.amdgpu.private,
	7453	else => .default,
	7454	};
	7455	}
	7456
	7457	const mips_clobber_overrides = std.StaticStringMap(enum {
	7458	@"$msair",
	7459	@"$msacsr",
	7460	@"$msaaccess",
	7461	@"$msasave",
	7462	@"$msamodify",
	7463	@"$msarequest",
	7464	@"$msamap",
	7465	@"$msaunmap",
	7466	@"$f0",
	7467	@"$f1",
	7468	@"$f2",
	7469	@"$f3",
	7470	@"$f4",
	7471	@"$f5",
	7472	@"$f6",
	7473	@"$f7",
	7474	@"$f8",
	7475	@"$f9",
	7476	@"$f10",
	7477	@"$f11",
	7478	@"$f12",
	7479	@"$f13",
	7480	@"$f14",
	7481	@"$f15",
	7482	@"$f16",
	7483	@"$f17",
	7484	@"$f18",
	7485	@"$f19",
	7486	@"$f20",
	7487	@"$f21",
	7488	@"$f22",
	7489	@"$f23",
	7490	@"$f24",
	7491	@"$f25",
	7492	@"$f26",
	7493	@"$f27",
	7494	@"$f28",
	7495	@"$f29",
	7496	@"$f30",
	7497	@"$f31",
	7498	@"$fcc0",
	7499	@"$fcc1",
	7500	@"$fcc2",
	7501	@"$fcc3",
	7502	@"$fcc4",
	7503	@"$fcc5",
	7504	@"$fcc6",
	7505	@"$fcc7",
	7506	@"$w0",
	7507	@"$w1",
	7508	@"$w2",
	7509	@"$w3",
	7510	@"$w4",
	7511	@"$w5",
	7512	@"$w6",
	7513	@"$w7",
	7514	@"$w8",
	7515	@"$w9",
	7516	@"$w10",
	7517	@"$w11",
	7518	@"$w12",
	7519	@"$w13",
	7520	@"$w14",
	7521	@"$w15",
	7522	@"$w16",
	7523	@"$w17",
	7524	@"$w18",
	7525	@"$w19",
	7526	@"$w20",
	7527	@"$w21",
	7528	@"$w22",
	7529	@"$w23",
	7530	@"$w24",
	7531	@"$w25",
	7532	@"$w26",
	7533	@"$w27",
	7534	@"$w28",
	7535	@"$w29",
	7536	@"$w30",
	7537	@"$w31",
	7538	@"$0",
	7539	@"$1",
	7540	@"$2",
	7541	@"$3",
	7542	@"$4",
	7543	@"$5",
	7544	@"$6",
	7545	@"$7",
	7546	@"$8",
	7547	@"$9",
	7548	@"$10",
	7549	@"$11",
	7550	@"$12",
	7551	@"$13",
	7552	@"$14",
	7553	@"$15",
	7554	@"$16",
	7555	@"$17",
	7556	@"$18",
	7557	@"$19",
	7558	@"$20",
	7559	@"$21",
	7560	@"$22",
	7561	@"$23",
	7562	@"$24",
	7563	@"$25",
	7564	@"$26",
	7565	@"$27",
	7566	@"$28",
	7567	@"$29",
	7568	@"$30",
	7569	@"$31",
	7570	}).initComptime(.{
	7571	.{ "msa_ir", .@"$msair" },
	7572	.{ "msa_csr", .@"$msacsr" },
	7573	.{ "msa_access", .@"$msaaccess" },
	7574	.{ "msa_save", .@"$msasave" },
	7575	.{ "msa_modify", .@"$msamodify" },
	7576	.{ "msa_request", .@"$msarequest" },
	7577	.{ "msa_map", .@"$msamap" },
	7578	.{ "msa_unmap", .@"$msaunmap" },
	7579	.{ "f0", .@"$f0" },
	7580	.{ "f1", .@"$f1" },
	7581	.{ "f2", .@"$f2" },
	7582	.{ "f3", .@"$f3" },
	7583	.{ "f4", .@"$f4" },
	7584	.{ "f5", .@"$f5" },
	7585	.{ "f6", .@"$f6" },
	7586	.{ "f7", .@"$f7" },
	7587	.{ "f8", .@"$f8" },
	7588	.{ "f9", .@"$f9" },
	7589	.{ "f10", .@"$f10" },
	7590	.{ "f11", .@"$f11" },
	7591	.{ "f12", .@"$f12" },
	7592	.{ "f13", .@"$f13" },
	7593	.{ "f14", .@"$f14" },
	7594	.{ "f15", .@"$f15" },
	7595	.{ "f16", .@"$f16" },
	7596	.{ "f17", .@"$f17" },
	7597	.{ "f18", .@"$f18" },
	7598	.{ "f19", .@"$f19" },
	7599	.{ "f20", .@"$f20" },
	7600	.{ "f21", .@"$f21" },
	7601	.{ "f22", .@"$f22" },
	7602	.{ "f23", .@"$f23" },
	7603	.{ "f24", .@"$f24" },
	7604	.{ "f25", .@"$f25" },
	7605	.{ "f26", .@"$f26" },
	7606	.{ "f27", .@"$f27" },
	7607	.{ "f28", .@"$f28" },
	7608	.{ "f29", .@"$f29" },
	7609	.{ "f30", .@"$f30" },
	7610	.{ "f31", .@"$f31" },
	7611	.{ "fcc0", .@"$fcc0" },
	7612	.{ "fcc1", .@"$fcc1" },
	7613	.{ "fcc2", .@"$fcc2" },
	7614	.{ "fcc3", .@"$fcc3" },
	7615	.{ "fcc4", .@"$fcc4" },
	7616	.{ "fcc5", .@"$fcc5" },
	7617	.{ "fcc6", .@"$fcc6" },
	7618	.{ "fcc7", .@"$fcc7" },
	7619	.{ "w0", .@"$w0" },
	7620	.{ "w1", .@"$w1" },
	7621	.{ "w2", .@"$w2" },
	7622	.{ "w3", .@"$w3" },
	7623	.{ "w4", .@"$w4" },
	7624	.{ "w5", .@"$w5" },
	7625	.{ "w6", .@"$w6" },
	7626	.{ "w7", .@"$w7" },
	7627	.{ "w8", .@"$w8" },
	7628	.{ "w9", .@"$w9" },
	7629	.{ "w10", .@"$w10" },
	7630	.{ "w11", .@"$w11" },
	7631	.{ "w12", .@"$w12" },
	7632	.{ "w13", .@"$w13" },
	7633	.{ "w14", .@"$w14" },
	7634	.{ "w15", .@"$w15" },
	7635	.{ "w16", .@"$w16" },
	7636	.{ "w17", .@"$w17" },
	7637	.{ "w18", .@"$w18" },
	7638	.{ "w19", .@"$w19" },
	7639	.{ "w20", .@"$w20" },
	7640	.{ "w21", .@"$w21" },
	7641	.{ "w22", .@"$w22" },
	7642	.{ "w23", .@"$w23" },
	7643	.{ "w24", .@"$w24" },
	7644	.{ "w25", .@"$w25" },
	7645	.{ "w26", .@"$w26" },
	7646	.{ "w27", .@"$w27" },
	7647	.{ "w28", .@"$w28" },
	7648	.{ "w29", .@"$w29" },
	7649	.{ "w30", .@"$w30" },
	7650	.{ "w31", .@"$w31" },
	7651	.{ "r0", .@"$0" },
	7652	.{ "r1", .@"$1" },
	7653	.{ "r2", .@"$2" },
	7654	.{ "r3", .@"$3" },
	7655	.{ "r4", .@"$4" },
	7656	.{ "r5", .@"$5" },
	7657	.{ "r6", .@"$6" },
	7658	.{ "r7", .@"$7" },
	7659	.{ "r8", .@"$8" },
	7660	.{ "r9", .@"$9" },
	7661	.{ "r10", .@"$10" },
	7662	.{ "r11", .@"$11" },
	7663	.{ "r12", .@"$12" },
	7664	.{ "r13", .@"$13" },
	7665	.{ "r14", .@"$14" },
	7666	.{ "r15", .@"$15" },
	7667	.{ "r16", .@"$16" },
	7668	.{ "r17", .@"$17" },
	7669	.{ "r18", .@"$18" },
	7670	.{ "r19", .@"$19" },
	7671	.{ "r20", .@"$20" },
	7672	.{ "r21", .@"$21" },
	7673	.{ "r22", .@"$22" },
	7674	.{ "r23", .@"$23" },
	7675	.{ "r24", .@"$24" },
	7676	.{ "r25", .@"$25" },
	7677	.{ "r26", .@"$26" },
	7678	.{ "r27", .@"$27" },
	7679	.{ "r28", .@"$28" },
	7680	.{ "r29", .@"$29" },
	7681	.{ "r30", .@"$30" },
	7682	.{ "r31", .@"$31" },
	7683	});
	7684
	7685	const std = @import("std");
	7686	const Allocator = std.mem.Allocator;
	7687	const Builder = std.zig.llvm.Builder;
	7688	const assert = std.debug.assert;
	7689	const math = std.math;
	7690
	7691	const x86_64_abi = @import("../x86_64/abi.zig");
	7692	const wasm_c_abi = @import("../wasm/abi.zig");
	7693	const aarch64_c_abi = @import("../aarch64/abi.zig");
	7694	const arm_c_abi = @import("../arm/abi.zig");
	7695	const riscv_c_abi = @import("../riscv64/abi.zig");
	7696	const mips_c_abi = @import("../mips/abi.zig");
	7697
	7698	const Zcu = @import("../../Zcu.zig");
	7699	const Air = @import("../../Air.zig");
	7700	const Package = @import("../../Package.zig");
	7701	const InternPool = @import("../../InternPool.zig");
	7702	const Value = @import("../../Value.zig");
	7703	const Type = @import("../../Type.zig");
	7704	const codegen = @import("../../codegen.zig");
	7705
	7706	const target_util = @import("../../target.zig");
	7707	const libcFloatPrefix = target_util.libcFloatPrefix;
	7708	const libcFloatSuffix = target_util.libcFloatSuffix;
	7709	const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
	7710	const compilerRtFloatAbbrev = target_util.compilerRtFloatAbbrev;
	7711
	7712	const llvm = @import("../llvm.zig");
	7713	const Object = llvm.Object;
	7714	const optional_layout_version = llvm.optional_layout_version;

src/codegen/riscv64/CodeGen.zig+5-5

...	...	@@ -1477,7 +1477,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1477	1477	=> try func.airCmp(inst, tag),
1478	1478
1479	1479	.cmp_vector => try func.airCmpVector(inst),
1480		.cmp_lt_errors_len => try func.airCmpLtErrorsLen(inst),
	1480	.cmp_lte_errors_len => try func.airCmpLteErrorsLen(inst),
1481	1481
1482	1482	.slice => try func.airSlice(inst),
1483	1483	.array_to_slice => try func.airArrayToSlice(inst),
...	...	@@ -4956,8 +4956,8 @@ fn genCall(
4956	4956	// on linking.
4957	4957	switch (info) {
4958	4958	.air => \|callee\| {
4959		if (try func.air.value(callee, pt)) \|func_value\| {
4960		const func_key = zcu.intern_pool.indexToKey(func_value.ip_index);
	4959	if (callee.toInterned()) \|func_ip_index\| {
	4960	const func_key = zcu.intern_pool.indexToKey(func_ip_index);
4961	4961	switch (switch (func_key) {
4962	4962	else => func_key,
4963	4963	.ptr => \|ptr\| if (ptr.byte_offset == 0) switch (ptr.base_addr) {
...	...	@@ -5186,11 +5186,11 @@ fn airCmpVector(func: *Func, inst: Air.Inst.Index) !void {
5186	5186	return func.fail("TODO implement airCmpVector for {}", .{func.target.cpu.arch});
5187	5187	}
5188	5188
5189		fn airCmpLtErrorsLen(func: *Func, inst: Air.Inst.Index) !void {
	5189	fn airCmpLteErrorsLen(func: *Func, inst: Air.Inst.Index) !void {
5190	5190	const un_op = func.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
5191	5191	const operand = try func.resolveInst(un_op);
5192	5192	_ = operand;
5193		const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLtErrorsLen for {}", .{func.target.cpu.arch});
	5193	const result: MCValue = if (func.liveness.isUnused(inst)) .unreach else return func.fail("TODO implement airCmpLteErrorsLen for {}", .{func.target.cpu.arch});
5194	5194	return func.finishAir(inst, result, .{ un_op, .none, .none });
5195	5195	}
5196	5196

src/codegen/sparc64/CodeGen.zig+5-5

...	...	@@ -545,7 +545,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
545	545	.cmp_gt => try self.airCmp(inst, .gt),
546	546	.cmp_neq => try self.airCmp(inst, .neq),
547	547	.cmp_vector => @panic("TODO try self.airCmpVector(inst)"),
548		.cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst),
	548	.cmp_lte_errors_len => try self.airCmpLteErrorsLen(inst),
549	549
550	550	.alloc => try self.airAlloc(inst),
551	551	.ret_ptr => try self.airRetPtr(inst),
...	...	@@ -1310,7 +1310,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1310	1310
1311	1311	// Due to incremental compilation, how function calls are generated depends
1312	1312	// on linking.
1313		if (try self.air.value(call.callee, pt)) \|func_value\| switch (ip.indexToKey(func_value.toIntern())) {
	1313	if (call.callee.toInterned()) \|func_ip_index\| switch (ip.indexToKey(func_ip_index)) {
1314	1314	.func => {
1315	1315	return self.fail("TODO implement calling functions", .{});
1316	1316	},
...	...	@@ -1425,11 +1425,11 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void {
1425	1425	return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1426	1426	}
1427	1427
1428		fn airCmpLtErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
	1428	fn airCmpLteErrorsLen(self: *Self, inst: Air.Inst.Index) !void {
1429	1429	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1430	1430	const operand = try self.resolveInst(un_op);
1431	1431	_ = operand;
1432		const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLtErrorsLen for {}", .{self.target.cpu.arch});
	1432	const result: MCValue = if (self.liveness.isUnused(inst)) .dead else return self.fail("TODO implement airCmpLteErrorsLen for {}", .{self.target.cpu.arch});
1433	1433	return self.finishAir(inst, result, .{ un_op, .none, .none });
1434	1434	}
1435	1435
...	...	@@ -4487,7 +4487,7 @@ fn resolveInst(self: *Self, ref: Air.Inst.Ref) InnerError!MCValue {
4487	4487	return self.getResolvedInstValue(inst);
4488	4488	}
4489	4489
4490		return self.genTypedValue((try self.air.value(ref, pt)).?);
	4490	return self.genTypedValue(.fromInterned(ref.toInterned().?));
4491	4491	}
4492	4492
4493	4493	fn ret(self: *Self, mcv: MCValue) !void {

src/codegen/spirv/CodeGen.zig+7-10

...	...	@@ -387,13 +387,12 @@ fn importExtendedSet(cg: *CodeGen) !Id {
387	387
388	388	/// Fetch the result-id for a previously generated instruction or constant.
389	389	fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id {
390		const pt = cg.pt;
391	390	const zcu = cg.module.zcu;
392	391	const ip = &zcu.intern_pool;
393		if (try cg.air.value(inst, pt)) \|val\| {
	392	if (inst.toInterned()) \|val_ip_index\| {
394	393	const ty = cg.typeOf(inst);
395	394	if (ty.zigTypeTag(zcu) == .@"fn") {
396		const fn_nav = switch (zcu.intern_pool.indexToKey(val.ip_index)) {
	395	const fn_nav = switch (zcu.intern_pool.indexToKey(val_ip_index)) {
397	396	.@"extern" => \|@"extern"\| @"extern".owner_nav,
398	397	.func => \|func\| func.owner_nav,
399	398	else => unreachable,
...	...	@@ -403,7 +402,7 @@ fn resolve(cg: *CodeGen, inst: Air.Inst.Ref) !Id {
403	402	return cg.module.declPtr(spv_decl_index).result_id;
404	403	}
405	404
406		return try cg.constant(ty, val, .direct);
	405	return try cg.constant(ty, .fromInterned(val_ip_index), .direct);
407	406	}
408	407	const index = inst.toIndex().?;
409	408	return cg.inst_results.get(index).?; // Assertion means instruction does not dominate usage.
...	...	@@ -5657,7 +5656,6 @@ fn airWrapOptional(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5657	5656
5658	5657	fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
5659	5658	const gpa = cg.module.gpa;
5660		const pt = cg.pt;
5661	5659	const zcu = cg.module.zcu;
5662	5660	const target = cg.module.zcu.getTarget();
5663	5661	const switch_br = cg.air.unwrapSwitch(inst);
...	...	@@ -5732,7 +5730,7 @@ fn airSwitchBr(cg: *CodeGen, inst: Air.Inst.Index) !void {
5732	5730	const label = case_labels.at(case.idx);
5733	5731
5734	5732	for (case.items) \|item\| {
5735		const value = (try cg.air.value(item, pt)) orelse unreachable;
	5733	const value: Value = .fromInterned(item.toInterned().?);
5736	5734	const int_val: u64 = switch (cond_ty.zigTypeTag(zcu)) {
5737	5735	.bool, .int => if (cond_ty.isSignedInt(zcu)) @bitCast(value.toSignedInt(zcu)) else value.toUnsignedInt(zcu),
5738	5736	.@"enum" => blk: {
...	...	@@ -5875,9 +5873,9 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5875	5873
5876	5874	if (std.mem.eql(u8, in.constraint, "c")) {
5877	5875	// constant
5878		const val = (try cg.air.value(in.operand, cg.pt)) orelse {
	5876	const val: Value = .fromInterned(in.operand.toInterned() orelse {
5879	5877	return cg.fail("assembly inputs with 'c' constraint have to be compile-time known", .{});
5880		};
	5878	});
5881	5879
5882	5880	// TODO: This entire function should be handled a bit better...
5883	5881	const ip = &zcu.intern_pool;
...	...	@@ -5911,8 +5909,7 @@ fn airAssembly(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
5911	5909	if (input_ty.zigTypeTag(zcu) == .type) {
5912	5910	// This assembly input is a type instead of a value.
5913	5911	// That's fine for now, just make sure to resolve it as such.
5914		const val = (try cg.air.value(in.operand, cg.pt)).?;
5915		const ty_id = try cg.resolveType(val.toType(), .direct);
	5912	const ty_id = try cg.resolveType(in.operand.toType(), .direct);
5916	5913	try ass.value_map.put(gpa, in.name, .{ .ty = ty_id });
5917	5914	} else {
5918	5915	const ty_id = try cg.resolveType(input_ty, .direct);

src/codegen/wasm/CodeGen.zig+5-5

...	...	@@ -303,7 +303,7 @@ fn resolveInst(cg: *CodeGen, ref: Air.Inst.Ref) InnerError!WValue {
303	303
304	304	const pt = cg.pt;
305	305	const zcu = pt.zcu;
306		const val = (try cg.air.value(ref, pt)).?;
	306	const val: Value = .fromInterned(ref.toInterned().?);
307	307	const ty = cg.typeOf(ref);
308	308	if (!ty.hasRuntimeBits(zcu) and !ty.isInt(zcu) and !ty.isError(zcu)) {
309	309	gop.value_ptr.* = .none;
...	...	@@ -1718,7 +1718,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1718	1718	.cmp_neq => cg.airCmp(inst, .neq),
1719	1719
1720	1720	.cmp_vector => cg.airCmpVector(inst),
1721		.cmp_lt_errors_len => cg.airCmpLtErrorsLen(inst),
	1721	.cmp_lte_errors_len => cg.airCmpLteErrorsLen(inst),
1722	1722
1723	1723	.array_elem_val => cg.airArrayElemVal(inst),
1724	1724	.array_to_slice => cg.airArrayToSlice(inst),
...	...	@@ -2006,7 +2006,7 @@ fn airCall(cg: *CodeGen, inst: Air.Inst.Index, modifier: std.builtin.CallModifie
2006	2006	const first_param_sret = firstParamSRet(fn_info.cc, Type.fromInterned(fn_info.return_type), zcu, cg.target);
2007	2007
2008	2008	const callee: ?InternPool.Nav.Index = blk: {
2009		const func_val = (try cg.air.value(call.callee, pt)) orelse break :blk null;
	2009	const func_val: Value = .fromInterned(call.callee.toInterned() orelse break :blk null);
2010	2010
2011	2011	switch (ip.indexToKey(func_val.toIntern())) {
2012	2012	inline .func, .@"extern" => \|x\| break :blk x.owner_nav,
...	...	@@ -4464,7 +4464,7 @@ fn lowerConstant(cg: *CodeGen, val: Value) InnerError!WValue {
4464	4464	.vector_type => {
4465	4465	assert(determineSimdStoreStrategy(ty, zcu, cg.target) == .direct);
4466	4466	var buf: [16]u8 = undefined;
4467		val.writeToMemory(pt, &buf) catch unreachable;
	4467	val.writeToMemory(zcu, &buf) catch unreachable;
4468	4468	return cg.storeSimdImmd(buf);
4469	4469	},
4470	4470	.struct_type => unreachable, // packed structs use `bitpack`
...	...	@@ -4841,7 +4841,7 @@ fn airCmpVector(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4841	4841	return cg.fail("TODO implement airCmpVector for wasm", .{});
4842	4842	}
4843	4843
4844		fn airCmpLtErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
	4844	fn airCmpLteErrorsLen(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
4845	4845	const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
4846	4846	const operand = try cg.resolveInst(un_op);
4847	4847

src/codegen/x86_64/CodeGen.zig+3-3

...	...	@@ -172921,7 +172921,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
172921	172921	try ops[0].finish(inst, &.{field_parent_ptr.field_ptr}, &ops, cg);
172922	172922	},
172923	172923	.wasm_memory_size, .wasm_memory_grow => unreachable,
172924		.cmp_lt_errors_len => \|air_tag\| {
	172924	.cmp_lte_errors_len => \|air_tag\| {
172925	172925	const un_op = air_datas[@intFromEnum(inst)].un_op;
172926	172926	var ops = try cg.tempsFromOperands(inst, .{un_op});
172927	172927	var res: [1]Temp = undefined;
...	...	@@ -176185,8 +176185,8 @@ fn genCall(self: *CodeGen, info: union(enum) {
176185	176185	// Due to incremental compilation, how function calls are generated depends
176186	176186	// on linking.
176187	176187	switch (info) {
176188		.air => \|callee\| if (try self.air.value(callee, pt)) \|func_value\| {
176189		const func_key = ip.indexToKey(func_value.ip_index);
	176188	.air => \|callee\| if (callee.toInterned()) \|func_ip_index\| {
	176189	const func_key = ip.indexToKey(func_ip_index);
176190	176190	switch (switch (func_key) {
176191	176191	else => func_key,
176192	176192	.ptr => \|ptr\| if (ptr.byte_offset == 0) switch (ptr.base_addr) {

src/link/MachO/Object.zig+24-5

...	...	@@ -328,7 +328,9 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void {
328	328	if (isPtrLiteral(sect)) continue;
329	329
330	330	const nlist_start = for (nlists, 0..) \|nlist, i\| {
331		if (nlist.nlist.n_sect - 1 == n_sect) break i;
	331	// We must ignore `alt_entry` (N_ALT_ENTRY) symbols here, because that flag indicates
	332	// that a symbol should not split subsections.
	333	if (nlist.nlist.n_sect - 1 == n_sect and !nlist.nlist.n_desc.alt_entry) break i;
332	334	} else nlists.len;
333	335	const nlist_end = for (nlists[nlist_start..], nlist_start..) \|nlist, i\| {
334	336	if (nlist.nlist.n_sect - 1 != n_sect) break i;
...	...	@@ -359,9 +361,24 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void {
359	361	const alias_start = idx;
360	362	const nlist = nlists[alias_start];
361	363
362		while (idx < nlist_end and
363		nlists[idx].nlist.n_value == nlist.nlist.n_value) : (idx += 1)
364		{}
	364	// Skip past any symbols which shouldn't terminate this subsection.
	365	while (true) {
	366	idx += 1;
	367	if (idx == nlist_end) {
	368	// This subsection contains the full remainder of the section.
	369	break;
	370	}
	371	if (nlists[idx].nlist.n_value == nlist.nlist.n_value) {
	372	// Multiple symbols at the same address---don't create zero-length subsections.
	373	continue;
	374	}
	375	if (nlists[idx].nlist.n_desc.alt_entry) {
	376	// N_ALT_ENTRY indicates that this symbol does not split subsections, and is
	377	// instead an "alternate entry point" into an existing subsection.
	378	continue;
	379	}
	380	break;
	381	}
365	382
366	383	const size = if (idx < nlist_end)
367	384	nlists[idx].nlist.n_value - nlist.nlist.n_value
...	...	@@ -385,7 +402,9 @@ fn initSubsections(self: *Object, allocator: Allocator, nlists: anytype) !void {
385	402	});
386	403
387	404	for (alias_start..idx) \|i\| {
388		self.symtab.items(.size)[nlists[i].idx] = size;
	405	if (!nlists[i].nlist.n_desc.alt_entry) {
	406	self.symtab.items(.size)[nlists[i].idx] = size;
	407	}
389	408	}
390	409	}
391	410

src/link/Wasm/Object.zig+35-31

...	...	@@ -969,6 +969,41 @@ pub fn parse(
969	969	func.type_index = func_type.ptr(ss).*;
970	970	}
971	971
	972	// Check for indirect function table in case of an MVP object file.
	973	legacy_indirect_function_table: {
	974	// If there is a symbol for each import table, this is not a legacy object file.
	975	if (ss.table_imports.items.len == table_import_symbol_count) break :legacy_indirect_function_table;
	976	if (table_import_symbol_count != 0) {
	977	return diags.failParse(path, "expected a table entry symbol for each of the {d} table(s), but instead got {d} symbols.", .{
	978	ss.table_imports.items.len, table_import_symbol_count,
	979	});
	980	}
	981	// MVP object files cannot have any table definitions, only imports
	982	// (for the indirect function table).
	983	const tables = wasm.object_tables.items[tables_start..];
	984	if (tables.len > 0) {
	985	return diags.failParse(path, "table definition without representing table symbols", .{});
	986	}
	987	if (ss.table_imports.items.len != 1) {
	988	return diags.failParse(path, "found more than one table import, but no representing table symbols", .{});
	989	}
	990	const table_import_name = ss.table_imports.items[0].name;
	991	if (table_import_name != wasm.preloaded_strings.__indirect_function_table) {
	992	return diags.failParse(path, "non-indirect function table import '{s}' is missing a corresponding symbol", .{
	993	table_import_name.slice(wasm),
	994	});
	995	}
	996
	997	try ss.symbol_table.append(gpa, .{
	998	.flags = .{
	999	.undefined = true,
	1000	.no_strip = true,
	1001	},
	1002	.name = table_import_name.toOptional(),
	1003	.pointee = .{ .table_import = @enumFromInt(0) },
	1004	});
	1005	}
	1006
972	1007	// Apply symbol table information.
973	1008	for (ss.symbol_table.items) \|symbol\| switch (symbol.pointee) {
974	1009	.function_import => \|index\| {
...	...	@@ -1331,37 +1366,6 @@ pub fn parse(
1331	1366	};
1332	1367	}
1333	1368
1334		// Check for indirect function table in case of an MVP object file.
1335		legacy_indirect_function_table: {
1336		// If there is a symbol for each import table, this is not a legacy object file.
1337		if (ss.table_imports.items.len == table_import_symbol_count) break :legacy_indirect_function_table;
1338		if (table_import_symbol_count != 0) {
1339		return diags.failParse(path, "expected a table entry symbol for each of the {d} table(s), but instead got {d} symbols.", .{
1340		ss.table_imports.items.len, table_import_symbol_count,
1341		});
1342		}
1343		// MVP object files cannot have any table definitions, only imports
1344		// (for the indirect function table).
1345		const tables = wasm.object_tables.items[tables_start..];
1346		if (tables.len > 0) {
1347		return diags.failParse(path, "table definition without representing table symbols", .{});
1348		}
1349		if (ss.table_imports.items.len != 1) {
1350		return diags.failParse(path, "found more than one table import, but no representing table symbols", .{});
1351		}
1352		const table_import_name = ss.table_imports.items[0].name;
1353		if (table_import_name != wasm.preloaded_strings.__indirect_function_table) {
1354		return diags.failParse(path, "non-indirect function table import '{s}' is missing a corresponding symbol", .{
1355		table_import_name.slice(wasm),
1356		});
1357		}
1358		const ptr = wasm.object_table_imports.getPtr(table_import_name).?;
1359		ptr.flags = .{
1360		.undefined = true,
1361		.no_strip = true,
1362		};
1363		}
1364
1365	1369	for (wasm.object_init_funcs.items[init_funcs_start..]) \|init_func\| {
1366	1370	const func = init_func.function_index.ptr(wasm);
1367	1371	const params = func.type_index.ptr(wasm).params.slice(wasm);

src/main.zig-4

...	...	@@ -3528,10 +3528,6 @@ fn buildOutputType(
3528	3528	fatal("--debug-incremental requires -fincremental", .{});
3529	3529	}
3530	3530
3531		if (incremental and create_module.resolved_options.use_llvm) {
3532		warn("-fincremental is currently unsupported by the LLVM backend; crashes or miscompilations are likely", .{});
3533		}
3534
3535	3531	const cache_mode: Compilation.CacheMode = b: {
3536	3532	// Once incremental compilation is the default, we'll want some smarter logic here,
3537	3533	// considering things like the backend in use and whether there's a ZCU.

test/incremental/add_decl+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/add_decl_namespaced+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/add_remove_struct_fields+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/add_remove_toplevel_fields+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/analysis_error_and_syntax_error+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/bad_import+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7
7	8	#update=initial version

test/incremental/change_embed_file+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_enum_tag_type+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_exports+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_fn_type+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#update=initial version
6	7	#file=main.zig
7	8	pub fn main() !void {

test/incremental/change_generic_line_number+1

...	...	@@ -1,5 +1,6 @@
1	1	#target=x86_64-linux-selfhosted
2	2	#target=x86_64-windows-selfhosted
	3	#target=x86_64-linux-llvm
3	4	#target=wasm32-wasi-selfhosted
4	5	#update=initial version
5	6	#file=main.zig

test/incremental/change_line_number+1

...	...	@@ -1,5 +1,6 @@
1	1	#target=x86_64-linux-selfhosted
2	2	#target=x86_64-windows-selfhosted
	3	#target=x86_64-linux-llvm
3	4	#target=wasm32-wasi-selfhosted
4	5	#update=initial version
5	6	#file=main.zig

test/incremental/change_module+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#module=foo=foo.zig
7	8

test/incremental/change_panic_handler+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#update=initial version
6	7	#file=main.zig
7	8	pub fn main() !u8 {

test/incremental/change_panic_handler_explicit+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#update=initial version
6	7	#file=main.zig
7	8	pub fn main() !u8 {

test/incremental/change_shift_op+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_struct_same_fields+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_zon_file+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/change_zon_file_no_result_type+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/compile_error_then_log+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version with compile error
7	8	#file=main.zig

test/incremental/compile_log+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7
7	8	#update=initial version with no compile log

test/incremental/delete_comptime_decls+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/dependency_on_type_of_inferred_global+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/fix_astgen_failure+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version with error
7	8	#file=main.zig

test/incremental/function_becomes_inline+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#update=non-inline version
6	7	#file=main.zig
7	8	pub fn main() !void {

test/incremental/hello+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/make_decl_pub+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/modify_inline_fn+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/move_src+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/no_change_preserves_tag_names+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	//#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/recursive_function_becomes_non_recursive+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/remove_enum_field+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/remove_invalid_union_backing_enum+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/temporary_parse_error+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/type_becomes_comptime_only+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/type_dependency_loop+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/incremental/unreferenced_error+1

...	...	@@ -2,6 +2,7 @@
2	2	#target=x86_64-windows-selfhosted
3	3	#target=x86_64-linux-cbe
4	4	#target=x86_64-windows-cbe
	5	#target=x86_64-linux-llvm
5	6	#target=wasm32-wasi-selfhosted
6	7	#update=initial version
7	8	#file=main.zig

test/tests.zig+23-3

...	...	@@ -2413,6 +2413,19 @@ pub fn addModuleTests(b: std.Build, options: ModuleTestOptions) Step {
2413	2413	const would_use_llvm = wouldUseLlvm(test_target.use_llvm, test_target.target, test_target.optimize_mode);
2414	2414	if (options.skip_llvm and would_use_llvm) continue;
2415	2415
	2416	if (would_use_llvm and (mem.eql(u8, options.name, "compiler-rt") or mem.eql(u8, options.name, "zigc"))) {
	2417	switch (test_target.optimize_mode) {
	2418	.Debug, .ReleaseSafe => {
	2419	// LLVM 21 is affected by multiple bugs in safe builds of compiler-rt:
	2420	// * https://codeberg.org/ziglang/zig/issues/31701
	2421	// * https://codeberg.org/ziglang/zig/issues/31702
	2422	// ...so for now, skip these tests.
	2423	continue;
	2424	},
	2425	.ReleaseSmall, .ReleaseFast => {},
	2426	}
	2427	}
	2428
2416	2429	const triple_txt = resolved_target.query.zigTriple(b.allocator) catch @panic("OOM");
2417	2430
2418	2431	if (options.test_target_filters.len > 0) {
...	...	@@ -2487,7 +2500,7 @@ fn addOneModuleTest(
2487	2500	.zig_lib_dir = b.path("lib"),
2488	2501	});
2489	2502	these_tests.linkage = test_target.linkage;
2490		if (options.no_builtin) these_tests.root_module.no_builtin = false;
	2503	if (options.no_builtin) these_tests.root_module.no_builtin = true;
2491	2504	if (options.build_options) \|build_options\| {
2492	2505	these_tests.root_module.addOptions("build_options", build_options);
2493	2506	}
...	...	@@ -2634,12 +2647,19 @@ pub fn wouldUseLlvm(use_llvm: ?bool, query: std.Target.Query, optimize_mode: Opt
2634	2647	}
2635	2648	const cpu_arch = query.cpu_arch orelse builtin.cpu.arch;
2636	2649	const os_tag = query.os_tag orelse builtin.os.tag;
	2650	const ofmt: std.Target.ObjectFormat = query.ofmt orelse .default(os_tag, cpu_arch);
2637	2651	switch (cpu_arch) {
2638		.x86_64 => if (os_tag.isBSD() or os_tag == .illumos or std.Target.ptrBitWidth_arch_abi(cpu_arch, query.abi orelse .none) != 64) return true,
	2652	.x86_64 => {
	2653	if (std.Target.ptrBitWidth_arch_abi(cpu_arch, query.abi orelse .none) != 64) return true;
	2654	if (os_tag.isBSD() or os_tag == .illumos) return true;
	2655	return switch (ofmt) {
	2656	.elf, .macho => return false,
	2657	else => return true,
	2658	};
	2659	},
2639	2660	.spirv32, .spirv64 => return false,
2640	2661	else => return true,
2641	2662	}
2642		return false;
2643	2663	}
2644	2664
2645	2665	const CAbiTestOptions = struct {