Meg's Mirrors

zig

1	const std = @import("std");
2	const Io = std.Io;
3	const assert = std.debug.assert;
4	const Allocator = std.mem.Allocator;
5	const DW = std.dwarf;
6	const Builder = std.zig.llvm.Builder;
7	const builtin = @import("builtin");
8	const build_options = @import("build_options");
9
10	const Air = @import("../Air.zig");
11	const codegen = @import("../codegen.zig");
12	const Compilation = @import("../Compilation.zig");
13	const dev = @import("../dev.zig");
14	const InternPool = @import("../InternPool.zig");
15	const link = @import("../link.zig");
16	const Module = @import("../Module.zig");
17	const target_util = @import("../target.zig");
18	const Type = @import("../Type.zig");
19	const Value = @import("../Value.zig");
20	const Zcu = @import("../Zcu.zig");
21	const aarch64_c_abi = @import("aarch64/abi.zig");
22	const FuncGen = @import("llvm/FuncGen.zig");
23	const isByRef = FuncGen.isByRef;
24	const fnReturnStrat = FuncGen.fnReturnStrat;
25	const iterateParamTypes = FuncGen.iterateParamTypes;
26	const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
27
28	const log = std.log.scoped(.codegen);
29	const bindings = if (build_options.have_llvm)
30	@import("llvm/bindings.zig")
31	else
32	@compileError("LLVM unavailable");
33
34	pub fn legalizeFeatures(target: *const std.Target) ?*const Air.Legalize.Features {
35	return switch (target.cpu.arch.endian()) {
36	inline else => |endian| comptime &.init(.{
37	.expand_int_from_float_safe = true,
38	.expand_int_from_float_optimized_safe = true,
39
40	.scalarize_bit_cast_array = true,
41	// LLVM's `bitcast` on vectors places element 0 in the least significant bits on
42	// little-endian targets, which matches our semantics; but it does the opposite on
43	// big-endian targets, so in that case we need to scalarize.
44	.scalarize_bit_cast_vector_non_elementwise = endian != .little,
45	}),
46	};
47	}
48
49	pub fn supportsTailCall(target: *const std.Target) bool {
50	return switch (target.cpu.arch) {
51	.wasm32, .wasm64 => target.cpu.has(.wasm, .tail_call),
52	// Although these ISAs support tail calls, LLVM does not support tail calls on them.
53	.mips, .mipsel, .mips64, .mips64el => false,
54	.powerpc, .powerpcle, .powerpc64, .powerpc64le => false,
55	else => true,
56	};
57	}
58
59	// Avoid depending on `bindings.CodeModel` in the bitcode-only case.
60	const CodeModel = enum {
61	default,
62	tiny,
63	small,
64	kernel,
65	medium,
66	large,
67	};
68
69	fn codeModel(model: std.lang.CodeModel, target: *const std.Target) CodeModel {
70	// Roughly match Clang's mapping of GCC code models to LLVM code models.
71	return switch (model) {
72	.default => .default,
73	.extreme, .large => .large,
74	.kernel => .kernel,
75	.medany => if (target.cpu.arch.isRISCV()) .medium else .large,
76	.medium => .medium,
77	.medmid => .medium,
78	.normal, .medlow, .small => .small,
79	.tiny => .tiny,
80	};
81	}
82
83	pub const Object = struct {
84	gpa: Allocator,
85	builder: Builder,
86
87	/// The basename of the object file which will emitted by LLVM for the ZCU. Once it it emitted,
88	/// this object file is passed to the active linker implementation as an ordinary link input.
89	///
90	/// For the full path, use `Compilation.resolveEmitPath` with `kind == .temp`.
91	out_bin_basename: []const u8,
92
93	/// This pool contains only types (and not `@as(type, undefined)`). It has two purposes:
94	///
95	/// * Lazily tracking ABI alignment of types, so that `@"align"` attributes can be set to a
96	/// type's ABI alignment before that type is fully resolved. Each type in the pool has a
97	/// corresponding entry in `lazy_abi_aligns`.
98	///
99	/// * If `!Object.builder.strip`, lazily tracking debug information types, so that debug
100	/// information can handle indirect self-reference (and so that debug information works
101	/// correctly across incremental updates). Each type has a corresponding entry in
102	/// `debug_types`, provided that `Object.builder.strip` is `false`.
103	type_pool: link.ConstPool,
104
105	/// Keyed on `link.ConstPool.Index`.
106	lazy_abi_aligns: std.ArrayList(Builder.Alignment.Lazy),
107
108	debug_compile_unit: Builder.Metadata.Optional,
109
110	debug_enums_fwd_ref: Builder.Metadata.Optional,
111	debug_globals_fwd_ref: Builder.Metadata.Optional,
112
113	debug_enums: std.ArrayList(Builder.Metadata),
114	debug_globals: std.ArrayList(Builder.Metadata),
115
116	debug_file_map: std.AutoHashMapUnmanaged(Zcu.File.Index, Builder.Metadata),
117
118	/// Keyed on `link.ConstPool.Index`.
119	debug_types: std.ArrayList(Builder.Metadata),
120	/// Initially `.none`, set if the type `anyerror` is lowered to a debug type. The type will not
121	/// actually be created until `emit`, which must resolve this reference with an appropriate enum
122	/// type from the global error set.
123	debug_anyerror_fwd_ref: Builder.Metadata.Optional,
124
125	zcu: *Zcu,
126	/// Maps a `Nav` to the corresponding LLVM global.
127	nav_map: std.AutoHashMapUnmanaged(InternPool.Nav.Index, Builder.Global.Index),
128	/// Same as `nav_map` but for UAVs (which are always global constants).
129	uav_map: std.AutoHashMapUnmanaged(struct {
130	val: InternPool.Index,
131	@"addrspace": std.lang.AddressSpace,
132	}, Builder.Variable.Index),
133	/// Same as `uav_map` but for llvm values not originating from the frontend.
134	const_map: std.AutoHashMapUnmanaged(Builder.Constant, Builder.Variable.Index),
135	/// Maps enum types to their corresponding LLVM functions for implementing the `tag_name` instruction.
136	enum_tag_name_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index),
137	/// Serves the same purpose as `enum_tag_name_map` but for the `is_named_enum_value` instruction.
138	named_enum_map: std.AutoHashMapUnmanaged(InternPool.Index, Builder.Function.Index),
139	/// Maps Zig types to LLVM types. The table memory is backed by the GPA of
140	/// the compiler.
141	/// TODO when InternPool garbage collection is implemented, this map needs
142	/// to be garbage collected as well.
143	type_map: TypeMap,
144	/// The LLVM global table which holds the names corresponding to Zig errors.
145	/// Note that the values are not added until `emit`, when all errors in
146	/// the compilation are known.
147	error_name_table: Builder.Variable.Index,
148	/// Constant variable whose value is the number of errors in the Zcu.
149	///
150	/// Initially `.none`---populated lazily by `getErrorsLen`.
151	///
152	/// If this is not `.none`, the variable's initializer is set in `emit`.
153	errors_len_variable: Builder.Variable.Index,
154
155	/// Values for `@llvm.used`.
156	used: std.ArrayList(Builder.Constant),
157
158	pub const Ptr = if (dev.env.supports(.llvm_backend)) *Object else noreturn;
159
160	const TypeMap = std.AutoHashMapUnmanaged(InternPool.Index, Builder.Type);
161
162	pub fn create(arena: Allocator, zcu: *Zcu) !Ptr {
163	dev.check(.llvm_backend);
164	const comp = zcu.comp;
165	const gpa = comp.gpa;
166	const target = zcu.getTarget();
167
168	var builder = try Builder.init(.{
169	.allocator = gpa,
170	.strip = comp.config.debug_format == .strip,
171	.name = comp.root_name,
172	.target = target,
173	});
174	errdefer builder.deinit();
175
176	const debug_compile_unit, const debug_enums_fwd_ref, const debug_globals_fwd_ref =
177	if (!builder.strip) debug_info: {
178	// We fully resolve all paths at this point to avoid lack of
179	// source line info in stack traces or lack of debugging
180	// information which, if relative paths were used, would be
181	// very location dependent.
182	// TODO: the only concern I have with this is WASI as either host or target, should
183	// we leave the paths as relative then?
184	// TODO: This is totally wrong. In dwarf, paths are encoded as relative to
185	// a particular directory, and then the directory path is specified elsewhere.
186	// In the compiler frontend we have it stored correctly in this
187	// way already, but here we throw all that sweet information
188	// into the garbage can by converting into absolute paths. What
189	// a terrible tragedy.
190	const compile_unit_dir = try zcu.main_mod.root.toAbsolute(&comp.dirs, arena);
191
192	const debug_file = try builder.debugFile(
193	try builder.metadataString(comp.root_name),
194	try builder.metadataString(compile_unit_dir),
195	);
196
197	const debug_enums_fwd_ref = try builder.debugForwardReference();
198	const debug_globals_fwd_ref = try builder.debugForwardReference();
199
200	const debug_compile_unit = try builder.debugCompileUnit(
201	debug_file,
202	// Don't use the version string here; LLVM misparses it when it
203	// includes the git revision.
204	try builder.metadataStringFmt("zig {d}.{d}.{d}", .{
205	build_options.semver.major,
206	build_options.semver.minor,
207	build_options.semver.patch,
208	}),
209	debug_enums_fwd_ref,
210	debug_globals_fwd_ref,
211	.{ .optimized = comp.root_mod.optimize_mode != .debug },
212	);
213
214	try builder.addNamedMetadata(try builder.string("llvm.dbg.cu"), &.{debug_compile_unit});
215	break :debug_info .{
216	debug_compile_unit.toOptional(),
217	debug_enums_fwd_ref.toOptional(),
218	debug_globals_fwd_ref.toOptional(),
219	};
220	} else .{
221	Builder.Metadata.Optional.none,
222	Builder.Metadata.Optional.none,
223	Builder.Metadata.Optional.none,
224	};
225
226	const obj = try arena.create(Object);
227	obj.* = .{
228	.gpa = gpa,
229	.builder = builder,
230	.out_bin_basename = try std.zig.binNameAlloc(arena, .{
231	.root_name = try std.fmt.allocPrint(arena, "{s}_zcu", .{comp.root_name}),
232	.cpu_arch = target.cpu.arch,
233	.os_tag = target.os.tag,
234	.ofmt = target.ofmt,
235	.abi = target.abi,
236	.output_mode = .Obj,
237	}),
238	.type_pool = .empty,
239	.lazy_abi_aligns = .empty,
240	.debug_compile_unit = debug_compile_unit,
241	.debug_enums_fwd_ref = debug_enums_fwd_ref,
242	.debug_globals_fwd_ref = debug_globals_fwd_ref,
243	.debug_enums = .empty,
244	.debug_globals = .empty,
245	.debug_file_map = .empty,
246	.debug_types = .empty,
247	.debug_anyerror_fwd_ref = .none,
248	.zcu = zcu,
249	.nav_map = .empty,
250	.uav_map = .empty,
251	.const_map = .empty,
252	.enum_tag_name_map = .empty,
253	.named_enum_map = .empty,
254	.type_map = .empty,
255	.error_name_table = .none,
256	.errors_len_variable = .none,
257	.used = .empty,
258	};
259	return obj;
260	}
261
262	pub fn deinit(o: *Object) void {
263	const gpa = o.gpa;
264	o.type_pool.deinit(gpa);
265	o.lazy_abi_aligns.deinit(gpa);
266	o.debug_enums.deinit(gpa);
267	o.debug_globals.deinit(gpa);
268	o.debug_file_map.deinit(gpa);
269	o.debug_types.deinit(gpa);
270	o.nav_map.deinit(gpa);
271	o.uav_map.deinit(gpa);
272	o.const_map.deinit(gpa);
273	o.enum_tag_name_map.deinit(gpa);
274	o.named_enum_map.deinit(gpa);
275	o.type_map.deinit(gpa);
276	o.builder.deinit();
277	o.* = undefined;
278	}
279
280	fn genErrorNameTable(o: *Object) Allocator.Error!void {
281	// If o.error_name_table is null, then it was not referenced by any instructions.
282	if (o.error_name_table == .none) return;
283
284	const zcu = o.zcu;
285	const ip = &zcu.intern_pool;
286
287	const error_name_list = ip.global_error_set.getNamesFromMainThread();
288	const llvm_errors = try zcu.gpa.alloc(Builder.Constant, 1 + error_name_list.len);
289	defer zcu.gpa.free(llvm_errors);
290
291	// TODO: Address space
292	const slice_ty = Type.slice_const_u8_sentinel_0;
293	const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
294	const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
295	const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
296
297	llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
298	for (llvm_errors[1..], error_name_list) |*llvm_error, name| {
299	const name_string = try o.builder.stringNull(name.toSlice(ip));
300	const name_init = try o.builder.stringConst(name_string);
301	const name_llvm_variable = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
302	try name_llvm_variable.setInitializer(name_init, &o.builder);
303	name_llvm_variable.setMutability(.constant, &o.builder);
304	name_llvm_variable.setAlignment(comptime .fromByteUnits(1), &o.builder);
305	const llvm_global = name_llvm_variable.ptrConst(&o.builder).global;
306	llvm_global.setLinkage(.private, &o.builder);
307	llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
308
309	llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{
310	name_llvm_variable.toConst(&o.builder),
311	try o.builder.intConst(llvm_usize_ty, name_string.slice(&o.builder).?.len - 1),
312	});
313	}
314
315	try o.error_name_table.setInitializer(
316	try o.builder.arrayConst(llvm_table_ty, llvm_errors),
317	&o.builder,
318	);
319	}
320
321	fn genModuleLevelAssembly(object: *Object) Allocator.Error!void {
322	const b = &object.builder;
323	const gpa = b.gpa;
324	b.module_asm.clearRetainingCapacity();
325	for (object.zcu.global_assembly.values()) |assembly| {
326	try b.module_asm.ensureUnusedCapacity(gpa, assembly.len + 1);
327	b.module_asm.appendSliceAssumeCapacity(assembly);
328	b.module_asm.appendAssumeCapacity('\n');
329	}
330	if (b.module_asm.last()) |last| {
331	if (last != '\n') try b.module_asm.append(gpa, '\n');
332	}
333	}
334
335	pub const EmitOptions = struct {
336	pre_ir_path: ?[]const u8,
337	pre_bc_path: ?[]const u8,
338	bin_path: ?[:0]const u8,
339	asm_path: ?[:0]const u8,
340	post_ir_path: ?[:0]const u8,
341	post_bc_path: ?[]const u8,
342
343	is_debug: bool,
344	is_small: bool,
345	time_report: ?*Compilation.TimeReport,
346	sanitize_thread: bool,
347	fuzz: bool,
348	lto: std.zig.LtoMode,
349	};
350
351	pub fn emit(o: *Object, pt: Zcu.PerThread, options: EmitOptions) error{ AlreadyReported, OutOfMemory }!void {
352	const zcu = o.zcu;
353	const comp = zcu.comp;
354	const io = comp.io;
355	const diags = &comp.link_diags;
356
357	{
358	if (o.errors_len_variable != .none) {
359	const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
360	const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
361	try o.errors_len_variable.setInitializer(init_val, &o.builder);
362	}
363	try o.genErrorNameTable();
364	try o.genModuleLevelAssembly();
365
366	if (o.used.items.len > 0) {
367	const array_llvm_ty = try o.builder.arrayType(o.used.items.len, .ptr);
368	const init_val = try o.builder.arrayConst(array_llvm_ty, o.used.items);
369	const compiler_used_variable = try o.builder.addVariable(
370	try o.builder.strtabString("llvm.used"),
371	array_llvm_ty,
372	.default,
373	);
374	try compiler_used_variable.setInitializer(init_val, &o.builder);
375	compiler_used_variable.setSection(try o.builder.string("llvm.metadata"), &o.builder);
376	compiler_used_variable.ptrConst(&o.builder).global.setLinkage(.appending, &o.builder);
377	}
378
379	if (!o.builder.strip) {
380	if (o.debug_anyerror_fwd_ref.unwrap()) |fwd_ref| {
381	const debug_anyerror_type = try o.lowerDebugAnyerrorType();
382	o.builder.resolveDebugForwardReference(fwd_ref, debug_anyerror_type);
383	}
384
385	try o.flushTypePool(pt);
386
387	o.builder.resolveDebugForwardReference(
388	o.debug_enums_fwd_ref.unwrap().?,
389	try o.builder.metadataTuple(o.debug_enums.items),
390	);
391
392	o.builder.resolveDebugForwardReference(
393	o.debug_globals_fwd_ref.unwrap().?,
394	try o.builder.metadataTuple(o.debug_globals.items),
395	);
396	}
397	}
398
399	{
400	var module_flags = try std.array_list.Managed(Builder.Metadata).initCapacity(o.gpa, 8);
401	defer module_flags.deinit();
402
403	const behavior_error = try o.builder.metadataConstant(try o.builder.intConst(.i32, 1));
404	const behavior_warning = try o.builder.metadataConstant(try o.builder.intConst(.i32, 2));
405	const behavior_max = try o.builder.metadataConstant(try o.builder.intConst(.i32, 7));
406	const behavior_min = try o.builder.metadataConstant(try o.builder.intConst(.i32, 8));
407
408	if (target_util.llvmMachineAbi(&comp.root_mod.resolved_target.result)) |abi| {
409	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
410	behavior_error,
411	(try o.builder.metadataString("target-abi")).toMetadata(),
412	(try o.builder.metadataString(abi)).toMetadata(),
413	}));
414	}
415
416	const pic_level = target_util.picLevel(&comp.root_mod.resolved_target.result);
417	if (comp.root_mod.pic) {
418	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
419	behavior_min,
420	(try o.builder.metadataString("PIC Level")).toMetadata(),
421	try o.builder.metadataConstant(try o.builder.intConst(.i32, pic_level)),
422	}));
423	}
424
425	if (comp.config.pie) {
426	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
427	behavior_max,
428	(try o.builder.metadataString("PIE Level")).toMetadata(),
429	try o.builder.metadataConstant(try o.builder.intConst(.i32, pic_level)),
430	}));
431	}
432
433	if (comp.root_mod.code_model != .default) {
434	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
435	behavior_error,
436	(try o.builder.metadataString("Code Model")).toMetadata(),
437	try o.builder.metadataConstant(try o.builder.intConst(.i32, @as(
438	i32,
439	switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
440	.default => unreachable,
441	.tiny => 0,
442	.small => 1,
443	.kernel => 2,
444	.medium => 3,
445	.large => 4,
446	},
447	))),
448	}));
449	}
450
451	if (!o.builder.strip) {
452	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
453	behavior_warning,
454	(try o.builder.metadataString("Debug Info Version")).toMetadata(),
455	try o.builder.metadataConstant(try o.builder.intConst(.i32, 3)),
456	}));
457
458	switch (comp.config.debug_format) {
459	.strip => unreachable,
460	.dwarf => |f| {
461	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
462	behavior_max,
463	(try o.builder.metadataString("Dwarf Version")).toMetadata(),
464	try o.builder.metadataConstant(try o.builder.intConst(.i32, 4)),
465	}));
466
467	if (f == .@"64") {
468	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
469	behavior_max,
470	(try o.builder.metadataString("DWARF64")).toMetadata(),
471	try o.builder.metadataConstant(.@"1"),
472	}));
473	}
474	},
475	.code_view => {
476	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
477	behavior_warning,
478	(try o.builder.metadataString("CodeView")).toMetadata(),
479	try o.builder.metadataConstant(.@"1"),
480	}));
481	},
482	}
483	}
484
485	const target = &comp.root_mod.resolved_target.result;
486	if (target.os.tag == .windows and (target.cpu.arch == .x86_64 or target.cpu.arch == .x86)) {
487	// Add the "RegCallv4" flag so that any functions using `x86_regcallcc` use regcall
488	// v4, which is essentially a requirement on Windows. See corresponding logic in
489	// `toLlvmCallConvTag`.
490	module_flags.appendAssumeCapacity(try o.builder.metadataTuple(&.{
491	behavior_max,
492	(try o.builder.metadataString("RegCallv4")).toMetadata(),
493	try o.builder.metadataConstant(.@"1"),
494	}));
495	}
496
497	try o.builder.addNamedMetadata(try o.builder.string("llvm.module.flags"), module_flags.items);
498	}
499
500	const target_triple_sentinel =
501	try o.gpa.dupeSentinel(u8, o.builder.target_triple.slice(&o.builder).?, 0);
502	defer o.gpa.free(target_triple_sentinel);
503
504	const emit_asm_msg = options.asm_path orelse "(none)";
505	const emit_bin_msg = options.bin_path orelse "(none)";
506	const post_llvm_ir_msg = options.post_ir_path orelse "(none)";
507	const post_llvm_bc_msg = options.post_bc_path orelse "(none)";
508	log.debug("emit LLVM object asm={s} bin={s} ir={s} bc={s}", .{
509	emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg,
510	});
511
512	const context, const module = emit: {
513	if (options.pre_ir_path) |path| {
514	if (std.mem.eql(u8, path, "-")) {
515	o.builder.dump(io);
516	} else {
517	o.builder.printToFilePath(io, Io.Dir.cwd(), path) catch |err| {
518	log.err("failed printing LLVM module to \"{s}\": {t}", .{ path, err });
519	};
520	}
521	}
522
523	const bitcode = try o.builder.toBitcode(o.gpa, .{
524	.name = "zig",
525	.version = build_options.semver,
526	});
527	defer o.gpa.free(bitcode);
528
529	if (options.pre_bc_path) |path| {
530	var file = Io.Dir.cwd().createFile(io, path, .{}) catch |err|
531	return diags.fail("failed to create '{s}': {t}", .{ path, err });
532	defer file.close(io);
533
534	const ptr: [*]const u8 = @ptrCast(bitcode.ptr);
535	file.writeStreamingAll(io, ptr[0..(bitcode.len * 4)]) catch |err|
536	return diags.fail("failed to write to '{s}': {t}", .{ path, err });
537	}
538
539	if (options.asm_path == null and options.bin_path == null and
540	options.post_ir_path == null and options.post_bc_path == null) return;
541
542	if (options.post_bc_path) |path| {
543	var file = Io.Dir.cwd().createFile(io, path, .{}) catch |err|
544	return diags.fail("failed to create '{s}': {t}", .{ path, err });
545	defer file.close(io);
546
547	const ptr: [*]const u8 = @ptrCast(bitcode.ptr);
548	file.writeStreamingAll(io, ptr[0..(bitcode.len * 4)]) catch |err|
549	return diags.fail("failed to write to '{s}': {t}", .{ path, err });
550	}
551
552	if (!build_options.have_llvm or !comp.config.use_lib_llvm) {
553	return diags.fail("emitting without libllvm not implemented", .{});
554	}
555
556	initializeLLVMTarget(io, comp.root_mod.resolved_target.result.cpu.arch);
557
558	const context: *bindings.Context = .create();
559	errdefer context.dispose();
560
561	const bitcode_memory_buffer = bindings.MemoryBuffer.createMemoryBufferWithMemoryRange(
562	@ptrCast(bitcode.ptr),
563	bitcode.len * 4,
564	"BitcodeBuffer",
565	bindings.Bool.False,
566	);
567	defer bitcode_memory_buffer.dispose();
568
569	context.enableBrokenDebugInfoCheck();
570
571	var module: *bindings.Module = undefined;
572	if (context.parseBitcodeInContext2(bitcode_memory_buffer, &module).toBool() or context.getBrokenDebugInfo()) {
573	return diags.fail("Failed to parse bitcode", .{});
574	}
575	break :emit .{ context, module };
576	};
577	defer context.dispose();
578
579	var target: *bindings.Target = undefined;
580	var error_message: [*:0]const u8 = undefined;
581	if (bindings.Target.getFromTriple(target_triple_sentinel, &target, &error_message).toBool()) {
582	defer bindings.disposeMessage(error_message);
583	return diags.fail("LLVM failed to parse '{s}': {s}", .{ target_triple_sentinel, error_message });
584	}
585
586	const optimize_mode = comp.root_mod.optimize_mode;
587
588	const opt_level: bindings.CodeGenOptLevel = if (optimize_mode == .debug)
589	.None
590	else
591	.Aggressive;
592
593	const reloc_mode: bindings.RelocMode = if (comp.root_mod.pic)
594	.PIC
595	else if (comp.config.link_mode == .dynamic)
596	bindings.RelocMode.DynamicNoPIC
597	else
598	.Static;
599
600	const code_model: bindings.CodeModel = switch (codeModel(comp.root_mod.code_model, &comp.root_mod.resolved_target.result)) {
601	.default => .Default,
602	.tiny => .Tiny,
603	.small => .Small,
604	.kernel => .Kernel,
605	.medium => .Medium,
606	.large => .Large,
607	};
608
609	const float_abi: bindings.TargetMachine.FloatABI = if (comp.root_mod.resolved_target.result.abi.float() == .hard)
610	.Hard
611	else
612	.Soft;
613
614	var target_machine = bindings.TargetMachine.create(
615	target,
616	target_triple_sentinel,
617	if (comp.root_mod.resolved_target.result.cpu.model.llvm_name) |s| s.ptr else null,
618	comp.root_mod.resolved_target.llvm_cpu_features.?,
619	opt_level,
620	reloc_mode,
621	code_model,
622	comp.function_sections,
623	comp.data_sections,
624	float_abi,
625	if (target_util.llvmMachineAbi(&comp.root_mod.resolved_target.result)) |s| s.ptr else null,
626	target_util.useEmulatedTls(&comp.root_mod.resolved_target.result),
627	);
628	errdefer target_machine.dispose();
629
630	if (comp.llvm_opt_bisect_limit >= 0) {
631	context.setOptBisectLimit(comp.llvm_opt_bisect_limit);
632	}
633
634	// Unfortunately, LLVM shits the bed when we ask for both binary and assembly.
635	// So we call the entire pipeline multiple times if this is requested.
636	// var error_message: [*:0]const u8 = undefined;
637	var lowered_options: bindings.TargetMachine.EmitOptions = .{
638	.is_debug = options.is_debug,
639	.is_small = options.is_small,
640	.time_report_out = null, // set below to make sure it's only set for a single `emitToFile`
641	.tsan = options.sanitize_thread,
642	.lto = switch (options.lto) {
643	.none => .None,
644	.thin => .ThinPreLink,
645	.full => .FullPreLink,
646	},
647	.allow_fast_isel = true,
648	// LLVM's RISC-V backend for some reason enables the machine outliner by default even
649	// though it's clearly not ready and produces multiple miscompilations in our std tests.
650	.allow_machine_outliner = !comp.root_mod.resolved_target.result.cpu.arch.isRISCV(),
651	.asm_filename = null,
652	.bin_filename = if (options.bin_path) |x| x.ptr else null,
653	.llvm_ir_filename = if (options.post_ir_path) |x| x.ptr else null,
654	.bitcode_filename = null,
655
656	// `.coverage` value is only used when `.sancov` is enabled.
657	.sancov = options.fuzz or comp.config.san_cov_trace_pc_guard,
658	.coverage = .{
659	.CoverageType = .Edge,
660	// Works in tandem with Inline8bitCounters or InlineBoolFlag.
661	// Zig does not yet implement its own version of this but it
662	// needs to for better fuzzing logic.
663	.IndirectCalls = false,
664	.TraceBB = false,
665	.TraceCmp = false,
666	.TraceDiv = false,
667	.TraceGep = false,
668	.Use8bitCounters = false,
669	.TracePC = false,
670	.TracePCGuard = comp.config.san_cov_trace_pc_guard,
671	// Zig emits its own inline 8-bit counters instrumentation.
672	.Inline8bitCounters = false,
673	.InlineBoolFlag = false,
674	// Zig emits its own PC table instrumentation.
675	.PCTable = false,
676	.NoPrune = false,
677	// Workaround for https://github.com/llvm/llvm-project/pull/106464
678	.StackDepth = true,
679	.TraceLoads = false,
680	.TraceStores = false,
681	.CollectControlFlow = false,
682	},
683	};
684	if (options.asm_path != null and options.bin_path != null) {
685	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
686	defer bindings.disposeMessage(error_message);
687	return diags.fail("LLVM failed to emit bin={s} ir={s}: {s}", .{
688	emit_bin_msg, post_llvm_ir_msg, error_message,
689	});
690	}
691	lowered_options.bin_filename = null;
692	lowered_options.llvm_ir_filename = null;
693	}
694
695	var time_report_c_str: [*:0]u8 = undefined;
696	if (options.time_report != null) {
697	lowered_options.time_report_out = &time_report_c_str;
698	}
699
700	lowered_options.asm_filename = if (options.asm_path) |x| x.ptr else null;
701	if (target_machine.emitToFile(module, &error_message, &lowered_options)) {
702	defer bindings.disposeMessage(error_message);
703	return diags.fail("LLVM failed to emit asm={s} bin={s} ir={s} bc={s}: {s}", .{
704	emit_asm_msg, emit_bin_msg, post_llvm_ir_msg, post_llvm_bc_msg, error_message,
705	});
706	}
707	if (options.time_report) |tr| {
708	defer std.c.free(time_report_c_str);
709	const time_report_data = std.mem.span(time_report_c_str);
710	assert(tr.llvm_pass_timings.len == 0);
711	tr.llvm_pass_timings = try comp.gpa.dupe(u8, time_report_data);
712	}
713	}
714
715	pub fn updateFunc(
716	o: *Object,
717	pt: Zcu.PerThread,
718	func_index: InternPool.Index,
719	air: *const Air,
720	liveness: *const ?Air.Liveness,
721	) Zcu.CodegenFailError!void {
722	const zcu = o.zcu;
723	const comp = zcu.comp;
724	const gpa = comp.gpa;
725	const ip = &zcu.intern_pool;
726	const func = zcu.funcInfo(func_index);
727	const nav = ip.getNav(func.owner_nav);
728	const file_scope = zcu.navFileScopeIndex(func.owner_nav);
729	const owner_mod = zcu.fileByIndex(file_scope).mod.?;
730	const fn_ty = Type.fromInterned(func.ty);
731	const fn_info = zcu.typeToFunc(fn_ty).?;
732	const target = &owner_mod.resolved_target.result;
733
734	const gop = try o.nav_map.getOrPut(gpa, func.owner_nav);
735	if (!gop.found_existing) {
736	errdefer assert(o.nav_map.remove(func.owner_nav));
737	// First time lowering this NAV! Create a fresh global.
738	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
739	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
740	.type = .void, // placeholder; populated below
741	.kind = .{ .alias = .none }, // placeholder; populated below
742	});
743	}
744	const llvm_global = gop.value_ptr.*;
745
746	const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
747	.function => |function| function, // re-use existing `Builder.Function`
748	.replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder),
749	};
750	{
751	const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
752	global.type = try o.lowerType(fn_ty, .in_memory);
753	global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
754	global.linkage = if (o.builder.strip) .private else .internal;
755	global.visibility = .default;
756	global.dll_storage_class = .default;
757	global.unnamed_addr = .unnamed_addr;
758	}
759	llvm_function.setAlignment(nav.resolved.?.@"align".toLlvm(), &o.builder);
760	llvm_function.setSection(s: {
761	const section = nav.resolved.?.@"linksection".toSlice(ip) orelse break :s .none;
762	break :s try o.builder.string(section);
763	}, &o.builder);
764
765	var attributes: Builder.FunctionAttributes.Wip = .{};
766	defer attributes.deinit(&o.builder);
767
768	// Function attributes that are independent of analysis results of the function body.
769	try o.addCommonFnAttributes(
770	&attributes,
771	owner_mod,
772	// Some backends don't respect the `naked` attribute in `TargetFrameLowering::hasFP()`,
773	// so for these backends, LLVM will happily emit code that accesses the stack through
774	// the frame pointer. This is nonsensical since what the `naked` attribute does is
775	// suppress generation of the prologue and epilogue, and the prologue is where the
776	// frame pointer normally gets set up. At time of writing, this is the case for at
777	// least x86 and RISC-V.
778	owner_mod.omit_frame_pointer or fn_info.cc == .naked,
779	);
780
781	try o.addCallingConventionFnAttributes(pt, llvm_function, &attributes, if (nav.getExtern(ip)) |@"extern"| .{
782	.name = nav.name.toSlice(ip),
783	.lib_name = @"extern".lib_name.toSlice(ip),
784	} else null, .fromIntern(fn_info, ip));
785
786	const func_analysis = func.analysisUnordered(ip);
787	if (func_analysis.is_noinline) {
788	try attributes.addFnAttr(.@"noinline", &o.builder);
789	} else {
790	_ = try attributes.removeFnAttr(.@"noinline");
791	}
792
793	if (func_analysis.branch_hint == .cold) {
794	try attributes.addFnAttr(.cold, &o.builder);
795	} else {
796	_ = try attributes.removeFnAttr(.cold);
797	}
798
799	if (owner_mod.sanitize_thread and !func_analysis.disable_instrumentation) {
800	try attributes.addFnAttr(.sanitize_thread, &o.builder);
801	} else {
802	_ = try attributes.removeFnAttr(.sanitize_thread);
803	}
804	const is_naked = fn_info.cc == .naked;
805	if (!func_analysis.disable_instrumentation and !is_naked) {
806	if (owner_mod.fuzz) {
807	try attributes.addFnAttr(.optforfuzzing, &o.builder);
808	}
809	_ = try attributes.removeFnAttr(.skipprofile);
810	_ = try attributes.removeFnAttr(.nosanitize_coverage);
811	} else {
812	_ = try attributes.removeFnAttr(.optforfuzzing);
813	try attributes.addFnAttr(.skipprofile, &o.builder);
814	try attributes.addFnAttr(.nosanitize_coverage, &o.builder);
815	}
816
817	const disable_intrinsics = func_analysis.disable_intrinsics or owner_mod.no_builtin;
818	if (disable_intrinsics) {
819	// The intent here is for compiler-rt and libc functions to not generate
820	// infinite recursion. For example, if we are compiling the memcpy function,
821	// and llvm detects that the body is equivalent to memcpy, it may replace the
822	// body of memcpy with a call to memcpy, which would then cause a stack
823	// overflow instead of performing memcpy.
824	try attributes.addFnAttr(.{ .string = .{
825	.kind = try o.builder.string("no-builtins"),
826	.value = .empty,
827	} }, &o.builder);
828	}
829
830	// TODO: disable this if safety is off for the function scope
831	const ssp_buf_size = owner_mod.stack_protector;
832	if (ssp_buf_size != 0) {
833	try attributes.addFnAttr(.sspstrong, &o.builder);
834	try attributes.addFnAttr(.{ .string = .{
835	.kind = try o.builder.string("stack-protector-buffer-size"),
836	.value = try o.builder.fmt("{d}", .{ssp_buf_size}),
837	} }, &o.builder);
838	}
839
840	// TODO: disable this if safety is off for the function scope
841	if (owner_mod.stack_check) {
842	try attributes.addFnAttr(.{ .string = .{
843	.kind = try o.builder.string("probe-stack"),
844	.value = try o.builder.string("__zig_probe_stack"),
845	} }, &o.builder);
846	} else if (target.os.tag == .uefi) {
847	try attributes.addFnAttr(.{ .string = .{
848	.kind = try o.builder.string("no-stack-arg-probe"),
849	.value = .empty,
850	} }, &o.builder);
851	}
852
853	const file, const subprogram = if (!owner_mod.strip) debug_info: {
854	const file = try o.getDebugFile(file_scope);
855
856	const line_number = zcu.navSrcLine(func.owner_nav) + 1;
857	const is_internal_linkage = ip.indexToKey(nav.resolved.?.value) != .@"extern";
858	const debug_decl_type = try o.getDebugType(pt, fn_ty);
859
860	const subprogram = try o.builder.debugSubprogram(
861	file,
862	try o.builder.metadataString(nav.name.toSlice(ip)),
863	try o.builder.metadataString(nav.fqn.toSlice(ip)),
864	line_number,
865	line_number + func.lbrace_line,
866	debug_decl_type,
867	.{
868	.di_flags = .{
869	.StaticMember = true,
870	.NoReturn = fn_info.return_type == .noreturn_type,
871	},
872	.sp_flags = .{
873	.Optimized = owner_mod.optimize_mode != .debug,
874	.Definition = true,
875	.LocalToUnit = is_internal_linkage,
876	},
877	},
878	o.debug_compile_unit.unwrap().?,
879	);
880	llvm_function.setSubprogram(subprogram, &o.builder);
881	break :debug_info .{ file, subprogram };
882	} else .{ undefined, undefined };
883
884	const fuzz: ?FuncGen.Fuzz = f: {
885	if (!owner_mod.fuzz) break :f null;
886	if (func_analysis.disable_instrumentation) break :f null;
887	if (is_naked) break :f null;
888	if (comp.config.san_cov_trace_pc_guard) break :f null;
889
890	// The void type used here is a placeholder to be replaced with an
891	// array of the appropriate size after the POI count is known.
892
893	// Due to error "members of llvm.compiler.used must be named", this global needs a name.
894	const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
895	const counters_variable = try o.builder.addVariable(anon_name, .void, .default);
896	try o.used.append(gpa, counters_variable.toConst(&o.builder));
897	counters_variable.ptrConst(&o.builder).global.setLinkage(.private, &o.builder);
898	counters_variable.setAlignment(comptime .fromByteUnits(1), &o.builder);
899
900	if (target.ofmt == .macho) {
901	counters_variable.setSection(try o.builder.string("__DATA,__sancov_cntrs"), &o.builder);
902	} else {
903	counters_variable.setSection(try o.builder.string("__sancov_cntrs"), &o.builder);
904	}
905
906	break :f .{
907	.counters_variable = counters_variable,
908	.pcs = .empty,
909	};
910	};
911
912	var fg: FuncGen = .{
913	.object = o,
914	.nav_index = func.owner_nav,
915	.pt = pt,
916	.gpa = gpa,
917	.air = air.*,
918	.liveness = liveness.*.?,
919	.wip = try .init(&o.builder, .{
920	.function = llvm_function,
921	.strip = owner_mod.strip,
922	}),
923	.is_naked = fn_info.cc == .naked,
924	.fuzz = fuzz,
925	.arg_index = 0,
926	.arg_inline_index = 0,
927	.func_inst_table = .empty,
928	.blocks = .empty,
929	.loops = .empty,
930	.switch_dispatch_info = .empty,
931	.sync_scope = if (owner_mod.single_threaded) .singlethread else .system,
932	.file = file,
933	.scope = subprogram,
934	.inlined_at = .none,
935	.base_line = zcu.navSrcLine(func.owner_nav),
936	.prev_dbg_line = 0,
937	.prev_dbg_column = 0,
938	.disable_intrinsics = disable_intrinsics,
939	.allowzero_access = false,
940
941	.ret_ptr = undefined, // populated by `genMainBody`
942	.err_ret_trace = undefined, // populated by `genMainBody`
943	.args = undefined, // populated by `genMainBody`
944	};
945	defer fg.deinit();
946
947	fg.wip.cursor = .{ .block = try fg.wip.block(0, "Entry") };
948
949	try fg.genMainBody();
950
951	// If we saw any loads or stores involving `allowzero` pointers, we need to mark the whole
952	// function as considering null pointers valid so that LLVM's optimizers don't remove these
953	// operations on the assumption that they're undefined behavior.
954	if (fg.allowzero_access) {
955	try attributes.addFnAttr(.null_pointer_is_valid, &o.builder);
956	} else {
957	_ = try attributes.removeFnAttr(.null_pointer_is_valid);
958	}
959
960	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
961
962	if (fg.fuzz) |*f| {
963	{
964	const array_llvm_ty = try o.builder.arrayType(f.pcs.items.len, .i8);
965	f.counters_variable.ptrConst(&o.builder).global.ptr(&o.builder).type = array_llvm_ty;
966	const zero_init = try o.builder.zeroInitConst(array_llvm_ty);
967	try f.counters_variable.setInitializer(zero_init, &o.builder);
968	}
969
970	const array_llvm_ty = try o.builder.arrayType(f.pcs.items.len, .ptr);
971	const init_val = try o.builder.arrayConst(array_llvm_ty, f.pcs.items);
972	// Due to error "members of llvm.compiler.used must be named", this global needs a name.
973	const anon_name = try o.builder.strtabStringFmt("__sancov_gen_.{d}", .{o.used.items.len});
974	const pcs_variable = try o.builder.addVariable(anon_name, array_llvm_ty, .default);
975	try pcs_variable.setInitializer(init_val, &o.builder);
976	pcs_variable.setMutability(.constant, &o.builder);
977	pcs_variable.setSection(switch (target.ofmt) {
978	.macho => try o.builder.string("__DATA,__sancov_pcs1"),
979	else => try o.builder.string("__sancov_pcs1"),
980	}, &o.builder);
981	pcs_variable.setAlignment(Type.usize.abiAlignment(zcu).toLlvm(), &o.builder);
982	const pcs_global = pcs_variable.ptrConst(&o.builder).global;
983	pcs_global.setLinkage(.private, &o.builder);
984	try o.used.append(gpa, pcs_global.toConst());
985	}
986
987	try fg.wip.finish();
988	try o.flushTypePool(pt);
989	}
990
991	fn workaroundPrivateSymbolBugs(target: *const std.Target, resolved: *const InternPool.Nav.Resolved) bool {
992	// https://codeberg.org/ziglang/zig/issues/31865
993	return target.cpu.arch.isAARCH64() and target.ofmt == .coff and resolved.@"threadlocal";
994	}
995
996	pub fn updateNav(o: *Object, pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) !void {
997	const zcu = o.zcu;
998	const ip = &zcu.intern_pool;
999	const comp = zcu.comp;
1000	const gpa = comp.gpa;
1001
1002	const nav = ip.getNav(nav_id);
1003	const resolved = nav.resolved.?;
1004
1005	const opt_extern: ?InternPool.Key.Extern = switch (ip.indexToKey(resolved.value)) {
1006	.@"extern" => |@"extern"| @"extern",
1007	else => null,
1008	};
1009	const nav_ty: Type = .fromInterned(resolved.type);
1010	const llvm_ty: Builder.Type = if (opt_extern != null) ty: {
1011	// We *must* lower this declaration no matter what. If it has a type we can't actually
1012	// represent (because it doesn't have runtime bits), we instead lower as the zero-size
1013	// type `[0 x i8]`. I don't think the type on an extern declaration actually does much
1014	// anyway.
1015	if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
1016	break :ty try o.builder.arrayType(0, .i8);
1017	} else if (nav_ty.hasRuntimeBits(zcu)) ty: {
1018	break :ty try o.lowerType(nav_ty, .in_memory);
1019	} else {
1020	// This is a non-extern zero-bit `Nav`---we're not interested in it.
1021	// TODO: we might need to rethink this a little under incremental compilation. If a
1022	// declaration becomes zero-bit, we can't just leave its old value there, because it
1023	// might now be ill-formed.
1024	return;
1025	};
1026
1027	const gop = try o.nav_map.getOrPut(gpa, nav_id);
1028	if (!gop.found_existing) {
1029	errdefer assert(o.nav_map.remove(nav_id));
1030	// First time lowering this NAV! Create a fresh global.
1031	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
1032	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
1033	.type = .void, // placeholder; populated below
1034	.kind = .{ .alias = .none }, // placeholder; populated below
1035	});
1036	}
1037	const llvm_global = gop.value_ptr.*;
1038
1039	llvm_global.ptr(&o.builder).type = llvm_ty;
1040	llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(resolved.@"addrspace", zcu.getTarget());
1041
1042	if (opt_extern) |@"extern"| {
1043	const name = name: {
1044	const name_slice = nav.name.toSlice(ip);
1045	if (zcu.getTarget().cpu.arch.isWasm() and nav_ty.zigTypeTag(zcu) == .@"fn") {
1046	if (@"extern".lib_name.toSlice(ip)) |lib_name_slice| {
1047	if (!std.mem.eql(u8, lib_name_slice, "c")) {
1048	break :name try o.builder.strtabStringFmt("{s}|{s}", .{ name_slice, lib_name_slice });
1049	}
1050	}
1051	}
1052	break :name try o.builder.strtabString(name_slice);
1053	};
1054	if (o.builder.getGlobal(name)) |other_global| {
1055	if (other_global != llvm_global) {
1056	// Another global already has this name; just use it in place of this global.
1057	try llvm_global.replace(other_global, &o.builder);
1058	return;
1059	}
1060	}
1061	try llvm_global.rename(name, &o.builder);
1062	llvm_global.ptr(&o.builder).unnamed_addr = .default;
1063	llvm_global.ptr(&o.builder).dll_storage_class = switch (@"extern".is_dll_import) {
1064	true => .dllimport,
1065	false => .default,
1066	};
1067	llvm_global.ptr(&o.builder).linkage = switch (@"extern".linkage) {
1068	.internal => if (o.builder.strip and !workaroundPrivateSymbolBugs(zcu.getTarget(), &resolved)) .private else .internal,
1069	.strong => .external,
1070	.weak => .extern_weak,
1071	.link_once => unreachable,
1072	};
1073	llvm_global.ptr(&o.builder).visibility = .fromSymbolVisibility(@"extern".visibility);
1074	} else {
1075	llvm_global.ptr(&o.builder).linkage = if (o.builder.strip and !workaroundPrivateSymbolBugs(zcu.getTarget(), &resolved)) .private else .internal;
1076	llvm_global.ptr(&o.builder).visibility = .default;
1077	llvm_global.ptr(&o.builder).dll_storage_class = .default;
1078	llvm_global.ptr(&o.builder).unnamed_addr = .unnamed_addr;
1079	}
1080
1081	const llvm_section: Builder.String = if (resolved.@"linksection".toSlice(ip)) |section| s: {
1082	break :s try o.builder.string(section);
1083	} else .none;
1084
1085	// Actual function bodies with AIR go through `updateFunc` instead, so the only functions we
1086	// can see are extern functions or other comptime function body values (e.g. undefined). Of
1087	// these, only extern functions need to be lowered to LLVM functions.
1088	if (opt_extern != null and nav_ty.zigTypeTag(zcu) == .@"fn" and nav_ty.fnHasRuntimeBits(zcu)) {
1089	const fn_info = zcu.typeToFunc(nav_ty).?;
1090	const llvm_function: Builder.Function.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
1091	.function => |function| function, // re-use existing `Builder.Function`
1092	.replaced, .alias, .variable => try llvm_global.toNewFunction(&o.builder),
1093	};
1094	llvm_function.setAlignment(resolved.@"align".toLlvm(), &o.builder);
1095	llvm_function.setSection(llvm_section, &o.builder);
1096	var attributes: Builder.FunctionAttributes.Wip = .{};
1097	defer attributes.deinit(&o.builder);
1098	try o.addCallingConventionFnAttributes(pt, llvm_function, &attributes, .{
1099	.name = nav.name.toSlice(ip),
1100	.lib_name = opt_extern.?.lib_name.toSlice(ip),
1101	}, .fromIntern(fn_info, ip));
1102	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
1103	} else {
1104	const file_scope = nav.srcInst(ip).resolveFile(ip);
1105	const mod = zcu.fileByIndex(file_scope).mod.?;
1106
1107	const llvm_variable: Builder.Variable.Index = switch (llvm_global.ptrConst(&o.builder).kind) {
1108	.variable => |variable| variable, // re-use existing `Builder.Variable`
1109	.replaced, .alias, .function => try llvm_global.toNewVariable(&o.builder),
1110	};
1111	llvm_variable.setAlignment(switch (resolved.@"align") {
1112	.none => nav_ty.abiAlignment(zcu).toLlvm(),
1113	else => |a| a.toLlvm(),
1114	}, &o.builder);
1115	llvm_variable.setSection(llvm_section, &o.builder);
1116	llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
1117	try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
1118	llvm_variable.setThreadLocal(tl: {
1119	if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
1120	break :tl .default;
1121	}, &o.builder);
1122
1123	if (!mod.strip) {
1124	const debug_file = try o.getDebugFile(file_scope);
1125	const debug_global_var_expr = try o.builder.debugGlobalVarExpression(
1126	try o.builder.debugGlobalVar(
1127	try o.builder.metadataString(nav.name.toSlice(ip)), // Name
1128	try o.builder.metadataString(nav.fqn.toSlice(ip)), // Linkage name
1129	debug_file, // File
1130	debug_file, // Scope
1131	zcu.navSrcLine(nav_id) + 1,
1132	try o.getDebugType(pt, nav_ty),
1133	llvm_variable,
1134	.{ .local = llvm_global.ptrConst(&o.builder).linkage == .internal },
1135	),
1136	try o.builder.debugExpression(&.{}),
1137	);
1138	llvm_variable.setGlobalVariableExpression(debug_global_var_expr, &o.builder);
1139	try o.debug_globals.append(o.gpa, debug_global_var_expr);
1140	}
1141	}
1142	}
1143
1144	fn flushTypePool(o: *Object, pt: Zcu.PerThread) Allocator.Error!void {
1145	try o.type_pool.flushPending(pt, .{ .llvm = o });
1146	}
1147
1148	pub fn updateExports(
1149	o: *Object,
1150	export_indices: []const Zcu.Export.Index,
1151	) link.Error!void {
1152	const zcu = o.zcu;
1153	const ip = &zcu.intern_pool;
1154	for (export_indices) |export_index| {
1155	const ty: Type, const llvm_ptr: Builder.Constant = switch (export_index.ptr(zcu).exported) {
1156	.nav => |nav| exp: {
1157	const nav_ty: Type = .fromInterned(ip.getNav(nav).resolved.?.type);
1158	const nav_ref = try o.lowerNavRef(nav);
1159	break :exp .{ nav_ty, nav_ref };
1160	},
1161	.uav => |uav| exp: {
1162	const uav_ty = Value.fromInterned(uav).typeOf(zcu);
1163	const uav_ref = try o.lowerUavRef(
1164	uav,
1165	uav_ty.abiAlignment(zcu).toLlvm(),
1166	target_util.defaultAddressSpace(zcu.getTarget(), .global_constant),
1167	);
1168	break :exp .{ uav_ty, uav_ref };
1169	},
1170	};
1171	switch (llvm_ptr.unwrap()) {
1172	.global => |global| try o.addGlobalExport(global, ty, export_index),
1173	.constant => @panic("LLVM TODO: export zero-bit value"),
1174	}
1175	}
1176	}
1177
1178	fn addGlobalExport(
1179	o: *Object,
1180	llvm_global: Builder.Global.Index,
1181	ty: Type,
1182	export_index: Zcu.Export.Index,
1183	) link.Error!void {
1184	const zcu = o.zcu;
1185	const comp = zcu.comp;
1186	const ip = &zcu.intern_pool;
1187
1188	const exp = export_index.ptr(zcu);
1189
1190	// If we're on COFF and linking with LLD, the linker cares about our exports to determine the subsystem in use.
1191	coff_export_flags: {
1192	const lf = comp.bin_file orelse break :coff_export_flags;
1193	const lld = lf.cast(.lld) orelse break :coff_export_flags;
1194	const coff = switch (lld.ofmt) {
1195	.elf, .wasm => break :coff_export_flags,
1196	.coff => |*coff| coff,
1197	};
1198	if (ty.zigTypeTag(zcu) != .@"fn") break :coff_export_flags;
1199	const flags = &coff.lld_export_flags;
1200	if (exp.opts.name.eqlSlice("main", ip)) flags.c_main = true;
1201	if (exp.opts.name.eqlSlice("WinMain", ip)) flags.winmain = true;
1202	if (exp.opts.name.eqlSlice("wWinMain", ip)) flags.wwinmain = true;
1203	if (exp.opts.name.eqlSlice("WinMainCRTStartup", ip)) flags.winmain_crt_startup = true;
1204	if (exp.opts.name.eqlSlice("wWinMainCRTStartup", ip)) flags.wwinmain_crt_startup = true;
1205	if (exp.opts.name.eqlSlice("DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;
1206	if (exp.opts.name.eqlSlice("_DllMainCRTStartup", ip)) flags.dllmain_crt_startup = true;
1207	}
1208
1209	// If the export specifies a linksection, set the exported variable's section to that one.
1210	// This is kind of a hack because `std.lang.ExportOptions.section` doesn't actually make
1211	// much sense: the linksection should be associated with the declaration itself rather than
1212	// some particular symbol it is exported as!
1213	if (exp.opts.section.toSlice(ip)) |section_slice| {
1214	const variable = &llvm_global.ptrConst(&o.builder).kind.variable;
1215	variable.setSection(try o.builder.string(section_slice), &o.builder);
1216	}
1217
1218	const arch = comp.root_mod.resolved_target.result.cpu.arch;
1219	const workaround_alias_bugs = arch == .amdgcn or arch == .nvptx or arch == .nvptx64;
1220
1221	const llvm_global_ty = llvm_global.typeOf(&o.builder);
1222
1223	// All exports are represented as aliases to the original global.
1224
1225	// TODO: we currently do not delete old exports. To do that we'll need to track which
1226	// globals actually *are* exports.
1227
1228	const exp_name = try o.builder.strtabString(exp.opts.name.toSlice(ip));
1229
1230	// Our goal is to make an alias with the name `exp_name`, but if that name is already
1231	// taken by some existing global, we need to figure out what to do with that existing
1232	// global.
1233	//
1234	// The name, aliasee, and type will be set within this block. Other properties of the
1235	// alias will be set below.
1236	const alias_global: Builder.Global.Index = global: {
1237
1238	// WORKAROUND (see https://github.com/llvm/llvm-project/issues/213504, https://github.com/llvm/llvm-project/issues/214835)
1239	// For NVPTX, LLVM throws "NVPTX aliasee must be a non-kernel function definition" if we try to alias a kernel
1240	// On AMDGCN, LLVM does not generate an alias for the kernel descriptor symbol on associated functions
1241	// To solve these, we rename the global
1242	if (workaround_alias_bugs) {
1243	try llvm_global.rename(exp_name, &o.builder);
1244	break :global llvm_global;
1245	}
1246
1247	const existing_global = o.builder.getGlobal(exp_name) orelse {
1248	// There is no existing global with this name, so make a new alias.
1249	const alias = try o.builder.addAlias(
1250	exp_name,
1251	llvm_global_ty,
1252	llvm_global.ptrConst(&o.builder).addr_space,
1253	llvm_global.toConst(),
1254	);
1255	break :global alias.ptrConst(&o.builder).global;
1256	};
1257	// There is an existing global with this name, so we can't just create an alias. We
1258	// need to figure out what to do with the existing global instead.
1259	switch (existing_global.ptrConst(&o.builder).kind) {
1260	.alias => |alias| {
1261	// We can just repurpose the existing alias.
1262	alias.setAliasee(llvm_global.toConst(), &o.builder);
1263	alias.ptrConst(&o.builder).global.ptr(&o.builder).type = llvm_global.typeOf(&o.builder);
1264	alias.ptrConst(&o.builder).global.ptr(&o.builder).addr_space = llvm_global.ptrConst(&o.builder).addr_space;
1265	break :global existing_global;
1266	},
1267	.variable, .function => {
1268	// This must be an extern, which is no good to us---we need an alias. The
1269	// extern should refer to the value we're exporting, so replace it with the
1270	// exported value. That will free up the name for us to create a new alias.
1271	// We need to make a new global which is an alias. Replace this existing one
1272	// with the target global, making the name available and fixing references
1273	// to this global to point to the target.
1274	try existing_global.replace(llvm_global, &o.builder);
1275	// The name is now free, so create an alias.
1276	const alias = try o.builder.addAlias(
1277	exp_name,
1278	llvm_global_ty,
1279	llvm_global.ptrConst(&o.builder).addr_space,
1280	llvm_global.toConst(),
1281	);
1282	break :global alias.ptrConst(&o.builder).global;
1283	},
1284	.replaced => unreachable, // a replaced global would have lost the name `exp_name`
1285	}
1286	};
1287
1288	// We need the alias to *not* be `unnamed_addr` to ensure that the alias address equals
1289	// the address of the original global.
1290	alias_global.setUnnamedAddr(.default, &o.builder);
1291
1292	if (comp.config.dll_export_fns and exp.opts.visibility != .hidden)
1293	alias_global.setDllStorageClass(.dllexport, &o.builder);
1294	alias_global.setLinkage(switch (exp.opts.linkage) {
1295	.internal => if (o.builder.strip) .private else .internal, // we still did useful work in replacing an existing symbol if there was one
1296	.strong => .external,
1297	.weak => .weak_odr,
1298	.link_once => .linkonce_odr,
1299	}, &o.builder);
1300	alias_global.setVisibility(switch (exp.opts.visibility) {
1301	.default => .default,
1302	.hidden => .hidden,
1303	.protected => .protected,
1304	}, &o.builder);
1305	}
1306
1307	pub fn updateContainerType(o: *Object, pt: Zcu.PerThread, ty: InternPool.Index, success: bool) Allocator.Error!void {
1308	_ = o.type_map.remove(ty);
1309	try o.type_pool.updateContainerType(pt, .{ .llvm = o }, ty, success);
1310	if (o.named_enum_map.get(ty)) |llvm_function| {
1311	try o.updateIsNamedEnumValueFunction(.fromInterned(ty), llvm_function);
1312	}
1313	if (o.enum_tag_name_map.get(ty)) |llvm_function| {
1314	try o.updateEnumTagNameFunction(.fromInterned(ty), llvm_function);
1315	}
1316	}
1317
1318	/// Should only be called by the `link.ConstPool` implementation.
1319	///
1320	/// `val` is always a type because `o.type_pool` only contains types.
1321	pub fn addConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1322	_ = pt;
1323	const zcu = o.zcu;
1324	const gpa = zcu.comp.gpa;
1325	assert(zcu.intern_pool.typeOf(val) == .type_type);
1326
1327	{
1328	assert(@backingInt(index) == o.lazy_abi_aligns.items.len);
1329	try o.lazy_abi_aligns.ensureUnusedCapacity(gpa, 1);
1330	const fwd_ref = try o.builder.alignmentForwardReference();
1331	o.lazy_abi_aligns.appendAssumeCapacity(fwd_ref);
1332	}
1333
1334	if (!o.builder.strip) {
1335	assert(@backingInt(index) == o.debug_types.items.len);
1336	try o.debug_types.ensureUnusedCapacity(gpa, 1);
1337	const fwd_ref = try o.builder.debugForwardReference();
1338	o.debug_types.appendAssumeCapacity(fwd_ref);
1339	if (val == .anyerror_type) {
1340	assert(o.debug_anyerror_fwd_ref.is_none);
1341	o.debug_anyerror_fwd_ref = fwd_ref.toOptional();
1342	}
1343	}
1344	}
1345	/// Should only be called by the `link.ConstPool` implementation.
1346	///
1347	/// `val` is always a type because `o.type_pool` only contains types.
1348	pub fn updateConstIncomplete(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1349	const zcu = o.zcu;
1350	assert(zcu.intern_pool.typeOf(val) == .type_type);
1351
1352	const ty: Type = .fromInterned(val);
1353
1354	{
1355	const fwd_ref = o.lazy_abi_aligns.items[@backingInt(index)];
1356	o.builder.resolveAlignmentForwardReference(fwd_ref, .fromByteUnits(1));
1357	}
1358
1359	if (!o.builder.strip) {
1360	assert(val != .anyerror_type);
1361	const fwd_ref = o.debug_types.items[@backingInt(index)];
1362	const name_str = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)});
1363	// If `ty` is a function, use a dummy *function* type to prevent existing debug
1364	// subprograms from becoming ill-formed.
1365	const debug_incomplete_type = switch (ty.zigTypeTag(zcu)) {
1366	.@"fn" => try o.builder.debugSubroutineType(null),
1367	else => try o.builder.debugSignedType(name_str, 0),
1368	};
1369	o.builder.resolveDebugForwardReference(fwd_ref, debug_incomplete_type);
1370	}
1371	}
1372	/// Should only be called by the `link.ConstPool` implementation.
1373	///
1374	/// `val` is always a type because `o.type_pool` only contains types.
1375	pub fn updateConst(o: *Object, pt: Zcu.PerThread, index: link.ConstPool.Index, val: InternPool.Index) Allocator.Error!void {
1376	const zcu = o.zcu;
1377	assert(zcu.intern_pool.typeOf(val) == .type_type);
1378
1379	const ty: Type = .fromInterned(val);
1380
1381	{
1382	const fwd_ref = o.lazy_abi_aligns.items[@backingInt(index)];
1383	o.builder.resolveAlignmentForwardReference(fwd_ref, ty.abiAlignment(zcu).toLlvm());
1384	}
1385
1386	if (!o.builder.strip) {
1387	const fwd_ref = o.debug_types.items[@backingInt(index)];
1388	if (val == .anyerror_type) {
1389	// Don't lower this now; it will be populated in `emit` instead.
1390	assert(o.debug_anyerror_fwd_ref == fwd_ref.toOptional());
1391	} else {
1392	const debug_type = try o.lowerDebugType(pt, ty, fwd_ref);
1393	o.builder.resolveDebugForwardReference(fwd_ref, debug_type);
1394	}
1395	}
1396	}
1397
1398	pub fn getDebugFile(o: *Object, file_index: Zcu.File.Index) Allocator.Error!Builder.Metadata {
1399	const gpa = o.gpa;
1400	const gop = try o.debug_file_map.getOrPut(gpa, file_index);
1401	errdefer assert(o.debug_file_map.remove(file_index));
1402	if (gop.found_existing) return gop.value_ptr.*;
1403
1404	const dirs = o.zcu.comp.dirs;
1405	const path = o.zcu.fileByIndex(file_index).path;
1406	const root_path: ?[]const u8 = switch (path.root) {
1407	.zig_lib => dirs.zig_lib.path,
1408	.global_cache => dirs.global_cache.path,
1409	.local_cache => dirs.local_cache.path,
1410	.build_root => dirs.build_root.path,
1411	.none => null,
1412	};
1413
1414	const file = if (root_path) |root|
1415	try o.builder.debugFile(
1416	try o.builder.metadataString(path.sub_path),
1417	try o.builder.metadataString(root),
1418	)
1419	else blk: {
1420	const relative = try std.fs.path.relative(gpa, dirs.cwd, null, dirs.cwd, path.sub_path);
1421	defer gpa.free(relative);
1422	break :blk try o.builder.debugFile(
1423	try o.builder.metadataString(relative),
1424	try o.builder.metadataString(dirs.cwd),
1425	);
1426	};
1427
1428	gop.value_ptr.* = file;
1429	return file;
1430	}
1431
1432	pub fn getDebugType(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Metadata {
1433	assert(!o.builder.strip);
1434	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
1435	return o.debug_types.items[@backingInt(index)];
1436	}
1437
1438	/// In codegen logic, instead of calling this directly, use `getDebugType` to get a forward
1439	/// reference which will be populated only when all necessary type resolution is complete.
1440	fn lowerDebugType(
1441	o: *Object,
1442	pt: Zcu.PerThread,
1443	ty: Type,
1444	ty_fwd_ref: Builder.Metadata,
1445	) Allocator.Error!Builder.Metadata {
1446	assert(!o.builder.strip);
1447
1448	const gpa = o.gpa;
1449	const zcu = o.zcu;
1450	const target = zcu.getTarget();
1451	const ip = &zcu.intern_pool;
1452
1453	const name = try o.builder.metadataStringFmt("{f}", .{ty.fmt(pt)});
1454
1455	// lldb cannot handle non-byte-sized types, so in the logic below, bit sizes are padded up.
1456	// For instance, `bool` is considered to be 8 bits, and `u60` is considered to be 64 bits.
1457
1458	// I tried using variants (DW_TAG_variant_part + DW_TAG_variant) to encode error unions,
1459	// tagged unions, etc; this would have told debuggers which field was active, which could
1460	// improve UX significantly. GDB handles this perfectly fine, but unfortunately, LLDB has no
1461	// handling for variants at all, and will never print fields in them, so I opted not to use
1462	// them for now.
1463
1464	switch (ty.zigTypeTag(zcu)) {
1465	.void,
1466	.noreturn,
1467	.comptime_int,
1468	.comptime_float,
1469	.type,
1470	.undefined,
1471	.null,
1472	.enum_literal,
1473	=> return o.builder.debugSignedType(name, 0),
1474
1475	.float => return o.builder.debugFloatType(name, ty.floatBits(target)),
1476
1477	.bool => return o.builder.debugBoolType(name, 8),
1478
1479	.int => {
1480	const info = ty.intInfo(zcu);
1481	const bits = ty.abiSize(zcu) * 8;
1482	return switch (info.signedness) {
1483	.signed => try o.builder.debugSignedType(name, bits),
1484	.unsigned => try o.builder.debugUnsignedType(name, bits),
1485	};
1486	},
1487
1488	.pointer => {
1489	const ptr_size = Type.ptrAbiSize(zcu.getTarget());
1490	const ptr_align = Type.ptrAbiAlignment(zcu.getTarget());
1491
1492	if (ty.isSlice(zcu)) {
1493	const debug_ptr_type = try o.builder.debugMemberType(
1494	try o.builder.metadataString("ptr"),
1495	null, // file
1496	ty_fwd_ref,
1497	0, // line
1498	try o.getDebugType(pt, ty.slicePtrFieldType(zcu)),
1499	ptr_size * 8,
1500	ptr_align.toByteUnits().? * 8,
1501	0, // offset
1502	);
1503
1504	const debug_len_type = try o.builder.debugMemberType(
1505	try o.builder.metadataString("len"),
1506	null, // file
1507	ty_fwd_ref,
1508	0, // line
1509	try o.getDebugType(pt, .usize),
1510	ptr_size * 8,
1511	ptr_align.toByteUnits().? * 8,
1512	ptr_size * 8,
1513	);
1514
1515	return o.builder.debugStructType(
1516	name,
1517	null, // file
1518	o.debug_compile_unit.unwrap().?, // scope
1519	0, // line
1520	null, // underlying type
1521	ptr_size * 2 * 8,
1522	ptr_align.toByteUnits().? * 8,
1523	try o.builder.metadataTuple(&.{
1524	debug_ptr_type,
1525	debug_len_type,
1526	}),
1527	);
1528	}
1529
1530	return o.builder.debugPointerType(
1531	name,
1532	null, // file
1533	o.debug_compile_unit.unwrap().?, // scope
1534	0, // line
1535	try o.getDebugType(pt, ty.childType(zcu)),
1536	ptr_size * 8,
1537	ptr_align.toByteUnits().? * 8,
1538	0, // offset
1539	);
1540	},
1541	.array => return o.builder.debugArrayType(
1542	name,
1543	null, // file
1544	o.debug_compile_unit.unwrap().?, // scope
1545	0, // line
1546	try o.getDebugType(pt, ty.childType(zcu)),
1547	ty.abiSize(zcu) * 8,
1548	ty.abiAlignment(zcu).toByteUnits().? * 8,
1549	try o.builder.metadataTuple(&.{
1550	try o.builder.debugSubrange(
1551	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
1552	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.arrayLen(zcu))),
1553	),
1554	}),
1555	),
1556	.vector => {
1557	const elem_ty = ty.childType(zcu);
1558	// Vector elements cannot be padded since that would make
1559	// @bitSizeOf(elem) * len > @bitSizOf(vec).
1560	// Neither gdb nor lldb seem to be able to display non-byte sized
1561	// vectors properly.
1562	const debug_elem_type = switch (elem_ty.zigTypeTag(zcu)) {
1563	.int => blk: {
1564	const info = elem_ty.intInfo(zcu);
1565	break :blk switch (info.signedness) {
1566	.signed => try o.builder.debugSignedType(name, info.bits),
1567	.unsigned => try o.builder.debugUnsignedType(name, info.bits),
1568	};
1569	},
1570	.bool => try o.builder.debugBoolType(try o.builder.metadataString("bool"), 1),
1571	// We don't pad pointers or floats, so we can lower those normally.
1572	.pointer, .optional, .float => try o.getDebugType(pt, elem_ty),
1573	else => unreachable,
1574	};
1575
1576	return o.builder.debugVectorType(
1577	name,
1578	null, // file
1579	o.debug_compile_unit.unwrap().?, // scope
1580	0, // line
1581	debug_elem_type,
1582	ty.abiSize(zcu) * 8,
1583	ty.abiAlignment(zcu).toByteUnits().? * 8,
1584	try o.builder.metadataTuple(&.{
1585	try o.builder.debugSubrange(
1586	try o.builder.metadataConstant(try o.builder.intConst(.i64, 0)),
1587	try o.builder.metadataConstant(try o.builder.intConst(.i64, ty.vectorLen(zcu))),
1588	),
1589	}),
1590	);
1591	},
1592	.optional => {
1593	const payload_ty = ty.optionalChild(zcu);
1594	if (ty.optionalReprIsPayload(zcu)) {
1595	return o.builder.debugTypedefType(
1596	name,
1597	null, // file
1598	o.debug_compile_unit.unwrap().?, // scope
1599	0, // line
1600	try o.getDebugType(pt, payload_ty),
1601	ty.abiSize(zcu) * 8,
1602	ty.abiAlignment(zcu).toByteUnits().? * 8,
1603	0, // offset
1604	);
1605	}
1606
1607	const payload_size = payload_ty.abiSize(zcu);
1608
1609	const non_null_ty = Type.u8;
1610	const non_null_size = non_null_ty.abiSize(zcu);
1611	const non_null_align = non_null_ty.abiAlignment(zcu);
1612	const non_null_offset = non_null_align.forward(payload_size);
1613
1614	const debug_payload_type = try o.builder.debugMemberType(
1615	try o.builder.metadataString("payload"),
1616	null, // file
1617	ty_fwd_ref, // scope
1618	0, // line
1619	try o.getDebugType(pt, payload_ty),
1620	payload_size * 8,
1621	payload_ty.abiAlignment(zcu).toByteUnits().? * 8,
1622	0, // offset
1623	);
1624
1625	const debug_some_type = try o.builder.debugMemberType(
1626	try o.builder.metadataString("some"),
1627	null,
1628	ty_fwd_ref,
1629	0,
1630	try o.getDebugType(pt, non_null_ty),
1631	non_null_size * 8,
1632	non_null_align.toByteUnits().? * 8,
1633	non_null_offset * 8,
1634	);
1635
1636	return o.builder.debugStructType(
1637	name,
1638	null, // file
1639	o.debug_compile_unit.unwrap().?, // scope
1640	0, // line
1641	null, // underlying type
1642	ty.abiSize(zcu) * 8,
1643	ty.abiAlignment(zcu).toByteUnits().? * 8,
1644	try o.builder.metadataTuple(&.{
1645	debug_payload_type,
1646	debug_some_type,
1647	}),
1648	);
1649	},
1650	.error_union => {
1651	const error_ty = ty.errorUnionSet(zcu);
1652	const payload_ty = ty.errorUnionPayload(zcu);
1653
1654	const error_size = error_ty.abiSize(zcu);
1655	const error_align = error_ty.abiAlignment(zcu);
1656	const payload_size = payload_ty.abiSize(zcu);
1657	const payload_align = payload_ty.abiAlignment(zcu);
1658
1659	const error_offset: u64, const payload_offset: u64 = offsets: {
1660	if (error_align.compare(.gt, payload_align)) {
1661	break :offsets .{ 0, payload_align.forward(error_size) };
1662	} else {
1663	break :offsets .{ error_align.forward(payload_size), 0 };
1664	}
1665	};
1666
1667	const error_field = try o.builder.debugMemberType(
1668	try o.builder.metadataString("error"),
1669	null, // file
1670	ty_fwd_ref,
1671	0, // line
1672	try o.getDebugType(pt, error_ty),
1673	error_size * 8,
1674	error_align.toByteUnits().? * 8,
1675	error_offset * 8,
1676	);
1677	const payload_field = try o.builder.debugMemberType(
1678	try o.builder.metadataString("payload"),
1679	null, // file
1680	ty_fwd_ref, // scope
1681	0, // line
1682	try o.getDebugType(pt, payload_ty),
1683	payload_size * 8,
1684	payload_align.toByteUnits().? * 8,
1685	payload_offset * 8,
1686	);
1687
1688	return o.builder.debugStructType(
1689	name,
1690	null, // File
1691	o.debug_compile_unit.unwrap().?, // Scope
1692	0, // Line
1693	null, // Underlying type
1694	ty.abiSize(zcu) * 8,
1695	ty.abiAlignment(zcu).toByteUnits().? * 8,
1696	try o.builder.metadataTuple(&.{ error_field, payload_field }),
1697	);
1698	},
1699	.error_set => {
1700	assert(ty.toIntern() != .anyerror_type); // handled specially in `updateConst`; will be populated by `emit` instead
1701	// Error sets are just named wrappers around `anyerror`.
1702	return o.builder.debugTypedefType(
1703	name,
1704	null, // file
1705	o.debug_compile_unit.unwrap().?, // scope
1706	0, // line
1707	try o.getDebugType(pt, .anyerror),
1708	ty.abiSize(zcu) * 8,
1709	ty.abiAlignment(zcu).toByteUnits().? * 8,
1710	0, // offset
1711	);
1712	},
1713	.@"fn" => {
1714	if (!ty.fnHasRuntimeBits(zcu)) {
1715	// Use a dummy *function* type to prevent existing debug subprograms from
1716	// becoming ill-formed.
1717	return o.builder.debugSubroutineType(null);
1718	}
1719
1720	const fn_info = zcu.typeToFunc(ty).?;
1721
1722	var debug_param_types: std.ArrayList(Builder.Metadata) = try .initCapacity(gpa, 3 + fn_info.param_types.len);
1723	defer debug_param_types.deinit(gpa);
1724
1725	// Return type goes first.
1726	if (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type)) == .sret) {
1727	// Actual return type is void, then first arg is the sret pointer.
1728	const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
1729	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
1730	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty));
1731	} else {
1732	const ret_ty: Type = .fromInterned(fn_info.return_type);
1733	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ret_ty));
1734	}
1735
1736	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
1737	// Stack trace pointer.
1738	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .ptr_usize));
1739	}
1740
1741	for (fn_info.param_types.get(ip)) |param_ty_ip| {
1742	const param_ty: Type = .fromInterned(param_ty_ip);
1743	if (!param_ty.hasRuntimeBits(zcu)) continue;
1744	if (isByRef(param_ty, zcu)) {
1745	const ptr_ty = try pt.singleConstPtrType(param_ty);
1746	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, ptr_ty));
1747	} else {
1748	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, param_ty));
1749	}
1750	}
1751
1752	return o.builder.debugSubroutineType(
1753	try o.builder.metadataTuple(debug_param_types.items),
1754	);
1755	},
1756	.@"struct" => {
1757	if (ty.isTuple(zcu)) {
1758	const tuple = ip.indexToKey(ty.toIntern()).tuple_type;
1759	var fields: std.ArrayList(Builder.Metadata) = .empty;
1760	defer fields.deinit(gpa);
1761
1762	try fields.ensureUnusedCapacity(gpa, tuple.types.len);
1763
1764	comptime assert(struct_layout_version == 2);
1765	var offset: u64 = 0;
1766
1767	for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty_ip, field_val, i| {
1768	const field_ty: Type = .fromInterned(field_ty_ip);
1769	if (field_val != .none or !field_ty.hasRuntimeBits(zcu)) continue;
1770
1771	const field_size = field_ty.abiSize(zcu);
1772	const field_align = field_ty.abiAlignment(zcu);
1773	const field_offset = field_align.forward(offset);
1774	offset = field_offset + field_size;
1775
1776	fields.appendAssumeCapacity(try o.builder.debugMemberType(
1777	try o.builder.metadataStringFmt("{d}", .{i}),
1778	null, // file
1779	ty_fwd_ref,
1780	0, // line
1781	try o.getDebugType(pt, field_ty),
1782	field_size * 8,
1783	field_align.toByteUnits().? * 8,
1784	field_offset * 8,
1785	));
1786	}
1787
1788	return o.builder.debugStructType(
1789	name,
1790	null, // file
1791	o.debug_compile_unit.unwrap().?,
1792	0, // line
1793	null, // underlying type
1794	ty.abiSize(zcu) * 8,
1795	(ty.abiAlignment(zcu).toByteUnits() orelse 0) * 8,
1796	try o.builder.metadataTuple(fields.items),
1797	);
1798	}
1799
1800	const struct_type = zcu.typeToStruct(ty).?;
1801
1802	const file = try o.getDebugFile(struct_type.zir_index.resolveFile(ip));
1803	const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
1804	try o.namespaceToDebugScope(pt, parent_namespace)
1805	else
1806	file;
1807
1808	const line = ty.typeDeclSrcLine(zcu).? + 1;
1809
1810	var fields: std.ArrayList(Builder.Metadata) = .empty;
1811	defer fields.deinit(gpa);
1812
1813	switch (struct_type.layout) {
1814	.@"packed" => {
1815	try fields.ensureTotalCapacityPrecise(gpa, 1);
1816	fields.appendAssumeCapacity(try o.builder.debugMemberType(
1817	try o.builder.metadataString("bits"),
1818	null, // file
1819	ty_fwd_ref,
1820	0, // line
1821	try o.getDebugType(pt, .fromInterned(struct_type.packed_backing_int_type)),
1822	ty.abiSize(zcu) * 8,
1823	ty.abiAlignment(zcu).toByteUnits().? * 8,
1824	0, // offset
1825	));
1826	},
1827	.auto, .@"extern" => {
1828	comptime assert(struct_layout_version == 2);
1829	try fields.ensureTotalCapacityPrecise(gpa, struct_type.field_types.len);
1830	var it = struct_type.iterateRuntimeOrder(ip);
1831	while (it.next()) |field_index| {
1832	const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]);
1833	if (!field_ty.hasRuntimeBits(zcu)) continue;
1834	const field_size = field_ty.abiSize(zcu);
1835	const field_align = switch (ty.explicitFieldAlignment(field_index, zcu)) {
1836	.none => field_ty.abiAlignment(zcu),
1837	else => |a| a,
1838	};
1839	const field_offset = struct_type.field_offsets.get(ip)[field_index];
1840	const field_name = struct_type.field_names.get(ip)[field_index];
1841	fields.appendAssumeCapacity(try o.builder.debugMemberType(
1842	try o.builder.metadataString(field_name.toSlice(ip)),
1843	null, // file
1844	ty_fwd_ref,
1845	0, // line
1846	try o.getDebugType(pt, field_ty),
1847	field_size * 8,
1848	field_align.toByteUnits().? * 8,
1849	field_offset * 8,
1850	));
1851	}
1852	},
1853	}
1854
1855	return o.builder.debugStructType(
1856	name,
1857	file,
1858	scope,
1859	line,
1860	null, // underlying type
1861	ty.abiSize(zcu) * 8,
1862	ty.abiAlignment(zcu).toByteUnits().? * 8,
1863	try o.builder.metadataTuple(fields.items),
1864	);
1865	},
1866	.@"union" => {
1867	const union_type = ip.loadUnionType(ty.toIntern());
1868
1869	const file = try o.getDebugFile(union_type.zir_index.resolveFile(ip));
1870	const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
1871	try o.namespaceToDebugScope(pt, parent_namespace)
1872	else
1873	file;
1874
1875	const line = ty.typeDeclSrcLine(zcu).? + 1;
1876
1877	const enum_tag_ty: Type = .fromInterned(union_type.enum_tag_type);
1878
1879	if (union_type.layout == .@"packed") {
1880	const bitpack_field = try o.builder.debugMemberType(
1881	try o.builder.metadataString("bits"),
1882	null, // file
1883	ty_fwd_ref,
1884	0, // line
1885	try o.getDebugType(pt, .fromInterned(union_type.packed_backing_int_type)),
1886	ty.abiSize(zcu) * 8,
1887	ty.abiAlignment(zcu).toByteUnits().? * 8,
1888	0, // offset
1889	);
1890	return o.builder.debugStructType(
1891	name,
1892	file,
1893	scope,
1894	line,
1895	null, // underlying type
1896	ty.abiSize(zcu) * 8,
1897	ty.abiAlignment(zcu).toByteUnits().? * 8,
1898	try o.builder.metadataTuple(&.{bitpack_field}),
1899	);
1900	}
1901
1902	const layout = Type.getUnionLayout(union_type, zcu);
1903
1904	if (layout.payload_size == 0) {
1905	const fields_tuple: ?Builder.Metadata = fields: {
1906	if (layout.tag_size == 0) break :fields null;
1907	break :fields try o.builder.metadataTuple(&.{
1908	try o.builder.debugMemberType(
1909	try o.builder.metadataString("tag"),
1910	null, // file
1911	ty_fwd_ref,
1912	0, // line
1913	try o.getDebugType(pt, enum_tag_ty),
1914	layout.tag_size * 8,
1915	layout.tag_align.toByteUnits().? * 8,
1916	0, // offset
1917	),
1918	});
1919	};
1920	return o.builder.debugStructType(
1921	name,
1922	file,
1923	scope,
1924	line,
1925	null, // underlying type
1926	ty.abiSize(zcu) * 8,
1927	ty.abiAlignment(zcu).toByteUnits().? * 8,
1928	fields_tuple,
1929	);
1930	}
1931
1932	var fields: std.ArrayList(Builder.Metadata) = try .initCapacity(gpa, union_type.field_types.len);
1933	defer fields.deinit(gpa);
1934
1935	const payload_fwd_ref = if (layout.tag_size == 0)
1936	ty_fwd_ref
1937	else
1938	try o.builder.debugForwardReference();
1939
1940	for (0..union_type.field_types.len) |field_index| {
1941	const field_ty = union_type.field_types.get(ip)[field_index];
1942
1943	const field_size = Type.fromInterned(field_ty).abiSize(zcu);
1944	const field_align: InternPool.Alignment = ty.explicitFieldAlignment(field_index, zcu);
1945
1946	const field_name = enum_tag_ty.enumFieldName(field_index, zcu);
1947	fields.appendAssumeCapacity(try o.builder.debugMemberType(
1948	try o.builder.metadataString(field_name.toSlice(ip)),
1949	null, // file
1950	payload_fwd_ref,
1951	0, // line
1952	try o.getDebugType(pt, .fromInterned(field_ty)),
1953	field_size * 8,
1954	(field_align.toByteUnits() orelse 0) * 8,
1955	0, // offset
1956	));
1957	}
1958
1959	const debug_payload_type = try o.builder.debugUnionType(
1960	payload_name: {
1961	if (layout.tag_size == 0) break :payload_name name;
1962	break :payload_name try o.builder.metadataStringFmt("{f}:Payload", .{ty.fmt(pt)});
1963	},
1964	file,
1965	scope,
1966	line,
1967	null, // underlying type
1968	layout.payload_size * 8,
1969	ty.abiAlignment(zcu).toByteUnits().? * 8,
1970	try o.builder.metadataTuple(fields.items),
1971	);
1972
1973	if (layout.tag_size == 0) {
1974	return debug_payload_type;
1975	}
1976
1977	o.builder.resolveDebugForwardReference(payload_fwd_ref, debug_payload_type);
1978
1979	const tag_offset: u64, const payload_offset: u64 = offsets: {
1980	if (layout.tag_align.compare(.gte, layout.payload_align)) {
1981	break :offsets .{ 0, layout.payload_align.forward(layout.tag_size) };
1982	} else {
1983	break :offsets .{ layout.tag_align.forward(layout.payload_size), 0 };
1984	}
1985	};
1986
1987	const tag_member_type = try o.builder.debugMemberType(
1988	try o.builder.metadataString("tag"),
1989	null, // file
1990	ty_fwd_ref,
1991	0, // line
1992	try o.getDebugType(pt, enum_tag_ty),
1993	layout.tag_size * 8,
1994	layout.tag_align.toByteUnits().? * 8,
1995	tag_offset * 8,
1996	);
1997
1998	const payload_member_type = try o.builder.debugMemberType(
1999	try o.builder.metadataString("payload"),
2000	null, // file
2001	ty_fwd_ref,
2002	0, // line
2003	debug_payload_type,
2004	layout.payload_size * 8,
2005	layout.payload_align.toByteUnits().? * 8,
2006	payload_offset * 8,
2007	);
2008
2009	const full_fields: [2]Builder.Metadata =
2010	if (layout.tag_align.compare(.gte, layout.payload_align))
2011	.{ tag_member_type, payload_member_type }
2012	else
2013	.{ payload_member_type, tag_member_type };
2014
2015	return o.builder.debugStructType(
2016	name,
2017	file,
2018	scope,
2019	line,
2020	null, // underlying type
2021	ty.abiSize(zcu) * 8,
2022	ty.abiAlignment(zcu).toByteUnits().? * 8,
2023	try o.builder.metadataTuple(&full_fields),
2024	);
2025	},
2026	.@"enum" => {
2027	const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2028	const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
2029	try o.namespaceToDebugScope(pt, parent_namespace)
2030	else
2031	file;
2032
2033	const line = ty.typeDeclSrcLine(zcu).? + 1;
2034
2035	if (!ty.hasRuntimeBits(zcu)) {
2036	return o.builder.debugStructType(
2037	name,
2038	file,
2039	scope,
2040	line,
2041	null, // underlying type
2042	ty.abiSize(zcu) * 8,
2043	ty.abiAlignment(zcu).toByteUnits().? * 8,
2044	null, // fields
2045	);
2046	}
2047
2048	const enum_type = ip.loadEnumType(ty.toIntern());
2049	const enumerators = try gpa.alloc(Builder.Metadata, enum_type.field_names.len);
2050	defer gpa.free(enumerators);
2051
2052	const int_ty: Type = .fromInterned(enum_type.int_tag_type);
2053	const int_info = ty.intInfo(zcu);
2054	assert(int_info.bits != 0);
2055
2056	for (enumerators, enum_type.field_names.get(ip), 0..) |*out, field_name, field_index| {
2057	var space: Value.BigIntSpace = undefined;
2058	const field_val: std.math.big.int.Const = switch (enum_type.field_values.len) {
2059	0 => std.math.big.int.Mutable.init(&space.limbs, field_index).toConst(),
2060	else => Value.fromInterned(enum_type.field_values.get(ip)[field_index]).toBigInt(&space, zcu),
2061	};
2062	out.* = try o.builder.debugEnumerator(
2063	try o.builder.metadataString(field_name.toSlice(ip)),
2064	int_info.signedness == .unsigned,
2065	int_info.bits,
2066	field_val,
2067	);
2068	}
2069
2070	const debug_enum_type = try o.builder.debugEnumerationType(
2071	name,
2072	file,
2073	scope,
2074	line,
2075	try o.getDebugType(pt, int_ty),
2076	ty.abiSize(zcu) * 8,
2077	ty.abiAlignment(zcu).toByteUnits().? * 8,
2078	try o.builder.metadataTuple(enumerators),
2079	);
2080	try o.debug_enums.append(gpa, debug_enum_type);
2081	return debug_enum_type;
2082	},
2083	.@"opaque" => {
2084	if (ty.toIntern() == .anyopaque_type) {
2085	return o.builder.debugSignedType(name, 0);
2086	}
2087
2088	const file = try o.getDebugFile(ty.typeDeclInstAllowGeneratedTag(zcu).?.resolveFile(ip));
2089	const scope = if (ty.getParentNamespace(zcu).unwrap()) |parent_namespace|
2090	try o.namespaceToDebugScope(pt, parent_namespace)
2091	else
2092	file;
2093
2094	const line = ty.typeDeclSrcLine(zcu).? + 1;
2095
2096	return o.builder.debugStructType(
2097	name,
2098	file,
2099	scope,
2100	line,
2101	null, // underlying type
2102	0, // size
2103	ty.abiAlignment(zcu).toByteUnits().? * 8,
2104	null, // fields
2105	);
2106	},
2107	.frame => @panic("TODO implement lowerDebugType for Frame types"),
2108	.@"anyframe" => @panic("TODO implement lowerDebugType for AnyFrame types"),
2109	.spirv => unreachable,
2110	}
2111	}
2112
2113	/// Called in `emit` so that the global error set is fully populated.
2114	fn lowerDebugAnyerrorType(o: *Object) Allocator.Error!Builder.Metadata {
2115	const zcu = o.zcu;
2116	const ip = &zcu.intern_pool;
2117	const gpa = zcu.comp.gpa;
2118
2119	const error_set_bits = zcu.errorSetBits();
2120	const error_names = ip.global_error_set.getNamesFromMainThread();
2121
2122	const enumerators = try gpa.alloc(Builder.Metadata, error_names.len + 1);
2123	defer gpa.free(enumerators);
2124
2125	// The value 0 means "no error" in optionals and error unions.
2126	enumerators[0] = try o.builder.debugEnumerator(
2127	try o.builder.metadataString("null"),
2128	true, // unsigned,
2129	error_set_bits,
2130	.{ .limbs = &.{0}, .positive = true }, // zero
2131	);
2132
2133	for (enumerators[1..], error_names, 1..) |*out, error_name, error_value| {
2134	var space: Value.BigIntSpace = undefined;
2135	var bigint: std.math.big.int.Mutable = .init(&space.limbs, error_value);
2136	out.* = try o.builder.debugEnumerator(
2137	try o.builder.metadataStringFmt("error.{f}", .{error_name.fmtId(ip)}),
2138	true, // unsigned
2139	error_set_bits,
2140	bigint.toConst(),
2141	);
2142	}
2143
2144	const debug_enum_type = try o.builder.debugEnumerationType(
2145	try o.builder.metadataString("anyerror"),
2146	null, // file
2147	o.debug_compile_unit.unwrap().?, // scope
2148	0, // line
2149	try o.builder.debugUnsignedType(null, error_set_bits),
2150	Type.anyerror.abiSize(zcu) * 8,
2151	Type.anyerror.abiAlignment(zcu).toByteUnits().? * 8,
2152	try o.builder.metadataTuple(enumerators),
2153	);
2154	try o.debug_enums.append(gpa, debug_enum_type);
2155	return debug_enum_type;
2156	}
2157
2158	fn namespaceToDebugScope(o: *Object, pt: Zcu.PerThread, namespace_index: InternPool.NamespaceIndex) !Builder.Metadata {
2159	const zcu = o.zcu;
2160	const namespace = zcu.namespacePtr(namespace_index);
2161	if (namespace.parent == .none) return o.getDebugFile(namespace.file_scope);
2162	return o.getDebugType(pt, .fromInterned(namespace.owner_type));
2163	}
2164
2165	fn addCommonFnAttributes(
2166	o: *Object,
2167	attributes: *Builder.FunctionAttributes.Wip,
2168	owner_mod: *Module,
2169	omit_frame_pointer: bool,
2170	) Allocator.Error!void {
2171	if (!owner_mod.red_zone) {
2172	try attributes.addFnAttr(.noredzone, &o.builder);
2173	}
2174	if (omit_frame_pointer) {
2175	try attributes.addFnAttr(.{ .string = .{
2176	.kind = try o.builder.string("frame-pointer"),
2177	.value = try o.builder.string("none"),
2178	} }, &o.builder);
2179	} else {
2180	try attributes.addFnAttr(.{ .string = .{
2181	.kind = try o.builder.string("frame-pointer"),
2182	.value = try o.builder.string("all"),
2183	} }, &o.builder);
2184	}
2185	try attributes.addFnAttr(.nounwind, &o.builder);
2186	if (owner_mod.unwind_tables != .none) {
2187	try attributes.addFnAttr(
2188	.{ .uwtable = if (owner_mod.unwind_tables == .async) .async else .sync },
2189	&o.builder,
2190	);
2191	}
2192	if (owner_mod.optimize_mode == .small) {
2193	try attributes.addFnAttr(.minsize, &o.builder);
2194	try attributes.addFnAttr(.optsize, &o.builder);
2195	}
2196	const target = &owner_mod.resolved_target.result;
2197	if (target.cpu.model.llvm_name) |s| {
2198	try attributes.addFnAttr(.{ .string = .{
2199	.kind = try o.builder.string("target-cpu"),
2200	.value = try o.builder.string(s),
2201	} }, &o.builder);
2202	}
2203	if (owner_mod.resolved_target.llvm_cpu_features) |s| {
2204	try attributes.addFnAttr(.{ .string = .{
2205	.kind = try o.builder.string("target-features"),
2206	.value = try o.builder.string(std.mem.span(s)),
2207	} }, &o.builder);
2208	}
2209	if (target.abi.float() == .soft) {
2210	// `use-soft-float` means "use software routines for floating point computations". In
2211	// other words, it configures how LLVM lowers basic float instructions like `fcmp`,
2212	// `fadd`, etc. The float calling convention is configured on `TargetMachine` and is
2213	// mostly an orthogonal concept, although obviously we do need hardware float operations
2214	// to actually be able to pass float values in float registers.
2215	//
2216	// Ideally, we would support something akin to the `-mfloat-abi=softfp` option that GCC
2217	// and Clang support for Arm32 and CSKY. We don't currently expose such an option in
2218	// Zig, and using CPU features as the source of truth for this makes for a miserable
2219	// user experience since people expect e.g. `arm-linux-gnueabi` to mean full soft float
2220	// unless the compiler has explicitly been told otherwise. (And note that our baseline
2221	// CPU models almost all include FPU features!)
2222	//
2223	// Revisit this at some point.
2224	try attributes.addFnAttr(.{ .string = .{
2225	.kind = try o.builder.string("use-soft-float"),
2226	.value = try o.builder.string("true"),
2227	} }, &o.builder);
2228
2229	// This prevents LLVM from using FPU/SIMD code for things like `memcpy`. As for the
2230	// above, this should be revisited if `softfp` support is added.
2231	try attributes.addFnAttr(.noimplicitfloat, &o.builder);
2232	}
2233	}
2234
2235	pub fn addCallingConventionFnAttributes(
2236	o: *Object,
2237	pt: Zcu.PerThread,
2238	llvm_function: Builder.Function.Index,
2239	attributes: *Builder.FunctionAttributes.Wip,
2240	opt_extern: ?struct {
2241	name: []const u8,
2242	lib_name: ?[]const u8 = null,
2243	},
2244	fn_info: FuncInfo,
2245	) Allocator.Error!void {
2246	const zcu = o.zcu;
2247	const target = zcu.getTarget();
2248
2249	if (fn_info.cc == .async) {
2250	@panic("TODO: LLVM backend lower async function");
2251	}
2252
2253	if (target.cpu.arch.isWasm()) if (opt_extern) |@"extern"| {
2254	try attributes.addFnAttr(.{ .string = .{
2255	.kind = try o.builder.string("wasm-import-name"),
2256	.value = try o.builder.string(@"extern".name),
2257	} }, &o.builder);
2258	if (@"extern".lib_name) |lib_name| {
2259	if (!std.mem.eql(u8, lib_name, "c")) try attributes.addFnAttr(.{ .string = .{
2260	.kind = try o.builder.string("wasm-import-module"),
2261	.value = try o.builder.string(lib_name),
2262	} }, &o.builder);
2263	}
2264	};
2265
2266	const cc_info = toLlvmCallConv(fn_info.cc, target).?;
2267
2268	llvm_function.setCallConv(cc_info.llvm_cc, &o.builder);
2269
2270	if (cc_info.align_stack) {
2271	try attributes.addFnAttr(.{ .string = .{ .kind = try o.builder.string("stackrealign"), .value = .empty } }, &o.builder);
2272	}
2273
2274	if (cc_info.naked) {
2275	try attributes.addFnAttr(.naked, &o.builder);
2276	}
2277
2278	switch (fn_info.cc) {
2279	inline .riscv64_interrupt,
2280	.riscv32_interrupt,
2281	.mips_interrupt,
2282	.mips64_interrupt,
2283	=> |info| {
2284	try attributes.addFnAttr(.{ .string = .{
2285	.kind = try o.builder.string("interrupt"),
2286	.value = try o.builder.string(@tagName(info.mode)),
2287	} }, &o.builder);
2288	},
2289	.arm_interrupt,
2290	=> |info| {
2291	try attributes.addFnAttr(.{ .string = .{
2292	.kind = try o.builder.string("interrupt"),
2293	.value = try o.builder.string(switch (info.type) {
2294	.generic => "",
2295	.irq => "IRQ",
2296	.fiq => "FIQ",
2297	.swi => "SWI",
2298	.abort => "ABORT",
2299	.undef => "UNDEF",
2300	}),
2301	} }, &o.builder);
2302	},
2303	// these function attributes serve as a backup against any mistakes LLVM makes.
2304	// clang sets both the function's calling convention and the function attributes
2305	// in its backend, so future patches to the AVR backend could end up checking only one,
2306	// possibly breaking our support. it's safer to just emit both.
2307	.avr_interrupt, .avr_signal, .csky_interrupt => {
2308	try attributes.addFnAttr(.{ .string = .{
2309	.kind = try o.builder.string(switch (fn_info.cc) {
2310	.avr_interrupt,
2311	.csky_interrupt,
2312	=> "interrupt",
2313	.avr_signal => "signal",
2314	else => unreachable,
2315	}),
2316	.value = .empty,
2317	} }, &o.builder);
2318	},
2319	else => {},
2320	}
2321
2322	if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
2323
2324	var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types);
2325	if (try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type)) == .sret) {
2326	try o.addSRetFnAttributes(
2327	attributes,
2328	try o.lowerType(.fromInterned(fn_info.return_type), .in_memory),
2329	Type.fromInterned(fn_info.return_type).abiAlignment(zcu).toLlvm(),
2330	.declaration,
2331	);
2332	it.llvm_index += 1;
2333	} else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {
2334	.signed => try attributes.addRetAttr(.signext, &o.builder),
2335	.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
2336	};
2337
2338	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
2339	if (err_return_tracing) {
2340	try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2341	it.llvm_index += 1;
2342	}
2343
2344	var remaining_inreg_int = cc_info.inreg_int_params;
2345	var remaining_inreg_float = cc_info.inreg_float_params;
2346
2347	while (try it.next()) |lowering| switch (lowering) {
2348	.byval => {
2349	const param_index = it.zig_index - 1;
2350	const param_ty: Type = .fromInterned(fn_info.param_types[param_index]);
2351	if (!isByRef(param_ty, zcu)) {
2352	try o.addByValParamAttrs(pt, attributes, param_ty, param_index, fn_info, it.llvm_index - 1);
2353	}
2354
2355	if (remaining_inreg_int > 0 and
2356	(param_ty.isPtrAtRuntime(zcu) or
2357	(param_ty.isAbiInt(zcu) and param_ty.abiSize(zcu) <= Type.usize.abiSize(zcu))))
2358	{
2359	try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2360	remaining_inreg_int -= 1;
2361	}
2362
2363	if (remaining_inreg_float > 0 and
2364	param_ty.zigTypeTag(zcu) == .float)
2365	{
2366	try attributes.addParamAttr(it.llvm_index - 1, .inreg, &o.builder);
2367	remaining_inreg_float -= 1;
2368	}
2369	},
2370	.byref => {
2371	const param_ty: Type = .fromInterned(fn_info.param_types[it.zig_index - 1]);
2372	try o.addByRefParamAttrs(attributes, it.llvm_index - 1, it.byval_attr, param_ty);
2373	},
2374	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2375	.slice => {
2376	const param_ty: Type = .fromInterned(fn_info.param_types[it.zig_index - 1]);
2377	const ptr_info = param_ty.ptrInfo(zcu);
2378	const llvm_ptr_index = it.llvm_index - 2;
2379	if (std.math.cast(u5, it.zig_index - 1)) |i| {
2380	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
2381	try attributes.addParamAttr(llvm_ptr_index, .@"noalias", &o.builder);
2382	}
2383	}
2384	if (param_ty.zigTypeTag(zcu) != .optional and
2385	!ptr_info.flags.is_allowzero and
2386	ptr_info.flags.address_space == .generic)
2387	{
2388	try attributes.addParamAttr(llvm_ptr_index, .nonnull, &o.builder);
2389	}
2390	if (ptr_info.flags.is_const) {
2391	try attributes.addParamAttr(llvm_ptr_index, .readonly, &o.builder);
2392	}
2393	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
2394	else => |a| .wrap(a.toLlvm()),
2395	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
2396	};
2397	try attributes.addParamAttr(llvm_ptr_index, .{ .@"align" = elem_align }, &o.builder);
2398	},
2399	// No attributes needed for these.
2400	.no_bits,
2401	.abi_sized_int,
2402	.multiple_llvm_types,
2403	.float_array,
2404	.i32_array,
2405	.i64_array,
2406	=> continue,
2407	};
2408	}
2409
2410	pub fn addSRetFnAttributes(
2411	o: *Object,
2412	attributes: *Builder.FunctionAttributes.Wip,
2413	ret_ty: Builder.Type,
2414	ret_align: Builder.Alignment,
2415	location: enum { declaration, callsite },
2416	) Allocator.Error!void {
2417	try attributes.addParamAttr(0, .dead_on_unwind, &o.builder);
2418	switch (location) {
2419	.declaration => try attributes.addParamAttr(0, .@"noalias", &o.builder),
2420	.callsite => {},
2421	}
2422	try attributes.addParamAttr(0, .writeonly, &o.builder);
2423	try attributes.addParamAttr(0, .{ .captures = .none }, &o.builder);
2424	try attributes.addParamAttr(0, .{ .sret = ret_ty }, &o.builder);
2425	try attributes.addParamAttr(0, .{ .@"align" = .wrap(ret_align) }, &o.builder);
2426	}
2427
2428	pub const TypeRepr = enum {
2429	/// The representation of the type when it is being manipulated as a value in a function.
2430	/// e.g. Zig `u90` -> LLVM `i90`
2431	as_value,
2432	/// The representation of the type when it is loaded from or stored to memory.
2433	/// e.g. Zig `u90` -> LLVM `i96`
2434	memory_access,
2435	/// The representation of the type when it is in memory.
2436	/// e.g. Zig `u90` -> LLVM `[12 x i8]`
2437	in_memory,
2438	};
2439
2440	pub fn intType(o: *Object, bits: u16, repr: TypeRepr) Allocator.Error!Builder.Type {
2441	switch (repr) {
2442	.as_value => return o.builder.intType(bits),
2443	.memory_access, .in_memory => {},
2444	}
2445	const target = o.zcu.getTarget();
2446	const abi_size = std.zig.target.intByteSize(target, bits);
2447	const llvm_bit_width = @as(u20, 8) * abi_size;
2448	switch (repr) {
2449	.as_value => unreachable,
2450	.memory_access => {},
2451	.in_memory => {
2452	const zig_align = std.zig.target.intAlignment(target, bits);
2453	const llvm_align = o.builder.data_layout.getIntegerSpec(llvm_bit_width).abi_align;
2454	if (zig_align < llvm_align.toByteUnits().?) return o.builder.arrayType(abi_size, .i8);
2455	},
2456	}
2457	return o.builder.intType(llvm_bit_width);
2458	}
2459
2460	pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
2461	return o.intType(o.zcu.errorSetBits(), repr);
2462	}
2463
2464	pub const SoftF80Layout = struct {
2465	alignment: InternPool.Alignment,
2466	/// byte offset of u64 field
2467	mantissa_offset: u64,
2468	/// byte offset of u16 field
2469	exponent_offset: u64,
2470	llvm_fields_len: u32,
2471
2472	pub const LlvmFieldTag = enum { mantissa, exponent, padding };
2473	};
2474	pub fn softF80Layout(o: *Object, opts: struct {
2475	llvm_field_tags_buf: []SoftF80Layout.LlvmFieldTag = &.{},
2476	llvm_field_types_buf: []Builder.Type = &.{},
2477	}) Allocator.Error!SoftF80Layout {
2478	const zcu = o.zcu;
2479	const target = zcu.getTarget();
2480	assert(std.zig.target.compilerRtFloatAbi(target, 80) == .soft);
2481	// Current compiler rt soft abi, which is not yet affected by endianness for simplicity:
2482	//
2483	// typedef struct { uint64_t mantissa; uint16_t exponent; } f80;
2484	//
2485	var layout: SoftF80Layout = .{
2486	.alignment = Type.f80.abiAlignment(zcu),
2487	.mantissa_offset = undefined,
2488	.exponent_offset = undefined,
2489	.llvm_fields_len = 0,
2490	};
2491	var offset: u64 = 0;
2492	for ([2]SoftF80Layout.LlvmFieldTag{ .mantissa, .exponent }, [2]Type{ .u64, .u16 }) |field_tag, field_type| {
2493	const field_align = field_type.abiAlignment(zcu);
2494	assert(field_align.compareStrict(.lte, layout.alignment));
2495	const field_offset = field_align.forward(offset);
2496	switch (field_offset - offset) {
2497	0 => {},
2498	else => |padding| {
2499	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2500	opts.llvm_field_tags_buf[layout.llvm_fields_len] = .padding;
2501	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2502	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.builder.arrayType(padding, .i8);
2503	layout.llvm_fields_len += 1;
2504	},
2505	}
2506	switch (field_tag) {
2507	.mantissa => layout.mantissa_offset = field_offset,
2508	.exponent => layout.exponent_offset = field_offset,
2509	.padding => unreachable,
2510	}
2511	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2512	opts.llvm_field_tags_buf[layout.llvm_fields_len] = field_tag;
2513	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2514	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.lowerType(field_type, .in_memory);
2515	layout.llvm_fields_len += 1;
2516	offset = field_offset + field_type.abiSize(zcu);
2517	}
2518	const end = layout.alignment.forward(offset);
2519	assert(end == Type.f80.abiSize(zcu));
2520	switch (end - offset) {
2521	0 => {},
2522	else => |padding| {
2523	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2524	opts.llvm_field_tags_buf[layout.llvm_fields_len] = .padding;
2525	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2526	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.builder.arrayType(padding, .i8);
2527	layout.llvm_fields_len += 1;
2528	},
2529	}
2530	return layout;
2531	}
2532
2533	pub const SoftF128Layout = struct {
2534	alignment: InternPool.Alignment,
2535	/// byte offset of u64 field
2536	lo_offset: u64,
2537	/// byte offset of u64 field
2538	hi_offset: u64,
2539	llvm_fields_len: u32,
2540
2541	pub const LlvmFieldTag = enum { lo, hi, padding };
2542	};
2543	pub fn softF128Layout(o: *Object, opts: struct {
2544	llvm_field_tags_buf: []SoftF128Layout.LlvmFieldTag = &.{},
2545	llvm_field_types_buf: []Builder.Type = &.{},
2546	}) Allocator.Error!SoftF128Layout {
2547	const zcu = o.zcu;
2548	const target = zcu.getTarget();
2549	assert(std.zig.target.compilerRtFloatAbi(target, 128) == .soft);
2550	// Current compiler rt soft abi:
2551	//
2552	// #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
2553	// typedef struct { uint64_t hi, lo; } f128;
2554	// #else
2555	// typedef struct { uint64_t lo, hi; } f128;
2556	// #endif
2557	//
2558	var layout: SoftF128Layout = .{
2559	.alignment = Type.f128.abiAlignment(zcu),
2560	.lo_offset = undefined,
2561	.hi_offset = undefined,
2562	.llvm_fields_len = 0,
2563	};
2564	var offset: u64 = 0;
2565	for (@as([2]SoftF128Layout.LlvmFieldTag, switch (target.cpu.arch.endian()) {
2566	.big => .{ .hi, .lo },
2567	.little => .{ .lo, .hi },
2568	}), [2]Type{ .u64, .u64 }) |field_tag, field_type| {
2569	const field_align = field_type.abiAlignment(zcu);
2570	assert(field_align.compareStrict(.lte, layout.alignment));
2571	const field_offset = field_align.forward(offset);
2572	switch (field_offset - offset) {
2573	0 => {},
2574	else => |padding| {
2575	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2576	opts.llvm_field_tags_buf[layout.llvm_fields_len] = .padding;
2577	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2578	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.builder.arrayType(padding, .i8);
2579	layout.llvm_fields_len += 1;
2580	},
2581	}
2582	switch (field_tag) {
2583	.lo => layout.lo_offset = field_offset,
2584	.hi => layout.hi_offset = field_offset,
2585	.padding => unreachable,
2586	}
2587	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2588	opts.llvm_field_tags_buf[layout.llvm_fields_len] = field_tag;
2589	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2590	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.lowerType(field_type, .in_memory);
2591	layout.llvm_fields_len += 1;
2592	offset = field_offset + field_type.abiSize(zcu);
2593	}
2594	const end = layout.alignment.forward(offset);
2595	assert(end == Type.f128.abiSize(zcu));
2596	switch (end - offset) {
2597	0 => {},
2598	else => |padding| {
2599	if (layout.llvm_fields_len < opts.llvm_field_tags_buf.len)
2600	opts.llvm_field_tags_buf[layout.llvm_fields_len] = .padding;
2601	if (layout.llvm_fields_len < opts.llvm_field_types_buf.len)
2602	opts.llvm_field_types_buf[layout.llvm_fields_len] = try o.builder.arrayType(padding, .i8);
2603	layout.llvm_fields_len += 1;
2604	},
2605	}
2606	return layout;
2607	}
2608
2609	pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
2610	const zcu = o.zcu;
2611	const target = zcu.getTarget();
2612	const ip = &zcu.intern_pool;
2613
2614	switch (repr) {
2615	.as_value => assert(!isByRef(t, zcu)), // by-ref types must only be manipulated in memory
2616	.memory_access, .in_memory => {},
2617	}
2618
2619	return switch (t.toIntern()) {
2620	.u0_type => unreachable, // no runtime bits
2621	.u1_type, .bool_type => try o.intType(1, repr),
2622	.u8_type, .i8_type => try o.intType(8, repr),
2623	.u16_type, .i16_type => try o.intType(16, repr),
2624	.u29_type => try o.intType(29, repr),
2625	.u32_type, .i32_type => try o.intType(32, repr),
2626	.u64_type, .i64_type => try o.intType(64, repr),
2627	.u80_type => try o.intType(80, repr),
2628	.u128_type, .i128_type => try o.intType(128, repr),
2629	.usize_type, .isize_type => try o.intType(target.ptrBitWidth(), repr),
2630	.c_char_type => try o.intType(target.cTypeBitSize(.char).?, repr),
2631	.c_short_type => try o.intType(target.cTypeBitSize(.short).?, repr),
2632	.c_ushort_type => try o.intType(target.cTypeBitSize(.ushort).?, repr),
2633	.c_int_type => try o.intType(target.cTypeBitSize(.int).?, repr),
2634	.c_uint_type => try o.intType(target.cTypeBitSize(.uint).?, repr),
2635	.c_long_type => try o.intType(target.cTypeBitSize(.long).?, repr),
2636	.c_ulong_type => try o.intType(target.cTypeBitSize(.ulong).?, repr),
2637	.c_longlong_type => try o.intType(target.cTypeBitSize(.longlong).?, repr),
2638	.c_ulonglong_type => try o.intType(target.cTypeBitSize(.ulonglong).?, repr),
2639	.c_longdouble_type,
2640	.f16_type,
2641	.f32_type,
2642	.f64_type,
2643	.f80_type,
2644	.f128_type,
2645	=> switch (t.floatBits(target)) {
2646	16 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
2647	.hard => .half,
2648	.soft => .i16,
2649	},
2650	32 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
2651	.hard => .float,
2652	.soft => .i32,
2653	},
2654	64 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
2655	.hard => .double,
2656	.soft => .i64,
2657	},
2658	80 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
2659	.hard => .x86_fp80,
2660	.soft => {
2661	var llvm_field_types_buf: [5]Builder.Type = undefined;
2662	const f80_layout = try o.softF80Layout(.{
2663	.llvm_field_types_buf = &llvm_field_types_buf,
2664	});
2665	return o.builder.structType(
2666	.normal,
2667	llvm_field_types_buf[0..f80_layout.llvm_fields_len],
2668	);
2669	},
2670	},
2671	128 => |bits| switch (std.zig.target.compilerRtFloatAbi(target, bits)) {
2672	.hard => .fp128,
2673	.soft => {
2674	var llvm_field_types_buf: [5]Builder.Type = undefined;
2675	const f128_layout = try o.softF128Layout(.{
2676	.llvm_field_types_buf = &llvm_field_types_buf,
2677	});
2678	return o.builder.structType(
2679	.normal,
2680	llvm_field_types_buf[0..f128_layout.llvm_fields_len],
2681	);
2682	},
2683	},
2684	else => unreachable,
2685	},
2686	.anyopaque_type => {
2687	// This is unreachable except when used as the type for an extern global.
2688	// For example: `@extern(*anyopaque, .{ .name = "foo"})` should produce
2689	// @foo = external global i8
2690	return .i8;
2691	},
2692	.anyerror_type => try o.errorIntType(repr),
2693	.void_type => unreachable, // no runtime bits
2694	.type_type => unreachable, // no runtime bits
2695	.comptime_int_type => unreachable, // no runtime bits
2696	.comptime_float_type => unreachable, // no runtime bits
2697	.noreturn_type => unreachable, // no runtime bits
2698	.null_type => unreachable, // no runtime bits
2699	.undefined_type => unreachable, // no runtime bits
2700	.enum_literal_type => unreachable, // no runtime bits
2701	.optional_noreturn_type => unreachable, // no runtime bits
2702	.empty_tuple_type => unreachable, // no runtime bits
2703	.anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
2704	.ptr_usize_type,
2705	.ptr_const_comptime_int_type,
2706	.manyptr_u8_type,
2707	.manyptr_const_u8_type,
2708	.manyptr_const_u8_sentinel_0_type,
2709	=> .ptr,
2710	.slice_const_u8_type,
2711	.slice_const_u8_sentinel_0_type,
2712	=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
2713	.anyerror_void_error_union_type,
2714	.adhoc_inferred_error_set_type,
2715	=> try o.errorIntType(repr),
2716	.generic_poison_type => unreachable,
2717	// values, not types
2718	.undef,
2719	.undef_bool,
2720	.undef_usize,
2721	.undef_u1,
2722	.zero,
2723	.zero_usize,
2724	.zero_u1,
2725	.zero_u8,
2726	.one,
2727	.one_usize,
2728	.one_u1,
2729	.one_u8,
2730	.four_u8,
2731	.negative_one,
2732	.void_value,
2733	.unreachable_value,
2734	.null_value,
2735	.bool_true,
2736	.bool_false,
2737	.empty_tuple,
2738	.none,
2739	=> unreachable,
2740	else => switch (ip.indexToKey(t.toIntern())) {
2741	.int_type => |int_type| o.intType(int_type.bits, repr),
2742	.ptr_type => |ptr_type| type: {
2743	const ptr_ty = try o.builder.ptrType(
2744	toLlvmAddressSpace(ptr_type.flags.address_space, target),
2745	);
2746	break :type switch (ptr_type.flags.size) {
2747	.one, .many, .c => ptr_ty,
2748	.slice => try o.builder.structType(.normal, &.{
2749	ptr_ty,
2750	try o.lowerType(.usize, repr),
2751	}),
2752	};
2753	},
2754	.array_type => |array_type| o.builder.arrayType(
2755	array_type.lenIncludingSentinel(),
2756	try o.lowerType(.fromInterned(array_type.child), repr),
2757	),
2758	.vector_type => |vector_type| if (isByRef(t, zcu)) {
2759	const child_llvm_ty = try o.lowerType(.fromInterned(vector_type.child), repr);
2760	return o.builder.arrayType(vector_type.len, child_llvm_ty);
2761	} else {
2762	const child_llvm_ty = try o.lowerType(.fromInterned(vector_type.child), .as_value);
2763	return o.builder.vectorType(.normal, vector_type.len, child_llvm_ty);
2764	},
2765	.opt_type => |child_ty| {
2766	// Must stay in sync with `opt_payload` logic in `lowerPtr`.
2767	switch (Type.fromInterned(child_ty).classify(zcu)) {
2768	.no_possible_value, .fully_comptime => unreachable,
2769	.one_possible_value => return .i8,
2770	.runtime, .partially_comptime => {},
2771	}
2772
2773	if (t.optionalReprIsPayload(zcu)) {
2774	return o.lowerType(.fromInterned(child_ty), repr);
2775	}
2776
2777	const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
2778
2779	comptime assert(optional_layout_version == 3);
2780	var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
2781	var fields_len: usize = 2;
2782	const offset = Type.fromInterned(child_ty).abiSize(zcu) + 1;
2783	const abi_size = t.abiSize(zcu);
2784	const padding_len = abi_size - offset;
2785	if (padding_len > 0) {
2786	fields[2] = try o.builder.arrayType(padding_len, .i8);
2787	fields_len = 3;
2788	}
2789	return o.builder.structType(.normal, fields[0..fields_len]);
2790	},
2791	.anyframe_type => @panic("TODO implement lowerType for AnyFrame types"),
2792	.error_union_type => |error_union_type| {
2793	// Must stay in sync with `codegen.errUnionPayloadOffset`.
2794	// See logic in `lowerPtr`.
2795	const error_type = try o.errorIntType(repr);
2796
2797	switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
2798	.fully_comptime => unreachable,
2799	.no_possible_value, .one_possible_value => return error_type,
2800	.runtime, .partially_comptime => {},
2801	}
2802
2803	const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
2804
2805	const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
2806	const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
2807
2808	const payload_size = Type.fromInterned(error_union_type.payload_type).abiSize(zcu);
2809	const error_size = std.zig.target.intByteSize(target, zcu.errorSetBits());
2810
2811	var fields: [3]Builder.Type = undefined;
2812	var fields_len: usize = 2;
2813	const padding_len = if (error_align.compare(.gt, payload_align)) pad: {
2814	fields[0] = error_type;
2815	fields[1] = payload_type;
2816	const payload_end =
2817	payload_align.forward(error_size) +
2818	payload_size;
2819	const abi_size = error_align.forward(payload_end);
2820	break :pad abi_size - payload_end;
2821	} else pad: {
2822	fields[0] = payload_type;
2823	fields[1] = error_type;
2824	const error_end =
2825	error_align.forward(payload_size) +
2826	error_size;
2827	const abi_size = payload_align.forward(error_end);
2828	break :pad abi_size - error_end;
2829	};
2830	if (padding_len > 0) {
2831	fields[2] = try o.builder.arrayType(padding_len, .i8);
2832	fields_len = 3;
2833	}
2834	return o.builder.structType(.normal, fields[0..fields_len]);
2835	},
2836	.simple_type => unreachable,
2837	.struct_type => {
2838	const struct_type = ip.loadStructType(t.toIntern());
2839
2840	if (struct_type.layout == .@"packed") {
2841	return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
2842	}
2843
2844	if (o.type_map.get(t.toIntern())) |value| return value;
2845
2846	assert(struct_type.size > 0);
2847
2848	var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
2849	defer llvm_field_types.deinit(o.gpa);
2850	// Although we can estimate how much capacity to add, these cannot be
2851	// relied upon because of the recursive calls to lowerType below.
2852	try llvm_field_types.ensureUnusedCapacity(o.gpa, struct_type.field_types.len);
2853
2854	comptime assert(struct_layout_version == 2);
2855	var offset: u64 = 0;
2856	var struct_kind: Builder.Type.Structure.Kind = .normal;
2857	// When we encounter a zero-bit field, we place it here so we know to map it to the next non-zero-bit field (if any).
2858	var it = struct_type.iterateRuntimeOrder(ip);
2859	var max_field_ty_align: InternPool.Alignment = .@"1";
2860	while (it.next()) |field_index| {
2861	const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[field_index]);
2862	const field_ty_align = field_ty.abiAlignment(zcu);
2863	max_field_ty_align = max_field_ty_align.maxStrict(field_ty_align);
2864
2865	const prev_offset = offset;
2866	offset = struct_type.field_offsets.get(ip)[field_index];
2867	if (@ctz(offset) < field_ty_align.toLog2Units()) {
2868	struct_kind = .@"packed"; // prevent unexpected padding before this field
2869	}
2870
2871	const padding_len = offset - prev_offset;
2872	if (padding_len > 0) try llvm_field_types.append(
2873	o.gpa,
2874	try o.builder.arrayType(padding_len, .i8),
2875	);
2876
2877	if (!field_ty.hasRuntimeBits(zcu)) continue;
2878
2879	try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
2880
2881	offset += field_ty.abiSize(zcu);
2882	}
2883	{
2884	const prev_offset = offset;
2885	offset = struct_type.alignment.forward(offset);
2886	const padding_len = offset - prev_offset;
2887	if (padding_len > 0) try llvm_field_types.append(
2888	o.gpa,
2889	try o.builder.arrayType(padding_len, .i8),
2890	);
2891	if (@ctz(offset) < max_field_ty_align.toLog2Units()) {
2892	struct_kind = .@"packed"; // prevent unexpected trailing padding
2893	}
2894	}
2895
2896	const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
2897	try o.type_map.put(o.gpa, t.toIntern(), ty);
2898
2899	o.builder.namedTypeSetBody(
2900	ty,
2901	try o.builder.structType(struct_kind, llvm_field_types.items),
2902	);
2903	return ty;
2904	},
2905	.tuple_type => |tuple_type| {
2906	var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
2907	defer llvm_field_types.deinit(o.gpa);
2908	// Although we can estimate how much capacity to add, these cannot be
2909	// relied upon because of the recursive calls to lowerType below.
2910	try llvm_field_types.ensureUnusedCapacity(o.gpa, tuple_type.types.len);
2911
2912	comptime assert(struct_layout_version == 2);
2913	var offset: u64 = 0;
2914	var big_align: InternPool.Alignment = .@"1";
2915
2916	for (
2917	tuple_type.types.get(ip),
2918	tuple_type.values.get(ip),
2919	) |field_ty, field_val| {
2920	if (field_val != .none) continue;
2921
2922	const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
2923	big_align = big_align.max(field_align);
2924	const prev_offset = offset;
2925	offset = field_align.forward(offset);
2926
2927	const padding_len = offset - prev_offset;
2928	if (padding_len > 0) try llvm_field_types.append(
2929	o.gpa,
2930	try o.builder.arrayType(padding_len, .i8),
2931	);
2932	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
2933	continue;
2934	}
2935	try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
2936
2937	offset += Type.fromInterned(field_ty).abiSize(zcu);
2938	}
2939	{
2940	const prev_offset = offset;
2941	offset = big_align.forward(offset);
2942	const padding_len = offset - prev_offset;
2943	if (padding_len > 0) try llvm_field_types.append(
2944	o.gpa,
2945	try o.builder.arrayType(padding_len, .i8),
2946	);
2947	}
2948	assert(offset > 0);
2949	return o.builder.structType(.normal, llvm_field_types.items);
2950	},
2951	.union_type => {
2952	const union_obj = ip.loadUnionType(t.toIntern());
2953
2954	if (union_obj.layout == .@"packed") {
2955	return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
2956	}
2957
2958	const layout = Type.getUnionLayout(union_obj, zcu);
2959
2960	if (layout.payload_size == 0) {
2961	return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
2962	}
2963
2964	if (o.type_map.get(t.toIntern())) |value| return value;
2965
2966	assert(union_obj.size > 0);
2967
2968	const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
2969	const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
2970
2971	const payload_ty = ty: {
2972	if (layout.most_aligned_field_size == layout.payload_size) {
2973	break :ty aligned_field_llvm_ty;
2974	}
2975	const padding_len = if (layout.tag_size == 0)
2976	layout.abi_size - layout.most_aligned_field_size
2977	else
2978	layout.payload_size - layout.most_aligned_field_size;
2979	break :ty try o.builder.structType(.@"packed", &.{
2980	aligned_field_llvm_ty,
2981	try o.builder.arrayType(padding_len, .i8),
2982	});
2983	};
2984
2985	if (layout.tag_size == 0) {
2986	const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
2987	try o.type_map.put(o.gpa, t.toIntern(), ty);
2988
2989	o.builder.namedTypeSetBody(
2990	ty,
2991	try o.builder.structType(.normal, &.{payload_ty}),
2992	);
2993	return ty;
2994	}
2995	const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
2996
2997	// Put the tag before or after the payload depending on which one's
2998	// alignment is greater.
2999	var llvm_fields: [3]Builder.Type = undefined;
3000	var llvm_fields_len: usize = 2;
3001
3002	if (layout.tag_align.compare(.gte, layout.payload_align)) {
3003	llvm_fields = .{ enum_tag_ty, payload_ty, .none };
3004	} else {
3005	llvm_fields = .{ payload_ty, enum_tag_ty, .none };
3006	}
3007
3008	// Insert padding to make the LLVM struct ABI size match the Zig union ABI size.
3009	if (layout.padding != 0) {
3010	llvm_fields[llvm_fields_len] = try o.builder.arrayType(layout.padding, .i8);
3011	llvm_fields_len += 1;
3012	}
3013
3014	const ty = try o.builder.opaqueType(try o.builder.string(t.containerTypeName(ip).toSlice(ip)));
3015	try o.type_map.put(o.gpa, t.toIntern(), ty);
3016
3017	o.builder.namedTypeSetBody(
3018	ty,
3019	try o.builder.structType(.normal, llvm_fields[0..llvm_fields_len]),
3020	);
3021	return ty;
3022	},
3023	.opaque_type, .spirv_type => unreachable, // no runtime bits
3024	.enum_type => try o.intType(t.backingIntType(zcu).intInfo(zcu).bits, repr),
3025	.func_type => |func_type| {
3026	assert(t.fnHasRuntimeBits(zcu));
3027	return o.lowerFnType(.fromIntern(func_type, ip));
3028	},
3029	.error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
3030	// values, not types
3031	.undef,
3032	.simple_value,
3033	.@"extern",
3034	.func,
3035	.int,
3036	.err,
3037	.error_union,
3038	.enum_literal,
3039	.enum_tag,
3040	.float,
3041	.ptr,
3042	.slice,
3043	.opt,
3044	.aggregate,
3045	.un,
3046	.bitpack,
3047	// memoization, not types
3048	.memoized_call,
3049	=> unreachable,
3050	},
3051	};
3052	}
3053
3054	pub const FuncInfo = struct {
3055	cc: std.lang.CallingConvention,
3056	noalias_bits: u32 = 0,
3057	param_types: []const InternPool.Index,
3058	return_type: InternPool.Index = .void_type,
3059	is_var_args: bool = false,
3060
3061	pub fn fromIntern(fn_info: InternPool.Key.FuncType, ip: *InternPool) FuncInfo {
3062	return .{
3063	.cc = fn_info.cc,
3064	.noalias_bits = fn_info.noalias_bits,
3065	.param_types = fn_info.param_types.get(ip),
3066	.return_type = fn_info.return_type,
3067	.is_var_args = fn_info.is_var_args,
3068	};
3069	}
3070	};
3071	pub fn lowerFnType(o: *Object, fn_info: FuncInfo) Allocator.Error!Builder.Type {
3072	const zcu = o.zcu;
3073	const target = zcu.getTarget();
3074
3075	const ret_strat = try fnReturnStrat(o, fn_info.cc, .fromInterned(fn_info.return_type));
3076
3077	var llvm_params: std.ArrayList(Builder.Type) = .empty;
3078	defer llvm_params.deinit(o.gpa);
3079
3080	if (ret_strat == .sret) {
3081	try llvm_params.append(o.gpa, .ptr);
3082	}
3083
3084	if (fn_info.cc == .auto and zcu.comp.config.any_error_tracing) {
3085	// First parameter is a pointer to `std.lang.StackTrace`.
3086	const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(.generic, target));
3087	try llvm_params.append(o.gpa, llvm_ptr_ty);
3088	}
3089
3090	var it = iterateParamTypes(o, fn_info.cc, fn_info.param_types);
3091	while (try it.next()) |lowering| switch (lowering) {
3092	.no_bits => continue,
3093	.byval => {
3094	const param_ty = Type.fromInterned(fn_info.param_types[it.zig_index - 1]);
3095	try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .memory_access else .as_value));
3096	},
3097	.byref, .byref_mut => {
3098	try llvm_params.append(o.gpa, .ptr);
3099	},
3100	.abi_sized_int => {
3101	const param_ty = Type.fromInterned(fn_info.param_types[it.zig_index - 1]);
3102	try llvm_params.append(o.gpa, try o.builder.intType(
3103	@intCast(param_ty.abiSize(zcu) * 8),
3104	));
3105	},
3106	.slice => {
3107	const param_ty = Type.fromInterned(fn_info.param_types[it.zig_index - 1]);
3108	try llvm_params.appendSlice(o.gpa, &.{
3109	try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3110	try o.lowerType(.usize, .as_value),
3111	});
3112	},
3113	.multiple_llvm_types => {
3114	try llvm_params.appendSlice(o.gpa, it.types_buffer[0..it.types_len]);
3115	},
3116	.float_array => |count| {
3117	const param_ty = Type.fromInterned(fn_info.param_types[it.zig_index - 1]);
3118	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .memory_access);
3119	try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3120	},
3121	.i32_array, .i64_array => |arr_len| {
3122	try llvm_params.append(o.gpa, try o.builder.arrayType(arr_len, switch (lowering) {
3123	.i32_array => .i32,
3124	.i64_array => .i64,
3125	else => unreachable,
3126	}));
3127	},
3128	};
3129
3130	const llvm_ret_ty: Builder.Type = switch (ret_strat) {
3131	.void, .sret => .void,
3132	.by_val => try o.lowerType(.fromInterned(fn_info.return_type), .as_value),
3133	.mem_cast => |llvm_ret_ty| llvm_ret_ty,
3134	};
3135	const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
3136	true => .vararg,
3137	false => .normal,
3138	};
3139	return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
3140	}
3141
3142	pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
3143	const zcu = o.zcu;
3144	const ip = &zcu.intern_pool;
3145	const target = zcu.getTarget();
3146
3147	const val: Value = .fromInterned(arg_val);
3148	const val_key = ip.indexToKey(val.toIntern());
3149
3150	const ty: Type = .fromInterned(val_key.typeOf());
3151	ty.assertHasLayout(zcu);
3152	assert(ty.hasRuntimeBits(zcu));
3153
3154	return switch (val_key) {
3155	.int_type,
3156	.ptr_type,
3157	.array_type,
3158	.vector_type,
3159	.opt_type,
3160	.anyframe_type,
3161	.error_union_type,
3162	.simple_type,
3163	.struct_type,
3164	.tuple_type,
3165	.union_type,
3166	.opaque_type,
3167	.spirv_type,
3168	.enum_type,
3169	.func_type,
3170	.error_set_type,
3171	.inferred_error_set_type,
3172	=> unreachable, // types, not values
3173
3174	.undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
3175	.simple_value => |simple_value| switch (simple_value) {
3176	.void => unreachable, // non-runtime value
3177	.null => unreachable, // non-runtime value
3178	.@"unreachable" => unreachable, // non-runtime value
3179
3180	.false => switch (repr) {
3181	.as_value => .false,
3182	.in_memory, .memory_access => try o.builder.intConst(.i8, 0),
3183	},
3184	.true => switch (repr) {
3185	.as_value => .true,
3186	.in_memory, .memory_access => try o.builder.intConst(.i8, 1),
3187	},
3188	},
3189	.enum_literal => unreachable, // non-runtime value
3190	.@"extern" => unreachable, // non-runtime value
3191	.func => unreachable, // non-runtime value
3192	.int => {
3193	var bigint_space: Value.BigIntSpace = undefined;
3194	const bigint = val.toBigInt(&bigint_space, zcu);
3195	const llvm_int_ty = try o.lowerType(ty, repr);
3196	if (llvm_int_ty.isInteger(&o.builder))
3197	return o.builder.bigIntConst(llvm_int_ty, bigint);
3198	const buffer = try o.gpa.alloc(u8, llvm_int_ty.aggregateLen(&o.builder));
3199	defer o.gpa.free(buffer);
3200	bigint.writeTwosComplement(buffer, target.cpu.arch.endian());
3201	return o.builder.stringConst(try o.builder.string(buffer));
3202	},
3203	.err => |err| {
3204	const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
3205	return o.builder.intConst(try o.errorIntType(repr), int);
3206	},
3207	.error_union => |error_union| {
3208	const llvm_error_ty = try o.errorIntType(repr);
3209	const llvm_error_value = switch (error_union.val) {
3210	.err_name => |name| try o.builder.intConst(
3211	llvm_error_ty,
3212	zcu.intern_pool.getErrorValueIfExists(name).?,
3213	),
3214	.payload => try o.builder.intConst(llvm_error_ty, 0),
3215	};
3216
3217	const payload_type = ty.errorUnionPayload(zcu);
3218	if (!payload_type.hasRuntimeBits(zcu)) {
3219	// We use the error type directly as the type.
3220	return llvm_error_value;
3221	}
3222
3223	const payload_align = payload_type.abiAlignment(zcu);
3224	const error_align = Type.errorAbiAlignment(zcu);
3225	const llvm_payload_value = switch (error_union.val) {
3226	.err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
3227	.payload => |payload| try o.lowerValue(payload, repr),
3228	};
3229
3230	var fields: [3]Builder.Type = undefined;
3231	var vals: [3]Builder.Constant = undefined;
3232	if (error_align.compare(.gt, payload_align)) {
3233	vals[0] = llvm_error_value;
3234	vals[1] = llvm_payload_value;
3235	} else {
3236	vals[0] = llvm_payload_value;
3237	vals[1] = llvm_error_value;
3238	}
3239	fields[0] = vals[0].typeOf(&o.builder);
3240	fields[1] = vals[1].typeOf(&o.builder);
3241
3242	const llvm_ty = try o.lowerType(ty, repr);
3243	const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3244	if (llvm_ty_fields.len > 2) {
3245	assert(llvm_ty_fields.len == 3);
3246	fields[2] = llvm_ty_fields[2];
3247	vals[2] = try o.builder.undefConst(fields[2]);
3248	}
3249	return o.builder.structConst(try o.builder.structType(
3250	llvm_ty.structKind(&o.builder),
3251	fields[0..llvm_ty_fields.len],
3252	), vals[0..llvm_ty_fields.len]);
3253	},
3254	.enum_tag => |enum_tag| o.lowerValue(enum_tag.int, repr),
3255	.float => switch (ty.floatBits(target)) {
3256	else => unreachable,
3257	16 => try o.f16Const(val.toFloat(f16, zcu)),
3258	32 => try o.f32Const(val.toFloat(f32, zcu)),
3259	64 => try o.f64Const(val.toFloat(f64, zcu)),
3260	80 => try o.f80Const(val.toFloat(f80, zcu)),
3261	128 => try o.f128Const(val.toFloat(f128, zcu)),
3262	},
3263	.ptr => try o.lowerPtr(arg_val, 0),
3264	.slice => |slice| return o.builder.structConst(try o.lowerType(ty, repr), &.{
3265	try o.lowerValue(slice.ptr, repr),
3266	try o.lowerValue(slice.len, repr),
3267	}),
3268	.opt => |opt| {
3269	comptime assert(optional_layout_version == 3);
3270	const payload_ty = ty.optionalChild(zcu);
3271
3272	const non_null_bit = try o.builder.intConst(.i8, @intFromBool(opt.val != .none));
3273	if (!payload_ty.hasRuntimeBits(zcu)) {
3274	return non_null_bit;
3275	}
3276	const llvm_ty = try o.lowerType(ty, repr);
3277	if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3278	.none => switch (llvm_ty.tag(&o.builder)) {
3279	.integer => try o.builder.intConst(llvm_ty, 0),
3280	.pointer => try o.builder.nullConst(llvm_ty),
3281	.structure => try o.builder.zeroInitConst(llvm_ty),
3282	else => unreachable,
3283	},
3284	else => |payload| try o.lowerValue(payload, repr),
3285	};
3286	assert(payload_ty.zigTypeTag(zcu) != .@"fn");
3287
3288	var fields: [3]Builder.Type = undefined;
3289	var vals: [3]Builder.Constant = undefined;
3290	vals[0] = switch (opt.val) {
3291	.none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
3292	else => |payload| try o.lowerValue(payload, repr),
3293	};
3294	vals[1] = non_null_bit;
3295	fields[0] = vals[0].typeOf(&o.builder);
3296	fields[1] = vals[1].typeOf(&o.builder);
3297
3298	const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3299	if (llvm_ty_fields.len > 2) {
3300	assert(llvm_ty_fields.len == 3);
3301	fields[2] = llvm_ty_fields[2];
3302	vals[2] = try o.builder.undefConst(fields[2]);
3303	}
3304	return o.builder.structConst(try o.builder.structType(
3305	llvm_ty.structKind(&o.builder),
3306	fields[0..llvm_ty_fields.len],
3307	), vals[0..llvm_ty_fields.len]);
3308	},
3309	.bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val, repr),
3310	.aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {
3311	.array_type => |array_type| switch (aggregate.storage) {
3312	.bytes => |bytes| try o.builder.stringConst(try o.builder.string(
3313	bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3314	)),
3315	.elems => |elems| {
3316	const array_ty = try o.lowerType(ty, repr);
3317	const elem_ty = array_ty.childType(&o.builder);
3318	assert(elems.len == array_ty.aggregateLen(&o.builder));
3319
3320	const ExpectedContents = extern struct {
3321	vals: [Builder.expected_fields_len]Builder.Constant,
3322	fields: [Builder.expected_fields_len]Builder.Type,
3323	};
3324	var bfa_buf: ExpectedContents = undefined;
3325	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), o.gpa);
3326	const allocator = bfa.allocator();
3327	const vals = try allocator.alloc(Builder.Constant, elems.len);
3328	defer allocator.free(vals);
3329	const fields = try allocator.alloc(Builder.Type, elems.len);
3330	defer allocator.free(fields);
3331
3332	var need_unnamed = false;
3333	for (vals, fields, elems) |*result_val, *result_field, elem| {
3334	result_val.* = try o.lowerValue(elem, repr);
3335	result_field.* = result_val.typeOf(&o.builder);
3336	if (result_field.* != elem_ty) need_unnamed = true;
3337	}
3338	return if (need_unnamed) try o.builder.structConst(
3339	try o.builder.structType(.normal, fields),
3340	vals,
3341	) else try o.builder.arrayConst(array_ty, vals);
3342	},
3343	.repeated_elem => |elem| {
3344	const len: usize = @intCast(array_type.len);
3345	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3346	const array_ty = try o.lowerType(ty, repr);
3347	const elem_ty = array_ty.childType(&o.builder);
3348
3349	const ExpectedContents = extern struct {
3350	vals: [Builder.expected_fields_len]Builder.Constant,
3351	fields: [Builder.expected_fields_len]Builder.Type,
3352	};
3353	var bfa_buf: ExpectedContents = undefined;
3354	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), o.gpa);
3355	const allocator = bfa.allocator();
3356	const vals = try allocator.alloc(Builder.Constant, len_including_sentinel);
3357	defer allocator.free(vals);
3358	const fields = try allocator.alloc(Builder.Type, len_including_sentinel);
3359	defer allocator.free(fields);
3360
3361	var need_unnamed = false;
3362	@memset(vals[0..len], try o.lowerValue(elem, repr));
3363	@memset(fields[0..len], vals[0].typeOf(&o.builder));
3364	if (fields[0] != elem_ty) need_unnamed = true;
3365
3366	if (array_type.sentinel != .none) {
3367	vals[len] = try o.lowerValue(array_type.sentinel, repr);
3368	fields[len] = vals[len].typeOf(&o.builder);
3369	if (fields[len] != elem_ty) need_unnamed = true;
3370	}
3371
3372	return if (need_unnamed) try o.builder.structConst(
3373	try o.builder.structType(.@"packed", fields),
3374	vals,
3375	) else try o.builder.arrayConst(array_ty, vals);
3376	},
3377	},
3378	.vector_type => |vector_type| {
3379	const vector_ty = try o.lowerType(ty, repr);
3380	const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
3381	var bfa_buf: ExpectedContents = undefined;
3382	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), o.gpa);
3383	const allocator = bfa.allocator();
3384	const is_by_ref = isByRef(ty, zcu);
3385	switch (aggregate.storage) {
3386	.bytes, .elems => {
3387	const vals = try allocator.alloc(Builder.Constant, vector_type.len);
3388	defer allocator.free(vals);
3389
3390	switch (aggregate.storage) {
3391	.bytes => |bytes| for (vals, bytes.toSlice(vector_type.len, ip)) |*result_val, byte| {
3392	result_val.* = try o.builder.intConst(.i8, byte);
3393	},
3394	.elems => |elems| for (vals, elems) |*result_val, elem| {
3395	result_val.* = try o.lowerValue(elem, if (is_by_ref) repr else .as_value);
3396	},
3397	.repeated_elem => unreachable,
3398	}
3399	return if (is_by_ref)
3400	o.builder.arrayConst(vector_ty, vals)
3401	else
3402	o.builder.vectorConst(vector_ty, vals);
3403	},
3404	.repeated_elem => |elem| if (is_by_ref) {
3405	const vals = try allocator.alloc(Builder.Constant, vector_type.len);
3406	defer allocator.free(vals);
3407	@memset(vals, try o.lowerValue(elem, repr));
3408	return o.builder.arrayConst(vector_ty, vals);
3409	} else return o.builder.splatConst(vector_ty, try o.lowerValue(elem, .as_value)),
3410	}
3411	},
3412	.tuple_type => |tuple| {
3413	const struct_ty = try o.lowerType(ty, repr);
3414	const llvm_len = struct_ty.aggregateLen(&o.builder);
3415
3416	const ExpectedContents = extern struct {
3417	vals: [Builder.expected_fields_len]Builder.Constant,
3418	fields: [Builder.expected_fields_len]Builder.Type,
3419	};
3420	var bfa_buf: ExpectedContents = undefined;
3421	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), o.gpa);
3422	const allocator = bfa.allocator();
3423	const vals = try allocator.alloc(Builder.Constant, llvm_len);
3424	defer allocator.free(vals);
3425	const fields = try allocator.alloc(Builder.Type, llvm_len);
3426	defer allocator.free(fields);
3427
3428	comptime assert(struct_layout_version == 2);
3429	var llvm_index: usize = 0;
3430	var offset: u64 = 0;
3431	var big_align: InternPool.Alignment = .@"1";
3432	var need_unnamed = false;
3433	for (
3434	tuple.types.get(ip),
3435	tuple.values.get(ip),
3436	0..,
3437	) |field_ty, field_comptime_val, field_index| {
3438	if (field_comptime_val != .none) continue;
3439	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) continue;
3440
3441	const field_align = Type.fromInterned(field_ty).abiAlignment(zcu);
3442	big_align = big_align.max(field_align);
3443	const prev_offset = offset;
3444	offset = field_align.forward(offset);
3445
3446	const padding_len = offset - prev_offset;
3447	if (padding_len > 0) {
3448	// TODO make this and all other padding elsewhere in debug
3449	// builds be 0xaa not undef.
3450	fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
3451	vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
3452	assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
3453	llvm_index += 1;
3454	}
3455
3456	vals[llvm_index] = switch (aggregate.storage) {
3457	.bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3458	.elems => |elems| try o.lowerValue(elems[field_index], repr),
3459	.repeated_elem => |elem| try o.lowerValue(elem, repr),
3460	};
3461	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3462	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
3463	need_unnamed = true;
3464	llvm_index += 1;
3465
3466	offset += Type.fromInterned(field_ty).abiSize(zcu);
3467	}
3468	{
3469	const prev_offset = offset;
3470	offset = big_align.forward(offset);
3471	const padding_len = offset - prev_offset;
3472	if (padding_len > 0) {
3473	fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
3474	vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
3475	assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
3476	llvm_index += 1;
3477	}
3478	}
3479	assert(llvm_index == llvm_len);
3480
3481	return o.builder.structConst(if (need_unnamed)
3482	try o.builder.structType(struct_ty.structKind(&o.builder), fields)
3483	else
3484	struct_ty, vals);
3485	},
3486	.struct_type => {
3487	const struct_type = ip.loadStructType(ty.toIntern());
3488	const struct_ty = try o.lowerType(ty, repr);
3489	assert(struct_type.layout != .@"packed");
3490	const llvm_len = struct_ty.aggregateLen(&o.builder);
3491
3492	const ExpectedContents = extern struct {
3493	vals: [Builder.expected_fields_len]Builder.Constant,
3494	fields: [Builder.expected_fields_len]Builder.Type,
3495	};
3496	var bfa_buf: ExpectedContents = undefined;
3497	var bfa: std.heap.BufferFirstAllocator = .init(@ptrCast(&bfa_buf), o.gpa);
3498	const allocator = bfa.allocator();
3499	const vals = try allocator.alloc(Builder.Constant, llvm_len);
3500	defer allocator.free(vals);
3501	const fields = try allocator.alloc(Builder.Type, llvm_len);
3502	defer allocator.free(fields);
3503
3504	comptime assert(struct_layout_version == 2);
3505	var llvm_index: usize = 0;
3506	var offset: u64 = 0;
3507	var need_unnamed = false;
3508	var field_it = struct_type.iterateRuntimeOrder(ip);
3509	while (field_it.next()) |field_index| {
3510	const field_ty = Type.fromInterned(struct_type.field_types.get(ip)[field_index]);
3511	const prev_offset = offset;
3512	offset = struct_type.field_offsets.get(ip)[field_index];
3513
3514	const padding_len = offset - prev_offset;
3515	if (padding_len > 0) {
3516	// TODO make this and all other padding elsewhere in debug
3517	// builds be 0xaa not undef.
3518	fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
3519	vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
3520	assert(fields[llvm_index] ==
3521	struct_ty.structFields(&o.builder)[llvm_index]);
3522	llvm_index += 1;
3523	}
3524
3525	if (!field_ty.hasRuntimeBits(zcu)) {
3526	// This is a zero-bit field - we only needed it for the alignment.
3527	continue;
3528	}
3529
3530	vals[llvm_index] = switch (aggregate.storage) {
3531	.bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3532	.elems => |elems| try o.lowerValue(elems[field_index], repr),
3533	.repeated_elem => |elem| try o.lowerValue(elem, repr),
3534	};
3535	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3536	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
3537	need_unnamed = true;
3538	llvm_index += 1;
3539
3540	offset += field_ty.abiSize(zcu);
3541	}
3542	{
3543	const prev_offset = offset;
3544	offset = struct_type.alignment.forward(offset);
3545	const padding_len = offset - prev_offset;
3546	if (padding_len > 0) {
3547	fields[llvm_index] = try o.builder.arrayType(padding_len, .i8);
3548	vals[llvm_index] = try o.builder.undefConst(fields[llvm_index]);
3549	assert(fields[llvm_index] == struct_ty.structFields(&o.builder)[llvm_index]);
3550	llvm_index += 1;
3551	}
3552	}
3553	assert(llvm_index == llvm_len);
3554
3555	return o.builder.structConst(if (need_unnamed)
3556	try o.builder.structType(struct_ty.structKind(&o.builder), fields)
3557	else
3558	struct_ty, vals);
3559	},
3560	else => unreachable,
3561	},
3562	.un => |un| {
3563	const union_ty = try o.lowerType(ty, repr);
3564	const layout = ty.unionGetLayout(zcu);
3565	if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
3566
3567	const union_obj = zcu.typeToUnion(ty).?;
3568	const container_layout = union_obj.layout;
3569	assert(container_layout != .@"packed");
3570
3571	var need_unnamed = false;
3572	const payload = if (un.tag != .none) p: {
3573	const field_index = zcu.unionTagFieldIndex(union_obj, Value.fromInterned(un.tag)).?;
3574	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[field_index]);
3575
3576	// Sometimes we must make an unnamed struct because LLVM does
3577	// not support bitcasting our payload struct to the true union payload type.
3578	// Instead we use an unnamed struct and every reference to the global
3579	// must pointer cast to the expected type before accessing the union.
3580	need_unnamed = layout.most_aligned_field != field_index;
3581
3582	if (!field_ty.hasRuntimeBits(zcu)) {
3583	const padding_len = layout.payload_size;
3584	break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3585	}
3586	const payload = try o.lowerValue(un.val, repr);
3587	const payload_ty = payload.typeOf(&o.builder);
3588	if (payload_ty != union_ty.structFields(&o.builder)[
3589	@intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
3590	]) need_unnamed = true;
3591	const field_size = field_ty.abiSize(zcu);
3592	if (field_size == layout.payload_size) break :p payload;
3593	const padding_len = layout.payload_size - field_size;
3594	const padding_ty = try o.builder.arrayType(padding_len, .i8);
3595	break :p try o.builder.structConst(
3596	try o.builder.structType(.@"packed", &.{ payload_ty, padding_ty }),
3597	&.{ payload, try o.builder.undefConst(padding_ty) },
3598	);
3599	} else p: {
3600	assert(layout.tag_size == 0);
3601	const union_val = try o.lowerValue(un.val, repr);
3602	need_unnamed = true;
3603	break :p union_val;
3604	};
3605
3606	const payload_ty = payload.typeOf(&o.builder);
3607	if (layout.tag_size == 0) return o.builder.structConst(if (need_unnamed)
3608	try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3609	else
3610	union_ty, &.{payload});
3611	const tag = try o.lowerValue(un.tag, repr);
3612	const tag_ty = tag.typeOf(&o.builder);
3613	var fields: [3]Builder.Type = undefined;
3614	var vals: [3]Builder.Constant = undefined;
3615	var len: usize = 2;
3616	if (layout.tag_align.compare(.gte, layout.payload_align)) {
3617	fields = .{ tag_ty, payload_ty, undefined };
3618	vals = .{ tag, payload, undefined };
3619	} else {
3620	fields = .{ payload_ty, tag_ty, undefined };
3621	vals = .{ payload, tag, undefined };
3622	}
3623	if (layout.padding != 0) {
3624	fields[2] = try o.builder.arrayType(layout.padding, .i8);
3625	vals[2] = try o.builder.undefConst(fields[2]);
3626	len = 3;
3627	}
3628	return o.builder.structConst(if (need_unnamed)
3629	try o.builder.structType(union_ty.structKind(&o.builder), fields[0..len])
3630	else
3631	union_ty, vals[0..len]);
3632	},
3633	.memoized_call => unreachable,
3634	};
3635	}
3636
3637	pub fn f16Const(o: *Object, val: f16) Allocator.Error!Builder.Constant {
3638	return switch (std.zig.target.compilerRtFloatAbi(o.zcu.getTarget(), 16)) {
3639	.hard => o.builder.halfConst(val),
3640	.soft => o.builder.intConst(.i16, @as(u16, @bitCast(val))),
3641	};
3642	}
3643
3644	pub fn f32Const(o: *Object, val: f32) Allocator.Error!Builder.Constant {
3645	return switch (std.zig.target.compilerRtFloatAbi(o.zcu.getTarget(), 32)) {
3646	.hard => o.builder.floatConst(val),
3647	.soft => o.builder.intConst(.i32, @as(u32, @bitCast(val))),
3648	};
3649	}
3650
3651	pub fn f64Const(o: *Object, val: f64) Allocator.Error!Builder.Constant {
3652	return switch (std.zig.target.compilerRtFloatAbi(o.zcu.getTarget(), 64)) {
3653	.hard => o.builder.doubleConst(val),
3654	.soft => o.builder.intConst(.i64, @as(u64, @bitCast(val))),
3655	};
3656	}
3657
3658	pub fn f80Const(o: *Object, val: f80) Allocator.Error!Builder.Constant {
3659	switch (std.zig.target.compilerRtFloatAbi(o.zcu.getTarget(), 80)) {
3660	.hard => return o.builder.x86_fp80Const(val),
3661	.soft => {},
3662	}
3663	var llvm_field_tags_buf: [5]SoftF80Layout.LlvmFieldTag = undefined;
3664	var llvm_field_types_buf: [5]Builder.Type = undefined;
3665	const f80_layout = try o.softF80Layout(.{
3666	.llvm_field_tags_buf = &llvm_field_tags_buf,
3667	.llvm_field_types_buf = &llvm_field_types_buf,
3668	});
3669	const llvm_field_types = llvm_field_types_buf[0..f80_layout.llvm_fields_len];
3670	const f80_llvm_ty = try o.builder.structType(.normal, llvm_field_types);
3671	const f80_repr: packed struct { mantissa: u64, exponent: u16 } = @bitCast(val);
3672	var llvm_field_vals_buf: [5]Builder.Constant = undefined;
3673	const llvm_field_vals = llvm_field_vals_buf[0..f80_layout.llvm_fields_len];
3674	for (
3675	llvm_field_vals,
3676	llvm_field_tags_buf[0..f80_layout.llvm_fields_len],
3677	llvm_field_types,
3678	) |*llvm_field_val, llvm_field_tag, llvm_field_type|
3679	llvm_field_val.* = switch (llvm_field_tag) {
3680	.mantissa => try o.builder.intConst(llvm_field_type, f80_repr.mantissa),
3681	.exponent => try o.builder.intConst(llvm_field_type, f80_repr.exponent),
3682	.padding => try o.builder.undefConst(llvm_field_type),
3683	};
3684	return o.builder.structConst(f80_llvm_ty, llvm_field_vals);
3685	}
3686
3687	pub fn f128Const(o: *Object, val: f128) Allocator.Error!Builder.Constant {
3688	switch (std.zig.target.compilerRtFloatAbi(o.zcu.getTarget(), 128)) {
3689	.hard => return o.builder.fp128Const(val),
3690	.soft => {},
3691	}
3692	var llvm_field_tags_buf: [5]SoftF128Layout.LlvmFieldTag = undefined;
3693	var llvm_field_types_buf: [5]Builder.Type = undefined;
3694	const f128_layout = try o.softF128Layout(.{
3695	.llvm_field_tags_buf = &llvm_field_tags_buf,
3696	.llvm_field_types_buf = &llvm_field_types_buf,
3697	});
3698	const llvm_field_types = llvm_field_types_buf[0..f128_layout.llvm_fields_len];
3699	const f128_llvm_ty = try o.builder.structType(.normal, llvm_field_types);
3700	const f128_repr: packed struct { lo: u64, hi: u64 } = @bitCast(val);
3701	var llvm_field_vals_buf: [5]Builder.Constant = undefined;
3702	const llvm_field_vals = llvm_field_vals_buf[0..f128_layout.llvm_fields_len];
3703	for (
3704	llvm_field_vals,
3705	llvm_field_tags_buf[0..f128_layout.llvm_fields_len],
3706	llvm_field_types,
3707	) |*llvm_field_val, llvm_field_tag, llvm_field_type|
3708	llvm_field_val.* = switch (llvm_field_tag) {
3709	.lo => try o.builder.intConst(llvm_field_type, f128_repr.lo),
3710	.hi => try o.builder.intConst(llvm_field_type, f128_repr.hi),
3711	.padding => try o.builder.undefConst(llvm_field_type),
3712	};
3713	return o.builder.structConst(f128_llvm_ty, llvm_field_vals);
3714	}
3715
3716	pub fn lowerConstRef(
3717	o: *Object,
3718	constant: Builder.Constant,
3719	@"align": Builder.Alignment,
3720	) Allocator.Error!Builder.Constant {
3721	assert(@"align" != .default);
3722	const zcu = o.zcu;
3723	const gpa = zcu.comp.gpa;
3724	const gop = try o.const_map.getOrPut(gpa, constant);
3725	if (gop.found_existing) {
3726	// Keep the greater of the two alignments.
3727	const llvm_variable = gop.value_ptr.*;
3728	const llvm_old_align = llvm_variable.getAlignment(&o.builder);
3729	const llvm_new_align = llvm_old_align.max(@"align");
3730	llvm_variable.setAlignment(llvm_new_align, &o.builder);
3731	return llvm_variable.ptrConst(&o.builder).global.toConst();
3732	}
3733	errdefer assert(o.const_map.remove(constant));
3734
3735	const llvm_ty = constant.typeOf(&o.builder);
3736	const llvm_addrspace = toLlvmAddressSpace(.generic, zcu.getTarget());
3737	const llvm_variable = try o.builder.addVariable(.empty, llvm_ty, llvm_addrspace);
3738	gop.value_ptr.* = llvm_variable;
3739	try llvm_variable.setInitializer(constant, &o.builder);
3740	llvm_variable.setMutability(.constant, &o.builder);
3741	llvm_variable.setAlignment(@"align", &o.builder);
3742	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
3743	llvm_global.setLinkage(.private, &o.builder);
3744	llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
3745	return llvm_global.toConst();
3746	}
3747
3748	fn lowerPtr(
3749	o: *Object,
3750	ptr_val: InternPool.Index,
3751	prev_offset: u64,
3752	) Allocator.Error!Builder.Constant {
3753	const zcu = o.zcu;
3754	const ptr = zcu.intern_pool.indexToKey(ptr_val).ptr;
3755	const offset: u64 = prev_offset + ptr.byte_offset;
3756	return switch (ptr.base_addr) {
3757	.nav => |nav| {
3758	const base_ptr = try o.lowerNavRef(nav);
3759	return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
3760	try o.builder.intConst(.i64, offset),
3761	});
3762	},
3763	.uav => |uav| {
3764	const orig_ptr_ty: Type = .fromInterned(uav.orig_ty);
3765	const base_ptr = try o.lowerUavRef(
3766	uav.val,
3767	orig_ptr_ty.ptrAlignment(zcu).toLlvm(),
3768	orig_ptr_ty.ptrAddressSpace(zcu),
3769	);
3770	return o.builder.gepConst(.inbounds, .i8, base_ptr, null, &.{
3771	try o.builder.intConst(.i64, offset),
3772	});
3773	},
3774	.int => try o.builder.castConst(
3775	.inttoptr,
3776	try o.builder.intConst(try o.lowerType(.usize, .as_value), offset),
3777	try o.lowerType(.fromInterned(ptr.ty), .as_value),
3778	),
3779	.eu_payload => |eu_ptr| try o.lowerPtr(
3780	eu_ptr,
3781	offset + codegen.errUnionPayloadOffset(
3782	Value.fromInterned(eu_ptr).typeOf(zcu).childType(zcu),
3783	zcu,
3784	),
3785	),
3786	.opt_payload => |opt_ptr| try o.lowerPtr(opt_ptr, offset),
3787	.field => |field| {
3788	const agg_ty = Value.fromInterned(field.base).typeOf(zcu).childType(zcu);
3789	const field_off: u64 = switch (agg_ty.zigTypeTag(zcu)) {
3790	.pointer => off: {
3791	assert(agg_ty.isSlice(zcu));
3792	break :off switch (field.index) {
3793	Value.slice_ptr_index => 0,
3794	Value.slice_len_index => @divExact(zcu.getTarget().ptrBitWidth(), 8),
3795	else => unreachable,
3796	};
3797	},
3798	.@"struct", .@"union" => switch (agg_ty.containerLayout(zcu)) {
3799	.auto => agg_ty.structFieldOffset(@intCast(field.index), zcu),
3800	.@"extern", .@"packed" => unreachable,
3801	},
3802	else => unreachable,
3803	};
3804	return o.lowerPtr(field.base, offset + field_off);
3805	},
3806	.arr_elem => |arr_elem| {
3807	const base_ptr_ty = Value.fromInterned(arr_elem.base).typeOf(zcu);
3808	assert(base_ptr_ty.ptrSize(zcu) == .many);
3809	const elem_size = base_ptr_ty.childType(zcu).abiSize(zcu);
3810	return o.lowerPtr(arr_elem.base, offset + elem_size * arr_elem.index);
3811	},
3812	.comptime_field => unreachable,
3813	.comptime_alloc => unreachable,
3814	};
3815	}
3816
3817	pub fn lowerPtrToVoid(
3818	o: *Object,
3819	/// Must not be `.default`.
3820	@"align": Builder.Alignment,
3821	@"addrspace": std.lang.AddressSpace,
3822	) Allocator.Error!Builder.Constant {
3823	const addr: u64 = @"align".toByteUnits().?;
3824	const llvm_usize = try o.lowerType(.usize, .as_value);
3825	const llvm_addr = try o.builder.intConst(llvm_usize, addr);
3826	const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
3827	return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
3828	}
3829
3830	pub fn lowerUavRef(
3831	o: *Object,
3832	uav_val: InternPool.Index,
3833	/// Must not be `.default`.
3834	@"align": Builder.Alignment,
3835	@"addrspace": std.lang.AddressSpace,
3836	) Allocator.Error!Builder.Constant {
3837	assert(@"align" != .default);
3838
3839	const zcu = o.zcu;
3840	const ip = &zcu.intern_pool;
3841	const gpa = zcu.comp.gpa;
3842
3843	const uav_ty: Type = .fromInterned(ip.typeOf(uav_val));
3844
3845	switch (ip.indexToKey(uav_val)) {
3846	.func => unreachable, // should be using a Nav ref
3847	.@"extern" => unreachable, // should be using a Nav ref
3848	else => {},
3849	}
3850
3851	if (!uav_ty.hasRuntimeBits(zcu)) {
3852	return o.lowerPtrToVoid(@"align", @"addrspace");
3853	}
3854
3855	const llvm_addrspace = toLlvmAddressSpace(@"addrspace", zcu.getTarget());
3856
3857	const gop = try o.uav_map.getOrPut(gpa, .{ .val = uav_val, .@"addrspace" = @"addrspace" });
3858	if (gop.found_existing) {
3859	// Keep the greater of the two alignments.
3860	const llvm_variable = gop.value_ptr.*;
3861	const llvm_old_align = llvm_variable.getAlignment(&o.builder);
3862	const llvm_new_align = llvm_old_align.max(@"align");
3863	llvm_variable.setAlignment(llvm_new_align, &o.builder);
3864	return llvm_variable.ptrConst(&o.builder).global.toConst();
3865	}
3866	errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
3867
3868	const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@backingInt(uav_val)});
3869	const llvm_variable = try o.builder.addVariable(llvm_name, .void, llvm_addrspace);
3870	gop.value_ptr.* = llvm_variable;
3871	try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
3872	llvm_variable.setMutability(.constant, &o.builder);
3873	llvm_variable.setAlignment(@"align", &o.builder);
3874	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
3875	llvm_global.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
3876	llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
3877	return llvm_global.toConst();
3878	}
3879
3880	pub fn lowerNavRef(o: *Object, nav_id: InternPool.Nav.Index) Allocator.Error!Builder.Constant {
3881	const zcu = o.zcu;
3882	const ip = &zcu.intern_pool;
3883	const gpa = zcu.comp.gpa;
3884
3885	const nav = ip.getNav(nav_id);
3886	const nav_ty: Type = .fromInterned(nav.resolved.?.type);
3887	if (!nav_ty.isRuntimeFnOrHasRuntimeBits(zcu) and nav.getExtern(ip) == null) {
3888	const nav_align = switch (nav.resolved.?.@"align") {
3889	.none => nav_ty.abiAlignment(zcu),
3890	else => |a| a,
3891	};
3892	return o.lowerPtrToVoid(nav_align.toLlvm(), nav.resolved.?.@"addrspace");
3893	}
3894
3895	const gop = try o.nav_map.getOrPut(gpa, nav_id);
3896	if (!gop.found_existing) {
3897	errdefer assert(o.nav_map.remove(nav_id));
3898	// The NAV hasn't been lowered yet, so generate a placeholder global whose details will
3899	// be filled in later.
3900	const llvm_name = try o.builder.strtabString(nav.fqn.toSlice(ip));
3901	gop.value_ptr.* = try o.builder.addGlobal(llvm_name, .{
3902	.type = .void, // placeholder; populated by `updateNav`/`updateFunc`
3903	.kind = .{ .alias = .none }, // placeholder; populated by `updateNav`/`updateFunc`
3904	});
3905	}
3906	const llvm_global = gop.value_ptr.*;
3907
3908	// We need to make sure the global's address space is up to date, because that affects the
3909	// type of a pointer to this global. But everything else about the global will be populated
3910	// by `updateNav` or `updateFunc`.
3911	llvm_global.ptr(&o.builder).addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", zcu.getTarget());
3912	return llvm_global.toConst();
3913	}
3914
3915	pub fn addByValParamAttrs(
3916	o: *Object,
3917	pt: Zcu.PerThread,
3918	attributes: *Builder.FunctionAttributes.Wip,
3919	param_ty: Type,
3920	param_index: u32,
3921	fn_info: FuncInfo,
3922	llvm_arg_i: u32,
3923	) Allocator.Error!void {
3924	const zcu = o.zcu;
3925	if (param_ty.isPtrAtRuntime(zcu)) {
3926	const ptr_info = param_ty.ptrInfo(zcu);
3927	if (std.math.cast(u5, param_index)) |i| {
3928	if (@as(u1, @truncate(fn_info.noalias_bits >> i)) != 0) {
3929	try attributes.addParamAttr(llvm_arg_i, .@"noalias", &o.builder);
3930	}
3931	}
3932	if (!param_ty.isPtrLikeOptional(zcu) and
3933	!ptr_info.flags.is_allowzero and
3934	ptr_info.flags.address_space == .generic)
3935	{
3936	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
3937	}
3938	switch (fn_info.cc) {
3939	else => {},
3940	.x86_64_interrupt,
3941	.x86_interrupt,
3942	=> {
3943	const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
3944	try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
3945	},
3946	}
3947	if (ptr_info.flags.is_const) {
3948	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
3949	}
3950	const elem_align: Builder.Alignment.Lazy = switch (ptr_info.flags.alignment) {
3951	else => |a| .wrap(a.toLlvm()),
3952	.none => try o.lazyAbiAlignment(pt, .fromInterned(ptr_info.child)),
3953	};
3954	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = elem_align }, &o.builder);
3955	} else if (ccAbiPromoteInt(fn_info.cc, zcu, param_ty)) |s| switch (s) {
3956	.signed => try attributes.addParamAttr(llvm_arg_i, .signext, &o.builder),
3957	.unsigned => try attributes.addParamAttr(llvm_arg_i, .zeroext, &o.builder),
3958	};
3959	}
3960
3961	pub const Byval = struct { alignment: InternPool.Alignment = .none };
3962	pub fn addByRefParamAttrs(
3963	o: *Object,
3964	attributes: *Builder.FunctionAttributes.Wip,
3965	llvm_arg_i: u32,
3966	maybe_byval: ?Byval,
3967	param_ty: Type,
3968	) Allocator.Error!void {
3969	const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
3970	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
3971	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
3972	try attributes.addParamAttr(llvm_arg_i, .noundef, &o.builder);
3973	const alignment = if (maybe_byval) |byval| alignment: {
3974	try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
3975	break :alignment byval.alignment;
3976	} else .none;
3977	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(switch (alignment) {
3978	.none => param_ty.abiAlignment(o.zcu),
3979	else => alignment,
3980	}.toLlvm()) }, &o.builder);
3981	}
3982
3983	pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
3984	if (o.error_name_table != .none) return o.error_name_table;
3985
3986	const name = try o.builder.strtabString("__zig_error_name_table");
3987	// TODO: Address space
3988	const llvm_variable = try o.builder.addVariable(name, .ptr, .default);
3989	llvm_variable.setMutability(.constant, &o.builder);
3990	llvm_variable.setAlignment(
3991	Type.slice_const_u8_sentinel_0.abiAlignment(o.zcu).toLlvm(),
3992	&o.builder,
3993	);
3994	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
3995	llvm_global.setLinkage(.private, &o.builder);
3996	llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
3997
3998	o.error_name_table = llvm_variable;
3999	return llvm_variable;
4000	}
4001
4002	pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
4003	const builder = &o.builder;
4004	if (o.errors_len_variable == .none) {
4005	const llvm_err_int_ty = try o.errorIntType(.in_memory);
4006	const name = try builder.strtabString("__zig_errors_len");
4007	const llvm_variable = try builder.addVariable(name, llvm_err_int_ty, .default);
4008	llvm_variable.setMutability(.constant, builder);
4009	llvm_variable.setAlignment(Type.errorAbiAlignment(o.zcu).toLlvm(), builder);
4010	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
4011	llvm_global.setLinkage(.private, builder);
4012	llvm_global.setUnnamedAddr(.unnamed_addr, builder);
4013	o.errors_len_variable = llvm_variable;
4014	}
4015	return o.errors_len_variable;
4016	}
4017
4018	pub fn getEnumTagNameFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index {
4019	const zcu = o.zcu;
4020	const ip = &zcu.intern_pool;
4021
4022	const gop = try o.enum_tag_name_map.getOrPut(o.gpa, enum_ty.toIntern());
4023	if (gop.found_existing) return gop.value_ptr.*;
4024	errdefer assert(o.enum_tag_name_map.remove(enum_ty.toIntern()));
4025	const llvm_function = try o.builder.addFunction(
4026	// Dummy function type; `updateEnumTagNameFunction` will replace it with the correct type.
4027	// TODO: change the builder API so we don't need to do this.
4028	try o.builder.fnType(.void, &.{}, .normal),
4029	try o.builder.strtabStringFmt("__zig_tag_name_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
4030	toLlvmAddressSpace(.generic, zcu.getTarget()),
4031	);
4032	gop.value_ptr.* = llvm_function;
4033	try o.updateEnumTagNameFunction(enum_ty, llvm_function);
4034	return llvm_function;
4035	}
4036	fn updateEnumTagNameFunction(
4037	o: *Object,
4038	enum_ty: Type,
4039	llvm_function: Builder.Function.Index,
4040	) Allocator.Error!void {
4041	const zcu = o.zcu;
4042	const ip = &zcu.intern_pool;
4043	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4044
4045	const llvm_usize_ty = try o.lowerType(.usize, .as_value);
4046	const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .as_value);
4047	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .as_value);
4048
4049	llvm_function.ptrConst(&o.builder).global.ptr(&o.builder).type =
4050	try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
4051
4052	var attributes: Builder.FunctionAttributes.Wip = .{};
4053	defer attributes.deinit(&o.builder);
4054	try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
4055
4056	llvm_function.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
4057	llvm_function.setCallConv(.fastcc, &o.builder);
4058	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
4059
4060	var wip = try Builder.WipFunction.init(&o.builder, .{
4061	.function = llvm_function,
4062	.strip = true,
4063	});
4064	defer wip.deinit();
4065	wip.cursor = .{ .block = try wip.block(0, "Entry") };
4066
4067	const bad_value_block = try wip.block(1, "BadValue");
4068	const tag_int_value = wip.arg(0);
4069	var wip_switch = try wip.@"switch"(
4070	tag_int_value,
4071	bad_value_block,
4072	@intCast(loaded_enum.field_names.len),
4073	.none,
4074	);
4075	defer wip_switch.finish(&wip);
4076
4077	for (0..loaded_enum.field_names.len) |field_index| {
4078	const name = try o.builder.stringNull(loaded_enum.field_names.get(ip)[field_index].toSlice(ip));
4079	const name_init = try o.builder.stringConst(name);
4080	const name_llvm_variable = try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default);
4081	try name_llvm_variable.setInitializer(name_init, &o.builder);
4082	name_llvm_variable.setMutability(.constant, &o.builder);
4083	name_llvm_variable.setAlignment(comptime .fromByteUnits(1), &o.builder);
4084	const name_llvm_global = name_llvm_variable.ptrConst(&o.builder).global;
4085	name_llvm_global.setLinkage(.private, &o.builder);
4086	name_llvm_global.setUnnamedAddr(.unnamed_addr, &o.builder);
4087
4088	const name_val = try o.builder.structValue(llvm_ret_ty, &.{
4089	name_llvm_global.toConst(),
4090	try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len - 1),
4091	});
4092
4093	const return_block = try wip.block(1, "Name");
4094	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
4095	.none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
4096	else => |tag_val_ip| try o.lowerValue(tag_val_ip, .as_value),
4097	};
4098	try wip_switch.addCase(llvm_tag_val, return_block, &wip);
4099
4100	wip.cursor = .{ .block = return_block };
4101	_ = try wip.ret(name_val);
4102	}
4103
4104	wip.cursor = .{ .block = bad_value_block };
4105	_ = try wip.@"unreachable"();
4106
4107	try wip.finish();
4108	}
4109
4110	pub fn lazyAbiAlignment(o: *Object, pt: Zcu.PerThread, ty: Type) Allocator.Error!Builder.Alignment.Lazy {
4111	const index = try o.type_pool.get(pt, .{ .llvm = o }, ty.toIntern());
4112	return o.lazy_abi_aligns.items[@backingInt(index)];
4113	}
4114
4115	pub fn getIsNamedEnumValueFunction(o: *Object, enum_ty: Type) Allocator.Error!Builder.Function.Index {
4116	const zcu = o.zcu;
4117	const ip = &zcu.intern_pool;
4118
4119	const gop = try o.named_enum_map.getOrPut(o.gpa, enum_ty.toIntern());
4120	if (gop.found_existing) return gop.value_ptr.*;
4121	errdefer assert(o.named_enum_map.remove(enum_ty.toIntern()));
4122	const llvm_function = try o.builder.addFunction(
4123	// Dummy function type; `updateIsNamedEnumValue` will replace it with the correct type.
4124	// TODO: change the builder API so we don't need to do this.
4125	try o.builder.fnType(.void, &.{}, .normal),
4126	try o.builder.strtabStringFmt("__zig_is_named_enum_value_{f}", .{enum_ty.containerTypeName(ip).fmt(ip)}),
4127	toLlvmAddressSpace(.generic, zcu.getTarget()),
4128	);
4129	gop.value_ptr.* = llvm_function;
4130	try o.updateIsNamedEnumValueFunction(enum_ty, llvm_function);
4131	return llvm_function;
4132	}
4133	fn updateIsNamedEnumValueFunction(
4134	o: *Object,
4135	enum_ty: Type,
4136	llvm_function: Builder.Function.Index,
4137	) Allocator.Error!void {
4138	const zcu = o.zcu;
4139	const ip = &zcu.intern_pool;
4140	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4141
4142	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .as_value);
4143	llvm_function.ptrConst(&o.builder).global.ptr(&o.builder).type =
4144	try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
4145
4146	var attributes: Builder.FunctionAttributes.Wip = .{};
4147	defer attributes.deinit(&o.builder);
4148	try o.addCommonFnAttributes(&attributes, zcu.root_mod, zcu.root_mod.omit_frame_pointer);
4149
4150	llvm_function.setLinkage(if (o.builder.strip) .private else .internal, &o.builder);
4151	llvm_function.setCallConv(.fastcc, &o.builder);
4152	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
4153
4154	var wip: Builder.WipFunction = try .init(&o.builder, .{
4155	.function = llvm_function,
4156	.strip = true,
4157	});
4158	defer wip.deinit();
4159	wip.cursor = .{ .block = try wip.block(0, "Entry") };
4160
4161	const named_block = try wip.block(@intCast(loaded_enum.field_names.len), "Named");
4162	const unnamed_block = try wip.block(1, "Unnamed");
4163	const tag_int_value = wip.arg(0);
4164	var wip_switch = try wip.@"switch"(tag_int_value, unnamed_block, @intCast(loaded_enum.field_names.len), .none);
4165	defer wip_switch.finish(&wip);
4166
4167	if (loaded_enum.field_values.len > 0) {
4168	for (loaded_enum.field_values.get(ip)) |tag_val_ip| {
4169	const llvm_tag_val = try o.lowerValue(tag_val_ip, .as_value);
4170	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4171	}
4172	} else {
4173	// Auto-numbered.
4174	for (0..loaded_enum.field_names.len) |field_index| {
4175	const llvm_tag_val = try o.builder.intConst(llvm_int_ty, field_index);
4176	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4177	}
4178	}
4179
4180	wip.cursor = .{ .block = named_block };
4181	_ = try wip.ret(.true);
4182
4183	wip.cursor = .{ .block = unnamed_block };
4184	_ = try wip.ret(.false);
4185
4186	try wip.finish();
4187	}
4188
4189	pub fn getLibcFunction(
4190	o: *Object,
4191	pt: Zcu.PerThread,
4192	fn_name: Builder.StrtabString,
4193	fn_info: FuncInfo,
4194	) Allocator.Error!Builder.Function.Index {
4195	if (o.builder.getGlobal(fn_name)) |global| return switch (global.ptrConst(&o.builder).kind) {
4196	.alias => |alias| alias.getAliasee(&o.builder).ptrConst(&o.builder).kind.function,
4197	.function => |function| function,
4198	.variable, .replaced => unreachable,
4199	};
4200	const llvm_function = try o.builder.addFunction(
4201	try o.lowerFnType(fn_info),
4202	fn_name,
4203	toLlvmAddressSpace(.generic, o.zcu.getTarget()),
4204	);
4205	var attributes: Builder.FunctionAttributes.Wip = .{};
4206	defer attributes.deinit(&o.builder);
4207	try o.addCallingConventionFnAttributes(pt, llvm_function, &attributes, .{
4208	.name = fn_name.slice(&o.builder).?,
4209	}, fn_info);
4210	llvm_function.setAttributes(try attributes.finish(&o.builder), &o.builder);
4211	return llvm_function;
4212	}
4213	};
4214
4215	const CallingConventionInfo = struct {
4216	/// The LLVM calling convention to use.
4217	llvm_cc: Builder.CallConv,
4218	/// Whether to use an `alignstack` attribute to forcibly re-align the stack pointer in the function's prologue.
4219	align_stack: bool,
4220	/// Whether the function needs a `naked` attribute.
4221	naked: bool,
4222	/// How many leading register-sized integer parameters to apply the `inreg` attribute to.
4223	inreg_int_params: u2 = 0,
4224	/// How many leading floating-point parameters to apply the `inreg` attribute to.
4225	inreg_float_params: u3 = 0,
4226	};
4227
4228	pub fn toLlvmCallConv(cc: std.lang.CallingConvention, target: *const std.Target) ?CallingConventionInfo {
4229	const llvm_cc = toLlvmCallConvTag(cc, target) orelse return null;
4230	const incoming_stack_alignment: ?u64, const inreg_int_params: u2, const inreg_float_params: u3 = switch (cc) {
4231	.x86_fastcall => |opts| .{ opts.incoming_stack_alignment, 2, 0 },
4232	.x86_vectorcall => |opts| .{ opts.incoming_stack_alignment, 2, 6 },
4233	inline else => |pl| switch (@TypeOf(pl)) {
4234	void => .{ null, 0, 0 },
4235	std.lang.CallingConvention.ArcInterruptOptions,
4236	std.lang.CallingConvention.ArmInterruptOptions,
4237	std.lang.CallingConvention.RiscvInterruptOptions,
4238	std.lang.CallingConvention.ShInterruptOptions,
4239	std.lang.CallingConvention.MicroblazeInterruptOptions,
4240	std.lang.CallingConvention.MipsInterruptOptions,
4241	std.lang.CallingConvention.CommonOptions,
4242	=> .{ pl.incoming_stack_alignment, 0, 0 },
4243	std.lang.CallingConvention.X86RegparmOptions => .{ pl.incoming_stack_alignment, pl.register_params, 0 },
4244	std.lang.CallingConvention.SpirvKernelOptions,
4245	std.lang.CallingConvention.SpirvFragmentOptions,
4246	std.lang.CallingConvention.SpirvMeshOptions,
4247	=> .{ null, 0, 0 },
4248	else => @compileError("TODO: toLlvmCallConv(." ++ @tagName(pl) ++ ")"),
4249	},
4250	};
4251	return .{
4252	.llvm_cc = llvm_cc,
4253	.align_stack = if (incoming_stack_alignment) |a| need_align: {
4254	const normal_stack_align = target.stackAlignment();
4255	break :need_align a < normal_stack_align;
4256	} else false,
4257	.naked = cc == .naked,
4258	.inreg_int_params = inreg_int_params,
4259	.inreg_float_params = inreg_float_params,
4260	};
4261	}
4262	pub fn toLlvmCallConvTag(cc_tag: std.lang.CallingConvention.Tag, target: *const std.Target) ?Builder.CallConv {
4263	if (target.cCallingConvention()) |default_c| {
4264	if (cc_tag == default_c) {
4265	return .ccc;
4266	}
4267	}
4268	return switch (cc_tag) {
4269	.@"inline" => unreachable,
4270	.auto, .async => .fastcc,
4271	.naked => .ccc,
4272	.x86_64_sysv => .x86_64_sysvcc,
4273	.x86_64_win => .win64cc,
4274	.x86_64_regcall_v3_sysv => if (target.cpu.arch == .x86_64 and target.os.tag != .windows)
4275	.x86_regcallcc
4276	else
4277	null,
4278	.x86_64_regcall_v4_win => if (target.cpu.arch == .x86_64 and target.os.tag == .windows)
4279	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
4280	else
4281	null,
4282	.x86_64_vectorcall => .x86_vectorcallcc,
4283	.x86_64_interrupt => .x86_intrcc,
4284	.x86_64_preserve_none => .preserve_nonecc,
4285	.x86_stdcall => .x86_stdcallcc,
4286	.x86_fastcall => .x86_fastcallcc,
4287	.x86_thiscall => .x86_thiscallcc,
4288	.x86_regcall_v3 => if (target.cpu.arch == .x86 and target.os.tag != .windows)
4289	.x86_regcallcc
4290	else
4291	null,
4292	.x86_regcall_v4_win => if (target.cpu.arch == .x86 and target.os.tag == .windows)
4293	.x86_regcallcc // we use the "RegCallv4" module flag to make this correct
4294	else
4295	null,
4296	.x86_vectorcall => .x86_vectorcallcc,
4297	.x86_interrupt => .x86_intrcc,
4298	.aarch64_vfabi => .aarch64_vector_pcs,
4299	.aarch64_vfabi_sve => .aarch64_sve_vector_pcs,
4300	.aarch64_preserve_none => .preserve_nonecc,
4301	.arm_aapcs => .arm_aapcscc,
4302	.arm_aapcs_vfp => .arm_aapcs_vfpcc,
4303	.riscv64_lp64_v => .riscv_vectorcallcc,
4304	.riscv32_ilp32_v => .riscv_vectorcallcc,
4305	.avr_builtin => .avr_builtincc,
4306	.avr_signal => .avr_signalcc,
4307	.avr_interrupt => .avr_intrcc,
4308	.m68k_rtd => .m68k_rtdcc,
4309	.m68k_interrupt => .m68k_intrcc,
4310	.msp430_interrupt => .msp430_intrcc,
4311	.amdgcn_kernel => .amdgpu_kernel,
4312	.amdgcn_cs => .amdgpu_cs,
4313	.nvptx_device => .ptx_device,
4314	.nvptx_kernel => .ptx_kernel,
4315
4316	// Calling conventions which LLVM uses function attributes for.
4317	.riscv64_interrupt,
4318	.riscv32_interrupt,
4319	.arm_interrupt,
4320	.mips64_interrupt,
4321	.mips_interrupt,
4322	.csky_interrupt,
4323	=> .ccc,
4324
4325	// All the calling conventions which LLVM does not have a general representation for.
4326	// Note that these are often still supported through the `cCallingConvention` path above via `ccc`.
4327	.x86_16_cdecl,
4328	.x86_16_stdcall,
4329	.x86_16_regparmcall,
4330	.x86_16_interrupt,
4331	.x86_sysv,
4332	.x86_win,
4333	.x86_mingw,
4334	.x86_thiscall_mingw,
4335	.x86_64_x32,
4336	.aarch64_aapcs,
4337	.aarch64_aapcs_darwin,
4338	.aarch64_aapcs_win,
4339	.alpha_osf,
4340	.microblaze_std,
4341	.microblaze_interrupt,
4342	.mips64_n64,
4343	.mips64_n32,
4344	.mips_o32,
4345	.riscv64_lp64,
4346	.riscv32_ilp32,
4347	.sparc64_sysv,
4348	.sparc_sysv,
4349	.powerpc64_elf,
4350	.powerpc64_elf_altivec,
4351	.powerpc64_elf_v2,
4352	.powerpc_sysv,
4353	.powerpc_sysv_altivec,
4354	.powerpc_aix,
4355	.powerpc_aix_altivec,
4356	.wasm_mvp,
4357	.arc_sysv,
4358	.arc_interrupt,
4359	.avr_gnu,
4360	.bpf_std,
4361	.csky_sysv,
4362	.ez80_cet,
4363	.ez80_tiflags,
4364	.hexagon_sysv,
4365	.hexagon_sysv_hvx,
4366	.hppa_elf,
4367	.hppa64_elf,
4368	.kvx_lp64,
4369	.kvx_ilp32,
4370	.lanai_sysv,
4371	.loongarch64_lp64,
4372	.loongarch32_ilp32,
4373	.m68k_sysv,
4374	.m68k_gnu,
4375	.m88k_sysv,
4376	.msp430_eabi,
4377	.or1k_sysv,
4378	.propeller_sysv,
4379	.s390x_sysv,
4380	.s390x_sysv_vx,
4381	.sh_gnu,
4382	.sh_renesas,
4383	.sh_interrupt,
4384	.ve_sysv,
4385	.xcore_xs1,
4386	.xcore_xs2,
4387	.xtensa_call0,
4388	.xtensa_windowed,
4389	.amdgcn_device,
4390	.spirv_device,
4391	.spirv_kernel,
4392	.spirv_fragment,
4393	.spirv_vertex,
4394	.spirv_task,
4395	.spirv_mesh,
4396	.spork8,
4397	=> null,
4398	};
4399	}
4400
4401	/// Convert a zig-address space to an llvm address space.
4402	pub fn toLlvmAddressSpace(address_space: std.lang.AddressSpace, target: *const std.Target) Builder.AddrSpace {
4403	for (llvmAddrSpaceInfo(target)) |info| if (info.zig == address_space) return info.llvm;
4404	unreachable;
4405	}
4406
4407	const AddrSpaceInfo = struct {
4408	zig: ?std.lang.AddressSpace,
4409	llvm: Builder.AddrSpace,
4410	non_integral: bool = false,
4411	size: ?u16 = null,
4412	abi: ?u16 = null,
4413	pref: ?u16 = null,
4414	idx: ?u16 = null,
4415	force_in_data_layout: bool = false,
4416	};
4417	fn llvmAddrSpaceInfo(target: *const std.Target) []const AddrSpaceInfo {
4418	return switch (target.cpu.arch) {
4419	.x86, .x86_64 => &.{
4420	.{ .zig = .generic, .llvm = .default },
4421	.{ .zig = .gs, .llvm = Builder.AddrSpace.x86.gs },
4422	.{ .zig = .fs, .llvm = Builder.AddrSpace.x86.fs },
4423	.{ .zig = .ss, .llvm = Builder.AddrSpace.x86.ss },
4424	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_sptr, .size = 32, .abi = 32, .force_in_data_layout = true },
4425	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr32_uptr, .size = 32, .abi = 32, .force_in_data_layout = true },
4426	.{ .zig = null, .llvm = Builder.AddrSpace.x86.ptr64, .size = 64, .abi = 64, .force_in_data_layout = true },
4427	},
4428	.nvptx, .nvptx64 => &.{
4429	.{ .zig = .generic, .llvm = Builder.AddrSpace.nvptx.generic },
4430	.{ .zig = .global, .llvm = Builder.AddrSpace.nvptx.global },
4431	.{ .zig = .constant, .llvm = Builder.AddrSpace.nvptx.constant },
4432	.{ .zig = .param, .llvm = Builder.AddrSpace.nvptx.param },
4433	.{ .zig = .shared, .llvm = Builder.AddrSpace.nvptx.shared },
4434	.{ .zig = .local, .llvm = Builder.AddrSpace.nvptx.local },
4435	},
4436	.amdgcn => &.{
4437	.{ .zig = .generic, .llvm = Builder.AddrSpace.amdgpu.flat, .force_in_data_layout = true },
4438	.{ .zig = .global, .llvm = Builder.AddrSpace.amdgpu.global, .force_in_data_layout = true },
4439	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.region, .size = 32, .abi = 32 },
4440	.{ .zig = .shared, .llvm = Builder.AddrSpace.amdgpu.local, .size = 32, .abi = 32 },
4441	.{ .zig = .constant, .llvm = Builder.AddrSpace.amdgpu.constant, .force_in_data_layout = true },
4442	.{ .zig = .local, .llvm = Builder.AddrSpace.amdgpu.private, .size = 32, .abi = 32 },
4443	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_32bit, .size = 32, .abi = 32 },
4444	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_fat_pointer, .non_integral = true, .size = 160, .abi = 256, .idx = 32 },
4445	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_resource, .non_integral = true, .size = 128, .abi = 128 },
4446	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.buffer_strided_pointer, .non_integral = true, .size = 192, .abi = 256, .idx = 32 },
4447	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_0 },
4448	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_1 },
4449	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_2 },
4450	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_3 },
4451	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_4 },
4452	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_5 },
4453	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_6 },
4454	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_7 },
4455	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_8 },
4456	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_9 },
4457	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_10 },
4458	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_11 },
4459	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_12 },
4460	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_13 },
4461	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_14 },
4462	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.constant_buffer_15 },
4463	.{ .zig = null, .llvm = Builder.AddrSpace.amdgpu.streamout_register },
4464	},
4465	.avr => &.{
4466	.{ .zig = .generic, .llvm = Builder.AddrSpace.avr.data, .abi = 8 },
4467	.{ .zig = .flash, .llvm = Builder.AddrSpace.avr.program, .abi = 8 },
4468	.{ .zig = .flash1, .llvm = Builder.AddrSpace.avr.program1, .abi = 8 },
4469	.{ .zig = .flash2, .llvm = Builder.AddrSpace.avr.program2, .abi = 8 },
4470	.{ .zig = .flash3, .llvm = Builder.AddrSpace.avr.program3, .abi = 8 },
4471	.{ .zig = .flash4, .llvm = Builder.AddrSpace.avr.program4, .abi = 8 },
4472	.{ .zig = .flash5, .llvm = Builder.AddrSpace.avr.program5, .abi = 8 },
4473	},
4474	.wasm32, .wasm64 => &.{
4475	.{ .zig = .generic, .llvm = Builder.AddrSpace.wasm.default, .force_in_data_layout = true },
4476	.{ .zig = null, .llvm = Builder.AddrSpace.wasm.variable, .non_integral = true },
4477	.{ .zig = .externref, .llvm = Builder.AddrSpace.wasm.externref, .non_integral = true, .size = 8, .abi = 8 },
4478	.{ .zig = .funcref, .llvm = Builder.AddrSpace.wasm.funcref, .non_integral = true, .size = 8, .abi = 8 },
4479	},
4480	.m68k => &.{
4481	.{ .zig = .generic, .llvm = .default, .abi = 16, .pref = 32 },
4482	},
4483	else => &.{
4484	.{ .zig = .generic, .llvm = .default },
4485	},
4486	};
4487	}
4488
4489	/// On some targets, global values that are in the generic address space must be generated into a
4490	/// different address space, and then cast back to the generic address space.
4491	fn llvmDefaultGlobalAddressSpace(target: *const std.Target) Builder.AddrSpace {
4492	return switch (target.cpu.arch) {
4493	// On amdgcn, globals must be explicitly allocated and uploaded so that the program can access
4494	// them.
4495	.amdgcn => Builder.AddrSpace.amdgpu.global,
4496	else => .default,
4497	};
4498	}
4499
4500	/// Return the actual address space that a value should be stored in if its a global address space.
4501	/// When a value is placed in the resulting address space, it needs to be cast back into wanted_address_space.
4502	fn toLlvmGlobalAddressSpace(wanted_address_space: std.lang.AddressSpace, target: *const std.Target) Builder.AddrSpace {
4503	return switch (wanted_address_space) {
4504	.generic => llvmDefaultGlobalAddressSpace(target),
4505	else => |as| toLlvmAddressSpace(as, target),
4506	};
4507	}
4508
4509	/// We need to insert extra padding if LLVM's isn't enough.
4510	/// However we don't want to ever call LLVMABIAlignmentOfType or
4511	/// LLVMABISizeOfType because these functions will trip assertions
4512	/// when using them for self-referential types. So our strategy is
4513	/// to use non-packed llvm structs but to emit all padding explicitly.
4514	/// We can do this because for all types, Zig ABI alignment >= LLVM ABI
4515	/// alignment.
4516	const struct_layout_version = 2;
4517
4518	// TODO: Restore the non_null field to i1 once
4519	// https://github.com/llvm/llvm-project/issues/56585/ is fixed
4520	pub const optional_layout_version = 3;
4521
4522	var target_registry_mutex: std.Io.Mutex = .init;
4523
4524	pub fn initializeLLVMTarget(io: Io, arch: std.Target.Cpu.Arch) void {
4525	// Repeated initialization is safe, as targets which have already been registered will be skipped.
4526	// It is however the client's responsibility to synchronize registry access.
4527	target_registry_mutex.lockUncancelable(io);
4528	defer target_registry_mutex.unlock(io);
4529
4530	switch (arch) {
4531	.aarch64, .aarch64_be => {
4532	bindings.LLVMInitializeAArch64Target();
4533	bindings.LLVMInitializeAArch64TargetInfo();
4534	bindings.LLVMInitializeAArch64TargetMC();
4535	bindings.LLVMInitializeAArch64AsmPrinter();
4536	bindings.LLVMInitializeAArch64AsmParser();
4537	},
4538	.amdgcn => {
4539	bindings.LLVMInitializeAMDGPUTarget();
4540	bindings.LLVMInitializeAMDGPUTargetInfo();
4541	bindings.LLVMInitializeAMDGPUTargetMC();
4542	bindings.LLVMInitializeAMDGPUAsmPrinter();
4543	bindings.LLVMInitializeAMDGPUAsmParser();
4544	},
4545	.thumb, .thumbeb, .arm, .armeb => {
4546	bindings.LLVMInitializeARMTarget();
4547	bindings.LLVMInitializeARMTargetInfo();
4548	bindings.LLVMInitializeARMTargetMC();
4549	bindings.LLVMInitializeARMAsmPrinter();
4550	bindings.LLVMInitializeARMAsmParser();
4551	},
4552	.avr => {
4553	bindings.LLVMInitializeAVRTarget();
4554	bindings.LLVMInitializeAVRTargetInfo();
4555	bindings.LLVMInitializeAVRTargetMC();
4556	bindings.LLVMInitializeAVRAsmPrinter();
4557	bindings.LLVMInitializeAVRAsmParser();
4558	},
4559	.bpfel, .bpfeb => {
4560	bindings.LLVMInitializeBPFTarget();
4561	bindings.LLVMInitializeBPFTargetInfo();
4562	bindings.LLVMInitializeBPFTargetMC();
4563	bindings.LLVMInitializeBPFAsmPrinter();
4564	bindings.LLVMInitializeBPFAsmParser();
4565	},
4566	.hexagon => {
4567	bindings.LLVMInitializeHexagonTarget();
4568	bindings.LLVMInitializeHexagonTargetInfo();
4569	bindings.LLVMInitializeHexagonTargetMC();
4570	bindings.LLVMInitializeHexagonAsmPrinter();
4571	bindings.LLVMInitializeHexagonAsmParser();
4572	},
4573	.lanai => {
4574	bindings.LLVMInitializeLanaiTarget();
4575	bindings.LLVMInitializeLanaiTargetInfo();
4576	bindings.LLVMInitializeLanaiTargetMC();
4577	bindings.LLVMInitializeLanaiAsmPrinter();
4578	bindings.LLVMInitializeLanaiAsmParser();
4579	},
4580	.mips, .mipsel, .mips64, .mips64el => {
4581	bindings.LLVMInitializeMipsTarget();
4582	bindings.LLVMInitializeMipsTargetInfo();
4583	bindings.LLVMInitializeMipsTargetMC();
4584	bindings.LLVMInitializeMipsAsmPrinter();
4585	bindings.LLVMInitializeMipsAsmParser();
4586	},
4587	.msp430 => {
4588	bindings.LLVMInitializeMSP430Target();
4589	bindings.LLVMInitializeMSP430TargetInfo();
4590	bindings.LLVMInitializeMSP430TargetMC();
4591	bindings.LLVMInitializeMSP430AsmPrinter();
4592	bindings.LLVMInitializeMSP430AsmParser();
4593	},
4594	.nvptx, .nvptx64 => {
4595	bindings.LLVMInitializeNVPTXTarget();
4596	bindings.LLVMInitializeNVPTXTargetInfo();
4597	bindings.LLVMInitializeNVPTXTargetMC();
4598	bindings.LLVMInitializeNVPTXAsmPrinter();
4599	// There is no LLVMInitializeNVPTXAsmParser function available.
4600	},
4601	.powerpc, .powerpcle, .powerpc64, .powerpc64le => {
4602	bindings.LLVMInitializePowerPCTarget();
4603	bindings.LLVMInitializePowerPCTargetInfo();
4604	bindings.LLVMInitializePowerPCTargetMC();
4605	bindings.LLVMInitializePowerPCAsmPrinter();
4606	bindings.LLVMInitializePowerPCAsmParser();
4607	},
4608	.riscv32, .riscv32be, .riscv64, .riscv64be => {
4609	bindings.LLVMInitializeRISCVTarget();
4610	bindings.LLVMInitializeRISCVTargetInfo();
4611	bindings.LLVMInitializeRISCVTargetMC();
4612	bindings.LLVMInitializeRISCVAsmPrinter();
4613	bindings.LLVMInitializeRISCVAsmParser();
4614	},
4615	.sparc, .sparc64 => {
4616	bindings.LLVMInitializeSparcTarget();
4617	bindings.LLVMInitializeSparcTargetInfo();
4618	bindings.LLVMInitializeSparcTargetMC();
4619	bindings.LLVMInitializeSparcAsmPrinter();
4620	bindings.LLVMInitializeSparcAsmParser();
4621	},
4622	.s390x => {
4623	bindings.LLVMInitializeSystemZTarget();
4624	bindings.LLVMInitializeSystemZTargetInfo();
4625	bindings.LLVMInitializeSystemZTargetMC();
4626	bindings.LLVMInitializeSystemZAsmPrinter();
4627	bindings.LLVMInitializeSystemZAsmParser();
4628	},
4629	.wasm32, .wasm64 => {
4630	bindings.LLVMInitializeWebAssemblyTarget();
4631	bindings.LLVMInitializeWebAssemblyTargetInfo();
4632	bindings.LLVMInitializeWebAssemblyTargetMC();
4633	bindings.LLVMInitializeWebAssemblyAsmPrinter();
4634	bindings.LLVMInitializeWebAssemblyAsmParser();
4635	},
4636	.x86, .x86_64 => {
4637	bindings.LLVMInitializeX86Target();
4638	bindings.LLVMInitializeX86TargetInfo();
4639	bindings.LLVMInitializeX86TargetMC();
4640	bindings.LLVMInitializeX86AsmPrinter();
4641	bindings.LLVMInitializeX86AsmParser();
4642	},
4643	.xtensa => {
4644	if (build_options.llvm_has_xtensa) {
4645	bindings.LLVMInitializeXtensaTarget();
4646	bindings.LLVMInitializeXtensaTargetInfo();
4647	bindings.LLVMInitializeXtensaTargetMC();
4648	bindings.LLVMInitializeXtensaAsmPrinter();
4649	bindings.LLVMInitializeXtensaAsmParser();
4650	}
4651	},
4652	.xcore => {
4653	bindings.LLVMInitializeXCoreTarget();
4654	bindings.LLVMInitializeXCoreTargetInfo();
4655	bindings.LLVMInitializeXCoreTargetMC();
4656	bindings.LLVMInitializeXCoreAsmPrinter();
4657	// There is no LLVMInitializeXCoreAsmParser function.
4658	},
4659	.m68k => {
4660	if (build_options.llvm_has_m68k) {
4661	bindings.LLVMInitializeM68kTarget();
4662	bindings.LLVMInitializeM68kTargetInfo();
4663	bindings.LLVMInitializeM68kTargetMC();
4664	bindings.LLVMInitializeM68kAsmPrinter();
4665	bindings.LLVMInitializeM68kAsmParser();
4666	}
4667	},
4668	.csky => {
4669	if (build_options.llvm_has_csky) {
4670	bindings.LLVMInitializeCSKYTarget();
4671	bindings.LLVMInitializeCSKYTargetInfo();
4672	bindings.LLVMInitializeCSKYTargetMC();
4673	// There is no LLVMInitializeCSKYAsmPrinter function.
4674	bindings.LLVMInitializeCSKYAsmParser();
4675	}
4676	},
4677	.ve => {
4678	bindings.LLVMInitializeVETarget();
4679	bindings.LLVMInitializeVETargetInfo();
4680	bindings.LLVMInitializeVETargetMC();
4681	bindings.LLVMInitializeVEAsmPrinter();
4682	bindings.LLVMInitializeVEAsmParser();
4683	},
4684	.arc => {
4685	if (build_options.llvm_has_arc) {
4686	bindings.LLVMInitializeARCTarget();
4687	bindings.LLVMInitializeARCTargetInfo();
4688	bindings.LLVMInitializeARCTargetMC();
4689	bindings.LLVMInitializeARCAsmPrinter();
4690	// There is no LLVMInitializeARCAsmParser function.
4691	}
4692	},
4693	.loongarch32, .loongarch64 => {
4694	bindings.LLVMInitializeLoongArchTarget();
4695	bindings.LLVMInitializeLoongArchTargetInfo();
4696	bindings.LLVMInitializeLoongArchTargetMC();
4697	bindings.LLVMInitializeLoongArchAsmPrinter();
4698	bindings.LLVMInitializeLoongArchAsmParser();
4699	},
4700	.spirv32,
4701	.spirv64,
4702	=> {
4703	bindings.LLVMInitializeSPIRVTarget();
4704	bindings.LLVMInitializeSPIRVTargetInfo();
4705	bindings.LLVMInitializeSPIRVTargetMC();
4706	bindings.LLVMInitializeSPIRVAsmPrinter();
4707	},
4708
4709	// LLVM does does not have a backend for these.
4710	.alpha,
4711	.arceb,
4712	.ez80,
4713	.hppa,
4714	.hppa64,
4715	.kalimba,
4716	.kvx,
4717	.m88k,
4718	.microblaze,
4719	.microblazeel,
4720	.or1k,
4721	.propeller,
4722	.sh,
4723	.sheb,
4724	.spork8,
4725	.x86_16,
4726	.xtensaeb,
4727	=> unreachable,
4728	}
4729	}