Commit 6d2c8349c53707f9eeee7554bd7717667718ce4d | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-17 12:11:23+01:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 10:20:56+01:00
log	6d2c8349c53707f9eeee7554bd7717667718ce4d
tree	e1ce8fdf46a3f0282c0ba32833b76cb7a1bf2184
parent	f9a6149b34af662a0d524f2f64f00e8857afc13b
signature	Commit is signed but in an unrecognized format.

compiler: fully switch to new `@bitCast` semantics

29 files changed, 3159 insertions(+), 2673 deletions(-)

CMakeLists.txt
src
- Air.zig
- Air
  - Legalize.zig
  - Liveness.zig
  - Liveness
    Verify.zig
  - Verify.zig
  - print.zig
- Sema.zig
- Sema
- Type.zig
- Value.zig
- Zcu
  - PerThread.zig
- codegen
  - aarch64
    Select.zig
    abi.zig
  - arm
    abi.zig
  - c.zig
  - c
    type.zig
  - llvm.zig
  - llvm
    FuncGen.zig
  - mips
    abi.zig
  - riscv64
    CodeGen.zig
    abi.zig
  - sparc64
    CodeGen.zig
  - spirv
    CodeGen.zig
  - wasm
    CodeGen.zig
  - x86_64
    CodeGen.zig

CMakeLists.txt+1-1

...	...	@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES
342	342	src/Package/Module.zig
343	343	src/RangeSet.zig
344	344	src/Sema.zig
345		src/Sema/bitcast.zig
	345	src/Sema/reinterpret.zig
346	346	src/Sema/comptime_ptr_access.zig
347	347	src/Sema/type_resolution.zig
348	348	src/Type.zig

src/Air.zig+67-12

...	...	@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");
17	17
18	18	pub const Legalize = @import("Air/Legalize.zig");
19	19	pub const Liveness = @import("Air/Liveness.zig");
	20	pub const Verify = @import("Air/Verify.zig");
20	21
21	22	instructions: std.MultiArrayList(Inst).Slice,
22	23	/// The meaning of this data is determined by `Inst.Tag` value.
...	...	@@ -276,10 +277,50 @@ pub const Inst = struct {
276	277	/// Boolean or binary NOT.
277	278	/// Uses the `ty_op` field.
278	279	not,
279		/// Reinterpret the bits of a value as a different type. This is like `@bitCast` but
280		/// also supports enums and pointers.
	280	/// Implements `@bitCast`.
	281	///
	282	/// Uses the `ty_op` field.
	283	bit_cast,
	284	/// Cast a pointer to a different pointer type. The result type is a slice iff the operand
	285	/// type is a slice (the length of the slice does not change). All other pointer attributes
	286	/// except for the address space may change.
	287	///
	288	/// Supports vectors of pointers.
	289	///
	290	/// Uses the `ty_op` field.
	291	ptr_cast,
	292	/// Cast an integer to a pointer (not a slice). Operand type is always `usize`.
	293	///
	294	/// Supports vectors of integers.
	295	///
	296	/// Uses the `ty_op` field.
	297	ptr_from_int,
	298	/// Cast a pointer (not a slice) to an integer. Result type is always `usize`.
	299	///
	300	/// Supports vectors of pointers.
	301	///
	302	/// Uses the `ty_op` field.
	303	int_from_ptr,
	304	/// Cast an error set `E1` to a different error set `E2`, or cast an error union `E1!T` to
	305	/// an error union `E2!T` with the same payload type but a different error set type.
	306	///
	307	/// Uses the `ty_op` field.
	308	error_cast,
	309	/// Cast an integer to an error set type. The integer operand type is unsigned and has bit
	310	/// width equal to `zcu.errorSetBits()`.
	311	///
	312	/// Uses the `ty_op` field.
	313	error_from_int,
	314	/// Cast an error set to an integer type. The integer destination type is unsigned and has
	315	/// bit width equal to `zcu.errorSetBits()`.
	316	///
	317	/// Uses the `ty_op` field.
	318	int_from_error,
	319	/// Cast an enum value to a tagged union, whose tag type is that enum, and which has no
	320	/// payload bits (i.e. all payloads are equivalent to `void`).
	321	///
281	322	/// Uses the `ty_op` field.
282		bitcast,
	323	union_from_enum,
283	324	/// A block runs its body which always ends with a `noreturn` instruction,
284	325	/// so the only way to proceed to the code after the `block` is to encounter a `br`
285	326	/// that targets this `block`. If the `block` type is `noreturn`,
...	...	@@ -589,13 +630,13 @@ pub const Inst = struct {
589	630	/// the integer tag type of the enum.
590	631	/// See `trunc` for integer truncation.
591	632	/// Uses the `ty_op` field.
592		intcast,
593		/// Like `intcast`, but includes two safety checks:
	633	int_cast,
	634	/// Like `int_cast`, but includes two safety checks:
594	635	/// * triggers a safety panic if the cast truncates bits
595	636	/// * triggers a safety panic if the destination type is an exhaustive enum
596	637	/// and the operand is not a valid value of this type; i.e. equivalent to
597	638	/// a safety check based on `.is_named_enum_value`
598		intcast_safe,
	639	int_cast_safe,
599	640	/// Truncate higher bits from an integer, resulting in an integer type with the same
600	641	/// sign but an equal or smaller number of bits.
601	642	/// Uses the `ty_op` field.
...	...	@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: const Air, inst: Air.Inst.Index, ip: const InternPool)
1667	1708	=> return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
1668	1709
1669	1710	.not,
1670		.bitcast,
	1711	.bit_cast,
	1712	.ptr_cast,
	1713	.ptr_from_int,
	1714	.int_from_ptr,
	1715	.error_cast,
	1716	.error_from_int,
	1717	.int_from_error,
	1718	.union_from_enum,
1671	1719	.load,
1672	1720	.fpext,
1673	1721	.fptrunc,
1674		.intcast,
1675		.intcast_safe,
	1722	.int_cast,
	1723	.int_cast_safe,
1676	1724	.trunc,
1677	1725	.optional_payload,
1678	1726	.optional_payload_ptr,
...	...	@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1913	1961	.add_safe,
1914	1962	.sub_safe,
1915	1963	.mul_safe,
1916		.intcast_safe,
	1964	.int_cast_safe,
1917	1965	.int_from_float_safe,
1918	1966	.int_from_float_optimized_safe,
1919	1967	.legalize_vec_store_elem,
...	...	@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1965	2013	.shl_sat,
1966	2014	.xor,
1967	2015	.not,
1968		.bitcast,
	2016	.bit_cast,
	2017	.ptr_cast,
	2018	.ptr_from_int,
	2019	.int_from_ptr,
	2020	.error_cast,
	2021	.error_from_int,
	2022	.int_from_error,
	2023	.union_from_enum,
1969	2024	.ret_addr,
1970	2025	.frame_addr,
1971	2026	.clz,
...	...	@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2009	2064	.is_non_err,
2010	2065	.fptrunc,
2011	2066	.fpext,
2012		.intcast,
	2067	.int_cast,
2013	2068	.trunc,
2014	2069	.optional_payload,
2015	2070	.optional_payload_ptr,

src/Air/Legalize.zig+133-74

...	...	@@ -75,6 +75,9 @@ pub const Feature = enum {
75	75	scalarize_shl_sat,
76	76	scalarize_xor,
77	77	scalarize_not,
	78	scalarize_ptr_cast,
	79	scalarize_ptr_from_int,
	80	scalarize_int_from_ptr,
78	81	scalarize_clz,
79	82	scalarize_ctz,
80	83	scalarize_popcount,
...	...	@@ -100,8 +103,8 @@ pub const Feature = enum {
100	103	scalarize_cmp_vector_optimized,
101	104	scalarize_fptrunc,
102	105	scalarize_fpext,
103		scalarize_intcast,
104		scalarize_intcast_safe,
	106	scalarize_int_cast,
	107	scalarize_int_cast_safe,
105	108	scalarize_trunc,
106	109	scalarize_int_from_float,
107	110	scalarize_int_from_float_optimized,
...	...	@@ -115,24 +118,24 @@ pub const Feature = enum {
115	118	scalarize_select,
116	119	scalarize_mul_add,
117	120
118		// Below are several different features for scalarizing `bitcast` in different scenarios. It is
	121	// Below are several different features for scalarizing `bit_cast` in different scenarios. It is
119	122	// valid to enable any combination of these features.
120	123
121		/// Scalarize `bitcast` where the operand or result type is an array.
122		scalarize_bitcast_array,
123		/// Scalarize `bitcast` where either:
	124	/// Scalarize `bit_cast` where the operand or result type is an array.
	125	scalarize_bit_cast_array,
	126	/// Scalarize `bit_cast` where either:
124	127	///
125	128	/// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
126	129	/// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
127	130	///
128		/// This effectively scalarizes any `bitcast` to/from a vector, unless the operation can be
	131	/// This effectively scalarizes any `bit_cast` to/from a vector, unless the operation can be
129	132	/// performed by bitcasting each vector element and returning a vector of the results.
130	133	///
131	134	/// If this feature is enabled, the following AIR instruction tags may be emitted:
132	135	/// * `.legalize_vec_elem_val`
133	136	/// * `.legalize_vec_store_elem`
134		scalarize_bitcast_vector_non_elementwise,
135		/// Scalarize `bitcast` where the operand or result type is an array or vector whose element
	137	scalarize_bit_cast_vector_non_elementwise,
	138	/// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
136	139	/// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
137	140	/// need to sign- or zero-extend multiple elements to populate "padding" bits.
138	141	///
...	...	@@ -142,18 +145,18 @@ pub const Feature = enum {
142	145	/// If this feature is enabled, the following AIR instruction tags may be emitted:
143	146	/// * `.legalize_vec_elem_val`
144	147	/// * `.legalize_vec_store_elem`
145		scalarize_bitcast_padded_elems,
	148	scalarize_bit_cast_padded_elems,
146	149
147	150	/// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
148	151	unsplat_shift_rhs,
149	152	/// Legalize reduce of a one element vector to a bitcast.
150		reduce_one_elem_to_bitcast,
	153	reduce_one_elem_to_bit_cast,
151	154	/// Legalize splat to a one element vector to a bitcast.
152		splat_one_elem_to_bitcast,
	155	splat_one_elem_to_bit_cast,
153	156
154		/// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.
155		/// Not compatible with `scalarize_intcast_safe`.
156		expand_intcast_safe,
	157	/// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
	158	/// Not compatible with `scalarize_int_cast_safe`.
	159	expand_int_cast_safe,
157	160	/// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
158	161	/// Not compatible with `scalarize_int_from_float_safe`.
159	162	expand_int_from_float_safe,
...	...	@@ -178,9 +181,9 @@ pub const Feature = enum {
178	181	/// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
179	182	/// first byte of memory until bit pointers know their backing type.
180	183	expand_packed_store,
181		/// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.
	184	/// Replace `struct_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
182	185	expand_packed_struct_field_val,
183		/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
	186	/// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
184	187	expand_packed_aggregate_init,
185	188
186	189	/// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
...	...	@@ -274,8 +277,11 @@ pub const Feature = enum {
274	277	.cmp_vector_optimized => .scalarize_cmp_vector_optimized,
275	278	.fptrunc => .scalarize_fptrunc,
276	279	.fpext => .scalarize_fpext,
277		.intcast => .scalarize_intcast,
278		.intcast_safe => .scalarize_intcast_safe,
	280	.int_cast => .scalarize_int_cast,
	281	.int_cast_safe => .scalarize_int_cast_safe,
	282	.ptr_cast => .scalarize_ptr_cast,
	283	.ptr_from_int => .scalarize_ptr_from_int,
	284	.int_from_ptr => .scalarize_int_from_ptr,
279	285	.trunc => .scalarize_trunc,
280	286	.int_from_float => .scalarize_int_from_float,
281	287	.int_from_float_optimized => .scalarize_int_from_float_optimized,
...	...	@@ -495,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
495	501	.popcount,
496	502	.byte_swap,
497	503	.bit_reverse,
498		.intcast,
	504	.int_cast,
	505	.ptr_cast,
	506	.ptr_from_int,
	507	.int_from_ptr,
499	508	.trunc,
500	509	=> \|air_tag\| if (l.features.has(comptime .scalarize(air_tag))) {
501	510	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
...	...	@@ -569,19 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
569	578	},
570	579	}
571	580	},
572		.bitcast => if (l.features.hasAny(&.{
573		.scalarize_bitcast_array,
574		.scalarize_bitcast_vector_non_elementwise,
575		.scalarize_bitcast_padded_elems,
	581	.bit_cast => if (l.features.hasAny(&.{
	582	.scalarize_bit_cast_array,
	583	.scalarize_bit_cast_vector_non_elementwise,
	584	.scalarize_bit_cast_padded_elems,
576	585	})) {
577	586	if (try l.scalarizeBitcastBlockPayload(inst)) \|payload\| {
578	587	continue :inst l.replaceInst(inst, .block, payload);
579	588	}
580	589	},
581		.intcast_safe => if (l.features.has(.expand_intcast_safe)) {
582		assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both
	590	.int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
	591	assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
583	592	continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
584		} else if (l.features.has(.scalarize_intcast_safe)) {
	593	} else if (l.features.has(.scalarize_int_cast_safe)) {
585	594	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
586	595	if (ty_op.ty.toType().isVector(zcu)) {
587	596	continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
...	...	@@ -797,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
797	806	inline .reduce, .reduce_optimized => \|air_tag\| {
798	807	const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
799	808	const vector_ty = l.typeOf(reduce.operand);
800		if (l.features.has(.reduce_one_elem_to_bitcast)) {
	809	if (l.features.has(.reduce_one_elem_to_bit_cast)) {
801	810	switch (vector_ty.vectorLen(zcu)) {
802	811	0 => unreachable,
803		1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	812	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
804	813	.ty = .fromType(vector_ty.childType(zcu)),
805	814	.operand = reduce.operand,
806	815	} }),
...	...	@@ -817,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
817	826	.soft_float => unreachable, // the operand is not a scalar
818	827	}
819	828	},
820		.splat => if (l.features.has(.splat_one_elem_to_bitcast)) {
	829	.splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
821	830	const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
822	831	switch (ty_op.ty.toType().vectorLen(zcu)) {
823	832	0 => unreachable,
824		1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	833	1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
825	834	.ty = ty_op.ty,
826	835	.operand = ty_op.operand,
827	836	} }),
...	...	@@ -887,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
887	896	const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
888	897	if (field_bits == struct_bits) {
889	898	// Just bitcast this field.
890		continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
	899	continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
891	900	.ty = .fromType(agg_ty),
892	901	.operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),
893	902	} });
...	...	@@ -934,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
934	943	.legalize_vec_store_elem,
935	944	.legalize_compiler_rt_call,
936	945	.spirv_runtime_array_len,
	946	.error_cast,
	947	.error_from_int,
	948	.int_from_error,
	949	.union_from_enum,
937	950	=> {},
938	951	}
939	952	}
...	...	@@ -956,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz
956	969
957	970	if (result_is_array) {
958	971	// This is only allowed when legalizing an elementwise bitcast.
959		assert(orig.tag == .bitcast);
	972	assert(orig.tag == .bit_cast);
960	973	assert(form == .ty_op);
961	974	}
962	975
...	...	@@ -1474,7 +1487,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1474	1487	var operand_to_int: bool = true;
1475	1488	var int_to_dest: bool = true;
1476	1489
1477		if (l.features.has(.scalarize_bitcast_array)) {
	1490	if (l.features.has(.scalarize_bit_cast_array)) {
1478	1491	if (operand_tag == .array) {
1479	1492	operand_to_dest = false;
1480	1493	operand_to_int = false;
...	...	@@ -1485,7 +1498,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1485	1498	}
1486	1499	}
1487	1500
1488		if (l.features.has(.scalarize_bitcast_vector_non_elementwise)) {
	1501	if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
1489	1502	if (operand_tag == .vector) operand_to_int = false;
1490	1503	if (dest_tag == .vector) int_to_dest = false;
1491	1504
...	...	@@ -1499,7 +1512,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1499	1512	}
1500	1513	}
1501	1514
1502		if (l.features.has(.scalarize_bitcast_padded_elems)) {
	1515	if (l.features.has(.scalarize_bit_cast_padded_elems)) {
1503	1516	if (operand_tag == .array or operand_tag == .vector) {
1504	1517	const elem_ty = operand_ty.childType(zcu);
1505	1518	if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
...	...	@@ -1554,6 +1567,12 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1554	1567	break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1555	1568	}
1556	1569
	1570	if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
	1571	_ = main_block.stealCapacity(18);
	1572	const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
	1573	break :uint_val main_block.addBitCast(l, uint_ty, elem);
	1574	}
	1575
1557	1576	// %1 = block({
1558	1577	// %2 = alloc(*usize)
1559	1578	// %3 = alloc(*uN)
...	...	@@ -1562,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1562	1581	// %6 = loop({
1563	1582	// %7 = load(%2)
1564	1583	// %8 = array_elem_val(orig_operand, %7)
1565		// %9 = bitcast(uE, %8)
1566		// %10 = intcast(uN, %9)
	1584	// %9 = bit_cast(uE, %8)
	1585	// %10 = int_cast(uN, %9)
1567	1586	// %11 = load(%3)
1568	1587	// %12 = shl_exact(%11, <uS, E>)
1569	1588	// %13 = bit_or(%12, %10)
...	...	@@ -1613,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1613	1632	index_val,
1614	1633	).toRef();
1615	1634	const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1616		const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();
	1635	const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
1617	1636	const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1618	1637	const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1619	1638	const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
...	...	@@ -1648,6 +1667,19 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1648	1667	_ = main_block.stealCapacity(17);
1649	1668	const result = main_block.addBitCast(l, dest_ty, uint_val);
1650	1669	main_block.addBr(l, orig_inst, result);
	1670	} else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
	1671	_ = main_block.stealCapacity(16);
	1672	const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
	1673	const aggregate_init_payload_start = l.air_extra.items.len;
	1674	try l.air_extra.append(zcu.gpa, @intFromEnum(elem));
	1675	const result = main_block.add(l, .{
	1676	.tag = .aggregate_init,
	1677	.data = .{ .ty_pl = .{
	1678	.ty = .fromType(dest_ty),
	1679	.payload = @intCast(aggregate_init_payload_start),
	1680	} },
	1681	}).toRef();
	1682	main_block.addBr(l, orig_inst, result);
1651	1683	} else {
1652	1684	// %1 = alloc(*usize)
1653	1685	// %2 = alloc(*@Vector(N, Result))
...	...	@@ -1655,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1655	1687	// %4 = loop({
1656	1688	// %5 = load(%1)
1657	1689	// %6 = mul(%5, <usize, E>)
1658		// %7 = intcast(uS, %6)
	1690	// %7 = int_cast(uS, %6)
1659	1691	// %8 = shr(uint_val, %7)
1660	1692	// %9 = trunc(uE, %8)
1661		// %10 = bitcast(Result, %9)
	1693	// %10 = bit_cast(Result, %9)
1662	1694	// %11 = legalize_vec_store_elem(%2, %5, %10)
1663	1695	// %12 = cmp_eq(%5, <usize, vec_len>)
1664	1696	// %13 = cond_br(%12, {
...	...	@@ -1687,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1687	1719
1688	1720	const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1689	1721	const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1690		const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();
	1722	const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
1691	1723	const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
1692	1724	const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1693	1725	const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
...	...	@@ -2065,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2065	2097	// %5 = call(@panic.invalidEnumValue, [])
2066	2098	// %6 = unreach()
2067	2099	// }, {
2068		// %7 = intcast(@res_ty, %y)
	2100	// %7 = int_cast(@res_ty, %y)
2069	2101	// %8 = is_named_enum_value(%7)
2070	2102	// %9 = cond_br(%8, {
2071	2103	// %10 = br(%x, %7)
...	...	@@ -2087,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2087	2119	// %6 = call(@panic.invalidEnumValue, [])
2088	2120	// %7 = unreach()
2089	2121	// }, {
2090		// %8 = intcast(@res_ty, %y)
	2122	// %8 = int_cast(@res_ty, %y)
2091	2123	// %9 = br(%x, %8)
2092	2124	// })
2093	2125	// })
...	...	@@ -2140,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2140	2172	cur_block = &condbr.else_block;
2141	2173	}
2142	2174
2143		// Now we know we're in-range, we can intcast:
	2175	// Now we know we're in-range, we can int_cast:
2144	2176	const cast_inst = cur_block.add(l, .{
2145		.tag = .intcast,
	2177	.tag = .int_cast,
2146	2178	.data = .{ .ty_op = .{
2147	2179	.ty = Air.internedToRef(dest_ty.toIntern()),
2148	2180	.operand = operand_ref,
...	...	@@ -2313,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2313	2345	// %1 = add_with_overflow(%x, %y)
2314	2346	// %2 = struct_field_val(%1, .@"1")
2315	2347	// %3 = reduce(%2, .@"or")
2316		// %4 = bitcast(%3, @bool_type)
	2348	// %4 = bit_cast(%3, @bool_type)
2317	2349	// %5 = cond_br(%4, {
2318	2350	// %6 = call(@panic.integerOverflow, [])
2319	2351	// %7 = unreach()
...	...	@@ -2419,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins
2419	2451	.tag = .load,
2420	2452	.data = .{ .ty_op = .{
2421	2453	.ty = Air.internedToRef(load_ty.toIntern()),
2422		.operand = res_block.addBitCast(l, load_ptr_ty: {
	2454	.operand = res_block.addPtrCast(l, load_ptr_ty: {
2423	2455	var load_ptr_info = ptr_info;
2424	2456	load_ptr_info.child = load_ty.toIntern();
2425	2457	load_ptr_info.flags.vector_index = .none;
...	...	@@ -2462,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2462	2494
2463	2495	var res_block: Block = .init(&inst_buf);
2464	2496	{
2465		const backing_ptr_inst = res_block.add(l, .{
2466		.tag = .bitcast,
2467		.data = .{ .ty_op = .{
2468		.ty = Air.internedToRef((load_store_ptr_ty: {
2469		var load_ptr_info = ptr_info;
2470		load_ptr_info.child = load_store_ty.toIntern();
2471		load_ptr_info.flags.vector_index = .none;
2472		load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2473		break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2474		}).toIntern()),
2475		.operand = orig_bin_op.lhs,
2476		} },
2477		});
	2497	const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
	2498	var load_ptr_info = ptr_info;
	2499	load_ptr_info.child = load_store_ty.toIntern();
	2500	load_ptr_info.flags.vector_index = .none;
	2501	load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
	2502	break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
	2503	}, orig_bin_op.lhs);
2478	2504	_ = res_block.add(l, .{
2479	2505	.tag = .store,
2480	2506	.data = .{ .bin_op = .{
2481		.lhs = backing_ptr_inst.toRef(),
	2507	.lhs = backing_ptr,
2482	2508	.rhs = res_block.add(l, .{
2483	2509	.tag = .bit_or,
2484	2510	.data = .{ .bin_op = .{
...	...	@@ -2489,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2489	2515	.tag = .load,
2490	2516	.data = .{ .ty_op = .{
2491	2517	.ty = Air.internedToRef(load_store_ty.toIntern()),
2492		.operand = backing_ptr_inst.toRef(),
	2518	.operand = backing_ptr,
2493	2519	} },
2494	2520	}).toRef(),
2495	2521	.rhs = Air.internedToRef((keep_mask: {
...	...	@@ -2518,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2518	2544	.tag = .shl_exact,
2519	2545	.data = .{ .bin_op = .{
2520	2546	.lhs = res_block.add(l, .{
2521		.tag = .intcast,
	2547	.tag = .int_cast,
2522	2548	.data = .{ .ty_op = .{
2523	2549	.ty = Air.internedToRef(load_store_ty.toIntern()),
2524	2550	.operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
...	...	@@ -2616,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro
2616	2642
2617	2643	const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2618	2644	const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2619		const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();
	2645	const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
2620	2646	cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2621	2647	}
2622	2648
...	...	@@ -2805,18 +2831,51 @@ const Block = struct {
2805	2831	});
2806	2832	}
2807	2833
2808		/// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for
	2834	/// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2809	2835	/// no-op casts.
2810	2836	fn addBitCast(
2811	2837	b: *Block,
2812	2838	l: *Legalize,
2813		ty: Type,
	2839	result_ty: Type,
2814	2840	operand: Air.Inst.Ref,
2815	2841	) Air.Inst.Ref {
2816		if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{
2817		.tag = .bitcast,
	2842	const zcu = l.pt.zcu;
	2843	const operand_ty = l.typeOf(operand);
	2844	assert(!operand_ty.isPtrAtRuntime(zcu));
	2845	assert(!operand_ty.isSliceAtRuntime(zcu));
	2846	assert(!result_ty.isPtrAtRuntime(zcu));
	2847	assert(!result_ty.isSliceAtRuntime(zcu));
	2848	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
	2849	.tag = .bit_cast,
	2850	.data = .{ .ty_op = .{
	2851	.ty = .fromType(result_ty),
	2852	.operand = operand,
	2853	} },
	2854	}).toRef();
	2855	_ = b.stealCapacity(1);
	2856	return operand;
	2857	}
	2858
	2859	/// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
	2860	/// no-op casts.
	2861	fn addPtrCast(
	2862	b: *Block,
	2863	l: *Legalize,
	2864	result_ty: Type,
	2865	operand: Air.Inst.Ref,
	2866	) Air.Inst.Ref {
	2867	const zcu = l.pt.zcu;
	2868	const operand_ty = l.typeOf(operand);
	2869	if (operand_ty.isSliceAtRuntime(zcu)) {
	2870	assert(result_ty.isSliceAtRuntime(zcu));
	2871	} else {
	2872	assert(operand_ty.isPtrAtRuntime(zcu));
	2873	assert(result_ty.isPtrAtRuntime(zcu));
	2874	}
	2875	if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
	2876	.tag = .ptr_cast,
2818	2877	.data = .{ .ty_op = .{
2819		.ty = Air.internedToRef(ty.toIntern()),
	2878	.ty = .fromType(result_ty),
2820	2879	.operand = operand,
2821	2880	} },
2822	2881	}).toRef();
...	...	@@ -3157,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3157	3216	var main_block: Block = .init(&inst_buf);
3158	3217	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3159	3218
3160		const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
	3219	const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3161	3220	const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3162	3221	const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3163	3222	main_block.addBr(l, orig_inst, casted_result);
...	...	@@ -3184,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3184	3243	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3185	3244
3186	3245	const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3187		break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
	3246	break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3188	3247	} else ext: {
3189	3248	_ = main_block.stealCapacity(1);
3190	3249	break :ext ty_op.operand;
...	...	@@ -3249,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3249	3308	try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3250	3309
3251	3310	const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3252		const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();
	3311	const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
3253	3312	main_block.addBr(l, orig_inst, casted_val);
3254	3313
3255	3314	return .{ .block_payload = .{ .ty_pl = .{
...	...	@@ -3273,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3273	3332	const bits_val = try pt.intValue(.usize, dest_info.bits);
3274	3333	_ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3275	3334	const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3276		const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();
	3335	const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
3277	3336	main_block.addBr(l, orig_inst, result_val);
3278	3337
3279	3338	return .{ .block_payload = .{ .ty_pl = .{

src/Air/Liveness.zig+10-3

...	...	@@ -488,12 +488,19 @@ fn analyzeInst(
488	488	=> return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
489	489
490	490	.not,
491		.bitcast,
	491	.bit_cast,
	492	.ptr_cast,
	493	.ptr_from_int,
	494	.int_from_ptr,
	495	.error_cast,
	496	.error_from_int,
	497	.int_from_error,
	498	.union_from_enum,
492	499	.load,
493	500	.fpext,
494	501	.fptrunc,
495		.intcast,
496		.intcast_safe,
	502	.int_cast,
	503	.int_cast_safe,
497	504	.trunc,
498	505	.optional_payload,
499	506	.optional_payload_ptr,

src/Air/Liveness/Verify.zig+10-3

...	...	@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
78	78
79	79	// unary
80	80	.not,
81		.bitcast,
	81	.bit_cast,
	82	.ptr_cast,
	83	.ptr_from_int,
	84	.int_from_ptr,
	85	.error_cast,
	86	.error_from_int,
	87	.int_from_error,
	88	.union_from_enum,
82	89	.load,
83	90	.fpext,
84	91	.fptrunc,
85		.intcast,
86		.intcast_safe,
	92	.int_cast,
	93	.int_cast_safe,
87	94	.trunc,
88	95	.optional_payload,
89	96	.optional_payload_ptr,

src/Air/Verify.zig created+465

...	...	@@ -0,0 +1,465 @@
	1	/// Verifies that AIR is valid, in that every instruction has valid operands and types. In compiler
	2	/// builds with debug extensions, this is run on all AIR, both before `Air.Legalize` is run and (if
	3	/// it is run) after it.
	4	///
	5	/// This verification pass is currently highly incomplete---expand it as needed.
	6	const Verify = @This();
	7
	8	zcu: *Zcu,
	9	func_index: InternPool.Index,
	10	ret_ty: Type,
	11	air: *const Air,
	12	cur_inst: Air.Inst.Index,
	13
	14	pub fn run(pt: Zcu.PerThread, func_index: InternPool.Index, air: *const Air) void {
	15	if (!@import("build_options").enable_debug_extensions) {
	16	// `Air.Verify` is a debugging feature---it should not be used in release builds because it
	17	// has little benefit and negatively affects compiler performance.
	18	return;
	19	}
	20
	21	const zcu = pt.zcu;
	22
	23	const func_ty: Type = Value.fromInterned(func_index).typeOf(zcu);
	24	const ret_ty = func_ty.fnReturnType(zcu);
	25
	26	var verify: Verify = .{
	27	.zcu = zcu,
	28	.func_index = func_index,
	29	.ret_ty = ret_ty,
	30	.air = air,
	31	.cur_inst = undefined, // populated by `body(...)`
	32	};
	33	verify.body(air.getMainBody()) catch \|verify_err\| switch (verify_err) {
	34	error.VerifyFail => {
	35	const ip = &zcu.intern_pool;
	36	const func_nav = ip.indexToKey(func_index).func.owner_nav;
	37	const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
	38	log.info("AIR for '{s}':", .{func_fqn});
	39	const io = zcu.comp.io;
	40	const stderr = io.lockStderr(&.{}, null) catch \|err\| switch (err) {
	41	error.Canceled => return io.recancel(),
	42	};
	43	defer io.unlockStderr();
	44	air.write(&stderr.file_writer.interface, pt, null) catch \|err\| switch (err) {
	45	error.WriteFailed => switch (stderr.file_writer.err.?) {
	46	error.Canceled => return io.recancel(),
	47	else => {},
	48	},
	49	};
	50	},
	51	};
	52	}
	53
	54	const Error = error{VerifyFail};
	55
	56	fn fail(verify: *Verify, msg: []const u8) Error {
	57	const ip = &verify.zcu.intern_pool;
	58	const func_nav = ip.indexToKey(verify.func_index).func.owner_nav;
	59	const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
	60	log.err("'{s}', %{d}: {s}", .{ func_fqn, verify.cur_inst, msg });
	61	return error.VerifyFail;
	62	}
	63
	64	fn body(verify: *Verify, body_insts: []const Air.Inst.Index) Error!void {
	65	const zcu = verify.zcu;
	66	const ip = &zcu.intern_pool;
	67	const air = verify.air;
	68	const tags = air.instructions.items(.tag);
	69	const data = air.instructions.items(.data);
	70	for (body_insts, 0..) \|inst, body_index\| {
	71	verify.cur_inst = inst;
	72	switch (tags[@intFromEnum(inst)]) {
	73	.block => {
	74	const block = air.unwrapBlock(inst);
	75	try verify.body(block.body);
	76	},
	77	.dbg_inline_block => {
	78	const block = air.unwrapDbgBlock(inst);
	79	try verify.body(block.body);
	80	},
	81	.@"try", .try_cold => {
	82	const @"try" = air.unwrapTry(inst);
	83	try verify.body(@"try".else_body);
	84	},
	85	.try_ptr, .try_ptr_cold => {
	86	const try_ptr = air.unwrapTryPtr(inst);
	87	try verify.body(try_ptr.else_body);
	88	},
	89	.loop => {
	90	const block = air.unwrapBlock(inst);
	91	try verify.body(block.body);
	92	},
	93	.cond_br => {
	94	const cond_br = air.unwrapCondBr(inst);
	95	try verify.body(cond_br.then_body);
	96	try verify.body(cond_br.else_body);
	97	},
	98	.switch_br, .loop_switch_br => {
	99	const switch_br = air.unwrapSwitch(inst);
	100	var it = switch_br.iterateCases();
	101	while (it.next()) \|case\| {
	102	try verify.body(case.body);
	103	}
	104	const else_body = it.elseBody();
	105	if (else_body.len > 0) {
	106	try verify.body(else_body);
	107	}
	108	},
	109	.ret, .ret_safe => {
	110	const operand = data[@intFromEnum(inst)].un_op;
	111	if (air.typeOf(operand, ip).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
	112	},
	113	.ret_load => {
	114	const operand = data[@intFromEnum(inst)].un_op;
	115	const ptr_ty = air.typeOf(operand, ip);
	116	if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("operand is not a pointer");
	117	if (ptr_ty.ptrSize(zcu) != .one) return verify.fail("pointer size is not '.one'");
	118	if (ptr_ty.childType(zcu).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
	119	},
	120
	121	.bit_cast => {
	122	const ty_op = data[@intFromEnum(inst)].ty_op;
	123	const operand_ty = air.typeOf(ty_op.operand, ip);
	124	const result_ty = ty_op.ty.toType();
	125	// Enums are allowed here even if their backing type is implicit.
	126	if (!operand_ty.hasBitRepresentation(zcu) and operand_ty.zigTypeTag(zcu) != .@"enum") {
	127	return verify.fail("bad operand type");
	128	}
	129	if (!result_ty.hasBitRepresentation(zcu) and result_ty.zigTypeTag(zcu) != .@"enum") {
	130	return verify.fail("bad result type");
	131	}
	132	if (operand_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type (pointer)");
	133	if (result_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type (pointer)");
	134	if (operand_ty.bitSize(zcu) != result_ty.bitSize(zcu)) return verify.fail("bit size mismatch");
	135	},
	136	.ptr_cast => {
	137	const ty_op = data[@intFromEnum(inst)].ty_op;
	138	const operand_ty = air.typeOf(ty_op.operand, ip);
	139	const result_ty = ty_op.ty.toType();
	140	const operand_scalar_ty = operand_ty.scalarType(zcu);
	141	const result_scalar_ty = result_ty.scalarType(zcu);
	142	if (operand_ty.isSliceAtRuntime(zcu)) {
	143	if (!result_ty.isSliceAtRuntime(zcu)) return verify.fail("operand is slice, but result is not");
	144	} else {
	145	if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
	146	if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("operand is pointer, but result is not");
	147	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	148	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	149	}
	150	if (operand_scalar_ty.ptrAddressSpace(zcu) != result_scalar_ty.ptrAddressSpace(zcu)) {
	151	return verify.fail("illegal change to address space");
	152	}
	153	},
	154	.ptr_from_int => {
	155	const ty_op = data[@intFromEnum(inst)].ty_op;
	156	const operand_ty = air.typeOf(ty_op.operand, ip);
	157	const result_ty = ty_op.ty.toType();
	158	const operand_scalar_ty = operand_ty.scalarType(zcu);
	159	const result_scalar_ty = result_ty.scalarType(zcu);
	160	if (operand_scalar_ty.toIntern() != .usize_type) return verify.fail("bad operand type");
	161	if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type");
	162	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	163	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	164	},
	165	.int_from_ptr => {
	166	const ty_op = data[@intFromEnum(inst)].ty_op;
	167	const operand_ty = air.typeOf(ty_op.operand, ip);
	168	const result_ty = ty_op.ty.toType();
	169	const operand_scalar_ty = operand_ty.scalarType(zcu);
	170	const result_scalar_ty = result_ty.scalarType(zcu);
	171	if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
	172	if (result_scalar_ty.toIntern() != .usize_type) return verify.fail("bad result type");
	173	if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
	174	if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
	175	},
	176	.error_cast => {
	177	const ty_op = data[@intFromEnum(inst)].ty_op;
	178	const operand_ty = air.typeOf(ty_op.operand, ip);
	179	const result_ty = ty_op.ty.toType();
	180	switch (operand_ty.zigTypeTag(zcu)) {
	181	else => return verify.fail("bad operand type"),
	182	.error_union => {
	183	if (result_ty.zigTypeTag(zcu) != .error_union) {
	184	return verify.fail("operand is error union, but result is not");
	185	}
	186	if (operand_ty.errorUnionPayload(zcu).toIntern() != result_ty.errorUnionPayload(zcu).toIntern()) {
	187	return verify.fail("error union payload type differs");
	188	}
	189	},
	190	.error_set => if (result_ty.zigTypeTag(zcu) != .error_set) {
	191	return verify.fail("operand is error set, but result is not");
	192	},
	193	}
	194	},
	195	.error_from_int => {
	196	const ty_op = data[@intFromEnum(inst)].ty_op;
	197	const operand_ty = air.typeOf(ty_op.operand, ip);
	198	const result_ty = ty_op.ty.toType();
	199	if (!operand_ty.isUnsignedInt(zcu)) return verify.fail("bad operand type");
	200	if (operand_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad operand bit size");
	201	if (result_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad result type");
	202	},
	203	.int_from_error => {
	204	const ty_op = data[@intFromEnum(inst)].ty_op;
	205	const operand_ty = air.typeOf(ty_op.operand, ip);
	206	const result_ty = ty_op.ty.toType();
	207	if (operand_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad operand type");
	208	if (!result_ty.isUnsignedInt(zcu)) return verify.fail("bad result type");
	209	if (result_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad result bit size");
	210	},
	211	.union_from_enum => {
	212	const ty_op = data[@intFromEnum(inst)].ty_op;
	213	const operand_ty = air.typeOf(ty_op.operand, ip);
	214	const result_ty = ty_op.ty.toType();
	215	if (operand_ty.zigTypeTag(zcu) != .@"enum") return verify.fail("bad operand type");
	216	if (result_ty.zigTypeTag(zcu) != .@"union") return verify.fail("bad result type");
	217	const union_tag_ty = result_ty.unionTagType(zcu) orelse return verify.fail("union type is not tagged");
	218	if (union_tag_ty.toIntern() != operand_ty.toIntern()) return verify.fail("union tag type does not match operand type");
	219	},
	220
	221	.ptr_elem_ptr => {
	222	const ty_pl = data[@intFromEnum(inst)].ty_pl;
	223	const bin_op = air.extraData(Air.Bin, ty_pl.payload).data;
	224	const ptr_ty = air.typeOf(bin_op.lhs, ip);
	225	const result_ty = ty_pl.ty.toType();
	226	if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad pointer type");
	227	if (result_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad result type");
	228	const ptr_info = ptr_ty.ptrInfo(zcu);
	229	const result_ptr_info = result_ty.ptrInfo(zcu);
	230	if (ptr_info.packed_offset.host_size != 0) return verify.fail("pointer type is bitpacked pointer");
	231	if (result_ptr_info.packed_offset.host_size != 0) return verify.fail("result type is bitpacked pointer");
	232	},
	233
	234	.arg,
	235	.add,
	236	.add_safe,
	237	.add_optimized,
	238	.add_wrap,
	239	.add_sat,
	240	.sub,
	241	.sub_safe,
	242	.sub_optimized,
	243	.sub_wrap,
	244	.sub_sat,
	245	.mul,
	246	.mul_safe,
	247	.mul_optimized,
	248	.mul_wrap,
	249	.mul_sat,
	250	.div_float,
	251	.div_float_optimized,
	252	.div_trunc,
	253	.div_trunc_optimized,
	254	.div_floor,
	255	.div_floor_optimized,
	256	.div_exact,
	257	.div_exact_optimized,
	258	.rem,
	259	.rem_optimized,
	260	.mod,
	261	.mod_optimized,
	262	.ptr_add,
	263	.ptr_sub,
	264	.max,
	265	.min,
	266	.add_with_overflow,
	267	.sub_with_overflow,
	268	.mul_with_overflow,
	269	.shl_with_overflow,
	270	.alloc,
	271	.inferred_alloc,
	272	.inferred_alloc_comptime,
	273	.ret_ptr,
	274	.assembly,
	275	.bit_and,
	276	.bit_or,
	277	.shr,
	278	.shr_exact,
	279	.shl,
	280	.shl_exact,
	281	.shl_sat,
	282	.xor,
	283	.not,
	284	.repeat,
	285	.br,
	286	.trap,
	287	.breakpoint,
	288	.ret_addr,
	289	.frame_addr,
	290	.call,
	291	.call_always_tail,
	292	.call_never_tail,
	293	.call_never_inline,
	294	.clz,
	295	.ctz,
	296	.popcount,
	297	.byte_swap,
	298	.bit_reverse,
	299	.sqrt,
	300	.sin,
	301	.cos,
	302	.tan,
	303	.exp,
	304	.exp2,
	305	.log,
	306	.log2,
	307	.log10,
	308	.abs,
	309	.floor,
	310	.ceil,
	311	.round,
	312	.trunc_float,
	313	.neg,
	314	.neg_optimized,
	315	.cmp_lt,
	316	.cmp_lt_optimized,
	317	.cmp_lte,
	318	.cmp_lte_optimized,
	319	.cmp_eq,
	320	.cmp_eq_optimized,
	321	.cmp_gte,
	322	.cmp_gte_optimized,
	323	.cmp_gt,
	324	.cmp_gt_optimized,
	325	.cmp_neq,
	326	.cmp_neq_optimized,
	327	.cmp_vector,
	328	.cmp_vector_optimized,
	329	.switch_dispatch,
	330	.dbg_stmt,
	331	.dbg_empty_stmt,
	332	.dbg_var_ptr,
	333	.dbg_var_val,
	334	.dbg_arg_inline,
	335	.is_null,
	336	.is_non_null,
	337	.is_null_ptr,
	338	.is_non_null_ptr,
	339	.is_err,
	340	.is_non_err,
	341	.is_err_ptr,
	342	.is_non_err_ptr,
	343	.load,
	344	.store,
	345	.store_safe,
	346	.unreach,
	347	.fptrunc,
	348	.fpext,
	349	.int_cast,
	350	.int_cast_safe,
	351	.trunc,
	352	.optional_payload,
	353	.optional_payload_ptr,
	354	.optional_payload_ptr_set,
	355	.wrap_optional,
	356	.unwrap_errunion_payload,
	357	.unwrap_errunion_err,
	358	.unwrap_errunion_payload_ptr,
	359	.unwrap_errunion_err_ptr,
	360	.errunion_payload_ptr_set,
	361	.wrap_errunion_payload,
	362	.wrap_errunion_err,
	363	.struct_field_ptr,
	364	.struct_field_ptr_index_0,
	365	.struct_field_ptr_index_1,
	366	.struct_field_ptr_index_2,
	367	.struct_field_ptr_index_3,
	368	.struct_field_val,
	369	.set_union_tag,
	370	.get_union_tag,
	371	.slice,
	372	.slice_len,
	373	.slice_ptr,
	374	.ptr_slice_len_ptr,
	375	.ptr_slice_ptr_ptr,
	376	.array_elem_val,
	377	.slice_elem_val,
	378	.slice_elem_ptr,
	379	.ptr_elem_val,
	380	.array_to_slice,
	381	.int_from_float,
	382	.int_from_float_optimized,
	383	.int_from_float_safe,
	384	.int_from_float_optimized_safe,
	385	.float_from_int,
	386	.reduce,
	387	.reduce_optimized,
	388	.splat,
	389	.shuffle_one,
	390	.shuffle_two,
	391	.select,
	392	.memset,
	393	.memset_safe,
	394	.memcpy,
	395	.memmove,
	396	.cmpxchg_weak,
	397	.cmpxchg_strong,
	398	.atomic_load,
	399	.atomic_store_unordered,
	400	.atomic_store_monotonic,
	401	.atomic_store_release,
	402	.atomic_store_seq_cst,
	403	.atomic_rmw,
	404	.is_named_enum_value,
	405	.tag_name,
	406	.error_name,
	407	.error_set_has_value,
	408	.aggregate_init,
	409	.union_init,
	410	.prefetch,
	411	.mul_add,
	412	.field_parent_ptr,
	413	.wasm_memory_size,
	414	.wasm_memory_grow,
	415	.cmp_lte_errors_len,
	416	.err_return_trace,
	417	.set_err_return_trace,
	418	.addrspace_cast,
	419	.save_err_return_trace_index,
	420	.runtime_nav_ptr,
	421	.c_va_arg,
	422	.c_va_copy,
	423	.c_va_end,
	424	.c_va_start,
	425	.spirv_runtime_array_len,
	426	.work_item_id,
	427	.work_group_size,
	428	.work_group_id,
	429	.legalize_vec_store_elem,
	430	.legalize_vec_elem_val,
	431	.legalize_compiler_rt_call,
	432	=> {},
	433	}
	434	if (air.typeOfIndex(inst, ip).isNoReturn(zcu)) {
	435	if (body_index == body_insts.len - 1) return;
	436
	437	// HACK: right now, we emit the safety check for noreturn functions returning in a weird
	438	// way, where the `call` instruction is `noreturn` but there are still instructions
	439	// following it. We need to figure out a better way to represent that! That safety check
	440	// probably just needs to live exclusively in backends; putting AIR instructions after a
	441	// call implies that we have e.g. a valid stack at that point, which we can't actually
	442	// assume when the user has gotten a function's ABI wrong.
	443	switch (tags[@intFromEnum(inst)]) {
	444	.call,
	445	.call_always_tail,
	446	.call_never_tail,
	447	.call_never_inline,
	448	=> continue,
	449	else => {},
	450	}
	451
	452	return verify.fail("body contains instructions after noreturn");
	453	}
	454	}
	455	return verify.fail("body does not terminate noreturn");
	456	}
	457
	458	const std = @import("std");
	459	const log = std.log.scoped(.air_verify);
	460
	461	const Zcu = @import("../Zcu.zig");
	462	const InternPool = @import("../InternPool.zig");
	463	const Air = @import("../Air.zig");
	464	const Type = @import("../Type.zig");
	465	const Value = @import("../Value.zig");

src/Air/print.zig+10-3

...	...	@@ -232,12 +232,19 @@ const Writer = struct {
232	232	.arg => try w.writeArg(s, inst),
233	233
234	234	.not,
235		.bitcast,
	235	.bit_cast,
	236	.ptr_cast,
	237	.ptr_from_int,
	238	.int_from_ptr,
	239	.error_cast,
	240	.error_from_int,
	241	.int_from_error,
	242	.union_from_enum,
236	243	.load,
237	244	.fptrunc,
238	245	.fpext,
239		.intcast,
240		.intcast_safe,
	246	.int_cast,
	247	.int_cast_safe,
241	248	.trunc,
242	249	.optional_payload,
243	250	.optional_payload_ptr,

src/Sema.zig+376-478

...	...	@@ -583,16 +583,6 @@ pub const Block = struct {
583	583	});
584	584	}
585	585
586		fn addBitCast(block: *Block, ty: Type, operand: Air.Inst.Ref) Allocator.Error!Air.Inst.Ref {
587		return block.addInst(.{
588		.tag = .bitcast,
589		.data = .{ .ty_op = .{
590		.ty = Air.internedToRef(ty.toIntern()),
591		.operand = operand,
592		} },
593		});
594		}
595
596	586	fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {
597	587	return block.addInst(.{
598	588	.tag = tag,
...	...	@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
3113	3103	// https://github.com/ziglang/zig/issues/6597
3114	3104	if (sema.resolveValue(operand)) \|operand_val\| {
3115	3105	if (!operand_val.isNull(zcu)) {
3116		break :single_ptr try sema.coerceInMemory(operand_val, single_ptr_ty);
	3106	break :single_ptr .fromValue(try pt.getCoerced(operand_val, single_ptr_ty));
3117	3107	}
3118	3108	}
3119	3109	if (block.wantSafety()) {
3120	3110	const is_non_null = try block.addUnOp(.is_non_null, operand);
3121	3111	try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);
3122	3112	}
3123		const single_ptr = try block.addBitCast(single_ptr_ty, operand);
	3113	const single_ptr = try block.addTyOp(.ptr_cast, single_ptr_ty, operand);
3124	3114	try sema.checkKnownAllocPtr(block, operand, single_ptr);
3125	3115	break :single_ptr single_ptr;
3126	3116	},
...	...	@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: Sema, block: Block, alloc: Air.Inst.Ref,
3586	3576	.{ .elem = idx_val.toUnsignedInt(zcu) },
3587	3577	};
3588	3578	},
3589		.bitcast => .{
	3579	.ptr_cast => .{
3590	3580	tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,
3591	3581	.same_addr,
3592	3582	},
...	...	@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(
3729	3719	// This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to
3730	3720	// this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this
3731	3721	// too because it doesn't really matter what they become.
3732		const nop_inst: Air.Inst = .{ .tag = .bitcast, .data = .{ .ty_op = .{
	3722	const nop_inst: Air.Inst = .{ .tag = .ptr_from_int, .data = .{ .ty_op = .{
3733	3723	.ty = .fromIntern(alloc_ty.toIntern()),
3734	3724	.operand = .zero_usize,
3735	3725	} } };
...	...	@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: Sema, block: Block, alloc: Air.Inst.Ref) CompileError!Ai
3779	3769	return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());
3780	3770	}
3781	3771
3782		return block.addBitCast(const_ptr_ty, alloc);
	3772	return block.addTyOp(.ptr_cast, const_ptr_ty, alloc);
3783	3773	}
3784	3774
3785	3775	fn zirAllocInferredComptime(
...	...	@@ -7330,7 +7320,7 @@ fn analyzeCall(
7330	7320	if (resolved_ty == .none) break :r result_raw;
7331	7321	// TODO: mutate in place the previous instruction if possible
7332	7322	// rather than adding a bitcast instruction.
7333		break :r try block.addBitCast(.fromInterned(resolved_ty), result_raw);
	7323	break :r try block.addTyOp(.error_cast, .fromInterned(resolved_ty), result_raw);
7334	7324	};
7335	7325
7336	7326	if (block.isComptime()) {
...	...	@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstD
7636	7626	}
7637	7627
7638	7628	try sema.requireRuntimeBlock(block, src, operand_src);
7639		return block.addBitCast(err_int_ty, operand);
	7629	return block.addTyOp(.int_from_error, err_int_ty, operand);
7640	7630	}
7641	7631
7642	7632	fn zirErrorFromInt(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
...	...	@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstD
7674	7664	const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);
7675	7665	try sema.addSafetyCheck(block, src, ok, .invalid_error_code);
7676	7666	}
7677		return block.addInst(.{
7678		.tag = .bitcast,
7679		.data = .{ .ty_op = .{
7680		.ty = .anyerror_type,
7681		.operand = operand,
7682		} },
7683		});
	7667	return block.addTyOp(.error_from_int, .anyerror, operand);
7684	7668	}
7685	7669
7686	7670	fn zirMergeErrorSets(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
7863	7847	}
7864	7848
7865	7849	try sema.requireRuntimeBlock(block, src, operand_src);
7866		return block.addBitCast(int_tag_ty, enum_tag);
	7850	return block.addTyOp(.bit_cast, int_tag_ty, enum_tag);
7867	7851	}
7868	7852
7869	7853	fn zirEnumFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
7922	7906	try sema.requireRuntimeBlock(block, src, operand_src);
7923	7907	if (block.wantSafety()) {
7924	7908	try sema.preparePanicId(src, .invalid_enum_value);
7925		return block.addTyOp(.intcast_safe, dest_ty, operand);
	7909	return block.addTyOp(.int_cast_safe, dest_ty, operand);
7926	7910	}
7927		return block.addTyOp(.intcast, dest_ty, operand);
	7911	return block.addTyOp(.int_cast, dest_ty, operand);
7928	7912	}
7929	7913
7930	7914	/// Pointer in, pointer out.
...	...	@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!
9152	9136	try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);
9153	9137	try sema.validateRuntimeValue(block, ptr_src, operand);
9154	9138	try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);
9155		return block.addBitCast(dest_ty, operand);
	9139	return block.addTyOp(.int_from_ptr, dest_ty, operand);
9156	9140	}
9157	9141
9158	9142	fn zirFieldPtrLoad(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -9314,9 +9298,9 @@ fn intCast(
9314	9298	try sema.requireRuntimeBlock(block, src, operand_src);
9315	9299	if (block.wantSafety()) {
9316	9300	try sema.preparePanicId(src, .integer_out_of_bounds);
9317		return block.addTyOp(.intcast_safe, dest_ty, operand);
	9301	return block.addTyOp(.int_cast_safe, dest_ty, operand);
9318	9302	}
9319		return block.addTyOp(.intcast, dest_ty, operand);
	9303	return block.addTyOp(.int_cast, dest_ty, operand);
9320	9304	}
9321	9305
9322	9306	fn zirBitcast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
9330	9314	const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");
9331	9315	const operand = sema.resolveInst(extra.rhs);
9332	9316	const operand_ty = sema.typeOf(operand);
9333		switch (dest_ty.zigTypeTag(zcu)) {
9334		.@"anyframe",
9335		.comptime_float,
9336		.comptime_int,
9337		.enum_literal,
9338		.error_set,
9339		.error_union,
9340		.@"fn",
9341		.frame,
9342		.noreturn,
9343		.null,
9344		.@"opaque",
9345		.spirv,
9346		.optional,
9347		.type,
9348		.undefined,
9349		.void,
9350		=> return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9351
9352		.@"enum" => {
9353		const msg = msg: {
9354		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9355		errdefer msg.destroy(sema.gpa);
9356		switch (operand_ty.zigTypeTag(zcu)) {
9357		.int, .comptime_int => try sema.errNote(src, msg, "use @enumFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9358		else => {},
9359		}
9360
9361		break :msg msg;
9362		};
9363		return sema.failWithOwnedErrorMsg(block, msg);
9364		},
9365	9317
9366		.pointer => {
9367		const msg = msg: {
9368		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9369		errdefer msg.destroy(sema.gpa);
9370		switch (operand_ty.zigTypeTag(zcu)) {
9371		.int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9372		.pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
9373		else => {},
9374		}
9375
9376		break :msg msg;
9377		};
9378		return sema.failWithOwnedErrorMsg(block, msg);
9379		},
9380		.@"struct", .@"union" => if (dest_ty.containerLayout(zcu) == .auto) {
9381		const container = switch (dest_ty.zigTypeTag(zcu)) {
9382		.@"struct" => "struct",
9383		.@"union" => "union",
9384		else => unreachable,
9385		};
9386		return sema.fail(block, src, "cannot @bitCast to '{f}'; {s} does not have a guaranteed in-memory layout", .{
9387		dest_ty.fmt(pt), container,
9388		});
9389		},
9390		.array => {
9391		const elem_ty = dest_ty.childType(zcu);
9392		if (!elem_ty.hasWellDefinedLayout(zcu)) {
9393		const msg = msg: {
9394		const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9395		errdefer msg.destroy(sema.gpa);
9396		try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9397		break :msg msg;
9398		};
9399		return sema.failWithOwnedErrorMsg(block, msg);
	9318	// Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
	9319	switch (dest_ty.scalarType(zcu).zigTypeTag(zcu)) {
	9320	.pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
	9321	const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
	9322	errdefer msg.destroy(sema.gpa);
	9323	switch (operand_ty.zigTypeTag(zcu)) {
	9324	.int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
	9325	.pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
	9326	else => {},
9400	9327	}
9401		},
9402	9328
9403		.bool,
9404		.float,
9405		.int,
9406		.vector,
9407		=> {},
9408		}
9409		switch (operand_ty.zigTypeTag(zcu)) {
9410		.@"anyframe",
9411		.comptime_float,
9412		.comptime_int,
9413		.enum_literal,
9414		.error_set,
9415		.error_union,
9416		.@"fn",
9417		.frame,
9418		.noreturn,
9419		.null,
9420		.@"opaque",
9421		.spirv,
9422		.optional,
9423		.type,
9424		.undefined,
9425		.void,
9426		=> return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)}),
9427
9428		.@"enum" => {
9429		const msg = msg: {
9430		const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9431		errdefer msg.destroy(sema.gpa);
9432		switch (dest_ty.zigTypeTag(zcu)) {
9433		.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromEnum to cast to '{f}'", .{dest_ty.fmt(pt)}),
9434		else => {},
9435		}
9436
9437		break :msg msg;
9438		};
9439		return sema.failWithOwnedErrorMsg(block, msg);
	9329	break :msg msg;
	9330	}),
	9331	.array => switch (dest_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
	9332	.pointer, .optional => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
	9333	else => {},
9440	9334	},
9441		.pointer => {
9442		const msg = msg: {
9443		const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9444		errdefer msg.destroy(sema.gpa);
9445		switch (dest_ty.zigTypeTag(zcu)) {
9446		.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9447		.pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9448		else => {},
9449		}
	9335	else => {},
	9336	}
	9337	if (!dest_ty.hasBitRepresentation(zcu)) {
	9338	return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
	9339	}
9450	9340
9451		break :msg msg;
9452		};
9453		return sema.failWithOwnedErrorMsg(block, msg);
9454		},
9455		.@"struct", .@"union" => if (operand_ty.containerLayout(zcu) == .auto) {
9456		const container = switch (operand_ty.zigTypeTag(zcu)) {
9457		.@"struct" => "struct",
9458		.@"union" => "union",
9459		else => unreachable,
9460		};
9461		return sema.fail(block, operand_src, "cannot @bitCast from '{f}'; {s} does not have a guaranteed in-memory layout", .{
9462		operand_ty.fmt(pt), container,
9463		});
9464		},
9465		.array => {
9466		const elem_ty = operand_ty.childType(zcu);
9467		if (!elem_ty.hasWellDefinedLayout(zcu)) {
9468		const msg = msg: {
9469		const msg = try sema.errMsg(src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9470		errdefer msg.destroy(sema.gpa);
9471		try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9472		break :msg msg;
9473		};
9474		return sema.failWithOwnedErrorMsg(block, msg);
	9341	// Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
	9342	switch (operand_ty.scalarType(zcu).zigTypeTag(zcu)) {
	9343	.pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
	9344	const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
	9345	errdefer msg.destroy(sema.gpa);
	9346	switch (dest_ty.zigTypeTag(zcu)) {
	9347	.int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
	9348	.pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
	9349	else => {},
9475	9350	}
	9351	break :msg msg;
	9352	}),
	9353	.array => switch (operand_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
	9354	.pointer, .optional => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{dest_ty.fmt(pt)}),
	9355	else => {},
9476	9356	},
9477
9478		.bool,
9479		.float,
9480		.int,
9481		.vector,
9482		=> {},
	9357	else => {},
	9358	}
	9359	if (!operand_ty.hasBitRepresentation(zcu)) {
	9360	return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9483	9361	}
9484		return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node), operand_src);
	9362
	9363	return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node));
9485	9364	}
9486	9365
9487	9366	fn zirFloatCast(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(
12026	11905	.@"inline" => unreachable, // handled above
12027	11906	.has_ranges => unreachable, // not possible for error set
12028	11907	.special => {
12029		if (else_err_ty) \|err_ty\| {
12030		break :payload_ref try sema.bitCast(case_block, err_ty, loaded_operand, operand_src, null);
12031		} else {
	11908	const capture_err_ty = else_err_ty orelse {
12032	11909	try sema.analyzeUnreachable(case_block, operand_src, false);
12033	11910	break :payload_ref .unreachable_value;
	11911	};
	11912	if (sema.resolveValue(loaded_operand)) \|err_val\| {
	11913	break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
	11914	.ty = capture_err_ty.toIntern(),
	11915	.name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
	11916	} }));
	11917	} else {
	11918	break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
12034	11919	}
12035	11920	},
12036	11921	.item_refs => \|item_refs\| {
...	...	@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(
12040	11925	const item_val = sema.resolveValue(item_ref).?;
12041	11926	names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});
12042	11927	}
12043		const narrowed_ty = try pt.errorSetFromUnsortedNames(names.keys());
12044		break :payload_ref try sema.bitCast(case_block, narrowed_ty, loaded_operand, operand_src, null);
	11928	const capture_err_ty = try pt.errorSetFromUnsortedNames(names.keys());
	11929	if (sema.resolveValue(loaded_operand)) \|err_val\| {
	11930	break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
	11931	.ty = capture_err_ty.toIntern(),
	11932	.name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
	11933	} }));
	11934	} else {
	11935	break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
	11936	}
12045	11937	},
12046	11938	}
12047	11939	}
...	...	@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12259	12151	return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);
12260	12152	}
12261	12153
12262		// We may have to emit a switch block which coerces the operand to the capture type.
12263		// If we can, try to avoid that using in-memory coercions.
12264		const first_non_imc = in_mem: {
12265		for (field_indices, 0..) \|field_idx, i\| {
12266		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12267		if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12268		break :in_mem i;
12269		}
12270		}
12271		// All fields are in-memory coercible to the resolved type!
12272		// Just take the first field and bitcast the result.
12273		const uncoerced = try case_block.addStructFieldVal(loaded_operand, first_field_index, first_field_ty);
12274		return case_block.addBitCast(capture_ty, uncoerced);
12275		};
12276
12277	12154	// By-val capture with heterogeneous types which are not all in-memory coercible to
12278	12155	// the resolved capture type. We finally have to fall back to the ugly method.
12279	12156
12280		// However, let's first track which operands are in-memory coercible. There may well
12281		// be several, and we can squash all of these cases into the same switch prong using
12282		// a simple bitcast. We'll make this the 'else' prong.
12283
12284		var in_mem_coercible: std.bit_set.Dynamic = try .initFull(sema.arena, field_indices.len);
12285		in_mem_coercible.unset(first_non_imc);
12286		{
12287		const next = first_non_imc + 1;
12288		for (field_indices[next..], next..) \|field_idx, i\| {
12289		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12290		if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12291		in_mem_coercible.unset(i);
12292		}
12293		}
12294		}
12295
12296	12157	const capture_block_inst = try case_block.addInstAsIndex(.{
12297	12158	.tag = .block,
12298	12159	.data = .{
...	...	@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12303	12164	},
12304	12165	});
12305	12166
12306		const prong_count = field_indices.len - in_mem_coercible.count();
12307
12308		const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
	12167	const estimated_extra = field_indices.len * 6 + (field_indices.len / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12309	12168	var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);
12310	12169	defer cases_extra.deinit(gpa);
12311	12170
12312	12171	{
12313	12172	// All branch hints are `.none`, so just add zero elems.
12314	12173	comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);
12315		const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;
	12174	const need_elems = std.math.divCeil(usize, field_indices.len + 1, 10) catch unreachable;
12316	12175	try cases_extra.appendNTimes(gpa, 0, need_elems);
12317	12176	}
12318	12177
12319	12178	{
12320		// Non-bitcast cases
12321		var it = in_mem_coercible.iterator(.{ .kind = .unset });
12322		while (it.next()) \|idx\| {
	12179	for (field_indices, item_refs, 0..) \|field_index, item, item_index\| {
12323	12180	var coerce_block = case_block.makeSubBlock();
12324	12181	defer coerce_block.instructions.deinit(sema.gpa);
12325	12182
...	...	@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12328	12185	.offset = .{ .switch_case_item = .{
12329	12186	.switch_node_offset = switch_node_offset,
12330	12187	.case_idx = capture_src.offset.switch_capture.case_idx,
12331		.item_idx = .{ .kind = .single, .value = @intCast(idx) },
	12188	.item_idx = .{ .kind = .single, .value = @intCast(item_index) },
12332	12189	} },
12333	12190	};
12334	12191
12335		const field_idx = field_indices[idx];
12336		const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12337		const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_idx, field_ty);
	12192	const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_index]);
	12193	const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_index, field_ty);
12338	12194	const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);
12339	12195	_ = try coerce_block.addBr(capture_block_inst, coerced);
12340	12196
...	...	@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12346	12202	.ranges_len = 0,
12347	12203	.body_len = @intCast(coerce_block.instructions.items.len),
12348	12204	}));
12349		cases_extra.appendAssumeCapacity(@intFromEnum(item_refs[idx])); // item
	12205	cases_extra.appendAssumeCapacity(@intFromEnum(item)); // item
12350	12206	cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body
12351	12207	}
12352	12208	}
12353	12209	const else_body_len = len: {
12354		// 'else' prong uses a bitcast
12355		var coerce_block = case_block.makeSubBlock();
12356		defer coerce_block.instructions.deinit(sema.gpa);
12357
12358		const first_imc_item_idx = in_mem_coercible.findFirstSet().?;
12359		const first_imc_field_idx = field_indices[first_imc_item_idx];
12360		const first_imc_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[first_imc_field_idx]);
12361		const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, first_imc_field_idx, first_imc_field_ty);
12362		const coerced = try coerce_block.addBitCast(capture_ty, uncoerced);
12363		_ = try coerce_block.addBr(capture_block_inst, coerced);
12364
12365		try cases_extra.appendSlice(gpa, @ptrCast(coerce_block.instructions.items));
12366		break :len coerce_block.instructions.items.len;
	12210	// 'else' prong is unreachable
	12211	const result_index: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
	12212	try sema.air_instructions.append(gpa, .{ .tag = .unreach, .data = .{ .no_op = {} } });
	12213	try cases_extra.append(gpa, @intFromEnum(result_index));
	12214	break :len 1;
12367	12215	};
12368	12216
12369	12217	try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +
...	...	@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12378	12226	.pl_op = .{
12379	12227	.operand = undefined, // set by switch below
12380	12228	.payload = sema.addExtraAssumeCapacity(Air.SwitchBr{
12381		.cases_len = @intCast(prong_count),
	12229	.cases_len = @intCast(field_indices.len),
12382	12230	.else_body_len = @intCast(else_body_len),
12383	12231	}),
12384	12232	},
...	...	@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(
12488	12336	// being switched on if their prong body is `=> comptime unreachable,`.
12489	12337	switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {
12490	12338	.ok => if (sema.resolveValue(uncoerced)) \|uncoerced_val\| {
12491		break :item_ref try sema.coerceInMemory(uncoerced_val, item_ty);
	12339	break :item_ref .fromValue(try pt.getCoerced(uncoerced_val, item_ty));
12492	12340	},
12493	12341	.missing_error => if (prong_is_comptime_unreach) {
12494	12342	break :item_ref uncoerced;
...	...	@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13394	13242	defer trash_block.instructions.deinit(sema.gpa);
13395	13243
13396	13244	const instructions = [_]Air.Inst.Ref{
13397		try trash_block.addBitCast(lhs_info.elem_type, .void_value),
13398		try trash_block.addBitCast(rhs_info.elem_type, .void_value),
	13245	try trash_block.addTyOp(.bit_cast, lhs_info.elem_type, .void_value),
	13246	try trash_block.addTyOp(.bit_cast, rhs_info.elem_type, .void_value),
13399	13247	};
13400	13248	break :t try sema.resolvePeerTypes(block, src, &instructions, .{
13401	13249	.override = &[_]?LazySrcLoc{ lhs_src, rhs_src },
...	...	@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13552	13400	});
13553	13401
13554	13402	const many_ty = slice_ty.slicePtrFieldType(zcu);
13555		const many_alloc = try block.addBitCast(many_ty, mutable_alloc);
	13403	const many_alloc = try block.addTyOp(.ptr_cast, many_ty, mutable_alloc);
13556	13404
13557	13405	// lhs_dest_slice = dest[0..lhs.len]
13558	13406	if (lhs_len > 0) {
...	...	@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13601	13449	try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13602	13450	}
13603	13451
13604		return block.addBitCast(constant_alloc_ty, mutable_alloc);
	13452	return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13605	13453	}
13606	13454
13607	13455	var elem_i: u32 = 0;
...	...	@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Ai
13634	13482	try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13635	13483	}
13636	13484
13637		return block.addBitCast(constant_alloc_ty, mutable_alloc);
	13485	return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13638	13486	}
13639	13487
13640	13488	const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);
...	...	@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(
14917	14765	try sema.requireRuntimeBlock(block, src, runtime_src);
14918	14766	try sema.checkLogicalPtrOperation(block, src, lhs_ty);
14919	14767	try sema.checkLogicalPtrOperation(block, src, rhs_ty);
14920		const lhs_int = try block.addBitCast(.usize, lhs);
14921		const rhs_int = try block.addBitCast(.usize, rhs);
	14768	const lhs_int = try block.addTyOp(.int_from_ptr, .usize, lhs);
	14769	const rhs_int = try block.addTyOp(.int_from_ptr, .usize, rhs);
14922	14770	const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);
14923	14771	return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));
14924	14772	}
...	...	@@ -15186,15 +15034,76 @@ fn zirAsm(
15186	15034	break :out_ty sema.typeOf(inst).childType(zcu);
15187	15035	}
15188	15036	};
15189		if (!out_ty.hasWellDefinedLayout(zcu)) {
15190		return sema.failWithOwnedErrorMsg(block, msg: {
15191		const msg = try sema.errMsg(output_src, "invalid inline assembly output type; '{f}' does not have a guaranteed in-memory layout", .{
15192		out_ty.fmt(pt),
15193		});
	15037	switch (out_ty.zigTypeTag(zcu)) {
	15038	.int, .float, .bool, .vector => {},
	15039
	15040	.pointer => if (out_ty.isSlice(zcu)) return sema.failWithOwnedErrorMsg(block, msg: {
	15041	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15194	15042	errdefer msg.destroy(gpa);
15195		try sema.addDeclaredHereNote(msg, out_ty);
	15043	try sema.errNote(output_src, msg, "consider separate outputs for 'ptr' and 'len'", .{});
15196	15044	break :msg msg;
15197		});
	15045	}),
	15046
	15047	.optional => if (!out_ty.isPtrLikeOptional(zcu)) {
	15048	return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15049	},
	15050
	15051	.@"enum" => switch (ip.loadEnumType(out_ty.toIntern()).int_tag_mode) {
	15052	.explicit => {},
	15053	.auto => return sema.failWithOwnedErrorMsg(block, msg: {
	15054	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15055	errdefer msg.destroy(gpa);
	15056	try sema.errNote(out_ty.srcLoc(zcu), msg, "integer tag type of enum is inferred", .{});
	15057	try sema.errNote(out_ty.srcLoc(zcu), msg, "consider explicitly specifying the integer tag type", .{});
	15058	break :msg msg;
	15059	}),
	15060	},
	15061
	15062	.@"struct" => switch (out_ty.containerLayout(zcu)) {
	15063	.@"packed" => {},
	15064	.auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
	15065	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15066	errdefer msg.destroy(gpa);
	15067	try sema.errNote(output_src, msg, "struct types cannot be passed to inline assembly", .{});
	15068	try sema.addDeclaredHereNote(msg, out_ty);
	15069	break :msg msg;
	15070	}),
	15071	},
	15072
	15073	.@"union" => switch (out_ty.containerLayout(zcu)) {
	15074	.@"packed" => {},
	15075	.auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
	15076	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15077	errdefer msg.destroy(gpa);
	15078	try sema.errNote(output_src, msg, "union types cannot be passed to inline assembly", .{});
	15079	try sema.addDeclaredHereNote(msg, out_ty);
	15080	break :msg msg;
	15081	}),
	15082	},
	15083
	15084	.array => return sema.failWithOwnedErrorMsg(block, msg: {
	15085	const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
	15086	errdefer msg.destroy(gpa);
	15087	try sema.errNote(output_src, msg, "array types cannot be passed to inline assembly", .{});
	15088	break :msg msg;
	15089	}),
	15090
	15091	.void,
	15092	.type,
	15093	.noreturn,
	15094	.comptime_float,
	15095	.comptime_int,
	15096	.undefined,
	15097	.null,
	15098	.error_union,
	15099	.error_set,
	15100	.@"fn",
	15101	.@"opaque",
	15102	.frame,
	15103	.@"anyframe",
	15104	.enum_literal,
	15105	.spirv,
	15106	=> return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)}),
15198	15107	}
15199	15108
15200	15109	const constraint = sema.code.nullTerminatedString(output.data.constraint);
...	...	@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
15598	15507	const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
15599	15508	const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
15600	15509	const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);
15601		switch (operand_ty.zigTypeTag(zcu)) {
15602		.@"fn",
15603		.noreturn,
15604		.undefined,
15605		.null,
15606		.@"opaque",
15607		.spirv,
15608		.type,
15609		.enum_literal,
15610		.comptime_float,
15611		.comptime_int,
15612		=> return sema.fail(block, operand_src, "no size available for type '{f}'", .{operand_ty.fmt(pt)}),
15613
15614		.void,
15615		=> return .zero,
15616
15617		.bool,
15618		.int,
15619		.float,
15620		.pointer,
15621		.array,
15622		.@"struct",
15623		.optional,
15624		.error_union,
15625		.error_set,
15626		.@"enum",
15627		.@"union",
15628		.vector,
15629		.frame,
15630		.@"anyframe",
15631		=> {},
	15510	if (!operand_ty.hasBitRepresentation(zcu) and
	15511	// TODO: allow these types too for now because this is used in some places. We need to
	15512	// figure out whether we think errors and auto-enums have bit representations!
	15513	operand_ty.zigTypeTag(zcu) != .error_set and
	15514	operand_ty.zigTypeTag(zcu) != .@"enum")
	15515	{
	15516	return sema.fail(block, operand_src, "no bit size available for type '{f}'", .{operand_ty.fmt(pt)});
15632	15517	}
15633	15518	try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);
15634	15519	return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));
...	...	@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
18179	18064	} else 0;
18180	18065
18181	18066	if (host_size != 0) {
	18067	try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
	18068	if (elem_ty.unpackable(zcu)) \|reason\| return sema.failWithOwnedErrorMsg(block, msg: {
	18069	const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
	18070	errdefer msg.destroy(sema.gpa);
	18071	try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
	18072	break :msg msg;
	18073	});
	18074	const elem_bit_size = elem_ty.bitSize(zcu);
18182	18075	if (bit_offset >= host_size * 8) {
18183	18076	return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{
18184	18077	elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,
18185	18078	});
18186	18079	}
18187		try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18188		const elem_bit_size = elem_ty.bitSize(zcu);
18189	18080	if (elem_bit_size > host_size * 8 - bit_offset) {
18190	18081	return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{
18191	18082	elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,
...	...	@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air
18201	18092	return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});
18202	18093	}
18203	18094
18204		if (host_size != 0) {
18205		if (elem_ty.unpackable(zcu)) \|reason\| return sema.failWithOwnedErrorMsg(block, msg: {
18206		const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18207		errdefer msg.destroy(sema.gpa);
18208		try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18209		break :msg msg;
18210		});
18211		}
18212
18213	18095	const ty = try pt.ptrType(.{
18214	18096	.child = elem_ty.toIntern(),
18215	18097	.sentinel = sentinel,
...	...	@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!A
18379	18261	const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);
18380	18262
18381	18263	if (union_ty.containerLayout(zcu) == .@"packed") {
18382		return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node), payload_src);
	18264	return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node));
18383	18265	}
18384	18266
18385	18267	if (sema.resolveValue(payload)) \|payload_val\| {
...	...	@@ -18516,7 +18398,7 @@ fn zirStructInit(
18516	18398	const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);
18517	18399
18518	18400	if (resolved_ty.containerLayout(zcu) == .@"packed") {
18519		const union_val = try sema.bitCast(block, resolved_ty, init_inst, src, field_src);
	18401	const union_val = try sema.bitCast(block, resolved_ty, init_inst, src);
18520	18402	const result_val = try sema.coerce(block, result_ty, union_val, src);
18521	18403	if (is_ref) {
18522	18404	return sema.analyzeRef(block, src, result_val, .none);
...	...	@@ -18680,20 +18562,15 @@ fn finishStructInit(
18680	18562	},
18681	18563	.@"packed" => {
18682	18564	const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));
	18565	@memset(buf, 0);
18683	18566	var bit_offset: u16 = 0;
18684	18567	for (field_inits) \|field_init\| {
18685	18568	const field_val = sema.resolveValue(field_init).?;
18686		field_val.writeToPackedMemory(zcu, buf, bit_offset) catch \|err\| switch (err) {
18687		error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
18688		error.OutOfMemory => \|e\| return e,
18689		};
	18569	field_val.writeToPackedMemory(zcu, buf, bit_offset);
18690	18570	bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
18691	18571	}
18692	18572	assert(bit_offset == struct_ty.bitSize(zcu));
18693		const struct_val = Value.readFromPackedMemory(struct_ty, pt, buf, 0, sema.arena) catch \|err\| switch (err) {
18694		error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
18695		error.OutOfMemory => \|e\| return e,
18696		};
	18573	const struct_val: Value = try .readFromPackedMemory(struct_ty, pt, buf, 0);
18697	18574	const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);
18698	18575	return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);
18699	18576	},
...	...	@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError
19387	19264	}
19388	19265	return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());
19389	19266	}
19390		return block.addBitCast(dest_ty, operand);
	19267	return block.addTyOp(.bit_cast, dest_ty, operand);
19391	19268	}
19392	19269
19393	19270	fn zirErrorName(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!
21245	21122	try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);
21246	21123	}
21247	21124	}
21248		return block.addBitCast(dest_ty, operand_coerced);
	21125	return block.addTyOp(.ptr_from_int, dest_ty, operand_coerced);
21249	21126	}
21250	21127
21251	21128	fn ptrFromIntVal(
...	...	@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData
21444	21321	.error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),
21445	21322	else => unreachable,
21446	21323	};
21447		const err_int_inst = try block.addBitCast(err_int_ty, err_code_inst);
	21324	const err_int_inst = try block.addTyOp(.int_from_error, err_int_ty, err_code_inst);
21448	21325	if (dest_tag == .error_union) {
21449	21326	const zero_err = try pt.intRef(err_int_ty, 0);
21450	21327	const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);
...	...	@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: Sema, block: Block, extended: Zir.Inst.Extended.InstData
21464	21341	}
21465	21342
21466	21343	if (operand_tag == .error_set and dest_tag == .error_union) {
21467		const err_val = try block.addBitCast(dest_err_ty, operand);
	21344	const err_val = try block.addTyOp(.error_cast, dest_err_ty, operand);
21468	21345	return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);
21469	21346	} else {
21470		return block.addBitCast(dest_ty, operand);
	21347	return block.addTyOp(.error_cast, dest_ty, operand);
21471	21348	}
21472	21349	}
21473	21350
...	...	@@ -22015,7 +21892,7 @@ fn ptrCastFull(
22015	21892	// `operand_ptr` converted to an integer, for safety checks.
22016	21893	const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {
22017	21894	assert(need_operand_ptr);
22018		break :i try block.addBitCast(.usize, operand_ptr);
	21895	break :i try block.addTyOp(.int_from_ptr, .usize, operand_ptr);
22019	21896	} else .none;
22020	21897
22021	21898	if (need_null_check) {
...	...	@@ -22042,8 +21919,8 @@ fn ptrCastFull(
22042	21919
22043	21920	if (dest_info.flags.size == .slice) {
22044	21921	if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {
22045		// Fast path: just bitcast!
22046		return block.addBitCast(dest_ty, operand);
	21922	// Fast path: just pointer cast!
	21923	return block.addTyOp(.ptr_cast, dest_ty, operand);
22047	21924	}
22048	21925
22049	21926	// We need to deconstruct the slice (if applicable) and reconstruct it.
...	...	@@ -22101,7 +21978,7 @@ fn ptrCastFull(
22101	21978	else => unreachable,
22102	21979	};
22103	21980	const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {
22104		break :ptr try block.addBitCast(want_ptr_ty, operand_ptr);
	21981	break :ptr try block.addTyOp(.ptr_cast, want_ptr_ty, operand_ptr);
22105	21982	} else operand_ptr;
22106	21983
22107	21984	return block.addInst(.{
...	...	@@ -22116,12 +21993,11 @@ fn ptrCastFull(
22116	21993	});
22117	21994	} else {
22118	21995	assert(need_operand_ptr);
22119		// We just need to bitcast the pointer, if necessary.
22120		// It might not be necessary, since we might have just needed the `addrspace_cast`.
	21996	// We just need a ptr_cast, if even that (we might only have needed the `addrspace_cast`).
22121	21997	const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())
22122	21998	operand_ptr
22123	21999	else
22124		try block.addBitCast(dest_ty, operand_ptr);
	22000	try block.addTyOp(.ptr_cast, dest_ty, operand_ptr);
22125	22001
22126	22002	try sema.checkKnownAllocPtr(block, operand, result);
22127	22003	return result;
...	...	@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: Sema, block: Block, extended: Zir.Inst.Extended.Inst
22157	22033	}
22158	22034
22159	22035	try sema.requireRuntimeBlock(block, src, null);
22160		const new_ptr = try block.addBitCast(dest_ty, operand);
	22036	const new_ptr = try block.addTyOp(.ptr_cast, dest_ty, operand);
22161	22037	try sema.checkKnownAllocPtr(block, operand, new_ptr);
22162	22038	return new_ptr;
22163	22039	}
...	...	@@ -24259,7 +24135,7 @@ fn analyzeMinMax(
24259	24135	// where we have refined the range, so we should be doing an intcast.
24260	24136	assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);
24261	24137	assert(result_scalar_ty.zigTypeTag(zcu) == .int);
24262		return block.addTyOp(.intcast, result_ty, cur_result);
	24138	return block.addTyOp(.int_cast, result_ty, cur_result);
24263	24139	}
24264	24140
24265	24141	fn upgradeToArrayPtr(sema: Sema, block: Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {
...	...	@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: Sema, block: Block, ptr: Air.Inst.Ref, len: u64) !A
24289	24165	try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)
24290	24166	else
24291	24167	ptr;
24292		return block.addBitCast(new_ty, non_slice_ptr);
	24168	return block.addTyOp(.ptr_cast, new_ty, non_slice_ptr);
24293	24169	}
24294	24170
24295	24171	fn zirMemcpy(
...	...	@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(
25087	24963	const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);
25088	24964	return Air.internedToRef(casted_ptr_val.toIntern());
25089	24965	} else {
25090		return block.addBitCast(result_ptr_ty, uncasted_ptr);
	24966	return block.addTyOp(.ptr_cast, result_ptr_ty, uncasted_ptr);
25091	24967	}
25092	24968	}
25093	24969
...	...	@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(
25533	25409	.param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),
25534	25410	else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
25535	25411	},
25536		.vector => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
	25412	.vector => try sema.errNote(src_loc, msg, "vectors have no guaranteed in-memory representation", .{}),
25537	25413	.optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),
25538	25414	}
25539	25415	}
...	...	@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(
25801	25677	.address_space = ptr_info.flags.address_space,
25802	25678	},
25803	25679	});
25804		const many_ptr = try parent_block.addBitCast(many_ptr_ty, ptr);
	25680	const many_ptr = try parent_block.addTyOp(.ptr_cast, many_ptr_ty, ptr);
25805	25681	break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);
25806	25682	},
25807	25683	.many => unreachable,
...	...	@@ -26278,7 +26154,7 @@ fn fieldPtr(
26278	26154	},
26279	26155	.packed_offset = ptr_ptr_info.packed_offset,
26280	26156	});
26281		return sema.bitCast(block, result_ty, object_ptr, src, null);
	26157	return block.addTyOp(.ptr_cast, result_ty, object_ptr);
26282	26158	} else {
26283	26159	return sema.fail(
26284	26160	block,
...	...	@@ -26996,16 +26872,13 @@ fn unionFieldVal(
26996	26872	break :msg msg;
26997	26873	});
26998	26874	},
26999		.@"extern" => if (try sema.bitCastVal(union_val, field_ty, 0, 0, 0)) \|field_val\| {
	26875	.@"extern" => if (try sema.castMemory(union_val, field_ty, 0)) \|field_val\| {
27000	26876	return .fromValue(field_val);
27001	26877	} else {
27002	26878	// Runtime-known due to a pointer-to-integer conversion.
27003	26879	},
27004	26880	.@"packed" => {
27005		const field_val = try sema.bitCastVal(union_val, field_ty, 0, union_ty.bitSize(zcu), 0) orelse {
27006		unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
27007		};
27008		return .fromValue(field_val);
	26881	return .fromValue(try sema.bitCastVal(union_val, field_ty));
27009	26882	},
27010	26883	}
27011	26884	}
...	...	@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(
27104	26977
27105	26978	if (child_ty.abiSize(zcu) == 0) {
27106	26979	// zero-bit child type; just bitcast the pointer
27107		return block.addBitCast(result_ty, indexable);
	26980	return block.addTyOp(.ptr_cast, result_ty, indexable);
27108	26981	}
27109	26982
27110	26983	return block.addPtrElemPtr(indexable, elem_index, result_ty);
...	...	@@ -27401,68 +27274,36 @@ fn elemPtrVector(
27401	27274	}
27402	27275
27403	27276	const elem_ty = vector_ty.childType(zcu);
27404		const elem_bits = elem_ty.bitSize(zcu);
27405		// Exiting this block means the operation is a runtime one.
27406		const elem_ptr_ty: Type = if (elem_bits < 8 or !std.math.isPowerOfTwo(elem_bits)) elem_ptr_ty: {
27407		// Use a packed pointer (i.e. vector_index != 0)
27408		const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27409		const elem_ptr_ty = try pt.ptrType(.{
27410		.child = elem_ty.toIntern(),
27411		.flags = .{
27412		.size = .one,
27413		.alignment = vector_ptr_info.flags.alignment,
27414		.is_const = vector_ptr_info.flags.is_const,
27415		.is_volatile = vector_ptr_info.flags.is_volatile,
27416		.is_allowzero = vector_ptr_info.flags.is_allowzero,
27417		.address_space = vector_ptr_info.flags.address_space,
27418		.vector_index = @enumFromInt(index),
27419		},
27420		.packed_offset = .{
27421		.host_size = @intCast(vector_len),
27422		.bit_offset = 0,
27423		},
27424		});
27425		if (maybe_vector_ptr_val) \|ptr_val\| {
27426		if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27427		return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27428		}
27429		break :elem_ptr_ty elem_ptr_ty;
27430		} else elem_ptr_ty: {
27431		// Use a normal pointer (i.e. vector_index == 0)
27432		const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27433		const elem_ptr_ty = try pt.ptrType(.{
27434		.child = elem_ty.toIntern(),
27435		.flags = .{
27436		.size = .one,
27437		// TODO: this logic was ported from old code, but it's bogus. This entire block will
27438		// go away when https://github.com/ziglang/zig/issues/24061 is implemented anyway.
27439		.alignment = switch (vector_ptr_info.flags.alignment) {
27440		.none => .none,
27441		else => \|vec_align\| switch (index * elem_ty.abiSize(zcu)) {
27442		0 => vec_align,
27443		else => \|byte_offset\| .minStrict(vec_align, .fromLog2Units(@ctz(byte_offset))),
27444		},
27445		},
27446		.is_const = vector_ptr_info.flags.is_const,
27447		.is_volatile = vector_ptr_info.flags.is_volatile,
27448		.is_allowzero = vector_ptr_info.flags.is_allowzero,
27449		.address_space = vector_ptr_info.flags.address_space,
27450		},
27451		});
27452		if (maybe_vector_ptr_val) \|ptr_val\| {
27453		if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27454		const bit_offset = index * @divExact(elem_ty.bitSize(zcu), 8);
27455		return .fromValue(try ptr_val.getOffsetPtr(bit_offset, elem_ptr_ty, pt));
27456		}
27457		break :elem_ptr_ty elem_ptr_ty;
27458		};
	27277
	27278	const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
	27279	const elem_ptr_ty = try pt.ptrType(.{
	27280	.child = elem_ty.toIntern(),
	27281	.flags = .{
	27282	.size = .one,
	27283	.alignment = vector_ptr_info.flags.alignment,
	27284	.is_const = vector_ptr_info.flags.is_const,
	27285	.is_volatile = vector_ptr_info.flags.is_volatile,
	27286	.is_allowzero = vector_ptr_info.flags.is_allowzero,
	27287	.address_space = vector_ptr_info.flags.address_space,
	27288	.vector_index = @enumFromInt(index),
	27289	},
	27290	.packed_offset = .{
	27291	.host_size = @intCast(vector_len),
	27292	.bit_offset = 0,
	27293	},
	27294	});
	27295
	27296	if (maybe_vector_ptr_val) \|ptr_val\| {
	27297	if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
	27298	return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
	27299	}
27459	27300
27460	27301	if (!init) {
27461	27302	try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);
27462	27303	try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);
27463	27304	}
27464	27305
27465		return block.addPtrElemPtr(vector_ptr, elem_index, elem_ptr_ty);
	27306	return block.addTyOp(.ptr_cast, elem_ptr_ty, vector_ptr);
27466	27307	}
27467	27308
27468	27309	fn elemPtrSpirvRuntimeArray(
...	...	@@ -27544,7 +27385,7 @@ fn elemPtrArray(
27544	27385
27545	27386	if (array_ty.childType(zcu).abiSize(zcu) == 0) {
27546	27387	// zero-bit child type; just bitcast the pointer
27547		return block.addBitCast(elem_ptr_ty, array_ptr);
	27388	return block.addTyOp(.ptr_cast, elem_ptr_ty, array_ptr);
27548	27389	}
27549	27390
27550	27391	return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);
...	...	@@ -27670,7 +27511,7 @@ fn elemPtrSlice(
27670	27511	if (elem_ty.abiSize(zcu) == 0) {
27671	27512	// zero-bit child type; just extract the pointer and bitcast it
27672	27513	const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);
27673		return block.addBitCast(elem_ptr_ty, slice_ptr);
	27514	return block.addTyOp(.ptr_cast, elem_ptr_ty, slice_ptr);
27674	27515	}
27675	27516	return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);
27676	27517	}
...	...	@@ -27753,12 +27594,31 @@ fn coerceExtra(
27753	27594	var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);
27754	27595	if (in_memory_result == .ok) {
27755	27596	if (maybe_inst_val) \|val\| {
27756		return sema.coerceInMemory(val, dest_ty);
27757		}
27758		try sema.requireRuntimeBlock(block, inst_src, null);
27759		const new_val = try block.addBitCast(dest_ty, inst);
27760		try sema.checkKnownAllocPtr(block, inst, new_val);
27761		return new_val;
	27597	return .fromValue(try pt.getCoerced(val, dest_ty));
	27598	}
	27599	const coerced: Air.Inst.Ref = switch (in_memory_result.ok) {
	27600	.none => coerced: {
	27601	const @"addrspace" = target_util.defaultAddressSpace(zcu.getTarget(), .local);
	27602	const src_ptr_ty = try pt.ptrType(.{
	27603	.child = inst_ty.toIntern(),
	27604	.flags = .{ .size = .one, .address_space = @"addrspace" },
	27605	});
	27606	const dest_ptr_ty = try pt.ptrType(.{
	27607	.child = dest_ty.toIntern(),
	27608	.flags = .{ .size = .one, .address_space = @"addrspace" },
	27609	});
	27610	const ptr = try block.addTy(.alloc, src_ptr_ty);
	27611	_ = try block.addBinOp(.store_safe, ptr, inst);
	27612	const casted_ptr = try block.addTyOp(.ptr_cast, dest_ptr_ty, ptr);
	27613	break :coerced try block.addTyOp(.load, dest_ty, casted_ptr);
	27614	},
	27615	.same_type => unreachable, // we checked for equal types just above
	27616	.bit_cast => try block.addTyOp(.bit_cast, dest_ty, inst),
	27617	.ptr_cast => try block.addTyOp(.ptr_cast, dest_ty, inst),
	27618	.error_cast => try block.addTyOp(.error_cast, dest_ty, inst),
	27619	};
	27620	try sema.checkKnownAllocPtr(block, inst, coerced);
	27621	return coerced;
27762	27622	}
27763	27623
27764	27624	switch (dest_ty.zigTypeTag(zcu)) {
...	...	@@ -27872,8 +27732,8 @@ fn coerceExtra(
27872	27732
27873	27733	if (dest_info.sentinel != .none) {
27874	27734	if (array_ty.sentinel(zcu)) \|inst_sent\| {
27875		if (Air.internedToRef(dest_info.sentinel) !=
27876		try sema.coerceInMemory(inst_sent, dst_elem_type))
	27735	if (dest_info.sentinel !=
	27736	(try pt.getCoerced(inst_sent, dst_elem_type)).toIntern())
27877	27737	{
27878	27738	in_memory_result = .{ .ptr_sentinel = .{
27879	27739	.actual = inst_sent,
...	...	@@ -28067,8 +27927,8 @@ fn coerceExtra(
28067	27927	}
28068	27928
28069	27929	if (dest_info.sentinel == .none or inst_info.sentinel == .none or
28070		Air.internedToRef(dest_info.sentinel) !=
28071		try sema.coerceInMemory(Value.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child)))
	27930	dest_info.sentinel !=
	27931	(try pt.getCoerced(.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child))).toIntern())
28072	27932	break :p;
28073	27933
28074	27934	const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);
...	...	@@ -28117,7 +27977,7 @@ fn coerceExtra(
28117	27977	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
28118	27978	{
28119	27979	try sema.requireRuntimeBlock(block, inst_src, null);
28120		return block.addTyOp(.intcast, dest_ty, inst);
	27980	return block.addTyOp(.int_cast, dest_ty, inst);
28121	27981	}
28122	27982	},
28123	27983	else => {},
...	...	@@ -28379,16 +28239,8 @@ fn coerceExtra(
28379	28239	return sema.failWithOwnedErrorMsg(block, msg);
28380	28240	}
28381	28241
28382		fn coerceInMemory(
28383		sema: *Sema,
28384		val: Value,
28385		dst_ty: Type,
28386		) CompileError!Air.Inst.Ref {
28387		return Air.internedToRef((try sema.pt.getCoerced(val, dst_ty)).toIntern());
28388		}
28389
28390	28242	const InMemoryCoercionResult = union(enum) {
28391		ok,
	28243	ok: Strategy,
28392	28244	no_match: Pair,
28393	28245	int_not_coercible: Int,
28394	28246	comptime_int_not_coercible: TypeValuePair,
...	...	@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {
28424	28276	double_ptr_to_anyopaque: Pair,
28425	28277	slice_to_anyopaque: Pair,
28426	28278
	28279	const Strategy = enum {
	28280	/// There isn't a special strategy for this particular coercion---we'll just need to
	28281	/// reinterpret the bytes in memory.
	28282	none,
	28283
	28284	/// The source and destination types are equal, so no explicit cast operation is necessary.
	28285	same_type,
	28286	/// The coercion can be lowered to `Air.Inst.Tag.bit_cast`.
	28287	bit_cast,
	28288	/// The coercion can be lowered to `Air.Inst.Tag.ptr_cast`.
	28289	ptr_cast,
	28290	/// The coercion can be lowered to `Air.Inst.Tag.error_cast`.
	28291	error_cast,
	28292	};
	28293
28427	28294	const Pair = struct {
28428	28295	actual: Type,
28429	28296	wanted: Type,
...	...	@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(
28797	28664	}
28798	28665
28799	28666	if (dest_ty.eql(src_ty))
28800		return .ok;
	28667	return .{ .ok = .same_type };
28801	28668
28802	28669	const dest_tag = dest_ty.zigTypeTag(zcu);
28803	28670	const src_tag = src_ty.zigTypeTag(zcu);
...	...	@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(
28810	28677	if (dest_info.signedness == src_info.signedness and
28811	28678	dest_info.bits == src_info.bits)
28812	28679	{
28813		return .ok;
	28680	return .{ .ok = .bit_cast };
28814	28681	}
28815	28682
28816	28683	if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or
...	...	@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(
28818	28685	(dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or
28819	28686	(dest_info.signedness == .unsigned and src_info.signedness == .signed))
28820	28687	{
28821		return InMemoryCoercionResult{ .int_not_coercible = .{
	28688	return .{ .int_not_coercible = .{
28822	28689	.actual_signedness = src_info.signedness,
28823	28690	.wanted_signedness = dest_info.signedness,
28824	28691	.actual_bits = src_info.bits,
...	...	@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(
28841	28708	const dest_bits = dest_ty.floatBits(target);
28842	28709	const src_bits = src_ty.floatBits(target);
28843	28710	if (dest_bits == src_bits) {
28844		return .ok;
	28711	return .{ .ok = .bit_cast };
28845	28712	}
28846	28713	}
28847	28714
...	...	@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(
28864	28731	if (dest_tag == .error_union and src_tag == .error_union) {
28865	28732	const dest_payload = dest_ty.errorUnionPayload(zcu);
28866	28733	const src_payload = src_ty.errorUnionPayload(zcu);
28867		const child = try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null);
28868		if (child != .ok) {
28869		return .{ .error_union_payload = .{
28870		.child = try child.dupe(sema.arena),
	28734	const payload_strat = switch (try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null)) {
	28735	.ok => \|strat\| strat,
	28736	else => \|payload_result\| return .{ .error_union_payload = .{
	28737	.child = try payload_result.dupe(sema.arena),
28871	28738	.actual = src_payload,
28872	28739	.wanted = dest_payload,
28873		} };
	28740	} },
	28741	};
	28742	switch (try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null)) {
	28743	.ok => {},
	28744	else => \|err_set_result\| return err_set_result,
28874	28745	}
28875		return try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null);
	28746	return switch (payload_strat) {
	28747	.same_type => .{ .ok = .error_cast },
	28748	else => .{ .ok = .none },
	28749	};
28876	28750	}
28877	28751
28878	28752	// Error Sets
28879	28753	if (dest_tag == .error_set and src_tag == .error_set) {
28880		const res1 = try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src);
28881		if (!dest_is_mut or res1 != .ok) return res1;
28882		// src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
28883		const res2 = try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src);
28884		return res2;
	28754	switch (try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src)) {
	28755	.ok => \|strat\| assert(strat == .error_cast),
	28756	else => \|result\| return result,
	28757	}
	28758	if (dest_is_mut) {
	28759	// src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
	28760	switch (try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src)) {
	28761	.ok => \|strat\| assert(strat == .error_cast),
	28762	else => \|result\| return result,
	28763	}
	28764	}
	28765	return .{ .ok = .error_cast };
28885	28766	}
28886	28767
28887	28768	// Arrays
...	...	@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(
28896	28777	}
28897	28778
28898	28779	const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);
28899		switch (child) {
28900		.ok => {},
28901		.no_match => return child,
	28780	const child_strat = switch (child) {
	28781	.ok => \|strat\| strat,
	28782	.no_match => \|no_match\| return .{ .no_match = no_match },
28902	28783	else => {
28903	28784	return .{ .array_elem = .{
28904	28785	.child = try child.dupe(sema.arena),
...	...	@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(
28906	28787	.wanted = dest_info.elem_type,
28907	28788	} };
28908	28789	},
28909		}
	28790	};
28910	28791	const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or
28911	28792	(src_info.sentinel != null and
28912	28793	dest_info.sentinel != null and
...	...	@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(
28922	28803	.ty = dest_info.elem_type,
28923	28804	} };
28924	28805	}
28925		return .ok;
	28806	return .{ .ok = switch (child_strat) {
	28807	.bit_cast => .bit_cast,
	28808	else => .none,
	28809	} };
28926	28810	}
28927	28811
28928	28812	// Vectors
...	...	@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(
28938	28822
28939	28823	const dest_elem_ty = dest_ty.scalarType(zcu);
28940	28824	const src_elem_ty = src_ty.scalarType(zcu);
28941		const child = try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null);
28942		if (child != .ok) {
28943		return .{ .vector_elem = .{
28944		.child = try child.dupe(sema.arena),
	28825	switch (try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null)) {
	28826	.ok => \|child_strat\| return .{ .ok = switch (child_strat) {
	28827	.bit_cast => .bit_cast,
	28828	.ptr_cast => .ptr_cast,
	28829	else => .none,
	28830	} },
	28831	else => \|child_result\| return .{ .vector_elem = .{
	28832	.child = try child_result.dupe(sema.arena),
28945	28833	.actual = src_elem_ty,
28946	28834	.wanted = dest_elem_ty,
28947		} };
	28835	} },
28948	28836	}
28949
28950		return .ok;
28951	28837	}
28952	28838
28953	28839	// Optionals
...	...	@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(
28971	28857	} };
28972	28858	}
28973	28859
28974		return .ok;
	28860	return .{ .ok = .none };
28975	28861	}
28976	28862
28977	28863	// Tuples (with in-memory-coercible fields)
...	...	@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(
28985	28871	const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);
28986	28872	if (field != .ok) break :tuple;
28987	28873	}
28988		return .ok;
	28874	return .{ .ok = .none };
28989	28875	}
28990	28876
28991	28877	return .{ .no_match = .{
...	...	@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(
29008	28894	const ip = &zcu.intern_pool;
29009	28895
29010	28896	const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {
29011		.anyerror_type => return .ok,
	28897	.anyerror_type => return .{ .ok = .error_cast },
29012	28898	.adhoc_inferred_error_set_type => {
29013	28899	// We are trying to coerce an error set to the current function's
29014	28900	// inferred error set.
29015	28901	const dst_ies = sema.fn_ret_ty_ies.?;
29016	28902	try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29017		return .ok;
	28903	return .{ .ok = .error_cast };
29018	28904	},
29019	28905	else => \|err_set_ty\| switch (ip.indexToKey(err_set_ty)) {
29020	28906	.inferred_error_set_type => \|func_index\| {
...	...	@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(
29023	28909	// We are trying to coerce an error set to the current function's
29024	28910	// inferred error set.
29025	28911	try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29026		return .ok;
	28912	return .{ .ok = .error_cast };
29027	28913	}
29028	28914	}
29029	28915	try sema.ensureFuncIesResolved(block, dest_src, func_index);
...	...	@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(
29062	28948	) };
29063	28949	}
29064	28950
29065		return .ok;
	28951	return .{ .ok = .error_cast };
29066	28952	}
29067	28953
29068	28954	fn coerceInMemoryAllowedFns(
...	...	@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(
29179	29065	}
29180	29066	}
29181	29067
29182		return .ok;
	29068	return .{ .ok = .none };
29183	29069	}
29184	29070
29185	29071	fn callconvCoerceAllowed(
...	...	@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(
29256	29142	const ok_ptr_size = src_info.flags.size == dest_info.flags.size or
29257	29143	src_info.flags.size == .c or dest_info.flags.size == .c;
29258	29144	if (!ok_ptr_size) {
29259		return InMemoryCoercionResult{ .ptr_size = .{
	29145	return .{ .ptr_size = .{
29260	29146	.actual = src_info.flags.size,
29261	29147	.wanted = dest_info.flags.size,
29262	29148	} };
...	...	@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(
29392	29278	break :a dest_child.abiAlignment(zcu);
29393	29279	} else dest_info.flags.alignment;
29394	29280	if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {
29395		return InMemoryCoercionResult{ .ptr_alignment = .{
	29281	return .{ .ptr_alignment = .{
29396	29282	.actual = src_align,
29397	29283	.wanted = dest_align,
29398	29284	} };
29399	29285	}
29400	29286	}
29401	29287
29402		return .ok;
	29288	return .{ .ok = .ptr_cast };
29403	29289	}
29404	29290
29405	29291	fn coerceVarArgParam(
...	...	@@ -29703,7 +29589,6 @@ fn bitCast(
29703	29589	dest_ty: Type,
29704	29590	inst: Air.Inst.Ref,
29705	29591	inst_src: LazySrcLoc,
29706		operand_src: ?LazySrcLoc,
29707	29592	) CompileError!Air.Inst.Ref {
29708	29593	const pt = sema.pt;
29709	29594	const zcu = pt.zcu;
...	...	@@ -29712,6 +29597,11 @@ fn bitCast(
29712	29597	old_ty.assertHasLayout(zcu);
29713	29598	try sema.ensureLayoutResolved(dest_ty, inst_src, .init);
29714	29599
	29600	assert(old_ty.hasBitRepresentation(zcu));
	29601	assert(dest_ty.hasBitRepresentation(zcu));
	29602	assert(old_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
	29603	assert(dest_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
	29604
29715	29605	const dest_bits = dest_ty.bitSize(zcu);
29716	29606	const old_bits = old_ty.bitSize(zcu);
29717	29607
...	...	@@ -29725,20 +29615,30 @@ fn bitCast(
29725	29615	}
29726	29616
29727	29617	if (sema.resolveValue(inst)) \|val\| {
29728		if (val.isUndef(zcu))
29729		return pt.undefRef(dest_ty);
29730		if (old_ty.zigTypeTag(zcu) == .error_set and dest_ty.zigTypeTag(zcu) == .error_set) {
29731		// Special case: we sometimes call `bitCast` on error set values, but they
29732		// don't have a well-defined layout, so we can't use `bitCastVal` on them.
29733		return Air.internedToRef((try pt.getCoerced(val, dest_ty)).toIntern());
29734		}
29735		if (try sema.bitCastVal(val, dest_ty, 0, 0, 0)) \|result_val\| {
29736		return Air.internedToRef(result_val.toIntern());
29737		}
	29618	return .fromValue(try sema.bitCastVal(val, dest_ty));
29738	29619	}
29739		try sema.requireRuntimeBlock(block, inst_src, operand_src);
29740	29620	try sema.validateRuntimeValue(block, inst_src, inst);
29741		return block.addBitCast(dest_ty, inst);
	29621	return block.addTyOp(.bit_cast, dest_ty, inst);
	29622	}
	29623
	29624	/// Supports only types which `@bitCast` supports, so pointers are not supported.
	29625	pub fn bitCastVal(
	29626	sema: *Sema,
	29627	val: Value,
	29628	dest_ty: Type,
	29629	) Allocator.Error!Value {
	29630	const pt = sema.pt;
	29631	const zcu = pt.zcu;
	29632	const bit_size = dest_ty.bitSize(zcu);
	29633	assert(val.typeOf(zcu).bitSize(zcu) == bit_size);
	29634	if (val.isUndef(zcu)) {
	29635	return pt.undefValue(dest_ty);
	29636	} else {
	29637	const buf = try sema.arena.alloc(u8, @intCast((bit_size + 7) / 8));
	29638	@memset(buf, 0);
	29639	val.writeToPackedMemory(zcu, buf, 0);
	29640	return .readFromPackedMemory(dest_ty, pt, buf, 0);
	29641	}
29742	29642	}
29743	29643
29744	29644	fn coerceArrayPtrToSlice(
...	...	@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(
29855	29755	);
29856	29756	}
29857	29757	try sema.requireRuntimeBlock(block, inst_src, null);
29858		const inst_allows_zero = inst_ty.zigTypeTag(zcu) != .pointer or inst_ty.ptrAllowsZero(zcu);
29859		if (block.wantSafety() and inst_allows_zero and !dest_ty.ptrAllowsZero(zcu)) {
	29758	const maybe_zero: bool = switch (inst_ty.toIntern()) {
	29759	.usize_type, .isize_type => true,
	29760	else => inst_ty.ptrAllowsZero(zcu),
	29761	};
	29762	if (block.wantSafety() and maybe_zero and !dest_ty.ptrAllowsZero(zcu)) {
29860	29763	try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);
29861	29764	const actual_ptr = if (inst_ty.isSlice(zcu))
29862	29765	try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)
29863	29766	else
29864	29767	inst;
29865		const ptr_int = try block.addBitCast(.usize, actual_ptr);
	29768	const ptr_int = try block.addTyOp(.int_from_ptr, .usize, actual_ptr);
29866	29769	const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);
29867	29770	const ok = if (inst_ty.isSlice(zcu)) ok: {
29868	29771	const len = try sema.analyzeSliceLen(block, inst_src, inst);
...	...	@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(
29871	29774	} else is_non_zero;
29872	29775	try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);
29873	29776	}
29874		const new_ptr = try sema.bitCast(block, dest_ty, inst, inst_src, null);
	29777	const new_ptr: Air.Inst.Ref = switch (inst_ty.toIntern()) {
	29778	.usize_type => try block.addTyOp(.ptr_from_int, dest_ty, inst),
	29779	.isize_type => new_ptr: {
	29780	const usize_inst = try block.addTyOp(.bit_cast, .usize, inst);
	29781	break :new_ptr try block.addTyOp(.ptr_from_int, dest_ty, usize_inst);
	29782	},
	29783	else => try block.addTyOp(.ptr_cast, dest_ty, inst),
	29784	};
29875	29785	try sema.checkKnownAllocPtr(block, inst, new_ptr);
29876	29786	return new_ptr;
29877	29787	}
...	...	@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(
29968	29878	return .fromValue(opv);
29969	29879	} else {
29970	29880	// The union layout is just the tag, so we can bitcast the enum straight to the union.
29971		return block.addBitCast(union_ty, enum_tag);
	29881	return block.addTyOp(.union_from_enum, union_ty, enum_tag);
29972	29882	}
29973	29883	}
29974	29884
...	...	@@ -30017,18 +29927,6 @@ fn coerceArrayLike(
30017	29927	const inst_ty = sema.typeOf(inst);
30018	29928	const target = zcu.getTarget();
30019	29929
30020		// try coercion of the whole array
30021		const in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, null);
30022		if (in_memory_result == .ok) {
30023		if (sema.resolveValue(inst)) \|inst_val\| {
30024		// These types share the same comptime value representation.
30025		return sema.coerceInMemory(inst_val, dest_ty);
30026		}
30027		try sema.requireRuntimeBlock(block, inst_src, null);
30028		return block.addBitCast(dest_ty, inst);
30029		}
30030
30031		// otherwise, try element by element
30032	29930	const inst_len = inst_ty.arrayLen(zcu);
30033	29931	const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));
30034	29932	if (dest_len != inst_len) {
...	...	@@ -30055,7 +29953,7 @@ fn coerceArrayLike(
30055	29953	(dst_info.signedness == .signed and dst_info.bits > src_info.bits))
30056	29954	{
30057	29955	try sema.requireRuntimeBlock(block, inst_src, null);
30058		return block.addTyOp(.intcast, dest_ty, inst);
	29956	return block.addTyOp(.int_cast, dest_ty, inst);
30059	29957	}
30060	29958	},
30061	29959	.float => if (inst_elem_ty.isRuntimeFloat()) {
...	...	@@ -30582,7 +30480,7 @@ fn analyzeRef(
30582	30480
30583	30481	// Cast to the constant pointer type. We do this directly rather than going via `coerce` to
30584	30482	// avoid errors in the `block.isComptime()` case.
30585		return block.addBitCast(ptr_type, alloc);
	30483	return block.addTyOp(.ptr_cast, ptr_type, alloc);
30586	30484	}
30587	30485
30588	30486	fn analyzeLoad(
...	...	@@ -31334,7 +31232,7 @@ fn analyzeSlice(
31334	31232
31335	31233	const opt_new_ptr_val = sema.resolveValue(new_ptr);
31336	31234	const new_ptr_val = opt_new_ptr_val orelse {
31337		const result = try block.addBitCast(return_ty, new_ptr);
	31235	const result = try block.addTyOp(.ptr_cast, return_ty, new_ptr);
31338	31236	if (block.wantSafety()) {
31339	31237	// requirement: slicing C ptr is non-null
31340	31238	if (ptr_ptr_child_ty.isCPtr(zcu)) {
...	...	@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {
34366	34264	}
34367	34265	}
34368	34266
34369		pub const bitCastVal = @import("Sema/bitcast.zig").bitCast;
34370		pub const bitCastSpliceVal = @import("Sema/bitcast.zig").bitCastSplice;
	34267	pub const castMemory = @import("Sema/reinterpret.zig").castMemory;
	34268	pub const spliceMemory = @import("Sema/reinterpret.zig").spliceMemory;
34371	34269
34372	34270	const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;
34373	34271	const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;

src/Sema/LowerZon.zig+4-11

...	...	@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool
815	815	.@"packed" => result: {
816	816	const arena = self.sema.arena;
817	817	const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));
	818	@memset(buf, 0);
818	819	var bit_offset: u16 = 0;
819	820	for (field_values) \|field_ip\| {
820	821	const field_val: Value = .fromInterned(field_ip);
821		field_val.writeToPackedMemory(zcu, buf, bit_offset) catch \|err\| switch (err) {
822		error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
823		error.OutOfMemory => \|e\| return e,
824		};
	822	field_val.writeToPackedMemory(zcu, buf, bit_offset);
825	823	bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
826	824	}
827	825	assert(bit_offset == res_ty.bitSize(zcu));
828		break :result Value.readFromPackedMemory(res_ty, pt, buf, 0, arena) catch \|err\| switch (err) {
829		error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
830		error.OutOfMemory => \|e\| return e,
831		};
	826	break :result try .readFromPackedMemory(res_ty, pt, buf, 0);
832	827	},
833	828	};
834	829	return result.toIntern();
...	...	@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool.
981	976	};
982	977	const result: Value = switch (union_info.layout) {
983	978	.auto, .@"extern" => try pt.unionValue(res_ty, tag, val),
984		.@"packed" => try self.sema.bitCastVal(val, res_ty, 0, 0, 0) orelse {
985		unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
986		},
	979	.@"packed" => try self.sema.bitCastVal(val, res_ty),
987	980	};
988	981	return result.toIntern();
989	982	}

src/Sema/bitcast.zig deleted-774

...	...	@@ -1,774 +0,0 @@
1		//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2		//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3		//! a sequence of values in packed memory, and then unflatten through a combination of special
4		//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5		//!
6		//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7		//! have their fields reversed when represented as packed memory on such targets.
8
9		/// If `host_bits` is `0`, attempts to convert the memory at offset
10		/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11		/// ignoring `bit_offset`.
12		///
13		/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14		/// non-packed value consisting of `host_bits` bits. A value of type
15		/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16		/// into this value.
17		///
18		/// Returns `null` if the operation must be performed at runtime.
19		pub fn bitCast(
20		sema: *Sema,
21		val: Value,
22		dest_ty: Type,
23		byte_offset: u64,
24		host_bits: u64,
25		bit_offset: u64,
26		) CompileError!?Value {
27		return bitCastInner(sema, val, dest_ty, byte_offset, host_bits, bit_offset) catch \|err\| switch (err) {
28		error.ReinterpretDeclRef => return null,
29		error.IllDefinedMemoryLayout => unreachable,
30		error.Unimplemented => @panic("unimplemented bitcast"),
31		else => \|e\| return e,
32		};
33		}
34
35		/// Uses bitcasting to splice the value `splice_val` into `val`,
36		/// replacing overlapping bits and returning the modified value.
37		///
38		/// If `host_bits` is `0`, splices `splice_val` at an offset
39		/// `byte_offset` bytes into the virtual memory of `val`, ignoring
40		/// `bit_offset`.
41		///
42		/// Otherwise, `byte_offset` is an offset into bytes into `val` to
43		/// a non-packed value consisting of `host_bits` bits. The value
44		/// `splice_val` will be placed at a packed offset of `bit_offset`
45		/// into this value.
46		pub fn bitCastSplice(
47		sema: *Sema,
48		val: Value,
49		splice_val: Value,
50		byte_offset: u64,
51		host_bits: u64,
52		bit_offset: u64,
53		) CompileError!?Value {
54		return bitCastSpliceInner(sema, val, splice_val, byte_offset, host_bits, bit_offset) catch \|err\| switch (err) {
55		error.ReinterpretDeclRef => return null,
56		error.IllDefinedMemoryLayout => unreachable,
57		error.Unimplemented => @panic("unimplemented bitcast"),
58		else => \|e\| return e,
59		};
60		}
61
62		const BitCastError = CompileError \|\| error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented };
63
64		fn bitCastInner(
65		sema: *Sema,
66		val: Value,
67		dest_ty: Type,
68		byte_offset: u64,
69		host_bits: u64,
70		bit_offset: u64,
71		) BitCastError!Value {
72		const pt = sema.pt;
73		const zcu = pt.zcu;
74		const endian = zcu.getTarget().cpu.arch.endian();
75
76		if (dest_ty.toIntern() == val.typeOf(zcu).toIntern() and bit_offset == 0) {
77		return val;
78		}
79
80		const val_ty = val.typeOf(zcu);
81
82		val_ty.assertHasLayout(zcu);
83		dest_ty.assertHasLayout(zcu);
84
85		assert(val_ty.hasWellDefinedLayout(zcu));
86
87		const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
88		.{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
89		else
90		.{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
91
92		const skip_bits = switch (endian) {
93		.little => bit_offset + byte_offset * 8,
94		.big => if (host_bits > 0)
95		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
96		else
97		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - dest_ty.bitSize(zcu),
98		};
99
100		var unpack: UnpackValueBits = .{
101		.pt = sema.pt,
102		.arena = sema.arena,
103		.skip_bits = skip_bits,
104		.remaining_bits = dest_ty.bitSize(zcu),
105		.unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
106		};
107		switch (endian) {
108		.little => {
109		try unpack.add(val);
110		try unpack.padding(abi_pad_bits);
111		},
112		.big => {
113		try unpack.padding(abi_pad_bits);
114		try unpack.add(val);
115		},
116		}
117		try unpack.padding(host_pad_bits);
118
119		var pack: PackValueBits = .{
120		.pt = sema.pt,
121		.arena = sema.arena,
122		.unpacked = unpack.unpacked.items,
123		};
124		return pack.get(dest_ty);
125		}
126
127		fn bitCastSpliceInner(
128		sema: *Sema,
129		val: Value,
130		splice_val: Value,
131		byte_offset: u64,
132		host_bits: u64,
133		bit_offset: u64,
134		) BitCastError!Value {
135		const pt = sema.pt;
136		const zcu = pt.zcu;
137		const endian = zcu.getTarget().cpu.arch.endian();
138		const val_ty = val.typeOf(zcu);
139		const splice_val_ty = splice_val.typeOf(zcu);
140
141		val_ty.assertHasLayout(zcu);
142		splice_val_ty.assertHasLayout(zcu);
143
144		const splice_bits = splice_val_ty.bitSize(zcu);
145
146		const splice_offset = switch (endian) {
147		.little => bit_offset + byte_offset * 8,
148		.big => if (host_bits > 0)
149		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
150		else
151		val_ty.abiSize(zcu) * 8 - byte_offset * 8 - splice_bits,
152		};
153
154		assert(splice_offset + splice_bits <= val_ty.abiSize(zcu) * 8);
155
156		const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
157		.{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
158		else
159		.{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
160
161		var unpack: UnpackValueBits = .{
162		.pt = pt,
163		.arena = sema.arena,
164		.skip_bits = 0,
165		.remaining_bits = splice_offset,
166		.unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
167		};
168		switch (endian) {
169		.little => {
170		try unpack.add(val);
171		try unpack.padding(abi_pad_bits);
172		},
173		.big => {
174		try unpack.padding(abi_pad_bits);
175		try unpack.add(val);
176		},
177		}
178		try unpack.padding(host_pad_bits);
179
180		unpack.remaining_bits = splice_bits;
181		try unpack.add(splice_val);
182
183		unpack.skip_bits = splice_offset + splice_bits;
184		unpack.remaining_bits = val_ty.abiSize(zcu) * 8 - splice_offset - splice_bits;
185		switch (endian) {
186		.little => {
187		try unpack.add(val);
188		try unpack.padding(abi_pad_bits);
189		},
190		.big => {
191		try unpack.padding(abi_pad_bits);
192		try unpack.add(val);
193		},
194		}
195		try unpack.padding(host_pad_bits);
196
197		var pack: PackValueBits = .{
198		.pt = pt,
199		.arena = sema.arena,
200		.unpacked = unpack.unpacked.items,
201		};
202		switch (endian) {
203		.little => {},
204		.big => try pack.padding(abi_pad_bits),
205		}
206		return pack.get(val_ty);
207		}
208
209		/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
210		/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
211		/// of values in packed memory - therefore, on big-endian targets, the first element of this
212		/// list contains bits from the final byte of the value.
213		const UnpackValueBits = struct {
214		pt: Zcu.PerThread,
215		arena: Allocator,
216		skip_bits: u64,
217		remaining_bits: u64,
218		extra_bits: u64 = undefined,
219		unpacked: std.array_list.Managed(InternPool.Index),
220
221		fn add(unpack: *UnpackValueBits, val: Value) BitCastError!void {
222		const pt = unpack.pt;
223		const zcu = pt.zcu;
224		const endian = zcu.getTarget().cpu.arch.endian();
225		const ip = &zcu.intern_pool;
226
227		if (unpack.remaining_bits == 0) {
228		return;
229		}
230
231		const ty = val.typeOf(zcu);
232		const bit_size = ty.bitSize(zcu);
233
234		if (unpack.skip_bits >= bit_size) {
235		unpack.skip_bits -= bit_size;
236		return;
237		}
238
239		switch (ip.indexToKey(val.toIntern())) {
240		.int_type,
241		.ptr_type,
242		.array_type,
243		.vector_type,
244		.opt_type,
245		.anyframe_type,
246		.error_union_type,
247		.simple_type,
248		.struct_type,
249		.tuple_type,
250		.union_type,
251		.opaque_type,
252		.spirv_type,
253		.enum_type,
254		.func_type,
255		.error_set_type,
256		.inferred_error_set_type,
257		.@"extern",
258		.func,
259		.err,
260		.error_union,
261		.enum_literal,
262		.slice,
263		.memoized_call,
264		=> unreachable, // ill-defined layout or not real values
265
266		.undef,
267		.int,
268		.enum_tag,
269		.simple_value,
270		.float,
271		.ptr,
272		.opt,
273		=> try unpack.primitive(val),
274
275		.bitpack => \|bitpack\| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
276
277		.aggregate => switch (ty.zigTypeTag(zcu)) {
278		.vector => {
279		const len: usize = @intCast(ty.arrayLen(zcu));
280		for (0..len) \|i\| {
281		// We reverse vector elements in packed memory on BE targets.
282		const real_idx = switch (endian) {
283		.little => i,
284		.big => len - i - 1,
285		};
286		const elem_val = try val.elemValue(pt, real_idx);
287		try unpack.add(elem_val);
288		}
289		},
290		.array => {
291		// Each element is padded up to its ABI size. Padding bits are undefined.
292		// The final element does not have trailing padding.
293		// Elements are reversed in packed memory on BE targets.
294		const elem_ty = ty.childType(zcu);
295		const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
296		const len = ty.arrayLen(zcu);
297		const maybe_sent = ty.sentinel(zcu);
298
299		if (endian == .big) if (maybe_sent) \|s\| {
300		try unpack.add(s);
301		if (len != 0) try unpack.padding(pad_bits);
302		};
303
304		for (0..@intCast(len)) \|i\| {
305		// We reverse array elements in packed memory on BE targets.
306		const real_idx = switch (endian) {
307		.little => i,
308		.big => len - i - 1,
309		};
310		const elem_val = try val.elemValue(pt, @intCast(real_idx));
311		try unpack.add(elem_val);
312		if (i != len - 1) try unpack.padding(pad_bits);
313		}
314
315		if (endian == .little) if (maybe_sent) \|s\| {
316		if (len != 0) try unpack.padding(pad_bits);
317		try unpack.add(s);
318		};
319		},
320		.@"struct" => switch (ty.containerLayout(zcu)) {
321		.auto => unreachable, // ill-defined layout
322		.@"extern" => switch (endian) {
323		.little => {
324		var cur_bit_off: u64 = 0;
325		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
326		while (it.next()) \|field_idx\| {
327		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
328		const pad_bits = want_bit_off - cur_bit_off;
329		const field_val = try val.fieldValue(pt, field_idx);
330		try unpack.padding(pad_bits);
331		try unpack.add(field_val);
332		cur_bit_off = want_bit_off + field_val.typeOf(zcu).bitSize(zcu);
333		}
334		// Add trailing padding bits.
335		try unpack.padding(bit_size - cur_bit_off);
336		},
337		.big => {
338		var cur_bit_off: u64 = bit_size;
339		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
340		while (it.next()) \|field_idx\| {
341		const field_val = try val.fieldValue(pt, field_idx);
342		const field_ty = field_val.typeOf(zcu);
343		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
344		const pad_bits = cur_bit_off - want_bit_off;
345		try unpack.padding(pad_bits);
346		try unpack.add(field_val);
347		cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
348		}
349		assert(cur_bit_off == 0);
350		},
351		},
352		.@"packed" => {
353		// Just add all fields in order. There are no padding bits.
354		// This is identical between LE and BE targets.
355		for (0..ty.structFieldCount(zcu)) \|i\| {
356		const field_val = try val.fieldValue(pt, i);
357		try unpack.add(field_val);
358		}
359		},
360		},
361		else => unreachable,
362		},
363
364		.un => \|un\| {
365		// We actually don't care about the tag here!
366		// Instead, we just need to write the payload value, plus any necessary padding.
367		// This correctly handles the case where `tag == .none`, since the payload is then
368		// either an integer or a byte array, both of which we can unpack.
369		const payload_val = Value.fromInterned(un.val);
370		const pad_bits = bit_size - payload_val.typeOf(zcu).bitSize(zcu);
371		if (endian == .little or ty.containerLayout(zcu) == .@"packed") {
372		try unpack.add(payload_val);
373		try unpack.padding(pad_bits);
374		} else {
375		try unpack.padding(pad_bits);
376		try unpack.add(payload_val);
377		}
378		},
379		}
380		}
381
382		fn padding(unpack: *UnpackValueBits, pad_bits: u64) BitCastError!void {
383		if (pad_bits == 0) return;
384		const pt = unpack.pt;
385		// Figure out how many full bytes and leftover bits there are.
386		const bytes = pad_bits / 8;
387		const bits = pad_bits % 8;
388		// Add undef u8 values for the bytes...
389		const undef_u8 = try pt.undefValue(Type.u8);
390		for (0..@intCast(bytes)) \|_\| {
391		try unpack.primitive(undef_u8);
392		}
393		// ...and an undef int for the leftover bits.
394		if (bits == 0) return;
395		const bits_ty = try pt.intType(.unsigned, @intCast(bits));
396		const bits_val = try pt.undefValue(bits_ty);
397		try unpack.primitive(bits_val);
398		}
399
400		fn primitive(unpack: *UnpackValueBits, val: Value) BitCastError!void {
401		const pt = unpack.pt;
402		const zcu = pt.zcu;
403
404		if (unpack.remaining_bits == 0) {
405		return;
406		}
407
408		const ty = val.typeOf(pt.zcu);
409		const bit_size = ty.bitSize(zcu);
410
411		// Note that this skips all zero-bit types.
412		if (unpack.skip_bits >= bit_size) {
413		unpack.skip_bits -= bit_size;
414		return;
415		}
416
417		if (unpack.skip_bits > 0) {
418		const skip = unpack.skip_bits;
419		unpack.skip_bits = 0;
420		return unpack.splitPrimitive(val, skip, bit_size - skip);
421		}
422
423		if (unpack.remaining_bits < bit_size) {
424		return unpack.splitPrimitive(val, 0, unpack.remaining_bits);
425		}
426
427		unpack.remaining_bits -\|= bit_size;
428
429		try unpack.unpacked.append(val.toIntern());
430		}
431
432		fn splitPrimitive(unpack: *UnpackValueBits, val: Value, bit_offset: u64, bit_count: u64) BitCastError!void {
433		const pt = unpack.pt;
434		const zcu = pt.zcu;
435		const ty = val.typeOf(pt.zcu);
436
437		const val_bits = ty.bitSize(zcu);
438		assert(bit_offset + bit_count <= val_bits);
439
440		switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
441		// In the `ptr` case, this will return `error.ReinterpretDeclRef`
442		// if we're trying to split a non-integer pointer value.
443		.int, .float, .enum_tag, .ptr, .opt => {
444		// This @intCast is okay because no primitive can exceed the size of a u16.
445		const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
446		const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
447		try val.writeToPackedMemory(zcu, buf, 0);
448		const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
449		try unpack.primitive(sub_val);
450		},
451		.undef => try unpack.padding(bit_count),
452		// The only values here with runtime bits are `true` and `false.
453		// These are both 1 bit, so will never need truncating.
454		.simple_value => unreachable,
455		else => unreachable, // zero-bit or not primitives
456		}
457		}
458		};
459
460		/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBits`),
461		/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
462		/// values and of pointers which align in virtual memory.
463		const PackValueBits = struct {
464		pt: Zcu.PerThread,
465		arena: Allocator,
466		bit_offset: u64 = 0,
467		unpacked: []const InternPool.Index,
468
469		fn get(pack: *PackValueBits, ty: Type) BitCastError!Value {
470		const pt = pack.pt;
471		const zcu = pt.zcu;
472		const endian = zcu.getTarget().cpu.arch.endian();
473		const ip = &zcu.intern_pool;
474		const arena = pack.arena;
475		switch (ty.zigTypeTag(zcu)) {
476		.vector => {
477		// Elements are bit-packed.
478		const len = ty.arrayLen(zcu);
479		const elem_ty = ty.childType(zcu);
480		const elems = try arena.alloc(InternPool.Index, @intCast(len));
481		// We reverse vector elements in packed memory on BE targets.
482		switch (endian) {
483		.little => for (elems) \|*elem\| {
484		elem.* = (try pack.get(elem_ty)).toIntern();
485		},
486		.big => {
487		var i = elems.len;
488		while (i > 0) {
489		i -= 1;
490		elems[i] = (try pack.get(elem_ty)).toIntern();
491		}
492		},
493		}
494		return pt.aggregateValue(ty, elems);
495		},
496		.array => {
497		// Each element is padded up to its ABI size. The final element does not have trailing padding.
498		const len = ty.arrayLen(zcu);
499		const elem_ty = ty.childType(zcu);
500		const maybe_sent = ty.sentinel(zcu);
501		const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
502		const elems = try arena.alloc(InternPool.Index, @intCast(len));
503
504		if (endian == .big and maybe_sent != null) {
505		// TODO: validate sentinel was preserved!
506		try pack.padding(elem_ty.bitSize(zcu));
507		if (len != 0) try pack.padding(pad_bits);
508		}
509
510		for (0..elems.len) \|i\| {
511		const real_idx = switch (endian) {
512		.little => i,
513		.big => len - i - 1,
514		};
515		elems[@intCast(real_idx)] = (try pack.get(elem_ty)).toIntern();
516		if (i != len - 1) try pack.padding(pad_bits);
517		}
518
519		if (endian == .little and maybe_sent != null) {
520		// TODO: validate sentinel was preserved!
521		if (len != 0) try pack.padding(pad_bits);
522		try pack.padding(elem_ty.bitSize(zcu));
523		}
524
525		return pt.aggregateValue(ty, elems);
526		},
527		.@"struct" => switch (ty.containerLayout(zcu)) {
528		.auto => unreachable, // ill-defined layout
529		.@"extern" => {
530		const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
531		@memset(elems, .none);
532		switch (endian) {
533		.little => {
534		var cur_bit_off: u64 = 0;
535		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
536		while (it.next()) \|field_idx\| {
537		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
538		try pack.padding(want_bit_off - cur_bit_off);
539		const field_ty = ty.fieldType(field_idx, zcu);
540		elems[field_idx] = (try pack.get(field_ty)).toIntern();
541		cur_bit_off = want_bit_off + field_ty.bitSize(zcu);
542		}
543		try pack.padding(ty.bitSize(zcu) - cur_bit_off);
544		},
545		.big => {
546		var cur_bit_off: u64 = ty.bitSize(zcu);
547		var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
548		while (it.next()) \|field_idx\| {
549		const field_ty = ty.fieldType(field_idx, zcu);
550		const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
551		try pack.padding(cur_bit_off - want_bit_off);
552		elems[field_idx] = (try pack.get(field_ty)).toIntern();
553		cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
554		}
555		assert(cur_bit_off == 0);
556		},
557		}
558		// Any fields which do not have runtime bits should be OPV or comptime fields.
559		// Fill those values now.
560		for (elems, 0..) \|*elem, field_idx\| {
561		if (elem.* != .none) continue;
562		const val = (try ty.structFieldValueComptime(pt, field_idx)).?;
563		elem.* = val.toIntern();
564		}
565		return pt.aggregateValue(ty, elems);
566		},
567		.@"packed" => {
568		const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
569		if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
570		return pt.bitpackValue(ty, backing_int_val);
571		},
572		},
573		.@"union" => switch (ty.containerLayout(zcu)) {
574		.auto => unreachable, // ill-defined layout
575		.@"extern" => {
576		// We will attempt to read as the backing representation. If this emits
577		// `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
578		// We will also attempt smaller fields when we get `undefined`, as if some bits are
579		// defined we want to include them.
580		// TODO: this is very very bad. We need a more sophisticated union representation.
581
582		const prev_unpacked = pack.unpacked;
583		const prev_bit_offset = pack.bit_offset;
584
585		const backing_ty = try ty.externUnionBackingType(pt);
586
587		backing: {
588		const backing_val = pack.get(backing_ty) catch \|err\| switch (err) {
589		error.ReinterpretDeclRef => {
590		pack.unpacked = prev_unpacked;
591		pack.bit_offset = prev_bit_offset;
592		break :backing;
593		},
594		else => \|e\| return e,
595		};
596		if (backing_val.isUndef(zcu)) {
597		pack.unpacked = prev_unpacked;
598		pack.bit_offset = prev_bit_offset;
599		break :backing;
600		}
601		return Value.fromInterned(try pt.internUnion(.{
602		.ty = ty.toIntern(),
603		.tag = .none,
604		.val = backing_val.toIntern(),
605		}));
606		}
607
608		const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
609		for (field_order, 0..) \|f, i\| f. = @intCast(i);
610		// Sort `field_order` to put the fields with the largest bit sizes first.
611		const SizeSortCtx = struct {
612		zcu: *Zcu,
613		field_types: []const InternPool.Index,
614		fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
615		const a_ty = Type.fromInterned(ctx.field_types[a_idx]);
616		const b_ty = Type.fromInterned(ctx.field_types[b_idx]);
617		return a_ty.bitSize(ctx.zcu) > b_ty.bitSize(ctx.zcu);
618		}
619		};
620		std.mem.sortUnstable(u32, field_order, SizeSortCtx{
621		.zcu = zcu,
622		.field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
623		}, SizeSortCtx.lessThan);
624
625		const padding_after = endian == .little or ty.containerLayout(zcu) == .@"packed";
626
627		for (field_order) \|field_idx\| {
628		const field_ty = Type.fromInterned(zcu.typeToUnion(ty).?.field_types.get(ip)[field_idx]);
629		const pad_bits = ty.bitSize(zcu) - field_ty.bitSize(zcu);
630		if (!padding_after) try pack.padding(pad_bits);
631		const field_val = pack.get(field_ty) catch \|err\| switch (err) {
632		error.ReinterpretDeclRef => {
633		pack.unpacked = prev_unpacked;
634		pack.bit_offset = prev_bit_offset;
635		continue;
636		},
637		else => \|e\| return e,
638		};
639		if (padding_after) try pack.padding(pad_bits);
640		if (field_val.isUndef(zcu)) {
641		pack.unpacked = prev_unpacked;
642		pack.bit_offset = prev_bit_offset;
643		continue;
644		}
645		const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_idx);
646		return Value.fromInterned(try pt.internUnion(.{
647		.ty = ty.toIntern(),
648		.tag = tag_val.toIntern(),
649		.val = field_val.toIntern(),
650		}));
651		}
652
653		// No field could represent the value. Just do whatever happens when we try to read
654		// the backing type - either `undefined` or `error.ReinterpretDeclRef`.
655		const backing_val = try pack.get(backing_ty);
656		return Value.fromInterned(try pt.internUnion(.{
657		.ty = ty.toIntern(),
658		.tag = .none,
659		.val = backing_val.toIntern(),
660		}));
661		},
662		.@"packed" => {
663		const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
664		if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
665		return pt.bitpackValue(ty, backing_int_val);
666		},
667		},
668		else => return pack.primitive(ty),
669		}
670		}
671
672		fn padding(pack: *PackValueBits, pad_bits: u64) BitCastError!void {
673		_ = pack.prepareBits(pad_bits);
674		}
675
676		fn primitive(pack: *PackValueBits, want_ty: Type) BitCastError!Value {
677		const pt = pack.pt;
678		const zcu = pt.zcu;
679
680		if (try want_ty.onePossibleValue(pt)) \|opv\| return opv;
681
682		const vals, const bit_offset = pack.prepareBits(want_ty.bitSize(zcu));
683
684		for (vals) \|val\| {
685		if (!Value.fromInterned(val).isUndef(zcu)) break;
686		} else {
687		// All bits of the value are `undefined`.
688		return pt.undefValue(want_ty);
689		}
690
691		// TODO: we need to decide how to handle partially-undef values here.
692		// Currently, a value with some undefined bits becomes `0xAA` so that we
693		// preserve the well-defined bits, because we can't currently represent
694		// a partially-undefined primitive (e.g. an int with some undef bits).
695		// In future, we probably want to take one of these two routes:
696		// * Define that if any bits are `undefined`, the entire value is `undefined`.
697		// This is a major breaking change, and probably a footgun.
698		// * Introduce tracking for partially-undef values at comptime.
699		// This would complicate a lot of operations in Sema, such as basic
700		// arithmetic.
701		// This design complexity is tracked by #19634.
702
703		ptr_cast: {
704		if (vals.len != 1) break :ptr_cast;
705		const val = Value.fromInterned(vals[0]);
706		if (!val.typeOf(zcu).isPtrAtRuntime(zcu)) break :ptr_cast;
707		if (!want_ty.isPtrAtRuntime(zcu)) break :ptr_cast;
708		return pt.getCoerced(val, want_ty);
709		}
710
711		// Reinterpret via an in-memory buffer.
712
713		var buf_bits: u64 = 0;
714		for (vals) \|ip_val\| {
715		const val = Value.fromInterned(ip_val);
716		const ty = val.typeOf(pt.zcu);
717		buf_bits += ty.bitSize(zcu);
718		}
719
720		const buf = try pack.arena.alloc(u8, @intCast((buf_bits + 7) / 8));
721		// We will skip writing undefined values, so mark the buffer as `0xAA` so we get "undefined" bits.
722		@memset(buf, 0xAA);
723		var cur_bit_off: usize = 0;
724		for (vals) \|ip_val\| {
725		const val = Value.fromInterned(ip_val);
726		const ty = val.typeOf(zcu);
727		if (!val.isUndef(zcu)) {
728		try val.writeToPackedMemory(zcu, buf, cur_bit_off);
729		}
730		cur_bit_off += @intCast(ty.bitSize(zcu));
731		}
732
733		return Value.readFromPackedMemory(want_ty, pt, buf, @intCast(bit_offset), pack.arena);
734		}
735
736		fn prepareBits(pack: *PackValueBits, need_bits: u64) struct { []const InternPool.Index, u64 } {
737		if (need_bits == 0) return .{ &.{}, 0 };
738
739		const pt = pack.pt;
740		const zcu = pt.zcu;
741
742		var bits: u64 = 0;
743		var len: usize = 0;
744		while (bits < pack.bit_offset + need_bits) {
745		bits += Value.fromInterned(pack.unpacked[len]).typeOf(pt.zcu).bitSize(zcu);
746		len += 1;
747		}
748
749		const result_vals = pack.unpacked[0..len];
750		const result_offset = pack.bit_offset;
751
752		const extra_bits = bits - pack.bit_offset - need_bits;
753		if (extra_bits == 0) {
754		pack.unpacked = pack.unpacked[len..];
755		pack.bit_offset = 0;
756		} else {
757		pack.unpacked = pack.unpacked[len - 1 ..];
758		pack.bit_offset = Value.fromInterned(pack.unpacked[0]).typeOf(pt.zcu).bitSize(zcu) - extra_bits;
759		}
760
761		return .{ result_vals, result_offset };
762		}
763		};
764
765		const std = @import("std");
766		const Allocator = std.mem.Allocator;
767		const assert = std.debug.assert;
768
769		const Sema = @import("../Sema.zig");
770		const Zcu = @import("../Zcu.zig");
771		const InternPool = @import("../InternPool.zig");
772		const Type = @import("../Type.zig");
773		const Value = @import("../Value.zig");
774		const CompileError = Zcu.CompileError;

src/Sema/comptime_ptr_access.zig+173-148

...	...	@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {
14	14	pub fn loadComptimePtr(sema: Sema, block: Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {
15	15	const pt = sema.pt;
16	16	const zcu = pt.zcu;
	17
17	18	const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
18		// TODO: host size for vectors is terrible
19		const host_bits = switch (ptr_info.flags.vector_index) {
20		.none => ptr_info.packed_offset.host_size * 8,
21		else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
22		};
23		const bit_offset = if (host_bits != 0) bit_offset: {
24		const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
25		const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
26		.none => 0,
27		else => \|idx\| switch (pt.zcu.getTarget().cpu.arch.endian()) {
28		.little => child_bits * @intFromEnum(idx),
29		.big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
30		},
31		};
32		if (child_bits + bit_offset > host_bits) {
	19	const elem_ty: Type = .fromInterned(ptr_info.child);
	20	const host_size = ptr_info.packed_offset.host_size;
	21
	22	if (host_size == 0) {
	23	return loadComptimePtrInner(sema, block, src, ptr, elem_ty, 0);
	24	}
	25
	26	assert(elem_ty.hasBitRepresentation(zcu));
	27	if (ptr_info.flags.vector_index == .none) {
	28	if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
33	29	return .exceeds_host_size;
34	30	}
35		break :bit_offset bit_offset;
36		} else 0;
37		return loadComptimePtrInner(sema, block, src, ptr, bit_offset, host_bits, Type.fromInterned(ptr_info.child), 0);
	31	const load_ty: Type = try pt.intType(.unsigned, host_size * 8);
	32	const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
	33	else => \|result\| return result,
	34	.success => \|mv\| mv,
	35	};
	36	const backing_int_val = try backing_int_mv.intern(pt, sema.arena);
	37	const buf = try sema.arena.alloc(u8, host_size);
	38	@memset(buf, 0);
	39	backing_int_val.writeToPackedMemory(zcu, buf, 0);
	40	const result_val: Value = try .readFromPackedMemory(elem_ty, pt, buf, ptr_info.packed_offset.bit_offset);
	41	return .{ .success = .{ .interned = result_val.toIntern() } };
	42	}
	43	if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
	44	return .exceeds_host_size;
	45	}
	46	const load_ty: Type = try pt.vectorType(.{
	47	.len = host_size,
	48	.child = elem_ty.toIntern(),
	49	});
	50	const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
	51	else => \|result\| return result,
	52	.success => \|mv\| mv,
	53	};
	54	const vector_val = try vector_mv.intern(pt, sema.arena);
	55	const result_val = try vector_val.elemValue(pt, @intFromEnum(ptr_info.flags.vector_index));
	56	return .{ .success = .{ .interned = result_val.toIntern() } };
38	57	}
39	58
40	59	pub const ComptimeStoreResult = union(enum) {
...	...	@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {
52	71	};
53	72
54	73	/// Perform a comptime load of value `store_val` to a pointer.
55		/// The pointer's type is ignored.
	74	///
	75	/// Asserts that the type of `store_val` equals the element type of the pointer type.
56	76	pub fn storeComptimePtr(
57	77	sema: *Sema,
58	78	block: *Block,
...	...	@@ -62,42 +82,84 @@ pub fn storeComptimePtr(
62	82	) !ComptimeStoreResult {
63	83	const pt = sema.pt;
64	84	const zcu = pt.zcu;
65		const ptr_info = ptr.typeOf(zcu).ptrInfo(zcu);
66		assert(store_val.typeOf(zcu).toIntern() == ptr_info.child);
67	85
68		{
69		const store_ty: Type = .fromInterned(ptr_info.child);
70		if (!store_ty.comptimeOnly(zcu) and !store_ty.hasRuntimeBits(zcu)) {
71		// zero-bit store; nothing to do
72		return .success;
73		}
	86	const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
	87	const elem_ty: Type = .fromInterned(ptr_info.child);
	88	const host_size = ptr_info.packed_offset.host_size;
	89	assert(store_val.typeOf(zcu).toIntern() == elem_ty.toIntern());
	90
	91	if (host_size == 0) {
	92	return storeComptimePtrInner(sema, block, src, ptr, store_val);
74	93	}
75	94
76		// TODO: host size for vectors is terrible
77		const host_bits = switch (ptr_info.flags.vector_index) {
78		.none => ptr_info.packed_offset.host_size * 8,
79		else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
80		};
81		const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
82		.none => 0,
83		else => \|idx\| switch (zcu.getTarget().cpu.arch.endian()) {
84		.little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
85		.big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian
86		},
87		};
88		const pseudo_store_ty = if (host_bits > 0) t: {
89		const need_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
90		if (need_bits + bit_offset > host_bits) {
	95	assert(elem_ty.hasBitRepresentation(zcu));
	96	if (ptr_info.flags.vector_index == .none) {
	97	if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
91	98	return .exceeds_host_size;
92	99	}
93		break :t try sema.pt.intType(.unsigned, @intCast(host_bits));
94		} else Type.fromInterned(ptr_info.child);
	100	const backing_ty: Type = try pt.intType(.unsigned, host_size * 8);
	101	const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, backing_ty, 0)) {
	102	.success => \|mv\| mv,
	103	.runtime_load => return .runtime_store,
	104	inline else => \|payload, tag\| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
	105	};
	106	const old_backing_int_val = try backing_int_mv.intern(pt, sema.arena);
	107	const buf = try sema.arena.alloc(u8, host_size);
	108	@memset(buf, 0);
	109	old_backing_int_val.writeToPackedMemory(zcu, buf, 0);
	110	// Write the new element...
	111	store_val.writeToPackedMemory(zcu, buf, ptr_info.packed_offset.bit_offset);
	112	// ...then read the resulting backing integer value...
	113	const new_backing_int_val: Value = try .readFromPackedMemory(backing_ty, pt, buf, 0);
	114	// ...and store that back into memory
	115	return storeComptimePtrInner(sema, block, src, ptr, new_backing_int_val);
	116	}
	117
	118	if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
	119	return .exceeds_host_size;
	120	}
	121	const vec_ty: Type = try pt.vectorType(.{
	122	.len = host_size,
	123	.child = elem_ty.toIntern(),
	124	});
	125	const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, vec_ty, 0)) {
	126	.success => \|mv\| mv,
	127	.runtime_load => return .runtime_store,
	128	inline else => \|payload, tag\| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
	129	};
	130	const old_vector_val = try vector_mv.intern(pt, sema.arena);
	131	const elems_buf = try sema.arena.alloc(InternPool.Index, host_size);
	132	for (elems_buf, 0..) \|*elem, elem_index\| {
	133	const elem_val = try old_vector_val.elemValue(pt, elem_index);
	134	elem.* = elem_val.toIntern();
	135	}
	136	elems_buf[@intFromEnum(ptr_info.flags.vector_index)] = store_val.toIntern();
	137	const new_vector_val = try pt.aggregateValue(vec_ty, elems_buf);
	138	return storeComptimePtrInner(sema, block, src, ptr, new_vector_val);
	139	}
95	140
96		const strat = try prepareComptimePtrStore(sema, block, src, ptr, pseudo_store_ty, 0);
	141	/// Like `storeComptimePtr`, except ignores the type of `ptr`, instead treating it as a single-item
	142	/// pointer to `store_val.typeOf(zcu)`.
	143	fn storeComptimePtrInner(
	144	sema: *Sema,
	145	block: *Block,
	146	src: LazySrcLoc,
	147	ptr: Value,
	148	store_val: Value,
	149	) !ComptimeStoreResult {
	150	const pt = sema.pt;
	151	const zcu = pt.zcu;
	152	const store_ty = store_val.typeOf(zcu);
	153
	154	if (store_ty.classify(zcu) == .one_possible_value) {
	155	// zero-bit store; nothing to do
	156	return .success;
	157	}
	158
	159	const strat = try prepareComptimePtrStore(sema, block, src, ptr, store_ty, 0);
97	160
98	161	// Propagate errors and handle comptime fields.
99	162	switch (strat) {
100		.direct, .index, .flat_index, .reinterpret => {},
101	163	.comptime_field => {
102	164	// To "store" to a comptime field, just perform a load of the field
103	165	// and see if the store value matches.
...	...	@@ -125,79 +187,60 @@ pub fn storeComptimePtr(
125	187	.inactive_union_field => return .inactive_union_field,
126	188	.needed_well_defined => \|ty\| return .{ .needed_well_defined = ty },
127	189	.out_of_bounds => \|ty\| return .{ .out_of_bounds = ty },
128		}
129
130		// Check the store is not inside a runtime condition
131		try checkComptimeVarStore(sema, block, src, strat.alloc());
132
133		if (host_bits == 0) {
134		// We can attempt a direct store depending on the strategy.
135		switch (strat) {
136		.direct => \|direct\| {
137		const want_ty = direct.val.typeOf(zcu);
138		const coerced_store_val = try pt.getCoerced(store_val, want_ty);
139		direct.val.* = .{ .interned = coerced_store_val.toIntern() };
140		return .success;
141		},
142		.index => \|index\| {
143		const want_ty = index.val.typeOf(zcu).childType(zcu);
144		const coerced_store_val = try pt.getCoerced(store_val, want_ty);
145		try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
146		return .success;
147		},
148		.flat_index => \|flat\| {
149		const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
150		const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
151		{
152		var next_idx: u64 = 0;
153		var skip: u64 = 0;
154		try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
155		}
156		for (flat_elems, 0..) \|elem, idx\| {
157		// TODO: recursiveIndex in a loop does a lot of redundant work!
158		// Better would be to gather all the store targets into an array.
159		var index: u64 = flat.flat_elem_index + idx;
160		const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
161		try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
162		}
163		return .success;
164		},
165		.reinterpret => {},
166		else => unreachable,
167		}
168		}
169	190
170		// Either there is a bit offset, or the strategy required reinterpreting.
171		// Therefore, we must perform a bitcast.
	191	.direct => \|direct\| {
	192	try checkComptimeVarStore(sema, block, src, direct.alloc);
	193	const want_ty = direct.val.typeOf(zcu);
	194	const coerced_store_val = try pt.getCoerced(store_val, want_ty);
	195	direct.val.* = .{ .interned = coerced_store_val.toIntern() };
	196	return .success;
	197	},
172	198
173		const val_ptr: *MutableValue, const byte_offset: u64 = switch (strat) {
174		.direct => \|direct\| .{ direct.val, 0 },
175		.index => \|index\| .{
176		index.val,
177		index.elem_index * index.val.typeOf(zcu).childType(zcu).abiSize(zcu),
	199	.index => \|index\| {
	200	try checkComptimeVarStore(sema, block, src, index.alloc);
	201	const want_ty = index.val.typeOf(zcu).childType(zcu);
	202	const coerced_store_val = try pt.getCoerced(store_val, want_ty);
	203	try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
	204	return .success;
178	205	},
179		.flat_index => \|flat\| .{ flat.val, flat.flat_elem_index * flat.val.typeOf(zcu).arrayBase(zcu)[0].abiSize(zcu) },
180		.reinterpret => \|reinterpret\| .{ reinterpret.val, reinterpret.byte_offset },
181		else => unreachable,
182		};
183	206
184		if (!val_ptr.typeOf(zcu).hasWellDefinedLayout(zcu)) {
185		return .{ .needed_well_defined = val_ptr.typeOf(zcu) };
186		}
	207	.flat_index => \|flat\| {
	208	try checkComptimeVarStore(sema, block, src, flat.alloc);
	209	const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
	210	const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
	211	{
	212	var next_idx: u64 = 0;
	213	var skip: u64 = 0;
	214	try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
	215	}
	216	for (flat_elems, 0..) \|elem, idx\| {
	217	// TODO: recursiveIndex in a loop does a lot of redundant work!
	218	// Better would be to gather all the store targets into an array.
	219	var index: u64 = flat.flat_elem_index + idx;
	220	const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
	221	try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
	222	}
	223	return .success;
	224	},
187	225
188		if (!store_val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
189		return .{ .needed_well_defined = store_val.typeOf(zcu) };
	226	.reinterpret => \|reinterpret\| {
	227	try checkComptimeVarStore(sema, block, src, reinterpret.alloc);
	228	if (!reinterpret.val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
	229	return .{ .needed_well_defined = reinterpret.val.typeOf(zcu) };
	230	}
	231	if (!store_ty.hasWellDefinedLayout(zcu)) {
	232	return .{ .needed_well_defined = store_ty };
	233	}
	234	const old_val = try reinterpret.val.intern(pt, sema.arena);
	235	const new_val = try sema.spliceMemory(
	236	old_val,
	237	store_val,
	238	reinterpret.byte_offset,
	239	) orelse return .runtime_store;
	240	reinterpret.val.* = .{ .interned = new_val.toIntern() };
	241	return .success;
	242	},
190	243	}
191
192		const new_val = try sema.bitCastSpliceVal(
193		try val_ptr.intern(pt, sema.arena),
194		store_val,
195		byte_offset,
196		host_bits,
197		bit_offset,
198		) orelse return .runtime_store;
199		val_ptr.* = .{ .interned = new_val.toIntern() };
200		return .success;
201	244	}
202	245
203	246	/// Perform a comptime load of type `load_ty` from a pointer.
...	...	@@ -207,8 +250,6 @@ fn loadComptimePtrInner(
207	250	block: *Block,
208	251	src: LazySrcLoc,
209	252	ptr_val: Value,
210		bit_offset: u64,
211		host_bits: u64,
212	253	load_ty: Type,
213	254	/// If `load_ty` is an array, this is the number of array elements to skip
214	255	/// before `load_ty`. Otherwise, it is ignored and may be `undefined`.
...	...	@@ -244,7 +285,7 @@ fn loadComptimePtrInner(
244	285	.eu_payload => \|base_ptr_ip\| val: {
245	286	const base_ptr = Value.fromInterned(base_ptr_ip);
246	287	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
247		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	288	switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
248	289	.success => \|eu_val\| switch (eu_val.unpackErrorUnion(zcu)) {
249	290	.undef => return .undef,
250	291	.err => \|err\| return .{ .err_payload = err },
...	...	@@ -256,7 +297,7 @@ fn loadComptimePtrInner(
256	297	.opt_payload => \|base_ptr_ip\| val: {
257	298	const base_ptr = Value.fromInterned(base_ptr_ip);
258	299	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
259		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	300	switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
260	301	.success => \|eu_val\| switch (eu_val.unpackOptional(zcu)) {
261	302	.undef => return .undef,
262	303	.null => return .null_payload,
...	...	@@ -283,7 +324,7 @@ fn loadComptimePtrInner(
283	324	.child = base_ty.toIntern(),
284	325	});
285	326
286		switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, want_ty, base_index.index)) {
	327	switch (try loadComptimePtrInner(sema, block, src, base_ptr, want_ty, base_index.index)) {
287	328	.success => \|arr_val\| break :val arr_val,
288	329	else => \|err\| return err,
289	330	}
...	...	@@ -293,7 +334,7 @@ fn loadComptimePtrInner(
293	334	const base_ty = base_ptr.typeOf(zcu).childType(zcu);
294	335
295	336	// Field of a slice, or of an auto-layout struct or union.
296		const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {
	337	const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
297	338	.success => \|val\| val,
298	339	else => \|err\| return err,
299	340	};
...	...	@@ -324,7 +365,7 @@ fn loadComptimePtrInner(
324	365	},
325	366	};
326	367
327		if (ptr.byte_offset == 0 and host_bits == 0) {
	368	if (ptr.byte_offset == 0) {
328	369	if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {
329	370	if (.ok == try sema.coerceInMemoryAllowed(
330	371	block,
...	...	@@ -343,8 +384,6 @@ fn loadComptimePtrInner(
343	384	}
344	385
345	386	restructure_array: {
346		if (host_bits != 0) break :restructure_array;
347
348	387	// We might also be changing the length of an array, or restructuring it.
349	388	// e.g. [1][2][3]T -> [3][2]T.
350	389	// This case is important because it's permitted for types with ill-defined layouts.
...	...	@@ -402,7 +441,7 @@ fn loadComptimePtrInner(
402	441	cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;
403	442	}
404	443
405		const need_bytes = if (host_bits > 0) (host_bits + 7) / 8 else load_ty.abiSize(zcu);
	444	const need_bytes = load_ty.abiSize(zcu);
406	445
407	446	if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {
408	447	return .{ .out_of_bounds = cur_val.typeOf(zcu) };
...	...	@@ -453,7 +492,7 @@ fn loadComptimePtrInner(
453	492	},
454	493	.@"struct" => switch (cur_ty.containerLayout(zcu)) {
455	494	.auto => unreachable, // ill-defined layout
456		.@"packed" => break, // let the bitcast logic handle this
	495	.@"packed" => break, // let the memory reinterpret logic handle this
457	496	.@"extern" => for (0..cur_ty.structFieldCount(zcu)) \|field_idx\| {
458	497	const start_off = cur_ty.structFieldOffset(field_idx, zcu);
459	498	const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...	...	@@ -466,9 +505,9 @@ fn loadComptimePtrInner(
466	505	},
467	506	.@"union" => switch (cur_ty.containerLayout(zcu)) {
468	507	.auto => unreachable, // ill-defined layout
469		.@"packed" => break, // let the bitcast logic handle this
	508	.@"packed" => break, // let the memory reinterpret logic handle this
470	509	.@"extern" => {
471		// TODO: we have to let bitcast logic handle this for now.
	510	// TODO: we have to let the memory reinterpret logic handle this for now.
472	511	// Otherwise, we might traverse into a union field which doesn't allow pointers.
473	512	// Figure out a solution!
474	513	if (true) break;
...	...	@@ -495,27 +534,13 @@ fn loadComptimePtrInner(
495	534
496	535	// Fast path: check again if we're now at the type we want to load.
497	536	// If so, just return the loaded value.
498		if (cur_offset == 0 and host_bits == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
	537	if (cur_offset == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
499	538	return .{ .success = cur_val };
500	539	}
501	540
502		var bitcast_src_val = try cur_val.intern(sema.pt, sema.arena);
503
504		if (host_bits != 0) {
505		const src_bit_size = bitcast_src_val.typeOf(zcu).bitSize(zcu);
506		if (src_bit_size > host_bits) {
507		const truncate_ty = try pt.intType(.unsigned, @intCast(host_bits));
508		bitcast_src_val = try pt.getCoerced(bitcast_src_val, truncate_ty);
509		}
510		}
511
512		const result_val = try sema.bitCastVal(
513		bitcast_src_val,
514		load_ty,
515		cur_offset,
516		host_bits,
517		bit_offset,
518		) orelse return .runtime_load;
	541	// Otherwise, use the memory reinterpretation logic to pull out the bytes we need.
	542	const reinterpret_val = try cur_val.intern(pt, sema.arena);
	543	const result_val = try sema.castMemory(reinterpret_val, load_ty, cur_offset) orelse return .runtime_load;
519	544	return .{ .success = .{ .interned = result_val.toIntern() } };
520	545	}
521	546
...	...	@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {
546	571	val: *MutableValue,
547	572	flat_elem_index: u64,
548	573	},
549		/// This value should be reinterpreted using bitcast logic to perform the
	574	/// This value should be reinterpreted using `Sema.spliceMemory` to perform
550	575	/// store. Only returned if `store_ty` and the type of `val` both have
551	576	/// well-defined layouts.
552	577	reinterpret: struct {
...	...	@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(
886	911	},
887	912	.@"struct" => switch (cur_ty.containerLayout(zcu)) {
888	913	.auto => unreachable, // ill-defined layout
889		.@"packed" => break, // let the bitcast logic handle this
	914	.@"packed" => break, // let the memory reinterp logic handle this
890	915	.@"extern" => for (0..cur_ty.structFieldCount(zcu)) \|field_idx\| {
891	916	const start_off = cur_ty.structFieldOffset(field_idx, zcu);
892	917	const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...	...	@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(
899	924	},
900	925	.@"union" => switch (cur_ty.containerLayout(zcu)) {
901	926	.auto => unreachable, // ill-defined layout
902		.@"packed" => break, // let the bitcast logic handle this
	927	.@"packed" => break, // let the memory reinterp logic handle this
903	928	.@"extern" => {
904		// TODO: we have to let bitcast logic handle this for now.
	929	// TODO: we have to let the memory reinterp logic handle this for now.
905	930	// Otherwise, we might traverse into a union field which doesn't allow pointers.
906	931	// Figure out a solution!
907	932	if (true) break;

src/Sema/reinterpret.zig created+576

...	...	@@ -0,0 +1,576 @@
	1	//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
	2	//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
	3	//! a sequence of values in packed memory, and then unflatten through a combination of special
	4	//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
	5	//!
	6	//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
	7	//! have their fields reversed when represented as packed memory on such targets.
	8
	9	/// If `host_bits` is `0`, attempts to convert the memory at offset
	10	/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
	11	/// ignoring `bit_offset`.
	12	///
	13	/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
	14	/// non-packed value consisting of `host_bits` bits. A value of type
	15	/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
	16	/// into this value.
	17	///
	18	/// Returns `null` if the operation must be performed at runtime.
	19	pub fn castMemory(
	20	sema: *Sema,
	21	val: Value,
	22	dest_ty: Type,
	23	byte_offset: u64,
	24	) CompileError!?Value {
	25	const pt = sema.pt;
	26	const zcu = pt.zcu;
	27
	28	const val_ty = val.typeOf(zcu);
	29
	30	if (dest_ty.toIntern() == val_ty.toIntern()) {
	31	assert(byte_offset == 0);
	32	return val;
	33	}
	34
	35	val_ty.assertHasLayout(zcu);
	36	dest_ty.assertHasLayout(zcu);
	37
	38	var unpack: UnpackValueBytes = .{
	39	.pt = pt,
	40	.arena = sema.arena,
	41	.skip_bytes = byte_offset,
	42	.remaining_bytes = dest_ty.abiSize(zcu),
	43	.unpacked = .init(sema.arena),
	44	};
	45	unpack.add(val) catch \|err\| switch (err) {
	46	error.ReinterpretDeclRef => return null,
	47	error.OutOfMemory => \|e\| return e,
	48	};
	49
	50	var pack: PackValueBytes = .{
	51	.pt = pt,
	52	.arena = sema.arena,
	53	.unpacked = unpack.unpacked.items,
	54	};
	55	return pack.get(dest_ty) catch \|err\| switch (err) {
	56	error.ReinterpretDeclRef => return null,
	57	error.OutOfMemory => \|e\| return e,
	58	};
	59	}
	60
	61	/// Splice the value `splice_val` into `val` at the given `byte_offset`, replacing overlapping bits
	62	/// and returning the modified value.
	63	pub fn spliceMemory(
	64	sema: *Sema,
	65	val: Value,
	66	splice_val: Value,
	67	byte_offset: u64,
	68	) CompileError!?Value {
	69	const pt = sema.pt;
	70	const zcu = pt.zcu;
	71	const val_ty = val.typeOf(zcu);
	72	const splice_val_ty = splice_val.typeOf(zcu);
	73
	74	val_ty.assertHasLayout(zcu);
	75	splice_val_ty.assertHasLayout(zcu);
	76
	77	var unpack: UnpackValueBytes = .{
	78	.pt = pt,
	79	.arena = sema.arena,
	80	.skip_bytes = 0,
	81	.remaining_bytes = byte_offset,
	82	.unpacked = .init(sema.arena),
	83	};
	84	unpack.add(val) catch \|err\| switch (err) {
	85	error.ReinterpretDeclRef => return null,
	86	error.OutOfMemory => \|e\| return e,
	87	};
	88
	89	const splice_len = splice_val_ty.abiSize(zcu);
	90
	91	unpack.remaining_bytes = splice_len;
	92	unpack.add(splice_val) catch \|err\| switch (err) {
	93	error.ReinterpretDeclRef => return null,
	94	error.OutOfMemory => \|e\| return e,
	95	};
	96
	97	unpack.skip_bytes = byte_offset + splice_len;
	98	unpack.remaining_bytes = val_ty.abiSize(zcu) * 8 - byte_offset - splice_len;
	99	unpack.add(val) catch \|err\| switch (err) {
	100	error.ReinterpretDeclRef => return null,
	101	error.OutOfMemory => \|e\| return e,
	102	};
	103
	104	var pack: PackValueBytes = .{
	105	.pt = pt,
	106	.arena = sema.arena,
	107	.unpacked = unpack.unpacked.items,
	108	};
	109	return pack.get(val_ty) catch \|err\| switch (err) {
	110	error.ReinterpretDeclRef => return null,
	111	error.OutOfMemory => \|e\| return e,
	112	};
	113	}
	114
	115	/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
	116	/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
	117	/// of values in packed memory - therefore, on big-endian targets, the first element of this
	118	/// list contains bits from the final byte of the value.
	119	const UnpackValueBytes = struct {
	120	pt: Zcu.PerThread,
	121	arena: Allocator,
	122	skip_bytes: u64,
	123	remaining_bytes: u64,
	124	unpacked: std.array_list.Managed(InternPool.Index),
	125
	126	fn add(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	127	const pt = unpack.pt;
	128	const zcu = pt.zcu;
	129	const ip = &zcu.intern_pool;
	130
	131	if (unpack.remaining_bytes == 0) {
	132	return;
	133	}
	134
	135	const ty = val.typeOf(zcu);
	136	const size = ty.abiSize(zcu);
	137
	138	if (unpack.skip_bytes >= size) {
	139	unpack.skip_bytes -= size;
	140	return;
	141	}
	142
	143	switch (ip.indexToKey(val.toIntern())) {
	144	.int_type,
	145	.ptr_type,
	146	.array_type,
	147	.vector_type,
	148	.opt_type,
	149	.anyframe_type,
	150	.error_union_type,
	151	.simple_type,
	152	.struct_type,
	153	.tuple_type,
	154	.union_type,
	155	.opaque_type,
	156	.spirv_type,
	157	.enum_type,
	158	.func_type,
	159	.error_set_type,
	160	.inferred_error_set_type,
	161	.@"extern",
	162	.func,
	163	.err,
	164	.error_union,
	165	.enum_literal,
	166	.slice,
	167	.memoized_call,
	168	=> unreachable, // ill-defined layout or not real values
	169
	170	.undef,
	171	.int,
	172	.enum_tag,
	173	.simple_value,
	174	.float,
	175	.ptr,
	176	.opt,
	177	=> try unpack.primitive(val),
	178
	179	.bitpack => \|bitpack\| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
	180
	181	.aggregate => switch (ty.zigTypeTag(zcu)) {
	182	.vector => unreachable, // ill-defined layout
	183	.array => {
	184	for (0..@intCast(ty.arrayLen(zcu))) \|elem_index\| {
	185	const elem_val = try val.elemValue(pt, @intCast(elem_index));
	186	try unpack.add(elem_val);
	187	}
	188	if (ty.sentinel(zcu)) \|s\| {
	189	try unpack.add(s);
	190	}
	191	},
	192	.@"struct" => switch (ty.containerLayout(zcu)) {
	193	.auto => unreachable, // ill-defined layout
	194	.@"packed" => unreachable, // uses `.bitpack`, not `.aggregate`
	195	.@"extern" => {
	196	var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
	197	var offset: u64 = 0;
	198	while (it.next()) \|field_index\| {
	199	const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
	200	const field_val = try val.fieldValue(pt, field_index);
	201	try unpack.padding(pad_bytes);
	202	try unpack.add(field_val);
	203	offset += pad_bytes + field_val.typeOf(zcu).abiSize(zcu);
	204	}
	205	try unpack.padding(size - offset);
	206	},
	207	},
	208	else => unreachable,
	209	},
	210
	211	.un => \|un\| {
	212	const payload_val = Value.fromInterned(un.val);
	213	const pad_bytes = size - payload_val.typeOf(zcu).abiSize(zcu);
	214	try unpack.add(payload_val);
	215	try unpack.padding(pad_bytes);
	216	},
	217	}
	218	}
	219
	220	fn padding(unpack: *UnpackValueBytes, num_bytes: u64) Allocator.Error!void {
	221	if (num_bytes == 0) return;
	222	const undef_u8 = try unpack.pt.undefValue(Type.u8);
	223	for (0..@intCast(num_bytes)) \|_\| {
	224	unpack.primitive(undef_u8) catch \|err\| switch (err) {
	225	error.OutOfMemory => \|e\| return e,
	226	error.ReinterpretDeclRef => unreachable,
	227	};
	228	}
	229	}
	230
	231	fn primitive(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	232	const pt = unpack.pt;
	233	const zcu = pt.zcu;
	234
	235	if (unpack.remaining_bytes == 0) {
	236	return;
	237	}
	238
	239	const ty = val.typeOf(pt.zcu);
	240	const size = ty.abiSize(zcu);
	241
	242	if (unpack.skip_bytes >= size) {
	243	unpack.skip_bytes -= size;
	244	return;
	245	}
	246
	247	if (unpack.skip_bytes > 0) {
	248	const offset = unpack.skip_bytes;
	249	unpack.skip_bytes = 0;
	250	return unpack.splitPrimitive(val, offset, @min(size - offset, unpack.remaining_bytes));
	251	}
	252
	253	if (unpack.remaining_bytes < size) {
	254	return unpack.splitPrimitive(val, 0, unpack.remaining_bytes);
	255	}
	256
	257	unpack.remaining_bytes -= size;
	258	try unpack.unpacked.append(val.toIntern());
	259	}
	260
	261	fn splitPrimitive(unpack: *UnpackValueBytes, val: Value, offset: u64, len: u64) (error{ReinterpretDeclRef} \|\| Allocator.Error)!void {
	262	const pt = unpack.pt;
	263	const zcu = pt.zcu;
	264	const ty = val.typeOf(pt.zcu);
	265
	266	assert(offset + len <= ty.abiSize(zcu));
	267
	268	try unpack.unpacked.ensureUnusedCapacity(@intCast(len));
	269	unpack.remaining_bytes -= len;
	270
	271	switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
	272	// In the `ptr` case, this will return `error.ReinterpretDeclRef`
	273	// if we're trying to split a non-integer pointer value.
	274	.int, .float, .enum_tag, .ptr, .opt => {
	275	const buf = try unpack.arena.alloc(u8, @intCast(ty.abiSize(zcu)));
	276	val.writeToMemory(zcu, buf) catch \|err\| switch (err) {
	277	error.IllDefinedMemoryLayout => unreachable,
	278	else => \|e\| return e,
	279	};
	280	for (buf[@intCast(offset)..][0..@intCast(len)]) \|byte_raw\| {
	281	const byte_val = try pt.intValue(.u8, byte_raw);
	282	unpack.unpacked.appendAssumeCapacity(byte_val.toIntern());
	283	}
	284	},
	285	.undef => {
	286	const undef_u8 = try pt.undefValue(.u8);
	287	for (0..@intCast(len)) \|_\| {
	288	unpack.unpacked.appendAssumeCapacity(undef_u8.toIntern());
	289	}
	290	},
	291	// The only values here with runtime bits are `true` and `false`.
	292	// These are both 1 byte, so will never need splitting.
	293	.simple_value => unreachable,
	294	else => unreachable, // zero-bit or not primitives
	295	}
	296	}
	297	};
	298
	299	/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBytes`),
	300	/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
	301	/// values and of pointers which align in virtual memory.
	302	const PackValueBytes = struct {
	303	pt: Zcu.PerThread,
	304	arena: Allocator,
	305	byte_offset: u64 = 0,
	306	unpacked: []const InternPool.Index,
	307
	308	fn get(pack: *PackValueBytes, ty: Type) (Allocator.Error \|\| error{ReinterpretDeclRef})!Value {
	309	const pt = pack.pt;
	310	const zcu = pt.zcu;
	311	const ip = &zcu.intern_pool;
	312	const arena = pack.arena;
	313	switch (ty.zigTypeTag(zcu)) {
	314	.vector => unreachable, // ill-defined layout
	315	.array => {
	316	// Each element is padded up to its ABI size. The final element does not have trailing padding.
	317	const elem_ty = ty.childType(zcu);
	318	const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
	319
	320	for (elems) \|*elem\| {
	321	elem.* = (try pack.get(elem_ty)).toIntern();
	322	}
	323
	324	if (ty.sentinel(zcu)) \|s\| {
	325	_ = s; // TODO: validate sentinel was preserved!
	326	pack.padding(elem_ty.abiSize(zcu));
	327	}
	328
	329	return pt.aggregateValue(ty, elems);
	330	},
	331	.@"struct" => switch (ty.containerLayout(zcu)) {
	332	.auto => unreachable, // ill-defined layout
	333	.@"extern" => {
	334	const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
	335	@memset(elems, .none);
	336	var offset: u64 = 0;
	337	var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
	338	while (it.next()) \|field_index\| {
	339	const field_ty = ty.fieldType(field_index, zcu);
	340	const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
	341	pack.padding(pad_bytes);
	342	elems[field_index] = (try pack.get(field_ty)).toIntern();
	343	offset += pad_bytes + field_ty.abiSize(zcu);
	344	}
	345	pack.padding(ty.abiSize(zcu) - offset);
	346	// Any fields which do not have runtime bits should be OPV or comptime fields.
	347	// Fill those values now.
	348	for (elems, 0..) \|*elem, field_index\| {
	349	if (elem.* != .none) continue;
	350	const val = (try ty.structFieldValueComptime(pt, field_index)).?;
	351	elem.* = val.toIntern();
	352	}
	353	return pt.aggregateValue(ty, elems);
	354	},
	355	.@"packed" => {
	356	const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
	357	if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
	358	return pt.bitpackValue(ty, backing_int_val);
	359	},
	360	},
	361	.@"union" => switch (ty.containerLayout(zcu)) {
	362	.auto => unreachable, // ill-defined layout
	363	.@"extern" => {
	364	// We will attempt to read as the backing representation. If this emits
	365	// `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
	366	// We will also attempt smaller fields when we get `undefined`, as if some bits are
	367	// defined we want to include them.
	368	// TODO: this is very very bad. We need a more sophisticated union representation.
	369
	370	const prev_unpacked = pack.unpacked;
	371	const prev_byte_offset = pack.byte_offset;
	372
	373	const backing_ty = try ty.externUnionBackingType(pt);
	374
	375	const backing_result: enum { undef, reinterpret_decl_ref } = backing: {
	376	const backing_val = pack.get(backing_ty) catch \|err\| switch (err) {
	377	error.ReinterpretDeclRef => break :backing .reinterpret_decl_ref,
	378	else => \|e\| return e,
	379	};
	380	if (backing_val.isUndef(zcu)) break :backing .undef;
	381	return .fromInterned(try pt.internUnion(.{
	382	.ty = ty.toIntern(),
	383	.tag = .none,
	384	.val = backing_val.toIntern(),
	385	}));
	386	};
	387
	388	const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
	389	for (field_order, 0..) \|f, i\| f. = @intCast(i);
	390	// Sort `field_order` to put the fields with the largest ABI sizes first.
	391	const SizeSortCtx = struct {
	392	zcu: *const Zcu,
	393	field_types: []const InternPool.Index,
	394	fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
	395	const a_ty: Type = .fromInterned(ctx.field_types[a_idx]);
	396	const b_ty: Type = .fromInterned(ctx.field_types[b_idx]);
	397	return a_ty.abiSize(ctx.zcu) > b_ty.abiSize(ctx.zcu);
	398	}
	399	};
	400	std.mem.sortUnstable(u32, field_order, SizeSortCtx{
	401	.zcu = zcu,
	402	.field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
	403	}, SizeSortCtx.lessThan);
	404
	405	for (field_order) \|field_index\| {
	406	pack.unpacked = prev_unpacked;
	407	pack.byte_offset = prev_byte_offset;
	408	const field_ty = ty.fieldType(field_index, zcu);
	409	const field_val = pack.get(field_ty) catch \|err\| switch (err) {
	410	error.ReinterpretDeclRef => continue,
	411	else => \|e\| return e,
	412	};
	413	if (field_val.isUndef(zcu)) continue;
	414	pack.padding(ty.abiSize(zcu) - field_ty.abiSize(zcu));
	415	const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_index);
	416	return pt.unionValue(ty, tag_val, field_val);
	417	}
	418
	419	// No field could represent the value. Just do whatever happens when we try to read
	420	// the backing type - either `undefined` or `error.ReinterpretDeclRef`.
	421	switch (backing_result) {
	422	.undef => return pt.undefValue(ty),
	423	.reinterpret_decl_ref => return error.ReinterpretDeclRef,
	424	}
	425	},
	426	.@"packed" => {
	427	const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
	428	if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
	429	return pt.bitpackValue(ty, backing_int_val);
	430	},
	431	},
	432	.@"enum" => {
	433	const tag_int_val = try pack.primitive(ty.intTagType(zcu));
	434	if (tag_int_val.isUndef(zcu)) return pt.undefValue(ty);
	435	return pt.enumValue(ty, tag_int_val.toIntern());
	436	},
	437	else => return pack.primitive(ty),
	438	}
	439	}
	440
	441	fn padding(pack: *PackValueBytes, num_bytes: u64) void {
	442	_ = pack.prepareBytes(num_bytes);
	443	}
	444
	445	fn primitive(pack: *PackValueBytes, want_ty: Type) (Allocator.Error \|\| error{ReinterpretDeclRef})!Value {
	446	const pt = pack.pt;
	447	const zcu = pt.zcu;
	448
	449	if (try want_ty.onePossibleValue(pt)) \|opv\| return opv;
	450
	451	const vals, const byte_offset = pack.prepareBytes(want_ty.abiSize(zcu));
	452
	453	for (vals) \|val\| {
	454	if (!Value.fromInterned(val).isUndef(zcu)) break;
	455	} else {
	456	// All bits of the value are `undefined`.
	457	return pt.undefValue(want_ty);
	458	}
	459
	460	// TODO: we need to decide how to handle partially-undef values here.
	461	// Currently, a value with some undefined bits becomes `0xAA` so that we
	462	// preserve the well-defined bits, because we can't currently represent
	463	// a partially-undefined primitive (e.g. an int with some undef bits).
	464	// In future, we probably want to take one of these two routes:
	465	// * Define that if any bits are `undefined`, the entire value is `undefined`.
	466	// This is a major breaking change, and probably a footgun.
	467	// * Introduce tracking for partially-undef values at comptime.
	468	// This would complicate a lot of operations in Sema, such as basic
	469	// arithmetic.
	470	// This design complexity is tracked by #19634.
	471
	472	if (vals.len == 1 and
	473	want_ty.isPtrAtRuntime(zcu) and
	474	Value.fromInterned(vals[0]).typeOf(zcu).isPtrAtRuntime(zcu))
	475	{
	476	return pt.getCoerced(.fromInterned(vals[0]), want_ty);
	477	}
	478
	479	// Reinterpret via an in-memory buffer.
	480
	481	var buf_len: u64 = 0;
	482	for (vals) \|ip_val\| {
	483	const val: Value = .fromInterned(ip_val);
	484	buf_len += val.typeOf(zcu).abiSize(zcu);
	485	}
	486
	487	const buf = try pack.arena.alloc(u8, @intCast(buf_len));
	488	{
	489	var offset: usize = 0;
	490	for (vals) \|ip_val\| {
	491	const val: Value = .fromInterned(ip_val);
	492	const ty = val.typeOf(zcu);
	493	const size = ty.abiSize(zcu);
	494	if (val.isUndef(zcu)) {
	495	@memset(buf[offset..][0..@intCast(size)], 0xAA);
	496	} else {
	497	val.writeToMemory(zcu, buf[offset..][0..@intCast(size)]) catch \|err\| switch (err) {
	498	error.IllDefinedMemoryLayout => unreachable,
	499	else => \|e\| return e,
	500	};
	501	}
	502	offset += @intCast(size);
	503	}
	504	}
	505	const bytes = buf[@intCast(byte_offset)..];
	506
	507	const target = zcu.getTarget();
	508	const endian = target.cpu.arch.endian();
	509	switch (want_ty.zigTypeTag(zcu)) {
	510	.bool => return .makeBool(bytes[0] != 0),
	511	.int => return .readIntFromMemory(want_ty, pt, bytes, pack.arena),
	512	.float => switch (want_ty.floatBits(target)) {
	513	16 => return pt.floatValue(want_ty, @as(f16, @bitCast(std.mem.readInt(u16, bytes[0..2], endian)))),
	514	32 => return pt.floatValue(want_ty, @as(f32, @bitCast(std.mem.readInt(u32, bytes[0..4], endian)))),
	515	64 => return pt.floatValue(want_ty, @as(f64, @bitCast(std.mem.readInt(u64, bytes[0..8], endian)))),
	516	80 => return pt.floatValue(want_ty, @as(f80, @bitCast(std.mem.readInt(u80, bytes[0..10], endian)))),
	517	128 => return pt.floatValue(want_ty, @as(f128, @bitCast(std.mem.readInt(u128, bytes[0..16], endian)))),
	518	else => unreachable,
	519	},
	520	.pointer => {
	521	assert(!want_ty.isSlice(zcu));
	522	const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
	523	return pt.ptrIntValue(want_ty, ptr_addr);
	524	},
	525	.optional => {
	526	assert(want_ty.isPtrLikeOptional(zcu));
	527	const ptr_ty = want_ty.optionalChild(zcu);
	528	const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
	529	return .fromInterned(try pt.intern(.{ .opt = .{
	530	.ty = want_ty.toIntern(),
	531	.val = if (ptr_addr == 0) .none else (try pt.ptrIntValue(ptr_ty, ptr_addr)).toIntern(),
	532	} }));
	533	},
	534	else => unreachable,
	535	}
	536	}
	537
	538	fn prepareBytes(pack: *PackValueBytes, need_bytes: u64) struct { []const InternPool.Index, u64 } {
	539	if (need_bytes == 0) return .{ &.{}, 0 };
	540
	541	const pt = pack.pt;
	542	const zcu = pt.zcu;
	543
	544	var bytes: u64 = 0;
	545	var len: usize = 0;
	546	while (bytes < pack.byte_offset + need_bytes) {
	547	bytes += Value.fromInterned(pack.unpacked[len]).typeOf(zcu).abiSize(zcu);
	548	len += 1;
	549	}
	550
	551	const result_vals = pack.unpacked[0..len];
	552	const result_offset = pack.byte_offset;
	553
	554	const extra_bytes = bytes - pack.byte_offset - need_bytes;
	555	if (extra_bytes == 0) {
	556	pack.unpacked = pack.unpacked[len..];
	557	pack.byte_offset = 0;
	558	} else {
	559	pack.unpacked = pack.unpacked[len - 1 ..];
	560	pack.byte_offset = Value.fromInterned(pack.unpacked[0]).typeOf(zcu).abiSize(zcu) - extra_bytes;
	561	}
	562
	563	return .{ result_vals, result_offset };
	564	}
	565	};
	566
	567	const std = @import("std");
	568	const Allocator = std.mem.Allocator;
	569	const assert = std.debug.assert;
	570
	571	const Sema = @import("../Sema.zig");
	572	const Zcu = @import("../Zcu.zig");
	573	const InternPool = @import("../InternPool.zig");
	574	const Type = @import("../Type.zig");
	575	const Value = @import("../Value.zig");
	576	const CompileError = Zcu.CompileError;

src/Type.zig+47-106

...	...	@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {
757	757	const ip = &zcu.intern_pool;
758	758	return switch (ip.indexToKey(ty.toIntern())) {
759	759	.int_type,
760		.vector_type,
761	760	=> true,
762	761
	762	.vector_type,
763	763	.error_union_type,
764	764	.error_set_type,
765	765	.inferred_error_set_type,
...	...	@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {
1241	1241	}
1242	1242
1243	1243	/// Asserts that `ty` is not an opaque or comptime-only type.
1244		/// Once #19755 is implemented, this query will only work on types with a defined bit-level representation.
1245	1244	pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {
1246		const target = zcu.getTarget();
1247		const ip = &zcu.intern_pool;
1248		assertHasLayout(ty, zcu);
1249		return switch (ip.indexToKey(ty.toIntern())) {
1250		.int_type => \|int_type\| int_type.bits,
1251		.ptr_type => \|ptr_type\| switch (ptr_type.flags.size) {
1252		.slice => target.ptrBitWidth() * 2,
1253		else => target.ptrBitWidth(),
1254		},
1255		.anyframe_type => target.ptrBitWidth(),
1256		.array_type => \|array_type\| {
1257		const elem_ty: Type = .fromInterned(array_type.child);
1258		const len = array_type.lenIncludingSentinel();
1259		return switch (zcu.comp.getZigBackend()) {
1260		.stage2_x86_64, .stage2_llvm => len * elem_ty.bitSize(zcu),
1261		// this case will be removed under #19755
1262		else => switch (len) {
1263		0 => 0,
1264		else => (len - 1) * 8 * elem_ty.abiSize(zcu) + elem_ty.bitSize(zcu),
1265		},
1266		};
1267		},
1268		.vector_type => \|vec\| vec.len * Type.fromInterned(vec.child).bitSize(zcu),
1269		.error_set_type, .inferred_error_set_type => zcu.errorSetBits(),
1270		.func_type => unreachable,
1271
1272		.simple_type => \|t\| switch (t) {
1273		.void => 0,
1274		.bool => 1,
1275		.anyerror, .adhoc_inferred_error_set => zcu.errorSetBits(),
1276		.usize, .isize => target.ptrBitWidth(),
1277
1278		.c_char => target.cTypeBitSize(.char),
1279		.c_short => target.cTypeBitSize(.short),
1280		.c_ushort => target.cTypeBitSize(.ushort),
1281		.c_int => target.cTypeBitSize(.int),
1282		.c_uint => target.cTypeBitSize(.uint),
1283		.c_long => target.cTypeBitSize(.long),
1284		.c_ulong => target.cTypeBitSize(.ulong),
1285		.c_longlong => target.cTypeBitSize(.longlong),
1286		.c_ulonglong => target.cTypeBitSize(.ulonglong),
1287		.c_longdouble => target.cTypeBitSize(.longdouble),
1288
1289		.f16 => 16,
1290		.f32 => 32,
1291		.f64 => 64,
1292		.f80 => 80,
1293		.f128 => 128,
1294
1295		.anyopaque => unreachable,
1296		.type => unreachable,
1297		.comptime_int => unreachable,
1298		.comptime_float => unreachable,
1299		.noreturn => unreachable,
1300		.null => unreachable,
1301		.undefined => unreachable,
1302		.enum_literal => unreachable,
1303		.generic_poison => unreachable,
1304		},
1305
1306		.struct_type => {
1307		const struct_obj = ip.loadStructType(ty.toIntern());
1308		switch (struct_obj.layout) {
1309		.@"packed" => return Type.fromInterned(struct_obj.packed_backing_int_type).bitSize(zcu),
1310		.auto, .@"extern" => return struct_obj.size * 8, // will be `unreachable` under #19755
1311		}
1312		},
1313		.union_type => {
1314		const union_obj = ip.loadUnionType(ty.toIntern());
1315		switch (union_obj.layout) {
1316		.@"packed" => return Type.fromInterned(union_obj.packed_backing_int_type).bitSize(zcu),
1317		.auto, .@"extern" => return union_obj.size * 8, // will be `unreachable` under #19755
1318		}
	1245	return switch (ty.zigTypeTag(zcu)) {
	1246	.void => 0,
	1247	.bool => 1,
	1248	.float => ty.floatBits(zcu.getTarget()),
	1249	.pointer, .optional => {
	1250	assert(ty.isPtrAtRuntime(zcu));
	1251	return zcu.getTarget().ptrBitWidth();
1319	1252	},
1320		.enum_type => Type.fromInterned(ip.loadEnumType(ty.toIntern()).int_tag_type).bitSize(zcu),
1321
1322		// will be `unreachable` under #19755
1323		.opt_type,
1324		.error_union_type,
1325		.tuple_type,
1326		=> ty.abiSize(zcu) * 8,
1327
1328		.opaque_type, .spirv_type => unreachable,
1329
1330		// values, not types
1331		.undef,
1332		.simple_value,
1333		.@"extern",
1334		.func,
1335		.int,
1336		.err,
1337		.error_union,
1338		.enum_literal,
1339		.enum_tag,
1340		.float,
1341		.ptr,
1342		.slice,
1343		.opt,
1344		.aggregate,
1345		.un,
1346		.bitpack,
1347		// memoization, not types
1348		.memoized_call,
1349		=> unreachable,
	1253	.array, .vector => ty.arrayLenIncludingSentinel(zcu) * ty.childType(zcu).bitSize(zcu),
	1254	else => ty.intInfo(zcu).bits,
1350	1255	};
1351	1256	}
1352	1257
...	...	@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {
1528	1433	/// * `[*]T`
1529	1434	/// * `[*c]T`
1530	1435	/// * `@SpirvType(.{ .runtime_array = T })`
	1436	/// * `*@SpirvType(.{ .runtime_array = T })`
1531	1437	pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {
1532	1438	const ip = &zcu.intern_pool;
1533	1439	return switch (ip.indexToKey(ty.toIntern())) {
...	...	@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3181	3087	.frame,
3182	3088	=> false,
3183	3089
	3090	.vector => position == .param_ty or position == .ret_ty,
	3091
3184	3092	.void => switch (position) {
3185	3093	.ret_ty,
3186	3094	.union_field,
...	...	@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3259	3167	.other,
3260	3168	=> ty.childType(zcu).validateExtern(.element, zcu),
3261	3169	},
3262		.vector => ty.childType(zcu).validateExtern(.element, zcu),
3263	3170	.optional => ty.isPtrLikeOptional(zcu),
3264	3171	};
3265	3172	}
...	...	@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {
3272	3179	};
3273	3180	}
3274	3181
	3182	/// Returns whether `ty` is considered by Zig to have a bit-level representation, meaning it is
	3183	/// allowed as the operand to `@bitSizeOf`. This is a superset of packable types.
	3184	pub fn hasBitRepresentation(ty: Type, zcu: *const Zcu) bool {
	3185	return switch (ty.zigTypeTag(zcu)) {
	3186	.@"fn",
	3187	.noreturn,
	3188	.undefined,
	3189	.null,
	3190	.@"opaque",
	3191	.spirv,
	3192	.type,
	3193	.enum_literal,
	3194	.comptime_float,
	3195	.comptime_int,
	3196	.error_set,
	3197	.error_union,
	3198	.frame,
	3199	.@"anyframe",
	3200	=> false,
	3201
	3202	.void,
	3203	.bool,
	3204	.int,
	3205	.float,
	3206	=> true,
	3207
	3208	.@"enum" => zcu.intern_pool.loadEnumType(ty.toIntern()).int_tag_mode == .explicit,
	3209	.pointer, .optional => ty.isPtrAtRuntime(zcu),
	3210	.@"struct", .@"union" => ty.containerLayout(zcu) == .@"packed",
	3211
	3212	.array, .vector => ty.childType(zcu).hasBitRepresentation(zcu),
	3213	};
	3214	}
	3215
3275	3216	/// Asserts that `ty` has resolved layout.
3276	3217	pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {
3277	3218	if (!std.debug.runtime_safety) {

src/Value.zig+135-134

...	...	@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {
248	248	pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
249	249	ReinterpretDeclRef,
250	250	IllDefinedMemoryLayout,
251		Unimplemented,
252	251	OutOfMemory,
253	252	}!void {
254	253	const target = zcu.getTarget();
...	...	@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
257	256	const ty = val.typeOf(zcu);
258	257	if (val.isUndef(zcu)) {
259	258	const size: usize = @intCast(ty.abiSize(zcu));
260		@memset(buffer[0..size], 0xaa);
	259	@memset(buffer[0..size], 0xAA);
261	260	return;
262	261	}
263		switch (ty.zigTypeTag(zcu)) {
	262	tag: switch (ty.zigTypeTag(zcu)) {
	263	.type => return error.IllDefinedMemoryLayout,
	264	.comptime_float => return error.IllDefinedMemoryLayout,
	265	.comptime_int => return error.IllDefinedMemoryLayout,
	266	.undefined => return error.IllDefinedMemoryLayout,
	267	.null => return error.IllDefinedMemoryLayout,
	268	.error_union => return error.IllDefinedMemoryLayout,
	269	.enum_literal => return error.IllDefinedMemoryLayout,
	270	.@"fn" => return error.IllDefinedMemoryLayout,
	271	.spirv => return error.IllDefinedMemoryLayout,
	272	.@"opaque" => unreachable,
	273	.frame => unreachable,
	274	.@"anyframe" => unreachable,
	275	.noreturn => unreachable,
264	276	.void => {},
265	277	.bool => {
266	278	buffer[0] = @intFromBool(val.toBool());
267	279	},
268		.int, .@"enum", .error_set, .pointer => \|tag\| {
269		const int_ty = if (tag == .pointer) int_ty: {
270		if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
271		if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
272		break :int_ty Type.usize;
273		} else ty;
274		const int_info = int_ty.intInfo(zcu);
275		const bits = int_info.bits;
276		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
277
	280	.pointer => {
	281	if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
	282	if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
	283	continue :tag .int;
	284	},
	285	.int, .@"enum", .error_set => {
278	286	var bigint_buffer: BigIntSpace = undefined;
279	287	const bigint = val.toBigInt(&bigint_buffer, zcu);
280		bigint.writeTwosComplement(buffer[0..byte_count], endian);
281		},
282		.float => switch (ty.floatBits(target)) {
283		16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
284		32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
285		64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
286		80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
287		128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
288		else => unreachable,
	288	bigint.writeTwosComplement(buffer[0..@intCast(ty.abiSize(zcu))], endian);
	289	},
	290	.float => {
	291	const float_bits = ty.floatBits(target);
	292	switch (float_bits) {
	293	16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
	294	32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
	295	64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
	296	80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
	297	128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
	298	else => unreachable,
	299	}
	300	const float_bytes = @divExact(float_bits, 8);
	301	const total_bytes: usize = @intCast(ty.abiSize(zcu));
	302	@memset(buffer[float_bytes..total_bytes], 0); // padding
289	303	},
290	304	.array => {
291	305	const aggregate = ip.indexToKey(val.toIntern()).aggregate;
...	...	@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
302	316	}
303	317	buf_off += elem_size;
304	318	}
	319	if (ty.sentinel(zcu)) \|sentinel_val\| {
	320	try sentinel_val.writeToMemory(zcu, buffer[buf_off..]);
	321	}
305	322	},
306		.vector => {
307		// We use byte_count instead of abi_size here, so that any padding bytes
308		// follow the data bytes, on both big- and little-endian systems.
309		const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
310		return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
311		},
	323	.vector => return error.IllDefinedMemoryLayout,
312	324	.@"struct" => {
313	325	const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
314	326	switch (struct_type.layout) {
315	327	.auto => return error.IllDefinedMemoryLayout,
316		.@"extern" => for (0..struct_type.field_types.len) \|field_index\| {
317		const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
318		const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
319		.bytes => \|bytes\| {
320		buffer[off] = bytes.at(field_index, ip);
321		continue;
322		},
323		.elems => \|elems\| elems[field_index],
324		.repeated_elem => \|elem\| elem,
325		});
326		try writeToMemory(field_val, zcu, buffer[off..]);
	328	.@"extern" => {
	329	var last_off: usize = 0;
	330	for (struct_type.field_types.get(ip), 0..) \|field_ty_ip, field_index\| {
	331	const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
	332	@memset(buffer[last_off..off], 0xAA);
	333	const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
	334	.bytes => \|bytes\| {
	335	buffer[off] = bytes.at(field_index, ip);
	336	continue;
	337	},
	338	.elems => \|elems\| elems[field_index],
	339	.repeated_elem => \|elem\| elem,
	340	});
	341	try writeToMemory(field_val, zcu, buffer[off..]);
	342	last_off = @intCast(off + Type.fromInterned(field_ty_ip).abiSize(zcu));
	343	}
	344	const struct_size: usize = @intCast(ty.abiSize(zcu));
	345	@memset(buffer[last_off..struct_size], 0xAA);
327	346	},
328	347	.@"packed" => {
329	348	const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
...	...	@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
335	354	.auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
336	355	.@"extern" => {
337	356	const payload_val = val.unionPayload(zcu);
	357	const payload_size: usize = @intCast(payload_val.typeOf(zcu).abiSize(zcu));
	358	const union_size: usize = @intCast(ty.abiSize(zcu));
	359	@memset(buffer[payload_size..union_size], 0xAA);
338	360	return writeToMemory(payload_val, zcu, buffer);
339	361	},
340	362	.@"packed" => {
...	...	@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
352	374	@memset(buffer[0..@intCast(byte_count)], 0); // null pointer
353	375	}
354	376	},
355		else => return error.Unimplemented,
356	377	}
357	378	}
358	379
...	...	@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
360	381	///
361	382	/// Both the start and the end of the provided buffer must be tight, since
362	383	/// big-endian packed memory layouts start at the end of the buffer.
	384	///
	385	/// Supports arrays and vectors, for which the value is written in logical bit
	386	/// order, i.e. with the first element at bit offset 0.
363	387	pub fn writeToPackedMemory(
364	388	val: Value,
365	389	zcu: *const Zcu,
366	390	buffer: []u8,
367	391	bit_offset: usize,
368		) error{ ReinterpretDeclRef, OutOfMemory }!void {
	392	) void {
369	393	const ip = &zcu.intern_pool;
370	394	const target = zcu.getTarget();
371	395	const endian = target.cpu.arch.endian();
...	...	@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(
392	416	},
393	417	.@"enum" => {
394	418	const int_val = val.intFromEnum(zcu);
395		return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
396		},
397		.pointer => {
398		assert(!ty.isSlice(zcu)); // No well defined layout.
399		if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
400		const addr = val.toUnsignedInt(zcu);
401		std.mem.writeVarPackedInt(buffer, bit_offset, zcu.getTarget().ptrBitWidth(), addr, endian);
	419	int_val.writeToPackedMemory(zcu, buffer, bit_offset);
402	420	},
403	421	.int => {
404	422	const bits = ty.intInfo(zcu).bits;
...	...	@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(
416	434	128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),
417	435	else => unreachable,
418	436	},
419		.vector => {
420		const elem_ty = ty.childType(zcu);
421		const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
422		const len: usize = @intCast(ty.arrayLen(zcu));
423
424		var bits: u16 = 0;
425		var elem_i: usize = 0;
426		const aggregate = ip.indexToKey(val.toIntern()).aggregate;
427		while (elem_i < len) : (elem_i += 1) {
428		// On big-endian systems, LLVM reverses the element order of vectors by default
429		const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
430		switch (aggregate.storage) {
431		.bytes => \|bytes\| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
432		.elems => \|elems\| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
433		.repeated_elem => \|elem\| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
434		}
435		bits += elem_bit_size;
436		}
437		},
438	437	.@"struct", .@"union" => {
439	438	assert(ty.containerLayout(zcu) == .@"packed");
440	439	const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
441		return int_val.writeToPackedMemory(zcu, buffer, bit_offset);
	440	int_val.writeToPackedMemory(zcu, buffer, bit_offset);
442	441	},
443		.optional => {
444		assert(ty.isPtrLikeOptional(zcu));
445		if (val.optionalValue(zcu)) \|ptr_val\| {
446		return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);
447		} else {
448		return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);
	442	.array, .vector => {
	443	const elem_bits: usize = @intCast(ty.childType(zcu).bitSize(zcu));
	444	const len: usize = @intCast(ty.arrayLen(zcu));
	445	var elem_bit_off: usize = bit_offset;
	446	switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
	447	.repeated_elem => \|elem_val_ip\| {
	448	const elem_val: Value = .fromInterned(elem_val_ip);
	449	for (0..len) \|_\| {
	450	elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
	451	elem_bit_off += elem_bits;
	452	}
	453	},
	454	.elems => \|elems\| for (elems[0..len]) \|elem_val_ip\| {
	455	const elem_val: Value = .fromInterned(elem_val_ip);
	456	elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
	457	elem_bit_off += elem_bits;
	458	},
	459	.bytes => \|bytes\| for (bytes.toSlice(len, ip)) \|raw_byte\| {
	460	std.mem.writeVarPackedInt(buffer, elem_bit_off, elem_bits, raw_byte, endian);
	461	elem_bit_off += elem_bits;
	462	},
	463	}
	464	if (ty.sentinel(zcu)) \|sentinel_val\| {
	465	sentinel_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
449	466	}
450	467	},
451		else => @panic("TODO implement writeToPackedMemory for more types"),
	468	else => unreachable,
452	469	}
453	470	}
454	471
455		/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.
	472	/// Load a Value from the contents of `buffer`, where `ty` is any integer type.
456	473	///
457	474	/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
458	475	/// the end of the value in memory.
459		pub fn readUintFromMemory(
	476	pub fn readIntFromMemory(
460	477	ty: Type,
461	478	pt: Zcu.PerThread,
462	479	buffer: []const u8,
...	...	@@ -465,23 +482,28 @@ pub fn readUintFromMemory(
465	482	const zcu = pt.zcu;
466	483	const endian = zcu.getTarget().cpu.arch.endian();
467	484
468		assert(ty.isUnsignedInt(zcu));
469		const bits = ty.intInfo(zcu).bits;
470		const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
	485	const int = ty.intInfo(zcu);
	486	const abi_size: usize = @intCast(ty.abiSize(zcu));
	487	const exact_buf = buffer[0..abi_size];
471	488
472		assert(buffer.len >= byte_count);
473
474		if (bits <= 64) {
475		const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);
476		const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));
477		return pt.intValue(ty, result);
	489	if (abi_size <= 8) {
	490	const shift: u6 = @intCast(64 - int.bits);
	491	switch (int.signedness) {
	492	.unsigned => {
	493	const x = std.mem.readVarInt(u64, exact_buf, endian);
	494	return pt.intValue(ty, (x << shift) >> shift);
	495	},
	496	.signed => {
	497	const x = std.mem.readVarInt(i64, exact_buf, endian);
	498	return pt.intValue(ty, (x << shift) >> shift);
	499	},
	500	}
478	501	} else {
479		const Limb = std.math.big.Limb;
480		const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);
481		const limbs_buffer = try arena.alloc(Limb, limb_count);
	502	const limb_count = std.math.big.int.calcTwosCompLimbCount(int.bits);
	503	const limbs_buffer = try arena.alloc(std.math.big.Limb, limb_count);
482	504
483	505	var bigint: BigIntMutable = .init(limbs_buffer, 0);
484		bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);
	506	bigint.readTwosComplement(exact_buf, int.bits, endian, int.signedness);
485	507	return pt.intValue_big(ty, bigint.toConst());
486	508	}
487	509	}
...	...	@@ -490,17 +512,17 @@ pub fn readUintFromMemory(
490	512	///
491	513	/// Both the start and the end of the provided buffer must be tight, since
492	514	/// big-endian packed memory layouts start at the end of the buffer.
	515	///
	516	/// Supports arrays and vectors, for which the value is read in logical bit
	517	/// order, i.e. with the first element at bit offset 0.
493	518	pub fn readFromPackedMemory(
494	519	ty: Type,
495	520	pt: Zcu.PerThread,
496	521	buffer: []const u8,
497	522	bit_offset: usize,
498		gpa: Allocator,
499		) error{
500		IllDefinedMemoryLayout,
501		OutOfMemory,
502		}!Value {
	523	) Allocator.Error!Value {
503	524	const zcu = pt.zcu;
	525	const gpa = zcu.comp.gpa;
504	526	const target = zcu.getTarget();
505	527	const endian = target.cpu.arch.endian();
506	528	switch (ty.zigTypeTag(zcu)) {
...	...	@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(
543	565	},
544	566	.@"enum" => {
545	567	const int_ty = ty.intTagType(zcu);
546		const int_val = try Value.readFromPackedMemory(int_ty, pt, buffer, bit_offset, gpa);
	568	const int_val: Value = try .readFromPackedMemory(int_ty, pt, buffer, bit_offset);
547	569	return pt.getCoerced(int_val, ty);
548	570	},
549	571	.float => return Value.fromInterned(try pt.intern(.{ .float = .{
...	...	@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(
557	579	else => unreachable,
558	580	},
559	581	} })),
560		.vector => {
561		const elem_ty = ty.childType(zcu);
562		const elems = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
563		defer gpa.free(elems);
564
565		var bits: u16 = 0;
566		const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
567		for (elems, 0..) \|_, i\| {
568		// On big-endian systems, LLVM reverses the element order of vectors by default
569		const tgt_elem_i = if (endian == .big) elems.len - i - 1 else i;
570		elems[tgt_elem_i] = (try readFromPackedMemory(elem_ty, pt, buffer, bit_offset + bits, gpa)).toIntern();
571		bits += elem_bit_size;
572		}
573		return pt.aggregateValue(ty, elems);
574		},
575	582	.@"struct", .@"union" => {
576	583	assert(ty.containerLayout(zcu) == .@"packed");
577		const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset, gpa);
	584	const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset);
578	585	return pt.bitpackValue(ty, int_val);
579	586	},
580		.pointer => {
581		assert(!ty.isSlice(zcu)); // No well defined layout.
582		const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
583		return pt.ptrIntValue(ty, addr);
584		},
585		.optional => {
586		assert(ty.isPtrLikeOptional(zcu));
587		const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);
588		return .fromInterned(try pt.intern(.{ .opt = .{
589		.ty = ty.toIntern(),
590		.val = if (addr == 0) .none else (try pt.ptrIntValue(ty.childType(zcu), addr)).toIntern(),
591		} }));
	587	.array, .vector => {
	588	const elem_ty = ty.childType(zcu);
	589	const elem_bits: usize = @intCast(elem_ty.bitSize(zcu));
	590	const elems_buf = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
	591	defer gpa.free(elems_buf);
	592	var elem_bit_off: usize = bit_offset;
	593	for (elems_buf) \|*elem\| {
	594	const elem_val = try readFromPackedMemory(elem_ty, pt, buffer, elem_bit_off);
	595	elem.* = elem_val.toIntern();
	596	elem_bit_off += elem_bits;
	597	}
	598	return pt.aggregateValue(ty, elems_buf);
592	599	},
593		else => @panic("TODO implement readFromPackedMemory for more types"),
	600	else => unreachable,
594	601	}
595	602	}
596	603
...	...	@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
887	894	const bfa = bfa_state.allocator();
888	895	const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
889	896	defer bfa.free(buf);
890		int_val.writeToPackedMemory(zcu, buf, 0) catch \|err\| switch (err) {
891		error.ReinterpretDeclRef => unreachable, // it's an integer
892		error.OutOfMemory => \|e\| return e,
893		};
894		return Value.readFromPackedMemory(field_ty, pt, buf, field_bit_offset, bfa) catch \|err\| switch (err) {
895		error.IllDefinedMemoryLayout => unreachable, // it's a bitpack
896		error.OutOfMemory => \|e\| return e,
897		};
	897	@memset(buf, 0);
	898	int_val.writeToPackedMemory(zcu, buf, 0);
	899	return .readFromPackedMemory(field_ty, pt, buf, field_bit_offset);
898	900	},
899	901	else => unreachable,
900	902	};
...	...	@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
1619	1621	// code late in compilation. So, this error handling is too aggressive and
1620	1622	// causes some false negatives, causing less-than-ideal code generation.
1621	1623	error.IllDefinedMemoryLayout => return null,
1622		error.Unimplemented => return null,
1623	1624	};
1624	1625	const first_byte = byte_buffer[0];
1625	1626	for (byte_buffer[1..]) \|byte\| {

src/Zcu/PerThread.zig+4

...	...	@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e
4544	4544	tracy_trace.addText(fqn.toSlice(ip));
4545	4545	tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});
4546	4546
	4547	Air.Verify.run(pt, func_index, air);
	4548
4547	4549	if (codegen.legalizeFeatures(pt, nav)) \|features\| {
4548	4550	try air.legalize(pt, features);
	4551	// Verify the AIR again post-legalization.
	4552	Air.Verify.run(pt, func_index, air);
4549	4553	}
4550	4554
4551	4555	var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))

src/codegen/aarch64/Select.zig+23-7

...	...	@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
292	292	.load,
293	293	.fptrunc,
294	294	.fpext,
295		.intcast,
296		.intcast_safe,
	295	.int_cast,
	296	.int_cast_safe,
297	297	.trunc,
298	298	.optional_payload,
299	299	.optional_payload_ptr,
...	...	@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
334	334	air_inst_index = air_body[air_body_index];
335	335	continue :air_tag air_tags[@intFromEnum(air_inst_index)];
336	336	},
337		.bitcast => {
	337	.bit_cast,
	338	.ptr_cast,
	339	.ptr_from_int,
	340	.int_from_ptr,
	341	.error_cast,
	342	.error_from_int,
	343	.int_from_error,
	344	.union_from_enum,
	345	=> {
338	346	const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
339	347	maybe_noop: {
340	348	if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
...	...	@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3190	3198	}
3191	3199	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
3192	3200	},
3193		.bitcast => \|air_tag\| {
	3201	.bit_cast,
	3202	.ptr_cast,
	3203	.ptr_from_int,
	3204	.int_from_ptr,
	3205	.error_cast,
	3206	.error_from_int,
	3207	.int_from_error,
	3208	.union_from_enum,
	3209	=> \|air_tag\| {
3194	3210	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
3195	3211	defer dst_vi.value.deref(isel);
3196	3212	const ty_op = air.data(air.inst_index).ty_op;
...	...	@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5221	5237	}
5222	5238	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
5223	5239	},
5224		.intcast => \|air_tag\| {
	5240	.int_cast => \|air_tag\| {
5225	5241	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
5226	5242	defer dst_vi.value.deref(isel);
5227	5243
...	...	@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5312	5328	}
5313	5329	if (air.next()) \|next_air_tag\| continue :air_tag next_air_tag;
5314	5330	},
5315		.intcast_safe => \|air_tag\| {
	5331	.int_cast_safe => \|air_tag\| {
5316	5332	if (isel.live_values.fetchRemove(air.inst_index)) \|dst_vi\| unused: {
5317	5333	defer dst_vi.value.deref(isel);
5318	5334
...	...	@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
11355	11371	if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11356	11372	constant.writeToMemory(zcu, buffer) catch \|err\| switch (err) {
11357	11373	error.OutOfMemory => \|e\| return e,
11358		error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,
	11374	error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
11359	11375	};
11360	11376	return true;
11361	11377	}

src/codegen/aarch64/abi.zig+2-2

...	...	@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
21	21	if (ty.containerLayout(zcu) == .@"packed") return .byval;
22	22	if (countFloats(ty, zcu)) \|float\| return .{ .float_array = float.count };
23	23
24		const bit_size = ty.bitSize(zcu);
	24	const bit_size = ty.abiSize(zcu) * 8;
25	25	if (bit_size > 128) return .memory;
26	26	if (bit_size > 64) return .double_integer;
27	27	return .integer;
...	...	@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
30	30	if (ty.containerLayout(zcu) == .@"packed") return .byval;
31	31	if (countFloats(ty, zcu)) \|float\| return .{ .float_array = float.count };
32	32
33		const bit_size = ty.bitSize(zcu);
	33	const bit_size = ty.abiSize(zcu) * 8;
34	34	if (bit_size > 128) return .memory;
35	35	if (bit_size > 64) return .double_integer;
36	36	return .integer;

src/codegen/arm/abi.zig+6-6

...	...	@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
30	30	const ip = &zcu.intern_pool;
31	31	switch (ty.zigTypeTag(zcu)) {
32	32	.@"struct" => {
33		const bit_size = ty.bitSize(zcu);
34	33	if (ty.containerLayout(zcu) == .@"packed") {
35		if (bit_size > 64) return .memory;
	34	if (ty.bitSize(zcu) > 64) return .memory;
36	35	return .byval;
37	36	}
	37	const bit_size = ty.abiSize(zcu) * 8;
38	38	if (bit_size > max_byval_size) return .memory;
39	39	const float_count = countFloats(ty, zcu, &maybe_float_bits);
40	40	if (float_count <= byval_float_count) return .byval;
...	...	@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
47	47	var i: u32 = 0;
48	48	while (i < fields) : (i += 1) {
49	49	const field_ty = ty.fieldType(i, zcu);
50		if (field_ty.bitSize(zcu) > 32) return Class.arrSize(bit_size, 64);
	50	if (field_ty.abiSize(zcu) > 4) return Class.arrSize(bit_size, 64);
51	51	}
52	52	return Class.arrSize(bit_size, 32);
53	53	},
54	54	.@"union" => {
55		const bit_size = ty.bitSize(zcu);
56	55	const union_obj = zcu.typeToUnion(ty).?;
57	56	if (union_obj.layout == .@"packed") {
58		if (bit_size > 64) return .memory;
	57	if (ty.bitSize(zcu) > 64) return .memory;
59	58	return .byval;
60	59	}
	60	const bit_size = ty.abiSize(zcu) * 8;
61	61	if (bit_size > max_byval_size) return .memory;
62	62	const float_count = countFloats(ty, zcu, &maybe_float_bits);
63	63	if (float_count <= byval_float_count) return .byval;
...	...	@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
67	67	}
68	68
69	69	for (union_obj.field_types.get(ip)) \|field_ty\| {
70		if (Type.fromInterned(field_ty).bitSize(zcu) > 32) {
	70	if (Type.fromInterned(field_ty).abiSize(zcu) > 4) {
71	71	return Class.arrSize(bit_size, 64);
72	72	}
73	73	}

src/codegen/c.zig+270-104

...	...	@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
27	27	return comptime switch (dev.env.supports(.legalize)) {
28	28	inline false, true => \|supports_legalize\| &.init(.{
29	29	// we don't currently ask zig1 to use safe optimization modes
30		.expand_intcast_safe = supports_legalize,
	30	.expand_int_cast_safe = supports_legalize,
31	31	.expand_int_from_float_safe = supports_legalize,
32	32	.expand_int_from_float_optimized_safe = supports_legalize,
33	33	.expand_add_safe = supports_legalize,
...	...	@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
38	38	.expand_packed_store = true,
39	39	.expand_packed_struct_field_val = true,
40	40	.expand_packed_aggregate_init = true,
	41
	42	.scalarize_bit_cast_array = true,
	43	.scalarize_bit_cast_vector_non_elementwise = true,
41	44	}),
42	45	};
43	46	}
...	...	@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2636	2639	// zig fmt: off
2637	2640	.inferred_alloc, .inferred_alloc_comptime => unreachable,
2638	2641
2639		// No "scalarize" legalizations are enabled, so these instructions never appear.
2640		.legalize_vec_elem_val => unreachable,
2641		.legalize_vec_store_elem => unreachable,
	2642	// Possible because `Air.Legalize.scalarize_bit_cast_vector_non_elementwise` is enabled.
	2643	.legalize_vec_elem_val => try airArrayElemVal(f, inst),
	2644	.legalize_vec_store_elem => try airLegalizeVecStoreElem(f, inst),
2642	2645	// No soft float legalizations are enabled.
2643	2646	.legalize_compiler_rt_call => unreachable,
2644	2647
...	...	@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2751	2754	.alloc => try airAlloc(f, inst),
2752	2755	.ret_ptr => try airRetPtr(f, inst),
2753	2756	.assembly => try airAsm(f, inst),
2754		.bitcast => try airBitcast(f, inst),
2755		.intcast => try airIntCast(f, inst),
	2757	.ptr_cast => try airPtrCast(f, inst),
	2758	.ptr_from_int => try airSimpleCast(f, inst),
	2759	.int_from_ptr => try airSimpleCast(f, inst),
	2760	.error_cast => try airNopCast(f, inst),
	2761	.error_from_int => try airNopCast(f, inst),
	2762	.int_from_error => try airNopCast(f, inst),
	2763	.union_from_enum => try airUnionFromEnum(f, inst),
	2764	.bit_cast => try airBitCast(f, inst),
	2765	.int_cast => try airIntCast(f, inst),
2756	2766	.trunc => try airTrunc(f, inst),
2757	2767	.load => try airLoad(f, inst),
2758	2768	.store => try airStore(f, inst, false),
...	...	@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2864	2874	.add_safe,
2865	2875	.sub_safe,
2866	2876	.mul_safe,
2867		.intcast_safe,
	2877	.int_cast_safe,
2868	2878	.int_from_float_safe,
2869	2879	.int_from_float_optimized_safe,
2870	2880	=> return f.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
3083	3093	return local;
3084	3094	}
3085	3095
	3096	fn airLegalizeVecStoreElem(f: *Function, inst: Air.Inst.Index) !CValue {
	3097	const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
	3098	const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
	3099
	3100	const vec_ptr = try f.resolveInst(pl_op.operand);
	3101	const index = try f.resolveInst(extra.lhs);
	3102	const elem = try f.resolveInst(extra.rhs);
	3103	try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs });
	3104
	3105	const w = &f.code.writer;
	3106
	3107	try f.writeCValueDerefMember(w, vec_ptr, .{ .identifier = "array" });
	3108	try w.writeByte('[');
	3109	try f.writeCValue(w, index, .other);
	3110	try w.writeAll("] = ");
	3111	try f.writeCValue(w, elem, .other);
	3112	try w.writeByte(';');
	3113	try f.newline();
	3114
	3115	return .none;
	3116	}
	3117
3086	3118	fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3087	3119	const pt = f.dg.pt;
3088	3120	const zcu = pt.zcu;
...	...	@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
3190	3222
3191	3223	try reap(f, inst, &.{ty_op.operand});
3192	3224
3193		const is_aligned = if (ptr_info.flags.alignment != .none)
3194		ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte)
3195		else
3196		true;
	3225	const is_aligned = switch (ptr_info.flags.alignment) {
	3226	.none => true,
	3227	else => \|ptr_align\| ptr_align.compare(.gte, src_ty.abiAlignment(zcu)),
	3228	};
3197	3229
3198	3230	const w = &f.code.writer;
3199	3231	const local = try f.allocLocal(inst, src_ty);
3200		const v = try Vectorize.start(f, inst, w, ptr_ty);
3201	3232
3202	3233	if (!is_aligned) {
3203	3234	try w.writeAll("memcpy(&");
3204	3235	try f.writeCValue(w, local, .other);
3205		try v.elem(f, w);
3206	3236	try w.writeAll(", (const char *)");
3207		try f.writeCValue(w, operand, .other);
3208		try v.elem(f, w);
	3237	switch (ptr_info.flags.vector_index) {
	3238	.none => try f.writeCValue(w, operand, .other),
	3239	else => \|index\| {
	3240	try w.writeByte('&');
	3241	try f.writeCValue(w, operand, .other);
	3242	try w.print("[{d}]", .{@intFromEnum(index)});
	3243	},
	3244	}
3209	3245	try w.writeAll(", sizeof(");
3210	3246	try f.renderType(w, src_ty);
3211	3247	try w.writeAll("))");
3212	3248	} else {
3213	3249	try f.writeCValue(w, local, .other);
3214		try v.elem(f, w);
3215	3250	try w.writeAll(" = ");
3216		try f.writeCValueDeref(w, operand);
3217		try v.elem(f, w);
	3251	switch (ptr_info.flags.vector_index) {
	3252	.none => try f.writeCValueDeref(w, operand),
	3253	else => \|index\| {
	3254	try f.writeCValue(w, operand, .other);
	3255	try w.print("[{d}]", .{@intFromEnum(index)});
	3256	},
	3257	}
3218	3258	}
3219	3259	try w.writeByte(';');
3220	3260	try f.newline();
3221		try v.end(f, inst, w);
3222	3261
3223	3262	return local;
3224	3263	}
...	...	@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3433	3472	// underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
3434	3473	assert(src_ty.eql(.fromInterned(ptr_info.child)));
3435	3474
3436		const v = try Vectorize.start(f, inst, w, ptr_ty);
3437	3475	try w.writeAll("memcpy((char *)");
3438		try f.writeCValue(w, ptr_val, .other);
3439		try v.elem(f, w);
	3476	switch (ptr_info.flags.vector_index) {
	3477	.none => try f.writeCValue(w, ptr_val, .other),
	3478	else => \|index\| {
	3479	try w.writeByte('&');
	3480	try f.writeCValue(w, ptr_val, .other);
	3481	try w.print("[{d}]", .{@intFromEnum(index)});
	3482	},
	3483	}
3440	3484	try w.writeAll(", &");
3441	3485	switch (src_val) {
3442	3486	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
3443	3487	else => try f.writeCValue(w, src_val, .other),
3444	3488	}
3445		try v.elem(f, w);
3446	3489	try w.writeAll(", sizeof(");
3447	3490	try f.renderType(w, src_ty);
3448	3491	try w.writeAll("));");
3449	3492	try f.newline();
3450		try v.end(f, inst, w);
3451	3493	} else {
3452	3494	switch (ptr_val) {
3453	3495	.local_ref => \|ptr_local_index\| switch (src_val) {
...	...	@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3457	3499	},
3458	3500	else => {},
3459	3501	}
3460		const v = try Vectorize.start(f, inst, w, ptr_ty);
3461		try f.writeCValueDeref(w, ptr_val);
3462		try v.elem(f, w);
	3502
	3503	switch (ptr_info.flags.vector_index) {
	3504	.none => try f.writeCValueDeref(w, ptr_val),
	3505	else => \|index\| {
	3506	try f.writeCValue(w, ptr_val, .other);
	3507	try w.print("[{d}]", .{@intFromEnum(index)});
	3508	},
	3509	}
3463	3510	try w.writeAll(" = ");
3464	3511	try f.writeCValue(w, src_val, .other);
3465		try v.elem(f, w);
3466	3512	try w.writeByte(';');
3467	3513	try f.newline();
3468		try v.end(f, inst, w);
3469	3514	}
3470	3515	return .none;
3471	3516	}
...	...	@@ -3613,9 +3658,9 @@ fn airCmpOp(
3613	3658	const lhs_ty = f.typeOf(data.lhs);
3614	3659	const scalar_ty = lhs_ty.scalarType(zcu);
3615	3660
3616		const scalar_bits = scalar_ty.bitSize(zcu);
3617		if (scalar_ty.isInt(zcu) and scalar_bits > 64)
3618		return airCmpBuiltinCall(
	3661	if (scalar_ty.isInt(zcu)) {
	3662	const scalar_bits = scalar_ty.bitSize(zcu);
	3663	if (scalar_bits > 64) return airCmpBuiltinCall(
3619	3664	f,
3620	3665	inst,
3621	3666	data,
...	...	@@ -3623,6 +3668,7 @@ fn airCmpOp(
3623	3668	.cmp,
3624	3669	if (scalar_bits > 128) .bits else .none,
3625	3670	);
	3671	}
3626	3672	if (scalar_ty.isRuntimeFloat())
3627	3673	return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);
3628	3674
...	...	@@ -3668,9 +3714,9 @@ fn airEquality(
3668	3714	const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
3669	3715
3670	3716	const operand_ty = f.typeOf(bin_op.lhs);
3671		const operand_bits = operand_ty.bitSize(zcu);
3672		if (operand_ty.isAbiInt(zcu) and operand_bits > 64)
3673		return airCmpBuiltinCall(
	3717	if (operand_ty.isAbiInt(zcu)) {
	3718	const operand_bits = operand_ty.bitSize(zcu);
	3719	if (operand_bits > 64) return airCmpBuiltinCall(
3674	3720	f,
3675	3721	inst,
3676	3722	bin_op,
...	...	@@ -3678,6 +3724,7 @@ fn airEquality(
3678	3724	.cmp,
3679	3725	if (operand_bits > 128) .bits else .none,
3680	3726	);
	3727	}
3681	3728	if (operand_ty.isRuntimeFloat())
3682	3729	return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);
3683	3730
...	...	@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
4258	4305	try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});
4259	4306	}
4260	4307
4261		fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {
	4308	fn airPtrCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4309	const zcu = f.dg.pt.zcu;
	4310
	4311	const dest_ty = f.typeOfIndex(inst);
	4312	const ptr_ty = switch (dest_ty.zigTypeTag(zcu)) {
	4313	.optional => dest_ty.childType(zcu),
	4314	.pointer => dest_ty,
	4315	else => unreachable,
	4316	};
	4317
	4318	if (!ptr_ty.isSlice(zcu)) {
	4319	return airSimpleCast(f, inst);
	4320	}
	4321
	4322	// For slice casts we need to assign both fields.
	4323
4262	4324	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4263		const inst_ty = f.typeOfIndex(inst);
	4325	const operand = try f.resolveInst(ty_op.operand);
	4326
	4327	const w = &f.code.writer;
	4328	const dest_local = try f.allocLocal(inst, dest_ty);
	4329
	4330	try f.writeCValueMember(w, dest_local, .{ .identifier = "ptr" });
	4331	try w.writeAll(" = (");
	4332	try f.renderType(w, ptr_ty.slicePtrFieldType(zcu));
	4333	try w.writeByte(')');
	4334	try f.writeCValueMember(w, operand, .{ .identifier = "ptr" });
	4335	try w.writeByte(';');
	4336	try f.newline();
	4337
	4338	try f.writeCValueMember(w, dest_local, .{ .identifier = "len" });
	4339	try w.writeAll(" = ");
	4340	try f.writeCValueMember(w, operand, .{ .identifier = "len" });
	4341	try w.writeByte(';');
	4342	try f.newline();
	4343
	4344	try reap(f, inst, &.{ty_op.operand});
	4345	return dest_local;
	4346	}
	4347
	4348	fn airSimpleCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4349	const zcu = f.dg.pt.zcu;
	4350
	4351	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4352	const dest_ty = f.typeOfIndex(inst);
	4353	const operand_ty = f.typeOf(ty_op.operand);
	4354	const operand = try f.resolveInst(ty_op.operand);
	4355
	4356	const w = &f.code.writer;
	4357	const dest_local = try f.allocLocal(inst, dest_ty);
	4358	const v: Vectorize = try .start(f, inst, w, operand_ty);
	4359	try f.writeCValue(w, dest_local, .other);
	4360	try v.elem(f, w);
	4361	try w.writeAll(" = (");
	4362	try f.renderType(w, dest_ty.scalarType(zcu));
	4363	try w.writeByte(')');
	4364	try f.writeCValue(w, operand, .other);
	4365	try v.elem(f, w);
	4366	try w.writeByte(';');
	4367	try f.newline();
	4368	try v.end(f, inst, w);
	4369
	4370	try reap(f, inst, &.{ty_op.operand});
	4371	return dest_local;
	4372	}
4264	4373
	4374	fn airNopCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4375	const zcu = f.dg.pt.zcu;
	4376
	4377	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4378	const dest_ty = f.typeOfIndex(inst);
	4379	const operand_ty = f.typeOf(ty_op.operand);
4265	4380	const operand = try f.resolveInst(ty_op.operand);
	4381
	4382	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	4383	assert(operand_ty.isAbiInt(zcu) == dest_ty.isAbiInt(zcu));
	4384
	4385	try reap(f, inst, &.{ty_op.operand});
	4386	return f.moveCValue(inst, dest_ty, operand);
	4387	}
	4388
	4389	fn airUnionFromEnum(f: *Function, inst: Air.Inst.Index) Error!CValue {
	4390	const zcu = f.dg.pt.zcu;
	4391
	4392	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4393	const dest_ty = f.typeOfIndex(inst);
4266	4394	const operand_ty = f.typeOf(ty_op.operand);
	4395	const operand = try f.resolveInst(ty_op.operand);
	4396
	4397	assert(dest_ty.zigTypeTag(zcu) == .@"union");
	4398	assert(operand_ty.zigTypeTag(zcu) == .@"enum");
	4399
	4400	const w = &f.code.writer;
	4401	const dest_local = try f.allocLocal(inst, dest_ty);
	4402	try f.writeCValueMember(w, dest_local, .{ .identifier = "tag" });
	4403	try w.writeAll(" = ");
	4404	try f.writeCValue(w, operand, .other);
	4405	try w.writeByte(';');
	4406	try f.newline();
4267	4407
4268		const bitcasted = try bitcast(f, inst_ty, operand, operand_ty);
4269	4408	try reap(f, inst, &.{ty_op.operand});
4270		return f.moveCValue(inst, inst_ty, bitcasted);
	4409	return dest_local;
4271	4410	}
4272	4411
4273		fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CValue {
	4412	fn airBitCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4274	4413	const pt = f.dg.pt;
4275	4414	const zcu = pt.zcu;
4276		const target = &f.dg.mod.resolved_target.result;
4277	4415	const w = &f.code.writer;
4278	4416
4279		if (operand_ty.isAbiInt(zcu) and dest_ty.isAbiInt(zcu)) {
4280		const src_info = dest_ty.intInfo(zcu);
4281		const dest_info = operand_ty.intInfo(zcu);
4282		if (src_info.signedness == dest_info.signedness and
4283		src_info.bits == dest_info.bits) return operand;
4284		}
	4417	const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4418	const dest_ty = f.typeOfIndex(inst);
4285	4419
4286		if (dest_ty.isPtrAtRuntime(zcu) or operand_ty.isPtrAtRuntime(zcu)) {
4287		const local = try f.allocLocal(null, dest_ty);
4288		try f.writeCValue(w, local, .other);
	4420	const operand = try f.resolveInst(ty_op.operand);
	4421	const operand_ty = f.typeOf(ty_op.operand);
	4422
	4423	const dest_local = try f.allocLocal(inst, dest_ty);
	4424
	4425	// Because we have `scalarize_bit_cast_array` and `scalarize_bit_cast_vector_non_elementwise`
	4426	// enabled, we usually only see scalars here. The only case in which we may see vectors is when
	4427	// the operation happens elementwise, which we can handle with `Vectorize`.
	4428	var v: Vectorize = try .start(f, inst, w, operand_ty);
	4429	const operand_scalar_ty = operand_ty.scalarType(zcu);
	4430	const dest_scalar_ty = dest_ty.scalarType(zcu);
	4431
	4432	// Some cases are handled with a simple cast:
	4433	// * float -> float
	4434	// * bool -> int
	4435	if ((operand_scalar_ty.isRuntimeFloat() and dest_scalar_ty.isRuntimeFloat()) or
	4436	(operand_scalar_ty.toIntern() == .bool_type and dest_scalar_ty.isAbiInt(zcu)))
	4437	{
	4438	try f.writeCValue(w, dest_local, .other);
	4439	try v.elem(f, w);
4289	4440	try w.writeAll(" = (");
4290		try f.renderType(w, dest_ty);
	4441	try f.renderType(w, dest_scalar_ty);
4291	4442	try w.writeByte(')');
4292	4443	try f.writeCValue(w, operand, .other);
	4444	try v.elem(f, w);
4293	4445	try w.writeByte(';');
4294	4446	try f.newline();
4295		return local;
4296		}
4297
4298		const local = try f.allocLocal(null, dest_ty);
4299		// On big-endian targets, copying ABI integers with padding bits is awkward, because the padding bits are at the low bytes of the value.
4300		// We need to offset the source or destination pointer appropriately and copy the right number of bytes.
4301		if (target.cpu.arch.endian() == .big and dest_ty.isAbiInt(zcu) and !operand_ty.isAbiInt(zcu)) {
4302		// e.g. [10]u8 -> u80. We need to offset the destination so that we copy to the least significant bits of the integer.
4303		const offset = dest_ty.abiSize(zcu) - operand_ty.abiSize(zcu);
4304		try w.writeAll("memcpy((char *)&");
4305		try f.writeCValue(w, local, .other);
4306		try w.print(" + {d}, &", .{offset});
4307		switch (operand) {
4308		.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4309		else => try f.writeCValue(w, operand, .other),
4310		}
4311		try w.print(", {d});", .{operand_ty.abiSize(zcu)});
4312		} else if (target.cpu.arch.endian() == .big and operand_ty.isAbiInt(zcu) and !dest_ty.isAbiInt(zcu)) {
4313		// e.g. u80 -> [10]u8. We need to offset the source so that we copy from the least significant bits of the integer.
4314		const offset = operand_ty.abiSize(zcu) - dest_ty.abiSize(zcu);
	4447	} else if (dest_scalar_ty.toIntern() == .bool_type) {
	4448	// If the result is a boolean type, just check if the operand is non-zero.
	4449	assert(operand_scalar_ty.isAbiInt(zcu));
	4450	try f.writeCValue(w, dest_local, .other);
	4451	try v.elem(f, w);
	4452	try w.writeAll(" = ");
	4453	try f.writeCValue(w, operand, .other);
	4454	try v.elem(f, w);
	4455	try w.writeAll(" != 0;");
	4456	try f.newline();
	4457	} else if (dest_scalar_ty.isRuntimeFloat()) {
	4458	// For int->float, just do a memcpy.
	4459	assert(operand_scalar_ty.isAbiInt(zcu));
4315	4460	try w.writeAll("memcpy(&");
4316		try f.writeCValue(w, local, .other);
4317		try w.writeAll(", (const char *)&");
	4461	try f.writeCValue(w, dest_local, .other);
	4462	try v.elem(f, w);
	4463	try w.writeAll(", &");
4318	4464	switch (operand) {
4319	4465	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4320	4466	else => try f.writeCValue(w, operand, .other),
4321	4467	}
4322		try w.print(" + {d}, {d});", .{ offset, dest_ty.abiSize(zcu) });
	4468	try v.elem(f, w);
	4469	try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
	4470	try f.newline();
4323	4471	} else {
	4472	// The only remaining possibility is that the result is an integer. We will need to use
	4473	// `zig_wrap_*` to correct the "padding" bits after we populate the value bits.
	4474	assert(dest_scalar_ty.isAbiInt(zcu));
	4475	assert(operand_scalar_ty.isRuntimeFloat() or operand_scalar_ty.isAbiInt(zcu));
	4476
	4477	// memcpy the value...
4324	4478	try w.writeAll("memcpy(&");
4325		try f.writeCValue(w, local, .other);
	4479	try f.writeCValue(w, dest_local, .other);
	4480	try v.elem(f, w);
4326	4481	try w.writeAll(", &");
4327	4482	switch (operand) {
4328	4483	.constant => \|val\| try f.dg.renderValueAsLvalue(w, val),
4329	4484	else => try f.writeCValue(w, operand, .other),
4330	4485	}
4331		try w.print(", {d});", .{@min(dest_ty.abiSize(zcu), operand_ty.abiSize(zcu))});
4332		}
4333
4334		try f.newline();
	4486	try v.elem(f, w);
	4487	try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
	4488	try f.newline();
4335	4489
4336		// Ensure padding bits have the expected value.
4337		if (dest_ty.isAbiInt(zcu)) {
4338		switch (CType.classifyInt(dest_ty, zcu)) {
	4490	// ...and ensure padding bits have the correct value.
	4491	switch (CType.classifyInt(dest_scalar_ty, zcu)) {
4339	4492	.void => unreachable, // opv
4340	4493	.small => {
4341		try f.writeCValue(w, local, .other);
	4494	try f.writeCValue(w, dest_local, .other);
	4495	try v.elem(f, w);
4342	4496	try w.writeAll(" = zig_wrap_");
4343		try f.dg.renderTypeForBuiltinFnName(w, dest_ty);
	4497	try f.dg.renderTypeForBuiltinFnName(w, dest_scalar_ty);
4344	4498	try w.writeByte('(');
4345		try f.writeCValue(w, local, .other);
4346		try f.dg.renderBuiltinInfo(w, dest_ty, .bits);
	4499	try f.writeCValue(w, dest_local, .other);
	4500	try v.elem(f, w);
	4501	try f.dg.renderBuiltinInfo(w, dest_scalar_ty, .bits);
4347	4502	try w.writeAll(");");
4348	4503	try f.newline();
4349	4504	},
4350	4505	.big => \|big\| {
4351		const dest_info = dest_ty.intInfo(zcu);
4352		const padding_index: u16 = switch (target.cpu.arch.endian()) {
	4506	const dest_info = dest_scalar_ty.intInfo(zcu);
	4507	const padding_index: u16 = switch (f.dg.mod.resolved_target.result.cpu.arch.endian()) {
4353	4508	.little => big.limbs_len - 1,
4354	4509	.big => 0,
4355	4510	};
4356	4511	const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;
4357	4512	if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {
4358		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4359		try w.print("[{d}] = zig_wrap_{c}{d}(", .{
	4513	try f.writeCValue(w, dest_local, .other);
	4514	try v.elem(f, w);
	4515	try w.print(".limbs[{d}] = zig_wrap_{c}{d}(", .{
4360	4516	padding_index,
4361	4517	signAbbrev(dest_info.signedness),
4362	4518	big.limb_size.bits(),
4363	4519	});
4364		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4365		try w.print("[{d}], {d});", .{ padding_index, wrap_bits });
	4520	try f.writeCValue(w, dest_local, .other);
	4521	try v.elem(f, w);
	4522	try w.print(".limbs[{d}], {d});", .{ padding_index, wrap_bits });
4366	4523	} else {
4367		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4368		try w.print("[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
	4524	try f.writeCValue(w, dest_local, .other);
	4525	try v.elem(f, w);
	4526	try w.print(".limbs[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
4369	4527	padding_index,
4370	4528	});
4371		try f.writeCValueMember(w, local, .{ .identifier = "limbs" });
4372		try w.print("[{d}]), {d}));", .{ padding_index, wrap_bits });
	4529	try f.writeCValue(w, dest_local, .other);
	4530	try v.elem(f, w);
	4531	try w.print(".limbs[{d}]), {d}));", .{ padding_index, wrap_bits });
4373	4532	try f.newline();
4374	4533	}
4375	4534	},
4376	4535	}
4377	4536	}
4378	4537
4379		return local;
	4538	try v.end(f, inst, w);
	4539
	4540	try reap(f, inst, &.{ty_op.operand});
	4541	return dest_local;
4380	4542	}
4381	4543
4382	4544	fn airTrap(f: *Function) !void {
...	...	@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
6151	6313	return .none;
6152	6314	}
6153	6315
6154		if (elem_abi_size == 1 and !dest_ty.isVolatilePtr(zcu)) {
6155		const bitcasted = try bitcast(f, .u8, value, elem_ty);
	6316	if (elem_abi_size == 1 and elem_ty.isAbiInt(zcu) and !dest_ty.isVolatilePtr(zcu)) {
6156	6317	try w.writeAll("memset(");
6157	6318	switch (dest_ty.ptrSize(zcu)) {
6158	6319	.slice => {
6159	6320	try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });
6160		try w.writeAll(", ");
6161		try f.writeCValue(w, bitcasted, .other);
	6321	try w.writeAll(", (const char )&");
	6322	switch (value) {
	6323	.constant => \|v\| try f.dg.renderValueAsLvalue(w, v),
	6324	else => try f.writeCValue(w, value, .other),
	6325	}
6162	6326	try w.writeAll(", ");
6163	6327	try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });
6164	6328	},
6165	6329	.one => {
6166	6330	try f.writeCValue(w, dest_slice, .other);
6167		try w.writeAll(", ");
6168		try f.writeCValue(w, bitcasted, .other);
	6331	try w.writeAll(", (const char )&");
	6332	switch (value) {
	6333	.constant => \|v\| try f.dg.renderValueAsLvalue(w, v),
	6334	else => try f.writeCValue(w, value, .other),
	6335	}
6169	6336	try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});
6170	6337	},
6171	6338	.many, .c => unreachable,
6172	6339	}
6173	6340	try w.writeAll(");");
6174	6341	try f.newline();
6175		try f.freeCValue(inst, bitcasted);
6176	6342	try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
6177	6343	return .none;
6178	6344	}

src/codegen/c/type.zig+15-19

...	...	@@ -514,40 +514,39 @@ pub const CType = union(enum) {
514	514	}
515	515	}
516	516	fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {
517		const is_ez80 = zcu.getTarget().cpu.arch == .ez80;
518		return switch (bits) {
	517	const target = zcu.getTarget();
	518	return switch (std.zig.target.intByteSize(target, bits)) {
519	519	0 => .void,
520		1...8 => switch (signedness) {
	520	1 => switch (signedness) {
521	521	.unsigned => .{ .small = .uint8_t },
522	522	.signed => .{ .small = .int8_t },
523	523	},
524		9...16 => switch (signedness) {
	524	2 => switch (signedness) {
525	525	.unsigned => .{ .small = .uint16_t },
526	526	.signed => .{ .small = .int16_t },
527	527	},
528		17...24 => switch (signedness) {
529		.unsigned => .{ .small = if (is_ez80) .uint24_t else .uint32_t },
530		.signed => .{ .small = if (is_ez80) .int24_t else .int32_t },
	528	3 => switch (signedness) {
	529	.unsigned => .{ .small = .uint24_t },
	530	.signed => .{ .small = .int24_t },
531	531	},
532		25...32 => switch (signedness) {
	532	4 => switch (signedness) {
533	533	.unsigned => .{ .small = .uint32_t },
534	534	.signed => .{ .small = .int32_t },
535	535	},
536		33...48 => switch (signedness) {
537		.unsigned => .{ .small = if (is_ez80) .uint48_t else .uint64_t },
538		.signed => .{ .small = if (is_ez80) .int48_t else .int64_t },
	536	6 => switch (signedness) {
	537	.unsigned => .{ .small = .uint48_t },
	538	.signed => .{ .small = .int48_t },
539	539	},
540		49...64 => switch (signedness) {
	540	8 => switch (signedness) {
541	541	.unsigned => .{ .small = .uint64_t },
542	542	.signed => .{ .small = .int64_t },
543	543	},
544		65...128 => switch (signedness) {
	544	16 => switch (signedness) {
545	545	.unsigned => .{ .small = .zig_u128 },
546	546	.signed => .{ .small = .zig_i128 },
547	547	},
548		else => {
	548	else => \|n\| {
549	549	@branchHint(.unlikely);
550		const target = zcu.getTarget();
551	550	const limb_bytes = std.zig.target.intAlignment(target, bits);
552	551	return .{ .big = .{
553	552	.limb_size = switch (limb_bytes) {
...	...	@@ -558,10 +557,7 @@ pub const CType = union(enum) {
558	557	16 => .@"128",
559	558	else => unreachable,
560	559	},
561		.limbs_len = @divExact(
562		std.zig.target.intByteSize(target, bits),
563		limb_bytes,
564		),
	560	.limbs_len = @divExact(n, limb_bytes),
565	561	} };
566	562	},
567	563	};

src/codegen/llvm.zig+138-115

...	...	@@ -21,8 +21,7 @@ const Zcu = @import("../Zcu.zig");
21	21	const aarch64_c_abi = @import("aarch64/abi.zig");
22	22	const FuncGen = @import("llvm/FuncGen.zig");
23	23	const isByRef = FuncGen.isByRef;
24		const firstParamSRet = FuncGen.firstParamSRet;
25		const lowerFnRetTy = FuncGen.lowerFnRetTy;
	24	const fnReturnStrat = FuncGen.fnReturnStrat;
26	25	const iterateParamTypes = FuncGen.iterateParamTypes;
27	26	const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
28	27
...	...	@@ -37,10 +36,10 @@ pub fn legalizeFeatures(_: const std.Target) ?const Air.Legalize.Features {
37	36	.expand_int_from_float_safe,
38	37	.expand_int_from_float_optimized_safe,
39	38
40		.scalarize_bitcast_array,
	39	.scalarize_bit_cast_array,
41	40	// Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest
42	41	// types are vectors with equal length (hence also with equal bits-per-element).
43		.scalarize_bitcast_vector_non_elementwise,
	42	.scalarize_bit_cast_vector_non_elementwise,
44	43	});
45	44	}
46	45
...	...	@@ -732,8 +731,8 @@ pub const Object = struct {
732	731
733	732	// TODO: Address space
734	733	const slice_ty = Type.slice_const_u8_sentinel_0;
735		const llvm_usize_ty = try o.lowerType(.usize);
736		const llvm_slice_ty = try o.lowerType(slice_ty);
	734	const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
	735	const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
737	736	const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
738	737
739	738	llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
...	...	@@ -799,7 +798,7 @@ pub const Object = struct {
799	798	{
800	799	if (o.errors_len_variable != .none) {
801	800	const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
802		const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);
	801	const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
803	802	try o.errors_len_variable.setInitializer(init_val, &o.builder);
804	803	}
805	804	try o.genErrorNameTable();
...	...	@@ -1191,7 +1190,7 @@ pub const Object = struct {
1191	1190	};
1192	1191	{
1193	1192	const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
1194		global.type = try o.lowerType(fn_ty);
	1193	global.type = try o.lowerType(fn_ty, .in_memory);
1195	1194	global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
1196	1195	global.linkage = if (o.builder.strip) .private else .internal;
1197	1196	global.visibility = .default;
...	...	@@ -1440,10 +1439,10 @@ pub const Object = struct {
1440	1439	// represent (because it doesn't have runtime bits), we instead lower as the zero-size
1441	1440	// type `[0 x i8]`. I don't think the type on an extern declaration actually does much
1442	1441	// anyway.
1443		if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);
	1442	if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
1444	1443	break :ty try o.builder.arrayType(0, .i8);
1445	1444	} else if (nav_ty.hasRuntimeBits(zcu)) ty: {
1446		break :ty try o.lowerType(nav_ty);
	1445	break :ty try o.lowerType(nav_ty, .in_memory);
1447	1446	} else {
1448	1447	// This is a non-extern zero-bit `Nav`---we're not interested in it.
1449	1448	// TODO: we might need to rethink this a little under incremental compilation. If a
...	...	@@ -1536,7 +1535,7 @@ pub const Object = struct {
1536	1535	llvm_variable.setAlignment(llvm_align, &o.builder);
1537	1536	llvm_variable.setSection(llvm_section, &o.builder);
1538	1537	llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
1539		try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);
	1538	try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
1540	1539	llvm_variable.setThreadLocal(tl: {
1541	1540	if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
1542	1541	break :tl .default;
...	...	@@ -2134,7 +2133,7 @@ pub const Object = struct {
2134	2133	defer debug_param_types.deinit(gpa);
2135	2134
2136	2135	// Return type goes first.
2137		if (firstParamSRet(fn_info, zcu, target)) {
	2136	if (try fnReturnStrat(o, fn_info) == .sret) {
2138	2137	// Actual return type is void, then first arg is the sret pointer.
2139	2138	const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
2140	2139	debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
...	...	@@ -2682,12 +2681,12 @@ pub const Object = struct {
2682	2681	if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
2683	2682
2684	2683	var it = iterateParamTypes(o, fn_info);
2685		if (firstParamSRet(fn_info, zcu, target)) {
	2684	if (try fnReturnStrat(o, fn_info) == .sret) {
2686	2685	// Sret pointers must not be address 0
2687	2686	try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2688	2687	try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
2689	2688
2690		const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));
	2689	const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type), .in_memory);
2691	2690	try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2692	2691	it.llvm_index += 1;
2693	2692	} else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) \|s\| switch (s) {
...	...	@@ -2729,9 +2728,7 @@ pub const Object = struct {
2729	2728	},
2730	2729	.byref => {
2731	2730	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
2732		const param_llvm_ty = try o.lowerType(param_ty);
2733		const alignment = param_ty.abiAlignment(zcu);
2734		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
	2731	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
2735	2732	},
2736	2733	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2737	2734	.slice => {
...	...	@@ -2841,14 +2838,31 @@ pub const Object = struct {
2841	2838	}
2842	2839	}
2843	2840
2844		pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {
2845		return o.builder.intType(o.zcu.errorSetBits());
	2841	pub const TypeRepr = enum {
	2842	/// The representation of the type when it is being manipulated as a value in a function.
	2843	/// e.g. Zig `u5` -> LLVM `i5`
	2844	by_value,
	2845	/// The representation of the type when it is stored in memory.
	2846	/// e.g. Zig `u5` -> LLVM `i8`
	2847	in_memory,
	2848	};
	2849
	2850	pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
	2851	return o.builder.intType(switch (repr) {
	2852	.by_value => o.zcu.errorSetBits(),
	2853	.in_memory => @intCast(Type.anyerror.abiSize(o.zcu) * 8),
	2854	});
2846	2855	}
2847	2856
2848		pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {
	2857	pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
2849	2858	const zcu = o.zcu;
2850	2859	const target = zcu.getTarget();
2851	2860	const ip = &zcu.intern_pool;
	2861
	2862	if (repr == .by_value) {
	2863	assert(!isByRef(t, zcu)); // by-ref types must only be manipulated in memory
	2864	}
	2865
2852	2866	return switch (t.toIntern()) {
2853	2867	.u0_type => unreachable, // no runtime bits
2854	2868	inline .u1_type,
...	...	@@ -2864,7 +2878,10 @@ pub const Object = struct {
2864	2878	.u80_type,
2865	2879	.u128_type,
2866	2880	.i128_type,
2867		=> \|tag\| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
	2881	=> \|tag\| switch (repr) {
	2882	.by_value => @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
	2883	.in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
	2884	},
2868	2885	.usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
2869	2886	inline .c_char_type,
2870	2887	.c_short_type,
...	...	@@ -2899,7 +2916,7 @@ pub const Object = struct {
2899	2916	return .i8;
2900	2917	},
2901	2918	.bool_type => .i1,
2902		.anyerror_type => try o.errorIntType(),
	2919	.anyerror_type => try o.errorIntType(repr),
2903	2920	.void_type => unreachable, // no runtime bits
2904	2921	.type_type => unreachable, // no runtime bits
2905	2922	.comptime_int_type => unreachable, // no runtime bits
...	...	@@ -2919,10 +2936,10 @@ pub const Object = struct {
2919	2936	=> .ptr,
2920	2937	.slice_const_u8_type,
2921	2938	.slice_const_u8_sentinel_0_type,
2922		=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),
	2939	=> try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
2923	2940	.anyerror_void_error_union_type,
2924	2941	.adhoc_inferred_error_set_type,
2925		=> try o.errorIntType(),
	2942	=> try o.errorIntType(repr),
2926	2943	.generic_poison_type => unreachable,
2927	2944	// values, not types
2928	2945	.undef,
...	...	@@ -2948,7 +2965,10 @@ pub const Object = struct {
2948	2965	.none,
2949	2966	=> unreachable,
2950	2967	else => switch (ip.indexToKey(t.toIntern())) {
2951		.int_type => \|int_type\| try o.builder.intType(int_type.bits),
	2968	.int_type => \|int_type\| switch (repr) {
	2969	.by_value => try o.builder.intType(int_type.bits),
	2970	.in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
	2971	},
2952	2972	.ptr_type => \|ptr_type\| type: {
2953	2973	const ptr_ty = try o.builder.ptrType(
2954	2974	toLlvmAddressSpace(ptr_type.flags.address_space, target),
...	...	@@ -2957,18 +2977,18 @@ pub const Object = struct {
2957	2977	.one, .many, .c => ptr_ty,
2958	2978	.slice => try o.builder.structType(.normal, &.{
2959	2979	ptr_ty,
2960		try o.lowerType(.usize),
	2980	try o.lowerType(.usize, repr),
2961	2981	}),
2962	2982	};
2963	2983	},
2964	2984	.array_type => \|array_type\| o.builder.arrayType(
2965	2985	array_type.lenIncludingSentinel(),
2966		try o.lowerType(.fromInterned(array_type.child)),
	2986	try o.lowerType(.fromInterned(array_type.child), repr),
2967	2987	),
2968	2988	.vector_type => \|vector_type\| o.builder.vectorType(
2969	2989	.normal,
2970	2990	vector_type.len,
2971		try o.lowerType(.fromInterned(vector_type.child)),
	2991	try o.lowerType(.fromInterned(vector_type.child), .by_value),
2972	2992	),
2973	2993	.opt_type => \|child_ty\| {
2974	2994	// Must stay in sync with `opt_payload` logic in `lowerPtr`.
...	...	@@ -2978,8 +2998,11 @@ pub const Object = struct {
2978	2998	.runtime, .partially_comptime => {},
2979	2999	}
2980	3000
2981		const payload_ty = try o.lowerType(.fromInterned(child_ty));
2982		if (t.optionalReprIsPayload(zcu)) return payload_ty;
	3001	if (t.optionalReprIsPayload(zcu)) {
	3002	return o.lowerType(.fromInterned(child_ty), repr);
	3003	}
	3004
	3005	const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
2983	3006
2984	3007	comptime assert(optional_layout_version == 3);
2985	3008	var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
...	...	@@ -2997,7 +3020,7 @@ pub const Object = struct {
2997	3020	.error_union_type => \|error_union_type\| {
2998	3021	// Must stay in sync with `codegen.errUnionPayloadOffset`.
2999	3022	// See logic in `lowerPtr`.
3000		const error_type = try o.errorIntType();
	3023	const error_type = try o.errorIntType(repr);
3001	3024
3002	3025	switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
3003	3026	.fully_comptime => unreachable,
...	...	@@ -3005,7 +3028,7 @@ pub const Object = struct {
3005	3028	.runtime, .partially_comptime => {},
3006	3029	}
3007	3030
3008		const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));
	3031	const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
3009	3032
3010	3033	const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3011	3034	const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
...	...	@@ -3040,16 +3063,14 @@ pub const Object = struct {
3040	3063	},
3041	3064	.simple_type => unreachable,
3042	3065	.struct_type => {
3043		if (o.type_map.get(t.toIntern())) \|value\| return value;
3044
3045	3066	const struct_type = ip.loadStructType(t.toIntern());
3046	3067
3047	3068	if (struct_type.layout == .@"packed") {
3048		const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));
3049		try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3050		return int_ty;
	3069	return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
3051	3070	}
3052	3071
	3072	if (o.type_map.get(t.toIntern())) \|value\| return value;
	3073
3053	3074	assert(struct_type.size > 0);
3054	3075
3055	3076	var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
...	...	@@ -3083,7 +3104,7 @@ pub const Object = struct {
3083	3104
3084	3105	if (!field_ty.hasRuntimeBits(zcu)) continue;
3085	3106
3086		try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));
	3107	try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
3087	3108
3088	3109	offset += field_ty.abiSize(zcu);
3089	3110	}
...	...	@@ -3139,7 +3160,7 @@ pub const Object = struct {
3139	3160	if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3140	3161	continue;
3141	3162	}
3142		try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));
	3163	try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
3143	3164
3144	3165	offset += Type.fromInterned(field_ty).abiSize(zcu);
3145	3166	}
...	...	@@ -3156,28 +3177,24 @@ pub const Object = struct {
3156	3177	return o.builder.structType(.normal, llvm_field_types.items);
3157	3178	},
3158	3179	.union_type => {
3159		if (o.type_map.get(t.toIntern())) \|value\| return value;
3160
3161	3180	const union_obj = ip.loadUnionType(t.toIntern());
3162	3181
3163	3182	if (union_obj.layout == .@"packed") {
3164		const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));
3165		try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3166		return int_ty;
	3183	return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
3167	3184	}
3168	3185
3169		assert(union_obj.size > 0);
3170
3171	3186	const layout = Type.getUnionLayout(union_obj, zcu);
3172	3187
3173	3188	if (layout.payload_size == 0) {
3174		const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
3175		try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3176		return enum_tag_ty;
	3189	return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3177	3190	}
3178	3191
	3192	if (o.type_map.get(t.toIntern())) \|value\| return value;
	3193
	3194	assert(union_obj.size > 0);
	3195
3179	3196	const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3180		const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);
	3197	const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
3181	3198
3182	3199	const payload_ty = ty: {
3183	3200	if (layout.most_aligned_field_size == layout.payload_size) {
...	...	@@ -3203,7 +3220,7 @@ pub const Object = struct {
3203	3220	);
3204	3221	return ty;
3205	3222	}
3206		const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));
	3223	const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3207	3224
3208	3225	// Put the tag before or after the payload depending on which one's
3209	3226	// alignment is greater.
...	...	@@ -3232,9 +3249,9 @@ pub const Object = struct {
3232	3249	return ty;
3233	3250	},
3234	3251	.opaque_type, .spirv_type => unreachable, // no runtime bits
3235		.enum_type => try o.lowerType(t.intTagType(zcu)),
	3252	.enum_type => try o.lowerType(t.intTagType(zcu), repr),
3236	3253	.func_type => \|func_type\| try o.lowerFnType(t, func_type),
3237		.error_set_type, .inferred_error_set_type => try o.errorIntType(),
	3254	.error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
3238	3255	// values, not types
3239	3256	.undef,
3240	3257	.simple_value,
...	...	@@ -3266,12 +3283,12 @@ pub const Object = struct {
3266	3283
3267	3284	assert(fn_ty.fnHasRuntimeBits(zcu));
3268	3285
3269		const ret_ty = try lowerFnRetTy(o, fn_info);
	3286	const ret_strat = try fnReturnStrat(o, fn_info);
3270	3287
3271	3288	var llvm_params: std.ArrayList(Builder.Type) = .empty;
3272	3289	defer llvm_params.deinit(o.gpa);
3273	3290
3274		if (firstParamSRet(fn_info, zcu, target)) {
	3291	if (ret_strat == .sret) {
3275	3292	try llvm_params.append(o.gpa, .ptr);
3276	3293	}
3277	3294
...	...	@@ -3286,7 +3303,7 @@ pub const Object = struct {
3286	3303	.no_bits => continue,
3287	3304	.byval => {
3288	3305	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3289		try llvm_params.append(o.gpa, try o.lowerType(param_ty));
	3306	try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .in_memory else .by_value));
3290	3307	},
3291	3308	.byref, .byref_mut => {
3292	3309	try llvm_params.append(o.gpa, .ptr);
...	...	@@ -3301,7 +3318,7 @@ pub const Object = struct {
3301	3318	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3302	3319	try llvm_params.appendSlice(o.gpa, &.{
3303	3320	try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3304		try o.lowerType(.usize),
	3321	try o.lowerType(.usize, .by_value),
3305	3322	});
3306	3323	},
3307	3324	.multiple_llvm_types => {
...	...	@@ -3309,7 +3326,7 @@ pub const Object = struct {
3309	3326	},
3310	3327	.float_array => \|count\| {
3311	3328	const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3312		const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);
	3329	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .in_memory);
3313	3330	try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3314	3331	},
3315	3332	.i32_array, .i64_array => \|arr_len\| {
...	...	@@ -3321,14 +3338,19 @@ pub const Object = struct {
3321	3338	},
3322	3339	};
3323	3340
3324		return o.builder.fnType(
3325		ret_ty,
3326		llvm_params.items,
3327		if (fn_info.is_var_args) .vararg else .normal,
3328		);
	3341	const llvm_ret_ty: Builder.Type = switch (ret_strat) {
	3342	.void, .sret => .void,
	3343	.by_val => try o.lowerType(.fromInterned(fn_info.return_type), .by_value),
	3344	.mem_cast => \|llvm_ret_ty\| llvm_ret_ty,
	3345	};
	3346	const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
	3347	true => .vararg,
	3348	false => .normal,
	3349	};
	3350	return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
3329	3351	}
3330	3352
3331		pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {
	3353	pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
3332	3354	const zcu = o.zcu;
3333	3355	const ip = &zcu.intern_pool;
3334	3356	const target = zcu.getTarget();
...	...	@@ -3360,7 +3382,7 @@ pub const Object = struct {
3360	3382	.inferred_error_set_type,
3361	3383	=> unreachable, // types, not values
3362	3384
3363		.undef => return o.builder.undefConst(try o.lowerType(ty)),
	3385	.undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
3364	3386	.simple_value => \|simple_value\| switch (simple_value) {
3365	3387	.void => unreachable, // non-runtime value
3366	3388	.null => unreachable, // non-runtime value
...	...	@@ -3375,15 +3397,15 @@ pub const Object = struct {
3375	3397	.int => {
3376	3398	var bigint_space: Value.BigIntSpace = undefined;
3377	3399	const bigint = val.toBigInt(&bigint_space, zcu);
3378		const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);
	3400	const llvm_int_ty = try o.lowerType(ty, repr);
3379	3401	return o.builder.bigIntConst(llvm_int_ty, bigint);
3380	3402	},
3381	3403	.err => \|err\| {
3382	3404	const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
3383		return o.builder.intConst(try o.errorIntType(), int);
	3405	return o.builder.intConst(try o.errorIntType(repr), int);
3384	3406	},
3385	3407	.error_union => \|error_union\| {
3386		const llvm_error_ty = try o.errorIntType();
	3408	const llvm_error_ty = try o.errorIntType(repr);
3387	3409	const llvm_error_value = switch (error_union.val) {
3388	3410	.err_name => \|name\| try o.builder.intConst(
3389	3411	llvm_error_ty,
...	...	@@ -3401,8 +3423,8 @@ pub const Object = struct {
3401	3423	const payload_align = payload_type.abiAlignment(zcu);
3402	3424	const error_align = Type.errorAbiAlignment(zcu);
3403	3425	const llvm_payload_value = switch (error_union.val) {
3404		.err_name => try o.builder.undefConst(try o.lowerType(payload_type)),
3405		.payload => \|payload\| try o.lowerValue(payload),
	3426	.err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
	3427	.payload => \|payload\| try o.lowerValue(payload, repr),
3406	3428	};
3407	3429
3408	3430	var fields: [3]Builder.Type = undefined;
...	...	@@ -3417,7 +3439,7 @@ pub const Object = struct {
3417	3439	fields[0] = vals[0].typeOf(&o.builder);
3418	3440	fields[1] = vals[1].typeOf(&o.builder);
3419	3441
3420		const llvm_ty = try o.lowerType(ty);
	3442	const llvm_ty = try o.lowerType(ty, repr);
3421	3443	const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3422	3444	if (llvm_ty_fields.len > 2) {
3423	3445	assert(llvm_ty_fields.len == 3);
...	...	@@ -3429,7 +3451,7 @@ pub const Object = struct {
3429	3451	fields[0..llvm_ty_fields.len],
3430	3452	), vals[0..llvm_ty_fields.len]);
3431	3453	},
3432		.enum_tag => \|enum_tag\| o.lowerValue(enum_tag.int),
	3454	.enum_tag => \|enum_tag\| o.lowerValue(enum_tag.int, repr),
3433	3455	.float => switch (ty.floatBits(target)) {
3434	3456	16 => if (backendSupportsF16(target))
3435	3457	try o.builder.halfConst(val.toFloat(f16, zcu))
...	...	@@ -3445,9 +3467,9 @@ pub const Object = struct {
3445	3467	else => unreachable,
3446	3468	},
3447	3469	.ptr => try o.lowerPtr(arg_val, 0),
3448		.slice => \|slice\| return o.builder.structConst(try o.lowerType(ty), &.{
3449		try o.lowerValue(slice.ptr),
3450		try o.lowerValue(slice.len),
	3470	.slice => \|slice\| return o.builder.structConst(try o.lowerType(ty, repr), &.{
	3471	try o.lowerValue(slice.ptr, repr),
	3472	try o.lowerValue(slice.len, repr),
3451	3473	}),
3452	3474	.opt => \|opt\| {
3453	3475	comptime assert(optional_layout_version == 3);
...	...	@@ -3457,7 +3479,7 @@ pub const Object = struct {
3457	3479	if (!payload_ty.hasRuntimeBits(zcu)) {
3458	3480	return non_null_bit;
3459	3481	}
3460		const llvm_ty = try o.lowerType(ty);
	3482	const llvm_ty = try o.lowerType(ty, repr);
3461	3483	if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3462	3484	.none => switch (llvm_ty.tag(&o.builder)) {
3463	3485	.integer => try o.builder.intConst(llvm_ty, 0),
...	...	@@ -3465,15 +3487,15 @@ pub const Object = struct {
3465	3487	.structure => try o.builder.zeroInitConst(llvm_ty),
3466	3488	else => unreachable,
3467	3489	},
3468		else => \|payload\| try o.lowerValue(payload),
	3490	else => \|payload\| try o.lowerValue(payload, repr),
3469	3491	};
3470	3492	assert(payload_ty.zigTypeTag(zcu) != .@"fn");
3471	3493
3472	3494	var fields: [3]Builder.Type = undefined;
3473	3495	var vals: [3]Builder.Constant = undefined;
3474	3496	vals[0] = switch (opt.val) {
3475		.none => try o.builder.undefConst(try o.lowerType(payload_ty)),
3476		else => \|payload\| try o.lowerValue(payload),
	3497	.none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
	3498	else => \|payload\| try o.lowerValue(payload, repr),
3477	3499	};
3478	3500	vals[1] = non_null_bit;
3479	3501	fields[0] = vals[0].typeOf(&o.builder);
...	...	@@ -3490,14 +3512,14 @@ pub const Object = struct {
3490	3512	fields[0..llvm_ty_fields.len],
3491	3513	), vals[0..llvm_ty_fields.len]);
3492	3514	},
3493		.bitpack => \|bitpack\| return o.lowerValue(bitpack.backing_int_val),
	3515	.bitpack => \|bitpack\| return o.lowerValue(bitpack.backing_int_val, repr),
3494	3516	.aggregate => \|aggregate\| switch (ip.indexToKey(ty.toIntern())) {
3495	3517	.array_type => \|array_type\| switch (aggregate.storage) {
3496	3518	.bytes => \|bytes\| try o.builder.stringConst(try o.builder.string(
3497	3519	bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3498	3520	)),
3499	3521	.elems => \|elems\| {
3500		const array_ty = try o.lowerType(ty);
	3522	const array_ty = try o.lowerType(ty, repr);
3501	3523	const elem_ty = array_ty.childType(&o.builder);
3502	3524	assert(elems.len == array_ty.aggregateLen(&o.builder));
3503	3525
...	...	@@ -3515,7 +3537,7 @@ pub const Object = struct {
3515	3537
3516	3538	var need_unnamed = false;
3517	3539	for (vals, fields, elems) \|result_val, result_field, elem\| {
3518		result_val.* = try o.lowerValue(elem);
	3540	result_val.* = try o.lowerValue(elem, repr);
3519	3541	result_field.* = result_val.typeOf(&o.builder);
3520	3542	if (result_field.* != elem_ty) need_unnamed = true;
3521	3543	}
...	...	@@ -3527,7 +3549,7 @@ pub const Object = struct {
3527	3549	.repeated_elem => \|elem\| {
3528	3550	const len: usize = @intCast(array_type.len);
3529	3551	const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3530		const array_ty = try o.lowerType(ty);
	3552	const array_ty = try o.lowerType(ty, repr);
3531	3553	const elem_ty = array_ty.childType(&o.builder);
3532	3554
3533	3555	const ExpectedContents = extern struct {
...	...	@@ -3543,12 +3565,12 @@ pub const Object = struct {
3543	3565	defer allocator.free(fields);
3544	3566
3545	3567	var need_unnamed = false;
3546		@memset(vals[0..len], try o.lowerValue(elem));
	3568	@memset(vals[0..len], try o.lowerValue(elem, repr));
3547	3569	@memset(fields[0..len], vals[0].typeOf(&o.builder));
3548	3570	if (fields[0] != elem_ty) need_unnamed = true;
3549	3571
3550	3572	if (array_type.sentinel != .none) {
3551		vals[len] = try o.lowerValue(array_type.sentinel);
	3573	vals[len] = try o.lowerValue(array_type.sentinel, repr);
3552	3574	fields[len] = vals[len].typeOf(&o.builder);
3553	3575	if (fields[len] != elem_ty) need_unnamed = true;
3554	3576	}
...	...	@@ -3560,7 +3582,7 @@ pub const Object = struct {
3560	3582	},
3561	3583	},
3562	3584	.vector_type => \|vector_type\| {
3563		const vector_ty = try o.lowerType(ty);
	3585	const vector_ty = try o.lowerType(ty, repr);
3564	3586	switch (aggregate.storage) {
3565	3587	.bytes, .elems => {
3566	3588	const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
...	...	@@ -3575,7 +3597,7 @@ pub const Object = struct {
3575	3597	result_val.* = try o.builder.intConst(.i8, byte);
3576	3598	},
3577	3599	.elems => \|elems\| for (vals, elems) \|*result_val, elem\| {
3578		result_val.* = try o.lowerValue(elem);
	3600	result_val.* = try o.lowerValue(elem, .by_value);
3579	3601	},
3580	3602	.repeated_elem => unreachable,
3581	3603	}
...	...	@@ -3583,12 +3605,12 @@ pub const Object = struct {
3583	3605	},
3584	3606	.repeated_elem => \|elem\| return o.builder.splatConst(
3585	3607	vector_ty,
3586		try o.lowerValue(elem),
	3608	try o.lowerValue(elem, .by_value),
3587	3609	),
3588	3610	}
3589	3611	},
3590	3612	.tuple_type => \|tuple\| {
3591		const struct_ty = try o.lowerType(ty);
	3613	const struct_ty = try o.lowerType(ty, repr);
3592	3614	const llvm_len = struct_ty.aggregateLen(&o.builder);
3593	3615
3594	3616	const ExpectedContents = extern struct {
...	...	@@ -3633,8 +3655,8 @@ pub const Object = struct {
3633	3655
3634	3656	vals[llvm_index] = switch (aggregate.storage) {
3635	3657	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3636		.elems => \|elems\| try o.lowerValue(elems[field_index]),
3637		.repeated_elem => \|elem\| try o.lowerValue(elem),
	3658	.elems => \|elems\| try o.lowerValue(elems[field_index], repr),
	3659	.repeated_elem => \|elem\| try o.lowerValue(elem, repr),
3638	3660	};
3639	3661	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3640	3662	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...	...	@@ -3663,7 +3685,7 @@ pub const Object = struct {
3663	3685	},
3664	3686	.struct_type => {
3665	3687	const struct_type = ip.loadStructType(ty.toIntern());
3666		const struct_ty = try o.lowerType(ty);
	3688	const struct_ty = try o.lowerType(ty, repr);
3667	3689	assert(struct_type.layout != .@"packed");
3668	3690	const llvm_len = struct_ty.aggregateLen(&o.builder);
3669	3691
...	...	@@ -3707,8 +3729,8 @@ pub const Object = struct {
3707	3729
3708	3730	vals[llvm_index] = switch (aggregate.storage) {
3709	3731	.bytes => \|bytes\| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3710		.elems => \|elems\| try o.lowerValue(elems[field_index]),
3711		.repeated_elem => \|elem\| try o.lowerValue(elem),
	3732	.elems => \|elems\| try o.lowerValue(elems[field_index], repr),
	3733	.repeated_elem => \|elem\| try o.lowerValue(elem, repr),
3712	3734	};
3713	3735	fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3714	3736	if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...	...	@@ -3738,9 +3760,9 @@ pub const Object = struct {
3738	3760	else => unreachable,
3739	3761	},
3740	3762	.un => \|un\| {
3741		const union_ty = try o.lowerType(ty);
	3763	const union_ty = try o.lowerType(ty, repr);
3742	3764	const layout = ty.unionGetLayout(zcu);
3743		if (layout.payload_size == 0) return o.lowerValue(un.tag);
	3765	if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
3744	3766
3745	3767	const union_obj = zcu.typeToUnion(ty).?;
3746	3768	const container_layout = union_obj.layout;
...	...	@@ -3761,7 +3783,7 @@ pub const Object = struct {
3761	3783	const padding_len = layout.payload_size;
3762	3784	break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3763	3785	}
3764		const payload = try o.lowerValue(un.val);
	3786	const payload = try o.lowerValue(un.val, repr);
3765	3787	const payload_ty = payload.typeOf(&o.builder);
3766	3788	if (payload_ty != union_ty.structFields(&o.builder)[
3767	3789	@intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
...	...	@@ -3776,7 +3798,7 @@ pub const Object = struct {
3776	3798	);
3777	3799	} else p: {
3778	3800	assert(layout.tag_size == 0);
3779		const union_val = try o.lowerValue(un.val);
	3801	const union_val = try o.lowerValue(un.val, repr);
3780	3802	need_unnamed = true;
3781	3803	break :p union_val;
3782	3804	};
...	...	@@ -3786,7 +3808,7 @@ pub const Object = struct {
3786	3808	try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3787	3809	else
3788	3810	union_ty, &.{payload});
3789		const tag = try o.lowerValue(un.tag);
	3811	const tag = try o.lowerValue(un.tag, repr);
3790	3812	const tag_ty = tag.typeOf(&o.builder);
3791	3813	var fields: [3]Builder.Type = undefined;
3792	3814	var vals: [3]Builder.Constant = undefined;
...	...	@@ -3840,8 +3862,8 @@ pub const Object = struct {
3840	3862	},
3841	3863	.int => try o.builder.castConst(
3842	3864	.inttoptr,
3843		try o.builder.intConst(try o.lowerType(.usize), offset),
3844		try o.lowerType(.fromInterned(ptr.ty)),
	3865	try o.builder.intConst(try o.lowerType(.usize, .by_value), offset),
	3866	try o.lowerType(.fromInterned(ptr.ty), .by_value),
3845	3867	),
3846	3868	.eu_payload => \|eu_ptr\| try o.lowerPtr(
3847	3869	eu_ptr,
...	...	@@ -3888,7 +3910,7 @@ pub const Object = struct {
3888	3910	@"addrspace": std.lang.AddressSpace,
3889	3911	) Allocator.Error!Builder.Constant {
3890	3912	const addr: u64 = @"align".toByteUnits().?;
3891		const llvm_usize = try o.lowerType(.usize);
	3913	const llvm_usize = try o.lowerType(.usize, .by_value);
3892	3914	const llvm_addr = try o.builder.intConst(llvm_usize, addr);
3893	3915	const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
3894	3916	return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
...	...	@@ -3932,11 +3954,11 @@ pub const Object = struct {
3932	3954	}
3933	3955	errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
3934	3956
3935		const llvm_ty = try o.lowerType(uav_ty);
	3957	const llvm_ty = try o.lowerType(uav_ty, .in_memory);
3936	3958	const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
3937	3959	const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
3938	3960	gop.value_ptr.* = llvm_variable;
3939		try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);
	3961	try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
3940	3962	llvm_variable.setMutability(.constant, &o.builder);
3941	3963	llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
3942	3964	const llvm_global = llvm_variable.ptrConst(&o.builder).global;
...	...	@@ -4008,7 +4030,7 @@ pub const Object = struct {
4008	4030	.x86_64_interrupt,
4009	4031	.x86_interrupt,
4010	4032	=> {
4011		const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));
	4033	const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
4012	4034	try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4013	4035	},
4014	4036	}
...	...	@@ -4030,14 +4052,15 @@ pub const Object = struct {
4030	4052	o: *Object,
4031	4053	attributes: *Builder.FunctionAttributes.Wip,
4032	4054	llvm_arg_i: u32,
4033		alignment: Builder.Alignment,
4034	4055	byval: bool,
4035		param_llvm_ty: Builder.Type,
	4056	param_ty: Type,
4036	4057	) Allocator.Error!void {
	4058	const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
	4059	const alignment = param_ty.abiAlignment(o.zcu).toLlvm();
4037	4060	try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4038	4061	try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4039	4062	try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);
4040		if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);
	4063	if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
4041	4064	}
4042	4065
4043	4066	pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
...	...	@@ -4062,7 +4085,7 @@ pub const Object = struct {
4062	4085	pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
4063	4086	const builder = &o.builder;
4064	4087	if (o.errors_len_variable == .none) {
4065		const llvm_err_int_ty = try o.errorIntType();
	4088	const llvm_err_int_ty = try o.errorIntType(.in_memory);
4066	4089	const name = try builder.strtabString("__zig_errors_len");
4067	4090	const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
4068	4091	variable_index.setMutability(.constant, builder);
...	...	@@ -4102,9 +4125,9 @@ pub const Object = struct {
4102	4125	const ip = &zcu.intern_pool;
4103	4126	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4104	4127
4105		const llvm_usize_ty = try o.lowerType(.usize);
4106		const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);
4107		const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4128	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	4129	const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .by_value);
	4130	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4108	4131
4109	4132	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4110	4133	try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
...	...	@@ -4153,7 +4176,7 @@ pub const Object = struct {
4153	4176	const return_block = try wip.block(1, "Name");
4154	4177	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
4155	4178	.none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
4156		else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	4179	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip, .by_value),
4157	4180	};
4158	4181	try wip_switch.addCase(llvm_tag_val, return_block, &wip);
4159	4182
...	...	@@ -4199,7 +4222,7 @@ pub const Object = struct {
4199	4222	const ip = &zcu.intern_pool;
4200	4223	const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
4201	4224
4202		const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));
	4225	const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4203	4226	function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4204	4227	try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
4205	4228
...	...	@@ -4226,7 +4249,7 @@ pub const Object = struct {
4226	4249
4227	4250	if (loaded_enum.field_values.len > 0) {
4228	4251	for (loaded_enum.field_values.get(ip)) \|tag_val_ip\| {
4229		const llvm_tag_val = try o.lowerValue(tag_val_ip);
	4252	const llvm_tag_val = try o.lowerValue(tag_val_ip, .by_value);
4230	4253	try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4231	4254	}
4232	4255	} else {

src/codegen/llvm/FuncGen.zig+467-579

...	...	@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
164	164	const zcu = o.zcu;
165	165	const ty = val.typeOf(zcu);
166	166	if (!isByRef(ty, zcu)) {
167		return o.lowerValue(val.toIntern());
	167	return o.lowerValue(val.toIntern(), .by_value);
168	168	} else {
169	169	// We need a pointer to a global constant, i.e. a UAV.
170	170	return o.lowerUavRef(
...	...	@@ -193,12 +193,13 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
193	193	var it = iterateParamTypes(o, fn_info);
194	194
195	195	// Populate `fg.ret_ptr`...
196		if (firstParamSRet(fn_info, zcu, zcu.getTarget())) {
197		fg.ret_ptr = fg.wip.arg(it.llvm_index);
198		it.llvm_index += 1;
199		} else {
200		fg.ret_ptr = .none;
201		}
	196	fg.ret_ptr = switch (try fnReturnStrat(o, fn_info)) {
	197	.sret => rp: {
	198	defer it.llvm_index += 1;
	199	break :rp fg.wip.arg(it.llvm_index);
	200	},
	201	else => .none,
	202	};
202	203	// ...and `fg.err_ret_trace`...
203	204	if (fn_info.cc == .auto and comp.config.any_error_tracing) {
204	205	fg.err_ret_trace = fg.wip.arg(it.llvm_index);
...	...	@@ -224,7 +225,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
224	225
225	226	if (isByRef(param_ty, zcu)) {
226	227	const alignment = param_ty.abiAlignment(zcu).toLlvm();
227		const arg_ptr = try fg.buildAlloca(try o.lowerType(param_ty), alignment);
	228	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
228	229	// We don't need to handle non-ABI-sized integer types in memory here since they
229	230	// are never by-ref.
230	231	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
...	...	@@ -248,9 +249,8 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
248	249	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
249	250	const param = fg.wip.arg(it.llvm_index - 1);
250	251
251		const param_llvm_ty = try o.lowerType(param_ty);
252	252	const alignment = param_ty.abiAlignment(zcu).toLlvm();
253		const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
	253	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
254	254	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
255	255
256	256	if (isByRef(param_ty, zcu)) {
...	...	@@ -264,7 +264,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
264	264	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
265	265	assert(!isByRef(param_ty, zcu));
266	266	const slice_val = try fg.wip.buildAggregate(
267		try o.lowerType(param_ty),
	267	try o.lowerType(param_ty, .by_value),
268	268	&.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
269	269	"",
270	270	);
...	...	@@ -291,11 +291,10 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
291	291	},
292	292	.float_array => {
293	293	const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
294		const param_llvm_ty = try o.lowerType(param_ty);
295	294	const param = fg.wip.arg(it.llvm_index - 1);
296	295
297	296	const alignment = param_ty.abiAlignment(zcu).toLlvm();
298		const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);
	297	const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
299	298	_ = try fg.wip.store(.normal, param, arg_ptr, alignment);
300	299
301	300	if (isByRef(param_ty, zcu)) {
...	...	@@ -349,13 +348,13 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
349	348	try fuzz.pcs.append(gpa, pc);
350	349	},
351	350	}
352		for (body, 0..) \|inst, i\| {
	351	for (body) \|inst\| {
353	352	if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
354	353
355	354	const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
356	355	// zig fmt: off
357	356
358		// Required due to `.scalarize_bitcast_vector_non_elementwise` being enabled.
	357	// Required due to `.scalarize_bit_cast_vector_non_elementwise` being enabled.
359	358	.legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),
360	359	.legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),
361	360
...	...	@@ -461,29 +460,36 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
461	460	.is_err => try self.airIsErr(inst, .ne, false),
462	461	.is_err_ptr => try self.airIsErr(inst, .ne, true),
463	462
464		.alloc => try self.airAlloc(inst),
465		.ret_ptr => try self.airRetPtr(inst),
466		.arg => try self.airArg(inst),
467		.bitcast => try self.airBitCast(inst),
468		.breakpoint => try self.airBreakpoint(inst),
469		.ret_addr => try self.airRetAddr(inst),
470		.frame_addr => try self.airFrameAddress(inst),
471		.@"try" => try self.airTry(inst, false),
472		.try_cold => try self.airTry(inst, true),
473		.try_ptr => try self.airTryPtr(inst, false),
474		.try_ptr_cold => try self.airTryPtr(inst, true),
475		.intcast => try self.airIntCast(inst, false),
476		.intcast_safe => try self.airIntCast(inst, true),
477		.trunc => try self.airTrunc(inst),
478		.fptrunc => try self.airFptrunc(inst),
479		.fpext => try self.airFpext(inst),
480		.load => try self.airLoad(inst),
481		.not => try self.airNot(inst),
482		.store => try self.airStore(inst, false),
483		.store_safe => try self.airStore(inst, true),
484		.assembly => try self.airAssembly(inst),
485		.slice_ptr => try self.airSliceField(inst, 0),
486		.slice_len => try self.airSliceField(inst, 1),
	463	.alloc => try self.airAlloc(inst),
	464	.ret_ptr => try self.airRetPtr(inst),
	465	.arg => try self.airArg(inst),
	466	.bit_cast => try self.airBitCast(inst),
	467	.ptr_cast => try self.airNopCast(inst),
	468	.ptr_from_int => try self.airPtrFromInt(inst),
	469	.int_from_ptr => try self.airIntFromPtr(inst),
	470	.error_cast => try self.airNopCast(inst),
	471	.error_from_int => try self.airNopCast(inst),
	472	.int_from_error => try self.airNopCast(inst),
	473	.union_from_enum => try self.airUnionFromEnum(inst),
	474	.breakpoint => try self.airBreakpoint(inst),
	475	.ret_addr => try self.airRetAddr(inst),
	476	.frame_addr => try self.airFrameAddress(inst),
	477	.@"try" => try self.airTry(inst, false),
	478	.try_cold => try self.airTry(inst, true),
	479	.try_ptr => try self.airTryPtr(inst, false),
	480	.try_ptr_cold => try self.airTryPtr(inst, true),
	481	.int_cast => try self.airIntCast(inst, false),
	482	.int_cast_safe => try self.airIntCast(inst, true),
	483	.trunc => try self.airTrunc(inst),
	484	.fptrunc => try self.airFptrunc(inst),
	485	.fpext => try self.airFpext(inst),
	486	.load => try self.airLoad(inst),
	487	.not => try self.airNot(inst),
	488	.store => try self.airStore(inst, false),
	489	.store_safe => try self.airStore(inst, true),
	490	.assembly => try self.airAssembly(inst),
	491	.slice_ptr => try self.airSliceField(inst, 0),
	492	.slice_len => try self.airSliceField(inst, 1),
487	493
488	494	.ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
489	495	.ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
...	...	@@ -561,9 +567,9 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
561	567	.set_err_return_trace => try self.airSetErrReturnTrace(inst),
562	568	.save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
563	569
564		.wrap_optional => try self.airWrapOptional(body[i..]),
565		.wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),
566		.wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),
	570	.wrap_optional => try self.airWrapOptional(inst),
	571	.wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
	572	.wrap_errunion_err => try self.airWrapErrUnionErr(inst),
567	573
568	574	.wasm_memory_size => try self.airWasmMemorySize(inst),
569	575	.wasm_memory_grow => try self.airWasmMemoryGrow(inst),
...	...	@@ -746,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
746	752	break :llvm_fn try self.resolveInst(air_call.callee);
747	753	};
748	754	const target = zcu.getTarget();
749		const sret = firstParamSRet(fn_info, zcu, target);
	755	const ret_strat = try fnReturnStrat(o, fn_info);
750	756
751	757	var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
752	758	defer llvm_args.deinit();
...	...	@@ -764,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
764	770	.no_suspend, .always_inline, .compile_time => unreachable,
765	771	}
766	772
767		const ret_ptr = if (sret) ret_ptr: {
768		const llvm_ret_ty = try o.lowerType(return_type);
769		try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);
	773	const sret_alloc: ?Builder.Value = switch (ret_strat) {
	774	.sret => sret_alloc: {
	775	try attributes.addParamAttr(0, .{ .sret = try o.lowerType(return_type, .in_memory) }, &o.builder);
770	776
771		const alignment = return_type.abiAlignment(zcu).toLlvm();
772		const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);
773		try llvm_args.append(ret_ptr);
774		break :ret_ptr ret_ptr;
775		} else ret_ptr: {
776		if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) \|s\| switch (s) {
777		.signed => try attributes.addRetAttr(.signext, &o.builder),
778		.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
779		};
780		break :ret_ptr null;
	777	const ptr = try self.buildZigAlloca(return_type, .none);
	778	try llvm_args.append(ptr);
	779	break :sret_alloc ptr;
	780	},
	781	else => sret_alloc: {
	782	if (ccAbiPromoteInt(fn_info.cc, zcu, .fromInterned(fn_info.return_type))) \|s\| switch (s) {
	783	.signed => try attributes.addRetAttr(.signext, &o.builder),
	784	.unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
	785	};
	786	break :sret_alloc null;
	787	},
781	788	};
782	789
783	790	const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
...	...	@@ -793,11 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
793	800	const arg = args[it.zig_index - 1];
794	801	const param_ty = self.typeOf(arg);
795	802	const llvm_arg = try self.resolveInst(arg);
796		const llvm_param_ty = try o.lowerType(param_ty);
797	803	if (isByRef(param_ty, zcu)) {
798	804	const alignment = param_ty.abiAlignment(zcu).toLlvm();
799	805	// We don't need to handle non-ABI-sized integer types in memory here since they are
800	806	// never by-ref.
	807	const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
801	808	const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
802	809	try llvm_args.append(loaded);
803	810	} else {
...	...	@@ -811,9 +818,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
811	818	if (isByRef(param_ty, zcu)) {
812	819	try llvm_args.append(llvm_arg);
813	820	} else {
814		const alignment = param_ty.abiAlignment(zcu).toLlvm();
815		const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
816		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
	821	const arg_ptr = try self.buildZigAlloca(param_ty, .none);
817	822	try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
818	823	try llvm_args.append(arg_ptr);
819	824	}
...	...	@@ -823,9 +828,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
823	828	const param_ty = self.typeOf(arg);
824	829	const llvm_arg = try self.resolveInst(arg);
825	830
826		const alignment = param_ty.abiAlignment(zcu).toLlvm();
827		const param_llvm_ty = try o.lowerType(param_ty);
828		const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
	831	const arg_ptr = try self.buildZigAlloca(param_ty, .none);
829	832	try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
830	833	try llvm_args.append(arg_ptr);
831	834	},
...	...	@@ -877,18 +880,16 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
877	880	const arg_ty = self.typeOf(arg);
878	881	const arg_val = try self.resolveInst(arg);
879	882
880		const arg_align = arg_ty.abiAlignment(zcu);
881
882	883	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
883		const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
	884	const ptr = try self.buildZigAlloca(arg_ty, .none);
884	885	try self.store(ptr, .none, arg_val, arg_ty, .normal);
885	886	break :ptr ptr;
886	887	} else arg_val;
887	888
888		const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);
	889	const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?, .in_memory);
889	890	const array_ty = try o.builder.arrayType(count, float_ty);
890	891
891		const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
	892	const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
892	893	try llvm_args.append(loaded);
893	894	},
894	895	.i32_array, .i64_array => \|arr_len\| {
...	...	@@ -897,16 +898,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
897	898	const arg_ty = self.typeOf(arg);
898	899	const arg_val = try self.resolveInst(arg);
899	900
900		const arg_align = arg_ty.abiAlignment(zcu);
901
902	901	const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
903		const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());
	902	const ptr = try self.buildZigAlloca(arg_ty, .none);
904	903	try self.store(ptr, .none, arg_val, arg_ty, .normal);
905	904	break :ptr ptr;
906	905	} else arg_val;
907	906
908	907	const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
909		const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");
	908	const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
910	909	try llvm_args.append(loaded);
911	910	},
912	911	};
...	...	@@ -916,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
916	915	{
917	916	// Add argument attributes.
918	917	it = iterateParamTypes(o, fn_info);
919		it.llvm_index += @intFromBool(sret);
	918	it.llvm_index += @intFromBool(ret_strat == .sret);
920	919	it.llvm_index += @intFromBool(err_return_tracing);
921	920	var remaining_inreg_int = cc_info.inreg_int_params;
922	921	var remaining_inreg_float = cc_info.inreg_float_params;
...	...	@@ -945,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
945	944	},
946	945	.byref => {
947	946	const param_index = it.zig_index - 1;
948		const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);
949		const param_llvm_ty = try o.lowerType(param_ty);
950		const alignment = param_ty.abiAlignment(zcu).toLlvm();
951		try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
	947	const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]);
	948	try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
952	949	},
953	950	.byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
954	951	// No attributes needed for these.
...	...	@@ -998,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
998	995	},
999	996	cc_info.llvm_cc,
1000	997	try attributes.finish(&o.builder),
1001		try o.lowerType(zig_fn_ty),
	998	try o.lowerType(zig_fn_ty, .by_value),
1002	999	llvm_fn,
1003	1000	llvm_args.items,
1004	1001	"",
...	...	@@ -1008,48 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
1008	1005	return .none;
1009	1006	}
1010	1007
1011		if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {
	1008	if (self.liveness.isUnused(inst)) {
1012	1009	return .none;
1013	1010	}
1014	1011
1015		const llvm_ret_ty = try o.lowerType(return_type);
1016		if (ret_ptr) \|rp\| {
1017		if (isByRef(return_type, zcu)) {
1018		return rp;
1019		} else {
1020		// our by-ref status disagrees with sret so we must load.
1021		return self.load(rp, .none, return_type, .normal);
1022		}
1023		}
1024
1025		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1026
1027		if (abi_ret_ty != llvm_ret_ty) {
1028		// In this case the function return type is honoring the calling convention by having
1029		// a different LLVM type than the usual one. We solve this here at the callsite
1030		// by using our canonical type, then loading it if necessary.
1031		const alignment = return_type.abiAlignment(zcu).toLlvm();
1032		const rp = try self.buildAlloca(abi_ret_ty, alignment);
1033		// We don't need to handle non-ABI-sized integer types in memory here since they can only be
1034		// returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1035		// `llvm_ret_ty` anyway.
1036		_ = try self.wip.store(.normal, call, rp, alignment);
1037		return if (isByRef(return_type, zcu))
1038		rp
1039		else
1040		try self.load(rp, .none, return_type, .normal);
1041		}
	1012	// We exit this `switch` if we have a pointer to the return value.
	1013	const ret_val_ptr: Builder.Value = switch (ret_strat) {
	1014	.void => return .none,
	1015	.by_val => return call,
1042	1016
	1017	.sret => sret_alloc.?,
	1018	.mem_cast => \|llvm_ret_ty\| ret_val_ptr: {
	1019	const alignment = return_type.abiAlignment(zcu).toLlvm();
	1020	const ptr = try self.buildAlloca(llvm_ret_ty, alignment);
	1021	_ = try self.wip.store(.normal, call, ptr, alignment);
	1022	break :ret_val_ptr ptr;
	1023	},
	1024	};
1043	1025	if (isByRef(return_type, zcu)) {
1044		// our by-ref status disagrees with sret so we must allocate, store,
1045		// and return the allocation pointer.
1046		const alignment = return_type.abiAlignment(zcu).toLlvm();
1047		const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1048		// We don't need to handle non-ABI-sized integer types here since they are never by-ref.
1049		_ = try self.wip.store(.normal, call, rp, alignment);
1050		return rp;
	1026	return ret_val_ptr;
1051	1027	} else {
1052		return call;
	1028	return self.load(ret_val_ptr, .none, return_type, .normal);
1053	1029	}
1054	1030	}
1055	1031
...	...	@@ -1059,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v
1059	1035	const target = zcu.getTarget();
1060	1036	const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));
1061	1037	const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
1062		const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty));
	1038	const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty), .by_value);
1063	1039
1064	1040	const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
1065	1041
...	...	@@ -1083,64 +1059,21 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1083	1059	const zcu = o.zcu;
1084	1060	const ip = &zcu.intern_pool;
1085	1061	const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1086		const ret_ty = self.typeOf(un_op);
1087
1088		if (self.ret_ptr != .none) {
1089		const operand = try self.resolveInst(un_op);
1090		const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
1091		if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1092		const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
1093		_ = try self.wip.callMemSet(
1094		self.ret_ptr,
1095		ret_ty.abiAlignment(zcu).toLlvm(),
1096		try o.builder.intValue(.i8, 0xaa),
1097		len,
1098		.normal,
1099		self.disable_intrinsics,
1100		);
1101		const owner_mod = self.ownerModule();
1102		if (owner_mod.valgrind) {
1103		try self.valgrindMarkUndef(self.ret_ptr, len);
1104		}
1105		_ = try self.wip.retVoid();
1106		return;
1107		}
1108
1109		const unwrapped_operand = operand.unwrap();
1110		const unwrapped_ret = self.ret_ptr.unwrap();
1111	1062
1112		// Return value was stored previously
1113		if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
1114		_ = try self.wip.retVoid();
1115		return;
1116		}
	1063	const ret_ty = self.typeOf(un_op);
1117	1064
1118		try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
1119		_ = try self.wip.retVoid();
1120		return;
1121		}
1122	1065	const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1123		if (!ret_ty.hasRuntimeBits(zcu)) {
1124		if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1125		// Functions with an empty error set are emitted with an error code
1126		// return type and return zero so they can be function pointers coerced
1127		// to functions that return anyerror.
1128		_ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1129		} else {
1130		_ = try self.wip.retVoid();
1131		}
1132		return;
1133		}
1134	1066
1135		const llvm_ret_ty = try o.lowerType(ret_ty);
1136		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1137		const operand = try self.resolveInst(un_op);
	1067	const ret_strat = try fnReturnStrat(o, fn_info);
1138	1068	const val_is_undef = if (un_op.toInterned()) \|i\| Value.fromInterned(i).isUndef(zcu) else false;
1139	1069	const ret_ty_align = ret_ty.abiAlignment(zcu);
1140	1070
1141	1071	if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1142		const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
1143		const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
	1072	const rp = switch (self.ret_ptr) {
	1073	.none => try self.buildZigAlloca(ret_ty, .none),
	1074	else => \|rp\| rp,
	1075	};
	1076	const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), ret_ty.abiSize(zcu));
1144	1077	_ = try self.wip.callMemSet(
1145	1078	rp,
1146	1079	ret_ty_align.toLlvm(),
...	...	@@ -1153,36 +1086,46 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1153	1086	if (owner_mod.valgrind) {
1154	1087	try self.valgrindMarkUndef(rp, len);
1155	1088	}
1156		if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1157		assert(!isByRef(ret_ty, zcu));
1158		// The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure
1159		// we load it from memory correctly.
1160		const loaded = try self.load(rp, .none, ret_ty, .normal);
1161		_ = try self.wip.ret(loaded);
1162		} else {
1163		const loaded = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1164		_ = try self.wip.ret(loaded);
	1089	switch (ret_strat) {
	1090	.void => unreachable, // value is undef so return type cannot be OPV
	1091	.sret => {
	1092	// We just stored directly to `self.ret_ptr`.
	1093	_ = try self.wip.retVoid();
	1094	},
	1095	.by_val => {
	1096	const loaded = try self.load(rp, .none, ret_ty, .normal);
	1097	_ = try self.wip.ret(loaded);
	1098	},
	1099	.mem_cast => \|llvm_abi_ret_ty\| {
	1100	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
	1101	_ = try self.wip.ret(loaded);
	1102	},
1165	1103	}
1166	1104	return;
1167	1105	}
1168	1106
1169		if (isByRef(ret_ty, zcu)) {
1170		// operand is a pointer however self.ret_ptr is null so that means we need to return a value.
1171		// No need to handle non-ABI-sized integer types in memory here since they are never by-ref.
1172		_ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, ret_ty_align.toLlvm(), ""));
1173		return;
1174		}
1175
1176		if (abi_ret_ty == llvm_ret_ty) {
1177		_ = try self.wip.ret(operand);
1178		} else {
1179		const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());
1180		try self.store(rp, .none, operand, ret_ty, .normal);
1181		// No need to handle non-ABI-sized integer types in memory here since they can only be
1182		// returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1183		// `llvm_ret_ty` anyway.
1184		const ret_val = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1185		_ = try self.wip.ret(ret_val);
	1107	switch (ret_strat) {
	1108	.void => _ = try self.wip.retVoid(),
	1109	.sret => {
	1110	const operand = try self.resolveInst(un_op);
	1111	try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
	1112	_ = try self.wip.retVoid();
	1113	},
	1114	.by_val => {
	1115	assert(!isByRef(ret_ty, zcu));
	1116	const operand = try self.resolveInst(un_op);
	1117	_ = try self.wip.ret(operand);
	1118	},
	1119	.mem_cast => \|llvm_ret_ty\| {
	1120	const operand = try self.resolveInst(un_op);
	1121	const ptr: Builder.Value = if (!isByRef(ret_ty, zcu)) ptr: {
	1122	const ptr = try self.buildZigAlloca(ret_ty, .none);
	1123	try self.store(ptr, .none, operand, ret_ty, .normal);
	1124	break :ptr ptr;
	1125	} else operand;
	1126	const ret_val = try self.wip.load(.normal, llvm_ret_ty, ptr, ret_ty_align.toLlvm(), "");
	1127	_ = try self.wip.ret(ret_val);
	1128	},
1186	1129	}
1187	1130	}
1188	1131
...	...	@@ -1194,23 +1137,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1194	1137	const ptr_ty = self.typeOf(un_op);
1195	1138	const ret_ty = ptr_ty.childType(zcu);
1196	1139	const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1197		if (!ret_ty.hasRuntimeBits(zcu) or self.ret_ptr != .none) {
1198		_ = try self.wip.retVoid();
1199		return;
1200		}
1201	1140	const ptr = try self.resolveInst(un_op);
1202		const llvm_ret_ty = try o.lowerType(ret_ty);
1203		const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1204		if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {
1205		assert(!isByRef(ret_ty, zcu));
1206		// The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure we
1207		// load it from memory correctly.
1208		const loaded = try self.load(ptr, .none, ret_ty, .normal);
1209		_ = try self.wip.ret(loaded);
1210		} else {
1211		const ret_ty_align = ret_ty.abiAlignment(zcu);
1212		const loaded = try self.wip.load(.normal, abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1213		_ = try self.wip.ret(loaded);
	1141	switch (try fnReturnStrat(o, fn_info)) {
	1142	.void => _ = try self.wip.retVoid(),
	1143	.sret => {
	1144	assert(self.ret_ptr != .none);
	1145	_ = try self.wip.retVoid();
	1146	},
	1147	.by_val => {
	1148	assert(self.ret_ptr == .none);
	1149	const loaded = try self.load(ptr, .none, ret_ty, .normal);
	1150	_ = try self.wip.ret(loaded);
	1151	},
	1152	.mem_cast => \|llvm_abi_ret_ty\| {
	1153	assert(self.ret_ptr == .none);
	1154	const ret_ty_align = ret_ty.abiAlignment(zcu);
	1155	const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
	1156	_ = try self.wip.ret(loaded);
	1157	},
1214	1158	}
1215	1159	}
1216	1160
...	...	@@ -1218,7 +1162,7 @@ fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
1218	1162	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1219	1163	const list = try self.resolveInst(ty_op.operand);
1220	1164	const arg_ty = ty_op.ty.toType();
1221		const llvm_arg_ty = try self.object.lowerType(arg_ty);
	1165	const llvm_arg_ty = try self.object.lowerType(arg_ty, .by_value);
1222	1166
1223	1167	return self.wip.vaArg(list, llvm_arg_ty, "");
1224	1168	}
...	...	@@ -1229,10 +1173,8 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
1229	1173	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1230	1174	const src_list = try self.resolveInst(ty_op.operand);
1231	1175	const va_list_ty = ty_op.ty.toType();
1232		const llvm_va_list_ty = try o.lowerType(va_list_ty);
1233	1176
1234		const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
1235		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1177	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1236	1178
1237	1179	_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1238	1180	return if (isByRef(va_list_ty, zcu))
...	...	@@ -1253,10 +1195,8 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
1253	1195	const o = self.object;
1254	1196	const zcu = o.zcu;
1255	1197	const va_list_ty = self.typeOfIndex(inst);
1256		const llvm_va_list_ty = try o.lowerType(va_list_ty);
1257	1198
1258		const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();
1259		const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
	1199	const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1260	1200
1261	1201	_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1262	1202	return if (isByRef(va_list_ty, zcu))
...	...	@@ -1434,18 +1374,10 @@ fn lowerBlock(
1434	1374
1435	1375	// Create a phi node only if the block returns a value.
1436	1376	if (have_block_result) {
1437		const raw_llvm_ty = try o.lowerType(inst_ty);
1438		const llvm_ty: Builder.Type = ty: {
1439		// If the zig tag type is a function, this represents an actual function body; not
1440		// a pointer to it. LLVM IR allows the call instruction to use function bodies instead
1441		// of function pointers, however the phi makes it a runtime value and therefore
1442		// the LLVM type has to be wrapped in a pointer.
1443		if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
1444		break :ty .ptr;
1445		}
1446		break :ty raw_llvm_ty;
	1377	const llvm_ty: Builder.Type = switch (isByRef(inst_ty, zcu)) {
	1378	true => .ptr,
	1379	false => try o.lowerType(inst_ty, .by_value),
1447	1380	};
1448
1449	1381	parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1450	1382	const phi = try self.wip.phi(llvm_ty, "");
1451	1383	phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
...	...	@@ -1551,7 +1483,7 @@ fn lowerSwitchDispatch(
1551	1483	const table_index = try self.wip.conv(
1552	1484	.unsigned,
1553	1485	try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1554		try o.lowerType(.usize),
	1486	try o.lowerType(.usize, .by_value),
1555	1487	"",
1556	1488	);
1557	1489	const target_ptr_ptr = try self.ptraddScaled(
...	...	@@ -1576,7 +1508,7 @@ fn lowerSwitchDispatch(
1576	1508	// The switch prongs will correspond to our scalar cases. Ranges will
1577	1509	// be handled by conditional branches in the `else` prong.
1578	1510
1579		const llvm_usize = try o.lowerType(.usize);
	1511	const llvm_usize = try o.lowerType(.usize, .by_value);
1580	1512	const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
1581	1513	try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1582	1514	else
...	...	@@ -1771,7 +1703,6 @@ fn lowerTry(
1771	1703	const zcu = o.zcu;
1772	1704	const payload_ty = err_union_ty.errorUnionPayload(zcu);
1773	1705	const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1774		const error_type = try o.errorIntType();
1775	1706
1776	1707	const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
1777	1708
...	...	@@ -1792,7 +1723,7 @@ fn lowerTry(
1792	1723	if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1793	1724	);
1794	1725	};
1795		const zero = try o.builder.intValue(error_type, 0);
	1726	const zero = try o.builder.intValue(try o.errorIntType(.by_value), 0);
1796	1727	const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
1797	1728
1798	1729	const return_block = try fg.wip.block(1, "TryRet");
...	...	@@ -1929,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod
1929	1860	const table_includes_else = item_count != table_len;
1930	1861
1931	1862	break :jmp_table .{
1932		.min = try o.lowerValue(min.toIntern()),
1933		.max = try o.lowerValue(max.toIntern()),
	1863	.min = try o.lowerValue(min.toIntern(), .by_value),
	1864	.max = try o.lowerValue(max.toIntern(), .by_value),
1934	1865	.in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1935	1866	.none, .cold => .none,
1936	1867	.unpredictable => .unpredictable,
...	...	@@ -2088,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2088	2019	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2089	2020	const operand_ty = self.typeOf(ty_op.operand);
2090	2021	const array_ty = operand_ty.childType(zcu);
2091		const llvm_usize = try o.lowerType(.usize);
	2022	const llvm_usize = try o.lowerType(.usize, .by_value);
2092	2023	const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
2093		const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst));
	2024	const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst), .by_value);
2094	2025	const operand = try self.resolveInst(ty_op.operand);
2095	2026	return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
2096	2027	}
...	...	@@ -2107,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value
2107	2038
2108	2039	const dest_ty = self.typeOfIndex(inst);
2109	2040	const dest_scalar_ty = dest_ty.scalarType(zcu);
2110		const dest_llvm_ty = try o.lowerType(dest_ty);
	2041	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
2111	2042	const target = zcu.getTarget();
2112	2043
2113	2044	if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
...	...	@@ -2175,7 +2106,7 @@ fn airIntFromFloat(
2175	2106
2176	2107	const dest_ty = self.typeOfIndex(inst);
2177	2108	const dest_scalar_ty = dest_ty.scalarType(zcu);
2178		const dest_llvm_ty = try o.lowerType(dest_ty);
	2109	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
2179	2110
2180	2111	if (intrinsicsAllowed(operand_scalar_ty, target)) {
2181	2112	// TODO set fast math flag
...	...	@@ -2209,7 +2140,7 @@ fn airIntFromFloat(
2209	2140	compiler_rt_dest_abbrev,
2210	2141	});
2211	2142
2212		const operand_llvm_ty = try o.lowerType(operand_ty);
	2143	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
2213	2144	const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
2214	2145	var result = try self.wip.call(
2215	2146	.normal,
...	...	@@ -2234,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B
2234	2165	fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2235	2166	const o = fg.object;
2236	2167	const zcu = o.zcu;
2237		const llvm_usize = try o.lowerType(.usize);
	2168	const llvm_usize = try o.lowerType(.usize, .by_value);
2238	2169	switch (ty.ptrSize(zcu)) {
2239	2170	.slice => {
2240	2171	const len = try fg.wip.extractValue(ptr, &.{1}, "");
...	...	@@ -2370,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
2370	2301	const base_ptr = try self.resolveInst(bin_op.lhs);
2371	2302	const rhs = try self.resolveInst(bin_op.rhs);
2372	2303
2373		const elem_ptr = ty_pl.ty.toType();
2374		if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
2375
2376	2304	return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2377	2305	}
2378	2306
...	...	@@ -2435,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2435	2363	},
2436	2364	.float => {
2437	2365	// bitcast int->float
2438		return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), "");
	2366	return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty, .by_value), "");
2439	2367	},
2440	2368	}
2441	2369	}
...	...	@@ -2465,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2465	2393	const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2466	2394	if (field_offset == 0) return field_ptr;
2467	2395
2468		const res_ty = try o.lowerType(ty_pl.ty.toType());
2469		const llvm_usize = try o.lowerType(.usize);
	2396	const res_ty = try o.lowerType(ty_pl.ty.toType(), .by_value);
	2397	const llvm_usize = try o.lowerType(.usize, .by_value);
2470	2398
2471	2399	const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2472	2400	const base_ptr_int = try self.wip.bin(
...	...	@@ -2590,8 +2518,7 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
2590	2518	// We avoid taking this path for naked functions because there's no guarantee that such
2591	2519	// functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
2592	2520
2593		const alignment = operand_ty.abiAlignment(zcu).toLlvm();
2594		const alloca = try self.buildAlloca(try o.lowerType(operand_ty), alignment);
	2521	const alloca = try self.buildZigAlloca(operand_ty, .none);
2595	2522	try self.store(alloca, .none, operand, operand_ty, .normal);
2596	2523	_ = try self.wip.callIntrinsic(
2597	2524	.normal,
...	...	@@ -2683,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2683	2610	const output_inst = try self.resolveInst(output.operand);
2684	2611	const output_ty = self.typeOf(output.operand);
2685	2612	assert(output_ty.zigTypeTag(zcu) == .pointer);
2686		const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu));
	2613	const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu), .by_value);
2687	2614
2688	2615	switch (constraint[0]) {
2689	2616	'=' => {},
...	...	@@ -2721,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2721	2648	llvm_ret_indirect[output.index] = false;
2722	2649
2723	2650	const ret_ty = self.typeOfIndex(inst);
2724		llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty);
	2651	llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty, .by_value);
2725	2652	llvm_ret_i += 1;
2726	2653	}
2727	2654
...	...	@@ -2760,7 +2687,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2760	2687	llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2761	2688	} else {
2762	2689	const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2763		const arg_llvm_ty = try o.lowerType(arg_ty);
	2690	const arg_llvm_ty = try o.lowerType(arg_ty, .by_value);
2764	2691	const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2765	2692	llvm_param_values[llvm_param_i] = load_inst;
2766	2693	llvm_param_types[llvm_param_i] = arg_llvm_ty;
...	...	@@ -2800,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2800	2727	llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2801	2728	if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
2802	2729
2803		break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu));
	2730	break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu), .by_value);
2804	2731	} else .none;
2805	2732
2806	2733	llvm_param_i += 1;
...	...	@@ -2814,7 +2741,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2814	2741	if (constraint[0] != '+') continue;
2815	2742
2816	2743	const rw_ty = self.typeOf(output.operand);
2817		const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu));
	2744	const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu), .by_value);
2818	2745	if (llvm_ret_indirect[output.index]) {
2819	2746	llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2820	2747	llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
...	...	@@ -3028,7 +2955,7 @@ fn airIsNonNull(
3028	2955	));
3029	2956	return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
3030	2957	}
3031		return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty)), "");
	2958	return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty, .by_value)), "");
3032	2959	}
3033	2960
3034	2961	comptime assert(optional_layout_version == 3);
...	...	@@ -3057,8 +2984,7 @@ fn airIsErr(
3057	2984	const operand_ty = self.typeOf(un_op);
3058	2985	const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3059	2986	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3060		const error_type = try o.errorIntType();
3061		const zero = try o.builder.intValue(error_type, 0);
	2987	const zero_err = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3062	2988
3063	2989	const access_kind: Builder.MemoryAccessKind =
3064	2990	if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -3079,7 +3005,7 @@ fn airIsErr(
3079	3005	try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
3080	3006	else
3081	3007	operand;
3082		return self.wip.icmp(cond, loaded, zero, "");
	3008	return self.wip.icmp(cond, loaded, zero_err, "");
3083	3009	}
3084	3010	assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
3085	3011
...	...	@@ -3089,7 +3015,7 @@ fn airIsErr(
3089	3015	.none;
3090	3016	const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3091	3017	const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
3092		return self.wip.icmp(cond, loaded, zero, "");
	3018	return self.wip.icmp(cond, loaded, zero_err, "");
3093	3019	}
3094	3020
3095	3021	fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -3228,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3228	3154	const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
3229	3155
3230	3156	const payload_ty = err_union_ty.errorUnionPayload(zcu);
3231		const non_error_val = try o.builder.intValue(try o.errorIntType(), 0);
	3157	const non_error_val = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3232	3158
3233	3159	const access_kind: Builder.MemoryAccessKind =
3234	3160	if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -3274,33 +3200,9 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
3274	3200	return self.load(field_ptr, field_align, field_ty, .normal);
3275	3201	}
3276	3202
3277		/// As an optimization, we want to avoid unnecessary copies of
3278		/// error union/optional types when returning from a function.
3279		/// Here, we scan forward in the current block, looking to see
3280		/// if the next instruction is a return (ignoring debug instructions).
3281		///
3282		/// The first instruction of `body_tail` is a wrap instruction.
3283		fn isNextRet(
3284		self: *FuncGen,
3285		body_tail: []const Air.Inst.Index,
3286		) bool {
3287		const air_tags = self.air.instructions.items(.tag);
3288		for (body_tail[1..]) \|body_inst\| {
3289		switch (air_tags[@intFromEnum(body_inst)]) {
3290		.ret => return true,
3291		.dbg_stmt => continue,
3292		else => return false,
3293		}
3294		}
3295		// The only way to get here is to hit the end of a loop instruction
3296		// (implicit repeat).
3297		return false;
3298		}
3299
3300		fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3203	fn airWrapOptional(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3301	3204	const o = self.object;
3302	3205	const zcu = o.zcu;
3303		const inst = body_tail[0];
3304	3206	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3305	3207	const payload_ty = self.typeOf(ty_op.operand);
3306	3208	comptime assert(optional_layout_version == 3);
...	...	@@ -3309,14 +3211,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3309	3211	const optional_ty = self.typeOfIndex(inst);
3310	3212	if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3311	3213	assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3312		const llvm_optional_ty = try o.lowerType(optional_ty);
3313		const optional_ptr = if (self.isNextRet(body_tail))
3314		self.ret_ptr
3315		else brk: {
3316		const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3317		const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3318		break :brk optional_ptr;
3319		};
	3214	const optional_ptr = try self.buildZigAlloca(optional_ty, .none);
3320	3215
3321	3216	const payload_ptr = optional_ptr; // payload always at offset 0
3322	3217	try self.store(payload_ptr, .none, operand, payload_ty, .normal);
...	...	@@ -3328,26 +3223,18 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3328	3223	return optional_ptr;
3329	3224	}
3330	3225
3331		fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3226	fn airWrapErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3332	3227	const o = self.object;
3333	3228	const zcu = o.zcu;
3334		const inst = body_tail[0];
3335	3229	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3336	3230	const err_un_ty = self.typeOfIndex(inst);
3337	3231	const operand = try self.resolveInst(ty_op.operand);
3338	3232	const payload_ty = self.typeOf(ty_op.operand);
3339	3233	assert(payload_ty.hasRuntimeBits(zcu));
3340	3234	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3341		const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0);
3342		const err_un_llvm_ty = try o.lowerType(err_un_ty);
3343
3344		const result_ptr = if (self.isNextRet(body_tail))
3345		self.ret_ptr
3346		else brk: {
3347		const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm();
3348		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3349		break :brk result_ptr;
3350		};
	3235	const ok_err_code = try o.builder.intValue(try o.errorIntType(.by_value), 0);
	3236
	3237	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3351	3238
3352	3239	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3353	3240	try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
...	...	@@ -3358,25 +3245,17 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All
3358	3245	return result_ptr;
3359	3246	}
3360	3247
3361		fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
	3248	fn airWrapErrUnionErr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3362	3249	const o = self.object;
3363	3250	const zcu = o.zcu;
3364		const inst = body_tail[0];
3365	3251	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3366	3252	const err_un_ty = self.typeOfIndex(inst);
3367	3253	const payload_ty = err_un_ty.errorUnionPayload(zcu);
3368	3254	const operand = try self.resolveInst(ty_op.operand);
3369	3255	if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3370	3256	assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3371		const err_un_llvm_ty = try o.lowerType(err_un_ty);
3372
3373		const result_ptr = if (self.isNextRet(body_tail))
3374		self.ret_ptr
3375		else brk: {
3376		const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3377		const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3378		break :brk result_ptr;
3379		};
	3257
	3258	const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3380	3259
3381	3260	const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3382	3261	try self.store(err_ptr, .none, operand, .anyerror, .normal);
...	...	@@ -3392,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3392	3271	const o = self.object;
3393	3272	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3394	3273	const index = pl_op.payload;
3395		const llvm_usize = try o.lowerType(.usize);
	3274	const llvm_usize = try o.lowerType(.usize, .by_value);
3396	3275	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3397	3276	try o.builder.intValue(.i32, index),
3398	3277	}, "");
...	...	@@ -3402,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3402	3281	const o = self.object;
3403	3282	const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3404	3283	const index = pl_op.payload;
3405		const llvm_isize = try o.lowerType(.isize);
	3284	const llvm_isize = try o.lowerType(.isize, .by_value);
3406	3285	return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3407	3286	try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3408	3287	}, "");
...	...	@@ -3429,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3429	3308	.normal,
3430	3309	.none,
3431	3310	if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3432		&.{try o.lowerType(inst_ty)},
	3311	&.{try o.lowerType(inst_ty, .by_value)},
3433	3312	&.{ lhs, rhs },
3434	3313	"",
3435	3314	);
...	...	@@ -3449,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3449	3328	.normal,
3450	3329	.none,
3451	3330	if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3452		&.{try o.lowerType(inst_ty)},
	3331	&.{try o.lowerType(inst_ty, .by_value)},
3453	3332	&.{ lhs, rhs },
3454	3333	"",
3455	3334	);
...	...	@@ -3461,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3461	3340	const ptr = try self.resolveInst(bin_op.lhs);
3462	3341	const len = try self.resolveInst(bin_op.rhs);
3463	3342	const inst_ty = self.typeOfIndex(inst);
3464		return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, "");
	3343	return self.wip.buildAggregate(try self.object.lowerType(inst_ty, .by_value), &.{ ptr, len }, "");
3465	3344	}
3466	3345
3467	3346	fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...	...	@@ -3492,7 +3371,7 @@ fn airSafeArithmetic(
3492	3371	const scalar_ty = inst_ty.scalarType(zcu);
3493	3372
3494	3373	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3495		const llvm_inst_ty = try o.lowerType(inst_ty);
	3374	const llvm_inst_ty = try o.lowerType(inst_ty, .by_value);
3496	3375	const results =
3497	3376	try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
3498	3377
...	...	@@ -3542,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3542	3421	.normal,
3543	3422	.none,
3544	3423	if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3545		&.{try o.lowerType(inst_ty)},
	3424	&.{try o.lowerType(inst_ty, .by_value)},
3546	3425	&.{ lhs, rhs },
3547	3426	"",
3548	3427	);
...	...	@@ -3581,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3581	3460	.normal,
3582	3461	.none,
3583	3462	if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3584		&.{try o.lowerType(inst_ty)},
	3463	&.{try o.lowerType(inst_ty, .by_value)},
3585	3464	&.{ lhs, rhs },
3586	3465	"",
3587	3466	);
...	...	@@ -3620,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3620	3499	.normal,
3621	3500	.none,
3622	3501	if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3623		&.{try o.lowerType(inst_ty)},
	3502	&.{try o.lowerType(inst_ty, .by_value)},
3624	3503	&.{ lhs, rhs, .@"0" },
3625	3504	"",
3626	3505	);
...	...	@@ -3664,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)
3664	3543	return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3665	3544	}
3666	3545	if (scalar_ty.isSignedInt(zcu)) {
3667		const scalar_llvm_ty = try o.lowerType(scalar_ty);
3668		const inst_llvm_ty = try o.lowerType(inst_ty);
	3546	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
	3547	const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3669	3548
3670	3549	const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3671	3550	var bfa_buf: ExpectedContents = undefined;
...	...	@@ -3739,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3739	3618	const lhs = try self.resolveInst(bin_op.lhs);
3740	3619	const rhs = try self.resolveInst(bin_op.rhs);
3741	3620	const inst_ty = self.typeOfIndex(inst);
3742		const inst_llvm_ty = try o.lowerType(inst_ty);
	3621	const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3743	3622	const scalar_ty = inst_ty.scalarType(zcu);
3744	3623
3745	3624	if (scalar_ty.isRuntimeFloat()) {
...	...	@@ -3768,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3768	3647	defer allocator.free(smin_big_int.limbs);
3769	3648	smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3770	3649	const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3771		try o.lowerType(scalar_ty),
	3650	try o.lowerType(scalar_ty, .by_value),
3772	3651	smin_big_int.toConst(),
3773	3652	));
3774	3653
...	...	@@ -3804,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3804	3683	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3805	3684	const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3806	3685	const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3807		const llvm_usize_ty = try o.lowerType(.usize);
	3686	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
3808	3687	const ptr_ty = self.typeOf(bin_op.lhs);
3809	3688	const elem_ty = ptr_ty.indexableElem(zcu);
3810	3689	const ptr = switch (ptr_ty.ptrSize(zcu)) {
...	...	@@ -3837,8 +3716,7 @@ fn airOverflow(
3837	3716	assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
3838	3717
3839	3718	const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3840		const llvm_inst_ty = try o.lowerType(inst_ty);
3841		const llvm_lhs_ty = try o.lowerType(lhs_ty);
	3719	const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
3842	3720	const results =
3843	3721	try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
3844	3722
...	...	@@ -3846,7 +3724,7 @@ fn airOverflow(
3846	3724	const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
3847	3725
3848	3726	const result_alignment = inst_ty.abiAlignment(zcu);
3849		const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment.toLlvm());
	3727	const alloca_inst = try self.buildZigAlloca(inst_ty, .none);
3850	3728
3851	3729	{
3852	3730	// Store to 'result: IntType' field
...	...	@@ -3911,7 +3789,7 @@ fn buildFloatCmp(
3911	3789	const zcu = o.zcu;
3912	3790	const target = zcu.getTarget();
3913	3791	const scalar_ty = ty.scalarType(zcu);
3914		const scalar_llvm_ty = try o.lowerType(scalar_ty);
	3792	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
3915	3793
3916	3794	if (intrinsicsAllowed(scalar_ty, target)) {
3917	3795	const cond: Builder.FloatCondition = switch (pred) {
...	...	@@ -4017,7 +3895,7 @@ fn buildFloatOp(
4017	3895	const zcu = o.zcu;
4018	3896	const target = zcu.getTarget();
4019	3897	const scalar_ty = ty.scalarType(zcu);
4020		const llvm_ty = try o.lowerType(ty);
	3898	const llvm_ty = try o.lowerType(ty, .by_value);
4021	3899
4022	3900	if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
4023	3901	// Some operations are dedicated LLVM instructions, not available as intrinsics
...	...	@@ -4122,7 +4000,7 @@ fn buildFloatOp(
4122	4000	}),
4123	4001	};
4124	4002
4125		const scalar_llvm_ty = try o.lowerType(scalar_ty);
	4003	const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
4126	4004	const libc_fn = try o.getLibcFunction(
4127	4005	fn_name,
4128	4006	@as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],
...	...	@@ -4176,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
4176	4054
4177	4055	const dest_ty = self.typeOfIndex(inst);
4178	4056	assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4179		const llvm_dest_ty = try o.lowerType(dest_ty);
4180	4057
4181		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4058	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4182	4059
4183	4060	const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4184	4061	const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
...	...	@@ -4189,7 +4066,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
4189	4066	const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
4190	4067
4191	4068	const result_alignment = dest_ty.abiAlignment(zcu);
4192		const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment.toLlvm());
	4069	const alloca_inst = try self.buildZigAlloca(dest_ty, .none);
4193	4070
4194	4071	{
4195	4072	// Store to 'result: IntType' field
...	...	@@ -4245,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
4245	4122	}
4246	4123	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4247	4124
4248		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4125	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4249	4126	return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4250	4127	.@"shl nsw"
4251	4128	else
...	...	@@ -4266,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4266	4143	// features which we do not use. Therefore this branch is currently impossible.
4267	4144	unreachable;
4268	4145	}
4269		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4146	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4270	4147	return self.wip.bin(.shl, lhs, casted_rhs, "");
4271	4148	}
4272	4149
...	...	@@ -4280,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4280	4157
4281	4158	const lhs_ty = self.typeOf(bin_op.lhs);
4282	4159	const lhs_info = lhs_ty.intInfo(zcu);
4283		const llvm_lhs_ty = try o.lowerType(lhs_ty);
4284		const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu));
	4160	const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
	4161	const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu), .by_value);
4285	4162
4286	4163	const rhs_ty = self.typeOf(bin_op.rhs);
4287	4164	if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
...	...	@@ -4291,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4291	4168	}
4292	4169	const rhs_info = rhs_ty.intInfo(zcu);
4293	4170	assert(rhs_info.signedness == .unsigned);
4294		const llvm_rhs_ty = try o.lowerType(rhs_ty);
4295		const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu));
	4171	const llvm_rhs_ty = try o.lowerType(rhs_ty, .by_value);
	4172	const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu), .by_value);
4296	4173
4297	4174	const result = try self.wip.callIntrinsic(
4298	4175	.normal,
...	...	@@ -4368,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!
4368	4245	}
4369	4246	const lhs_scalar_ty = lhs_ty.scalarType(zcu);
4370	4247
4371		const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");
	4248	const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4372	4249	const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
4373	4250
4374	4251	return self.wip.bin(if (is_exact)
...	...	@@ -4389,7 +4266,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4389	4266	.normal,
4390	4267	.none,
4391	4268	.abs,
4392		&.{try o.lowerType(operand_ty)},
	4269	&.{try o.lowerType(operand_ty, .by_value)},
4393	4270	&.{ operand, .false },
4394	4271	"",
4395	4272	),
...	...	@@ -4403,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4403	4280	const zcu = o.zcu;
4404	4281	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4405	4282	const dest_ty = fg.typeOfIndex(inst);
4406		const dest_llvm_ty = try o.lowerType(dest_ty);
	4283	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4407	4284	const operand = try fg.resolveInst(ty_op.operand);
4408	4285	const operand_ty = fg.typeOf(ty_op.operand);
4409	4286	const operand_info = operand_ty.intInfo(zcu);
...	...	@@ -4431,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4431	4308
4432	4309	if (!have_min_check and !have_max_check) break :bounds_check;
4433	4310
4434		const operand_llvm_ty = try o.lowerType(operand_ty);
4435		const operand_scalar_llvm_ty = try o.lowerType(operand_scalar);
	4311	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4312	const operand_scalar_llvm_ty = try o.lowerType(operand_scalar, .by_value);
4436	4313
4437	4314	const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4438	4315	assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
...	...	@@ -4510,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4510	4387	fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4511	4388	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4512	4389	const operand = try self.resolveInst(ty_op.operand);
4513		const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst));
	4390	const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst), .by_value);
4514	4391	return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4515	4392	}
4516	4393
...	...	@@ -4524,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4524	4401	const target = zcu.getTarget();
4525	4402
4526	4403	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4527		return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), "");
	4404	return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty, .by_value), "");
4528	4405	} else {
4529		const operand_llvm_ty = try o.lowerType(operand_ty);
4530		const dest_llvm_ty = try o.lowerType(dest_ty);
	4406	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4407	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4531	4408
4532	4409	const dest_bits = dest_ty.floatBits(target);
4533	4410	const src_bits = operand_ty.floatBits(target);
...	...	@@ -4558,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4558	4435	const target = zcu.getTarget();
4559	4436
4560	4437	if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4561		return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");
	4438	return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty, .by_value), "");
4562	4439	} else {
4563		const operand_llvm_ty = try o.lowerType(operand_ty);
4564		const dest_llvm_ty = try o.lowerType(dest_ty);
	4440	const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
	4441	const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4565	4442
4566	4443	const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
4567	4444	const src_bits = operand_ty.scalarType(zcu).floatBits(target);
...	...	@@ -4599,39 +4476,69 @@ fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4599	4476
4600	4477	// We have the following `Air.Legalize` features enabled:
4601	4478	//
4602		// * `.scalarize_bitcast_array`
4603		// * `.scalarize_bitcast_vector_non_elementwise`
	4479	// * `.scalarize_bit_cast_array`
	4480	// * `.scalarize_bit_cast_vector_non_elementwise`
4604	4481	//
4605		// That means the set of bitcasts we might see is limited to the following:
	4482	// That means the `bit_cast` instructions we might see are limited to the following:
4606	4483	//
4607	4484	// * bool/int/float <-> bool/int/float
4608	4485	// * `@Vector(n, A)` <-> `@Vector(n, B)`
4609		// * pointer <-> pointer
4610		// * pointer <-> int
4611		// * slice <-> slice
4612	4486	//
4613		// Most of these can be handled by LLVM's `bitcast` instruction. We will check for the few cases
4614		// that aren't, and otherwise use `bitcast`.
4615
4616		if (operand_ty.isSlice(zcu) and dest_ty.isSlice(zcu)) {
4617		// The slice types are the same type in LLVM IR, so this conversion is a nop.
4618		return operand;
4619		}
	4487	// All of these cases can be handled by LLVM's `bitcast` instruction.
4620	4488
4621	4489	assert(!isByRef(operand_ty, zcu));
4622	4490	assert(!isByRef(dest_ty, zcu));
4623	4491
4624		const llvm_dest_ty = try o.lowerType(dest_ty);
	4492	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4493	return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
	4494	}
4625	4495
4626		if (operand_ty.scalarType(zcu).zigTypeTag(zcu) == .int and dest_ty.scalarType(zcu).isPtrAtRuntime(zcu)) {
4627		return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4628		}
	4496	fn airNopCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4497	const zcu = fg.object.zcu;
	4498	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4499	const operand_ty = fg.typeOf(ty_op.operand);
	4500	const dest_ty = fg.typeOfIndex(inst);
	4501	assert(isByRef(operand_ty, zcu) == isByRef(dest_ty, zcu));
	4502	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	4503	return fg.resolveInst(ty_op.operand);
	4504	}
4629	4505
4630		if (operand_ty.scalarType(zcu).isPtrAtRuntime(zcu) and dest_ty.scalarType(zcu).zigTypeTag(zcu) == .int) {
4631		return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4632		}
	4506	fn airPtrFromInt(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4507	const o = fg.object;
	4508	const zcu = o.zcu;
	4509	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4510	const operand_ty = fg.typeOf(ty_op.operand);
	4511	const dest_ty = fg.typeOfIndex(inst);
	4512	assert(operand_ty.scalarType(zcu).toIntern() == .usize_type);
	4513	assert(dest_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4633	4514
4634		return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
	4515	const operand = try fg.resolveInst(ty_op.operand);
	4516	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4517	return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
	4518	}
	4519
	4520	fn airIntFromPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4521	const o = fg.object;
	4522	const zcu = o.zcu;
	4523	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4524	const operand_ty = fg.typeOf(ty_op.operand);
	4525	const dest_ty = fg.typeOfIndex(inst);
	4526	assert(operand_ty.scalarType(zcu).isPtrAtRuntime(zcu));
	4527	assert(dest_ty.scalarType(zcu).toIntern() == .usize_type);
	4528
	4529	const operand = try fg.resolveInst(ty_op.operand);
	4530	const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
	4531	return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
	4532	}
	4533
	4534	fn airUnionFromEnum(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
	4535	const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	4536	const enum_ty = fg.typeOf(ty_op.operand);
	4537	const union_ty = fg.typeOfIndex(inst);
	4538	const enum_val = try fg.resolveInst(ty_op.operand);
	4539	const union_ptr = try fg.buildZigAlloca(union_ty, .none);
	4540	try fg.store(union_ptr, .none, enum_val, enum_ty, .normal);
	4541	return union_ptr;
4635	4542	}
4636	4543
4637	4544	fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -4690,8 +4597,7 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4690	4597	"",
4691	4598	);
4692	4599	} else if (mod.optimize_mode == .Debug) {
4693		const alignment = inst_ty.abiAlignment(zcu).toLlvm();
4694		const alloca = try self.buildAlloca(try o.lowerType(inst_ty), alignment);
	4600	const alloca = try self.buildZigAlloca(inst_ty, .none);
4695	4601	try self.store(alloca, .none, arg_val, inst_ty, .normal);
4696	4602	_ = try self.wip.callIntrinsic(
4697	4603	.normal,
...	...	@@ -4733,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4733	4639	if (!elem_ty.hasRuntimeBits(zcu)) {
4734	4640	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4735	4641	}
4736		const llvm_elem_ty = try o.lowerType(elem_ty);
4737		return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4642	return self.buildZigAlloca(elem_ty, ptr_align);
4738	4643	}
4739	4644
4740	4645	fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -4747,8 +4652,19 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4747	4652	if (!elem_ty.hasRuntimeBits(zcu)) {
4748	4653	return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4749	4654	}
4750		const llvm_elem_ty = try o.lowerType(elem_ty);
4751		return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
	4655	return self.buildZigAlloca(elem_ty, ptr_align);
	4656	}
	4657
	4658	fn buildZigAlloca(fg: *FuncGen, ty: Type, @"align": InternPool.Alignment) Allocator.Error!Builder.Value {
	4659	const o = fg.object;
	4660	const resolved_align: InternPool.Alignment = switch (@"align") {
	4661	.none => ty.abiAlignment(o.zcu),
	4662	else => \|a\| a,
	4663	};
	4664	return fg.buildAlloca(
	4665	try o.lowerType(ty, .in_memory),
	4666	resolved_align.toLlvm(),
	4667	);
4752	4668	}
4753	4669
4754	4670	/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
...	...	@@ -4823,7 +4739,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu
4823	4739	return .none;
4824	4740	}
4825	4741
4826		const len = try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu));
	4742	const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), elem_ty.abiSize(zcu));
4827	4743	_ = try fg.wip.callMemSet(
4828	4744	ptr,
4829	4745	ptr_alignment.toLlvm(),
...	...	@@ -4858,7 +4774,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu
4858	4774	if (ptr_info.packed_offset.host_size != 0) {
4859	4775	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4860	4776	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4861		const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
	4777	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
4862	4778
4863	4779	const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
4864	4780
...	...	@@ -4936,14 +4852,14 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4936	4852
4937	4853	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4938	4854	const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4939		const llvm_backing_int_ty = try o.lowerType(backing_int_ty);
	4855	const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
4940	4856
4941	4857	const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
4942	4858
4943	4859	const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4944	4860	const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4945	4861	const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
4946		const elem_llvm_ty = try o.lowerType(elem_ty);
	4862	const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
4947	4863
4948	4864	if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
4949	4865	const same_size_int = try o.builder.intType(@intCast(elem_bits));
...	...	@@ -4993,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
4993	4909	fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4994	4910	_ = inst;
4995	4911	const o = self.object;
4996		const llvm_usize = try o.lowerType(.usize);
	4912	const llvm_usize = try o.lowerType(.usize, .by_value);
4997	4913	if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
4998	4914	// https://github.com/ziglang/zig/issues/11946
4999	4915	return o.builder.intValue(llvm_usize, 0);
...	...	@@ -5005,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
5005	4921	fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5006	4922	_ = inst;
5007	4923	const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
5008		return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), "");
	4924	return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize, .by_value), "");
5009	4925	}
5010	4926
5011	4927	fn airCmpxchg(
...	...	@@ -5022,7 +4938,7 @@ fn airCmpxchg(
5022	4938	var expected_value = try self.resolveInst(extra.expected_value);
5023	4939	var new_value = try self.resolveInst(extra.new_value);
5024	4940	const operand_ty = ptr_ty.childType(zcu);
5025		const llvm_operand_ty = try o.lowerType(operand_ty);
	4941	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5026	4942	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5027	4943	if (llvm_abi_ty != .none) {
5028	4944	// operand needs widening and truncating
...	...	@@ -5066,7 +4982,7 @@ fn airCmpxchg(
5066	4982	const non_null_bit = try self.wip.not(success_bit, "");
5067	4983
5068	4984	const payload_align = operand_ty.abiAlignment(zcu);
5069		const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align.toLlvm());
	4985	const alloca_inst = try self.buildZigAlloca(optional_ty, .none);
5070	4986
5071	4987	// Payload is always the first field at offset 0, so address is `alloca_inst`
5072	4988	try self.store(alloca_inst, .none, payload, operand_ty, .normal);
...	...	@@ -5092,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5092	5008	const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
5093	5009	const ordering = toLlvmAtomicOrdering(extra.ordering());
5094	5010	const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
5095		const llvm_operand_ty = try o.lowerType(operand_ty);
	5011	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5096	5012
5097	5013	const access_kind: Builder.MemoryAccessKind =
5098	5014	if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...	...	@@ -5121,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5121	5037
5122	5038	// If we are storing a pointer we need to convert to and from a plain old integer.
5123	5039	const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
5124		.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""),
	5040	.pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize, .by_value), ""),
5125	5041	else => operand,
5126	5042	};
5127	5043
...	...	@@ -5160,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
5160	5076	Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5161	5077	const access_kind: Builder.MemoryAccessKind =
5162	5078	if (info.flags.is_volatile) .@"volatile" else .normal;
5163		const elem_llvm_ty = try o.lowerType(elem_ty);
	5079	const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
5164	5080
5165	5081	self.maybeMarkAllowZeroAccess(info);
5166	5082
...	...	@@ -5306,10 +5222,13 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5306	5222	}
5307	5223	if (elem_ty.isAbiInt(zcu)) {
5308	5224	const info = elem_ty.intInfo(zcu);
5309		break :byte self.wip.conv(info.signedness, value, .i8, "");
	5225	break :byte try self.wip.conv(switch (info.signedness) {
	5226	.unsigned => .unsigned,
	5227	.signed => .signed,
	5228	}, value, .i8, "");
5310	5229	}
5311		if (elem_ty == .bool) {
5312		break :byte self.wip.cast(.zext, value, .i8, "");
	5230	if (elem_ty.toIntern() == .bool_type) {
	5231	break :byte try self.wip.cast(.zext, value, .i8, "");
5313	5232	}
5314	5233	break :intrinsic;
5315	5234	};
...	...	@@ -5459,15 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5459	5378	const layout = un_ty.unionGetLayout(zcu);
5460	5379	assert(layout.tag_size != 0);
5461	5380	const operand = try self.resolveInst(ty_op.operand);
5462		if (isByRef(un_ty, zcu)) {
5463		const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5464		return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5465		} else {
5466		// This is only possible if all fields are zero-bit, in which case `operand` is already an
5467		// integer value (the union is lowered as its enum tag).
5468		assert(layout.payload_size == 0);
5469		return operand;
5470		}
	5381	assert(isByRef(un_ty, zcu));
	5382	const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
	5383	return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5471	5384	}
5472	5385
5473	5386	fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
...	...	@@ -5497,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5497	5410	.normal,
5498	5411	.none,
5499	5412	intrinsic,
5500		&.{try o.lowerType(operand_ty)},
	5413	&.{try o.lowerType(operand_ty, .by_value)},
5501	5414	&.{ operand, .false },
5502	5415	"",
5503	5416	);
5504		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5417	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5505	5418	}
5506	5419
5507	5420	fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
...	...	@@ -5515,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5515	5428	.normal,
5516	5429	.none,
5517	5430	intrinsic,
5518		&.{try o.lowerType(operand_ty)},
	5431	&.{try o.lowerType(operand_ty, .by_value)},
5519	5432	&.{operand},
5520	5433	"",
5521	5434	);
5522		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5435	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5523	5436	}
5524	5437
5525	5438	fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5532,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5532	5445
5533	5446	const inst_ty = self.typeOfIndex(inst);
5534	5447	var operand = try self.resolveInst(ty_op.operand);
5535		var llvm_operand_ty = try o.lowerType(operand_ty);
	5448	var llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5536	5449
5537	5450	if (bits % 16 == 8) {
5538	5451	// If not an even byte-multiple, we need zero-extend + shift-left 1 byte
...	...	@@ -5553,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5553	5466
5554	5467	const result =
5555	5468	try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5556		return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");
	5469	return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5557	5470	}
5558	5471
5559	5472	fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5573,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui
5573	5486
5574	5487	for (0..names.len) \|name_index\| {
5575	5488	const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5576		const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int);
	5489	const this_tag_int_value = try o.builder.intConst(try o.errorIntType(.by_value), err_int);
5577	5490	try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5578	5491	}
5579	5492	self.wip.cursor = .{ .block = valid_block };
...	...	@@ -5632,7 +5545,7 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5632	5545	const slice_ty = self.typeOfIndex(inst);
5633	5546
5634	5547	// If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5635		const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");
	5548	const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize, .by_value), "");
5636	5549
5637	5550	const error_name_table_ptr = try o.getErrorNameTable();
5638	5551	const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
...	...	@@ -5643,7 +5556,7 @@ fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
5643	5556	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5644	5557	const scalar = try self.resolveInst(ty_op.operand);
5645	5558	const vector_ty = self.typeOfIndex(inst);
5646		return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, "");
	5559	return self.wip.splatVector(try self.object.lowerType(vector_ty, .by_value), scalar, "");
5647	5560	}
5648	5561
5649	5562	fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...	...	@@ -5666,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5666	5579	const operand = try fg.resolveInst(unwrapped.operand);
5667	5580	const mask = unwrapped.mask;
5668	5581	const operand_ty = fg.typeOf(unwrapped.operand);
5669		const llvm_operand_ty = try o.lowerType(operand_ty);
5670		const llvm_result_ty = try o.lowerType(unwrapped.result_ty);
5671		const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5582	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
	5583	const llvm_result_ty = try o.lowerType(unwrapped.result_ty, .by_value);
	5584	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5672	5585	const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
5673	5586	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5674	5587	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
...	...	@@ -5698,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5698	5611	.elem => llvm_poison_elem,
5699	5612	.value => \|val\| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
5700	5613	any_defined_comptime_value = true;
5701		break :elem try o.lowerValue(val);
	5614	break :elem try o.lowerValue(val, .by_value);
5702	5615	} else llvm_poison_elem,
5703	5616	};
5704	5617	}
...	...	@@ -5770,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5770	5683	const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
5771	5684
5772	5685	const mask = unwrapped.mask;
5773		const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));
	5686	const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5774	5687	const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5775	5688	const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5776	5689
...	...	@@ -5860,7 +5773,7 @@ fn buildReducedCall(
5860	5773	accum_init: Builder.Value,
5861	5774	) Allocator.Error!Builder.Value {
5862	5775	const o = self.object;
5863		const llvm_usize_ty = try o.lowerType(.usize);
	5776	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
5864	5777	const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
5865	5778	const llvm_result_ty = accum_init.typeOfWip(&self.wip);
5866	5779
...	...	@@ -5907,7 +5820,7 @@ fn buildReducedCall(
5907	5820	accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
5908	5821
5909	5822	self.wip.cursor = .{ .block = exit_block };
5910		return new_accum;
	5823	return accum.toValue();
5911	5824	}
5912	5825
5913	5826	fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...	...	@@ -5918,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A
5918	5831	const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
5919	5832	const operand = try self.resolveInst(reduce.operand);
5920	5833	const operand_ty = self.typeOf(reduce.operand);
5921		const llvm_operand_ty = try o.lowerType(operand_ty);
	5834	const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5922	5835	const scalar_ty = self.typeOfIndex(inst);
5923		const llvm_scalar_ty = try o.lowerType(scalar_ty);
	5836	const llvm_scalar_ty = try o.lowerType(scalar_ty, .by_value);
5924	5837
5925	5838	switch (reduce.operation) {
5926	5839	.And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
...	...	@@ -6027,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6027	5940	const result_ty = self.typeOfIndex(inst);
6028	5941	const len: usize = @intCast(result_ty.arrayLen(zcu));
6029	5942	const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
6030		const llvm_result_ty = try o.lowerType(result_ty);
6031	5943
6032	5944	switch (result_ty.zigTypeTag(zcu)) {
6033	5945	.vector => {
	5946	const llvm_result_ty = try o.lowerType(result_ty, .by_value);
6034	5947	var vector = try o.builder.poisonValue(llvm_result_ty);
6035	5948	for (elements, 0..) \|elem, i\| {
6036	5949	const index_u32 = try o.builder.intValue(.i32, i);
...	...	@@ -6072,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6072	5985	// TODO in debug builds init to undef so that the padding will be 0xaa
6073	5986	// even if we fully populate the fields.
6074	5987	const struct_align = result_ty.abiAlignment(zcu);
6075		const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());
	5988	const alloca_inst = try self.buildZigAlloca(result_ty, .none);
6076	5989
6077	5990	for (elements, 0..) \|elem, field_index\| {
6078	5991	if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
...	...	@@ -6093,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6093	6006	.array => {
6094	6007	assert(isByRef(result_ty, zcu));
6095	6008
6096		const alignment = result_ty.abiAlignment(zcu).toLlvm();
6097		const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
	6009	const alloca_inst = try self.buildZigAlloca(result_ty, .none);
6098	6010
6099	6011	const array_info = result_ty.arrayInfo(zcu);
6100	6012
...	...	@@ -6124,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6124	6036	const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6125	6037	const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6126	6038	const union_ty = self.typeOfIndex(inst);
6127		const union_llvm_ty = try o.lowerType(union_ty);
6128	6039	const union_obj = zcu.typeToUnion(union_ty).?;
6129	6040
6130	6041	assert(union_obj.layout != .@"packed");
...	...	@@ -6134,8 +6045,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6134	6045	assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6135	6046	assert(isByRef(union_ty, zcu));
6136	6047
6137		const alignment = layout.abi_align.toLlvm();
6138		const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
	6048	const result_ptr = try self.buildZigAlloca(union_ty, layout.abi_align);
6139	6049	const llvm_payload = try self.resolveInst(extra.init);
6140	6050	const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6141	6051	assert(field_ty.hasRuntimeBits(zcu));
...	...	@@ -6150,10 +6060,10 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6150	6060	const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6151	6061	const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6152	6062	.none => try o.builder.intConst(
6153		try o.lowerType(.fromInterned(union_obj.enum_tag_type)),
	6063	try o.lowerType(.fromInterned(union_obj.enum_tag_type), .by_value),
6154	6064	extra.field_index, // auto-numbered
6155	6065	),
6156		else => \|tag_val_ip\| try o.lowerValue(tag_val_ip),
	6066	else => \|tag_val_ip\| try o.lowerValue(tag_val_ip, .by_value),
6157	6067	};
6158	6068	const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6159	6069	try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
...	...	@@ -6215,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6215	6125	const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6216	6126	const inst_ty = self.typeOfIndex(inst);
6217	6127	const operand = try self.resolveInst(ty_op.operand);
6218		return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), "");
	6128	return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty, .by_value), "");
6219	6129	}
6220	6130
6221	6131	fn workIntrinsic(
...	...	@@ -6361,15 +6271,14 @@ fn load(
6361	6271	const abi_align = load_ty.abiAlignment(zcu);
6362	6272	const abi_size = load_ty.abiSize(zcu);
6363	6273
6364		const llvm_load_ty = try o.lowerType(load_ty);
6365	6274	const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6366	6275	.none => abi_align.toLlvm(),
6367	6276	else => \|a\| a.toLlvm(),
6368	6277	};
6369	6278
6370	6279	if (isByRef(load_ty, zcu)) {
6371		const llvm_usize_ty = try o.lowerType(.usize);
6372		const result_ptr = try fg.buildAlloca(llvm_load_ty, abi_align.toLlvm());
	6280	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
	6281	const result_ptr = try fg.buildZigAlloca(load_ty, .none);
6373	6282	_ = try fg.wip.callMemCpy(
6374	6283	result_ptr,
6375	6284	abi_align.toLlvm(),
...	...	@@ -6382,7 +6291,11 @@ fn load(
6382	6291	return result_ptr;
6383	6292	}
6384	6293
6385		if (load_ty.isAbiInt(zcu) and load_ty.bitSize(zcu) != abi_size * 8) {
	6294	const llvm_memory_ty = try o.lowerType(load_ty, .in_memory);
	6295	const llvm_value_ty = try o.lowerType(load_ty, .by_value);
	6296
	6297	if (llvm_memory_ty != llvm_value_ty) {
	6298	assert(load_ty.isAbiInt(zcu));
6386	6299	// `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6387	6300	// handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6388	6301	// semantics. However:
...	...	@@ -6395,8 +6308,7 @@ fn load(
6395	6308	//
6396	6309	// Therefore, we handle these memory accesses specially: in this case we will actually load
6397	6310	// the next-largest "natural" integer type and then truncate to `load_ty`.
6398		const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6399		const loaded = try fg.wip.load(access_kind, llvm_abi_ty, ptr, llvm_ptr_align, "");
	6311	const loaded = try fg.wip.load(access_kind, llvm_memory_ty, ptr, llvm_ptr_align, "");
6400	6312	// For packed structs, current Zig semantics don't really allow us to make the padding bits
6401	6313	// well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6402	6314	// implemented, but until then, do a normal trunc for packed types.
...	...	@@ -6406,11 +6318,11 @@ fn load(
6406	6318	.unsigned => .@"trunc nuw",
6407	6319	.signed => .@"trunc nsw",
6408	6320	},
6409		}, loaded, llvm_load_ty, "");
	6321	}, loaded, llvm_value_ty, "");
6410	6322	}
6411	6323
6412	6324	// `load_ty` is a simple by-val type which requires no special handling.
6413		return fg.wip.load(access_kind, llvm_load_ty, ptr, llvm_ptr_align, "");
	6325	return fg.wip.load(access_kind, llvm_value_ty, ptr, llvm_ptr_align, "");
6414	6326	}
6415	6327
6416	6328	/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
...	...	@@ -6438,7 +6350,7 @@ fn store(
6438	6350	};
6439	6351
6440	6352	if (isByRef(elem_ty, zcu)) {
6441		const llvm_usize_ty = try o.lowerType(.usize);
	6353	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6442	6354	_ = try fg.wip.callMemCpy(
6443	6355	ptr,
6444	6356	llvm_ptr_align,
...	...	@@ -6451,16 +6363,19 @@ fn store(
6451	6363	return;
6452	6364	}
6453	6365
6454		assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty));
	6366	assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty, .by_value));
6455	6367
6456		if (elem_ty.isAbiInt(zcu) and elem_ty.bitSize(zcu) != abi_size * 8) {
	6368	const llvm_memory_ty = try o.lowerType(elem_ty, .in_memory);
	6369	const llvm_value_ty = try o.lowerType(elem_ty, .by_value);
	6370
	6371	if (llvm_memory_ty != llvm_value_ty) {
	6372	assert(elem_ty.isAbiInt(zcu));
6457	6373	// `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6458	6374	// the corresponding comment in `FuncGen.load` for more details.
6459		const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6460	6375	const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
6461	6376	.unsigned => .zext,
6462	6377	.signed => .sext,
6463		}, elem, llvm_abi_ty, "");
	6378	}, elem, llvm_memory_ty, "");
6464	6379	_ = try fg.wip.storeAtomic(
6465	6380	access_kind,
6466	6381	extended,
...	...	@@ -6486,7 +6401,7 @@ fn store(
6486	6401	fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6487	6402	const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6488	6403	const o = fg.object;
6489		const usize_ty = try o.lowerType(.usize);
	6404	const usize_ty = try o.lowerType(.usize, .by_value);
6490	6405	const zero = try o.builder.intValue(usize_ty, 0);
6491	6406	const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6492	6407	const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
...	...	@@ -6508,7 +6423,7 @@ fn valgrindClientRequest(
6508	6423	const target = zcu.getTarget();
6509	6424	if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
6510	6425
6511		const llvm_usize = try o.lowerType(.usize);
	6426	const llvm_usize = try o.lowerType(.usize, .by_value);
6512	6427	const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
6513	6428
6514	6429	const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
...	...	@@ -6790,7 +6705,7 @@ const ParamTypeIterator = struct {
6790	6705	while (field_it.next()) \|field_index\| {
6791	6706	const field_ty = ty.fieldType(field_index, zcu);
6792	6707	if (!field_ty.hasRuntimeBits(zcu)) continue;
6793		it.types_buffer[it.types_len] = try it.object.lowerType(field_ty);
	6708	it.types_buffer[it.types_len] = try it.object.lowerType(field_ty, .by_value);
6794	6709	it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);
6795	6710	it.types_len += 1;
6796	6711	}
...	...	@@ -6807,7 +6722,7 @@ const ParamTypeIterator = struct {
6807	6722	it.llvm_index += 1;
6808	6723	return .byval;
6809	6724	} else {
6810		it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty)};
	6725	it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty, .by_value)};
6811	6726	it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };
6812	6727	it.types_len = 1;
6813	6728	it.llvm_index += 1;
...	...	@@ -6988,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para
6988	6903	};
6989	6904	}
6990	6905
6991		fn returnTypeByRef(zcu: Zcu, target: const std.Target, ty: Type) bool {
6992		if (isByRef(ty, zcu)) {
6993		return true;
6994		} else if (target.cpu.arch.isX86() and
6995		!target.cpu.has(.x86, .avx512f) and
6996		ty.totalVectorBits(zcu) >= 512)
6997		{
6998		// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6999		// "512-bit vector arguments require 'avx512f' for AVX512"
7000		return true;
7001		} else {
7002		return false;
7003		}
7004		}
7005
7006		pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: Zcu, target: const std.Target) bool {
7007		const return_type = Type.fromInterned(fn_info.return_type);
7008		if (!return_type.hasRuntimeBits(zcu)) return false;
7009
7010		return switch (fn_info.cc) {
7011		.auto => returnTypeByRef(zcu, target, return_type),
7012		.x86_64_sysv, .x86_64_x32 => firstParamSRetSystemV(return_type, zcu, target),
7013		.x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
7014		.x86_sysv, .x86_win => isByRef(return_type, zcu),
7015		.x86_stdcall => !isScalar(zcu, return_type),
7016		.x86_fastcall => firstParamSRetX86Fastcall(zcu, return_type),
7017		.wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
7018		.aarch64_aapcs,
7019		.aarch64_aapcs_darwin,
7020		.aarch64_aapcs_win,
7021		=> aarch64_c_abi.classifyType(return_type, zcu) == .memory,
7022		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7023		.memory, .i64_array => true,
7024		.i32_array => \|size\| size != 1,
7025		.byval => false,
7026		},
7027		.riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
7028		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7029		.memory, .i32_array => true,
7030		.byval => false,
7031		},
7032		else => false, // TODO: investigate other targets/callconvs
7033		};
7034		}
	6906	pub const FnReturnStrat = union(enum) {
	6907	/// The function return type is OPV (zero-bit), so the LLVM function return type is `void`.
	6908	void,
	6909	/// An sret parameter is used. The LLVM function return type is `void`.
	6910	sret,
	6911	/// The function's return type directly corresponds to the LLVM function return type.
	6912	///
	6913	/// The return type is by-val, i.e. `isByRef` returns `false`.
	6914	by_val,
	6915	/// The LLVM function returns the given `Builder.Type` by reinterpreting memory containing the
	6916	/// actual return value. The actual return type may be by-val or by-ref.
	6917	mem_cast: Builder.Type,
7035	6918
7036		fn firstParamSRetX86Fastcall(zcu: *Zcu, ty: Type) bool {
7037		if (isScalar(zcu, ty)) {
7038		return false;
7039		}
7040		const tag = ty.zigTypeTag(zcu);
7041		if (tag == .@"struct" or tag == .@"union") {
7042		const size = ty.abiSize(zcu);
7043		if (size == 1 or size == 2 or size == 4 or size == 8) {
7044		return false;
7045		}
	6919	fn forceByVal(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
	6920	if (!isByRef(ret_ty, o.zcu)) return .by_val;
	6921	return .{ .mem_cast = try o.lowerType(ret_ty, .in_memory) };
7046	6922	}
7047		return true;
7048		}
7049
7050		fn firstParamSRetSystemV(ty: Type, zcu: Zcu, target: const std.Target) bool {
7051		if (isScalar(zcu, ty)) return false;
7052		const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
7053		if (class[0] == .memory) return true;
7054		if (class[0] == .x87 and class[2] != .none) return true;
7055		return false;
7056		}
7057
	6923	};
7058	6924	/// In order to support the C calling convention, some return types need to be lowered
7059	6925	/// completely differently in the function prototype to honor the C ABI, and then
7060	6926	/// be effectively bitcasted to the actual return type.
7061		pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	6927	pub fn fnReturnStrat(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7062	6928	const zcu = o.zcu;
7063		const return_type = Type.fromInterned(fn_info.return_type);
7064		if (!return_type.hasRuntimeBits(zcu)) {
7065		assert(!return_type.isError(zcu));
7066		return .void;
7067		}
7068		const target = zcu.getTarget();
	6929	const ret_ty: Type = .fromInterned(fn_info.return_type);
	6930	ret_ty.assertHasLayout(zcu);
	6931	if (!ret_ty.hasRuntimeBits(zcu)) return .void;
7069	6932	switch (fn_info.cc) {
7070	6933	.@"inline" => unreachable,
7071		.auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type),
	6934	.auto => {
	6935	if (isByRef(ret_ty, zcu)) return .sret;
	6936
	6937	const target = zcu.getTarget();
	6938	if (target.cpu.arch.isX86() and
	6939	!target.cpu.has(.x86, .avx512f) and
	6940	ret_ty.totalVectorBits(zcu) >= 512)
	6941	{
	6942	// As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
	6943	// "512-bit vector arguments require 'avx512f' for AVX512"
	6944	return .sret;
	6945	}
	6946
	6947	return .by_val;
	6948	},
7072	6949	.x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),
7073	6950	.x86_64_win => return lowerWin64FnRetTy(o, fn_info),
7074		.x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void,
7075		.x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, return_type),
7076		.x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type),
7077		.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {
7078		.memory => return .void,
7079		.float_array => return o.lowerType(return_type),
7080		.byval => return o.lowerType(return_type),
7081		.integer => return .i64,
7082		.double_integer => return o.builder.arrayType(2, .i64),
	6951	.x86_stdcall => if (isScalar(zcu, ret_ty)) {
	6952	assert(!isByRef(ret_ty, zcu));
	6953	return .by_val;
	6954	} else return .sret,
	6955	.x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, ret_ty),
	6956	.x86_sysv, .x86_win => return if (isByRef(ret_ty, zcu)) .sret else .by_val,
	6957	.aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(ret_ty, zcu)) {
	6958	.memory => return .sret,
	6959	.float_array, .byval => return .forceByVal(o, ret_ty),
	6960	.integer => return .{ .mem_cast = .i64 },
	6961	.double_integer => return .{ .mem_cast = try o.builder.arrayType(2, .i64) },
7083	6962	},
7084		.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7085		.memory, .i64_array => return .void,
7086		.i32_array => \|len\| return if (len == 1) .i32 else .void,
7087		.byval => return o.lowerType(return_type),
	6963	.arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(ret_ty, zcu, .ret)) {
	6964	.memory, .i64_array => return .sret,
	6965	.i32_array => \|len\| return if (len == 1) .{ .mem_cast = .i32 } else .sret,
	6966	.byval => return .forceByVal(o, ret_ty),
7088	6967	},
7089		.mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7090		.memory, .i32_array => return .void,
7091		.byval => return o.lowerType(return_type),
	6968	.mips_o32 => switch (mips_c_abi.classifyType(ret_ty, zcu, .ret)) {
	6969	.memory, .i32_array => return .sret,
	6970	.byval => return .forceByVal(o, ret_ty),
7092	6971	},
7093		.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {
7094		.memory => return .void,
7095		.integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),
	6972	.riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(ret_ty, zcu)) {
	6973	.memory => return .sret,
	6974	.integer => return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7096	6975	.double_integer => {
7097	6976	const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7098	6977	.riscv64, .riscv64be => .i64,
7099	6978	.riscv32, .riscv32be => .i32,
7100	6979	else => unreachable,
7101	6980	};
7102		return o.builder.structType(.normal, &.{ integer, integer });
	6981	return .{ .mem_cast = try o.builder.structType(.normal, &.{ integer, integer }) };
7103	6982	},
7104		.byval => return o.lowerType(return_type),
	6983	.byval => return .forceByVal(o, ret_ty),
7105	6984	.fields => {
7106	6985	var types_len: usize = 0;
7107	6986	var types: [8]Builder.Type = undefined;
7108		for (0..return_type.structFieldCount(zcu)) \|field_index\| {
7109		const field_ty = return_type.fieldType(field_index, zcu);
	6987	for (0..ret_ty.structFieldCount(zcu)) \|field_index\| {
	6988	const field_ty = ret_ty.fieldType(field_index, zcu);
7110	6989	if (!field_ty.hasRuntimeBits(zcu)) continue;
7111		types[types_len] = try o.lowerType(field_ty);
	6990	types[types_len] = try o.lowerType(field_ty, .by_value);
7112	6991	types_len += 1;
7113	6992	}
7114		return o.builder.structType(.normal, types[0..types_len]);
	6993	return .{ .mem_cast = try o.builder.structType(.normal, types[0..types_len]) };
7115	6994	},
7116	6995	},
7117		.wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {
7118		.direct => \|scalar_ty\| return o.lowerType(scalar_ty),
7119		.indirect => return .void,
	6996	.wasm_mvp => switch (wasm_c_abi.classifyType(ret_ty, zcu)) {
	6997	.direct => \|scalar_ty\| if (scalar_ty.toIntern() == ret_ty.toIntern()) {
	6998	assert(!isByRef(ret_ty, zcu));
	6999	return .by_val;
	7000	} else {
	7001	return .{ .mem_cast = try o.lowerType(scalar_ty, .by_value) };
	7002	},
	7003	.indirect => return .sret,
7120	7004	},
7121	7005	// TODO investigate other callconvs
7122		else => return o.lowerType(return_type),
	7006	else => return .forceByVal(o, ret_ty),
7123	7007	}
7124	7008	}
7125	7009
7126		fn lowerX86FastcallFnRetTy(o: Object, zcu: Zcu, ty: Type) Allocator.Error!Builder.Type {
	7010	fn lowerX86FastcallFnRetTy(o: Object, zcu: Zcu, ty: Type) Allocator.Error!FnReturnStrat {
7127	7011	if (isScalar(zcu, ty)) {
7128		return o.lowerType(ty);
	7012	assert(!isByRef(ty, zcu));
	7013	return .by_val;
7129	7014	}
7130	7015	const tag = ty.zigTypeTag(zcu);
7131	7016	if (tag == .@"struct" or tag == .@"union") {
7132	7017	const size = ty.abiSize(zcu);
7133	7018	if (size == 1 or size == 2 or size == 4 or size == 8) {
7134		return o.builder.intType(@intCast(size * 8));
	7019	return .{ .mem_cast = try o.builder.intType(@intCast(size * 8)) };
7135	7020	}
7136	7021	}
7137		return .void;
	7022	return .sret;
7138	7023	}
7139	7024
7140		fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7025	fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7141	7026	const zcu = o.zcu;
7142		const return_type = Type.fromInterned(fn_info.return_type);
7143		switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
7144		.integer => {
7145		if (isScalar(zcu, return_type)) {
7146		return o.lowerType(return_type);
7147		} else {
7148		return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));
7149		}
	7027	const ret_ty = Type.fromInterned(fn_info.return_type);
	7028	switch (x86_64_abi.classifyWindows(ret_ty, zcu, zcu.getTarget(), .ret)) {
	7029	.integer => if (isScalar(zcu, ret_ty)) {
	7030	assert(!isByRef(ret_ty, zcu));
	7031	return .by_val;
	7032	} else {
	7033	return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) };
7150	7034	},
	7035	.win_i128 => return .{ .mem_cast = try o.builder.vectorType(.normal, 2, .i64) },
	7036	.memory => return .sret,
	7037
7151	7038	.sse,
7152	7039	.bool_vector_mask,
7153	7040	.integer_per_element,
...	...	@@ -7156,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7156	7043	.sse_per_xword,
7157	7044	.sse_per_yword,
7158	7045	.sse_per_zword,
7159		=> return o.lowerType(return_type),
	7046	=> {
	7047	assert(!isByRef(ret_ty, zcu));
	7048	return .by_val;
	7049	},
7160	7050	.sseup,
7161	7051	.x87,
7162	7052	.x87up,
...	...	@@ -7164,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7164	7054	.float,
7165	7055	.float_combine,
7166	7056	=> unreachable,
7167		.win_i128 => return o.builder.vectorType(.normal, 2, .i64),
7168		.memory => return .void,
7169	7057	}
7170	7058	}
7171	7059
7172		fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
	7060	fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7173	7061	const zcu = o.zcu;
7174	7062	const ip = &zcu.intern_pool;
7175		const return_type = Type.fromInterned(fn_info.return_type);
7176		return_type.assertHasLayout(zcu);
7177		if (isScalar(zcu, return_type)) {
7178		return o.lowerType(return_type);
	7063	const ret_ty = Type.fromInterned(fn_info.return_type);
	7064	if (isScalar(zcu, ret_ty)) {
	7065	assert(!isByRef(ret_ty, zcu));
	7066	return .by_val;
7179	7067	}
7180		const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);
	7068	const classes = x86_64_abi.classifySystemV(ret_ty, zcu, zcu.getTarget(), .ret);
7181	7069	var types_index: u32 = 0;
7182	7070	var types_buffer: [8]Builder.Type = undefined;
7183	7071	for (classes) \|class\| {
...	...	@@ -7207,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7207	7095	types_index += 1;
7208	7096	},
7209	7097	.x87 => {
7210		if (types_index != 0 or classes[2] != .none) return .void;
	7098	if (types_index != 0 or classes[2] != .none) return .sret;
7211	7099	types_buffer[types_index] = .x86_fp80;
7212	7100	types_index += 1;
7213	7101	},
7214	7102	.x87up => continue,
7215	7103	.none => break,
7216		.memory => return .void,
	7104	.memory => return .sret,
7217	7105	.win_i128 => unreachable, // windows only
7218	7106	.bool_vector_mask,
7219	7107	.integer_per_element,
...	...	@@ -7228,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7228	7116	const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7229	7117	if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7230	7118	assert(first_non_integer orelse classes.len == types_index);
7231		switch (ip.indexToKey(return_type.toIntern())) {
	7119	switch (ip.indexToKey(ret_ty.toIntern())) {
7232	7120	.struct_type => {
7233		const size = return_type.abiSize(zcu);
	7121	const size = ret_ty.abiSize(zcu);
7234	7122	assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7235	7123	if (size % 8 > 0) {
7236	7124	types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
...	...	@@ -7238,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7238	7126	},
7239	7127	else => {},
7240	7128	}
7241		if (types_index == 1) return types_buffer[0];
	7129	if (types_index == 1) return .{ .mem_cast = types_buffer[0] };
7242	7130	}
7243		return o.builder.structType(.normal, types_buffer[0..types_index]);
	7131	return .{ .mem_cast = try o.builder.structType(.normal, types_buffer[0..types_index]) };
7244	7132	}
7245	7133
7246	7134	/// This function deliberately does not handle `_BitInt` because it typically
...	...	@@ -7380,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7380	7268	},
7381	7269	.@"union" => switch (ty.containerLayout(zcu)) {
7382	7270	.@"packed" => false,
7383		else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),
	7271	else => ty.hasRuntimeBits(zcu),
7384	7272	},
7385	7273	};
7386	7274	}
...	...	@@ -7411,7 +7299,7 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7411	7299	fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7412	7300	if (offset == 0) return ptr;
7413	7301	const o = fg.object;
7414		const llvm_usize_ty = try o.lowerType(.usize);
	7302	const llvm_usize_ty = try o.lowerType(.usize, .by_value);
7415	7303	const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7416	7304	return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7417	7305	}

src/codegen/mips/abi.zig+4-5

...	...	@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
18	18	const max_direct_size = target.ptrBitWidth() * 2;
19	19	switch (ty.zigTypeTag(zcu)) {
20	20	.@"struct" => {
21		const bit_size = ty.bitSize(zcu);
22	21	if (ty.containerLayout(zcu) == .@"packed") {
23		if (bit_size > max_direct_size) return .memory;
	22	if (ty.bitSize(zcu) > max_direct_size) return .memory;
24	23	return .byval;
25	24	}
	25	const bit_size = ty.abiSize(zcu) * 8;
26	26	if (bit_size > max_direct_size) return .memory;
27	27	// TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28	28	const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29	29	return .{ .i32_array = count };
30	30	},
31	31	.@"union" => {
32		const bit_size = ty.bitSize(zcu);
33	32	if (ty.containerLayout(zcu) == .@"packed") {
34		if (bit_size > max_direct_size) return .memory;
	33	if (ty.bitSize(zcu) > max_direct_size) return .memory;
35	34	return .byval;
36	35	}
	36	const bit_size = ty.abiSize(zcu) * 8;
37	37	if (bit_size > max_direct_size) return .memory;
38
39	38	return .byval;
40	39	},
41	40	.bool => return .byval,

src/codegen/riscv64/CodeGen.zig+12-7

...	...	@@ -51,7 +51,7 @@ const InnerError = codegen.Error \|\| error{OutOfRegisters};
51	51
52	52	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
53	53	return comptime &.initMany(&.{
54		.expand_intcast_safe,
	54	.expand_int_cast_safe,
55	55	.expand_int_from_float_safe,
56	56	.expand_int_from_float_optimized_safe,
57	57	.expand_add_safe,
...	...	@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1453	1453	.add_safe,
1454	1454	.sub_safe,
1455	1455	.mul_safe,
1456		.intcast_safe,
	1456	.int_cast_safe,
1457	1457	.int_from_float_safe,
1458	1458	.int_from_float_optimized_safe,
1459	1459	=> return func.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1479	1479	.ret_ptr => try func.airRetPtr(inst),
1480	1480	.arg => try func.airArg(inst),
1481	1481	.assembly => try func.airAsm(inst),
1482		.bitcast => try func.airBitCast(inst),
	1482	.bit_cast => try func.airBitCast(inst),
	1483	.ptr_cast => try func.airBitCast(inst),
	1484	.ptr_from_int => try func.airBitCast(inst),
	1485	.int_from_ptr => try func.airBitCast(inst),
	1486	.error_cast => try func.airBitCast(inst),
	1487	.error_from_int => try func.airBitCast(inst),
	1488	.int_from_error => try func.airBitCast(inst),
	1489	.union_from_enum => try func.airBitCast(inst),
1483	1490	.block => try func.airBlock(inst),
1484	1491	.br => try func.airBr(inst),
1485	1492	.repeat => try func.airRepeat(inst),
...	...	@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1493	1500	.dbg_empty_stmt => func.finishAirBookkeeping(),
1494	1501	.fptrunc => try func.airFptrunc(inst),
1495	1502	.fpext => try func.airFpext(inst),
1496		.intcast => try func.airIntCast(inst),
	1503	.int_cast => try func.airIntCast(inst),
1497	1504	.trunc => try func.airTrunc(inst),
1498	1505	.is_non_null => try func.airIsNonNull(inst),
1499	1506	.is_non_null_ptr => try func.airIsNonNullPtr(inst),
...	...	@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3953	3960	const elem_ptr_ty = func.typeOfIndex(inst);
3954	3961	const base_ptr_ty = func.typeOf(extra.lhs);
3955	3962
3956		if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
3957		@panic("audit");
3958		}
	3963	assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
3959	3964
3960	3965	const base_ptr_mcv = try func.resolveInst(extra.lhs);
3961	3966	const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {

src/codegen/riscv64/abi.zig+5-6

...	...	@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
16	16	const max_byval_size = target.ptrBitWidth() * 2;
17	17	switch (ty.zigTypeTag(zcu)) {
18	18	.@"struct" => {
19		const bit_size = ty.bitSize(zcu);
20	19	if (ty.containerLayout(zcu) == .@"packed") {
21		if (bit_size > max_byval_size) return .memory;
	20	if (ty.bitSize(zcu) > max_byval_size) return .memory;
22	21	return .byval;
23	22	}
24	23
...	...	@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
40	39	}
41	40
42	41	// TODO this doesn't exactly match what clang produces but its better than nothing
	42	const bit_size = ty.abiSize(zcu) * 8;
43	43	if (bit_size > max_byval_size) return .memory;
44	44	if (bit_size > max_byval_size / 2) return .double_integer;
45	45	return .integer;
46	46	},
47	47	.@"union" => {
48		const bit_size = ty.bitSize(zcu);
49	48	if (ty.containerLayout(zcu) == .@"packed") {
50		if (bit_size > max_byval_size) return .memory;
	49	if (ty.bitSize(zcu) > max_byval_size) return .memory;
51	50	return .byval;
52	51	}
53	52	// TODO this doesn't exactly match what clang produces but its better than nothing
	53	const bit_size = ty.abiSize(zcu) * 8;
54	54	if (bit_size > max_byval_size) return .memory;
55	55	if (bit_size > max_byval_size / 2) return .double_integer;
56	56	return .integer;
...	...	@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {
153	153	},
154	154	.error_union => {
155	155	const payload_ty = ty.errorUnionPayload(zcu);
156		const payload_bits = payload_ty.bitSize(zcu);
157	156
158	157	// the error union itself
159	158	result[0] = .integer;
160	159
161	160	// anyerror!void can fit into one register
162		if (payload_bits == 0) return result;
	161	if (!payload_ty.hasRuntimeBits(zcu)) return result;
163	162
164	163	return memory_class;
165	164	},

src/codegen/sparc64/CodeGen.zig+11-4

...	...	@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
538	538	.ret_ptr => try self.airRetPtr(inst),
539	539	.arg => try self.airArg(inst),
540	540	.assembly => try self.airAsm(inst),
541		.bitcast => try self.airBitCast(inst),
	541	.bit_cast => try self.airBitCast(inst),
	542	.ptr_cast => try self.airBitCast(inst),
	543	.ptr_from_int => try self.airBitCast(inst),
	544	.int_from_ptr => try self.airBitCast(inst),
	545	.error_cast => try self.airBitCast(inst),
	546	.error_from_int => try self.airBitCast(inst),
	547	.int_from_error => try self.airBitCast(inst),
	548	.union_from_enum => try self.airBitCast(inst),
542	549	.block => try self.airBlock(inst),
543	550	.br => try self.airBr(inst),
544	551	.repeat => return self.fail("TODO implement `repeat`", .{}),
...	...	@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
550	557	.cond_br => try self.airCondBr(inst),
551	558	.fptrunc => @panic("TODO try self.airFptrunc(inst)"),
552	559	.fpext => @panic("TODO try self.airFpext(inst)"),
553		.intcast => try self.airIntCast(inst),
	560	.int_cast => try self.airIntCast(inst),
554	561	.trunc => try self.airTrunc(inst),
555	562	.is_non_null => try self.airIsNonNull(inst),
556	563	.is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
...	...	@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
689	696	.add_safe,
690	697	.sub_safe,
691	698	.mul_safe,
692		.intcast_safe,
	699	.int_cast_safe,
693	700	.int_from_float_safe,
694	701	.int_from_float_optimized_safe,
695	702	=> @panic("TODO implement safety_checked_instructions"),
...	...	@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1659	1666	const info_a = operand_ty.intInfo(zcu);
1660	1667	const info_b = self.typeOfIndex(inst).intInfo(zcu);
1661	1668	if (info_a.signedness != info_b.signedness)
1662		return self.fail("TODO gen intcast sign safety in semantic analysis", .{});
	1669	return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
1663	1670
1664	1671	if (info_a.bits == info_b.bits)
1665	1672	return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });

src/codegen/spirv/CodeGen.zig+67-28

...	...	@@ -34,7 +34,7 @@ const CodeGen = @This();
34	34
35	35	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
36	36	return comptime &.initMany(&.{
37		.expand_intcast_safe,
	37	.expand_int_cast_safe,
38	38	.expand_int_from_float_safe,
39	39	.expand_int_from_float_optimized_safe,
40	40	.expand_add_safe,
...	...	@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1848	1848	.pointer => {
1849	1849	const ptr_info = ty.ptrInfo(zcu);
1850	1850
1851		const child_ty: Type = .fromInterned(ptr_info.child);
	1851	const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
	1852	0 => .fromInterned(ptr_info.child),
	1853	else => switch (ptr_info.flags.vector_index) {
	1854	// Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
	1855	.none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
	1856	else => try pt.vectorType(.{
	1857	.child = ptr_info.child,
	1858	.len = ptr_info.packed_offset.host_size,
	1859	}),
	1860	},
	1861	};
1852	1862	const child_ty_id = try cg.resolveType(child_ty, .indirect);
1853	1863	const storage_class = cg.module.storageClass(ptr_info.flags.address_space);
1854	1864	const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);
...	...	@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
3847	3857	.min => try cg.airMinMax(inst, .min),
3848	3858	.max => try cg.airMinMax(inst, .max),
3849	3859
3850		.bitcast => try cg.airBitCast(inst),
3851		.intcast, .trunc => try cg.airIntCast(inst),
3852		.float_from_int => try cg.airFloatFromInt(inst),
3853		.int_from_float => try cg.airIntFromFloat(inst),
3854		.fpext, .fptrunc => try cg.airFloatCast(inst),
3855		.not => try cg.airNot(inst),
	3860	.bit_cast => try cg.airBitCast(inst),
	3861	.ptr_cast => try cg.airBitCast(inst),
	3862	.ptr_from_int => try cg.airBitCast(inst),
	3863	.int_from_ptr => try cg.airBitCast(inst),
	3864	.error_cast => try cg.airBitCast(inst),
	3865	.error_from_int => try cg.airBitCast(inst),
	3866	.int_from_error => try cg.airBitCast(inst),
	3867	.union_from_enum => try cg.airBitCast(inst),
	3868	.int_cast, .trunc => try cg.airIntCast(inst),
	3869	.float_from_int => try cg.airFloatFromInt(inst),
	3870	.int_from_float => try cg.airIntFromFloat(inst),
	3871	.fpext, .fptrunc => try cg.airFloatCast(inst),
	3872	.not => try cg.airNot(inst),
3856	3873
3857	3874	.array_to_slice => try cg.airArrayToSlice(inst),
3858	3875	.slice => try cg.airSlice(inst),
...	...	@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6913	6930	const zcu = cg.module.zcu;
6914	6931	const pt = cg.pt;
6915	6932	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6916		const ptr_ty = cg.typeOf(ty_op.operand);
6917		const ptr_info = ptr_ty.ptrInfo(zcu);
	6933
	6934	const ptr_info = cg.typeOf(ty_op.operand).ptrInfo(zcu);
	6935
6918	6936	const elem_ty = cg.typeOfIndex(inst);
6919		const operand = try cg.resolve(ty_op.operand);
6920		if (!ptr_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
	6937	const operand_ptr_id = try cg.resolve(ty_op.operand);
6921	6938
6922		if (cg.virtual_allocas.get(operand)) \|stored\| return stored.?;
	6939	assert(ptr_info.child == elem_ty.toIntern());
	6940
	6941	if (cg.virtual_allocas.get(operand_ptr_id)) \|stored\| return stored.?;
6923	6942
6924	6943	if (ptr_info.packed_offset.host_size != 0 and
6925	6944	ptr_info.flags.vector_index == .none)
...	...	@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6927	6946	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6928	6947	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6929	6948	const host_int_ty = try pt.intType(.unsigned, host_bits);
6930		const host_val = try cg.load(host_int_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	6949	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6931	6950	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6932	6951	const field_int_ty = try pt.intType(signedness, elem_bit_size);
6933	6952	const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
...	...	@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6946	6965	return try cg.bitCast(elem_ty, field_int_ty, result_id);
6947	6966	}
6948	6967
6949		return try cg.load(elem_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	6968	const ptr_id = switch (ptr_info.flags.vector_index) {
	6969	.none => operand_ptr_id,
	6970	else => \|index\| ptr_id: {
	6971	const elem_ptr_ty_id = try cg.module.ptrType(
	6972	try cg.resolveType(elem_ty, .indirect),
	6973	cg.module.storageClass(ptr_info.flags.address_space),
	6974	);
	6975	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
	6976	},
	6977	};
	6978	return try cg.load(elem_ty, ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6950	6979	}
6951	6980
6952	6981	fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6953	6982	const zcu = cg.module.zcu;
6954	6983	const pt = cg.pt;
6955	6984	const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6956		const ptr_ty = cg.typeOf(bin_op.lhs);
6957		const ptr_info = ptr_ty.ptrInfo(zcu);
6958		const elem_ty = ptr_ty.childType(zcu);
6959		const ptr = try cg.resolve(bin_op.lhs);
6960		const value = try cg.resolve(bin_op.rhs);
	6985	const ptr_info = cg.typeOf(bin_op.lhs).ptrInfo(zcu);
	6986	const elem_ty: Type = .fromInterned(ptr_info.child);
	6987	const operand_ptr_id = try cg.resolve(bin_op.lhs);
	6988	const value_id = try cg.resolve(bin_op.rhs);
6961	6989
6962		if (cg.virtual_allocas.getPtr(ptr)) \|slot\| {
6963		slot.* = value;
	6990	if (cg.virtual_allocas.getPtr(operand_ptr_id)) \|slot\| {
	6991	slot.* = value_id;
6964	6992	return;
6965	6993	}
6966	6994
...	...	@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6969	6997	{
6970	6998	const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6971	6999	const host_int_ty = try pt.intType(.unsigned, host_bits);
6972		const host_val = try cg.load(host_int_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7000	const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6973	7001	const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6974	7002	const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6975	7003	const field_int_ty = try pt.intType(signedness, elem_bit_size);
6976	7004
6977	7005	var value_as_int: Id = undefined;
6978	7006	if (elem_ty.ip_index == .bool_type) {
6979		value_as_int = try cg.convertToIndirect(.bool, value);
	7007	value_as_int = try cg.convertToIndirect(.bool, value_id);
6980	7008	value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
6981	7009	} else if (elem_ty.isInt(zcu)) {
6982		value_as_int = value;
	7010	value_as_int = value_id;
6983	7011	} else {
6984		value_as_int = try cg.bitCast(field_int_ty, elem_ty, value);
	7012	value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
6985	7013	}
6986	7014
6987	7015	const extended = blk: {
...	...	@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7002	7030	const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7003	7031	const combined_id = try combined.materialize(cg);
7004	7032
7005		try cg.store(host_int_ty, ptr, combined_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7033	try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7006	7034	return;
7007	7035	}
7008	7036
7009		try cg.store(elem_ty, ptr, value, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });
	7037	const ptr_id = switch (ptr_info.flags.vector_index) {
	7038	.none => operand_ptr_id,
	7039	else => \|index\| ptr_id: {
	7040	const elem_ptr_ty_id = try cg.module.ptrType(
	7041	try cg.resolveType(elem_ty, .indirect),
	7042	cg.module.storageClass(ptr_info.flags.address_space),
	7043	);
	7044	break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
	7045	},
	7046	};
	7047
	7048	try cg.store(elem_ty, ptr_id, value_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7010	7049	}
7011	7050
7012	7051	fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {

src/codegen/wasm/CodeGen.zig+96-25

...	...	@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
32	32
33	33	pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
34	34	return comptime &.initMany(&.{
35		.expand_intcast_safe,
	35	.expand_int_cast_safe,
36	36	.expand_int_from_float_safe,
37	37	.expand_int_from_float_optimized_safe,
38	38	.expand_add_safe,
...	...	@@ -108,7 +108,10 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
108	108	.scalarize_cmp_vector_optimized,
109	109	.scalarize_fptrunc,
110	110	.scalarize_fpext,
111		.scalarize_intcast,
	111	.scalarize_int_cast,
	112	.scalarize_ptr_cast,
	113	.scalarize_ptr_from_int,
	114	.scalarize_int_from_ptr,
112	115	.scalarize_trunc,
113	116	.scalarize_int_from_float,
114	117	.scalarize_int_from_float_optimized,
...	...	@@ -120,7 +123,7 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
120	123	.scalarize_select,
121	124	.scalarize_mul_add,
122	125
123		.scalarize_bitcast_padded_elems,
	126	.scalarize_bit_cast_padded_elems,
124	127	});
125	128	}
126	129
...	...	@@ -1551,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1551	1554	try cg.finishAir(inst, result, &.{ty_op.operand});
1552	1555	},
1553	1556
1554		.bitcast => cg.airBitcast(inst),
	1557	.ptr_cast => cg.airNopCast(inst),
	1558	.error_cast => cg.airNopCast(inst),
	1559	.error_from_int => cg.airNopCast(inst),
	1560	.int_from_error => cg.airNopCast(inst),
	1561	.ptr_from_int => cg.airNopCast(inst),
	1562	.int_from_ptr => cg.airIntFromPtr(inst),
1555	1563
1556		.intcast => {
	1564	.bit_cast => cg.airBitcast(inst),
	1565	.union_from_enum => cg.airBitcast(inst),
	1566
	1567	.int_cast => {
1557	1568	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1558	1569
1559	1570	const dest_ty = ty_op.ty.toType();
...	...	@@ -1561,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1561	1572	const src_ty = cg.typeOf(ty_op.operand);
1562	1573
1563	1574	if (dest_ty.zigTypeTag(zcu) == .vector) {
1564		return cg.fail("TODO: implement AIR op: intcast for vectors", .{});
	1575	return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1565	1576	}
1566	1577
1567	1578	const src_int_ty: IntType = .fromType(cg, src_ty);
...	...	@@ -1876,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1876	1887	.add_safe,
1877	1888	.sub_safe,
1878	1889	.mul_safe,
1879		.intcast_safe,
	1890	.int_cast_safe,
1880	1891	.int_from_float_safe,
1881	1892	.int_from_float_optimized_safe,
1882	1893	=> return cg.fail("TODO implement safety_checked_instructions", .{}),
...	...	@@ -2100,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2100	2111	const rhs = try cg.resolveInst(bin_op.rhs);
2101	2112	const ptr_ty = cg.typeOf(bin_op.lhs);
2102	2113	const ptr_info = ptr_ty.ptrInfo(zcu);
2103		const ty = ptr_ty.childType(zcu);
	2114	const elem_ty = ptr_ty.childType(zcu);
2104	2115
2105	2116	if (!safety and bin_op.rhs == .undef) {
2106	2117	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2107	2118	}
2108	2119
2109		assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_store
2110
2111		try cg.store(lhs, rhs, ty, 0);
2112
	2120	const offset: u32 = switch (ptr_info.flags.vector_index) {
	2121	.none => offset: {
	2122	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
	2123	break :offset 0;
	2124	},
	2125	else => \|index\| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
	2126	};
	2127	try cg.store(lhs, rhs, elem_ty, offset);
2113	2128	return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2114	2129	}
2115	2130
...	...	@@ -2122,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2122	2137	if (!ty.hasRuntimeBits(zcu)) return;
2123	2138
2124	2139	if (isByRef(ty, zcu, cg.target)) {
2125		return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });
	2140	const offset_ptr: WValue = switch (offset + lhs.offset()) {
	2141	0 => lhs,
	2142	else => \|total_offset\| ptr: {
	2143	try cg.emitWValue(lhs);
	2144	try cg.addImm32(total_offset);
	2145	try cg.addTag(.i32_add);
	2146	break :ptr .stack;
	2147	},
	2148	};
	2149	return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2126	2150	}
2127	2151
2128	2152	if (ty.zigTypeTag(zcu) == .vector) {
...	...	@@ -2134,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2134	2158	try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2135	2159	@intFromEnum(std.wasm.SimdOpcode.v128_store),
2136	2160	offset + lhs.offset(),
2137		@intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),
	2161	@intCast(ty.abiAlignment(zcu).toByteUnits().?),
2138	2162	});
2139	2163	return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2140	2164	}
...	...	@@ -2175,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2175	2199	const zcu = pt.zcu;
2176	2200	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2177	2201	const operand = try cg.resolveInst(ty_op.operand);
2178		const ty = ty_op.ty.toType();
	2202	const elem_ty = ty_op.ty.toType();
2179	2203	const ptr_ty = cg.typeOf(ty_op.operand);
2180	2204	const ptr_info = ptr_ty.ptrInfo(zcu);
2181	2205
2182		if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});
2183
2184		assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_load
	2206	assert(elem_ty.hasRuntimeBits(zcu));
2185	2207
2186		const result = try cg.load(operand, ty, 0);
	2208	const offset: u32 = switch (ptr_info.flags.vector_index) {
	2209	.none => offset: {
	2210	assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
	2211	break :offset 0;
	2212	},
	2213	else => \|index\| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
	2214	};
	2215	const result = try cg.load(operand, elem_ty, offset);
2187	2216	return cg.finishAir(inst, result, &.{ty_op.operand});
2188	2217	}
2189	2218
...	...	@@ -2192,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2192	2221	fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2193	2222	const zcu = cg.pt.zcu;
2194	2223	if (isByRef(ty, zcu, cg.target)) {
2195		const val = try cg.allocStack(ty);
2196		try cg.store(val, try operand.toLocal(cg, .usize), ty, 0);
2197		return val;
	2224	const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
	2225	0 => operand,
	2226	else => \|total_offset\| ptr: {
	2227	try cg.emitWValue(operand);
	2228	try cg.addImm32(total_offset);
	2229	try cg.addTag(.i32_add);
	2230	break :ptr .stack;
	2231	},
	2232	};
	2233	const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
	2234	const new_ptr = try cg.allocStack(ty);
	2235	try cg.store(new_ptr, src_ptr, ty, 0);
	2236	return new_ptr;
2198	2237	}
2199	2238
2200	2239	// load local's value from memory by its stack position
...	...	@@ -5236,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5236	5275	return cg.finishAir(inst, .none, &.{});
5237	5276	}
5238	5277
	5278	fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
	5279	const zcu = cg.pt.zcu;
	5280	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5281
	5282	const operand_ty = cg.typeOf(ty_op.operand);
	5283	const dest_ty = cg.typeOfIndex(inst);
	5284	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
	5285	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	5286	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
	5287
	5288	const operand = try cg.resolveInst(ty_op.operand);
	5289	const result = cg.reuseOperand(ty_op.operand, operand);
	5290	return cg.finishAir(inst, result, &.{ty_op.operand});
	5291	}
	5292
	5293	fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
	5294	const zcu = cg.pt.zcu;
	5295	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
	5296
	5297	const operand_ty = cg.typeOf(ty_op.operand);
	5298	const dest_ty = cg.typeOfIndex(inst);
	5299	assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
	5300	assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
	5301	assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
	5302
	5303	const operand = try cg.resolveInst(ty_op.operand);
	5304	const result = switch (operand) {
	5305	.stack_offset => try cg.buildPointerOffset(operand, 0, .new),
	5306	else => cg.reuseOperand(ty_op.operand, operand),
	5307	};
	5308	return cg.finishAir(inst, result, &.{ty_op.operand});
	5309	}
	5310
5239	5311	fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5240	5312	const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5241	5313	const operand = try cg.resolveInst(ty_op.operand);
...	...	@@ -6341,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6341	6413	}
6342	6414	}
6343	6415
6344		const elem_result = if (isByRef(elem_ty, zcu, cg.target))
	6416	const result = if (isByRef(elem_ty, zcu, cg.target))
6345	6417	.stack
6346	6418	else
6347	6419	try cg.load(.stack, elem_ty, 0);
6348
6349		return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
	6420	return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6350	6421	}
6351	6422
6352	6423	fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {

src/codegen/x86_64/CodeGen.zig+32-19

...	...	@@ -57,13 +57,13 @@ pub fn legalizeFeatures(_: const std.Target) const Air.Legalize.Features {
57	57	.scalarize_shuffle_two,
58	58	.scalarize_select,
59	59
60		.scalarize_bitcast_padded_elems,
	60	.scalarize_bit_cast_padded_elems,
61	61
62	62	//.unsplat_shift_rhs,
63		.reduce_one_elem_to_bitcast,
64		.splat_one_elem_to_bitcast,
	63	.reduce_one_elem_to_bit_cast,
	64	.splat_one_elem_to_bit_cast,
65	65
66		.expand_intcast_safe,
	66	.expand_int_cast_safe,
67	67	.expand_int_from_float_safe,
68	68	.expand_int_from_float_optimized_safe,
69	69	.expand_add_safe,
...	...	@@ -67434,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67434	67434	};
67435	67435	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
67436	67436	},
67437		.bitcast => try cg.airBitCast(inst),
	67437	.bit_cast,
	67438	.ptr_cast,
	67439	.ptr_from_int,
	67440	.int_from_ptr,
	67441	.error_cast,
	67442	.error_from_int,
	67443	.int_from_error,
	67444	.union_from_enum,
	67445	=> try cg.airBitCast(inst),
67438	67446	.block => {
67439	67447	const block = cg.air.unwrapBlock(inst);
67440	67448	if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
...	...	@@ -93375,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
93375	93383	};
93376	93384	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
93377	93385	},
93378		.intcast => \|air_tag\| {
	93386	.int_cast => \|air_tag\| {
93379	93387	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
93380	93388	const dst_ty = ty_op.ty.toType();
93381	93389	const src_ty = cg.typeOf(ty_op.operand);
...	...	@@ -98133,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
98133	98141	};
98134	98142	try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
98135	98143	},
98136		.intcast_safe => unreachable,
	98144	.int_cast_safe => unreachable,
98137	98145	.trunc => \|air_tag\| {
98138	98146	const ty_op = air_datas[@intFromEnum(inst)].ty_op;
98139	98147	var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
...	...	@@ -104351,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104351	104359	var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
104352	104360	try ops[0].toSlicePtr(cg);
104353	104361	const dst_ty = ty_pl.ty.toType();
104354		if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {
	104362	zero_offset: {
104355	104363	const elem_size = dst_ty.childType(zcu).abiSize(zcu);
104356	104364	if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;
104357	104365	while (true) for (&ops) \|*op\| {
...	...	@@ -179054,8 +179062,11 @@ fn genSetReg(
179054	179062	const zcu = pt.zcu;
179055	179063	const abi_size: u32 = @intCast(ty.abiSize(zcu));
179056	179064	const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));
179057		if (ty.bitSize(zcu) > dst_alias.size().bitSize(cg.target))
179058		return cg.fail("genSetReg called with a value larger than dst_reg", .{});
	179065	{
	179066	const ty_bit_size = if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else 8 * abi_size;
	179067	if (ty_bit_size > dst_alias.size().bitSize(cg.target))
	179068	return cg.fail("genSetReg called with a value larger than dst_reg", .{});
	179069	}
179059	179070	switch (src_mcv) {
179060	179071	.none,
179061	179072	.unreach,
...	...	@@ -180128,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
180128	180139	break :dst dst_mcv;
180129	180140	};
180130	180141
180131		if (dst_ty.isRuntimeFloat()) break :result dst_mcv;
	180142	switch (dst_ty.zigTypeTag(zcu)) {
	180143	.float, .error_union, .error_set, .vector => break :result dst_mcv,
	180144	.@"struct", .@"union" => if (dst_ty.containerLayout(zcu) != .@"packed") break :result dst_mcv,
	180145	.optional, .pointer => if (!dst_ty.isPtrAtRuntime(zcu)) break :result dst_mcv,
	180146	else => {},
	180147	}
180132	180148
180133	180149	if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
180134	180150	dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
180135	180151
180136	180152	const abi_size = dst_ty.abiSize(zcu);
180137	180153	const bit_size = dst_ty.bitSize(zcu);
180138		if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;
	180154	if (abi_size * 8 <= bit_size) break :result dst_mcv;
180139	180155
180140	180156	const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
180141	180157	const high_mcv: MCValue = switch (dst_mcv) {
...	...	@@ -182411,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
182411	182427	const zcu = pt.zcu;
182412	182428	const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
182413	182429	.signedness = .unsigned,
182414		.bits = @intCast(ty.bitSize(zcu)),
	182430	.bits = @intCast(if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else ty.abiSize(zcu) * 8),
182415	182431	};
182416	182432	const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
182417	182433	try self.spillEflagsIfOccupied();
...	...	@@ -182451,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {
182451	182467	};
182452	182468	}
182453	182469
182454		fn regExtraBits(self: *CodeGen, ty: Type) u64 {
182455		return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
182456		}
182457
182458	182470	fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
182459	182471	return switch (feature) {
182460	182472	.@"64bit" => switch (cg.target.cpu.arch) {
...	...	@@ -182570,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {
182570	182582	.bool_type => vector_type.len,
182571	182583	else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),
182572	182584	},
182573		else => @intCast(ty.bitSize(zcu)),
	182585	else => if (ty.hasBitRepresentation(zcu) or ty.isAbiInt(zcu)) @intCast(ty.bitSize(zcu)) else @intCast(ty.abiSize(zcu) * 8),
182574	182586	};
182575	182587	}
182576	182588
...	...	@@ -192243,7 +192255,8 @@ const Select = struct {
192243	192255	.src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
192244	192256	.@"8_size_sub_bit_size" => {
192245	192257	const ty = op.flags.base.ref.typeOf(s);
192246		break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));
	192258	const bit_size = s.cg.intInfo(ty).?.bits;
	192259	break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - bit_size);
192247	192260	},
192248	192261	.len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
192249	192262	.elem_limbs => @intCast(@divExact(