authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-17 12:11:23+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-06-24 10:20:56+01:00
log6d2c8349c53707f9eeee7554bd7717667718ce4d
treee1ce8fdf46a3f0282c0ba32833b76cb7a1bf2184
parentf9a6149b34af662a0d524f2f64f00e8857afc13b
signaturelock-open Commit is signed but in an unrecognized format.

compiler: fully switch to new `@bitCast` semantics

29 files changed, 3159 insertions(+), 2673 deletions(-)

CMakeLists.txt+1-1
...@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES...@@ -342,7 +342,7 @@ set(ZIG_STAGE2_SOURCES
342 src/Package/Module.zig342 src/Package/Module.zig
343 src/RangeSet.zig343 src/RangeSet.zig
344 src/Sema.zig344 src/Sema.zig
345 src/Sema/bitcast.zig345 src/Sema/reinterpret.zig
346 src/Sema/comptime_ptr_access.zig346 src/Sema/comptime_ptr_access.zig
347 src/Sema/type_resolution.zig347 src/Sema/type_resolution.zig
348 src/Type.zig348 src/Type.zig
src/Air.zig+67-12
...@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");...@@ -17,6 +17,7 @@ const print = @import("Air/print.zig");
1717
18pub const Legalize = @import("Air/Legalize.zig");18pub const Legalize = @import("Air/Legalize.zig");
19pub const Liveness = @import("Air/Liveness.zig");19pub const Liveness = @import("Air/Liveness.zig");
20pub const Verify = @import("Air/Verify.zig");
2021
21instructions: std.MultiArrayList(Inst).Slice,22instructions: std.MultiArrayList(Inst).Slice,
22/// The meaning of this data is determined by `Inst.Tag` value.23/// The meaning of this data is determined by `Inst.Tag` value.
...@@ -276,10 +277,50 @@ pub const Inst = struct {...@@ -276,10 +277,50 @@ pub const Inst = struct {
276 /// Boolean or binary NOT.277 /// Boolean or binary NOT.
277 /// Uses the `ty_op` field.278 /// Uses the `ty_op` field.
278 not,279 not,
279 /// Reinterpret the bits of a value as a different type. This is like `@bitCast` but280 /// Implements `@bitCast`.
280 /// also supports enums and pointers.281 ///
282 /// Uses the `ty_op` field.
283 bit_cast,
284 /// Cast a pointer to a different pointer type. The result type is a slice iff the operand
285 /// type is a slice (the length of the slice does not change). All other pointer attributes
286 /// except for the address space may change.
287 ///
288 /// Supports vectors of pointers.
289 ///
290 /// Uses the `ty_op` field.
291 ptr_cast,
292 /// Cast an integer to a pointer (not a slice). Operand type is always `usize`.
293 ///
294 /// Supports vectors of integers.
295 ///
296 /// Uses the `ty_op` field.
297 ptr_from_int,
298 /// Cast a pointer (not a slice) to an integer. Result type is always `usize`.
299 ///
300 /// Supports vectors of pointers.
301 ///
302 /// Uses the `ty_op` field.
303 int_from_ptr,
304 /// Cast an error set `E1` to a different error set `E2`, or cast an error union `E1!T` to
305 /// an error union `E2!T` with the same payload type but a different error set type.
306 ///
307 /// Uses the `ty_op` field.
308 error_cast,
309 /// Cast an integer to an error set type. The integer operand type is unsigned and has bit
310 /// width equal to `zcu.errorSetBits()`.
311 ///
312 /// Uses the `ty_op` field.
313 error_from_int,
314 /// Cast an error set to an integer type. The integer destination type is unsigned and has
315 /// bit width equal to `zcu.errorSetBits()`.
316 ///
317 /// Uses the `ty_op` field.
318 int_from_error,
319 /// Cast an enum value to a tagged union, whose tag type is that enum, and which has no
320 /// payload bits (i.e. all payloads are equivalent to `void`).
321 ///
281 /// Uses the `ty_op` field.322 /// Uses the `ty_op` field.
282 bitcast,323 union_from_enum,
283 /// A block runs its body which always ends with a `noreturn` instruction,324 /// A block runs its body which always ends with a `noreturn` instruction,
284 /// so the only way to proceed to the code after the `block` is to encounter a `br`325 /// so the only way to proceed to the code after the `block` is to encounter a `br`
285 /// that targets this `block`. If the `block` type is `noreturn`,326 /// that targets this `block`. If the `block` type is `noreturn`,
...@@ -589,13 +630,13 @@ pub const Inst = struct {...@@ -589,13 +630,13 @@ pub const Inst = struct {
589 /// the integer tag type of the enum.630 /// the integer tag type of the enum.
590 /// See `trunc` for integer truncation.631 /// See `trunc` for integer truncation.
591 /// Uses the `ty_op` field.632 /// Uses the `ty_op` field.
592 intcast,633 int_cast,
593 /// Like `intcast`, but includes two safety checks:634 /// Like `int_cast`, but includes two safety checks:
594 /// * triggers a safety panic if the cast truncates bits635 /// * triggers a safety panic if the cast truncates bits
595 /// * triggers a safety panic if the destination type is an exhaustive enum636 /// * triggers a safety panic if the destination type is an exhaustive enum
596 /// and the operand is not a valid value of this type; i.e. equivalent to637 /// and the operand is not a valid value of this type; i.e. equivalent to
597 /// a safety check based on `.is_named_enum_value`638 /// a safety check based on `.is_named_enum_value`
598 intcast_safe,639 int_cast_safe,
599 /// Truncate higher bits from an integer, resulting in an integer type with the same640 /// Truncate higher bits from an integer, resulting in an integer type with the same
600 /// sign but an equal or smaller number of bits.641 /// sign but an equal or smaller number of bits.
601 /// Uses the `ty_op` field.642 /// Uses the `ty_op` field.
...@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1667,12 +1708,19 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1667 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),1708 => return datas[@intFromEnum(inst)].ty_pl.ty.toType(),
16681709
1669 .not,1710 .not,
1670 .bitcast,1711 .bit_cast,
1712 .ptr_cast,
1713 .ptr_from_int,
1714 .int_from_ptr,
1715 .error_cast,
1716 .error_from_int,
1717 .int_from_error,
1718 .union_from_enum,
1671 .load,1719 .load,
1672 .fpext,1720 .fpext,
1673 .fptrunc,1721 .fptrunc,
1674 .intcast,1722 .int_cast,
1675 .intcast_safe,1723 .int_cast_safe,
1676 .trunc,1724 .trunc,
1677 .optional_payload,1725 .optional_payload,
1678 .optional_payload_ptr,1726 .optional_payload_ptr,
...@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1913,7 +1961,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1913 .add_safe,1961 .add_safe,
1914 .sub_safe,1962 .sub_safe,
1915 .mul_safe,1963 .mul_safe,
1916 .intcast_safe,1964 .int_cast_safe,
1917 .int_from_float_safe,1965 .int_from_float_safe,
1918 .int_from_float_optimized_safe,1966 .int_from_float_optimized_safe,
1919 .legalize_vec_store_elem,1967 .legalize_vec_store_elem,
...@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1965,7 +2013,14 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1965 .shl_sat,2013 .shl_sat,
1966 .xor,2014 .xor,
1967 .not,2015 .not,
1968 .bitcast,2016 .bit_cast,
2017 .ptr_cast,
2018 .ptr_from_int,
2019 .int_from_ptr,
2020 .error_cast,
2021 .error_from_int,
2022 .int_from_error,
2023 .union_from_enum,
1969 .ret_addr,2024 .ret_addr,
1970 .frame_addr,2025 .frame_addr,
1971 .clz,2026 .clz,
...@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -2009,7 +2064,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
2009 .is_non_err,2064 .is_non_err,
2010 .fptrunc,2065 .fptrunc,
2011 .fpext,2066 .fpext,
2012 .intcast,2067 .int_cast,
2013 .trunc,2068 .trunc,
2014 .optional_payload,2069 .optional_payload,
2015 .optional_payload_ptr,2070 .optional_payload_ptr,
src/Air/Legalize.zig+133-74
...@@ -75,6 +75,9 @@ pub const Feature = enum {...@@ -75,6 +75,9 @@ pub const Feature = enum {
75 scalarize_shl_sat,75 scalarize_shl_sat,
76 scalarize_xor,76 scalarize_xor,
77 scalarize_not,77 scalarize_not,
78 scalarize_ptr_cast,
79 scalarize_ptr_from_int,
80 scalarize_int_from_ptr,
78 scalarize_clz,81 scalarize_clz,
79 scalarize_ctz,82 scalarize_ctz,
80 scalarize_popcount,83 scalarize_popcount,
...@@ -100,8 +103,8 @@ pub const Feature = enum {...@@ -100,8 +103,8 @@ pub const Feature = enum {
100 scalarize_cmp_vector_optimized,103 scalarize_cmp_vector_optimized,
101 scalarize_fptrunc,104 scalarize_fptrunc,
102 scalarize_fpext,105 scalarize_fpext,
103 scalarize_intcast,106 scalarize_int_cast,
104 scalarize_intcast_safe,107 scalarize_int_cast_safe,
105 scalarize_trunc,108 scalarize_trunc,
106 scalarize_int_from_float,109 scalarize_int_from_float,
107 scalarize_int_from_float_optimized,110 scalarize_int_from_float_optimized,
...@@ -115,24 +118,24 @@ pub const Feature = enum {...@@ -115,24 +118,24 @@ pub const Feature = enum {
115 scalarize_select,118 scalarize_select,
116 scalarize_mul_add,119 scalarize_mul_add,
117120
118 // Below are several different features for scalarizing `bitcast` in different scenarios. It is121 // Below are several different features for scalarizing `bit_cast` in different scenarios. It is
119 // valid to enable any combination of these features.122 // valid to enable any combination of these features.
120123
121 /// Scalarize `bitcast` where the operand or result type is an array.124 /// Scalarize `bit_cast` where the operand or result type is an array.
122 scalarize_bitcast_array,125 scalarize_bit_cast_array,
123 /// Scalarize `bitcast` where either:126 /// Scalarize `bit_cast` where either:
124 ///127 ///
125 /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or128 /// * operand type is `@Vector(n, A), but result type is not `@Vector(n, B)`; or
126 /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`129 /// * result type is `@Vector(n, A), but operand type is not `@Vector(n, B)`
127 ///130 ///
128 /// This effectively scalarizes any `bitcast` to/from a vector, *unless* the operation can be131 /// This effectively scalarizes any `bit_cast` to/from a vector, *unless* the operation can be
129 /// performed by bitcasting each vector element and returning a vector of the results.132 /// performed by bitcasting each vector element and returning a vector of the results.
130 ///133 ///
131 /// If this feature is enabled, the following AIR instruction tags may be emitted:134 /// If this feature is enabled, the following AIR instruction tags may be emitted:
132 /// * `.legalize_vec_elem_val`135 /// * `.legalize_vec_elem_val`
133 /// * `.legalize_vec_store_elem`136 /// * `.legalize_vec_store_elem`
134 scalarize_bitcast_vector_non_elementwise,137 scalarize_bit_cast_vector_non_elementwise,
135 /// Scalarize `bitcast` where the operand or result type is an array or vector whose element138 /// Scalarize `bit_cast` where the operand or result type is an array or vector whose element
136 /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may139 /// type `E` has `@bitSizeOf(E) != 8 * @sizeOf(E)`. These are the cases where the backend may
137 /// need to sign- or zero-extend multiple elements to populate "padding" bits.140 /// need to sign- or zero-extend multiple elements to populate "padding" bits.
138 ///141 ///
...@@ -142,18 +145,18 @@ pub const Feature = enum {...@@ -142,18 +145,18 @@ pub const Feature = enum {
142 /// If this feature is enabled, the following AIR instruction tags may be emitted:145 /// If this feature is enabled, the following AIR instruction tags may be emitted:
143 /// * `.legalize_vec_elem_val`146 /// * `.legalize_vec_elem_val`
144 /// * `.legalize_vec_store_elem`147 /// * `.legalize_vec_store_elem`
145 scalarize_bitcast_padded_elems,148 scalarize_bit_cast_padded_elems,
146149
147 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)150 /// Legalize (shift lhs, (splat rhs)) -> (shift lhs, rhs)
148 unsplat_shift_rhs,151 unsplat_shift_rhs,
149 /// Legalize reduce of a one element vector to a bitcast.152 /// Legalize reduce of a one element vector to a bitcast.
150 reduce_one_elem_to_bitcast,153 reduce_one_elem_to_bit_cast,
151 /// Legalize splat to a one element vector to a bitcast.154 /// Legalize splat to a one element vector to a bitcast.
152 splat_one_elem_to_bitcast,155 splat_one_elem_to_bit_cast,
153156
154 /// Replace `intcast_safe` with an explicit safety check which `call`s the panic function on failure.157 /// Replace `int_cast_safe` with an explicit safety check which `call`s the panic function on failure.
155 /// Not compatible with `scalarize_intcast_safe`.158 /// Not compatible with `scalarize_int_cast_safe`.
156 expand_intcast_safe,159 expand_int_cast_safe,
157 /// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.160 /// Replace `int_from_float_safe` with an explicit safety check which `call`s the panic function on failure.
158 /// Not compatible with `scalarize_int_from_float_safe`.161 /// Not compatible with `scalarize_int_from_float_safe`.
159 expand_int_from_float_safe,162 expand_int_from_float_safe,
...@@ -178,9 +181,9 @@ pub const Feature = enum {...@@ -178,9 +181,9 @@ pub const Feature = enum {
178 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the181 /// Currently assumes little endian and a specific integer layout where the lsb of every integer is the lsb of the
179 /// first byte of memory until bit pointers know their backing type.182 /// first byte of memory until bit pointers know their backing type.
180 expand_packed_store,183 expand_packed_store,
181 /// Replace `struct_field_val` of a packed field with a `bitcast` to integer, `shr`, `trunc`, and `bitcast` to field type.184 /// Replace `struct_field_val` of a packed field with a `bit_cast` to integer, `shr`, `trunc`, and `bit_cast` to field type.
182 expand_packed_struct_field_val,185 expand_packed_struct_field_val,
183 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.186 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bit_cast`, `int_cast`, and `bit_or`.
184 expand_packed_aggregate_init,187 expand_packed_aggregate_init,
185188
186 /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float189 /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
...@@ -274,8 +277,11 @@ pub const Feature = enum {...@@ -274,8 +277,11 @@ pub const Feature = enum {
274 .cmp_vector_optimized => .scalarize_cmp_vector_optimized,277 .cmp_vector_optimized => .scalarize_cmp_vector_optimized,
275 .fptrunc => .scalarize_fptrunc,278 .fptrunc => .scalarize_fptrunc,
276 .fpext => .scalarize_fpext,279 .fpext => .scalarize_fpext,
277 .intcast => .scalarize_intcast,280 .int_cast => .scalarize_int_cast,
278 .intcast_safe => .scalarize_intcast_safe,281 .int_cast_safe => .scalarize_int_cast_safe,
282 .ptr_cast => .scalarize_ptr_cast,
283 .ptr_from_int => .scalarize_ptr_from_int,
284 .int_from_ptr => .scalarize_int_from_ptr,
279 .trunc => .scalarize_trunc,285 .trunc => .scalarize_trunc,
280 .int_from_float => .scalarize_int_from_float,286 .int_from_float => .scalarize_int_from_float,
281 .int_from_float_optimized => .scalarize_int_from_float_optimized,287 .int_from_float_optimized => .scalarize_int_from_float_optimized,
...@@ -495,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -495,7 +501,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
495 .popcount,501 .popcount,
496 .byte_swap,502 .byte_swap,
497 .bit_reverse,503 .bit_reverse,
498 .intcast,504 .int_cast,
505 .ptr_cast,
506 .ptr_from_int,
507 .int_from_ptr,
499 .trunc,508 .trunc,
500 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {509 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
501 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;510 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
...@@ -569,19 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -569,19 +578,19 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
569 },578 },
570 }579 }
571 },580 },
572 .bitcast => if (l.features.hasAny(&.{581 .bit_cast => if (l.features.hasAny(&.{
573 .scalarize_bitcast_array,582 .scalarize_bit_cast_array,
574 .scalarize_bitcast_vector_non_elementwise,583 .scalarize_bit_cast_vector_non_elementwise,
575 .scalarize_bitcast_padded_elems,584 .scalarize_bit_cast_padded_elems,
576 })) {585 })) {
577 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {586 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
578 continue :inst l.replaceInst(inst, .block, payload);587 continue :inst l.replaceInst(inst, .block, payload);
579 }588 }
580 },589 },
581 .intcast_safe => if (l.features.has(.expand_intcast_safe)) {590 .int_cast_safe => if (l.features.has(.expand_int_cast_safe)) {
582 assert(!l.features.has(.scalarize_intcast_safe)); // it doesn't make sense to do both591 assert(!l.features.has(.scalarize_int_cast_safe)); // it doesn't make sense to do both
583 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));592 continue :inst l.replaceInst(inst, .block, try l.safeIntcastBlockPayload(inst));
584 } else if (l.features.has(.scalarize_intcast_safe)) {593 } else if (l.features.has(.scalarize_int_cast_safe)) {
585 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;594 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
586 if (ty_op.ty.toType().isVector(zcu)) {595 if (ty_op.ty.toType().isVector(zcu)) {
587 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));596 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
...@@ -797,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -797,10 +806,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
797 inline .reduce, .reduce_optimized => |air_tag| {806 inline .reduce, .reduce_optimized => |air_tag| {
798 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;807 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
799 const vector_ty = l.typeOf(reduce.operand);808 const vector_ty = l.typeOf(reduce.operand);
800 if (l.features.has(.reduce_one_elem_to_bitcast)) {809 if (l.features.has(.reduce_one_elem_to_bit_cast)) {
801 switch (vector_ty.vectorLen(zcu)) {810 switch (vector_ty.vectorLen(zcu)) {
802 0 => unreachable,811 0 => unreachable,
803 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{812 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
804 .ty = .fromType(vector_ty.childType(zcu)),813 .ty = .fromType(vector_ty.childType(zcu)),
805 .operand = reduce.operand,814 .operand = reduce.operand,
806 } }),815 } }),
...@@ -817,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -817,11 +826,11 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
817 .soft_float => unreachable, // the operand is not a scalar826 .soft_float => unreachable, // the operand is not a scalar
818 }827 }
819 },828 },
820 .splat => if (l.features.has(.splat_one_elem_to_bitcast)) {829 .splat => if (l.features.has(.splat_one_elem_to_bit_cast)) {
821 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;830 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
822 switch (ty_op.ty.toType().vectorLen(zcu)) {831 switch (ty_op.ty.toType().vectorLen(zcu)) {
823 0 => unreachable,832 0 => unreachable,
824 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{833 1 => continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
825 .ty = ty_op.ty,834 .ty = ty_op.ty,
826 .operand = ty_op.operand,835 .operand = ty_op.operand,
827 } }),836 } }),
...@@ -887,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -887,7 +896,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
887 const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);896 const field_bits = agg_ty.fieldType(field_index, zcu).bitSize(zcu);
888 if (field_bits == struct_bits) {897 if (field_bits == struct_bits) {
889 // Just bitcast this field.898 // Just bitcast this field.
890 continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{899 continue :inst l.replaceInst(inst, .bit_cast, .{ .ty_op = .{
891 .ty = .fromType(agg_ty),900 .ty = .fromType(agg_ty),
892 .operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),901 .operand = @enumFromInt(l.air_extra.items[ty_pl.payload + field_index]),
893 } });902 } });
...@@ -934,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -934,6 +943,10 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
934 .legalize_vec_store_elem,943 .legalize_vec_store_elem,
935 .legalize_compiler_rt_call,944 .legalize_compiler_rt_call,
936 .spirv_runtime_array_len,945 .spirv_runtime_array_len,
946 .error_cast,
947 .error_from_int,
948 .int_from_error,
949 .union_from_enum,
937 => {},950 => {},
938 }951 }
939 }952 }
...@@ -956,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz...@@ -956,7 +969,7 @@ fn scalarizeBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, form: Scalariz
956969
957 if (result_is_array) {970 if (result_is_array) {
958 // This is only allowed when legalizing an elementwise bitcast.971 // This is only allowed when legalizing an elementwise bitcast.
959 assert(orig.tag == .bitcast);972 assert(orig.tag == .bit_cast);
960 assert(form == .ty_op);973 assert(form == .ty_op);
961 }974 }
962975
...@@ -1474,7 +1487,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1474,7 +1487,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1474 var operand_to_int: bool = true;1487 var operand_to_int: bool = true;
1475 var int_to_dest: bool = true;1488 var int_to_dest: bool = true;
14761489
1477 if (l.features.has(.scalarize_bitcast_array)) {1490 if (l.features.has(.scalarize_bit_cast_array)) {
1478 if (operand_tag == .array) {1491 if (operand_tag == .array) {
1479 operand_to_dest = false;1492 operand_to_dest = false;
1480 operand_to_int = false;1493 operand_to_int = false;
...@@ -1485,7 +1498,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1485,7 +1498,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1485 }1498 }
1486 }1499 }
14871500
1488 if (l.features.has(.scalarize_bitcast_vector_non_elementwise)) {1501 if (l.features.has(.scalarize_bit_cast_vector_non_elementwise)) {
1489 if (operand_tag == .vector) operand_to_int = false;1502 if (operand_tag == .vector) operand_to_int = false;
1490 if (dest_tag == .vector) int_to_dest = false;1503 if (dest_tag == .vector) int_to_dest = false;
14911504
...@@ -1499,7 +1512,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1499,7 +1512,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1499 }1512 }
1500 }1513 }
15011514
1502 if (l.features.has(.scalarize_bitcast_padded_elems)) {1515 if (l.features.has(.scalarize_bit_cast_padded_elems)) {
1503 if (operand_tag == .array or operand_tag == .vector) {1516 if (operand_tag == .array or operand_tag == .vector) {
1504 const elem_ty = operand_ty.childType(zcu);1517 const elem_ty = operand_ty.childType(zcu);
1505 if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {1518 if (elem_ty.bitSize(zcu) != 8 * elem_ty.abiSize(zcu)) {
...@@ -1554,6 +1567,12 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1554,6 +1567,12 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1554 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);1567 break :uint_val main_block.addBitCast(l, uint_ty, ty_op.operand);
1555 }1568 }
15561569
1570 if (operand_ty.arrayLenIncludingSentinel(zcu) == 1) {
1571 _ = main_block.stealCapacity(18);
1572 const elem = main_block.addBinOp(l, .array_elem_val, ty_op.operand, .zero_usize).toRef();
1573 break :uint_val main_block.addBitCast(l, uint_ty, elem);
1574 }
1575
1557 // %1 = block({1576 // %1 = block({
1558 // %2 = alloc(*usize)1577 // %2 = alloc(*usize)
1559 // %3 = alloc(*uN)1578 // %3 = alloc(*uN)
...@@ -1562,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1562,8 +1581,8 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1562 // %6 = loop({1581 // %6 = loop({
1563 // %7 = load(%2)1582 // %7 = load(%2)
1564 // %8 = array_elem_val(orig_operand, %7)1583 // %8 = array_elem_val(orig_operand, %7)
1565 // %9 = bitcast(uE, %8)1584 // %9 = bit_cast(uE, %8)
1566 // %10 = intcast(uN, %9)1585 // %10 = int_cast(uN, %9)
1567 // %11 = load(%3)1586 // %11 = load(%3)
1568 // %12 = shl_exact(%11, <uS, E>)1587 // %12 = shl_exact(%11, <uS, E>)
1569 // %13 = bit_or(%12, %10)1588 // %13 = bit_or(%12, %10)
...@@ -1613,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1613,7 +1632,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1613 index_val,1632 index_val,
1614 ).toRef();1633 ).toRef();
1615 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);1634 const elem_uint = loop.block.addBitCast(l, elem_uint_ty, raw_elem);
1616 const elem_extended = loop.block.addTyOp(l, .intcast, uint_ty, elem_uint).toRef();1635 const elem_extended = loop.block.addTyOp(l, .int_cast, uint_ty, elem_uint).toRef();
1617 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();1636 const old_result = loop.block.addTyOp(l, .load, uint_ty, result_ptr).toRef();
1618 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();1637 const shifted_result = loop.block.addBinOp(l, .shl_exact, old_result, .fromValue(elem_bits_val)).toRef();
1619 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();1638 const new_result = loop.block.addBinOp(l, .bit_or, shifted_result, elem_extended).toRef();
...@@ -1648,6 +1667,19 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1648,6 +1667,19 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1648 _ = main_block.stealCapacity(17);1667 _ = main_block.stealCapacity(17);
1649 const result = main_block.addBitCast(l, dest_ty, uint_val);1668 const result = main_block.addBitCast(l, dest_ty, uint_val);
1650 main_block.addBr(l, orig_inst, result);1669 main_block.addBr(l, orig_inst, result);
1670 } else if (dest_ty.arrayLenIncludingSentinel(zcu) == 1) {
1671 _ = main_block.stealCapacity(16);
1672 const elem = main_block.addBitCast(l, dest_ty.childType(zcu), uint_val);
1673 const aggregate_init_payload_start = l.air_extra.items.len;
1674 try l.air_extra.append(zcu.gpa, @intFromEnum(elem));
1675 const result = main_block.add(l, .{
1676 .tag = .aggregate_init,
1677 .data = .{ .ty_pl = .{
1678 .ty = .fromType(dest_ty),
1679 .payload = @intCast(aggregate_init_payload_start),
1680 } },
1681 }).toRef();
1682 main_block.addBr(l, orig_inst, result);
1651 } else {1683 } else {
1652 // %1 = alloc(*usize)1684 // %1 = alloc(*usize)
1653 // %2 = alloc(*@Vector(N, Result))1685 // %2 = alloc(*@Vector(N, Result))
...@@ -1655,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1655,10 +1687,10 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
1655 // %4 = loop({1687 // %4 = loop({
1656 // %5 = load(%1)1688 // %5 = load(%1)
1657 // %6 = mul(%5, <usize, E>)1689 // %6 = mul(%5, <usize, E>)
1658 // %7 = intcast(uS, %6)1690 // %7 = int_cast(uS, %6)
1659 // %8 = shr(uint_val, %7)1691 // %8 = shr(uint_val, %7)
1660 // %9 = trunc(uE, %8)1692 // %9 = trunc(uE, %8)
1661 // %10 = bitcast(Result, %9)1693 // %10 = bit_cast(Result, %9)
1662 // %11 = legalize_vec_store_elem(%2, %5, %10)1694 // %11 = legalize_vec_store_elem(%2, %5, %10)
1663 // %12 = cmp_eq(%5, <usize, vec_len>)1695 // %12 = cmp_eq(%5, <usize, vec_len>)
1664 // %13 = cond_br(%12, {1696 // %13 = cond_br(%12, {
...@@ -1687,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?...@@ -1687,7 +1719,7 @@ fn scalarizeBitcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!?
16871719
1688 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();1720 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1689 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();1721 const bit_offset = loop.block.addBinOp(l, .mul, index_val, .fromValue(try pt.intValue(.usize, elem_bits))).toRef();
1690 const casted_bit_offset = loop.block.addTyOp(l, .intcast, shift_ty, bit_offset).toRef();1722 const casted_bit_offset = loop.block.addTyOp(l, .int_cast, shift_ty, bit_offset).toRef();
1691 const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();1723 const shifted_uint = loop.block.addBinOp(l, .shr, uint_val, casted_bit_offset).toRef();
1692 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();1724 const elem_uint = loop.block.addTyOp(l, .trunc, elem_uint_ty, shifted_uint).toRef();
1693 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);1725 const elem_val = loop.block.addBitCast(l, elem_ty, elem_uint);
...@@ -2065,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2065,7 +2097,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2065 // %5 = call(@panic.invalidEnumValue, [])2097 // %5 = call(@panic.invalidEnumValue, [])
2066 // %6 = unreach()2098 // %6 = unreach()
2067 // }, {2099 // }, {
2068 // %7 = intcast(@res_ty, %y)2100 // %7 = int_cast(@res_ty, %y)
2069 // %8 = is_named_enum_value(%7)2101 // %8 = is_named_enum_value(%7)
2070 // %9 = cond_br(%8, {2102 // %9 = cond_br(%8, {
2071 // %10 = br(%x, %7)2103 // %10 = br(%x, %7)
...@@ -2087,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2087,7 +2119,7 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2087 // %6 = call(@panic.invalidEnumValue, [])2119 // %6 = call(@panic.invalidEnumValue, [])
2088 // %7 = unreach()2120 // %7 = unreach()
2089 // }, {2121 // }, {
2090 // %8 = intcast(@res_ty, %y)2122 // %8 = int_cast(@res_ty, %y)
2091 // %9 = br(%x, %8)2123 // %9 = br(%x, %8)
2092 // })2124 // })
2093 // })2125 // })
...@@ -2140,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2140,9 +2172,9 @@ fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2140 cur_block = &condbr.else_block;2172 cur_block = &condbr.else_block;
2141 }2173 }
21422174
2143 // Now we know we're in-range, we can intcast:2175 // Now we know we're in-range, we can int_cast:
2144 const cast_inst = cur_block.add(l, .{2176 const cast_inst = cur_block.add(l, .{
2145 .tag = .intcast,2177 .tag = .int_cast,
2146 .data = .{ .ty_op = .{2178 .data = .{ .ty_op = .{
2147 .ty = Air.internedToRef(dest_ty.toIntern()),2179 .ty = Air.internedToRef(dest_ty.toIntern()),
2148 .operand = operand_ref,2180 .operand = operand_ref,
...@@ -2313,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_...@@ -2313,7 +2345,7 @@ fn safeArithmeticBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, overflow_
2313 // %1 = add_with_overflow(%x, %y)2345 // %1 = add_with_overflow(%x, %y)
2314 // %2 = struct_field_val(%1, .@"1")2346 // %2 = struct_field_val(%1, .@"1")
2315 // %3 = reduce(%2, .@"or")2347 // %3 = reduce(%2, .@"or")
2316 // %4 = bitcast(%3, @bool_type)2348 // %4 = bit_cast(%3, @bool_type)
2317 // %5 = cond_br(%4, {2349 // %5 = cond_br(%4, {
2318 // %6 = call(@panic.integerOverflow, [])2350 // %6 = call(@panic.integerOverflow, [])
2319 // %7 = unreach()2351 // %7 = unreach()
...@@ -2419,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins...@@ -2419,7 +2451,7 @@ fn packedLoadBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Ins
2419 .tag = .load,2451 .tag = .load,
2420 .data = .{ .ty_op = .{2452 .data = .{ .ty_op = .{
2421 .ty = Air.internedToRef(load_ty.toIntern()),2453 .ty = Air.internedToRef(load_ty.toIntern()),
2422 .operand = res_block.addBitCast(l, load_ptr_ty: {2454 .operand = res_block.addPtrCast(l, load_ptr_ty: {
2423 var load_ptr_info = ptr_info;2455 var load_ptr_info = ptr_info;
2424 load_ptr_info.child = load_ty.toIntern();2456 load_ptr_info.child = load_ty.toIntern();
2425 load_ptr_info.flags.vector_index = .none;2457 load_ptr_info.flags.vector_index = .none;
...@@ -2462,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2462,23 +2494,17 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
24622494
2463 var res_block: Block = .init(&inst_buf);2495 var res_block: Block = .init(&inst_buf);
2464 {2496 {
2465 const backing_ptr_inst = res_block.add(l, .{2497 const backing_ptr = res_block.addPtrCast(l, load_store_ptr_ty: {
2466 .tag = .bitcast,2498 var load_ptr_info = ptr_info;
2467 .data = .{ .ty_op = .{2499 load_ptr_info.child = load_store_ty.toIntern();
2468 .ty = Air.internedToRef((load_store_ptr_ty: {2500 load_ptr_info.flags.vector_index = .none;
2469 var load_ptr_info = ptr_info;2501 load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2470 load_ptr_info.child = load_store_ty.toIntern();2502 break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2471 load_ptr_info.flags.vector_index = .none;2503 }, orig_bin_op.lhs);
2472 load_ptr_info.packed_offset = .{ .host_size = 0, .bit_offset = 0 };
2473 break :load_store_ptr_ty try pt.ptrType(load_ptr_info);
2474 }).toIntern()),
2475 .operand = orig_bin_op.lhs,
2476 } },
2477 });
2478 _ = res_block.add(l, .{2504 _ = res_block.add(l, .{
2479 .tag = .store,2505 .tag = .store,
2480 .data = .{ .bin_op = .{2506 .data = .{ .bin_op = .{
2481 .lhs = backing_ptr_inst.toRef(),2507 .lhs = backing_ptr,
2482 .rhs = res_block.add(l, .{2508 .rhs = res_block.add(l, .{
2483 .tag = .bit_or,2509 .tag = .bit_or,
2484 .data = .{ .bin_op = .{2510 .data = .{ .bin_op = .{
...@@ -2489,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2489,7 +2515,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2489 .tag = .load,2515 .tag = .load,
2490 .data = .{ .ty_op = .{2516 .data = .{ .ty_op = .{
2491 .ty = Air.internedToRef(load_store_ty.toIntern()),2517 .ty = Air.internedToRef(load_store_ty.toIntern()),
2492 .operand = backing_ptr_inst.toRef(),2518 .operand = backing_ptr,
2493 } },2519 } },
2494 }).toRef(),2520 }).toRef(),
2495 .rhs = Air.internedToRef((keep_mask: {2521 .rhs = Air.internedToRef((keep_mask: {
...@@ -2518,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In...@@ -2518,7 +2544,7 @@ fn packedStoreBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.In
2518 .tag = .shl_exact,2544 .tag = .shl_exact,
2519 .data = .{ .bin_op = .{2545 .data = .{ .bin_op = .{
2520 .lhs = res_block.add(l, .{2546 .lhs = res_block.add(l, .{
2521 .tag = .intcast,2547 .tag = .int_cast,
2522 .data = .{ .ty_op = .{2548 .data = .{ .ty_op = .{
2523 .ty = Air.internedToRef(load_store_ty.toIntern()),2549 .ty = Air.internedToRef(load_store_ty.toIntern()),
2524 .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),2550 .operand = res_block.addBitCast(l, operand_int_ty, orig_bin_op.rhs),
...@@ -2616,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro...@@ -2616,7 +2642,7 @@ fn packedAggregateInitBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Erro
26162642
2617 const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();2643 const shifted = main_block.addBinOp(l, .shl_exact, cur_uint, field_bit_size_ref).toRef();
2618 const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);2644 const field_as_uint = main_block.addBitCast(l, field_uint_ty, field_val);
2619 const field_extended = main_block.addTyOp(l, .intcast, uint_ty, field_as_uint).toRef();2645 const field_extended = main_block.addTyOp(l, .int_cast, uint_ty, field_as_uint).toRef();
2620 cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();2646 cur_uint = main_block.addBinOp(l, .bit_or, shifted, field_extended).toRef();
2621 }2647 }
26222648
...@@ -2805,18 +2831,51 @@ const Block = struct {...@@ -2805,18 +2831,51 @@ const Block = struct {
2805 });2831 });
2806 }2832 }
28072833
2808 /// Adds a `bitcast` instruction to `b`. This is a thin wrapper that omits the instruction for2834 /// Adds a `bit_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2809 /// no-op casts.2835 /// no-op casts.
2810 fn addBitCast(2836 fn addBitCast(
2811 b: *Block,2837 b: *Block,
2812 l: *Legalize,2838 l: *Legalize,
2813 ty: Type,2839 result_ty: Type,
2814 operand: Air.Inst.Ref,2840 operand: Air.Inst.Ref,
2815 ) Air.Inst.Ref {2841 ) Air.Inst.Ref {
2816 if (ty.toIntern() != l.typeOf(operand).toIntern()) return b.add(l, .{2842 const zcu = l.pt.zcu;
2817 .tag = .bitcast,2843 const operand_ty = l.typeOf(operand);
2844 assert(!operand_ty.isPtrAtRuntime(zcu));
2845 assert(!operand_ty.isSliceAtRuntime(zcu));
2846 assert(!result_ty.isPtrAtRuntime(zcu));
2847 assert(!result_ty.isSliceAtRuntime(zcu));
2848 if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
2849 .tag = .bit_cast,
2850 .data = .{ .ty_op = .{
2851 .ty = .fromType(result_ty),
2852 .operand = operand,
2853 } },
2854 }).toRef();
2855 _ = b.stealCapacity(1);
2856 return operand;
2857 }
2858
2859 /// Adds a `ptr_cast` instruction to `b`. This is a thin wrapper that omits the instruction for
2860 /// no-op casts.
2861 fn addPtrCast(
2862 b: *Block,
2863 l: *Legalize,
2864 result_ty: Type,
2865 operand: Air.Inst.Ref,
2866 ) Air.Inst.Ref {
2867 const zcu = l.pt.zcu;
2868 const operand_ty = l.typeOf(operand);
2869 if (operand_ty.isSliceAtRuntime(zcu)) {
2870 assert(result_ty.isSliceAtRuntime(zcu));
2871 } else {
2872 assert(operand_ty.isPtrAtRuntime(zcu));
2873 assert(result_ty.isPtrAtRuntime(zcu));
2874 }
2875 if (result_ty.toIntern() != operand_ty.toIntern()) return b.add(l, .{
2876 .tag = .ptr_cast,
2818 .data = .{ .ty_op = .{2877 .data = .{ .ty_op = .{
2819 .ty = Air.internedToRef(ty.toIntern()),2878 .ty = .fromType(result_ty),
2820 .operand = operand,2879 .operand = operand,
2821 } },2880 } },
2822 }).toRef();2881 }).toRef();
...@@ -3157,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3157,7 +3216,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3157 var main_block: Block = .init(&inst_buf);3216 var main_block: Block = .init(&inst_buf);
3158 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3217 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
31593218
3160 const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();3219 const extended_val = main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3161 const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});3220 const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3162 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());3221 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3163 main_block.addBr(l, orig_inst, casted_result);3222 main_block.addBr(l, orig_inst, casted_result);
...@@ -3184,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3184,7 +3243,7 @@ fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3184 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3243 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
31853244
3186 const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {3245 const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3187 break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();3246 break :ext main_block.addTyOp(l, .int_cast, extended_ty, ty_op.operand).toRef();
3188 } else ext: {3247 } else ext: {
3189 _ = main_block.stealCapacity(1);3248 _ = main_block.stealCapacity(1);
3190 break :ext ty_op.operand;3249 break :ext ty_op.operand;
...@@ -3249,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3249,7 +3308,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3249 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);3308 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
32503309
3251 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});3310 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3252 const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();3311 const casted_val = main_block.addTyOp(l, .int_cast, dest_ty, call_inst.toRef()).toRef();
3253 main_block.addBr(l, orig_inst, casted_val);3312 main_block.addBr(l, orig_inst, casted_val);
32543313
3255 return .{ .block_payload = .{ .ty_pl = .{3314 return .{ .block_payload = .{ .ty_pl = .{
...@@ -3273,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {...@@ -3273,7 +3332,7 @@ fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3273 const bits_val = try pt.intValue(.usize, dest_info.bits);3332 const bits_val = try pt.intValue(.usize, dest_info.bits);
3274 _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });3333 _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3275 const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();3334 const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3276 const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();3335 const result_val = main_block.addTyOp(l, .int_cast, dest_ty, extended_val).toRef();
3277 main_block.addBr(l, orig_inst, result_val);3336 main_block.addBr(l, orig_inst, result_val);
32783337
3279 return .{ .block_payload = .{ .ty_pl = .{3338 return .{ .block_payload = .{ .ty_pl = .{
src/Air/Liveness.zig+10-3
...@@ -488,12 +488,19 @@ fn analyzeInst(...@@ -488,12 +488,19 @@ fn analyzeInst(
488 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),488 => return analyzeFuncEnd(a, pass, data, inst, .{ .none, .none, .none }),
489489
490 .not,490 .not,
491 .bitcast,491 .bit_cast,
492 .ptr_cast,
493 .ptr_from_int,
494 .int_from_ptr,
495 .error_cast,
496 .error_from_int,
497 .int_from_error,
498 .union_from_enum,
492 .load,499 .load,
493 .fpext,500 .fpext,
494 .fptrunc,501 .fptrunc,
495 .intcast,502 .int_cast,
496 .intcast_safe,503 .int_cast_safe,
497 .trunc,504 .trunc,
498 .optional_payload,505 .optional_payload,
499 .optional_payload_ptr,506 .optional_payload_ptr,
src/Air/Liveness/Verify.zig+10-3
...@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -78,12 +78,19 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
7878
79 // unary79 // unary
80 .not,80 .not,
81 .bitcast,81 .bit_cast,
82 .ptr_cast,
83 .ptr_from_int,
84 .int_from_ptr,
85 .error_cast,
86 .error_from_int,
87 .int_from_error,
88 .union_from_enum,
82 .load,89 .load,
83 .fpext,90 .fpext,
84 .fptrunc,91 .fptrunc,
85 .intcast,92 .int_cast,
86 .intcast_safe,93 .int_cast_safe,
87 .trunc,94 .trunc,
88 .optional_payload,95 .optional_payload,
89 .optional_payload_ptr,96 .optional_payload_ptr,
src/Air/Verify.zig created+465
...@@ -0,0 +1,465 @@
1/// Verifies that AIR is valid, in that every instruction has valid operands and types. In compiler
2/// builds with debug extensions, this is run on all AIR, both before `Air.Legalize` is run and (if
3/// it is run) after it.
4///
5/// This verification pass is currently highly incomplete---expand it as needed.
6const Verify = @This();
7
8zcu: *Zcu,
9func_index: InternPool.Index,
10ret_ty: Type,
11air: *const Air,
12cur_inst: Air.Inst.Index,
13
14pub fn run(pt: Zcu.PerThread, func_index: InternPool.Index, air: *const Air) void {
15 if (!@import("build_options").enable_debug_extensions) {
16 // `Air.Verify` is a debugging feature---it should not be used in release builds because it
17 // has little benefit and negatively affects compiler performance.
18 return;
19 }
20
21 const zcu = pt.zcu;
22
23 const func_ty: Type = Value.fromInterned(func_index).typeOf(zcu);
24 const ret_ty = func_ty.fnReturnType(zcu);
25
26 var verify: Verify = .{
27 .zcu = zcu,
28 .func_index = func_index,
29 .ret_ty = ret_ty,
30 .air = air,
31 .cur_inst = undefined, // populated by `body(...)`
32 };
33 verify.body(air.getMainBody()) catch |verify_err| switch (verify_err) {
34 error.VerifyFail => {
35 const ip = &zcu.intern_pool;
36 const func_nav = ip.indexToKey(func_index).func.owner_nav;
37 const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
38 log.info("AIR for '{s}':", .{func_fqn});
39 const io = zcu.comp.io;
40 const stderr = io.lockStderr(&.{}, null) catch |err| switch (err) {
41 error.Canceled => return io.recancel(),
42 };
43 defer io.unlockStderr();
44 air.write(&stderr.file_writer.interface, pt, null) catch |err| switch (err) {
45 error.WriteFailed => switch (stderr.file_writer.err.?) {
46 error.Canceled => return io.recancel(),
47 else => {},
48 },
49 };
50 },
51 };
52}
53
54const Error = error{VerifyFail};
55
56fn fail(verify: *Verify, msg: []const u8) Error {
57 const ip = &verify.zcu.intern_pool;
58 const func_nav = ip.indexToKey(verify.func_index).func.owner_nav;
59 const func_fqn = ip.getNav(func_nav).fqn.toSlice(ip);
60 log.err("'{s}', %{d}: {s}", .{ func_fqn, verify.cur_inst, msg });
61 return error.VerifyFail;
62}
63
64fn body(verify: *Verify, body_insts: []const Air.Inst.Index) Error!void {
65 const zcu = verify.zcu;
66 const ip = &zcu.intern_pool;
67 const air = verify.air;
68 const tags = air.instructions.items(.tag);
69 const data = air.instructions.items(.data);
70 for (body_insts, 0..) |inst, body_index| {
71 verify.cur_inst = inst;
72 switch (tags[@intFromEnum(inst)]) {
73 .block => {
74 const block = air.unwrapBlock(inst);
75 try verify.body(block.body);
76 },
77 .dbg_inline_block => {
78 const block = air.unwrapDbgBlock(inst);
79 try verify.body(block.body);
80 },
81 .@"try", .try_cold => {
82 const @"try" = air.unwrapTry(inst);
83 try verify.body(@"try".else_body);
84 },
85 .try_ptr, .try_ptr_cold => {
86 const try_ptr = air.unwrapTryPtr(inst);
87 try verify.body(try_ptr.else_body);
88 },
89 .loop => {
90 const block = air.unwrapBlock(inst);
91 try verify.body(block.body);
92 },
93 .cond_br => {
94 const cond_br = air.unwrapCondBr(inst);
95 try verify.body(cond_br.then_body);
96 try verify.body(cond_br.else_body);
97 },
98 .switch_br, .loop_switch_br => {
99 const switch_br = air.unwrapSwitch(inst);
100 var it = switch_br.iterateCases();
101 while (it.next()) |case| {
102 try verify.body(case.body);
103 }
104 const else_body = it.elseBody();
105 if (else_body.len > 0) {
106 try verify.body(else_body);
107 }
108 },
109 .ret, .ret_safe => {
110 const operand = data[@intFromEnum(inst)].un_op;
111 if (air.typeOf(operand, ip).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
112 },
113 .ret_load => {
114 const operand = data[@intFromEnum(inst)].un_op;
115 const ptr_ty = air.typeOf(operand, ip);
116 if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("operand is not a pointer");
117 if (ptr_ty.ptrSize(zcu) != .one) return verify.fail("pointer size is not '.one'");
118 if (ptr_ty.childType(zcu).toIntern() != verify.ret_ty.toIntern()) return verify.fail("bad return type");
119 },
120
121 .bit_cast => {
122 const ty_op = data[@intFromEnum(inst)].ty_op;
123 const operand_ty = air.typeOf(ty_op.operand, ip);
124 const result_ty = ty_op.ty.toType();
125 // Enums are allowed here even if their backing type is implicit.
126 if (!operand_ty.hasBitRepresentation(zcu) and operand_ty.zigTypeTag(zcu) != .@"enum") {
127 return verify.fail("bad operand type");
128 }
129 if (!result_ty.hasBitRepresentation(zcu) and result_ty.zigTypeTag(zcu) != .@"enum") {
130 return verify.fail("bad result type");
131 }
132 if (operand_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type (pointer)");
133 if (result_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type (pointer)");
134 if (operand_ty.bitSize(zcu) != result_ty.bitSize(zcu)) return verify.fail("bit size mismatch");
135 },
136 .ptr_cast => {
137 const ty_op = data[@intFromEnum(inst)].ty_op;
138 const operand_ty = air.typeOf(ty_op.operand, ip);
139 const result_ty = ty_op.ty.toType();
140 const operand_scalar_ty = operand_ty.scalarType(zcu);
141 const result_scalar_ty = result_ty.scalarType(zcu);
142 if (operand_ty.isSliceAtRuntime(zcu)) {
143 if (!result_ty.isSliceAtRuntime(zcu)) return verify.fail("operand is slice, but result is not");
144 } else {
145 if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
146 if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("operand is pointer, but result is not");
147 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
148 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
149 }
150 if (operand_scalar_ty.ptrAddressSpace(zcu) != result_scalar_ty.ptrAddressSpace(zcu)) {
151 return verify.fail("illegal change to address space");
152 }
153 },
154 .ptr_from_int => {
155 const ty_op = data[@intFromEnum(inst)].ty_op;
156 const operand_ty = air.typeOf(ty_op.operand, ip);
157 const result_ty = ty_op.ty.toType();
158 const operand_scalar_ty = operand_ty.scalarType(zcu);
159 const result_scalar_ty = result_ty.scalarType(zcu);
160 if (operand_scalar_ty.toIntern() != .usize_type) return verify.fail("bad operand type");
161 if (!result_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad result type");
162 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
163 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
164 },
165 .int_from_ptr => {
166 const ty_op = data[@intFromEnum(inst)].ty_op;
167 const operand_ty = air.typeOf(ty_op.operand, ip);
168 const result_ty = ty_op.ty.toType();
169 const operand_scalar_ty = operand_ty.scalarType(zcu);
170 const result_scalar_ty = result_ty.scalarType(zcu);
171 if (!operand_scalar_ty.isPtrAtRuntime(zcu)) return verify.fail("bad operand type");
172 if (result_scalar_ty.toIntern() != .usize_type) return verify.fail("bad result type");
173 if (operand_ty.isVector(zcu) and !result_ty.isVector(zcu)) return verify.fail("operand is vector, but result is not");
174 if (!operand_ty.isVector(zcu) and result_ty.isVector(zcu)) return verify.fail("result is vector, but operand is not");
175 },
176 .error_cast => {
177 const ty_op = data[@intFromEnum(inst)].ty_op;
178 const operand_ty = air.typeOf(ty_op.operand, ip);
179 const result_ty = ty_op.ty.toType();
180 switch (operand_ty.zigTypeTag(zcu)) {
181 else => return verify.fail("bad operand type"),
182 .error_union => {
183 if (result_ty.zigTypeTag(zcu) != .error_union) {
184 return verify.fail("operand is error union, but result is not");
185 }
186 if (operand_ty.errorUnionPayload(zcu).toIntern() != result_ty.errorUnionPayload(zcu).toIntern()) {
187 return verify.fail("error union payload type differs");
188 }
189 },
190 .error_set => if (result_ty.zigTypeTag(zcu) != .error_set) {
191 return verify.fail("operand is error set, but result is not");
192 },
193 }
194 },
195 .error_from_int => {
196 const ty_op = data[@intFromEnum(inst)].ty_op;
197 const operand_ty = air.typeOf(ty_op.operand, ip);
198 const result_ty = ty_op.ty.toType();
199 if (!operand_ty.isUnsignedInt(zcu)) return verify.fail("bad operand type");
200 if (operand_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad operand bit size");
201 if (result_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad result type");
202 },
203 .int_from_error => {
204 const ty_op = data[@intFromEnum(inst)].ty_op;
205 const operand_ty = air.typeOf(ty_op.operand, ip);
206 const result_ty = ty_op.ty.toType();
207 if (operand_ty.zigTypeTag(zcu) != .error_set) return verify.fail("bad operand type");
208 if (!result_ty.isUnsignedInt(zcu)) return verify.fail("bad result type");
209 if (result_ty.bitSize(zcu) != zcu.errorSetBits()) return verify.fail("bad result bit size");
210 },
211 .union_from_enum => {
212 const ty_op = data[@intFromEnum(inst)].ty_op;
213 const operand_ty = air.typeOf(ty_op.operand, ip);
214 const result_ty = ty_op.ty.toType();
215 if (operand_ty.zigTypeTag(zcu) != .@"enum") return verify.fail("bad operand type");
216 if (result_ty.zigTypeTag(zcu) != .@"union") return verify.fail("bad result type");
217 const union_tag_ty = result_ty.unionTagType(zcu) orelse return verify.fail("union type is not tagged");
218 if (union_tag_ty.toIntern() != operand_ty.toIntern()) return verify.fail("union tag type does not match operand type");
219 },
220
221 .ptr_elem_ptr => {
222 const ty_pl = data[@intFromEnum(inst)].ty_pl;
223 const bin_op = air.extraData(Air.Bin, ty_pl.payload).data;
224 const ptr_ty = air.typeOf(bin_op.lhs, ip);
225 const result_ty = ty_pl.ty.toType();
226 if (ptr_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad pointer type");
227 if (result_ty.zigTypeTag(zcu) != .pointer) return verify.fail("bad result type");
228 const ptr_info = ptr_ty.ptrInfo(zcu);
229 const result_ptr_info = result_ty.ptrInfo(zcu);
230 if (ptr_info.packed_offset.host_size != 0) return verify.fail("pointer type is bitpacked pointer");
231 if (result_ptr_info.packed_offset.host_size != 0) return verify.fail("result type is bitpacked pointer");
232 },
233
234 .arg,
235 .add,
236 .add_safe,
237 .add_optimized,
238 .add_wrap,
239 .add_sat,
240 .sub,
241 .sub_safe,
242 .sub_optimized,
243 .sub_wrap,
244 .sub_sat,
245 .mul,
246 .mul_safe,
247 .mul_optimized,
248 .mul_wrap,
249 .mul_sat,
250 .div_float,
251 .div_float_optimized,
252 .div_trunc,
253 .div_trunc_optimized,
254 .div_floor,
255 .div_floor_optimized,
256 .div_exact,
257 .div_exact_optimized,
258 .rem,
259 .rem_optimized,
260 .mod,
261 .mod_optimized,
262 .ptr_add,
263 .ptr_sub,
264 .max,
265 .min,
266 .add_with_overflow,
267 .sub_with_overflow,
268 .mul_with_overflow,
269 .shl_with_overflow,
270 .alloc,
271 .inferred_alloc,
272 .inferred_alloc_comptime,
273 .ret_ptr,
274 .assembly,
275 .bit_and,
276 .bit_or,
277 .shr,
278 .shr_exact,
279 .shl,
280 .shl_exact,
281 .shl_sat,
282 .xor,
283 .not,
284 .repeat,
285 .br,
286 .trap,
287 .breakpoint,
288 .ret_addr,
289 .frame_addr,
290 .call,
291 .call_always_tail,
292 .call_never_tail,
293 .call_never_inline,
294 .clz,
295 .ctz,
296 .popcount,
297 .byte_swap,
298 .bit_reverse,
299 .sqrt,
300 .sin,
301 .cos,
302 .tan,
303 .exp,
304 .exp2,
305 .log,
306 .log2,
307 .log10,
308 .abs,
309 .floor,
310 .ceil,
311 .round,
312 .trunc_float,
313 .neg,
314 .neg_optimized,
315 .cmp_lt,
316 .cmp_lt_optimized,
317 .cmp_lte,
318 .cmp_lte_optimized,
319 .cmp_eq,
320 .cmp_eq_optimized,
321 .cmp_gte,
322 .cmp_gte_optimized,
323 .cmp_gt,
324 .cmp_gt_optimized,
325 .cmp_neq,
326 .cmp_neq_optimized,
327 .cmp_vector,
328 .cmp_vector_optimized,
329 .switch_dispatch,
330 .dbg_stmt,
331 .dbg_empty_stmt,
332 .dbg_var_ptr,
333 .dbg_var_val,
334 .dbg_arg_inline,
335 .is_null,
336 .is_non_null,
337 .is_null_ptr,
338 .is_non_null_ptr,
339 .is_err,
340 .is_non_err,
341 .is_err_ptr,
342 .is_non_err_ptr,
343 .load,
344 .store,
345 .store_safe,
346 .unreach,
347 .fptrunc,
348 .fpext,
349 .int_cast,
350 .int_cast_safe,
351 .trunc,
352 .optional_payload,
353 .optional_payload_ptr,
354 .optional_payload_ptr_set,
355 .wrap_optional,
356 .unwrap_errunion_payload,
357 .unwrap_errunion_err,
358 .unwrap_errunion_payload_ptr,
359 .unwrap_errunion_err_ptr,
360 .errunion_payload_ptr_set,
361 .wrap_errunion_payload,
362 .wrap_errunion_err,
363 .struct_field_ptr,
364 .struct_field_ptr_index_0,
365 .struct_field_ptr_index_1,
366 .struct_field_ptr_index_2,
367 .struct_field_ptr_index_3,
368 .struct_field_val,
369 .set_union_tag,
370 .get_union_tag,
371 .slice,
372 .slice_len,
373 .slice_ptr,
374 .ptr_slice_len_ptr,
375 .ptr_slice_ptr_ptr,
376 .array_elem_val,
377 .slice_elem_val,
378 .slice_elem_ptr,
379 .ptr_elem_val,
380 .array_to_slice,
381 .int_from_float,
382 .int_from_float_optimized,
383 .int_from_float_safe,
384 .int_from_float_optimized_safe,
385 .float_from_int,
386 .reduce,
387 .reduce_optimized,
388 .splat,
389 .shuffle_one,
390 .shuffle_two,
391 .select,
392 .memset,
393 .memset_safe,
394 .memcpy,
395 .memmove,
396 .cmpxchg_weak,
397 .cmpxchg_strong,
398 .atomic_load,
399 .atomic_store_unordered,
400 .atomic_store_monotonic,
401 .atomic_store_release,
402 .atomic_store_seq_cst,
403 .atomic_rmw,
404 .is_named_enum_value,
405 .tag_name,
406 .error_name,
407 .error_set_has_value,
408 .aggregate_init,
409 .union_init,
410 .prefetch,
411 .mul_add,
412 .field_parent_ptr,
413 .wasm_memory_size,
414 .wasm_memory_grow,
415 .cmp_lte_errors_len,
416 .err_return_trace,
417 .set_err_return_trace,
418 .addrspace_cast,
419 .save_err_return_trace_index,
420 .runtime_nav_ptr,
421 .c_va_arg,
422 .c_va_copy,
423 .c_va_end,
424 .c_va_start,
425 .spirv_runtime_array_len,
426 .work_item_id,
427 .work_group_size,
428 .work_group_id,
429 .legalize_vec_store_elem,
430 .legalize_vec_elem_val,
431 .legalize_compiler_rt_call,
432 => {},
433 }
434 if (air.typeOfIndex(inst, ip).isNoReturn(zcu)) {
435 if (body_index == body_insts.len - 1) return;
436
437 // HACK: right now, we emit the safety check for noreturn functions returning in a weird
438 // way, where the `call` instruction is `noreturn` but there are still instructions
439 // following it. We need to figure out a better way to represent that! That safety check
440 // probably just needs to live exclusively in backends; putting AIR instructions after a
441 // call implies that we have e.g. a valid stack at that point, which we can't actually
442 // assume when the user has gotten a function's ABI wrong.
443 switch (tags[@intFromEnum(inst)]) {
444 .call,
445 .call_always_tail,
446 .call_never_tail,
447 .call_never_inline,
448 => continue,
449 else => {},
450 }
451
452 return verify.fail("body contains instructions after noreturn");
453 }
454 }
455 return verify.fail("body does not terminate noreturn");
456}
457
458const std = @import("std");
459const log = std.log.scoped(.air_verify);
460
461const Zcu = @import("../Zcu.zig");
462const InternPool = @import("../InternPool.zig");
463const Air = @import("../Air.zig");
464const Type = @import("../Type.zig");
465const Value = @import("../Value.zig");
src/Air/print.zig+10-3
...@@ -232,12 +232,19 @@ const Writer = struct {...@@ -232,12 +232,19 @@ const Writer = struct {
232 .arg => try w.writeArg(s, inst),232 .arg => try w.writeArg(s, inst),
233233
234 .not,234 .not,
235 .bitcast,235 .bit_cast,
236 .ptr_cast,
237 .ptr_from_int,
238 .int_from_ptr,
239 .error_cast,
240 .error_from_int,
241 .int_from_error,
242 .union_from_enum,
236 .load,243 .load,
237 .fptrunc,244 .fptrunc,
238 .fpext,245 .fpext,
239 .intcast,246 .int_cast,
240 .intcast_safe,247 .int_cast_safe,
241 .trunc,248 .trunc,
242 .optional_payload,249 .optional_payload,
243 .optional_payload_ptr,250 .optional_payload_ptr,
src/Sema.zig+376-478
...@@ -583,16 +583,6 @@ pub const Block = struct {...@@ -583,16 +583,6 @@ pub const Block = struct {
583 });583 });
584 }584 }
585585
586 fn addBitCast(block: *Block, ty: Type, operand: Air.Inst.Ref) Allocator.Error!Air.Inst.Ref {
587 return block.addInst(.{
588 .tag = .bitcast,
589 .data = .{ .ty_op = .{
590 .ty = Air.internedToRef(ty.toIntern()),
591 .operand = operand,
592 } },
593 });
594 }
595
596 fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {586 fn addNoOp(block: *Block, tag: Air.Inst.Tag) error{OutOfMemory}!Air.Inst.Ref {
597 return block.addInst(.{587 return block.addInst(.{
598 .tag = tag,588 .tag = tag,
...@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -3113,14 +3103,14 @@ fn zirRefDeref(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
3113 // https://github.com/ziglang/zig/issues/65973103 // https://github.com/ziglang/zig/issues/6597
3114 if (sema.resolveValue(operand)) |operand_val| {3104 if (sema.resolveValue(operand)) |operand_val| {
3115 if (!operand_val.isNull(zcu)) {3105 if (!operand_val.isNull(zcu)) {
3116 break :single_ptr try sema.coerceInMemory(operand_val, single_ptr_ty);3106 break :single_ptr .fromValue(try pt.getCoerced(operand_val, single_ptr_ty));
3117 }3107 }
3118 }3108 }
3119 if (block.wantSafety()) {3109 if (block.wantSafety()) {
3120 const is_non_null = try block.addUnOp(.is_non_null, operand);3110 const is_non_null = try block.addUnOp(.is_non_null, operand);
3121 try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);3111 try sema.addSafetyCheck(block, src, is_non_null, .unwrap_null);
3122 }3112 }
3123 const single_ptr = try block.addBitCast(single_ptr_ty, operand);3113 const single_ptr = try block.addTyOp(.ptr_cast, single_ptr_ty, operand);
3124 try sema.checkKnownAllocPtr(block, operand, single_ptr);3114 try sema.checkKnownAllocPtr(block, operand, single_ptr);
3125 break :single_ptr single_ptr;3115 break :single_ptr single_ptr;
3126 },3116 },
...@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: *Sema, block: *Block, alloc: Air.Inst.Ref,...@@ -3586,7 +3576,7 @@ fn resolveComptimeKnownAllocPtr(sema: *Sema, block: *Block, alloc: Air.Inst.Ref,
3586 .{ .elem = idx_val.toUnsignedInt(zcu) },3576 .{ .elem = idx_val.toUnsignedInt(zcu) },
3587 };3577 };
3588 },3578 },
3589 .bitcast => .{3579 .ptr_cast => .{
3590 tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,3580 tmp_air.instructions.items(.data)[@intFromEnum(air_ptr)].ty_op.operand,
3591 .same_addr,3581 .same_addr,
3592 },3582 },
...@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(...@@ -3729,7 +3719,7 @@ fn finishResolveComptimeKnownAllocPtr(
3729 // This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to3719 // This instruction has type `alloc_ty`, meaning we can rewrite the `alloc` AIR instruction to
3730 // this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this3720 // this one to drop the side effect. We also need to rewrite the stores; we'll turn them to this
3731 // too because it doesn't really matter what they become.3721 // too because it doesn't really matter what they become.
3732 const nop_inst: Air.Inst = .{ .tag = .bitcast, .data = .{ .ty_op = .{3722 const nop_inst: Air.Inst = .{ .tag = .ptr_from_int, .data = .{ .ty_op = .{
3733 .ty = .fromIntern(alloc_ty.toIntern()),3723 .ty = .fromIntern(alloc_ty.toIntern()),
3734 .operand = .zero_usize,3724 .operand = .zero_usize,
3735 } } };3725 } } };
...@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: *Sema, block: *Block, alloc: Air.Inst.Ref) CompileError!Ai...@@ -3779,7 +3769,7 @@ fn makePtrConst(sema: *Sema, block: *Block, alloc: Air.Inst.Ref) CompileError!Ai
3779 return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());3769 return Air.internedToRef((try sema.pt.getCoerced(val, const_ptr_ty)).toIntern());
3780 }3770 }
37813771
3782 return block.addBitCast(const_ptr_ty, alloc);3772 return block.addTyOp(.ptr_cast, const_ptr_ty, alloc);
3783}3773}
37843774
3785fn zirAllocInferredComptime(3775fn zirAllocInferredComptime(
...@@ -7330,7 +7320,7 @@ fn analyzeCall(...@@ -7330,7 +7320,7 @@ fn analyzeCall(
7330 if (resolved_ty == .none) break :r result_raw;7320 if (resolved_ty == .none) break :r result_raw;
7331 // TODO: mutate in place the previous instruction if possible7321 // TODO: mutate in place the previous instruction if possible
7332 // rather than adding a bitcast instruction.7322 // rather than adding a bitcast instruction.
7333 break :r try block.addBitCast(.fromInterned(resolved_ty), result_raw);7323 break :r try block.addTyOp(.error_cast, .fromInterned(resolved_ty), result_raw);
7334 };7324 };
73357325
7336 if (block.isComptime()) {7326 if (block.isComptime()) {
...@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD...@@ -7636,7 +7626,7 @@ fn zirIntFromError(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD
7636 }7626 }
76377627
7638 try sema.requireRuntimeBlock(block, src, operand_src);7628 try sema.requireRuntimeBlock(block, src, operand_src);
7639 return block.addBitCast(err_int_ty, operand);7629 return block.addTyOp(.int_from_error, err_int_ty, operand);
7640}7630}
76417631
7642fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {7632fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!Air.Inst.Ref {
...@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD...@@ -7674,13 +7664,7 @@ fn zirErrorFromInt(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstD
7674 const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);7664 const ok = try block.addBinOp(.bit_and, is_lte_len, is_non_zero);
7675 try sema.addSafetyCheck(block, src, ok, .invalid_error_code);7665 try sema.addSafetyCheck(block, src, ok, .invalid_error_code);
7676 }7666 }
7677 return block.addInst(.{7667 return block.addTyOp(.error_from_int, .anyerror, operand);
7678 .tag = .bitcast,
7679 .data = .{ .ty_op = .{
7680 .ty = .anyerror_type,
7681 .operand = operand,
7682 } },
7683 });
7684}7668}
76857669
7686fn zirMergeErrorSets(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {7670fn zirMergeErrorSets(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -7863,7 +7847,7 @@ fn zirIntFromEnum(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
7863 }7847 }
78647848
7865 try sema.requireRuntimeBlock(block, src, operand_src);7849 try sema.requireRuntimeBlock(block, src, operand_src);
7866 return block.addBitCast(int_tag_ty, enum_tag);7850 return block.addTyOp(.bit_cast, int_tag_ty, enum_tag);
7867}7851}
78687852
7869fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {7853fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -7922,9 +7906,9 @@ fn zirEnumFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
7922 try sema.requireRuntimeBlock(block, src, operand_src);7906 try sema.requireRuntimeBlock(block, src, operand_src);
7923 if (block.wantSafety()) {7907 if (block.wantSafety()) {
7924 try sema.preparePanicId(src, .invalid_enum_value);7908 try sema.preparePanicId(src, .invalid_enum_value);
7925 return block.addTyOp(.intcast_safe, dest_ty, operand);7909 return block.addTyOp(.int_cast_safe, dest_ty, operand);
7926 }7910 }
7927 return block.addTyOp(.intcast, dest_ty, operand);7911 return block.addTyOp(.int_cast, dest_ty, operand);
7928}7912}
79297913
7930/// Pointer in, pointer out.7914/// Pointer in, pointer out.
...@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!...@@ -9152,7 +9136,7 @@ fn zirIntFromPtr(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
9152 try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);9136 try sema.requireRuntimeBlock(block, block.nodeOffset(inst_data.src_node), ptr_src);
9153 try sema.validateRuntimeValue(block, ptr_src, operand);9137 try sema.validateRuntimeValue(block, ptr_src, operand);
9154 try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);9138 try sema.checkLogicalPtrOperation(block, ptr_src, ptr_ty);
9155 return block.addBitCast(dest_ty, operand);9139 return block.addTyOp(.int_from_ptr, dest_ty, operand);
9156}9140}
91579141
9158fn zirFieldPtrLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9142fn zirFieldPtrLoad(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -9314,9 +9298,9 @@ fn intCast(...@@ -9314,9 +9298,9 @@ fn intCast(
9314 try sema.requireRuntimeBlock(block, src, operand_src);9298 try sema.requireRuntimeBlock(block, src, operand_src);
9315 if (block.wantSafety()) {9299 if (block.wantSafety()) {
9316 try sema.preparePanicId(src, .integer_out_of_bounds);9300 try sema.preparePanicId(src, .integer_out_of_bounds);
9317 return block.addTyOp(.intcast_safe, dest_ty, operand);9301 return block.addTyOp(.int_cast_safe, dest_ty, operand);
9318 }9302 }
9319 return block.addTyOp(.intcast, dest_ty, operand);9303 return block.addTyOp(.int_cast, dest_ty, operand);
9320}9304}
93219305
9322fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9306fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -9330,158 +9314,53 @@ fn zirBitcast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
9330 const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");9314 const dest_ty = try sema.resolveDestType(block, src, extra.lhs, .remove_eu_opt, "@bitCast");
9331 const operand = sema.resolveInst(extra.rhs);9315 const operand = sema.resolveInst(extra.rhs);
9332 const operand_ty = sema.typeOf(operand);9316 const operand_ty = sema.typeOf(operand);
9333 switch (dest_ty.zigTypeTag(zcu)) {
9334 .@"anyframe",
9335 .comptime_float,
9336 .comptime_int,
9337 .enum_literal,
9338 .error_set,
9339 .error_union,
9340 .@"fn",
9341 .frame,
9342 .noreturn,
9343 .null,
9344 .@"opaque",
9345 .spirv,
9346 .optional,
9347 .type,
9348 .undefined,
9349 .void,
9350 => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9351
9352 .@"enum" => {
9353 const msg = msg: {
9354 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9355 errdefer msg.destroy(sema.gpa);
9356 switch (operand_ty.zigTypeTag(zcu)) {
9357 .int, .comptime_int => try sema.errNote(src, msg, "use @enumFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9358 else => {},
9359 }
9360
9361 break :msg msg;
9362 };
9363 return sema.failWithOwnedErrorMsg(block, msg);
9364 },
93659317
9366 .pointer => {9318 // Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
9367 const msg = msg: {9319 switch (dest_ty.scalarType(zcu).zigTypeTag(zcu)) {
9368 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});9320 .pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
9369 errdefer msg.destroy(sema.gpa);9321 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9370 switch (operand_ty.zigTypeTag(zcu)) {9322 errdefer msg.destroy(sema.gpa);
9371 .int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),9323 switch (operand_ty.zigTypeTag(zcu)) {
9372 .pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),9324 .int, .comptime_int => try sema.errNote(src, msg, "use @ptrFromInt to cast from '{f}'", .{operand_ty.fmt(pt)}),
9373 else => {},9325 .pointer => try sema.errNote(src, msg, "use @ptrCast to cast from '{f}'", .{operand_ty.fmt(pt)}),
9374 }9326 else => {},
9375
9376 break :msg msg;
9377 };
9378 return sema.failWithOwnedErrorMsg(block, msg);
9379 },
9380 .@"struct", .@"union" => if (dest_ty.containerLayout(zcu) == .auto) {
9381 const container = switch (dest_ty.zigTypeTag(zcu)) {
9382 .@"struct" => "struct",
9383 .@"union" => "union",
9384 else => unreachable,
9385 };
9386 return sema.fail(block, src, "cannot @bitCast to '{f}'; {s} does not have a guaranteed in-memory layout", .{
9387 dest_ty.fmt(pt), container,
9388 });
9389 },
9390 .array => {
9391 const elem_ty = dest_ty.childType(zcu);
9392 if (!elem_ty.hasWellDefinedLayout(zcu)) {
9393 const msg = msg: {
9394 const msg = try sema.errMsg(src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9395 errdefer msg.destroy(sema.gpa);
9396 try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9397 break :msg msg;
9398 };
9399 return sema.failWithOwnedErrorMsg(block, msg);
9400 }9327 }
9401 },
94029328
9403 .bool,9329 break :msg msg;
9404 .float,9330 }),
9405 .int,9331 .array => switch (dest_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
9406 .vector,9332 .pointer, .optional => return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)}),
9407 => {},9333 else => {},
9408 }
9409 switch (operand_ty.zigTypeTag(zcu)) {
9410 .@"anyframe",
9411 .comptime_float,
9412 .comptime_int,
9413 .enum_literal,
9414 .error_set,
9415 .error_union,
9416 .@"fn",
9417 .frame,
9418 .noreturn,
9419 .null,
9420 .@"opaque",
9421 .spirv,
9422 .optional,
9423 .type,
9424 .undefined,
9425 .void,
9426 => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)}),
9427
9428 .@"enum" => {
9429 const msg = msg: {
9430 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9431 errdefer msg.destroy(sema.gpa);
9432 switch (dest_ty.zigTypeTag(zcu)) {
9433 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromEnum to cast to '{f}'", .{dest_ty.fmt(pt)}),
9434 else => {},
9435 }
9436
9437 break :msg msg;
9438 };
9439 return sema.failWithOwnedErrorMsg(block, msg);
9440 },9334 },
9441 .pointer => {9335 else => {},
9442 const msg = msg: {9336 }
9443 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});9337 if (!dest_ty.hasBitRepresentation(zcu)) {
9444 errdefer msg.destroy(sema.gpa);9338 return sema.fail(block, src, "cannot @bitCast to '{f}'", .{dest_ty.fmt(pt)});
9445 switch (dest_ty.zigTypeTag(zcu)) {9339 }
9446 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9447 .pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9448 else => {},
9449 }
94509340
9451 break :msg msg;9341 // Check for pointers before checking `hasBitRepresentation` so we can emit a better message for slices.
9452 };9342 switch (operand_ty.scalarType(zcu).zigTypeTag(zcu)) {
9453 return sema.failWithOwnedErrorMsg(block, msg);9343 .pointer, .optional => return sema.failWithOwnedErrorMsg(block, msg: {
9454 },9344 const msg = try sema.errMsg(operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9455 .@"struct", .@"union" => if (operand_ty.containerLayout(zcu) == .auto) {9345 errdefer msg.destroy(sema.gpa);
9456 const container = switch (operand_ty.zigTypeTag(zcu)) {9346 switch (dest_ty.zigTypeTag(zcu)) {
9457 .@"struct" => "struct",9347 .int, .comptime_int => try sema.errNote(operand_src, msg, "use @intFromPtr to cast to '{f}'", .{dest_ty.fmt(pt)}),
9458 .@"union" => "union",9348 .pointer => try sema.errNote(operand_src, msg, "use @ptrCast to cast to '{f}'", .{dest_ty.fmt(pt)}),
9459 else => unreachable,9349 else => {},
9460 };
9461 return sema.fail(block, operand_src, "cannot @bitCast from '{f}'; {s} does not have a guaranteed in-memory layout", .{
9462 operand_ty.fmt(pt), container,
9463 });
9464 },
9465 .array => {
9466 const elem_ty = operand_ty.childType(zcu);
9467 if (!elem_ty.hasWellDefinedLayout(zcu)) {
9468 const msg = msg: {
9469 const msg = try sema.errMsg(src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9470 errdefer msg.destroy(sema.gpa);
9471 try sema.errNote(src, msg, "array element type '{f}' does not have a guaranteed in-memory layout", .{elem_ty.fmt(pt)});
9472 break :msg msg;
9473 };
9474 return sema.failWithOwnedErrorMsg(block, msg);
9475 }9350 }
9351 break :msg msg;
9352 }),
9353 .array => switch (operand_ty.arrayBase(zcu)[0].zigTypeTag(zcu)) {
9354 .pointer, .optional => return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{dest_ty.fmt(pt)}),
9355 else => {},
9476 },9356 },
94779357 else => {},
9478 .bool,9358 }
9479 .float,9359 if (!operand_ty.hasBitRepresentation(zcu)) {
9480 .int,9360 return sema.fail(block, operand_src, "cannot @bitCast from '{f}'", .{operand_ty.fmt(pt)});
9481 .vector,
9482 => {},
9483 }9361 }
9484 return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node), operand_src);9362
9363 return sema.bitCast(block, dest_ty, operand, block.nodeOffset(inst_data.src_node));
9485}9364}
94869365
9487fn zirFloatCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {9366fn zirFloatCast(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(...@@ -12026,11 +11905,17 @@ fn analyzeSwitchCaptures(
12026 .@"inline" => unreachable, // handled above11905 .@"inline" => unreachable, // handled above
12027 .has_ranges => unreachable, // not possible for error set11906 .has_ranges => unreachable, // not possible for error set
12028 .special => {11907 .special => {
12029 if (else_err_ty) |err_ty| {11908 const capture_err_ty = else_err_ty orelse {
12030 break :payload_ref try sema.bitCast(case_block, err_ty, loaded_operand, operand_src, null);
12031 } else {
12032 try sema.analyzeUnreachable(case_block, operand_src, false);11909 try sema.analyzeUnreachable(case_block, operand_src, false);
12033 break :payload_ref .unreachable_value;11910 break :payload_ref .unreachable_value;
11911 };
11912 if (sema.resolveValue(loaded_operand)) |err_val| {
11913 break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
11914 .ty = capture_err_ty.toIntern(),
11915 .name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
11916 } }));
11917 } else {
11918 break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
12034 }11919 }
12035 },11920 },
12036 .item_refs => |item_refs| {11921 .item_refs => |item_refs| {
...@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(...@@ -12040,8 +11925,15 @@ fn analyzeSwitchCaptures(
12040 const item_val = sema.resolveValue(item_ref).?;11925 const item_val = sema.resolveValue(item_ref).?;
12041 names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});11926 names.putAssumeCapacityNoClobber(item_val.getErrorName(zcu).unwrap().?, {});
12042 }11927 }
12043 const narrowed_ty = try pt.errorSetFromUnsortedNames(names.keys());11928 const capture_err_ty = try pt.errorSetFromUnsortedNames(names.keys());
12044 break :payload_ref try sema.bitCast(case_block, narrowed_ty, loaded_operand, operand_src, null);11929 if (sema.resolveValue(loaded_operand)) |err_val| {
11930 break :payload_ref .fromIntern(try pt.intern(.{ .err = .{
11931 .ty = capture_err_ty.toIntern(),
11932 .name = zcu.intern_pool.indexToKey(err_val.toIntern()).err.name,
11933 } }));
11934 } else {
11935 break :payload_ref try case_block.addTyOp(.error_cast, capture_err_ty, loaded_operand);
11936 }
12045 },11937 },
12046 }11938 }
12047 }11939 }
...@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12259,40 +12151,9 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12259 return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);12151 return case_block.addStructFieldVal(loaded_operand, first_field_index, capture_ty);
12260 }12152 }
1226112153
12262 // We may have to emit a switch block which coerces the operand to the capture type.
12263 // If we can, try to avoid that using in-memory coercions.
12264 const first_non_imc = in_mem: {
12265 for (field_indices, 0..) |field_idx, i| {
12266 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12267 if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12268 break :in_mem i;
12269 }
12270 }
12271 // All fields are in-memory coercible to the resolved type!
12272 // Just take the first field and bitcast the result.
12273 const uncoerced = try case_block.addStructFieldVal(loaded_operand, first_field_index, first_field_ty);
12274 return case_block.addBitCast(capture_ty, uncoerced);
12275 };
12276
12277 // By-val capture with heterogeneous types which are not all in-memory coercible to12154 // By-val capture with heterogeneous types which are not all in-memory coercible to
12278 // the resolved capture type. We finally have to fall back to the ugly method.12155 // the resolved capture type. We finally have to fall back to the ugly method.
1227912156
12280 // However, let's first track which operands are in-memory coercible. There may well
12281 // be several, and we can squash all of these cases into the same switch prong using
12282 // a simple bitcast. We'll make this the 'else' prong.
12283
12284 var in_mem_coercible: std.bit_set.Dynamic = try .initFull(sema.arena, field_indices.len);
12285 in_mem_coercible.unset(first_non_imc);
12286 {
12287 const next = first_non_imc + 1;
12288 for (field_indices[next..], next..) |field_idx, i| {
12289 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);
12290 if (.ok != try sema.coerceInMemoryAllowed(case_block, capture_ty, field_ty, false, zcu.getTarget(), .unneeded, .unneeded, null)) {
12291 in_mem_coercible.unset(i);
12292 }
12293 }
12294 }
12295
12296 const capture_block_inst = try case_block.addInstAsIndex(.{12157 const capture_block_inst = try case_block.addInstAsIndex(.{
12297 .tag = .block,12158 .tag = .block,
12298 .data = .{12159 .data = .{
...@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12303,23 +12164,19 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12303 },12164 },
12304 });12165 });
1230512166
12306 const prong_count = field_indices.len - in_mem_coercible.count();12167 const estimated_extra = field_indices.len * 6 + (field_indices.len / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12307
12308 const estimated_extra = prong_count * 6 + (prong_count / 10); // 2 for Case, 1 item, probably 3 insts; plus hints
12309 var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);12168 var cases_extra = try std.ArrayList(u32).initCapacity(gpa, estimated_extra);
12310 defer cases_extra.deinit(gpa);12169 defer cases_extra.deinit(gpa);
1231112170
12312 {12171 {
12313 // All branch hints are `.none`, so just add zero elems.12172 // All branch hints are `.none`, so just add zero elems.
12314 comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);12173 comptime assert(@intFromEnum(std.lang.BranchHint.none) == 0);
12315 const need_elems = std.math.divCeil(usize, prong_count + 1, 10) catch unreachable;12174 const need_elems = std.math.divCeil(usize, field_indices.len + 1, 10) catch unreachable;
12316 try cases_extra.appendNTimes(gpa, 0, need_elems);12175 try cases_extra.appendNTimes(gpa, 0, need_elems);
12317 }12176 }
1231812177
12319 {12178 {
12320 // Non-bitcast cases12179 for (field_indices, item_refs, 0..) |field_index, item, item_index| {
12321 var it = in_mem_coercible.iterator(.{ .kind = .unset });
12322 while (it.next()) |idx| {
12323 var coerce_block = case_block.makeSubBlock();12180 var coerce_block = case_block.makeSubBlock();
12324 defer coerce_block.instructions.deinit(sema.gpa);12181 defer coerce_block.instructions.deinit(sema.gpa);
1232512182
...@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12328,13 +12185,12 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12328 .offset = .{ .switch_case_item = .{12185 .offset = .{ .switch_case_item = .{
12329 .switch_node_offset = switch_node_offset,12186 .switch_node_offset = switch_node_offset,
12330 .case_idx = capture_src.offset.switch_capture.case_idx,12187 .case_idx = capture_src.offset.switch_capture.case_idx,
12331 .item_idx = .{ .kind = .single, .value = @intCast(idx) },12188 .item_idx = .{ .kind = .single, .value = @intCast(item_index) },
12332 } },12189 } },
12333 };12190 };
1233412191
12335 const field_idx = field_indices[idx];12192 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_index]);
12336 const field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[field_idx]);12193 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_index, field_ty);
12337 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, field_idx, field_ty);
12338 const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);12194 const coerced = try sema.coerce(&coerce_block, capture_ty, uncoerced, case_src);
12339 _ = try coerce_block.addBr(capture_block_inst, coerced);12195 _ = try coerce_block.addBr(capture_block_inst, coerced);
1234012196
...@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12346,24 +12202,16 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12346 .ranges_len = 0,12202 .ranges_len = 0,
12347 .body_len = @intCast(coerce_block.instructions.items.len),12203 .body_len = @intCast(coerce_block.instructions.items.len),
12348 }));12204 }));
12349 cases_extra.appendAssumeCapacity(@intFromEnum(item_refs[idx])); // item12205 cases_extra.appendAssumeCapacity(@intFromEnum(item)); // item
12350 cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body12206 cases_extra.appendSliceAssumeCapacity(@ptrCast(coerce_block.instructions.items)); // body
12351 }12207 }
12352 }12208 }
12353 const else_body_len = len: {12209 const else_body_len = len: {
12354 // 'else' prong uses a bitcast12210 // 'else' prong is unreachable
12355 var coerce_block = case_block.makeSubBlock();12211 const result_index: Air.Inst.Index = @enumFromInt(sema.air_instructions.len);
12356 defer coerce_block.instructions.deinit(sema.gpa);12212 try sema.air_instructions.append(gpa, .{ .tag = .unreach, .data = .{ .no_op = {} } });
1235712213 try cases_extra.append(gpa, @intFromEnum(result_index));
12358 const first_imc_item_idx = in_mem_coercible.findFirstSet().?;12214 break :len 1;
12359 const first_imc_field_idx = field_indices[first_imc_item_idx];
12360 const first_imc_field_ty: Type = .fromInterned(union_obj.field_types.get(ip)[first_imc_field_idx]);
12361 const uncoerced = try coerce_block.addStructFieldVal(loaded_operand, first_imc_field_idx, first_imc_field_ty);
12362 const coerced = try coerce_block.addBitCast(capture_ty, uncoerced);
12363 _ = try coerce_block.addBr(capture_block_inst, coerced);
12364
12365 try cases_extra.appendSlice(gpa, @ptrCast(coerce_block.instructions.items));
12366 break :len coerce_block.instructions.items.len;
12367 };12215 };
1236812216
12369 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +12217 try sema.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.SwitchBr).@"struct".field_names.len +
...@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(...@@ -12378,7 +12226,7 @@ fn analyzeSwitchPayloadCaptureTaggedUnion(
12378 .pl_op = .{12226 .pl_op = .{
12379 .operand = undefined, // set by switch below12227 .operand = undefined, // set by switch below
12380 .payload = sema.addExtraAssumeCapacity(Air.SwitchBr{12228 .payload = sema.addExtraAssumeCapacity(Air.SwitchBr{
12381 .cases_len = @intCast(prong_count),12229 .cases_len = @intCast(field_indices.len),
12382 .else_body_len = @intCast(else_body_len),12230 .else_body_len = @intCast(else_body_len),
12383 }),12231 }),
12384 },12232 },
...@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(...@@ -12488,7 +12336,7 @@ fn resolveSwitchItem(
12488 // being switched on if their prong body is `=> comptime unreachable,`.12336 // being switched on if their prong body is `=> comptime unreachable,`.
12489 switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {12337 switch (try sema.coerceInMemoryAllowedErrorSets(block, item_ty, uncoerced_ty, item_src, item_src)) {
12490 .ok => if (sema.resolveValue(uncoerced)) |uncoerced_val| {12338 .ok => if (sema.resolveValue(uncoerced)) |uncoerced_val| {
12491 break :item_ref try sema.coerceInMemory(uncoerced_val, item_ty);12339 break :item_ref .fromValue(try pt.getCoerced(uncoerced_val, item_ty));
12492 },12340 },
12493 .missing_error => if (prong_is_comptime_unreach) {12341 .missing_error => if (prong_is_comptime_unreach) {
12494 break :item_ref uncoerced;12342 break :item_ref uncoerced;
...@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13394,8 +13242,8 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13394 defer trash_block.instructions.deinit(sema.gpa);13242 defer trash_block.instructions.deinit(sema.gpa);
1339513243
13396 const instructions = [_]Air.Inst.Ref{13244 const instructions = [_]Air.Inst.Ref{
13397 try trash_block.addBitCast(lhs_info.elem_type, .void_value),13245 try trash_block.addTyOp(.bit_cast, lhs_info.elem_type, .void_value),
13398 try trash_block.addBitCast(rhs_info.elem_type, .void_value),13246 try trash_block.addTyOp(.bit_cast, rhs_info.elem_type, .void_value),
13399 };13247 };
13400 break :t try sema.resolvePeerTypes(block, src, &instructions, .{13248 break :t try sema.resolvePeerTypes(block, src, &instructions, .{
13401 .override = &[_]?LazySrcLoc{ lhs_src, rhs_src },13249 .override = &[_]?LazySrcLoc{ lhs_src, rhs_src },
...@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13552,7 +13400,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13552 });13400 });
1355313401
13554 const many_ty = slice_ty.slicePtrFieldType(zcu);13402 const many_ty = slice_ty.slicePtrFieldType(zcu);
13555 const many_alloc = try block.addBitCast(many_ty, mutable_alloc);13403 const many_alloc = try block.addTyOp(.ptr_cast, many_ty, mutable_alloc);
1355613404
13557 // lhs_dest_slice = dest[0..lhs.len]13405 // lhs_dest_slice = dest[0..lhs.len]
13558 if (lhs_len > 0) {13406 if (lhs_len > 0) {
...@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13601,7 +13449,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13601 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);13449 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13602 }13450 }
1360313451
13604 return block.addBitCast(constant_alloc_ty, mutable_alloc);13452 return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13605 }13453 }
1360613454
13607 var elem_i: u32 = 0;13455 var elem_i: u32 = 0;
...@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai...@@ -13634,7 +13482,7 @@ fn zirArrayCat(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai
13634 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);13482 try sema.storePtr2(block, src, elem_ptr, src, init, lhs_src, .store);
13635 }13483 }
1363613484
13637 return block.addBitCast(constant_alloc_ty, mutable_alloc);13485 return block.addTyOp(.ptr_cast, constant_alloc_ty, mutable_alloc);
13638 }13486 }
1363913487
13640 const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);13488 const element_refs = try sema.arena.alloc(Air.Inst.Ref, result_len);
...@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(...@@ -14917,8 +14765,8 @@ fn analyzeArithmetic(
14917 try sema.requireRuntimeBlock(block, src, runtime_src);14765 try sema.requireRuntimeBlock(block, src, runtime_src);
14918 try sema.checkLogicalPtrOperation(block, src, lhs_ty);14766 try sema.checkLogicalPtrOperation(block, src, lhs_ty);
14919 try sema.checkLogicalPtrOperation(block, src, rhs_ty);14767 try sema.checkLogicalPtrOperation(block, src, rhs_ty);
14920 const lhs_int = try block.addBitCast(.usize, lhs);14768 const lhs_int = try block.addTyOp(.int_from_ptr, .usize, lhs);
14921 const rhs_int = try block.addBitCast(.usize, rhs);14769 const rhs_int = try block.addTyOp(.int_from_ptr, .usize, rhs);
14922 const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);14770 const address = try block.addBinOp(.sub_wrap, lhs_int, rhs_int);
14923 return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));14771 return try block.addBinOp(.div_exact, address, try pt.intRef(.usize, elem_size));
14924 }14772 }
...@@ -15186,15 +15034,76 @@ fn zirAsm(...@@ -15186,15 +15034,76 @@ fn zirAsm(
15186 break :out_ty sema.typeOf(inst).childType(zcu);15034 break :out_ty sema.typeOf(inst).childType(zcu);
15187 }15035 }
15188 };15036 };
15189 if (!out_ty.hasWellDefinedLayout(zcu)) {15037 switch (out_ty.zigTypeTag(zcu)) {
15190 return sema.failWithOwnedErrorMsg(block, msg: {15038 .int, .float, .bool, .vector => {},
15191 const msg = try sema.errMsg(output_src, "invalid inline assembly output type; '{f}' does not have a guaranteed in-memory layout", .{15039
15192 out_ty.fmt(pt),15040 .pointer => if (out_ty.isSlice(zcu)) return sema.failWithOwnedErrorMsg(block, msg: {
15193 });15041 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15194 errdefer msg.destroy(gpa);15042 errdefer msg.destroy(gpa);
15195 try sema.addDeclaredHereNote(msg, out_ty);15043 try sema.errNote(output_src, msg, "consider separate outputs for 'ptr' and 'len'", .{});
15196 break :msg msg;15044 break :msg msg;
15197 });15045 }),
15046
15047 .optional => if (!out_ty.isPtrLikeOptional(zcu)) {
15048 return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15049 },
15050
15051 .@"enum" => switch (ip.loadEnumType(out_ty.toIntern()).int_tag_mode) {
15052 .explicit => {},
15053 .auto => return sema.failWithOwnedErrorMsg(block, msg: {
15054 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15055 errdefer msg.destroy(gpa);
15056 try sema.errNote(out_ty.srcLoc(zcu), msg, "integer tag type of enum is inferred", .{});
15057 try sema.errNote(out_ty.srcLoc(zcu), msg, "consider explicitly specifying the integer tag type", .{});
15058 break :msg msg;
15059 }),
15060 },
15061
15062 .@"struct" => switch (out_ty.containerLayout(zcu)) {
15063 .@"packed" => {},
15064 .auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
15065 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15066 errdefer msg.destroy(gpa);
15067 try sema.errNote(output_src, msg, "struct types cannot be passed to inline assembly", .{});
15068 try sema.addDeclaredHereNote(msg, out_ty);
15069 break :msg msg;
15070 }),
15071 },
15072
15073 .@"union" => switch (out_ty.containerLayout(zcu)) {
15074 .@"packed" => {},
15075 .auto, .@"extern" => return sema.failWithOwnedErrorMsg(block, msg: {
15076 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15077 errdefer msg.destroy(gpa);
15078 try sema.errNote(output_src, msg, "union types cannot be passed to inline assembly", .{});
15079 try sema.addDeclaredHereNote(msg, out_ty);
15080 break :msg msg;
15081 }),
15082 },
15083
15084 .array => return sema.failWithOwnedErrorMsg(block, msg: {
15085 const msg = try sema.errMsg(output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)});
15086 errdefer msg.destroy(gpa);
15087 try sema.errNote(output_src, msg, "array types cannot be passed to inline assembly", .{});
15088 break :msg msg;
15089 }),
15090
15091 .void,
15092 .type,
15093 .noreturn,
15094 .comptime_float,
15095 .comptime_int,
15096 .undefined,
15097 .null,
15098 .error_union,
15099 .error_set,
15100 .@"fn",
15101 .@"opaque",
15102 .frame,
15103 .@"anyframe",
15104 .enum_literal,
15105 .spirv,
15106 => return sema.fail(block, output_src, "invalid inline assembly output type '{f}'", .{out_ty.fmt(pt)}),
15198 }15107 }
1519915108
15200 const constraint = sema.code.nullTerminatedString(output.data.constraint);15109 const constraint = sema.code.nullTerminatedString(output.data.constraint);
...@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -15598,37 +15507,13 @@ fn zirBitSizeOf(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
15598 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;15507 const inst_data = sema.code.instructions.items(.data)[@intFromEnum(inst)].un_node;
15599 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);15508 const operand_src = block.builtinCallArgSrc(inst_data.src_node, 0);
15600 const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);15509 const operand_ty = try sema.resolveType(block, operand_src, inst_data.operand);
15601 switch (operand_ty.zigTypeTag(zcu)) {15510 if (!operand_ty.hasBitRepresentation(zcu) and
15602 .@"fn",15511 // TODO: allow these types too for now because this is used in some places. We need to
15603 .noreturn,15512 // figure out whether we think errors and auto-enums have bit representations!
15604 .undefined,15513 operand_ty.zigTypeTag(zcu) != .error_set and
15605 .null,15514 operand_ty.zigTypeTag(zcu) != .@"enum")
15606 .@"opaque",15515 {
15607 .spirv,15516 return sema.fail(block, operand_src, "no bit size available for type '{f}'", .{operand_ty.fmt(pt)});
15608 .type,
15609 .enum_literal,
15610 .comptime_float,
15611 .comptime_int,
15612 => return sema.fail(block, operand_src, "no size available for type '{f}'", .{operand_ty.fmt(pt)}),
15613
15614 .void,
15615 => return .zero,
15616
15617 .bool,
15618 .int,
15619 .float,
15620 .pointer,
15621 .array,
15622 .@"struct",
15623 .optional,
15624 .error_union,
15625 .error_set,
15626 .@"enum",
15627 .@"union",
15628 .vector,
15629 .frame,
15630 .@"anyframe",
15631 => {},
15632 }15517 }
15633 try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);15518 try sema.ensureLayoutResolved(operand_ty, operand_src, .size_of);
15634 return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));15519 return .fromValue(try pt.intValue(.comptime_int, operand_ty.bitSize(zcu)));
...@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -18179,13 +18064,19 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
18179 } else 0;18064 } else 0;
1818018065
18181 if (host_size != 0) {18066 if (host_size != 0) {
18067 try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18068 if (elem_ty.unpackable(zcu)) |reason| return sema.failWithOwnedErrorMsg(block, msg: {
18069 const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18070 errdefer msg.destroy(sema.gpa);
18071 try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18072 break :msg msg;
18073 });
18074 const elem_bit_size = elem_ty.bitSize(zcu);
18182 if (bit_offset >= host_size * 8) {18075 if (bit_offset >= host_size * 8) {
18183 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{18076 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} starts {d} bits after the end of a {d} byte host integer", .{
18184 elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,18077 elem_ty.fmt(pt), bit_offset, bit_offset - host_size * 8, host_size,
18185 });18078 });
18186 }18079 }
18187 try sema.ensureLayoutResolved(elem_ty, elem_ty_src, .bit_ptr_child);
18188 const elem_bit_size = elem_ty.bitSize(zcu);
18189 if (elem_bit_size > host_size * 8 - bit_offset) {18080 if (elem_bit_size > host_size * 8 - bit_offset) {
18190 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{18081 return sema.fail(block, bitoffset_src, "packed type '{f}' at bit offset {d} ends {d} bits after the end of a {d} byte host integer", .{
18191 elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,18082 elem_ty.fmt(pt), bit_offset, elem_bit_size - (host_size * 8 - bit_offset), host_size,
...@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air...@@ -18201,15 +18092,6 @@ fn zirPtrType(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air
18201 return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});18092 return sema.fail(block, elem_ty_src, "indexable pointer to opaque type '{f}' not allowed", .{elem_ty.fmt(pt)});
18202 }18093 }
1820318094
18204 if (host_size != 0) {
18205 if (elem_ty.unpackable(zcu)) |reason| return sema.failWithOwnedErrorMsg(block, msg: {
18206 const msg = try sema.errMsg(elem_ty_src, "bit-pointer cannot refer to value of type '{f}'", .{elem_ty.fmt(pt)});
18207 errdefer msg.destroy(sema.gpa);
18208 try sema.explainWhyTypeIsUnpackable(msg, elem_ty_src, reason);
18209 break :msg msg;
18210 });
18211 }
18212
18213 const ty = try pt.ptrType(.{18095 const ty = try pt.ptrType(.{
18214 .child = elem_ty.toIntern(),18096 .child = elem_ty.toIntern(),
18215 .sentinel = sentinel,18097 .sentinel = sentinel,
...@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A...@@ -18379,7 +18261,7 @@ fn zirUnionInit(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
18379 const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);18261 const payload = try sema.coerce(block, field_ty, sema.resolveInst(extra.init), payload_src);
1838018262
18381 if (union_ty.containerLayout(zcu) == .@"packed") {18263 if (union_ty.containerLayout(zcu) == .@"packed") {
18382 return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node), payload_src);18264 return sema.bitCast(block, union_ty, payload, block.nodeOffset(inst_data.src_node));
18383 }18265 }
1838418266
18385 if (sema.resolveValue(payload)) |payload_val| {18267 if (sema.resolveValue(payload)) |payload_val| {
...@@ -18516,7 +18398,7 @@ fn zirStructInit(...@@ -18516,7 +18398,7 @@ fn zirStructInit(
18516 const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);18398 const init_inst = try sema.coerce(block, field_ty, uncoerced_init_inst, field_src);
1851718399
18518 if (resolved_ty.containerLayout(zcu) == .@"packed") {18400 if (resolved_ty.containerLayout(zcu) == .@"packed") {
18519 const union_val = try sema.bitCast(block, resolved_ty, init_inst, src, field_src);18401 const union_val = try sema.bitCast(block, resolved_ty, init_inst, src);
18520 const result_val = try sema.coerce(block, result_ty, union_val, src);18402 const result_val = try sema.coerce(block, result_ty, union_val, src);
18521 if (is_ref) {18403 if (is_ref) {
18522 return sema.analyzeRef(block, src, result_val, .none);18404 return sema.analyzeRef(block, src, result_val, .none);
...@@ -18680,20 +18562,15 @@ fn finishStructInit(...@@ -18680,20 +18562,15 @@ fn finishStructInit(
18680 },18562 },
18681 .@"packed" => {18563 .@"packed" => {
18682 const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));18564 const buf = try sema.arena.alloc(u8, @intCast((struct_ty.bitSize(zcu) + 7) / 8));
18565 @memset(buf, 0);
18683 var bit_offset: u16 = 0;18566 var bit_offset: u16 = 0;
18684 for (field_inits) |field_init| {18567 for (field_inits) |field_init| {
18685 const field_val = sema.resolveValue(field_init).?;18568 const field_val = sema.resolveValue(field_init).?;
18686 field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) {18569 field_val.writeToPackedMemory(zcu, buf, bit_offset);
18687 error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
18688 error.OutOfMemory => |e| return e,
18689 };
18690 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));18570 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
18691 }18571 }
18692 assert(bit_offset == struct_ty.bitSize(zcu));18572 assert(bit_offset == struct_ty.bitSize(zcu));
18693 const struct_val = Value.readFromPackedMemory(struct_ty, pt, buf, 0, sema.arena) catch |err| switch (err) {18573 const struct_val: Value = try .readFromPackedMemory(struct_ty, pt, buf, 0);
18694 error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
18695 error.OutOfMemory => |e| return e,
18696 };
18697 const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);18574 const final_val_ref = try sema.coerce(block, result_ty, .fromValue(struct_val), init_src);
18698 return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);18575 return sema.addConstantMaybeRef(sema.resolveValue(final_val_ref).?, is_ref);
18699 },18576 },
...@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError...@@ -19387,7 +19264,7 @@ fn zirIntFromBool(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
19387 }19264 }
19388 return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());19265 return Air.internedToRef((try pt.aggregateValue(dest_ty, new_elems)).toIntern());
19389 }19266 }
19390 return block.addBitCast(dest_ty, operand);19267 return block.addTyOp(.bit_cast, dest_ty, operand);
19391}19268}
1939219269
19393fn zirErrorName(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {19270fn zirErrorName(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!...@@ -21245,7 +21122,7 @@ fn zirPtrFromInt(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!
21245 try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);21122 try sema.addSafetyCheck(block, src, is_aligned, .incorrect_alignment);
21246 }21123 }
21247 }21124 }
21248 return block.addBitCast(dest_ty, operand_coerced);21125 return block.addTyOp(.ptr_from_int, dest_ty, operand_coerced);
21249}21126}
2125021127
21251fn ptrFromIntVal(21128fn ptrFromIntVal(
...@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData...@@ -21444,7 +21321,7 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData
21444 .error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),21321 .error_union => try block.addTyOp(.unwrap_errunion_err, operand_err_ty, operand),
21445 else => unreachable,21322 else => unreachable,
21446 };21323 };
21447 const err_int_inst = try block.addBitCast(err_int_ty, err_code_inst);21324 const err_int_inst = try block.addTyOp(.int_from_error, err_int_ty, err_code_inst);
21448 if (dest_tag == .error_union) {21325 if (dest_tag == .error_union) {
21449 const zero_err = try pt.intRef(err_int_ty, 0);21326 const zero_err = try pt.intRef(err_int_ty, 0);
21450 const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);21327 const is_zero = try block.addBinOp(.cmp_eq, err_int_inst, zero_err);
...@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData...@@ -21464,10 +21341,10 @@ fn zirErrorCast(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData
21464 }21341 }
2146521342
21466 if (operand_tag == .error_set and dest_tag == .error_union) {21343 if (operand_tag == .error_set and dest_tag == .error_union) {
21467 const err_val = try block.addBitCast(dest_err_ty, operand);21344 const err_val = try block.addTyOp(.error_cast, dest_err_ty, operand);
21468 return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);21345 return block.addTyOp(.wrap_errunion_err, dest_ty, err_val);
21469 } else {21346 } else {
21470 return block.addBitCast(dest_ty, operand);21347 return block.addTyOp(.error_cast, dest_ty, operand);
21471 }21348 }
21472}21349}
2147321350
...@@ -22015,7 +21892,7 @@ fn ptrCastFull(...@@ -22015,7 +21892,7 @@ fn ptrCastFull(
22015 // `operand_ptr` converted to an integer, for safety checks.21892 // `operand_ptr` converted to an integer, for safety checks.
22016 const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {21893 const operand_ptr_int: Air.Inst.Ref = if (need_null_check or need_align_check) i: {
22017 assert(need_operand_ptr);21894 assert(need_operand_ptr);
22018 break :i try block.addBitCast(.usize, operand_ptr);21895 break :i try block.addTyOp(.int_from_ptr, .usize, operand_ptr);
22019 } else .none;21896 } else .none;
2202021897
22021 if (need_null_check) {21898 if (need_null_check) {
...@@ -22042,8 +21919,8 @@ fn ptrCastFull(...@@ -22042,8 +21919,8 @@ fn ptrCastFull(
2204221919
22043 if (dest_info.flags.size == .slice) {21920 if (dest_info.flags.size == .slice) {
22044 if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {21921 if (src_info.flags.size == .slice and !flags.addrspace_cast and !slice_needs_len_change) {
22045 // Fast path: just bitcast!21922 // Fast path: just pointer cast!
22046 return block.addBitCast(dest_ty, operand);21923 return block.addTyOp(.ptr_cast, dest_ty, operand);
22047 }21924 }
2204821925
22049 // We need to deconstruct the slice (if applicable) and reconstruct it.21926 // We need to deconstruct the slice (if applicable) and reconstruct it.
...@@ -22101,7 +21978,7 @@ fn ptrCastFull(...@@ -22101,7 +21978,7 @@ fn ptrCastFull(
22101 else => unreachable,21978 else => unreachable,
22102 };21979 };
22103 const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {21980 const coerced_ptr = if (operand_ptr_ty.toIntern() != want_ptr_ty.toIntern()) ptr: {
22104 break :ptr try block.addBitCast(want_ptr_ty, operand_ptr);21981 break :ptr try block.addTyOp(.ptr_cast, want_ptr_ty, operand_ptr);
22105 } else operand_ptr;21982 } else operand_ptr;
2210621983
22107 return block.addInst(.{21984 return block.addInst(.{
...@@ -22116,12 +21993,11 @@ fn ptrCastFull(...@@ -22116,12 +21993,11 @@ fn ptrCastFull(
22116 });21993 });
22117 } else {21994 } else {
22118 assert(need_operand_ptr);21995 assert(need_operand_ptr);
22119 // We just need to bitcast the pointer, if necessary.21996 // We just need a ptr_cast, if even that (we might only have needed the `addrspace_cast`).
22120 // It might not be necessary, since we might have just needed the `addrspace_cast`.
22121 const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())21997 const result = if (sema.typeOf(operand_ptr).toIntern() == dest_ty.toIntern())
22122 operand_ptr21998 operand_ptr
22123 else21999 else
22124 try block.addBitCast(dest_ty, operand_ptr);22000 try block.addTyOp(.ptr_cast, dest_ty, operand_ptr);
2212522001
22126 try sema.checkKnownAllocPtr(block, operand, result);22002 try sema.checkKnownAllocPtr(block, operand, result);
22127 return result;22003 return result;
...@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst...@@ -22157,7 +22033,7 @@ fn zirPtrCastNoDest(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.Inst
22157 }22033 }
2215822034
22159 try sema.requireRuntimeBlock(block, src, null);22035 try sema.requireRuntimeBlock(block, src, null);
22160 const new_ptr = try block.addBitCast(dest_ty, operand);22036 const new_ptr = try block.addTyOp(.ptr_cast, dest_ty, operand);
22161 try sema.checkKnownAllocPtr(block, operand, new_ptr);22037 try sema.checkKnownAllocPtr(block, operand, new_ptr);
22162 return new_ptr;22038 return new_ptr;
22163}22039}
...@@ -24259,7 +24135,7 @@ fn analyzeMinMax(...@@ -24259,7 +24135,7 @@ fn analyzeMinMax(
24259 // where we have refined the range, so we should be doing an intcast.24135 // where we have refined the range, so we should be doing an intcast.
24260 assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);24136 assert(intermediate_scalar_ty.zigTypeTag(zcu) == .int);
24261 assert(result_scalar_ty.zigTypeTag(zcu) == .int);24137 assert(result_scalar_ty.zigTypeTag(zcu) == .int);
24262 return block.addTyOp(.intcast, result_ty, cur_result);24138 return block.addTyOp(.int_cast, result_ty, cur_result);
24263}24139}
2426424140
24265fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {24141fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !Air.Inst.Ref {
...@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !A...@@ -24289,7 +24165,7 @@ fn upgradeToArrayPtr(sema: *Sema, block: *Block, ptr: Air.Inst.Ref, len: u64) !A
24289 try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)24165 try block.addTyOp(.slice_ptr, ptr_ty.slicePtrFieldType(zcu), ptr)
24290 else24166 else
24291 ptr;24167 ptr;
24292 return block.addBitCast(new_ty, non_slice_ptr);24168 return block.addTyOp(.ptr_cast, new_ty, non_slice_ptr);
24293}24169}
2429424170
24295fn zirMemcpy(24171fn zirMemcpy(
...@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(...@@ -25087,7 +24963,7 @@ fn zirBuiltinExtern(
25087 const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);24963 const casted_ptr_val = try pt.getCoerced(uncasted_ptr_val, result_ptr_ty);
25088 return Air.internedToRef(casted_ptr_val.toIntern());24964 return Air.internedToRef(casted_ptr_val.toIntern());
25089 } else {24965 } else {
25090 return block.addBitCast(result_ptr_ty, uncasted_ptr);24966 return block.addTyOp(.ptr_cast, result_ptr_ty, uncasted_ptr);
25091 }24967 }
25092}24968}
2509324969
...@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(...@@ -25533,7 +25409,7 @@ pub fn explainWhyTypeIsNotExtern(
25533 .param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),25409 .param_ty => try sema.errNote(src_loc, msg, "arrays are not allowed as a parameter type", .{}),
25534 else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),25410 else => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),
25535 },25411 },
25536 .vector => try sema.explainWhyTypeIsNotExtern(msg, src_loc, ty.childType(zcu), .element),25412 .vector => try sema.errNote(src_loc, msg, "vectors have no guaranteed in-memory representation", .{}),
25537 .optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),25413 .optional => try sema.errNote(src_loc, msg, "non-pointer optionals have no guaranteed in-memory representation", .{}),
25538 }25414 }
25539}25415}
...@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(...@@ -25801,7 +25677,7 @@ fn addSafetyCheckSentinelMismatch(
25801 .address_space = ptr_info.flags.address_space,25677 .address_space = ptr_info.flags.address_space,
25802 },25678 },
25803 });25679 });
25804 const many_ptr = try parent_block.addBitCast(many_ptr_ty, ptr);25680 const many_ptr = try parent_block.addTyOp(.ptr_cast, many_ptr_ty, ptr);
25805 break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);25681 break :s try parent_block.addBinOp(.ptr_elem_val, many_ptr, sentinel_index);
25806 },25682 },
25807 .many => unreachable,25683 .many => unreachable,
...@@ -26278,7 +26154,7 @@ fn fieldPtr(...@@ -26278,7 +26154,7 @@ fn fieldPtr(
26278 },26154 },
26279 .packed_offset = ptr_ptr_info.packed_offset,26155 .packed_offset = ptr_ptr_info.packed_offset,
26280 });26156 });
26281 return sema.bitCast(block, result_ty, object_ptr, src, null);26157 return block.addTyOp(.ptr_cast, result_ty, object_ptr);
26282 } else {26158 } else {
26283 return sema.fail(26159 return sema.fail(
26284 block,26160 block,
...@@ -26996,16 +26872,13 @@ fn unionFieldVal(...@@ -26996,16 +26872,13 @@ fn unionFieldVal(
26996 break :msg msg;26872 break :msg msg;
26997 });26873 });
26998 },26874 },
26999 .@"extern" => if (try sema.bitCastVal(union_val, field_ty, 0, 0, 0)) |field_val| {26875 .@"extern" => if (try sema.castMemory(union_val, field_ty, 0)) |field_val| {
27000 return .fromValue(field_val);26876 return .fromValue(field_val);
27001 } else {26877 } else {
27002 // Runtime-known due to a pointer-to-integer conversion.26878 // Runtime-known due to a pointer-to-integer conversion.
27003 },26879 },
27004 .@"packed" => {26880 .@"packed" => {
27005 const field_val = try sema.bitCastVal(union_val, field_ty, 0, union_ty.bitSize(zcu), 0) orelse {26881 return .fromValue(try sema.bitCastVal(union_val, field_ty));
27006 unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
27007 };
27008 return .fromValue(field_val);
27009 },26882 },
27010 }26883 }
27011 }26884 }
...@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(...@@ -27104,7 +26977,7 @@ fn elemPtrOneLayerOnly(
2710426977
27105 if (child_ty.abiSize(zcu) == 0) {26978 if (child_ty.abiSize(zcu) == 0) {
27106 // zero-bit child type; just bitcast the pointer26979 // zero-bit child type; just bitcast the pointer
27107 return block.addBitCast(result_ty, indexable);26980 return block.addTyOp(.ptr_cast, result_ty, indexable);
27108 }26981 }
2710926982
27110 return block.addPtrElemPtr(indexable, elem_index, result_ty);26983 return block.addPtrElemPtr(indexable, elem_index, result_ty);
...@@ -27401,68 +27274,36 @@ fn elemPtrVector(...@@ -27401,68 +27274,36 @@ fn elemPtrVector(
27401 }27274 }
2740227275
27403 const elem_ty = vector_ty.childType(zcu);27276 const elem_ty = vector_ty.childType(zcu);
27404 const elem_bits = elem_ty.bitSize(zcu);27277
27405 // Exiting this block means the operation is a runtime one.27278 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27406 const elem_ptr_ty: Type = if (elem_bits < 8 or !std.math.isPowerOfTwo(elem_bits)) elem_ptr_ty: {27279 const elem_ptr_ty = try pt.ptrType(.{
27407 // Use a packed pointer (i.e. vector_index != 0)27280 .child = elem_ty.toIntern(),
27408 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);27281 .flags = .{
27409 const elem_ptr_ty = try pt.ptrType(.{27282 .size = .one,
27410 .child = elem_ty.toIntern(),27283 .alignment = vector_ptr_info.flags.alignment,
27411 .flags = .{27284 .is_const = vector_ptr_info.flags.is_const,
27412 .size = .one,27285 .is_volatile = vector_ptr_info.flags.is_volatile,
27413 .alignment = vector_ptr_info.flags.alignment,27286 .is_allowzero = vector_ptr_info.flags.is_allowzero,
27414 .is_const = vector_ptr_info.flags.is_const,27287 .address_space = vector_ptr_info.flags.address_space,
27415 .is_volatile = vector_ptr_info.flags.is_volatile,27288 .vector_index = @enumFromInt(index),
27416 .is_allowzero = vector_ptr_info.flags.is_allowzero,27289 },
27417 .address_space = vector_ptr_info.flags.address_space,27290 .packed_offset = .{
27418 .vector_index = @enumFromInt(index),27291 .host_size = @intCast(vector_len),
27419 },27292 .bit_offset = 0,
27420 .packed_offset = .{27293 },
27421 .host_size = @intCast(vector_len),27294 });
27422 .bit_offset = 0,27295
27423 },27296 if (maybe_vector_ptr_val) |ptr_val| {
27424 });27297 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27425 if (maybe_vector_ptr_val) |ptr_val| {27298 return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27426 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);27299 }
27427 return .fromValue(try pt.getCoerced(ptr_val, elem_ptr_ty));
27428 }
27429 break :elem_ptr_ty elem_ptr_ty;
27430 } else elem_ptr_ty: {
27431 // Use a normal pointer (i.e. vector_index == 0)
27432 const vector_ptr_info = vector_ptr_ty.ptrInfo(zcu);
27433 const elem_ptr_ty = try pt.ptrType(.{
27434 .child = elem_ty.toIntern(),
27435 .flags = .{
27436 .size = .one,
27437 // TODO: this logic was ported from old code, but it's bogus. This entire block will
27438 // go away when https://github.com/ziglang/zig/issues/24061 is implemented anyway.
27439 .alignment = switch (vector_ptr_info.flags.alignment) {
27440 .none => .none,
27441 else => |vec_align| switch (index * elem_ty.abiSize(zcu)) {
27442 0 => vec_align,
27443 else => |byte_offset| .minStrict(vec_align, .fromLog2Units(@ctz(byte_offset))),
27444 },
27445 },
27446 .is_const = vector_ptr_info.flags.is_const,
27447 .is_volatile = vector_ptr_info.flags.is_volatile,
27448 .is_allowzero = vector_ptr_info.flags.is_allowzero,
27449 .address_space = vector_ptr_info.flags.address_space,
27450 },
27451 });
27452 if (maybe_vector_ptr_val) |ptr_val| {
27453 if (ptr_val.isUndef(zcu)) return pt.undefRef(elem_ptr_ty);
27454 const bit_offset = index * @divExact(elem_ty.bitSize(zcu), 8);
27455 return .fromValue(try ptr_val.getOffsetPtr(bit_offset, elem_ptr_ty, pt));
27456 }
27457 break :elem_ptr_ty elem_ptr_ty;
27458 };
2745927300
27460 if (!init) {27301 if (!init) {
27461 try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);27302 try sema.validateRuntimeElemAccess(block, elem_index_src, elem_ty, vector_ptr_src);
27462 try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);27303 try sema.validateRuntimeValue(block, vector_ptr_src, vector_ptr);
27463 }27304 }
2746427305
27465 return block.addPtrElemPtr(vector_ptr, elem_index, elem_ptr_ty);27306 return block.addTyOp(.ptr_cast, elem_ptr_ty, vector_ptr);
27466}27307}
2746727308
27468fn elemPtrSpirvRuntimeArray(27309fn elemPtrSpirvRuntimeArray(
...@@ -27544,7 +27385,7 @@ fn elemPtrArray(...@@ -27544,7 +27385,7 @@ fn elemPtrArray(
2754427385
27545 if (array_ty.childType(zcu).abiSize(zcu) == 0) {27386 if (array_ty.childType(zcu).abiSize(zcu) == 0) {
27546 // zero-bit child type; just bitcast the pointer27387 // zero-bit child type; just bitcast the pointer
27547 return block.addBitCast(elem_ptr_ty, array_ptr);27388 return block.addTyOp(.ptr_cast, elem_ptr_ty, array_ptr);
27548 }27389 }
2754927390
27550 return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);27391 return block.addPtrElemPtr(array_ptr, elem_index, elem_ptr_ty);
...@@ -27670,7 +27511,7 @@ fn elemPtrSlice(...@@ -27670,7 +27511,7 @@ fn elemPtrSlice(
27670 if (elem_ty.abiSize(zcu) == 0) {27511 if (elem_ty.abiSize(zcu) == 0) {
27671 // zero-bit child type; just extract the pointer and bitcast it27512 // zero-bit child type; just extract the pointer and bitcast it
27672 const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);27513 const slice_ptr = try block.addTyOp(.slice_ptr, slice_ty.slicePtrFieldType(zcu), slice);
27673 return block.addBitCast(elem_ptr_ty, slice_ptr);27514 return block.addTyOp(.ptr_cast, elem_ptr_ty, slice_ptr);
27674 }27515 }
27675 return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);27516 return block.addSliceElemPtr(slice, elem_index, elem_ptr_ty);
27676}27517}
...@@ -27753,12 +27594,31 @@ fn coerceExtra(...@@ -27753,12 +27594,31 @@ fn coerceExtra(
27753 var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);27594 var in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, maybe_inst_val);
27754 if (in_memory_result == .ok) {27595 if (in_memory_result == .ok) {
27755 if (maybe_inst_val) |val| {27596 if (maybe_inst_val) |val| {
27756 return sema.coerceInMemory(val, dest_ty);27597 return .fromValue(try pt.getCoerced(val, dest_ty));
27757 }27598 }
27758 try sema.requireRuntimeBlock(block, inst_src, null);27599 const coerced: Air.Inst.Ref = switch (in_memory_result.ok) {
27759 const new_val = try block.addBitCast(dest_ty, inst);27600 .none => coerced: {
27760 try sema.checkKnownAllocPtr(block, inst, new_val);27601 const @"addrspace" = target_util.defaultAddressSpace(zcu.getTarget(), .local);
27761 return new_val;27602 const src_ptr_ty = try pt.ptrType(.{
27603 .child = inst_ty.toIntern(),
27604 .flags = .{ .size = .one, .address_space = @"addrspace" },
27605 });
27606 const dest_ptr_ty = try pt.ptrType(.{
27607 .child = dest_ty.toIntern(),
27608 .flags = .{ .size = .one, .address_space = @"addrspace" },
27609 });
27610 const ptr = try block.addTy(.alloc, src_ptr_ty);
27611 _ = try block.addBinOp(.store_safe, ptr, inst);
27612 const casted_ptr = try block.addTyOp(.ptr_cast, dest_ptr_ty, ptr);
27613 break :coerced try block.addTyOp(.load, dest_ty, casted_ptr);
27614 },
27615 .same_type => unreachable, // we checked for equal types just above
27616 .bit_cast => try block.addTyOp(.bit_cast, dest_ty, inst),
27617 .ptr_cast => try block.addTyOp(.ptr_cast, dest_ty, inst),
27618 .error_cast => try block.addTyOp(.error_cast, dest_ty, inst),
27619 };
27620 try sema.checkKnownAllocPtr(block, inst, coerced);
27621 return coerced;
27762 }27622 }
2776327623
27764 switch (dest_ty.zigTypeTag(zcu)) {27624 switch (dest_ty.zigTypeTag(zcu)) {
...@@ -27872,8 +27732,8 @@ fn coerceExtra(...@@ -27872,8 +27732,8 @@ fn coerceExtra(
2787227732
27873 if (dest_info.sentinel != .none) {27733 if (dest_info.sentinel != .none) {
27874 if (array_ty.sentinel(zcu)) |inst_sent| {27734 if (array_ty.sentinel(zcu)) |inst_sent| {
27875 if (Air.internedToRef(dest_info.sentinel) !=27735 if (dest_info.sentinel !=
27876 try sema.coerceInMemory(inst_sent, dst_elem_type))27736 (try pt.getCoerced(inst_sent, dst_elem_type)).toIntern())
27877 {27737 {
27878 in_memory_result = .{ .ptr_sentinel = .{27738 in_memory_result = .{ .ptr_sentinel = .{
27879 .actual = inst_sent,27739 .actual = inst_sent,
...@@ -28067,8 +27927,8 @@ fn coerceExtra(...@@ -28067,8 +27927,8 @@ fn coerceExtra(
28067 }27927 }
2806827928
28069 if (dest_info.sentinel == .none or inst_info.sentinel == .none or27929 if (dest_info.sentinel == .none or inst_info.sentinel == .none or
28070 Air.internedToRef(dest_info.sentinel) !=27930 dest_info.sentinel !=
28071 try sema.coerceInMemory(Value.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child)))27931 (try pt.getCoerced(.fromInterned(inst_info.sentinel), .fromInterned(dest_info.child))).toIntern())
28072 break :p;27932 break :p;
2807327933
28074 const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);27934 const slice_ptr = try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty);
...@@ -28117,7 +27977,7 @@ fn coerceExtra(...@@ -28117,7 +27977,7 @@ fn coerceExtra(
28117 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))27977 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))
28118 {27978 {
28119 try sema.requireRuntimeBlock(block, inst_src, null);27979 try sema.requireRuntimeBlock(block, inst_src, null);
28120 return block.addTyOp(.intcast, dest_ty, inst);27980 return block.addTyOp(.int_cast, dest_ty, inst);
28121 }27981 }
28122 },27982 },
28123 else => {},27983 else => {},
...@@ -28379,16 +28239,8 @@ fn coerceExtra(...@@ -28379,16 +28239,8 @@ fn coerceExtra(
28379 return sema.failWithOwnedErrorMsg(block, msg);28239 return sema.failWithOwnedErrorMsg(block, msg);
28380}28240}
2838128241
28382fn coerceInMemory(
28383 sema: *Sema,
28384 val: Value,
28385 dst_ty: Type,
28386) CompileError!Air.Inst.Ref {
28387 return Air.internedToRef((try sema.pt.getCoerced(val, dst_ty)).toIntern());
28388}
28389
28390const InMemoryCoercionResult = union(enum) {28242const InMemoryCoercionResult = union(enum) {
28391 ok,28243 ok: Strategy,
28392 no_match: Pair,28244 no_match: Pair,
28393 int_not_coercible: Int,28245 int_not_coercible: Int,
28394 comptime_int_not_coercible: TypeValuePair,28246 comptime_int_not_coercible: TypeValuePair,
...@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {...@@ -28424,6 +28276,21 @@ const InMemoryCoercionResult = union(enum) {
28424 double_ptr_to_anyopaque: Pair,28276 double_ptr_to_anyopaque: Pair,
28425 slice_to_anyopaque: Pair,28277 slice_to_anyopaque: Pair,
2842628278
28279 const Strategy = enum {
28280 /// There isn't a special strategy for this particular coercion---we'll just need to
28281 /// reinterpret the bytes in memory.
28282 none,
28283
28284 /// The source and destination types are equal, so no explicit cast operation is necessary.
28285 same_type,
28286 /// The coercion can be lowered to `Air.Inst.Tag.bit_cast`.
28287 bit_cast,
28288 /// The coercion can be lowered to `Air.Inst.Tag.ptr_cast`.
28289 ptr_cast,
28290 /// The coercion can be lowered to `Air.Inst.Tag.error_cast`.
28291 error_cast,
28292 };
28293
28427 const Pair = struct {28294 const Pair = struct {
28428 actual: Type,28295 actual: Type,
28429 wanted: Type,28296 wanted: Type,
...@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(...@@ -28797,7 +28664,7 @@ pub fn coerceInMemoryAllowed(
28797 }28664 }
2879828665
28799 if (dest_ty.eql(src_ty))28666 if (dest_ty.eql(src_ty))
28800 return .ok;28667 return .{ .ok = .same_type };
2880128668
28802 const dest_tag = dest_ty.zigTypeTag(zcu);28669 const dest_tag = dest_ty.zigTypeTag(zcu);
28803 const src_tag = src_ty.zigTypeTag(zcu);28670 const src_tag = src_ty.zigTypeTag(zcu);
...@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(...@@ -28810,7 +28677,7 @@ pub fn coerceInMemoryAllowed(
28810 if (dest_info.signedness == src_info.signedness and28677 if (dest_info.signedness == src_info.signedness and
28811 dest_info.bits == src_info.bits)28678 dest_info.bits == src_info.bits)
28812 {28679 {
28813 return .ok;28680 return .{ .ok = .bit_cast };
28814 }28681 }
2881528682
28816 if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or28683 if ((src_info.signedness == dest_info.signedness and dest_info.bits < src_info.bits) or
...@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(...@@ -28818,7 +28685,7 @@ pub fn coerceInMemoryAllowed(
28818 (dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or28685 (dest_info.signedness == .signed and src_info.signedness == .unsigned and dest_info.bits <= src_info.bits) or
28819 (dest_info.signedness == .unsigned and src_info.signedness == .signed))28686 (dest_info.signedness == .unsigned and src_info.signedness == .signed))
28820 {28687 {
28821 return InMemoryCoercionResult{ .int_not_coercible = .{28688 return .{ .int_not_coercible = .{
28822 .actual_signedness = src_info.signedness,28689 .actual_signedness = src_info.signedness,
28823 .wanted_signedness = dest_info.signedness,28690 .wanted_signedness = dest_info.signedness,
28824 .actual_bits = src_info.bits,28691 .actual_bits = src_info.bits,
...@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(...@@ -28841,7 +28708,7 @@ pub fn coerceInMemoryAllowed(
28841 const dest_bits = dest_ty.floatBits(target);28708 const dest_bits = dest_ty.floatBits(target);
28842 const src_bits = src_ty.floatBits(target);28709 const src_bits = src_ty.floatBits(target);
28843 if (dest_bits == src_bits) {28710 if (dest_bits == src_bits) {
28844 return .ok;28711 return .{ .ok = .bit_cast };
28845 }28712 }
28846 }28713 }
2884728714
...@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(...@@ -28864,24 +28731,38 @@ pub fn coerceInMemoryAllowed(
28864 if (dest_tag == .error_union and src_tag == .error_union) {28731 if (dest_tag == .error_union and src_tag == .error_union) {
28865 const dest_payload = dest_ty.errorUnionPayload(zcu);28732 const dest_payload = dest_ty.errorUnionPayload(zcu);
28866 const src_payload = src_ty.errorUnionPayload(zcu);28733 const src_payload = src_ty.errorUnionPayload(zcu);
28867 const child = try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null);28734 const payload_strat = switch (try sema.coerceInMemoryAllowed(block, dest_payload, src_payload, dest_is_mut, target, dest_src, src_src, null)) {
28868 if (child != .ok) {28735 .ok => |strat| strat,
28869 return .{ .error_union_payload = .{28736 else => |payload_result| return .{ .error_union_payload = .{
28870 .child = try child.dupe(sema.arena),28737 .child = try payload_result.dupe(sema.arena),
28871 .actual = src_payload,28738 .actual = src_payload,
28872 .wanted = dest_payload,28739 .wanted = dest_payload,
28873 } };28740 } },
28741 };
28742 switch (try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null)) {
28743 .ok => {},
28744 else => |err_set_result| return err_set_result,
28874 }28745 }
28875 return try sema.coerceInMemoryAllowed(block, dest_ty.errorUnionSet(zcu), src_ty.errorUnionSet(zcu), dest_is_mut, target, dest_src, src_src, null);28746 return switch (payload_strat) {
28747 .same_type => .{ .ok = .error_cast },
28748 else => .{ .ok = .none },
28749 };
28876 }28750 }
2887728751
28878 // Error Sets28752 // Error Sets
28879 if (dest_tag == .error_set and src_tag == .error_set) {28753 if (dest_tag == .error_set and src_tag == .error_set) {
28880 const res1 = try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src);28754 switch (try sema.coerceInMemoryAllowedErrorSets(block, dest_ty, src_ty, dest_src, src_src)) {
28881 if (!dest_is_mut or res1 != .ok) return res1;28755 .ok => |strat| assert(strat == .error_cast),
28882 // src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.28756 else => |result| return result,
28883 const res2 = try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src);28757 }
28884 return res2;28758 if (dest_is_mut) {
28759 // src -> dest is okay, but `dest_is_mut`, so it needs to be allowed in the other direction.
28760 switch (try sema.coerceInMemoryAllowedErrorSets(block, src_ty, dest_ty, src_src, dest_src)) {
28761 .ok => |strat| assert(strat == .error_cast),
28762 else => |result| return result,
28763 }
28764 }
28765 return .{ .ok = .error_cast };
28885 }28766 }
2888628767
28887 // Arrays28768 // Arrays
...@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(...@@ -28896,9 +28777,9 @@ pub fn coerceInMemoryAllowed(
28896 }28777 }
2889728778
28898 const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);28779 const child = try sema.coerceInMemoryAllowed(block, dest_info.elem_type, src_info.elem_type, dest_is_mut, target, dest_src, src_src, null);
28899 switch (child) {28780 const child_strat = switch (child) {
28900 .ok => {},28781 .ok => |strat| strat,
28901 .no_match => return child,28782 .no_match => |no_match| return .{ .no_match = no_match },
28902 else => {28783 else => {
28903 return .{ .array_elem = .{28784 return .{ .array_elem = .{
28904 .child = try child.dupe(sema.arena),28785 .child = try child.dupe(sema.arena),
...@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(...@@ -28906,7 +28787,7 @@ pub fn coerceInMemoryAllowed(
28906 .wanted = dest_info.elem_type,28787 .wanted = dest_info.elem_type,
28907 } };28788 } };
28908 },28789 },
28909 }28790 };
28910 const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or28791 const ok_sent = (dest_info.sentinel == null and src_info.sentinel == null) or
28911 (src_info.sentinel != null and28792 (src_info.sentinel != null and
28912 dest_info.sentinel != null and28793 dest_info.sentinel != null and
...@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(...@@ -28922,7 +28803,10 @@ pub fn coerceInMemoryAllowed(
28922 .ty = dest_info.elem_type,28803 .ty = dest_info.elem_type,
28923 } };28804 } };
28924 }28805 }
28925 return .ok;28806 return .{ .ok = switch (child_strat) {
28807 .bit_cast => .bit_cast,
28808 else => .none,
28809 } };
28926 }28810 }
2892728811
28928 // Vectors28812 // Vectors
...@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(...@@ -28938,16 +28822,18 @@ pub fn coerceInMemoryAllowed(
2893828822
28939 const dest_elem_ty = dest_ty.scalarType(zcu);28823 const dest_elem_ty = dest_ty.scalarType(zcu);
28940 const src_elem_ty = src_ty.scalarType(zcu);28824 const src_elem_ty = src_ty.scalarType(zcu);
28941 const child = try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null);28825 switch (try sema.coerceInMemoryAllowed(block, dest_elem_ty, src_elem_ty, dest_is_mut, target, dest_src, src_src, null)) {
28942 if (child != .ok) {28826 .ok => |child_strat| return .{ .ok = switch (child_strat) {
28943 return .{ .vector_elem = .{28827 .bit_cast => .bit_cast,
28944 .child = try child.dupe(sema.arena),28828 .ptr_cast => .ptr_cast,
28829 else => .none,
28830 } },
28831 else => |child_result| return .{ .vector_elem = .{
28832 .child = try child_result.dupe(sema.arena),
28945 .actual = src_elem_ty,28833 .actual = src_elem_ty,
28946 .wanted = dest_elem_ty,28834 .wanted = dest_elem_ty,
28947 } };28835 } },
28948 }28836 }
28949
28950 return .ok;
28951 }28837 }
2895228838
28953 // Optionals28839 // Optionals
...@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(...@@ -28971,7 +28857,7 @@ pub fn coerceInMemoryAllowed(
28971 } };28857 } };
28972 }28858 }
2897328859
28974 return .ok;28860 return .{ .ok = .none };
28975 }28861 }
2897628862
28977 // Tuples (with in-memory-coercible fields)28863 // Tuples (with in-memory-coercible fields)
...@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(...@@ -28985,7 +28871,7 @@ pub fn coerceInMemoryAllowed(
28985 const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);28871 const field = try sema.coerceInMemoryAllowed(block, dest_field_ty, src_field_ty, dest_is_mut, target, dest_src, src_src, null);
28986 if (field != .ok) break :tuple;28872 if (field != .ok) break :tuple;
28987 }28873 }
28988 return .ok;28874 return .{ .ok = .none };
28989 }28875 }
2899028876
28991 return .{ .no_match = .{28877 return .{ .no_match = .{
...@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29008,13 +28894,13 @@ fn coerceInMemoryAllowedErrorSets(
29008 const ip = &zcu.intern_pool;28894 const ip = &zcu.intern_pool;
2900928895
29010 const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {28896 const dest_set: InternPool.Key.ErrorSetType = err_set: switch (dest_ty.toIntern()) {
29011 .anyerror_type => return .ok,28897 .anyerror_type => return .{ .ok = .error_cast },
29012 .adhoc_inferred_error_set_type => {28898 .adhoc_inferred_error_set_type => {
29013 // We are trying to coerce an error set to the current function's28899 // We are trying to coerce an error set to the current function's
29014 // inferred error set.28900 // inferred error set.
29015 const dst_ies = sema.fn_ret_ty_ies.?;28901 const dst_ies = sema.fn_ret_ty_ies.?;
29016 try dst_ies.addErrorSet(src_ty, ip, sema.arena);28902 try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29017 return .ok;28903 return .{ .ok = .error_cast };
29018 },28904 },
29019 else => |err_set_ty| switch (ip.indexToKey(err_set_ty)) {28905 else => |err_set_ty| switch (ip.indexToKey(err_set_ty)) {
29020 .inferred_error_set_type => |func_index| {28906 .inferred_error_set_type => |func_index| {
...@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29023,7 +28909,7 @@ fn coerceInMemoryAllowedErrorSets(
29023 // We are trying to coerce an error set to the current function's28909 // We are trying to coerce an error set to the current function's
29024 // inferred error set.28910 // inferred error set.
29025 try dst_ies.addErrorSet(src_ty, ip, sema.arena);28911 try dst_ies.addErrorSet(src_ty, ip, sema.arena);
29026 return .ok;28912 return .{ .ok = .error_cast };
29027 }28913 }
29028 }28914 }
29029 try sema.ensureFuncIesResolved(block, dest_src, func_index);28915 try sema.ensureFuncIesResolved(block, dest_src, func_index);
...@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(...@@ -29062,7 +28948,7 @@ fn coerceInMemoryAllowedErrorSets(
29062 ) };28948 ) };
29063 }28949 }
2906428950
29065 return .ok;28951 return .{ .ok = .error_cast };
29066}28952}
2906728953
29068fn coerceInMemoryAllowedFns(28954fn coerceInMemoryAllowedFns(
...@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(...@@ -29179,7 +29065,7 @@ fn coerceInMemoryAllowedFns(
29179 }29065 }
29180 }29066 }
2918129067
29182 return .ok;29068 return .{ .ok = .none };
29183}29069}
2918429070
29185fn callconvCoerceAllowed(29071fn callconvCoerceAllowed(
...@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(...@@ -29256,7 +29142,7 @@ fn coerceInMemoryAllowedPtrs(
29256 const ok_ptr_size = src_info.flags.size == dest_info.flags.size or29142 const ok_ptr_size = src_info.flags.size == dest_info.flags.size or
29257 src_info.flags.size == .c or dest_info.flags.size == .c;29143 src_info.flags.size == .c or dest_info.flags.size == .c;
29258 if (!ok_ptr_size) {29144 if (!ok_ptr_size) {
29259 return InMemoryCoercionResult{ .ptr_size = .{29145 return .{ .ptr_size = .{
29260 .actual = src_info.flags.size,29146 .actual = src_info.flags.size,
29261 .wanted = dest_info.flags.size,29147 .wanted = dest_info.flags.size,
29262 } };29148 } };
...@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(...@@ -29392,14 +29278,14 @@ fn coerceInMemoryAllowedPtrs(
29392 break :a dest_child.abiAlignment(zcu);29278 break :a dest_child.abiAlignment(zcu);
29393 } else dest_info.flags.alignment;29279 } else dest_info.flags.alignment;
29394 if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {29280 if (dest_align.compare(if (dest_is_mut) .neq else .gt, src_align)) {
29395 return InMemoryCoercionResult{ .ptr_alignment = .{29281 return .{ .ptr_alignment = .{
29396 .actual = src_align,29282 .actual = src_align,
29397 .wanted = dest_align,29283 .wanted = dest_align,
29398 } };29284 } };
29399 }29285 }
29400 }29286 }
2940129287
29402 return .ok;29288 return .{ .ok = .ptr_cast };
29403}29289}
2940429290
29405fn coerceVarArgParam(29291fn coerceVarArgParam(
...@@ -29703,7 +29589,6 @@ fn bitCast(...@@ -29703,7 +29589,6 @@ fn bitCast(
29703 dest_ty: Type,29589 dest_ty: Type,
29704 inst: Air.Inst.Ref,29590 inst: Air.Inst.Ref,
29705 inst_src: LazySrcLoc,29591 inst_src: LazySrcLoc,
29706 operand_src: ?LazySrcLoc,
29707) CompileError!Air.Inst.Ref {29592) CompileError!Air.Inst.Ref {
29708 const pt = sema.pt;29593 const pt = sema.pt;
29709 const zcu = pt.zcu;29594 const zcu = pt.zcu;
...@@ -29712,6 +29597,11 @@ fn bitCast(...@@ -29712,6 +29597,11 @@ fn bitCast(
29712 old_ty.assertHasLayout(zcu);29597 old_ty.assertHasLayout(zcu);
29713 try sema.ensureLayoutResolved(dest_ty, inst_src, .init);29598 try sema.ensureLayoutResolved(dest_ty, inst_src, .init);
2971429599
29600 assert(old_ty.hasBitRepresentation(zcu));
29601 assert(dest_ty.hasBitRepresentation(zcu));
29602 assert(old_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
29603 assert(dest_ty.scalarType(zcu).zigTypeTag(zcu) != .pointer);
29604
29715 const dest_bits = dest_ty.bitSize(zcu);29605 const dest_bits = dest_ty.bitSize(zcu);
29716 const old_bits = old_ty.bitSize(zcu);29606 const old_bits = old_ty.bitSize(zcu);
2971729607
...@@ -29725,20 +29615,30 @@ fn bitCast(...@@ -29725,20 +29615,30 @@ fn bitCast(
29725 }29615 }
2972629616
29727 if (sema.resolveValue(inst)) |val| {29617 if (sema.resolveValue(inst)) |val| {
29728 if (val.isUndef(zcu))29618 return .fromValue(try sema.bitCastVal(val, dest_ty));
29729 return pt.undefRef(dest_ty);
29730 if (old_ty.zigTypeTag(zcu) == .error_set and dest_ty.zigTypeTag(zcu) == .error_set) {
29731 // Special case: we sometimes call `bitCast` on error set values, but they
29732 // don't have a well-defined layout, so we can't use `bitCastVal` on them.
29733 return Air.internedToRef((try pt.getCoerced(val, dest_ty)).toIntern());
29734 }
29735 if (try sema.bitCastVal(val, dest_ty, 0, 0, 0)) |result_val| {
29736 return Air.internedToRef(result_val.toIntern());
29737 }
29738 }29619 }
29739 try sema.requireRuntimeBlock(block, inst_src, operand_src);
29740 try sema.validateRuntimeValue(block, inst_src, inst);29620 try sema.validateRuntimeValue(block, inst_src, inst);
29741 return block.addBitCast(dest_ty, inst);29621 return block.addTyOp(.bit_cast, dest_ty, inst);
29622}
29623
29624/// Supports only types which `@bitCast` supports, so pointers are *not* supported.
29625pub fn bitCastVal(
29626 sema: *Sema,
29627 val: Value,
29628 dest_ty: Type,
29629) Allocator.Error!Value {
29630 const pt = sema.pt;
29631 const zcu = pt.zcu;
29632 const bit_size = dest_ty.bitSize(zcu);
29633 assert(val.typeOf(zcu).bitSize(zcu) == bit_size);
29634 if (val.isUndef(zcu)) {
29635 return pt.undefValue(dest_ty);
29636 } else {
29637 const buf = try sema.arena.alloc(u8, @intCast((bit_size + 7) / 8));
29638 @memset(buf, 0);
29639 val.writeToPackedMemory(zcu, buf, 0);
29640 return .readFromPackedMemory(dest_ty, pt, buf, 0);
29641 }
29742}29642}
2974329643
29744fn coerceArrayPtrToSlice(29644fn coerceArrayPtrToSlice(
...@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(...@@ -29855,14 +29755,17 @@ fn coerceCompatiblePtrs(
29855 );29755 );
29856 }29756 }
29857 try sema.requireRuntimeBlock(block, inst_src, null);29757 try sema.requireRuntimeBlock(block, inst_src, null);
29858 const inst_allows_zero = inst_ty.zigTypeTag(zcu) != .pointer or inst_ty.ptrAllowsZero(zcu);29758 const maybe_zero: bool = switch (inst_ty.toIntern()) {
29859 if (block.wantSafety() and inst_allows_zero and !dest_ty.ptrAllowsZero(zcu)) {29759 .usize_type, .isize_type => true,
29760 else => inst_ty.ptrAllowsZero(zcu),
29761 };
29762 if (block.wantSafety() and maybe_zero and !dest_ty.ptrAllowsZero(zcu)) {
29860 try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);29763 try sema.checkLogicalPtrOperation(block, inst_src, inst_ty);
29861 const actual_ptr = if (inst_ty.isSlice(zcu))29764 const actual_ptr = if (inst_ty.isSlice(zcu))
29862 try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)29765 try sema.analyzeSlicePtr(block, inst_src, inst, inst_ty)
29863 else29766 else
29864 inst;29767 inst;
29865 const ptr_int = try block.addBitCast(.usize, actual_ptr);29768 const ptr_int = try block.addTyOp(.int_from_ptr, .usize, actual_ptr);
29866 const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);29769 const is_non_zero = try block.addBinOp(.cmp_neq, ptr_int, .zero_usize);
29867 const ok = if (inst_ty.isSlice(zcu)) ok: {29770 const ok = if (inst_ty.isSlice(zcu)) ok: {
29868 const len = try sema.analyzeSliceLen(block, inst_src, inst);29771 const len = try sema.analyzeSliceLen(block, inst_src, inst);
...@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(...@@ -29871,7 +29774,14 @@ fn coerceCompatiblePtrs(
29871 } else is_non_zero;29774 } else is_non_zero;
29872 try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);29775 try sema.addSafetyCheck(block, inst_src, ok, .cast_to_null);
29873 }29776 }
29874 const new_ptr = try sema.bitCast(block, dest_ty, inst, inst_src, null);29777 const new_ptr: Air.Inst.Ref = switch (inst_ty.toIntern()) {
29778 .usize_type => try block.addTyOp(.ptr_from_int, dest_ty, inst),
29779 .isize_type => new_ptr: {
29780 const usize_inst = try block.addTyOp(.bit_cast, .usize, inst);
29781 break :new_ptr try block.addTyOp(.ptr_from_int, dest_ty, usize_inst);
29782 },
29783 else => try block.addTyOp(.ptr_cast, dest_ty, inst),
29784 };
29875 try sema.checkKnownAllocPtr(block, inst, new_ptr);29785 try sema.checkKnownAllocPtr(block, inst, new_ptr);
29876 return new_ptr;29786 return new_ptr;
29877}29787}
...@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(...@@ -29968,7 +29878,7 @@ fn coerceEnumToUnion(
29968 return .fromValue(opv);29878 return .fromValue(opv);
29969 } else {29879 } else {
29970 // The union layout is just the tag, so we can bitcast the enum straight to the union.29880 // The union layout is just the tag, so we can bitcast the enum straight to the union.
29971 return block.addBitCast(union_ty, enum_tag);29881 return block.addTyOp(.union_from_enum, union_ty, enum_tag);
29972 }29882 }
29973 }29883 }
2997429884
...@@ -30017,18 +29927,6 @@ fn coerceArrayLike(...@@ -30017,18 +29927,6 @@ fn coerceArrayLike(
30017 const inst_ty = sema.typeOf(inst);29927 const inst_ty = sema.typeOf(inst);
30018 const target = zcu.getTarget();29928 const target = zcu.getTarget();
3001929929
30020 // try coercion of the whole array
30021 const in_memory_result = try sema.coerceInMemoryAllowed(block, dest_ty, inst_ty, false, target, dest_ty_src, inst_src, null);
30022 if (in_memory_result == .ok) {
30023 if (sema.resolveValue(inst)) |inst_val| {
30024 // These types share the same comptime value representation.
30025 return sema.coerceInMemory(inst_val, dest_ty);
30026 }
30027 try sema.requireRuntimeBlock(block, inst_src, null);
30028 return block.addBitCast(dest_ty, inst);
30029 }
30030
30031 // otherwise, try element by element
30032 const inst_len = inst_ty.arrayLen(zcu);29930 const inst_len = inst_ty.arrayLen(zcu);
30033 const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));29931 const dest_len = try sema.usizeCast(block, dest_ty_src, dest_ty.arrayLen(zcu));
30034 if (dest_len != inst_len) {29932 if (dest_len != inst_len) {
...@@ -30055,7 +29953,7 @@ fn coerceArrayLike(...@@ -30055,7 +29953,7 @@ fn coerceArrayLike(
30055 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))29953 (dst_info.signedness == .signed and dst_info.bits > src_info.bits))
30056 {29954 {
30057 try sema.requireRuntimeBlock(block, inst_src, null);29955 try sema.requireRuntimeBlock(block, inst_src, null);
30058 return block.addTyOp(.intcast, dest_ty, inst);29956 return block.addTyOp(.int_cast, dest_ty, inst);
30059 }29957 }
30060 },29958 },
30061 .float => if (inst_elem_ty.isRuntimeFloat()) {29959 .float => if (inst_elem_ty.isRuntimeFloat()) {
...@@ -30582,7 +30480,7 @@ fn analyzeRef(...@@ -30582,7 +30480,7 @@ fn analyzeRef(
3058230480
30583 // Cast to the constant pointer type. We do this directly rather than going via `coerce` to30481 // Cast to the constant pointer type. We do this directly rather than going via `coerce` to
30584 // avoid errors in the `block.isComptime()` case.30482 // avoid errors in the `block.isComptime()` case.
30585 return block.addBitCast(ptr_type, alloc);30483 return block.addTyOp(.ptr_cast, ptr_type, alloc);
30586}30484}
3058730485
30588fn analyzeLoad(30486fn analyzeLoad(
...@@ -31334,7 +31232,7 @@ fn analyzeSlice(...@@ -31334,7 +31232,7 @@ fn analyzeSlice(
3133431232
31335 const opt_new_ptr_val = sema.resolveValue(new_ptr);31233 const opt_new_ptr_val = sema.resolveValue(new_ptr);
31336 const new_ptr_val = opt_new_ptr_val orelse {31234 const new_ptr_val = opt_new_ptr_val orelse {
31337 const result = try block.addBitCast(return_ty, new_ptr);31235 const result = try block.addTyOp(.ptr_cast, return_ty, new_ptr);
31338 if (block.wantSafety()) {31236 if (block.wantSafety()) {
31339 // requirement: slicing C ptr is non-null31237 // requirement: slicing C ptr is non-null
31340 if (ptr_ptr_child_ty.isCPtr(zcu)) {31238 if (ptr_ptr_child_ty.isCPtr(zcu)) {
...@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {...@@ -34366,8 +34264,8 @@ pub fn flushExports(sema: *Sema) !void {
34366 }34264 }
34367}34265}
3436834266
34369pub const bitCastVal = @import("Sema/bitcast.zig").bitCast;34267pub const castMemory = @import("Sema/reinterpret.zig").castMemory;
34370pub const bitCastSpliceVal = @import("Sema/bitcast.zig").bitCastSplice;34268pub const spliceMemory = @import("Sema/reinterpret.zig").spliceMemory;
3437134269
34372const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;34270const loadComptimePtr = @import("Sema/comptime_ptr_access.zig").loadComptimePtr;
34373const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;34271const ComptimeLoadResult = @import("Sema/comptime_ptr_access.zig").ComptimeLoadResult;
src/Sema/LowerZon.zig+4-11
...@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool...@@ -815,20 +815,15 @@ fn lowerStruct(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool
815 .@"packed" => result: {815 .@"packed" => result: {
816 const arena = self.sema.arena;816 const arena = self.sema.arena;
817 const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));817 const buf = try arena.alloc(u8, @intCast((res_ty.bitSize(zcu) + 7) / 8));
818 @memset(buf, 0);
818 var bit_offset: u16 = 0;819 var bit_offset: u16 = 0;
819 for (field_values) |field_ip| {820 for (field_values) |field_ip| {
820 const field_val: Value = .fromInterned(field_ip);821 const field_val: Value = .fromInterned(field_ip);
821 field_val.writeToPackedMemory(zcu, buf, bit_offset) catch |err| switch (err) {822 field_val.writeToPackedMemory(zcu, buf, bit_offset);
822 error.ReinterpretDeclRef => unreachable, // bitpack fields cannot be pointers
823 error.OutOfMemory => |e| return e,
824 };
825 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));823 bit_offset += @intCast(field_val.typeOf(zcu).bitSize(zcu));
826 }824 }
827 assert(bit_offset == res_ty.bitSize(zcu));825 assert(bit_offset == res_ty.bitSize(zcu));
828 break :result Value.readFromPackedMemory(res_ty, pt, buf, 0, arena) catch |err| switch (err) {826 break :result try .readFromPackedMemory(res_ty, pt, buf, 0);
829 error.IllDefinedMemoryLayout => unreachable, // bitpacks have well-defined layout
830 error.OutOfMemory => |e| return e,
831 };
832 },827 },
833 };828 };
834 return result.toIntern();829 return result.toIntern();
...@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool....@@ -981,9 +976,7 @@ fn lowerUnion(self: *LowerZon, node: Zoir.Node.Index, res_ty: Type) !InternPool.
981 };976 };
982 const result: Value = switch (union_info.layout) {977 const result: Value = switch (union_info.layout) {
983 .auto, .@"extern" => try pt.unionValue(res_ty, tag, val),978 .auto, .@"extern" => try pt.unionValue(res_ty, tag, val),
984 .@"packed" => try self.sema.bitCastVal(val, res_ty, 0, 0, 0) orelse {979 .@"packed" => try self.sema.bitCastVal(val, res_ty),
985 unreachable; // `null` is only possible if the input value contains a pointer, which a packed union cannot.
986 },
987 };980 };
988 return result.toIntern();981 return result.toIntern();
989}982}
src/Sema/bitcast.zig deleted-774
...@@ -1,774 +0,0 @@
1//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3//! a sequence of values in *packed* memory, and then unflatten through a combination of special
4//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5//!
6//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7//! have their fields reversed when represented as packed memory on such targets.
8
9/// If `host_bits` is `0`, attempts to convert the memory at offset
10/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11/// ignoring `bit_offset`.
12///
13/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14/// non-packed value consisting of `host_bits` bits. A value of type
15/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16/// into this value.
17///
18/// Returns `null` if the operation must be performed at runtime.
19pub fn bitCast(
20 sema: *Sema,
21 val: Value,
22 dest_ty: Type,
23 byte_offset: u64,
24 host_bits: u64,
25 bit_offset: u64,
26) CompileError!?Value {
27 return bitCastInner(sema, val, dest_ty, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
28 error.ReinterpretDeclRef => return null,
29 error.IllDefinedMemoryLayout => unreachable,
30 error.Unimplemented => @panic("unimplemented bitcast"),
31 else => |e| return e,
32 };
33}
34
35/// Uses bitcasting to splice the value `splice_val` into `val`,
36/// replacing overlapping bits and returning the modified value.
37///
38/// If `host_bits` is `0`, splices `splice_val` at an offset
39/// `byte_offset` bytes into the virtual memory of `val`, ignoring
40/// `bit_offset`.
41///
42/// Otherwise, `byte_offset` is an offset into bytes into `val` to
43/// a non-packed value consisting of `host_bits` bits. The value
44/// `splice_val` will be placed at a packed offset of `bit_offset`
45/// into this value.
46pub fn bitCastSplice(
47 sema: *Sema,
48 val: Value,
49 splice_val: Value,
50 byte_offset: u64,
51 host_bits: u64,
52 bit_offset: u64,
53) CompileError!?Value {
54 return bitCastSpliceInner(sema, val, splice_val, byte_offset, host_bits, bit_offset) catch |err| switch (err) {
55 error.ReinterpretDeclRef => return null,
56 error.IllDefinedMemoryLayout => unreachable,
57 error.Unimplemented => @panic("unimplemented bitcast"),
58 else => |e| return e,
59 };
60}
61
62const BitCastError = CompileError || error{ ReinterpretDeclRef, IllDefinedMemoryLayout, Unimplemented };
63
64fn bitCastInner(
65 sema: *Sema,
66 val: Value,
67 dest_ty: Type,
68 byte_offset: u64,
69 host_bits: u64,
70 bit_offset: u64,
71) BitCastError!Value {
72 const pt = sema.pt;
73 const zcu = pt.zcu;
74 const endian = zcu.getTarget().cpu.arch.endian();
75
76 if (dest_ty.toIntern() == val.typeOf(zcu).toIntern() and bit_offset == 0) {
77 return val;
78 }
79
80 const val_ty = val.typeOf(zcu);
81
82 val_ty.assertHasLayout(zcu);
83 dest_ty.assertHasLayout(zcu);
84
85 assert(val_ty.hasWellDefinedLayout(zcu));
86
87 const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
88 .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
89 else
90 .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
91
92 const skip_bits = switch (endian) {
93 .little => bit_offset + byte_offset * 8,
94 .big => if (host_bits > 0)
95 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
96 else
97 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - dest_ty.bitSize(zcu),
98 };
99
100 var unpack: UnpackValueBits = .{
101 .pt = sema.pt,
102 .arena = sema.arena,
103 .skip_bits = skip_bits,
104 .remaining_bits = dest_ty.bitSize(zcu),
105 .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
106 };
107 switch (endian) {
108 .little => {
109 try unpack.add(val);
110 try unpack.padding(abi_pad_bits);
111 },
112 .big => {
113 try unpack.padding(abi_pad_bits);
114 try unpack.add(val);
115 },
116 }
117 try unpack.padding(host_pad_bits);
118
119 var pack: PackValueBits = .{
120 .pt = sema.pt,
121 .arena = sema.arena,
122 .unpacked = unpack.unpacked.items,
123 };
124 return pack.get(dest_ty);
125}
126
127fn bitCastSpliceInner(
128 sema: *Sema,
129 val: Value,
130 splice_val: Value,
131 byte_offset: u64,
132 host_bits: u64,
133 bit_offset: u64,
134) BitCastError!Value {
135 const pt = sema.pt;
136 const zcu = pt.zcu;
137 const endian = zcu.getTarget().cpu.arch.endian();
138 const val_ty = val.typeOf(zcu);
139 const splice_val_ty = splice_val.typeOf(zcu);
140
141 val_ty.assertHasLayout(zcu);
142 splice_val_ty.assertHasLayout(zcu);
143
144 const splice_bits = splice_val_ty.bitSize(zcu);
145
146 const splice_offset = switch (endian) {
147 .little => bit_offset + byte_offset * 8,
148 .big => if (host_bits > 0)
149 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - host_bits + bit_offset
150 else
151 val_ty.abiSize(zcu) * 8 - byte_offset * 8 - splice_bits,
152 };
153
154 assert(splice_offset + splice_bits <= val_ty.abiSize(zcu) * 8);
155
156 const abi_pad_bits, const host_pad_bits = if (host_bits > 0)
157 .{ val_ty.abiSize(zcu) * 8 - host_bits, host_bits - val_ty.bitSize(zcu) }
158 else
159 .{ val_ty.abiSize(zcu) * 8 - val_ty.bitSize(zcu), 0 };
160
161 var unpack: UnpackValueBits = .{
162 .pt = pt,
163 .arena = sema.arena,
164 .skip_bits = 0,
165 .remaining_bits = splice_offset,
166 .unpacked = std.array_list.Managed(InternPool.Index).init(sema.arena),
167 };
168 switch (endian) {
169 .little => {
170 try unpack.add(val);
171 try unpack.padding(abi_pad_bits);
172 },
173 .big => {
174 try unpack.padding(abi_pad_bits);
175 try unpack.add(val);
176 },
177 }
178 try unpack.padding(host_pad_bits);
179
180 unpack.remaining_bits = splice_bits;
181 try unpack.add(splice_val);
182
183 unpack.skip_bits = splice_offset + splice_bits;
184 unpack.remaining_bits = val_ty.abiSize(zcu) * 8 - splice_offset - splice_bits;
185 switch (endian) {
186 .little => {
187 try unpack.add(val);
188 try unpack.padding(abi_pad_bits);
189 },
190 .big => {
191 try unpack.padding(abi_pad_bits);
192 try unpack.add(val);
193 },
194 }
195 try unpack.padding(host_pad_bits);
196
197 var pack: PackValueBits = .{
198 .pt = pt,
199 .arena = sema.arena,
200 .unpacked = unpack.unpacked.items,
201 };
202 switch (endian) {
203 .little => {},
204 .big => try pack.padding(abi_pad_bits),
205 }
206 return pack.get(val_ty);
207}
208
209/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
210/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
211/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
212/// list contains bits from the *final* byte of the value.
213const UnpackValueBits = struct {
214 pt: Zcu.PerThread,
215 arena: Allocator,
216 skip_bits: u64,
217 remaining_bits: u64,
218 extra_bits: u64 = undefined,
219 unpacked: std.array_list.Managed(InternPool.Index),
220
221 fn add(unpack: *UnpackValueBits, val: Value) BitCastError!void {
222 const pt = unpack.pt;
223 const zcu = pt.zcu;
224 const endian = zcu.getTarget().cpu.arch.endian();
225 const ip = &zcu.intern_pool;
226
227 if (unpack.remaining_bits == 0) {
228 return;
229 }
230
231 const ty = val.typeOf(zcu);
232 const bit_size = ty.bitSize(zcu);
233
234 if (unpack.skip_bits >= bit_size) {
235 unpack.skip_bits -= bit_size;
236 return;
237 }
238
239 switch (ip.indexToKey(val.toIntern())) {
240 .int_type,
241 .ptr_type,
242 .array_type,
243 .vector_type,
244 .opt_type,
245 .anyframe_type,
246 .error_union_type,
247 .simple_type,
248 .struct_type,
249 .tuple_type,
250 .union_type,
251 .opaque_type,
252 .spirv_type,
253 .enum_type,
254 .func_type,
255 .error_set_type,
256 .inferred_error_set_type,
257 .@"extern",
258 .func,
259 .err,
260 .error_union,
261 .enum_literal,
262 .slice,
263 .memoized_call,
264 => unreachable, // ill-defined layout or not real values
265
266 .undef,
267 .int,
268 .enum_tag,
269 .simple_value,
270 .float,
271 .ptr,
272 .opt,
273 => try unpack.primitive(val),
274
275 .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
276
277 .aggregate => switch (ty.zigTypeTag(zcu)) {
278 .vector => {
279 const len: usize = @intCast(ty.arrayLen(zcu));
280 for (0..len) |i| {
281 // We reverse vector elements in packed memory on BE targets.
282 const real_idx = switch (endian) {
283 .little => i,
284 .big => len - i - 1,
285 };
286 const elem_val = try val.elemValue(pt, real_idx);
287 try unpack.add(elem_val);
288 }
289 },
290 .array => {
291 // Each element is padded up to its ABI size. Padding bits are undefined.
292 // The final element does not have trailing padding.
293 // Elements are reversed in packed memory on BE targets.
294 const elem_ty = ty.childType(zcu);
295 const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
296 const len = ty.arrayLen(zcu);
297 const maybe_sent = ty.sentinel(zcu);
298
299 if (endian == .big) if (maybe_sent) |s| {
300 try unpack.add(s);
301 if (len != 0) try unpack.padding(pad_bits);
302 };
303
304 for (0..@intCast(len)) |i| {
305 // We reverse array elements in packed memory on BE targets.
306 const real_idx = switch (endian) {
307 .little => i,
308 .big => len - i - 1,
309 };
310 const elem_val = try val.elemValue(pt, @intCast(real_idx));
311 try unpack.add(elem_val);
312 if (i != len - 1) try unpack.padding(pad_bits);
313 }
314
315 if (endian == .little) if (maybe_sent) |s| {
316 if (len != 0) try unpack.padding(pad_bits);
317 try unpack.add(s);
318 };
319 },
320 .@"struct" => switch (ty.containerLayout(zcu)) {
321 .auto => unreachable, // ill-defined layout
322 .@"extern" => switch (endian) {
323 .little => {
324 var cur_bit_off: u64 = 0;
325 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
326 while (it.next()) |field_idx| {
327 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
328 const pad_bits = want_bit_off - cur_bit_off;
329 const field_val = try val.fieldValue(pt, field_idx);
330 try unpack.padding(pad_bits);
331 try unpack.add(field_val);
332 cur_bit_off = want_bit_off + field_val.typeOf(zcu).bitSize(zcu);
333 }
334 // Add trailing padding bits.
335 try unpack.padding(bit_size - cur_bit_off);
336 },
337 .big => {
338 var cur_bit_off: u64 = bit_size;
339 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
340 while (it.next()) |field_idx| {
341 const field_val = try val.fieldValue(pt, field_idx);
342 const field_ty = field_val.typeOf(zcu);
343 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
344 const pad_bits = cur_bit_off - want_bit_off;
345 try unpack.padding(pad_bits);
346 try unpack.add(field_val);
347 cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
348 }
349 assert(cur_bit_off == 0);
350 },
351 },
352 .@"packed" => {
353 // Just add all fields in order. There are no padding bits.
354 // This is identical between LE and BE targets.
355 for (0..ty.structFieldCount(zcu)) |i| {
356 const field_val = try val.fieldValue(pt, i);
357 try unpack.add(field_val);
358 }
359 },
360 },
361 else => unreachable,
362 },
363
364 .un => |un| {
365 // We actually don't care about the tag here!
366 // Instead, we just need to write the payload value, plus any necessary padding.
367 // This correctly handles the case where `tag == .none`, since the payload is then
368 // either an integer or a byte array, both of which we can unpack.
369 const payload_val = Value.fromInterned(un.val);
370 const pad_bits = bit_size - payload_val.typeOf(zcu).bitSize(zcu);
371 if (endian == .little or ty.containerLayout(zcu) == .@"packed") {
372 try unpack.add(payload_val);
373 try unpack.padding(pad_bits);
374 } else {
375 try unpack.padding(pad_bits);
376 try unpack.add(payload_val);
377 }
378 },
379 }
380 }
381
382 fn padding(unpack: *UnpackValueBits, pad_bits: u64) BitCastError!void {
383 if (pad_bits == 0) return;
384 const pt = unpack.pt;
385 // Figure out how many full bytes and leftover bits there are.
386 const bytes = pad_bits / 8;
387 const bits = pad_bits % 8;
388 // Add undef u8 values for the bytes...
389 const undef_u8 = try pt.undefValue(Type.u8);
390 for (0..@intCast(bytes)) |_| {
391 try unpack.primitive(undef_u8);
392 }
393 // ...and an undef int for the leftover bits.
394 if (bits == 0) return;
395 const bits_ty = try pt.intType(.unsigned, @intCast(bits));
396 const bits_val = try pt.undefValue(bits_ty);
397 try unpack.primitive(bits_val);
398 }
399
400 fn primitive(unpack: *UnpackValueBits, val: Value) BitCastError!void {
401 const pt = unpack.pt;
402 const zcu = pt.zcu;
403
404 if (unpack.remaining_bits == 0) {
405 return;
406 }
407
408 const ty = val.typeOf(pt.zcu);
409 const bit_size = ty.bitSize(zcu);
410
411 // Note that this skips all zero-bit types.
412 if (unpack.skip_bits >= bit_size) {
413 unpack.skip_bits -= bit_size;
414 return;
415 }
416
417 if (unpack.skip_bits > 0) {
418 const skip = unpack.skip_bits;
419 unpack.skip_bits = 0;
420 return unpack.splitPrimitive(val, skip, bit_size - skip);
421 }
422
423 if (unpack.remaining_bits < bit_size) {
424 return unpack.splitPrimitive(val, 0, unpack.remaining_bits);
425 }
426
427 unpack.remaining_bits -|= bit_size;
428
429 try unpack.unpacked.append(val.toIntern());
430 }
431
432 fn splitPrimitive(unpack: *UnpackValueBits, val: Value, bit_offset: u64, bit_count: u64) BitCastError!void {
433 const pt = unpack.pt;
434 const zcu = pt.zcu;
435 const ty = val.typeOf(pt.zcu);
436
437 const val_bits = ty.bitSize(zcu);
438 assert(bit_offset + bit_count <= val_bits);
439
440 switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
441 // In the `ptr` case, this will return `error.ReinterpretDeclRef`
442 // if we're trying to split a non-integer pointer value.
443 .int, .float, .enum_tag, .ptr, .opt => {
444 // This @intCast is okay because no primitive can exceed the size of a u16.
445 const int_ty = try unpack.pt.intType(.unsigned, @intCast(bit_count));
446 const buf = try unpack.arena.alloc(u8, @intCast((val_bits + 7) / 8));
447 try val.writeToPackedMemory(zcu, buf, 0);
448 const sub_val = try Value.readFromPackedMemory(int_ty, unpack.pt, buf, @intCast(bit_offset), unpack.arena);
449 try unpack.primitive(sub_val);
450 },
451 .undef => try unpack.padding(bit_count),
452 // The only values here with runtime bits are `true` and `false.
453 // These are both 1 bit, so will never need truncating.
454 .simple_value => unreachable,
455 else => unreachable, // zero-bit or not primitives
456 }
457 }
458};
459
460/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBits`),
461/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
462/// values and of pointers which align in virtual memory.
463const PackValueBits = struct {
464 pt: Zcu.PerThread,
465 arena: Allocator,
466 bit_offset: u64 = 0,
467 unpacked: []const InternPool.Index,
468
469 fn get(pack: *PackValueBits, ty: Type) BitCastError!Value {
470 const pt = pack.pt;
471 const zcu = pt.zcu;
472 const endian = zcu.getTarget().cpu.arch.endian();
473 const ip = &zcu.intern_pool;
474 const arena = pack.arena;
475 switch (ty.zigTypeTag(zcu)) {
476 .vector => {
477 // Elements are bit-packed.
478 const len = ty.arrayLen(zcu);
479 const elem_ty = ty.childType(zcu);
480 const elems = try arena.alloc(InternPool.Index, @intCast(len));
481 // We reverse vector elements in packed memory on BE targets.
482 switch (endian) {
483 .little => for (elems) |*elem| {
484 elem.* = (try pack.get(elem_ty)).toIntern();
485 },
486 .big => {
487 var i = elems.len;
488 while (i > 0) {
489 i -= 1;
490 elems[i] = (try pack.get(elem_ty)).toIntern();
491 }
492 },
493 }
494 return pt.aggregateValue(ty, elems);
495 },
496 .array => {
497 // Each element is padded up to its ABI size. The final element does not have trailing padding.
498 const len = ty.arrayLen(zcu);
499 const elem_ty = ty.childType(zcu);
500 const maybe_sent = ty.sentinel(zcu);
501 const pad_bits = elem_ty.abiSize(zcu) * 8 - elem_ty.bitSize(zcu);
502 const elems = try arena.alloc(InternPool.Index, @intCast(len));
503
504 if (endian == .big and maybe_sent != null) {
505 // TODO: validate sentinel was preserved!
506 try pack.padding(elem_ty.bitSize(zcu));
507 if (len != 0) try pack.padding(pad_bits);
508 }
509
510 for (0..elems.len) |i| {
511 const real_idx = switch (endian) {
512 .little => i,
513 .big => len - i - 1,
514 };
515 elems[@intCast(real_idx)] = (try pack.get(elem_ty)).toIntern();
516 if (i != len - 1) try pack.padding(pad_bits);
517 }
518
519 if (endian == .little and maybe_sent != null) {
520 // TODO: validate sentinel was preserved!
521 if (len != 0) try pack.padding(pad_bits);
522 try pack.padding(elem_ty.bitSize(zcu));
523 }
524
525 return pt.aggregateValue(ty, elems);
526 },
527 .@"struct" => switch (ty.containerLayout(zcu)) {
528 .auto => unreachable, // ill-defined layout
529 .@"extern" => {
530 const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
531 @memset(elems, .none);
532 switch (endian) {
533 .little => {
534 var cur_bit_off: u64 = 0;
535 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrder(ip);
536 while (it.next()) |field_idx| {
537 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8;
538 try pack.padding(want_bit_off - cur_bit_off);
539 const field_ty = ty.fieldType(field_idx, zcu);
540 elems[field_idx] = (try pack.get(field_ty)).toIntern();
541 cur_bit_off = want_bit_off + field_ty.bitSize(zcu);
542 }
543 try pack.padding(ty.bitSize(zcu) - cur_bit_off);
544 },
545 .big => {
546 var cur_bit_off: u64 = ty.bitSize(zcu);
547 var it = zcu.typeToStruct(ty).?.iterateRuntimeOrderReverse(ip);
548 while (it.next()) |field_idx| {
549 const field_ty = ty.fieldType(field_idx, zcu);
550 const want_bit_off = ty.structFieldOffset(field_idx, zcu) * 8 + field_ty.bitSize(zcu);
551 try pack.padding(cur_bit_off - want_bit_off);
552 elems[field_idx] = (try pack.get(field_ty)).toIntern();
553 cur_bit_off = want_bit_off - field_ty.bitSize(zcu);
554 }
555 assert(cur_bit_off == 0);
556 },
557 }
558 // Any fields which do not have runtime bits should be OPV or comptime fields.
559 // Fill those values now.
560 for (elems, 0..) |*elem, field_idx| {
561 if (elem.* != .none) continue;
562 const val = (try ty.structFieldValueComptime(pt, field_idx)).?;
563 elem.* = val.toIntern();
564 }
565 return pt.aggregateValue(ty, elems);
566 },
567 .@"packed" => {
568 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
569 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
570 return pt.bitpackValue(ty, backing_int_val);
571 },
572 },
573 .@"union" => switch (ty.containerLayout(zcu)) {
574 .auto => unreachable, // ill-defined layout
575 .@"extern" => {
576 // We will attempt to read as the backing representation. If this emits
577 // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
578 // We will also attempt smaller fields when we get `undefined`, as if some bits are
579 // defined we want to include them.
580 // TODO: this is very very bad. We need a more sophisticated union representation.
581
582 const prev_unpacked = pack.unpacked;
583 const prev_bit_offset = pack.bit_offset;
584
585 const backing_ty = try ty.externUnionBackingType(pt);
586
587 backing: {
588 const backing_val = pack.get(backing_ty) catch |err| switch (err) {
589 error.ReinterpretDeclRef => {
590 pack.unpacked = prev_unpacked;
591 pack.bit_offset = prev_bit_offset;
592 break :backing;
593 },
594 else => |e| return e,
595 };
596 if (backing_val.isUndef(zcu)) {
597 pack.unpacked = prev_unpacked;
598 pack.bit_offset = prev_bit_offset;
599 break :backing;
600 }
601 return Value.fromInterned(try pt.internUnion(.{
602 .ty = ty.toIntern(),
603 .tag = .none,
604 .val = backing_val.toIntern(),
605 }));
606 }
607
608 const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
609 for (field_order, 0..) |*f, i| f.* = @intCast(i);
610 // Sort `field_order` to put the fields with the largest bit sizes first.
611 const SizeSortCtx = struct {
612 zcu: *Zcu,
613 field_types: []const InternPool.Index,
614 fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
615 const a_ty = Type.fromInterned(ctx.field_types[a_idx]);
616 const b_ty = Type.fromInterned(ctx.field_types[b_idx]);
617 return a_ty.bitSize(ctx.zcu) > b_ty.bitSize(ctx.zcu);
618 }
619 };
620 std.mem.sortUnstable(u32, field_order, SizeSortCtx{
621 .zcu = zcu,
622 .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
623 }, SizeSortCtx.lessThan);
624
625 const padding_after = endian == .little or ty.containerLayout(zcu) == .@"packed";
626
627 for (field_order) |field_idx| {
628 const field_ty = Type.fromInterned(zcu.typeToUnion(ty).?.field_types.get(ip)[field_idx]);
629 const pad_bits = ty.bitSize(zcu) - field_ty.bitSize(zcu);
630 if (!padding_after) try pack.padding(pad_bits);
631 const field_val = pack.get(field_ty) catch |err| switch (err) {
632 error.ReinterpretDeclRef => {
633 pack.unpacked = prev_unpacked;
634 pack.bit_offset = prev_bit_offset;
635 continue;
636 },
637 else => |e| return e,
638 };
639 if (padding_after) try pack.padding(pad_bits);
640 if (field_val.isUndef(zcu)) {
641 pack.unpacked = prev_unpacked;
642 pack.bit_offset = prev_bit_offset;
643 continue;
644 }
645 const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_idx);
646 return Value.fromInterned(try pt.internUnion(.{
647 .ty = ty.toIntern(),
648 .tag = tag_val.toIntern(),
649 .val = field_val.toIntern(),
650 }));
651 }
652
653 // No field could represent the value. Just do whatever happens when we try to read
654 // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
655 const backing_val = try pack.get(backing_ty);
656 return Value.fromInterned(try pt.internUnion(.{
657 .ty = ty.toIntern(),
658 .tag = .none,
659 .val = backing_val.toIntern(),
660 }));
661 },
662 .@"packed" => {
663 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
664 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
665 return pt.bitpackValue(ty, backing_int_val);
666 },
667 },
668 else => return pack.primitive(ty),
669 }
670 }
671
672 fn padding(pack: *PackValueBits, pad_bits: u64) BitCastError!void {
673 _ = pack.prepareBits(pad_bits);
674 }
675
676 fn primitive(pack: *PackValueBits, want_ty: Type) BitCastError!Value {
677 const pt = pack.pt;
678 const zcu = pt.zcu;
679
680 if (try want_ty.onePossibleValue(pt)) |opv| return opv;
681
682 const vals, const bit_offset = pack.prepareBits(want_ty.bitSize(zcu));
683
684 for (vals) |val| {
685 if (!Value.fromInterned(val).isUndef(zcu)) break;
686 } else {
687 // All bits of the value are `undefined`.
688 return pt.undefValue(want_ty);
689 }
690
691 // TODO: we need to decide how to handle partially-undef values here.
692 // Currently, a value with some undefined bits becomes `0xAA` so that we
693 // preserve the well-defined bits, because we can't currently represent
694 // a partially-undefined primitive (e.g. an int with some undef bits).
695 // In future, we probably want to take one of these two routes:
696 // * Define that if any bits are `undefined`, the entire value is `undefined`.
697 // This is a major breaking change, and probably a footgun.
698 // * Introduce tracking for partially-undef values at comptime.
699 // This would complicate a lot of operations in Sema, such as basic
700 // arithmetic.
701 // This design complexity is tracked by #19634.
702
703 ptr_cast: {
704 if (vals.len != 1) break :ptr_cast;
705 const val = Value.fromInterned(vals[0]);
706 if (!val.typeOf(zcu).isPtrAtRuntime(zcu)) break :ptr_cast;
707 if (!want_ty.isPtrAtRuntime(zcu)) break :ptr_cast;
708 return pt.getCoerced(val, want_ty);
709 }
710
711 // Reinterpret via an in-memory buffer.
712
713 var buf_bits: u64 = 0;
714 for (vals) |ip_val| {
715 const val = Value.fromInterned(ip_val);
716 const ty = val.typeOf(pt.zcu);
717 buf_bits += ty.bitSize(zcu);
718 }
719
720 const buf = try pack.arena.alloc(u8, @intCast((buf_bits + 7) / 8));
721 // We will skip writing undefined values, so mark the buffer as `0xAA` so we get "undefined" bits.
722 @memset(buf, 0xAA);
723 var cur_bit_off: usize = 0;
724 for (vals) |ip_val| {
725 const val = Value.fromInterned(ip_val);
726 const ty = val.typeOf(zcu);
727 if (!val.isUndef(zcu)) {
728 try val.writeToPackedMemory(zcu, buf, cur_bit_off);
729 }
730 cur_bit_off += @intCast(ty.bitSize(zcu));
731 }
732
733 return Value.readFromPackedMemory(want_ty, pt, buf, @intCast(bit_offset), pack.arena);
734 }
735
736 fn prepareBits(pack: *PackValueBits, need_bits: u64) struct { []const InternPool.Index, u64 } {
737 if (need_bits == 0) return .{ &.{}, 0 };
738
739 const pt = pack.pt;
740 const zcu = pt.zcu;
741
742 var bits: u64 = 0;
743 var len: usize = 0;
744 while (bits < pack.bit_offset + need_bits) {
745 bits += Value.fromInterned(pack.unpacked[len]).typeOf(pt.zcu).bitSize(zcu);
746 len += 1;
747 }
748
749 const result_vals = pack.unpacked[0..len];
750 const result_offset = pack.bit_offset;
751
752 const extra_bits = bits - pack.bit_offset - need_bits;
753 if (extra_bits == 0) {
754 pack.unpacked = pack.unpacked[len..];
755 pack.bit_offset = 0;
756 } else {
757 pack.unpacked = pack.unpacked[len - 1 ..];
758 pack.bit_offset = Value.fromInterned(pack.unpacked[0]).typeOf(pt.zcu).bitSize(zcu) - extra_bits;
759 }
760
761 return .{ result_vals, result_offset };
762 }
763};
764
765const std = @import("std");
766const Allocator = std.mem.Allocator;
767const assert = std.debug.assert;
768
769const Sema = @import("../Sema.zig");
770const Zcu = @import("../Zcu.zig");
771const InternPool = @import("../InternPool.zig");
772const Type = @import("../Type.zig");
773const Value = @import("../Value.zig");
774const CompileError = Zcu.CompileError;
src/Sema/comptime_ptr_access.zig+173-148
...@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {...@@ -14,27 +14,46 @@ pub const ComptimeLoadResult = union(enum) {
14pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {14pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) !ComptimeLoadResult {
15 const pt = sema.pt;15 const pt = sema.pt;
16 const zcu = pt.zcu;16 const zcu = pt.zcu;
17
17 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);18 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
18 // TODO: host size for vectors is terrible19 const elem_ty: Type = .fromInterned(ptr_info.child);
19 const host_bits = switch (ptr_info.flags.vector_index) {20 const host_size = ptr_info.packed_offset.host_size;
20 .none => ptr_info.packed_offset.host_size * 8,21
21 else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),22 if (host_size == 0) {
22 };23 return loadComptimePtrInner(sema, block, src, ptr, elem_ty, 0);
23 const bit_offset = if (host_bits != 0) bit_offset: {24 }
24 const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);25
25 const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {26 assert(elem_ty.hasBitRepresentation(zcu));
26 .none => 0,27 if (ptr_info.flags.vector_index == .none) {
27 else => |idx| switch (pt.zcu.getTarget().cpu.arch.endian()) {28 if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
28 .little => child_bits * @intFromEnum(idx),
29 .big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian
30 },
31 };
32 if (child_bits + bit_offset > host_bits) {
33 return .exceeds_host_size;29 return .exceeds_host_size;
34 }30 }
35 break :bit_offset bit_offset;31 const load_ty: Type = try pt.intType(.unsigned, host_size * 8);
36 } else 0;32 const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
37 return loadComptimePtrInner(sema, block, src, ptr, bit_offset, host_bits, Type.fromInterned(ptr_info.child), 0);33 else => |result| return result,
34 .success => |mv| mv,
35 };
36 const backing_int_val = try backing_int_mv.intern(pt, sema.arena);
37 const buf = try sema.arena.alloc(u8, host_size);
38 @memset(buf, 0);
39 backing_int_val.writeToPackedMemory(zcu, buf, 0);
40 const result_val: Value = try .readFromPackedMemory(elem_ty, pt, buf, ptr_info.packed_offset.bit_offset);
41 return .{ .success = .{ .interned = result_val.toIntern() } };
42 }
43 if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
44 return .exceeds_host_size;
45 }
46 const load_ty: Type = try pt.vectorType(.{
47 .len = host_size,
48 .child = elem_ty.toIntern(),
49 });
50 const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, load_ty, 0)) {
51 else => |result| return result,
52 .success => |mv| mv,
53 };
54 const vector_val = try vector_mv.intern(pt, sema.arena);
55 const result_val = try vector_val.elemValue(pt, @intFromEnum(ptr_info.flags.vector_index));
56 return .{ .success = .{ .interned = result_val.toIntern() } };
38}57}
3958
40pub const ComptimeStoreResult = union(enum) {59pub const ComptimeStoreResult = union(enum) {
...@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {...@@ -52,7 +71,8 @@ pub const ComptimeStoreResult = union(enum) {
52};71};
5372
54/// Perform a comptime load of value `store_val` to a pointer.73/// Perform a comptime load of value `store_val` to a pointer.
55/// The pointer's type is ignored.74///
75/// Asserts that the type of `store_val` equals the element type of the pointer type.
56pub fn storeComptimePtr(76pub fn storeComptimePtr(
57 sema: *Sema,77 sema: *Sema,
58 block: *Block,78 block: *Block,
...@@ -62,42 +82,84 @@ pub fn storeComptimePtr(...@@ -62,42 +82,84 @@ pub fn storeComptimePtr(
62) !ComptimeStoreResult {82) !ComptimeStoreResult {
63 const pt = sema.pt;83 const pt = sema.pt;
64 const zcu = pt.zcu;84 const zcu = pt.zcu;
65 const ptr_info = ptr.typeOf(zcu).ptrInfo(zcu);
66 assert(store_val.typeOf(zcu).toIntern() == ptr_info.child);
6785
68 {86 const ptr_info = ptr.typeOf(pt.zcu).ptrInfo(pt.zcu);
69 const store_ty: Type = .fromInterned(ptr_info.child);87 const elem_ty: Type = .fromInterned(ptr_info.child);
70 if (!store_ty.comptimeOnly(zcu) and !store_ty.hasRuntimeBits(zcu)) {88 const host_size = ptr_info.packed_offset.host_size;
71 // zero-bit store; nothing to do89 assert(store_val.typeOf(zcu).toIntern() == elem_ty.toIntern());
72 return .success;90
73 }91 if (host_size == 0) {
92 return storeComptimePtrInner(sema, block, src, ptr, store_val);
74 }93 }
7594
76 // TODO: host size for vectors is terrible95 assert(elem_ty.hasBitRepresentation(zcu));
77 const host_bits = switch (ptr_info.flags.vector_index) {96 if (ptr_info.flags.vector_index == .none) {
78 .none => ptr_info.packed_offset.host_size * 8,97 if (ptr_info.packed_offset.bit_offset + elem_ty.bitSize(zcu) > host_size * 8) {
79 else => ptr_info.packed_offset.host_size * Type.fromInterned(ptr_info.child).bitSize(zcu),
80 };
81 const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) {
82 .none => 0,
83 else => |idx| switch (zcu.getTarget().cpu.arch.endian()) {
84 .little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx),
85 .big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian
86 },
87 };
88 const pseudo_store_ty = if (host_bits > 0) t: {
89 const need_bits = Type.fromInterned(ptr_info.child).bitSize(zcu);
90 if (need_bits + bit_offset > host_bits) {
91 return .exceeds_host_size;98 return .exceeds_host_size;
92 }99 }
93 break :t try sema.pt.intType(.unsigned, @intCast(host_bits));100 const backing_ty: Type = try pt.intType(.unsigned, host_size * 8);
94 } else Type.fromInterned(ptr_info.child);101 const backing_int_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, backing_ty, 0)) {
102 .success => |mv| mv,
103 .runtime_load => return .runtime_store,
104 inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
105 };
106 const old_backing_int_val = try backing_int_mv.intern(pt, sema.arena);
107 const buf = try sema.arena.alloc(u8, host_size);
108 @memset(buf, 0);
109 old_backing_int_val.writeToPackedMemory(zcu, buf, 0);
110 // Write the new element...
111 store_val.writeToPackedMemory(zcu, buf, ptr_info.packed_offset.bit_offset);
112 // ...then read the resulting backing integer value...
113 const new_backing_int_val: Value = try .readFromPackedMemory(backing_ty, pt, buf, 0);
114 // ...and store that back into memory
115 return storeComptimePtrInner(sema, block, src, ptr, new_backing_int_val);
116 }
117
118 if (@intFromEnum(ptr_info.flags.vector_index) >= host_size) {
119 return .exceeds_host_size;
120 }
121 const vec_ty: Type = try pt.vectorType(.{
122 .len = host_size,
123 .child = elem_ty.toIntern(),
124 });
125 const vector_mv = switch (try loadComptimePtrInner(sema, block, src, ptr, vec_ty, 0)) {
126 .success => |mv| mv,
127 .runtime_load => return .runtime_store,
128 inline else => |payload, tag| return @unionInit(ComptimeStoreResult, @tagName(tag), payload),
129 };
130 const old_vector_val = try vector_mv.intern(pt, sema.arena);
131 const elems_buf = try sema.arena.alloc(InternPool.Index, host_size);
132 for (elems_buf, 0..) |*elem, elem_index| {
133 const elem_val = try old_vector_val.elemValue(pt, elem_index);
134 elem.* = elem_val.toIntern();
135 }
136 elems_buf[@intFromEnum(ptr_info.flags.vector_index)] = store_val.toIntern();
137 const new_vector_val = try pt.aggregateValue(vec_ty, elems_buf);
138 return storeComptimePtrInner(sema, block, src, ptr, new_vector_val);
139}
95140
96 const strat = try prepareComptimePtrStore(sema, block, src, ptr, pseudo_store_ty, 0);141/// Like `storeComptimePtr`, except ignores the type of `ptr`, instead treating it as a single-item
142/// pointer to `store_val.typeOf(zcu)`.
143fn storeComptimePtrInner(
144 sema: *Sema,
145 block: *Block,
146 src: LazySrcLoc,
147 ptr: Value,
148 store_val: Value,
149) !ComptimeStoreResult {
150 const pt = sema.pt;
151 const zcu = pt.zcu;
152 const store_ty = store_val.typeOf(zcu);
153
154 if (store_ty.classify(zcu) == .one_possible_value) {
155 // zero-bit store; nothing to do
156 return .success;
157 }
158
159 const strat = try prepareComptimePtrStore(sema, block, src, ptr, store_ty, 0);
97160
98 // Propagate errors and handle comptime fields.161 // Propagate errors and handle comptime fields.
99 switch (strat) {162 switch (strat) {
100 .direct, .index, .flat_index, .reinterpret => {},
101 .comptime_field => {163 .comptime_field => {
102 // To "store" to a comptime field, just perform a load of the field164 // To "store" to a comptime field, just perform a load of the field
103 // and see if the store value matches.165 // and see if the store value matches.
...@@ -125,79 +187,60 @@ pub fn storeComptimePtr(...@@ -125,79 +187,60 @@ pub fn storeComptimePtr(
125 .inactive_union_field => return .inactive_union_field,187 .inactive_union_field => return .inactive_union_field,
126 .needed_well_defined => |ty| return .{ .needed_well_defined = ty },188 .needed_well_defined => |ty| return .{ .needed_well_defined = ty },
127 .out_of_bounds => |ty| return .{ .out_of_bounds = ty },189 .out_of_bounds => |ty| return .{ .out_of_bounds = ty },
128 }
129
130 // Check the store is not inside a runtime condition
131 try checkComptimeVarStore(sema, block, src, strat.alloc());
132
133 if (host_bits == 0) {
134 // We can attempt a direct store depending on the strategy.
135 switch (strat) {
136 .direct => |direct| {
137 const want_ty = direct.val.typeOf(zcu);
138 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
139 direct.val.* = .{ .interned = coerced_store_val.toIntern() };
140 return .success;
141 },
142 .index => |index| {
143 const want_ty = index.val.typeOf(zcu).childType(zcu);
144 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
145 try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
146 return .success;
147 },
148 .flat_index => |flat| {
149 const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
150 const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
151 {
152 var next_idx: u64 = 0;
153 var skip: u64 = 0;
154 try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
155 }
156 for (flat_elems, 0..) |elem, idx| {
157 // TODO: recursiveIndex in a loop does a lot of redundant work!
158 // Better would be to gather all the store targets into an array.
159 var index: u64 = flat.flat_elem_index + idx;
160 const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
161 try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
162 }
163 return .success;
164 },
165 .reinterpret => {},
166 else => unreachable,
167 }
168 }
169190
170 // Either there is a bit offset, or the strategy required reinterpreting.191 .direct => |direct| {
171 // Therefore, we must perform a bitcast.192 try checkComptimeVarStore(sema, block, src, direct.alloc);
193 const want_ty = direct.val.typeOf(zcu);
194 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
195 direct.val.* = .{ .interned = coerced_store_val.toIntern() };
196 return .success;
197 },
172198
173 const val_ptr: *MutableValue, const byte_offset: u64 = switch (strat) {199 .index => |index| {
174 .direct => |direct| .{ direct.val, 0 },200 try checkComptimeVarStore(sema, block, src, index.alloc);
175 .index => |index| .{201 const want_ty = index.val.typeOf(zcu).childType(zcu);
176 index.val,202 const coerced_store_val = try pt.getCoerced(store_val, want_ty);
177 index.elem_index * index.val.typeOf(zcu).childType(zcu).abiSize(zcu),203 try index.val.setElem(pt, sema.arena, @intCast(index.elem_index), .{ .interned = coerced_store_val.toIntern() });
204 return .success;
178 },205 },
179 .flat_index => |flat| .{ flat.val, flat.flat_elem_index * flat.val.typeOf(zcu).arrayBase(zcu)[0].abiSize(zcu) },
180 .reinterpret => |reinterpret| .{ reinterpret.val, reinterpret.byte_offset },
181 else => unreachable,
182 };
183206
184 if (!val_ptr.typeOf(zcu).hasWellDefinedLayout(zcu)) {207 .flat_index => |flat| {
185 return .{ .needed_well_defined = val_ptr.typeOf(zcu) };208 try checkComptimeVarStore(sema, block, src, flat.alloc);
186 }209 const store_elems = store_val.typeOf(zcu).arrayBase(zcu)[1];
210 const flat_elems = try sema.arena.alloc(InternPool.Index, @intCast(store_elems));
211 {
212 var next_idx: u64 = 0;
213 var skip: u64 = 0;
214 try flattenArray(sema, .{ .interned = store_val.toIntern() }, &skip, &next_idx, flat_elems);
215 }
216 for (flat_elems, 0..) |elem, idx| {
217 // TODO: recursiveIndex in a loop does a lot of redundant work!
218 // Better would be to gather all the store targets into an array.
219 var index: u64 = flat.flat_elem_index + idx;
220 const val_ptr, const final_idx = (try recursiveIndex(sema, flat.val, &index)).?;
221 try val_ptr.setElem(pt, sema.arena, @intCast(final_idx), .{ .interned = elem });
222 }
223 return .success;
224 },
187225
188 if (!store_val.typeOf(zcu).hasWellDefinedLayout(zcu)) {226 .reinterpret => |reinterpret| {
189 return .{ .needed_well_defined = store_val.typeOf(zcu) };227 try checkComptimeVarStore(sema, block, src, reinterpret.alloc);
228 if (!reinterpret.val.typeOf(zcu).hasWellDefinedLayout(zcu)) {
229 return .{ .needed_well_defined = reinterpret.val.typeOf(zcu) };
230 }
231 if (!store_ty.hasWellDefinedLayout(zcu)) {
232 return .{ .needed_well_defined = store_ty };
233 }
234 const old_val = try reinterpret.val.intern(pt, sema.arena);
235 const new_val = try sema.spliceMemory(
236 old_val,
237 store_val,
238 reinterpret.byte_offset,
239 ) orelse return .runtime_store;
240 reinterpret.val.* = .{ .interned = new_val.toIntern() };
241 return .success;
242 },
190 }243 }
191
192 const new_val = try sema.bitCastSpliceVal(
193 try val_ptr.intern(pt, sema.arena),
194 store_val,
195 byte_offset,
196 host_bits,
197 bit_offset,
198 ) orelse return .runtime_store;
199 val_ptr.* = .{ .interned = new_val.toIntern() };
200 return .success;
201}244}
202245
203/// Perform a comptime load of type `load_ty` from a pointer.246/// Perform a comptime load of type `load_ty` from a pointer.
...@@ -207,8 +250,6 @@ fn loadComptimePtrInner(...@@ -207,8 +250,6 @@ fn loadComptimePtrInner(
207 block: *Block,250 block: *Block,
208 src: LazySrcLoc,251 src: LazySrcLoc,
209 ptr_val: Value,252 ptr_val: Value,
210 bit_offset: u64,
211 host_bits: u64,
212 load_ty: Type,253 load_ty: Type,
213 /// If `load_ty` is an array, this is the number of array elements to skip254 /// If `load_ty` is an array, this is the number of array elements to skip
214 /// before `load_ty`. Otherwise, it is ignored and may be `undefined`.255 /// before `load_ty`. Otherwise, it is ignored and may be `undefined`.
...@@ -244,7 +285,7 @@ fn loadComptimePtrInner(...@@ -244,7 +285,7 @@ fn loadComptimePtrInner(
244 .eu_payload => |base_ptr_ip| val: {285 .eu_payload => |base_ptr_ip| val: {
245 const base_ptr = Value.fromInterned(base_ptr_ip);286 const base_ptr = Value.fromInterned(base_ptr_ip);
246 const base_ty = base_ptr.typeOf(zcu).childType(zcu);287 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
247 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {288 switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
248 .success => |eu_val| switch (eu_val.unpackErrorUnion(zcu)) {289 .success => |eu_val| switch (eu_val.unpackErrorUnion(zcu)) {
249 .undef => return .undef,290 .undef => return .undef,
250 .err => |err| return .{ .err_payload = err },291 .err => |err| return .{ .err_payload = err },
...@@ -256,7 +297,7 @@ fn loadComptimePtrInner(...@@ -256,7 +297,7 @@ fn loadComptimePtrInner(
256 .opt_payload => |base_ptr_ip| val: {297 .opt_payload => |base_ptr_ip| val: {
257 const base_ptr = Value.fromInterned(base_ptr_ip);298 const base_ptr = Value.fromInterned(base_ptr_ip);
258 const base_ty = base_ptr.typeOf(zcu).childType(zcu);299 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
259 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {300 switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
260 .success => |eu_val| switch (eu_val.unpackOptional(zcu)) {301 .success => |eu_val| switch (eu_val.unpackOptional(zcu)) {
261 .undef => return .undef,302 .undef => return .undef,
262 .null => return .null_payload,303 .null => return .null_payload,
...@@ -283,7 +324,7 @@ fn loadComptimePtrInner(...@@ -283,7 +324,7 @@ fn loadComptimePtrInner(
283 .child = base_ty.toIntern(),324 .child = base_ty.toIntern(),
284 });325 });
285326
286 switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, want_ty, base_index.index)) {327 switch (try loadComptimePtrInner(sema, block, src, base_ptr, want_ty, base_index.index)) {
287 .success => |arr_val| break :val arr_val,328 .success => |arr_val| break :val arr_val,
288 else => |err| return err,329 else => |err| return err,
289 }330 }
...@@ -293,7 +334,7 @@ fn loadComptimePtrInner(...@@ -293,7 +334,7 @@ fn loadComptimePtrInner(
293 const base_ty = base_ptr.typeOf(zcu).childType(zcu);334 const base_ty = base_ptr.typeOf(zcu).childType(zcu);
294335
295 // Field of a slice, or of an auto-layout struct or union.336 // Field of a slice, or of an auto-layout struct or union.
296 const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, 0, 0, base_ty, undefined)) {337 const agg_val = switch (try loadComptimePtrInner(sema, block, src, base_ptr, base_ty, undefined)) {
297 .success => |val| val,338 .success => |val| val,
298 else => |err| return err,339 else => |err| return err,
299 };340 };
...@@ -324,7 +365,7 @@ fn loadComptimePtrInner(...@@ -324,7 +365,7 @@ fn loadComptimePtrInner(
324 },365 },
325 };366 };
326367
327 if (ptr.byte_offset == 0 and host_bits == 0) {368 if (ptr.byte_offset == 0) {
328 if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {369 if (load_ty.zigTypeTag(zcu) != .array or array_offset == 0) {
329 if (.ok == try sema.coerceInMemoryAllowed(370 if (.ok == try sema.coerceInMemoryAllowed(
330 block,371 block,
...@@ -343,8 +384,6 @@ fn loadComptimePtrInner(...@@ -343,8 +384,6 @@ fn loadComptimePtrInner(
343 }384 }
344385
345 restructure_array: {386 restructure_array: {
346 if (host_bits != 0) break :restructure_array;
347
348 // We might also be changing the length of an array, or restructuring it.387 // We might also be changing the length of an array, or restructuring it.
349 // e.g. [1][2][3]T -> [3][2]T.388 // e.g. [1][2][3]T -> [3][2]T.
350 // This case is important because it's permitted for types with ill-defined layouts.389 // This case is important because it's permitted for types with ill-defined layouts.
...@@ -402,7 +441,7 @@ fn loadComptimePtrInner(...@@ -402,7 +441,7 @@ fn loadComptimePtrInner(
402 cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;441 cur_offset += load_ty.childType(zcu).abiSize(zcu) * array_offset;
403 }442 }
404443
405 const need_bytes = if (host_bits > 0) (host_bits + 7) / 8 else load_ty.abiSize(zcu);444 const need_bytes = load_ty.abiSize(zcu);
406445
407 if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {446 if (cur_offset + need_bytes > cur_val.typeOf(zcu).abiSize(zcu)) {
408 return .{ .out_of_bounds = cur_val.typeOf(zcu) };447 return .{ .out_of_bounds = cur_val.typeOf(zcu) };
...@@ -453,7 +492,7 @@ fn loadComptimePtrInner(...@@ -453,7 +492,7 @@ fn loadComptimePtrInner(
453 },492 },
454 .@"struct" => switch (cur_ty.containerLayout(zcu)) {493 .@"struct" => switch (cur_ty.containerLayout(zcu)) {
455 .auto => unreachable, // ill-defined layout494 .auto => unreachable, // ill-defined layout
456 .@"packed" => break, // let the bitcast logic handle this495 .@"packed" => break, // let the memory reinterpret logic handle this
457 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {496 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
458 const start_off = cur_ty.structFieldOffset(field_idx, zcu);497 const start_off = cur_ty.structFieldOffset(field_idx, zcu);
459 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);498 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...@@ -466,9 +505,9 @@ fn loadComptimePtrInner(...@@ -466,9 +505,9 @@ fn loadComptimePtrInner(
466 },505 },
467 .@"union" => switch (cur_ty.containerLayout(zcu)) {506 .@"union" => switch (cur_ty.containerLayout(zcu)) {
468 .auto => unreachable, // ill-defined layout507 .auto => unreachable, // ill-defined layout
469 .@"packed" => break, // let the bitcast logic handle this508 .@"packed" => break, // let the memory reinterpret logic handle this
470 .@"extern" => {509 .@"extern" => {
471 // TODO: we have to let bitcast logic handle this for now.510 // TODO: we have to let the memory reinterpret logic handle this for now.
472 // Otherwise, we might traverse into a union field which doesn't allow pointers.511 // Otherwise, we might traverse into a union field which doesn't allow pointers.
473 // Figure out a solution!512 // Figure out a solution!
474 if (true) break;513 if (true) break;
...@@ -495,27 +534,13 @@ fn loadComptimePtrInner(...@@ -495,27 +534,13 @@ fn loadComptimePtrInner(
495534
496 // Fast path: check again if we're now at the type we want to load.535 // Fast path: check again if we're now at the type we want to load.
497 // If so, just return the loaded value.536 // If so, just return the loaded value.
498 if (cur_offset == 0 and host_bits == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {537 if (cur_offset == 0 and cur_val.typeOf(zcu).toIntern() == load_ty.toIntern()) {
499 return .{ .success = cur_val };538 return .{ .success = cur_val };
500 }539 }
501540
502 var bitcast_src_val = try cur_val.intern(sema.pt, sema.arena);541 // Otherwise, use the memory reinterpretation logic to pull out the bytes we need.
503542 const reinterpret_val = try cur_val.intern(pt, sema.arena);
504 if (host_bits != 0) {543 const result_val = try sema.castMemory(reinterpret_val, load_ty, cur_offset) orelse return .runtime_load;
505 const src_bit_size = bitcast_src_val.typeOf(zcu).bitSize(zcu);
506 if (src_bit_size > host_bits) {
507 const truncate_ty = try pt.intType(.unsigned, @intCast(host_bits));
508 bitcast_src_val = try pt.getCoerced(bitcast_src_val, truncate_ty);
509 }
510 }
511
512 const result_val = try sema.bitCastVal(
513 bitcast_src_val,
514 load_ty,
515 cur_offset,
516 host_bits,
517 bit_offset,
518 ) orelse return .runtime_load;
519 return .{ .success = .{ .interned = result_val.toIntern() } };544 return .{ .success = .{ .interned = result_val.toIntern() } };
520}545}
521546
...@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {...@@ -546,7 +571,7 @@ const ComptimeStoreStrategy = union(enum) {
546 val: *MutableValue,571 val: *MutableValue,
547 flat_elem_index: u64,572 flat_elem_index: u64,
548 },573 },
549 /// This value should be reinterpreted using bitcast logic to perform the574 /// This value should be reinterpreted using `Sema.spliceMemory` to perform
550 /// store. Only returned if `store_ty` and the type of `val` both have575 /// store. Only returned if `store_ty` and the type of `val` both have
551 /// well-defined layouts.576 /// well-defined layouts.
552 reinterpret: struct {577 reinterpret: struct {
...@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(...@@ -886,7 +911,7 @@ fn prepareComptimePtrStore(
886 },911 },
887 .@"struct" => switch (cur_ty.containerLayout(zcu)) {912 .@"struct" => switch (cur_ty.containerLayout(zcu)) {
888 .auto => unreachable, // ill-defined layout913 .auto => unreachable, // ill-defined layout
889 .@"packed" => break, // let the bitcast logic handle this914 .@"packed" => break, // let the memory reinterp logic handle this
890 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {915 .@"extern" => for (0..cur_ty.structFieldCount(zcu)) |field_idx| {
891 const start_off = cur_ty.structFieldOffset(field_idx, zcu);916 const start_off = cur_ty.structFieldOffset(field_idx, zcu);
892 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);917 const end_off = start_off + cur_ty.fieldType(field_idx, zcu).abiSize(zcu);
...@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(...@@ -899,9 +924,9 @@ fn prepareComptimePtrStore(
899 },924 },
900 .@"union" => switch (cur_ty.containerLayout(zcu)) {925 .@"union" => switch (cur_ty.containerLayout(zcu)) {
901 .auto => unreachable, // ill-defined layout926 .auto => unreachable, // ill-defined layout
902 .@"packed" => break, // let the bitcast logic handle this927 .@"packed" => break, // let the memory reinterp logic handle this
903 .@"extern" => {928 .@"extern" => {
904 // TODO: we have to let bitcast logic handle this for now.929 // TODO: we have to let the memory reinterp logic handle this for now.
905 // Otherwise, we might traverse into a union field which doesn't allow pointers.930 // Otherwise, we might traverse into a union field which doesn't allow pointers.
906 // Figure out a solution!931 // Figure out a solution!
907 if (true) break;932 if (true) break;
src/Sema/reinterpret.zig created+576
...@@ -0,0 +1,576 @@
1//! This file contains logic for bit-casting arbitrary values at comptime, including splicing
2//! bits together for comptime stores of bit-pointers. The strategy is to "flatten" values to
3//! a sequence of values in *packed* memory, and then unflatten through a combination of special
4//! cases (particularly for pointers and `undefined` values) and in-memory buffer reinterprets.
5//!
6//! This is a little awkward on big-endian targets, as non-packed datastructures (e.g. `extern struct`)
7//! have their fields reversed when represented as packed memory on such targets.
8
9/// If `host_bits` is `0`, attempts to convert the memory at offset
10/// `byte_offset` into `val` to a non-packed value of type `dest_ty`,
11/// ignoring `bit_offset`.
12///
13/// Otherwise, `byte_offset` is an offset in bytes into `val` to a
14/// non-packed value consisting of `host_bits` bits. A value of type
15/// `dest_ty` will be interpreted at a packed offset of `bit_offset`
16/// into this value.
17///
18/// Returns `null` if the operation must be performed at runtime.
19pub fn castMemory(
20 sema: *Sema,
21 val: Value,
22 dest_ty: Type,
23 byte_offset: u64,
24) CompileError!?Value {
25 const pt = sema.pt;
26 const zcu = pt.zcu;
27
28 const val_ty = val.typeOf(zcu);
29
30 if (dest_ty.toIntern() == val_ty.toIntern()) {
31 assert(byte_offset == 0);
32 return val;
33 }
34
35 val_ty.assertHasLayout(zcu);
36 dest_ty.assertHasLayout(zcu);
37
38 var unpack: UnpackValueBytes = .{
39 .pt = pt,
40 .arena = sema.arena,
41 .skip_bytes = byte_offset,
42 .remaining_bytes = dest_ty.abiSize(zcu),
43 .unpacked = .init(sema.arena),
44 };
45 unpack.add(val) catch |err| switch (err) {
46 error.ReinterpretDeclRef => return null,
47 error.OutOfMemory => |e| return e,
48 };
49
50 var pack: PackValueBytes = .{
51 .pt = pt,
52 .arena = sema.arena,
53 .unpacked = unpack.unpacked.items,
54 };
55 return pack.get(dest_ty) catch |err| switch (err) {
56 error.ReinterpretDeclRef => return null,
57 error.OutOfMemory => |e| return e,
58 };
59}
60
61/// Splice the value `splice_val` into `val` at the given `byte_offset`, replacing overlapping bits
62/// and returning the modified value.
63pub fn spliceMemory(
64 sema: *Sema,
65 val: Value,
66 splice_val: Value,
67 byte_offset: u64,
68) CompileError!?Value {
69 const pt = sema.pt;
70 const zcu = pt.zcu;
71 const val_ty = val.typeOf(zcu);
72 const splice_val_ty = splice_val.typeOf(zcu);
73
74 val_ty.assertHasLayout(zcu);
75 splice_val_ty.assertHasLayout(zcu);
76
77 var unpack: UnpackValueBytes = .{
78 .pt = pt,
79 .arena = sema.arena,
80 .skip_bytes = 0,
81 .remaining_bytes = byte_offset,
82 .unpacked = .init(sema.arena),
83 };
84 unpack.add(val) catch |err| switch (err) {
85 error.ReinterpretDeclRef => return null,
86 error.OutOfMemory => |e| return e,
87 };
88
89 const splice_len = splice_val_ty.abiSize(zcu);
90
91 unpack.remaining_bytes = splice_len;
92 unpack.add(splice_val) catch |err| switch (err) {
93 error.ReinterpretDeclRef => return null,
94 error.OutOfMemory => |e| return e,
95 };
96
97 unpack.skip_bytes = byte_offset + splice_len;
98 unpack.remaining_bytes = val_ty.abiSize(zcu) * 8 - byte_offset - splice_len;
99 unpack.add(val) catch |err| switch (err) {
100 error.ReinterpretDeclRef => return null,
101 error.OutOfMemory => |e| return e,
102 };
103
104 var pack: PackValueBytes = .{
105 .pt = pt,
106 .arena = sema.arena,
107 .unpacked = unpack.unpacked.items,
108 };
109 return pack.get(val_ty) catch |err| switch (err) {
110 error.ReinterpretDeclRef => return null,
111 error.OutOfMemory => |e| return e,
112 };
113}
114
115/// Recurses through struct fields, array elements, etc, to get a sequence of "primitive" values
116/// which are bit-packed in memory to represent a single value. `unpacked` represents a series
117/// of values in *packed* memory - therefore, on big-endian targets, the first element of this
118/// list contains bits from the *final* byte of the value.
119const UnpackValueBytes = struct {
120 pt: Zcu.PerThread,
121 arena: Allocator,
122 skip_bytes: u64,
123 remaining_bytes: u64,
124 unpacked: std.array_list.Managed(InternPool.Index),
125
126 fn add(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
127 const pt = unpack.pt;
128 const zcu = pt.zcu;
129 const ip = &zcu.intern_pool;
130
131 if (unpack.remaining_bytes == 0) {
132 return;
133 }
134
135 const ty = val.typeOf(zcu);
136 const size = ty.abiSize(zcu);
137
138 if (unpack.skip_bytes >= size) {
139 unpack.skip_bytes -= size;
140 return;
141 }
142
143 switch (ip.indexToKey(val.toIntern())) {
144 .int_type,
145 .ptr_type,
146 .array_type,
147 .vector_type,
148 .opt_type,
149 .anyframe_type,
150 .error_union_type,
151 .simple_type,
152 .struct_type,
153 .tuple_type,
154 .union_type,
155 .opaque_type,
156 .spirv_type,
157 .enum_type,
158 .func_type,
159 .error_set_type,
160 .inferred_error_set_type,
161 .@"extern",
162 .func,
163 .err,
164 .error_union,
165 .enum_literal,
166 .slice,
167 .memoized_call,
168 => unreachable, // ill-defined layout or not real values
169
170 .undef,
171 .int,
172 .enum_tag,
173 .simple_value,
174 .float,
175 .ptr,
176 .opt,
177 => try unpack.primitive(val),
178
179 .bitpack => |bitpack| try unpack.primitive(.fromInterned(bitpack.backing_int_val)),
180
181 .aggregate => switch (ty.zigTypeTag(zcu)) {
182 .vector => unreachable, // ill-defined layout
183 .array => {
184 for (0..@intCast(ty.arrayLen(zcu))) |elem_index| {
185 const elem_val = try val.elemValue(pt, @intCast(elem_index));
186 try unpack.add(elem_val);
187 }
188 if (ty.sentinel(zcu)) |s| {
189 try unpack.add(s);
190 }
191 },
192 .@"struct" => switch (ty.containerLayout(zcu)) {
193 .auto => unreachable, // ill-defined layout
194 .@"packed" => unreachable, // uses `.bitpack`, not `.aggregate`
195 .@"extern" => {
196 var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
197 var offset: u64 = 0;
198 while (it.next()) |field_index| {
199 const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
200 const field_val = try val.fieldValue(pt, field_index);
201 try unpack.padding(pad_bytes);
202 try unpack.add(field_val);
203 offset += pad_bytes + field_val.typeOf(zcu).abiSize(zcu);
204 }
205 try unpack.padding(size - offset);
206 },
207 },
208 else => unreachable,
209 },
210
211 .un => |un| {
212 const payload_val = Value.fromInterned(un.val);
213 const pad_bytes = size - payload_val.typeOf(zcu).abiSize(zcu);
214 try unpack.add(payload_val);
215 try unpack.padding(pad_bytes);
216 },
217 }
218 }
219
220 fn padding(unpack: *UnpackValueBytes, num_bytes: u64) Allocator.Error!void {
221 if (num_bytes == 0) return;
222 const undef_u8 = try unpack.pt.undefValue(Type.u8);
223 for (0..@intCast(num_bytes)) |_| {
224 unpack.primitive(undef_u8) catch |err| switch (err) {
225 error.OutOfMemory => |e| return e,
226 error.ReinterpretDeclRef => unreachable,
227 };
228 }
229 }
230
231 fn primitive(unpack: *UnpackValueBytes, val: Value) (error{ReinterpretDeclRef} || Allocator.Error)!void {
232 const pt = unpack.pt;
233 const zcu = pt.zcu;
234
235 if (unpack.remaining_bytes == 0) {
236 return;
237 }
238
239 const ty = val.typeOf(pt.zcu);
240 const size = ty.abiSize(zcu);
241
242 if (unpack.skip_bytes >= size) {
243 unpack.skip_bytes -= size;
244 return;
245 }
246
247 if (unpack.skip_bytes > 0) {
248 const offset = unpack.skip_bytes;
249 unpack.skip_bytes = 0;
250 return unpack.splitPrimitive(val, offset, @min(size - offset, unpack.remaining_bytes));
251 }
252
253 if (unpack.remaining_bytes < size) {
254 return unpack.splitPrimitive(val, 0, unpack.remaining_bytes);
255 }
256
257 unpack.remaining_bytes -= size;
258 try unpack.unpacked.append(val.toIntern());
259 }
260
261 fn splitPrimitive(unpack: *UnpackValueBytes, val: Value, offset: u64, len: u64) (error{ReinterpretDeclRef} || Allocator.Error)!void {
262 const pt = unpack.pt;
263 const zcu = pt.zcu;
264 const ty = val.typeOf(pt.zcu);
265
266 assert(offset + len <= ty.abiSize(zcu));
267
268 try unpack.unpacked.ensureUnusedCapacity(@intCast(len));
269 unpack.remaining_bytes -= len;
270
271 switch (pt.zcu.intern_pool.indexToKey(val.toIntern())) {
272 // In the `ptr` case, this will return `error.ReinterpretDeclRef`
273 // if we're trying to split a non-integer pointer value.
274 .int, .float, .enum_tag, .ptr, .opt => {
275 const buf = try unpack.arena.alloc(u8, @intCast(ty.abiSize(zcu)));
276 val.writeToMemory(zcu, buf) catch |err| switch (err) {
277 error.IllDefinedMemoryLayout => unreachable,
278 else => |e| return e,
279 };
280 for (buf[@intCast(offset)..][0..@intCast(len)]) |byte_raw| {
281 const byte_val = try pt.intValue(.u8, byte_raw);
282 unpack.unpacked.appendAssumeCapacity(byte_val.toIntern());
283 }
284 },
285 .undef => {
286 const undef_u8 = try pt.undefValue(.u8);
287 for (0..@intCast(len)) |_| {
288 unpack.unpacked.appendAssumeCapacity(undef_u8.toIntern());
289 }
290 },
291 // The only values here with runtime bits are `true` and `false`.
292 // These are both 1 byte, so will never need splitting.
293 .simple_value => unreachable,
294 else => unreachable, // zero-bit or not primitives
295 }
296 }
297};
298
299/// Given a sequence of bit-packed values in packed memory (see `UnpackValueBytes`),
300/// reconstructs a value of an arbitrary type, with correct handling of `undefined`
301/// values and of pointers which align in virtual memory.
302const PackValueBytes = struct {
303 pt: Zcu.PerThread,
304 arena: Allocator,
305 byte_offset: u64 = 0,
306 unpacked: []const InternPool.Index,
307
308 fn get(pack: *PackValueBytes, ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
309 const pt = pack.pt;
310 const zcu = pt.zcu;
311 const ip = &zcu.intern_pool;
312 const arena = pack.arena;
313 switch (ty.zigTypeTag(zcu)) {
314 .vector => unreachable, // ill-defined layout
315 .array => {
316 // Each element is padded up to its ABI size. The final element does not have trailing padding.
317 const elem_ty = ty.childType(zcu);
318 const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
319
320 for (elems) |*elem| {
321 elem.* = (try pack.get(elem_ty)).toIntern();
322 }
323
324 if (ty.sentinel(zcu)) |s| {
325 _ = s; // TODO: validate sentinel was preserved!
326 pack.padding(elem_ty.abiSize(zcu));
327 }
328
329 return pt.aggregateValue(ty, elems);
330 },
331 .@"struct" => switch (ty.containerLayout(zcu)) {
332 .auto => unreachable, // ill-defined layout
333 .@"extern" => {
334 const elems = try arena.alloc(InternPool.Index, ty.structFieldCount(zcu));
335 @memset(elems, .none);
336 var offset: u64 = 0;
337 var it = ip.loadStructType(ty.toIntern()).iterateRuntimeOrder(ip);
338 while (it.next()) |field_index| {
339 const field_ty = ty.fieldType(field_index, zcu);
340 const pad_bytes = ty.structFieldOffset(field_index, zcu) - offset;
341 pack.padding(pad_bytes);
342 elems[field_index] = (try pack.get(field_ty)).toIntern();
343 offset += pad_bytes + field_ty.abiSize(zcu);
344 }
345 pack.padding(ty.abiSize(zcu) - offset);
346 // Any fields which do not have runtime bits should be OPV or comptime fields.
347 // Fill those values now.
348 for (elems, 0..) |*elem, field_index| {
349 if (elem.* != .none) continue;
350 const val = (try ty.structFieldValueComptime(pt, field_index)).?;
351 elem.* = val.toIntern();
352 }
353 return pt.aggregateValue(ty, elems);
354 },
355 .@"packed" => {
356 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
357 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
358 return pt.bitpackValue(ty, backing_int_val);
359 },
360 },
361 .@"union" => switch (ty.containerLayout(zcu)) {
362 .auto => unreachable, // ill-defined layout
363 .@"extern" => {
364 // We will attempt to read as the backing representation. If this emits
365 // `error.ReinterpretDeclRef`, we will try each union field, preferring larger ones.
366 // We will also attempt smaller fields when we get `undefined`, as if some bits are
367 // defined we want to include them.
368 // TODO: this is very very bad. We need a more sophisticated union representation.
369
370 const prev_unpacked = pack.unpacked;
371 const prev_byte_offset = pack.byte_offset;
372
373 const backing_ty = try ty.externUnionBackingType(pt);
374
375 const backing_result: enum { undef, reinterpret_decl_ref } = backing: {
376 const backing_val = pack.get(backing_ty) catch |err| switch (err) {
377 error.ReinterpretDeclRef => break :backing .reinterpret_decl_ref,
378 else => |e| return e,
379 };
380 if (backing_val.isUndef(zcu)) break :backing .undef;
381 return .fromInterned(try pt.internUnion(.{
382 .ty = ty.toIntern(),
383 .tag = .none,
384 .val = backing_val.toIntern(),
385 }));
386 };
387
388 const field_order = try pack.arena.alloc(u32, ty.unionTagTypeHypothetical(zcu).enumFieldCount(zcu));
389 for (field_order, 0..) |*f, i| f.* = @intCast(i);
390 // Sort `field_order` to put the fields with the largest ABI sizes first.
391 const SizeSortCtx = struct {
392 zcu: *const Zcu,
393 field_types: []const InternPool.Index,
394 fn lessThan(ctx: @This(), a_idx: u32, b_idx: u32) bool {
395 const a_ty: Type = .fromInterned(ctx.field_types[a_idx]);
396 const b_ty: Type = .fromInterned(ctx.field_types[b_idx]);
397 return a_ty.abiSize(ctx.zcu) > b_ty.abiSize(ctx.zcu);
398 }
399 };
400 std.mem.sortUnstable(u32, field_order, SizeSortCtx{
401 .zcu = zcu,
402 .field_types = zcu.typeToUnion(ty).?.field_types.get(ip),
403 }, SizeSortCtx.lessThan);
404
405 for (field_order) |field_index| {
406 pack.unpacked = prev_unpacked;
407 pack.byte_offset = prev_byte_offset;
408 const field_ty = ty.fieldType(field_index, zcu);
409 const field_val = pack.get(field_ty) catch |err| switch (err) {
410 error.ReinterpretDeclRef => continue,
411 else => |e| return e,
412 };
413 if (field_val.isUndef(zcu)) continue;
414 pack.padding(ty.abiSize(zcu) - field_ty.abiSize(zcu));
415 const tag_val = try pt.enumValueFieldIndex(ty.unionTagTypeHypothetical(zcu), field_index);
416 return pt.unionValue(ty, tag_val, field_val);
417 }
418
419 // No field could represent the value. Just do whatever happens when we try to read
420 // the backing type - either `undefined` or `error.ReinterpretDeclRef`.
421 switch (backing_result) {
422 .undef => return pt.undefValue(ty),
423 .reinterpret_decl_ref => return error.ReinterpretDeclRef,
424 }
425 },
426 .@"packed" => {
427 const backing_int_val = try pack.primitive(ty.bitpackBackingInt(zcu));
428 if (backing_int_val.isUndef(zcu)) return pt.undefValue(ty);
429 return pt.bitpackValue(ty, backing_int_val);
430 },
431 },
432 .@"enum" => {
433 const tag_int_val = try pack.primitive(ty.intTagType(zcu));
434 if (tag_int_val.isUndef(zcu)) return pt.undefValue(ty);
435 return pt.enumValue(ty, tag_int_val.toIntern());
436 },
437 else => return pack.primitive(ty),
438 }
439 }
440
441 fn padding(pack: *PackValueBytes, num_bytes: u64) void {
442 _ = pack.prepareBytes(num_bytes);
443 }
444
445 fn primitive(pack: *PackValueBytes, want_ty: Type) (Allocator.Error || error{ReinterpretDeclRef})!Value {
446 const pt = pack.pt;
447 const zcu = pt.zcu;
448
449 if (try want_ty.onePossibleValue(pt)) |opv| return opv;
450
451 const vals, const byte_offset = pack.prepareBytes(want_ty.abiSize(zcu));
452
453 for (vals) |val| {
454 if (!Value.fromInterned(val).isUndef(zcu)) break;
455 } else {
456 // All bits of the value are `undefined`.
457 return pt.undefValue(want_ty);
458 }
459
460 // TODO: we need to decide how to handle partially-undef values here.
461 // Currently, a value with some undefined bits becomes `0xAA` so that we
462 // preserve the well-defined bits, because we can't currently represent
463 // a partially-undefined primitive (e.g. an int with some undef bits).
464 // In future, we probably want to take one of these two routes:
465 // * Define that if any bits are `undefined`, the entire value is `undefined`.
466 // This is a major breaking change, and probably a footgun.
467 // * Introduce tracking for partially-undef values at comptime.
468 // This would complicate a lot of operations in Sema, such as basic
469 // arithmetic.
470 // This design complexity is tracked by #19634.
471
472 if (vals.len == 1 and
473 want_ty.isPtrAtRuntime(zcu) and
474 Value.fromInterned(vals[0]).typeOf(zcu).isPtrAtRuntime(zcu))
475 {
476 return pt.getCoerced(.fromInterned(vals[0]), want_ty);
477 }
478
479 // Reinterpret via an in-memory buffer.
480
481 var buf_len: u64 = 0;
482 for (vals) |ip_val| {
483 const val: Value = .fromInterned(ip_val);
484 buf_len += val.typeOf(zcu).abiSize(zcu);
485 }
486
487 const buf = try pack.arena.alloc(u8, @intCast(buf_len));
488 {
489 var offset: usize = 0;
490 for (vals) |ip_val| {
491 const val: Value = .fromInterned(ip_val);
492 const ty = val.typeOf(zcu);
493 const size = ty.abiSize(zcu);
494 if (val.isUndef(zcu)) {
495 @memset(buf[offset..][0..@intCast(size)], 0xAA);
496 } else {
497 val.writeToMemory(zcu, buf[offset..][0..@intCast(size)]) catch |err| switch (err) {
498 error.IllDefinedMemoryLayout => unreachable,
499 else => |e| return e,
500 };
501 }
502 offset += @intCast(size);
503 }
504 }
505 const bytes = buf[@intCast(byte_offset)..];
506
507 const target = zcu.getTarget();
508 const endian = target.cpu.arch.endian();
509 switch (want_ty.zigTypeTag(zcu)) {
510 .bool => return .makeBool(bytes[0] != 0),
511 .int => return .readIntFromMemory(want_ty, pt, bytes, pack.arena),
512 .float => switch (want_ty.floatBits(target)) {
513 16 => return pt.floatValue(want_ty, @as(f16, @bitCast(std.mem.readInt(u16, bytes[0..2], endian)))),
514 32 => return pt.floatValue(want_ty, @as(f32, @bitCast(std.mem.readInt(u32, bytes[0..4], endian)))),
515 64 => return pt.floatValue(want_ty, @as(f64, @bitCast(std.mem.readInt(u64, bytes[0..8], endian)))),
516 80 => return pt.floatValue(want_ty, @as(f80, @bitCast(std.mem.readInt(u80, bytes[0..10], endian)))),
517 128 => return pt.floatValue(want_ty, @as(f128, @bitCast(std.mem.readInt(u128, bytes[0..16], endian)))),
518 else => unreachable,
519 },
520 .pointer => {
521 assert(!want_ty.isSlice(zcu));
522 const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
523 return pt.ptrIntValue(want_ty, ptr_addr);
524 },
525 .optional => {
526 assert(want_ty.isPtrLikeOptional(zcu));
527 const ptr_ty = want_ty.optionalChild(zcu);
528 const ptr_addr = std.mem.readVarInt(u64, bytes[0..@intCast(want_ty.abiSize(zcu))], endian);
529 return .fromInterned(try pt.intern(.{ .opt = .{
530 .ty = want_ty.toIntern(),
531 .val = if (ptr_addr == 0) .none else (try pt.ptrIntValue(ptr_ty, ptr_addr)).toIntern(),
532 } }));
533 },
534 else => unreachable,
535 }
536 }
537
538 fn prepareBytes(pack: *PackValueBytes, need_bytes: u64) struct { []const InternPool.Index, u64 } {
539 if (need_bytes == 0) return .{ &.{}, 0 };
540
541 const pt = pack.pt;
542 const zcu = pt.zcu;
543
544 var bytes: u64 = 0;
545 var len: usize = 0;
546 while (bytes < pack.byte_offset + need_bytes) {
547 bytes += Value.fromInterned(pack.unpacked[len]).typeOf(zcu).abiSize(zcu);
548 len += 1;
549 }
550
551 const result_vals = pack.unpacked[0..len];
552 const result_offset = pack.byte_offset;
553
554 const extra_bytes = bytes - pack.byte_offset - need_bytes;
555 if (extra_bytes == 0) {
556 pack.unpacked = pack.unpacked[len..];
557 pack.byte_offset = 0;
558 } else {
559 pack.unpacked = pack.unpacked[len - 1 ..];
560 pack.byte_offset = Value.fromInterned(pack.unpacked[0]).typeOf(zcu).abiSize(zcu) - extra_bytes;
561 }
562
563 return .{ result_vals, result_offset };
564 }
565};
566
567const std = @import("std");
568const Allocator = std.mem.Allocator;
569const assert = std.debug.assert;
570
571const Sema = @import("../Sema.zig");
572const Zcu = @import("../Zcu.zig");
573const InternPool = @import("../InternPool.zig");
574const Type = @import("../Type.zig");
575const Value = @import("../Value.zig");
576const CompileError = Zcu.CompileError;
src/Type.zig+47-106
...@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {...@@ -757,9 +757,9 @@ pub fn hasWellDefinedLayout(ty: Type, zcu: *const Zcu) bool {
757 const ip = &zcu.intern_pool;757 const ip = &zcu.intern_pool;
758 return switch (ip.indexToKey(ty.toIntern())) {758 return switch (ip.indexToKey(ty.toIntern())) {
759 .int_type,759 .int_type,
760 .vector_type,
761 => true,760 => true,
762761
762 .vector_type,
763 .error_union_type,763 .error_union_type,
764 .error_set_type,764 .error_set_type,
765 .inferred_error_set_type,765 .inferred_error_set_type,
...@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {...@@ -1241,112 +1241,17 @@ pub fn errorAbiSize(zcu: *const Zcu) u64 {
1241}1241}
12421242
1243/// Asserts that `ty` is not an opaque or comptime-only type.1243/// Asserts that `ty` is not an opaque or comptime-only type.
1244/// Once #19755 is implemented, this query will only work on types with a defined bit-level representation.
1245pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {1244pub fn bitSize(ty: Type, zcu: *const Zcu) u64 {
1246 const target = zcu.getTarget();1245 return switch (ty.zigTypeTag(zcu)) {
1247 const ip = &zcu.intern_pool;1246 .void => 0,
1248 assertHasLayout(ty, zcu);1247 .bool => 1,
1249 return switch (ip.indexToKey(ty.toIntern())) {1248 .float => ty.floatBits(zcu.getTarget()),
1250 .int_type => |int_type| int_type.bits,1249 .pointer, .optional => {
1251 .ptr_type => |ptr_type| switch (ptr_type.flags.size) {1250 assert(ty.isPtrAtRuntime(zcu));
1252 .slice => target.ptrBitWidth() * 2,1251 return zcu.getTarget().ptrBitWidth();
1253 else => target.ptrBitWidth(),
1254 },
1255 .anyframe_type => target.ptrBitWidth(),
1256 .array_type => |array_type| {
1257 const elem_ty: Type = .fromInterned(array_type.child);
1258 const len = array_type.lenIncludingSentinel();
1259 return switch (zcu.comp.getZigBackend()) {
1260 .stage2_x86_64, .stage2_llvm => len * elem_ty.bitSize(zcu),
1261 // this case will be removed under #19755
1262 else => switch (len) {
1263 0 => 0,
1264 else => (len - 1) * 8 * elem_ty.abiSize(zcu) + elem_ty.bitSize(zcu),
1265 },
1266 };
1267 },
1268 .vector_type => |vec| vec.len * Type.fromInterned(vec.child).bitSize(zcu),
1269 .error_set_type, .inferred_error_set_type => zcu.errorSetBits(),
1270 .func_type => unreachable,
1271
1272 .simple_type => |t| switch (t) {
1273 .void => 0,
1274 .bool => 1,
1275 .anyerror, .adhoc_inferred_error_set => zcu.errorSetBits(),
1276 .usize, .isize => target.ptrBitWidth(),
1277
1278 .c_char => target.cTypeBitSize(.char),
1279 .c_short => target.cTypeBitSize(.short),
1280 .c_ushort => target.cTypeBitSize(.ushort),
1281 .c_int => target.cTypeBitSize(.int),
1282 .c_uint => target.cTypeBitSize(.uint),
1283 .c_long => target.cTypeBitSize(.long),
1284 .c_ulong => target.cTypeBitSize(.ulong),
1285 .c_longlong => target.cTypeBitSize(.longlong),
1286 .c_ulonglong => target.cTypeBitSize(.ulonglong),
1287 .c_longdouble => target.cTypeBitSize(.longdouble),
1288
1289 .f16 => 16,
1290 .f32 => 32,
1291 .f64 => 64,
1292 .f80 => 80,
1293 .f128 => 128,
1294
1295 .anyopaque => unreachable,
1296 .type => unreachable,
1297 .comptime_int => unreachable,
1298 .comptime_float => unreachable,
1299 .noreturn => unreachable,
1300 .null => unreachable,
1301 .undefined => unreachable,
1302 .enum_literal => unreachable,
1303 .generic_poison => unreachable,
1304 },
1305
1306 .struct_type => {
1307 const struct_obj = ip.loadStructType(ty.toIntern());
1308 switch (struct_obj.layout) {
1309 .@"packed" => return Type.fromInterned(struct_obj.packed_backing_int_type).bitSize(zcu),
1310 .auto, .@"extern" => return struct_obj.size * 8, // will be `unreachable` under #19755
1311 }
1312 },
1313 .union_type => {
1314 const union_obj = ip.loadUnionType(ty.toIntern());
1315 switch (union_obj.layout) {
1316 .@"packed" => return Type.fromInterned(union_obj.packed_backing_int_type).bitSize(zcu),
1317 .auto, .@"extern" => return union_obj.size * 8, // will be `unreachable` under #19755
1318 }
1319 },1252 },
1320 .enum_type => Type.fromInterned(ip.loadEnumType(ty.toIntern()).int_tag_type).bitSize(zcu),1253 .array, .vector => ty.arrayLenIncludingSentinel(zcu) * ty.childType(zcu).bitSize(zcu),
13211254 else => ty.intInfo(zcu).bits,
1322 // will be `unreachable` under #19755
1323 .opt_type,
1324 .error_union_type,
1325 .tuple_type,
1326 => ty.abiSize(zcu) * 8,
1327
1328 .opaque_type, .spirv_type => unreachable,
1329
1330 // values, not types
1331 .undef,
1332 .simple_value,
1333 .@"extern",
1334 .func,
1335 .int,
1336 .err,
1337 .error_union,
1338 .enum_literal,
1339 .enum_tag,
1340 .float,
1341 .ptr,
1342 .slice,
1343 .opt,
1344 .aggregate,
1345 .un,
1346 .bitpack,
1347 // memoization, not types
1348 .memoized_call,
1349 => unreachable,
1350 };1255 };
1351}1256}
13521257
...@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {...@@ -1528,6 +1433,7 @@ pub fn nullablePtrElem(ty: Type, zcu: *const Zcu) Type {
1528/// * `[*]T`1433/// * `[*]T`
1529/// * `[*c]T`1434/// * `[*c]T`
1530/// * `@SpirvType(.{ .runtime_array = T })`1435/// * `@SpirvType(.{ .runtime_array = T })`
1436/// * `*@SpirvType(.{ .runtime_array = T })`
1531pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {1437pub fn indexableElem(ty: Type, zcu: *const Zcu) Type {
1532 const ip = &zcu.intern_pool;1438 const ip = &zcu.intern_pool;
1533 return switch (ip.indexToKey(ty.toIntern())) {1439 return switch (ip.indexToKey(ty.toIntern())) {
...@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool...@@ -3181,6 +3087,8 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3181 .frame,3087 .frame,
3182 => false,3088 => false,
31833089
3090 .vector => position == .param_ty or position == .ret_ty,
3091
3184 .void => switch (position) {3092 .void => switch (position) {
3185 .ret_ty,3093 .ret_ty,
3186 .union_field,3094 .union_field,
...@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool...@@ -3259,7 +3167,6 @@ pub fn validateExtern(ty: Type, position: ExternPosition, zcu: *const Zcu) bool
3259 .other,3167 .other,
3260 => ty.childType(zcu).validateExtern(.element, zcu),3168 => ty.childType(zcu).validateExtern(.element, zcu),
3261 },3169 },
3262 .vector => ty.childType(zcu).validateExtern(.element, zcu),
3263 .optional => ty.isPtrLikeOptional(zcu),3170 .optional => ty.isPtrLikeOptional(zcu),
3264 };3171 };
3265}3172}
...@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {...@@ -3272,6 +3179,40 @@ fn validateExternCallconv(cc: std.lang.CallingConvention) bool {
3272 };3179 };
3273}3180}
32743181
3182/// Returns whether `ty` is considered by Zig to have a bit-level representation, meaning it is
3183/// allowed as the operand to `@bitSizeOf`. This is a superset of packable types.
3184pub fn hasBitRepresentation(ty: Type, zcu: *const Zcu) bool {
3185 return switch (ty.zigTypeTag(zcu)) {
3186 .@"fn",
3187 .noreturn,
3188 .undefined,
3189 .null,
3190 .@"opaque",
3191 .spirv,
3192 .type,
3193 .enum_literal,
3194 .comptime_float,
3195 .comptime_int,
3196 .error_set,
3197 .error_union,
3198 .frame,
3199 .@"anyframe",
3200 => false,
3201
3202 .void,
3203 .bool,
3204 .int,
3205 .float,
3206 => true,
3207
3208 .@"enum" => zcu.intern_pool.loadEnumType(ty.toIntern()).int_tag_mode == .explicit,
3209 .pointer, .optional => ty.isPtrAtRuntime(zcu),
3210 .@"struct", .@"union" => ty.containerLayout(zcu) == .@"packed",
3211
3212 .array, .vector => ty.childType(zcu).hasBitRepresentation(zcu),
3213 };
3214}
3215
3275/// Asserts that `ty` has resolved layout.3216/// Asserts that `ty` has resolved layout.
3276pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {3217pub fn assertHasLayout(ty: Type, zcu: *const Zcu) void {
3277 if (!std.debug.runtime_safety) {3218 if (!std.debug.runtime_safety) {
src/Value.zig+135-134
...@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {...@@ -248,7 +248,6 @@ pub fn toBool(val: Value) bool {
248pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{248pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
249 ReinterpretDeclRef,249 ReinterpretDeclRef,
250 IllDefinedMemoryLayout,250 IllDefinedMemoryLayout,
251 Unimplemented,
252 OutOfMemory,251 OutOfMemory,
253}!void {252}!void {
254 const target = zcu.getTarget();253 const target = zcu.getTarget();
...@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -257,35 +256,50 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
257 const ty = val.typeOf(zcu);256 const ty = val.typeOf(zcu);
258 if (val.isUndef(zcu)) {257 if (val.isUndef(zcu)) {
259 const size: usize = @intCast(ty.abiSize(zcu));258 const size: usize = @intCast(ty.abiSize(zcu));
260 @memset(buffer[0..size], 0xaa);259 @memset(buffer[0..size], 0xAA);
261 return;260 return;
262 }261 }
263 switch (ty.zigTypeTag(zcu)) {262 tag: switch (ty.zigTypeTag(zcu)) {
263 .type => return error.IllDefinedMemoryLayout,
264 .comptime_float => return error.IllDefinedMemoryLayout,
265 .comptime_int => return error.IllDefinedMemoryLayout,
266 .undefined => return error.IllDefinedMemoryLayout,
267 .null => return error.IllDefinedMemoryLayout,
268 .error_union => return error.IllDefinedMemoryLayout,
269 .enum_literal => return error.IllDefinedMemoryLayout,
270 .@"fn" => return error.IllDefinedMemoryLayout,
271 .spirv => return error.IllDefinedMemoryLayout,
272 .@"opaque" => unreachable,
273 .frame => unreachable,
274 .@"anyframe" => unreachable,
275 .noreturn => unreachable,
264 .void => {},276 .void => {},
265 .bool => {277 .bool => {
266 buffer[0] = @intFromBool(val.toBool());278 buffer[0] = @intFromBool(val.toBool());
267 },279 },
268 .int, .@"enum", .error_set, .pointer => |tag| {280 .pointer => {
269 const int_ty = if (tag == .pointer) int_ty: {281 if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;
270 if (ty.isSlice(zcu)) return error.IllDefinedMemoryLayout;282 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
271 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;283 continue :tag .int;
272 break :int_ty Type.usize;284 },
273 } else ty;285 .int, .@"enum", .error_set => {
274 const int_info = int_ty.intInfo(zcu);
275 const bits = int_info.bits;
276 const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);
277
278 var bigint_buffer: BigIntSpace = undefined;286 var bigint_buffer: BigIntSpace = undefined;
279 const bigint = val.toBigInt(&bigint_buffer, zcu);287 const bigint = val.toBigInt(&bigint_buffer, zcu);
280 bigint.writeTwosComplement(buffer[0..byte_count], endian);288 bigint.writeTwosComplement(buffer[0..@intCast(ty.abiSize(zcu))], endian);
281 },289 },
282 .float => switch (ty.floatBits(target)) {290 .float => {
283 16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),291 const float_bits = ty.floatBits(target);
284 32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),292 switch (float_bits) {
285 64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),293 16 => std.mem.writeInt(u16, buffer[0..2], @bitCast(val.toFloat(f16, zcu)), endian),
286 80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),294 32 => std.mem.writeInt(u32, buffer[0..4], @bitCast(val.toFloat(f32, zcu)), endian),
287 128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),295 64 => std.mem.writeInt(u64, buffer[0..8], @bitCast(val.toFloat(f64, zcu)), endian),
288 else => unreachable,296 80 => std.mem.writeInt(u80, buffer[0..10], @bitCast(val.toFloat(f80, zcu)), endian),
297 128 => std.mem.writeInt(u128, buffer[0..16], @bitCast(val.toFloat(f128, zcu)), endian),
298 else => unreachable,
299 }
300 const float_bytes = @divExact(float_bits, 8);
301 const total_bytes: usize = @intCast(ty.abiSize(zcu));
302 @memset(buffer[float_bytes..total_bytes], 0); // padding
289 },303 },
290 .array => {304 .array => {
291 const aggregate = ip.indexToKey(val.toIntern()).aggregate;305 const aggregate = ip.indexToKey(val.toIntern()).aggregate;
...@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -302,28 +316,33 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
302 }316 }
303 buf_off += elem_size;317 buf_off += elem_size;
304 }318 }
319 if (ty.sentinel(zcu)) |sentinel_val| {
320 try sentinel_val.writeToMemory(zcu, buffer[buf_off..]);
321 }
305 },322 },
306 .vector => {323 .vector => return error.IllDefinedMemoryLayout,
307 // We use byte_count instead of abi_size here, so that any padding bytes
308 // follow the data bytes, on both big- and little-endian systems.
309 const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8;
310 return writeToPackedMemory(val, zcu, buffer[0..byte_count], 0);
311 },
312 .@"struct" => {324 .@"struct" => {
313 const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;325 const struct_type = zcu.typeToStruct(ty) orelse return error.IllDefinedMemoryLayout;
314 switch (struct_type.layout) {326 switch (struct_type.layout) {
315 .auto => return error.IllDefinedMemoryLayout,327 .auto => return error.IllDefinedMemoryLayout,
316 .@"extern" => for (0..struct_type.field_types.len) |field_index| {328 .@"extern" => {
317 const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));329 var last_off: usize = 0;
318 const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {330 for (struct_type.field_types.get(ip), 0..) |field_ty_ip, field_index| {
319 .bytes => |bytes| {331 const off: usize = @intCast(ty.structFieldOffset(field_index, zcu));
320 buffer[off] = bytes.at(field_index, ip);332 @memset(buffer[last_off..off], 0xAA);
321 continue;333 const field_val = Value.fromInterned(switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
322 },334 .bytes => |bytes| {
323 .elems => |elems| elems[field_index],335 buffer[off] = bytes.at(field_index, ip);
324 .repeated_elem => |elem| elem,336 continue;
325 });337 },
326 try writeToMemory(field_val, zcu, buffer[off..]);338 .elems => |elems| elems[field_index],
339 .repeated_elem => |elem| elem,
340 });
341 try writeToMemory(field_val, zcu, buffer[off..]);
342 last_off = @intCast(off + Type.fromInterned(field_ty_ip).abiSize(zcu));
343 }
344 const struct_size: usize = @intCast(ty.abiSize(zcu));
345 @memset(buffer[last_off..struct_size], 0xAA);
327 },346 },
328 .@"packed" => {347 .@"packed" => {
329 const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;348 const int_index = ip.indexToKey(val.toIntern()).bitpack.backing_int_val;
...@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -335,6 +354,9 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
335 .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already354 .auto => return error.IllDefinedMemoryLayout, // Sema is supposed to have emitted a compile error already
336 .@"extern" => {355 .@"extern" => {
337 const payload_val = val.unionPayload(zcu);356 const payload_val = val.unionPayload(zcu);
357 const payload_size: usize = @intCast(payload_val.typeOf(zcu).abiSize(zcu));
358 const union_size: usize = @intCast(ty.abiSize(zcu));
359 @memset(buffer[payload_size..union_size], 0xAA);
338 return writeToMemory(payload_val, zcu, buffer);360 return writeToMemory(payload_val, zcu, buffer);
339 },361 },
340 .@"packed" => {362 .@"packed" => {
...@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -352,7 +374,6 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
352 @memset(buffer[0..@intCast(byte_count)], 0); // null pointer374 @memset(buffer[0..@intCast(byte_count)], 0); // null pointer
353 }375 }
354 },376 },
355 else => return error.Unimplemented,
356 }377 }
357}378}
358379
...@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{...@@ -360,12 +381,15 @@ pub fn writeToMemory(val: Value, zcu: *const Zcu, buffer: []u8) error{
360///381///
361/// Both the start and the end of the provided buffer must be tight, since382/// Both the start and the end of the provided buffer must be tight, since
362/// big-endian packed memory layouts start at the end of the buffer.383/// big-endian packed memory layouts start at the end of the buffer.
384///
385/// Supports arrays and vectors, for which the value is written in logical bit
386/// order, i.e. with the first element at bit offset 0.
363pub fn writeToPackedMemory(387pub fn writeToPackedMemory(
364 val: Value,388 val: Value,
365 zcu: *const Zcu,389 zcu: *const Zcu,
366 buffer: []u8,390 buffer: []u8,
367 bit_offset: usize,391 bit_offset: usize,
368) error{ ReinterpretDeclRef, OutOfMemory }!void {392) void {
369 const ip = &zcu.intern_pool;393 const ip = &zcu.intern_pool;
370 const target = zcu.getTarget();394 const target = zcu.getTarget();
371 const endian = target.cpu.arch.endian();395 const endian = target.cpu.arch.endian();
...@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(...@@ -392,13 +416,7 @@ pub fn writeToPackedMemory(
392 },416 },
393 .@"enum" => {417 .@"enum" => {
394 const int_val = val.intFromEnum(zcu);418 const int_val = val.intFromEnum(zcu);
395 return int_val.writeToPackedMemory(zcu, buffer, bit_offset);419 int_val.writeToPackedMemory(zcu, buffer, bit_offset);
396 },
397 .pointer => {
398 assert(!ty.isSlice(zcu)); // No well defined layout.
399 if (ip.getBackingAddrTag(val.toIntern()).? != .int) return error.ReinterpretDeclRef;
400 const addr = val.toUnsignedInt(zcu);
401 std.mem.writeVarPackedInt(buffer, bit_offset, zcu.getTarget().ptrBitWidth(), addr, endian);
402 },420 },
403 .int => {421 .int => {
404 const bits = ty.intInfo(zcu).bits;422 const bits = ty.intInfo(zcu).bits;
...@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(...@@ -416,47 +434,46 @@ pub fn writeToPackedMemory(
416 128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),434 128 => std.mem.writePackedInt(u128, buffer, bit_offset, @bitCast(val.toFloat(f128, zcu)), endian),
417 else => unreachable,435 else => unreachable,
418 },436 },
419 .vector => {
420 const elem_ty = ty.childType(zcu);
421 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
422 const len: usize = @intCast(ty.arrayLen(zcu));
423
424 var bits: u16 = 0;
425 var elem_i: usize = 0;
426 const aggregate = ip.indexToKey(val.toIntern()).aggregate;
427 while (elem_i < len) : (elem_i += 1) {
428 // On big-endian systems, LLVM reverses the element order of vectors by default
429 const tgt_elem_i = if (endian == .big) len - elem_i - 1 else elem_i;
430 switch (aggregate.storage) {
431 .bytes => |bytes| std.mem.writePackedInt(u8, buffer, bit_offset + bits, bytes.at(tgt_elem_i, ip), endian),
432 .elems => |elems| try Value.fromInterned(elems[tgt_elem_i]).writeToPackedMemory(zcu, buffer, bit_offset + bits),
433 .repeated_elem => |elem| try Value.fromInterned(elem).writeToPackedMemory(zcu, buffer, bit_offset + bits),
434 }
435 bits += elem_bit_size;
436 }
437 },
438 .@"struct", .@"union" => {437 .@"struct", .@"union" => {
439 assert(ty.containerLayout(zcu) == .@"packed");438 assert(ty.containerLayout(zcu) == .@"packed");
440 const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);439 const int_val: Value = .fromInterned(ip.indexToKey(val.toIntern()).bitpack.backing_int_val);
441 return int_val.writeToPackedMemory(zcu, buffer, bit_offset);440 int_val.writeToPackedMemory(zcu, buffer, bit_offset);
442 },441 },
443 .optional => {442 .array, .vector => {
444 assert(ty.isPtrLikeOptional(zcu));443 const elem_bits: usize = @intCast(ty.childType(zcu).bitSize(zcu));
445 if (val.optionalValue(zcu)) |ptr_val| {444 const len: usize = @intCast(ty.arrayLen(zcu));
446 return ptr_val.writeToPackedMemory(zcu, buffer, bit_offset);445 var elem_bit_off: usize = bit_offset;
447 } else {446 switch (ip.indexToKey(val.toIntern()).aggregate.storage) {
448 return Value.zero_usize.writeToPackedMemory(zcu, buffer, bit_offset);447 .repeated_elem => |elem_val_ip| {
448 const elem_val: Value = .fromInterned(elem_val_ip);
449 for (0..len) |_| {
450 elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
451 elem_bit_off += elem_bits;
452 }
453 },
454 .elems => |elems| for (elems[0..len]) |elem_val_ip| {
455 const elem_val: Value = .fromInterned(elem_val_ip);
456 elem_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
457 elem_bit_off += elem_bits;
458 },
459 .bytes => |bytes| for (bytes.toSlice(len, ip)) |raw_byte| {
460 std.mem.writeVarPackedInt(buffer, elem_bit_off, elem_bits, raw_byte, endian);
461 elem_bit_off += elem_bits;
462 },
463 }
464 if (ty.sentinel(zcu)) |sentinel_val| {
465 sentinel_val.writeToPackedMemory(zcu, buffer, elem_bit_off);
449 }466 }
450 },467 },
451 else => @panic("TODO implement writeToPackedMemory for more types"),468 else => unreachable,
452 }469 }
453}470}
454471
455/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type.472/// Load a Value from the contents of `buffer`, where `ty` is any integer type.
456///473///
457/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past474/// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past
458/// the end of the value in memory.475/// the end of the value in memory.
459pub fn readUintFromMemory(476pub fn readIntFromMemory(
460 ty: Type,477 ty: Type,
461 pt: Zcu.PerThread,478 pt: Zcu.PerThread,
462 buffer: []const u8,479 buffer: []const u8,
...@@ -465,23 +482,28 @@ pub fn readUintFromMemory(...@@ -465,23 +482,28 @@ pub fn readUintFromMemory(
465 const zcu = pt.zcu;482 const zcu = pt.zcu;
466 const endian = zcu.getTarget().cpu.arch.endian();483 const endian = zcu.getTarget().cpu.arch.endian();
467484
468 assert(ty.isUnsignedInt(zcu));485 const int = ty.intInfo(zcu);
469 const bits = ty.intInfo(zcu).bits;486 const abi_size: usize = @intCast(ty.abiSize(zcu));
470 const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8);487 const exact_buf = buffer[0..abi_size];
471488
472 assert(buffer.len >= byte_count);489 if (abi_size <= 8) {
473490 const shift: u6 = @intCast(64 - int.bits);
474 if (bits <= 64) {491 switch (int.signedness) {
475 const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian);492 .unsigned => {
476 const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits));493 const x = std.mem.readVarInt(u64, exact_buf, endian);
477 return pt.intValue(ty, result);494 return pt.intValue(ty, (x << shift) >> shift);
495 },
496 .signed => {
497 const x = std.mem.readVarInt(i64, exact_buf, endian);
498 return pt.intValue(ty, (x << shift) >> shift);
499 },
500 }
478 } else {501 } else {
479 const Limb = std.math.big.Limb;502 const limb_count = std.math.big.int.calcTwosCompLimbCount(int.bits);
480 const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb);503 const limbs_buffer = try arena.alloc(std.math.big.Limb, limb_count);
481 const limbs_buffer = try arena.alloc(Limb, limb_count);
482504
483 var bigint: BigIntMutable = .init(limbs_buffer, 0);505 var bigint: BigIntMutable = .init(limbs_buffer, 0);
484 bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned);506 bigint.readTwosComplement(exact_buf, int.bits, endian, int.signedness);
485 return pt.intValue_big(ty, bigint.toConst());507 return pt.intValue_big(ty, bigint.toConst());
486 }508 }
487}509}
...@@ -490,17 +512,17 @@ pub fn readUintFromMemory(...@@ -490,17 +512,17 @@ pub fn readUintFromMemory(
490///512///
491/// Both the start and the end of the provided buffer must be tight, since513/// Both the start and the end of the provided buffer must be tight, since
492/// big-endian packed memory layouts start at the end of the buffer.514/// big-endian packed memory layouts start at the end of the buffer.
515///
516/// Supports arrays and vectors, for which the value is read in logical bit
517/// order, i.e. with the first element at bit offset 0.
493pub fn readFromPackedMemory(518pub fn readFromPackedMemory(
494 ty: Type,519 ty: Type,
495 pt: Zcu.PerThread,520 pt: Zcu.PerThread,
496 buffer: []const u8,521 buffer: []const u8,
497 bit_offset: usize,522 bit_offset: usize,
498 gpa: Allocator,523) Allocator.Error!Value {
499) error{
500 IllDefinedMemoryLayout,
501 OutOfMemory,
502}!Value {
503 const zcu = pt.zcu;524 const zcu = pt.zcu;
525 const gpa = zcu.comp.gpa;
504 const target = zcu.getTarget();526 const target = zcu.getTarget();
505 const endian = target.cpu.arch.endian();527 const endian = target.cpu.arch.endian();
506 switch (ty.zigTypeTag(zcu)) {528 switch (ty.zigTypeTag(zcu)) {
...@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(...@@ -543,7 +565,7 @@ pub fn readFromPackedMemory(
543 },565 },
544 .@"enum" => {566 .@"enum" => {
545 const int_ty = ty.intTagType(zcu);567 const int_ty = ty.intTagType(zcu);
546 const int_val = try Value.readFromPackedMemory(int_ty, pt, buffer, bit_offset, gpa);568 const int_val: Value = try .readFromPackedMemory(int_ty, pt, buffer, bit_offset);
547 return pt.getCoerced(int_val, ty);569 return pt.getCoerced(int_val, ty);
548 },570 },
549 .float => return Value.fromInterned(try pt.intern(.{ .float = .{571 .float => return Value.fromInterned(try pt.intern(.{ .float = .{
...@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(...@@ -557,40 +579,25 @@ pub fn readFromPackedMemory(
557 else => unreachable,579 else => unreachable,
558 },580 },
559 } })),581 } })),
560 .vector => {
561 const elem_ty = ty.childType(zcu);
562 const elems = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
563 defer gpa.free(elems);
564
565 var bits: u16 = 0;
566 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
567 for (elems, 0..) |_, i| {
568 // On big-endian systems, LLVM reverses the element order of vectors by default
569 const tgt_elem_i = if (endian == .big) elems.len - i - 1 else i;
570 elems[tgt_elem_i] = (try readFromPackedMemory(elem_ty, pt, buffer, bit_offset + bits, gpa)).toIntern();
571 bits += elem_bit_size;
572 }
573 return pt.aggregateValue(ty, elems);
574 },
575 .@"struct", .@"union" => {582 .@"struct", .@"union" => {
576 assert(ty.containerLayout(zcu) == .@"packed");583 assert(ty.containerLayout(zcu) == .@"packed");
577 const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset, gpa);584 const int_val: Value = try .readFromPackedMemory(ty.bitpackBackingInt(zcu), pt, buffer, bit_offset);
578 return pt.bitpackValue(ty, int_val);585 return pt.bitpackValue(ty, int_val);
579 },586 },
580 .pointer => {587 .array, .vector => {
581 assert(!ty.isSlice(zcu)); // No well defined layout.588 const elem_ty = ty.childType(zcu);
582 const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);589 const elem_bits: usize = @intCast(elem_ty.bitSize(zcu));
583 return pt.ptrIntValue(ty, addr);590 const elems_buf = try gpa.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu)));
584 },591 defer gpa.free(elems_buf);
585 .optional => {592 var elem_bit_off: usize = bit_offset;
586 assert(ty.isPtrLikeOptional(zcu));593 for (elems_buf) |*elem| {
587 const addr = (try readFromPackedMemory(Type.usize, pt, buffer, bit_offset, gpa)).toUnsignedInt(zcu);594 const elem_val = try readFromPackedMemory(elem_ty, pt, buffer, elem_bit_off);
588 return .fromInterned(try pt.intern(.{ .opt = .{595 elem.* = elem_val.toIntern();
589 .ty = ty.toIntern(),596 elem_bit_off += elem_bits;
590 .val = if (addr == 0) .none else (try pt.ptrIntValue(ty.childType(zcu), addr)).toIntern(),597 }
591 } }));598 return pt.aggregateValue(ty, elems_buf);
592 },599 },
593 else => @panic("TODO implement readFromPackedMemory for more types"),600 else => unreachable,
594 }601 }
595}602}
596603
...@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {...@@ -887,14 +894,9 @@ pub fn fieldValue(val: Value, pt: Zcu.PerThread, index: usize) !Value {
887 const bfa = bfa_state.allocator();894 const bfa = bfa_state.allocator();
888 const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));895 const buf = try bfa.alloc(u8, @intCast((ty.bitSize(zcu) + 7) / 8));
889 defer bfa.free(buf);896 defer bfa.free(buf);
890 int_val.writeToPackedMemory(zcu, buf, 0) catch |err| switch (err) {897 @memset(buf, 0);
891 error.ReinterpretDeclRef => unreachable, // it's an integer898 int_val.writeToPackedMemory(zcu, buf, 0);
892 error.OutOfMemory => |e| return e,899 return .readFromPackedMemory(field_ty, pt, buf, field_bit_offset);
893 };
894 return Value.readFromPackedMemory(field_ty, pt, buf, field_bit_offset, bfa) catch |err| switch (err) {
895 error.IllDefinedMemoryLayout => unreachable, // it's a bitpack
896 error.OutOfMemory => |e| return e,
897 };
898 },900 },
899 else => unreachable,901 else => unreachable,
900 };902 };
...@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {...@@ -1619,7 +1621,6 @@ pub fn hasRepeatedByteRepr(val: Value, zcu: *const Zcu) !?u8 {
1619 // code late in compilation. So, this error handling is too aggressive and1621 // code late in compilation. So, this error handling is too aggressive and
1620 // causes some false negatives, causing less-than-ideal code generation.1622 // causes some false negatives, causing less-than-ideal code generation.
1621 error.IllDefinedMemoryLayout => return null,1623 error.IllDefinedMemoryLayout => return null,
1622 error.Unimplemented => return null,
1623 };1624 };
1624 const first_byte = byte_buffer[0];1625 const first_byte = byte_buffer[0];
1625 for (byte_buffer[1..]) |byte| {1626 for (byte_buffer[1..]) |byte| {
src/Zcu/PerThread.zig+4
...@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e...@@ -4544,8 +4544,12 @@ fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) e
4544 tracy_trace.addText(fqn.toSlice(ip));4544 tracy_trace.addText(fqn.toSlice(ip));
4545 tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});4545 tracy_trace.addTextFmt("func_ip_index={d}", .{func_index});
45464546
4547 Air.Verify.run(pt, func_index, air);
4548
4547 if (codegen.legalizeFeatures(pt, nav)) |features| {4549 if (codegen.legalizeFeatures(pt, nav)) |features| {
4548 try air.legalize(pt, features);4550 try air.legalize(pt, features);
4551 // Verify the AIR again post-legalization.
4552 Air.Verify.run(pt, func_index, air);
4549 }4553 }
45504554
4551 var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))4555 var liveness: ?Air.Liveness = if (codegen.wantsLiveness(pt, nav))
src/codegen/aarch64/Select.zig+23-7
...@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -292,8 +292,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
292 .load,292 .load,
293 .fptrunc,293 .fptrunc,
294 .fpext,294 .fpext,
295 .intcast,295 .int_cast,
296 .intcast_safe,296 .int_cast_safe,
297 .trunc,297 .trunc,
298 .optional_payload,298 .optional_payload,
299 .optional_payload_ptr,299 .optional_payload_ptr,
...@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -334,7 +334,15 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
334 air_inst_index = air_body[air_body_index];334 air_inst_index = air_body[air_body_index];
335 continue :air_tag air_tags[@intFromEnum(air_inst_index)];335 continue :air_tag air_tags[@intFromEnum(air_inst_index)];
336 },336 },
337 .bitcast => {337 .bit_cast,
338 .ptr_cast,
339 .ptr_from_int,
340 .int_from_ptr,
341 .error_cast,
342 .error_from_int,
343 .int_from_error,
344 .union_from_enum,
345 => {
338 const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;346 const ty_op = air_data[@intFromEnum(air_inst_index)].ty_op;
339 maybe_noop: {347 maybe_noop: {
340 if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;348 if (ty_op.ty.toInterned().? != isel.air.typeOf(ty_op.operand, ip).toIntern()) break :maybe_noop;
...@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -3190,7 +3198,15 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
3190 }3198 }
3191 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;3199 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
3192 },3200 },
3193 .bitcast => |air_tag| {3201 .bit_cast,
3202 .ptr_cast,
3203 .ptr_from_int,
3204 .int_from_ptr,
3205 .error_cast,
3206 .error_from_int,
3207 .int_from_error,
3208 .union_from_enum,
3209 => |air_tag| {
3194 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {3210 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
3195 defer dst_vi.value.deref(isel);3211 defer dst_vi.value.deref(isel);
3196 const ty_op = air.data(air.inst_index).ty_op;3212 const ty_op = air.data(air.inst_index).ty_op;
...@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -5221,7 +5237,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5221 }5237 }
5222 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;5238 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5223 },5239 },
5224 .intcast => |air_tag| {5240 .int_cast => |air_tag| {
5225 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {5241 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5226 defer dst_vi.value.deref(isel);5242 defer dst_vi.value.deref(isel);
52275243
...@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,...@@ -5312,7 +5328,7 @@ pub fn body(isel: *Select, air_body: []const Air.Inst.Index) error{ OutOfMemory,
5312 }5328 }
5313 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;5329 if (air.next()) |next_air_tag| continue :air_tag next_air_tag;
5314 },5330 },
5315 .intcast_safe => |air_tag| {5331 .int_cast_safe => |air_tag| {
5316 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {5332 if (isel.live_values.fetchRemove(air.inst_index)) |dst_vi| unused: {
5317 defer dst_vi.value.deref(isel);5333 defer dst_vi.value.deref(isel);
53185334
...@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem...@@ -11355,7 +11371,7 @@ fn writeToMemory(isel: *Select, constant: Constant, buffer: []u8) error{OutOfMem
11355 if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;11371 if (try isel.writeKeyToMemory(ip.indexToKey(constant.toIntern()), buffer)) return true;
11356 constant.writeToMemory(zcu, buffer) catch |err| switch (err) {11372 constant.writeToMemory(zcu, buffer) catch |err| switch (err) {
11357 error.OutOfMemory => |e| return e,11373 error.OutOfMemory => |e| return e,
11358 error.ReinterpretDeclRef, error.Unimplemented, error.IllDefinedMemoryLayout => return false,11374 error.ReinterpretDeclRef, error.IllDefinedMemoryLayout => return false,
11359 };11375 };
11360 return true;11376 return true;
11361}11377}
src/codegen/aarch64/abi.zig+2-2
...@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -21,7 +21,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
21 if (ty.containerLayout(zcu) == .@"packed") return .byval;21 if (ty.containerLayout(zcu) == .@"packed") return .byval;
22 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };22 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
2323
24 const bit_size = ty.bitSize(zcu);24 const bit_size = ty.abiSize(zcu) * 8;
25 if (bit_size > 128) return .memory;25 if (bit_size > 128) return .memory;
26 if (bit_size > 64) return .double_integer;26 if (bit_size > 64) return .double_integer;
27 return .integer;27 return .integer;
...@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -30,7 +30,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
30 if (ty.containerLayout(zcu) == .@"packed") return .byval;30 if (ty.containerLayout(zcu) == .@"packed") return .byval;
31 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };31 if (countFloats(ty, zcu)) |float| return .{ .float_array = float.count };
3232
33 const bit_size = ty.bitSize(zcu);33 const bit_size = ty.abiSize(zcu) * 8;
34 if (bit_size > 128) return .memory;34 if (bit_size > 128) return .memory;
35 if (bit_size > 64) return .double_integer;35 if (bit_size > 64) return .double_integer;
36 return .integer;36 return .integer;
src/codegen/arm/abi.zig+6-6
...@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -30,11 +30,11 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
30 const ip = &zcu.intern_pool;30 const ip = &zcu.intern_pool;
31 switch (ty.zigTypeTag(zcu)) {31 switch (ty.zigTypeTag(zcu)) {
32 .@"struct" => {32 .@"struct" => {
33 const bit_size = ty.bitSize(zcu);
34 if (ty.containerLayout(zcu) == .@"packed") {33 if (ty.containerLayout(zcu) == .@"packed") {
35 if (bit_size > 64) return .memory;34 if (ty.bitSize(zcu) > 64) return .memory;
36 return .byval;35 return .byval;
37 }36 }
37 const bit_size = ty.abiSize(zcu) * 8;
38 if (bit_size > max_byval_size) return .memory;38 if (bit_size > max_byval_size) return .memory;
39 const float_count = countFloats(ty, zcu, &maybe_float_bits);39 const float_count = countFloats(ty, zcu, &maybe_float_bits);
40 if (float_count <= byval_float_count) return .byval;40 if (float_count <= byval_float_count) return .byval;
...@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -47,17 +47,17 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
47 var i: u32 = 0;47 var i: u32 = 0;
48 while (i < fields) : (i += 1) {48 while (i < fields) : (i += 1) {
49 const field_ty = ty.fieldType(i, zcu);49 const field_ty = ty.fieldType(i, zcu);
50 if (field_ty.bitSize(zcu) > 32) return Class.arrSize(bit_size, 64);50 if (field_ty.abiSize(zcu) > 4) return Class.arrSize(bit_size, 64);
51 }51 }
52 return Class.arrSize(bit_size, 32);52 return Class.arrSize(bit_size, 32);
53 },53 },
54 .@"union" => {54 .@"union" => {
55 const bit_size = ty.bitSize(zcu);
56 const union_obj = zcu.typeToUnion(ty).?;55 const union_obj = zcu.typeToUnion(ty).?;
57 if (union_obj.layout == .@"packed") {56 if (union_obj.layout == .@"packed") {
58 if (bit_size > 64) return .memory;57 if (ty.bitSize(zcu) > 64) return .memory;
59 return .byval;58 return .byval;
60 }59 }
60 const bit_size = ty.abiSize(zcu) * 8;
61 if (bit_size > max_byval_size) return .memory;61 if (bit_size > max_byval_size) return .memory;
62 const float_count = countFloats(ty, zcu, &maybe_float_bits);62 const float_count = countFloats(ty, zcu, &maybe_float_bits);
63 if (float_count <= byval_float_count) return .byval;63 if (float_count <= byval_float_count) return .byval;
...@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -67,7 +67,7 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
67 }67 }
6868
69 for (union_obj.field_types.get(ip)) |field_ty| {69 for (union_obj.field_types.get(ip)) |field_ty| {
70 if (Type.fromInterned(field_ty).bitSize(zcu) > 32) {70 if (Type.fromInterned(field_ty).abiSize(zcu) > 4) {
71 return Class.arrSize(bit_size, 64);71 return Class.arrSize(bit_size, 64);
72 }72 }
73 }73 }
src/codegen/c.zig+270-104
...@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -27,7 +27,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
27 return comptime switch (dev.env.supports(.legalize)) {27 return comptime switch (dev.env.supports(.legalize)) {
28 inline false, true => |supports_legalize| &.init(.{28 inline false, true => |supports_legalize| &.init(.{
29 // we don't currently ask zig1 to use safe optimization modes29 // we don't currently ask zig1 to use safe optimization modes
30 .expand_intcast_safe = supports_legalize,30 .expand_int_cast_safe = supports_legalize,
31 .expand_int_from_float_safe = supports_legalize,31 .expand_int_from_float_safe = supports_legalize,
32 .expand_int_from_float_optimized_safe = supports_legalize,32 .expand_int_from_float_optimized_safe = supports_legalize,
33 .expand_add_safe = supports_legalize,33 .expand_add_safe = supports_legalize,
...@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -38,6 +38,9 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
38 .expand_packed_store = true,38 .expand_packed_store = true,
39 .expand_packed_struct_field_val = true,39 .expand_packed_struct_field_val = true,
40 .expand_packed_aggregate_init = true,40 .expand_packed_aggregate_init = true,
41
42 .scalarize_bit_cast_array = true,
43 .scalarize_bit_cast_vector_non_elementwise = true,
41 }),44 }),
42 };45 };
43}46}
...@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2636,9 +2639,9 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2636 // zig fmt: off2639 // zig fmt: off
2637 .inferred_alloc, .inferred_alloc_comptime => unreachable,2640 .inferred_alloc, .inferred_alloc_comptime => unreachable,
26382641
2639 // No "scalarize" legalizations are enabled, so these instructions never appear.2642 // Possible because `Air.Legalize.scalarize_bit_cast_vector_non_elementwise` is enabled.
2640 .legalize_vec_elem_val => unreachable,2643 .legalize_vec_elem_val => try airArrayElemVal(f, inst),
2641 .legalize_vec_store_elem => unreachable,2644 .legalize_vec_store_elem => try airLegalizeVecStoreElem(f, inst),
2642 // No soft float legalizations are enabled.2645 // No soft float legalizations are enabled.
2643 .legalize_compiler_rt_call => unreachable,2646 .legalize_compiler_rt_call => unreachable,
26442647
...@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2751,8 +2754,15 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2751 .alloc => try airAlloc(f, inst),2754 .alloc => try airAlloc(f, inst),
2752 .ret_ptr => try airRetPtr(f, inst),2755 .ret_ptr => try airRetPtr(f, inst),
2753 .assembly => try airAsm(f, inst),2756 .assembly => try airAsm(f, inst),
2754 .bitcast => try airBitcast(f, inst),2757 .ptr_cast => try airPtrCast(f, inst),
2755 .intcast => try airIntCast(f, inst),2758 .ptr_from_int => try airSimpleCast(f, inst),
2759 .int_from_ptr => try airSimpleCast(f, inst),
2760 .error_cast => try airNopCast(f, inst),
2761 .error_from_int => try airNopCast(f, inst),
2762 .int_from_error => try airNopCast(f, inst),
2763 .union_from_enum => try airUnionFromEnum(f, inst),
2764 .bit_cast => try airBitCast(f, inst),
2765 .int_cast => try airIntCast(f, inst),
2756 .trunc => try airTrunc(f, inst),2766 .trunc => try airTrunc(f, inst),
2757 .load => try airLoad(f, inst),2767 .load => try airLoad(f, inst),
2758 .store => try airStore(f, inst, false),2768 .store => try airStore(f, inst, false),
...@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -2864,7 +2874,7 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
2864 .add_safe,2874 .add_safe,
2865 .sub_safe,2875 .sub_safe,
2866 .mul_safe,2876 .mul_safe,
2867 .intcast_safe,2877 .int_cast_safe,
2868 .int_from_float_safe,2878 .int_from_float_safe,
2869 .int_from_float_optimized_safe,2879 .int_from_float_optimized_safe,
2870 => return f.fail("TODO implement safety_checked_instructions", .{}),2880 => return f.fail("TODO implement safety_checked_instructions", .{}),
...@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3083,6 +3093,28 @@ fn airArrayElemVal(f: *Function, inst: Air.Inst.Index) !CValue {
3083 return local;3093 return local;
3084}3094}
30853095
3096fn airLegalizeVecStoreElem(f: *Function, inst: Air.Inst.Index) !CValue {
3097 const pl_op = f.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3098 const extra = f.air.extraData(Air.Bin, pl_op.payload).data;
3099
3100 const vec_ptr = try f.resolveInst(pl_op.operand);
3101 const index = try f.resolveInst(extra.lhs);
3102 const elem = try f.resolveInst(extra.rhs);
3103 try reap(f, inst, &.{ pl_op.operand, extra.lhs, extra.rhs });
3104
3105 const w = &f.code.writer;
3106
3107 try f.writeCValueDerefMember(w, vec_ptr, .{ .identifier = "array" });
3108 try w.writeByte('[');
3109 try f.writeCValue(w, index, .other);
3110 try w.writeAll("] = ");
3111 try f.writeCValue(w, elem, .other);
3112 try w.writeByte(';');
3113 try f.newline();
3114
3115 return .none;
3116}
3117
3086fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {3118fn airAlloc(f: *Function, inst: Air.Inst.Index) !CValue {
3087 const pt = f.dg.pt;3119 const pt = f.dg.pt;
3088 const zcu = pt.zcu;3120 const zcu = pt.zcu;
...@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {...@@ -3190,35 +3222,42 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue {
31903222
3191 try reap(f, inst, &.{ty_op.operand});3223 try reap(f, inst, &.{ty_op.operand});
31923224
3193 const is_aligned = if (ptr_info.flags.alignment != .none)3225 const is_aligned = switch (ptr_info.flags.alignment) {
3194 ptr_info.flags.alignment.order(src_ty.abiAlignment(zcu)).compare(.gte)3226 .none => true,
3195 else3227 else => |ptr_align| ptr_align.compare(.gte, src_ty.abiAlignment(zcu)),
3196 true;3228 };
31973229
3198 const w = &f.code.writer;3230 const w = &f.code.writer;
3199 const local = try f.allocLocal(inst, src_ty);3231 const local = try f.allocLocal(inst, src_ty);
3200 const v = try Vectorize.start(f, inst, w, ptr_ty);
32013232
3202 if (!is_aligned) {3233 if (!is_aligned) {
3203 try w.writeAll("memcpy(&");3234 try w.writeAll("memcpy(&");
3204 try f.writeCValue(w, local, .other);3235 try f.writeCValue(w, local, .other);
3205 try v.elem(f, w);
3206 try w.writeAll(", (const char *)");3236 try w.writeAll(", (const char *)");
3207 try f.writeCValue(w, operand, .other);3237 switch (ptr_info.flags.vector_index) {
3208 try v.elem(f, w);3238 .none => try f.writeCValue(w, operand, .other),
3239 else => |index| {
3240 try w.writeByte('&');
3241 try f.writeCValue(w, operand, .other);
3242 try w.print("[{d}]", .{@intFromEnum(index)});
3243 },
3244 }
3209 try w.writeAll(", sizeof(");3245 try w.writeAll(", sizeof(");
3210 try f.renderType(w, src_ty);3246 try f.renderType(w, src_ty);
3211 try w.writeAll("))");3247 try w.writeAll("))");
3212 } else {3248 } else {
3213 try f.writeCValue(w, local, .other);3249 try f.writeCValue(w, local, .other);
3214 try v.elem(f, w);
3215 try w.writeAll(" = ");3250 try w.writeAll(" = ");
3216 try f.writeCValueDeref(w, operand);3251 switch (ptr_info.flags.vector_index) {
3217 try v.elem(f, w);3252 .none => try f.writeCValueDeref(w, operand),
3253 else => |index| {
3254 try f.writeCValue(w, operand, .other);
3255 try w.print("[{d}]", .{@intFromEnum(index)});
3256 },
3257 }
3218 }3258 }
3219 try w.writeByte(';');3259 try w.writeByte(';');
3220 try f.newline();3260 try f.newline();
3221 try v.end(f, inst, w);
32223261
3223 return local;3262 return local;
3224}3263}
...@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -3433,21 +3472,24 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3433 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).3472 // underlying type as the lhs (i.e. they must both be arrays of the same underlying type).
3434 assert(src_ty.eql(.fromInterned(ptr_info.child)));3473 assert(src_ty.eql(.fromInterned(ptr_info.child)));
34353474
3436 const v = try Vectorize.start(f, inst, w, ptr_ty);
3437 try w.writeAll("memcpy((char *)");3475 try w.writeAll("memcpy((char *)");
3438 try f.writeCValue(w, ptr_val, .other);3476 switch (ptr_info.flags.vector_index) {
3439 try v.elem(f, w);3477 .none => try f.writeCValue(w, ptr_val, .other),
3478 else => |index| {
3479 try w.writeByte('&');
3480 try f.writeCValue(w, ptr_val, .other);
3481 try w.print("[{d}]", .{@intFromEnum(index)});
3482 },
3483 }
3440 try w.writeAll(", &");3484 try w.writeAll(", &");
3441 switch (src_val) {3485 switch (src_val) {
3442 .constant => |val| try f.dg.renderValueAsLvalue(w, val),3486 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
3443 else => try f.writeCValue(w, src_val, .other),3487 else => try f.writeCValue(w, src_val, .other),
3444 }3488 }
3445 try v.elem(f, w);
3446 try w.writeAll(", sizeof(");3489 try w.writeAll(", sizeof(");
3447 try f.renderType(w, src_ty);3490 try f.renderType(w, src_ty);
3448 try w.writeAll("));");3491 try w.writeAll("));");
3449 try f.newline();3492 try f.newline();
3450 try v.end(f, inst, w);
3451 } else {3493 } else {
3452 switch (ptr_val) {3494 switch (ptr_val) {
3453 .local_ref => |ptr_local_index| switch (src_val) {3495 .local_ref => |ptr_local_index| switch (src_val) {
...@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -3457,15 +3499,18 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
3457 },3499 },
3458 else => {},3500 else => {},
3459 }3501 }
3460 const v = try Vectorize.start(f, inst, w, ptr_ty);3502
3461 try f.writeCValueDeref(w, ptr_val);3503 switch (ptr_info.flags.vector_index) {
3462 try v.elem(f, w);3504 .none => try f.writeCValueDeref(w, ptr_val),
3505 else => |index| {
3506 try f.writeCValue(w, ptr_val, .other);
3507 try w.print("[{d}]", .{@intFromEnum(index)});
3508 },
3509 }
3463 try w.writeAll(" = ");3510 try w.writeAll(" = ");
3464 try f.writeCValue(w, src_val, .other);3511 try f.writeCValue(w, src_val, .other);
3465 try v.elem(f, w);
3466 try w.writeByte(';');3512 try w.writeByte(';');
3467 try f.newline();3513 try f.newline();
3468 try v.end(f, inst, w);
3469 }3514 }
3470 return .none;3515 return .none;
3471}3516}
...@@ -3613,9 +3658,9 @@ fn airCmpOp(...@@ -3613,9 +3658,9 @@ fn airCmpOp(
3613 const lhs_ty = f.typeOf(data.lhs);3658 const lhs_ty = f.typeOf(data.lhs);
3614 const scalar_ty = lhs_ty.scalarType(zcu);3659 const scalar_ty = lhs_ty.scalarType(zcu);
36153660
3616 const scalar_bits = scalar_ty.bitSize(zcu);3661 if (scalar_ty.isInt(zcu)) {
3617 if (scalar_ty.isInt(zcu) and scalar_bits > 64)3662 const scalar_bits = scalar_ty.bitSize(zcu);
3618 return airCmpBuiltinCall(3663 if (scalar_bits > 64) return airCmpBuiltinCall(
3619 f,3664 f,
3620 inst,3665 inst,
3621 data,3666 data,
...@@ -3623,6 +3668,7 @@ fn airCmpOp(...@@ -3623,6 +3668,7 @@ fn airCmpOp(
3623 .cmp,3668 .cmp,
3624 if (scalar_bits > 128) .bits else .none,3669 if (scalar_bits > 128) .bits else .none,
3625 );3670 );
3671 }
3626 if (scalar_ty.isRuntimeFloat())3672 if (scalar_ty.isRuntimeFloat())
3627 return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);3673 return airCmpBuiltinCall(f, inst, data, operator, .operator, .none);
36283674
...@@ -3668,9 +3714,9 @@ fn airEquality(...@@ -3668,9 +3714,9 @@ fn airEquality(
3668 const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;3714 const bin_op = f.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
36693715
3670 const operand_ty = f.typeOf(bin_op.lhs);3716 const operand_ty = f.typeOf(bin_op.lhs);
3671 const operand_bits = operand_ty.bitSize(zcu);3717 if (operand_ty.isAbiInt(zcu)) {
3672 if (operand_ty.isAbiInt(zcu) and operand_bits > 64)3718 const operand_bits = operand_ty.bitSize(zcu);
3673 return airCmpBuiltinCall(3719 if (operand_bits > 64) return airCmpBuiltinCall(
3674 f,3720 f,
3675 inst,3721 inst,
3676 bin_op,3722 bin_op,
...@@ -3678,6 +3724,7 @@ fn airEquality(...@@ -3678,6 +3724,7 @@ fn airEquality(
3678 .cmp,3724 .cmp,
3679 if (operand_bits > 128) .bits else .none,3725 if (operand_bits > 128) .bits else .none,
3680 );3726 );
3727 }
3681 if (operand_ty.isRuntimeFloat())3728 if (operand_ty.isRuntimeFloat())
3682 return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);3729 return airCmpBuiltinCall(f, inst, bin_op, operator, .operator, .none);
36833730
...@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {...@@ -4258,125 +4305,240 @@ fn airSwitchDispatch(f: *Function, inst: Air.Inst.Index) !void {
4258 try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});4305 try w.print("goto zig_switch_{d}_loop;\n", .{@intFromEnum(br.block_inst)});
4259}4306}
42604307
4261fn airBitcast(f: *Function, inst: Air.Inst.Index) !CValue {4308fn airPtrCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4309 const zcu = f.dg.pt.zcu;
4310
4311 const dest_ty = f.typeOfIndex(inst);
4312 const ptr_ty = switch (dest_ty.zigTypeTag(zcu)) {
4313 .optional => dest_ty.childType(zcu),
4314 .pointer => dest_ty,
4315 else => unreachable,
4316 };
4317
4318 if (!ptr_ty.isSlice(zcu)) {
4319 return airSimpleCast(f, inst);
4320 }
4321
4322 // For slice casts we need to assign both fields.
4323
4262 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4324 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4263 const inst_ty = f.typeOfIndex(inst);4325 const operand = try f.resolveInst(ty_op.operand);
4326
4327 const w = &f.code.writer;
4328 const dest_local = try f.allocLocal(inst, dest_ty);
4329
4330 try f.writeCValueMember(w, dest_local, .{ .identifier = "ptr" });
4331 try w.writeAll(" = (");
4332 try f.renderType(w, ptr_ty.slicePtrFieldType(zcu));
4333 try w.writeByte(')');
4334 try f.writeCValueMember(w, operand, .{ .identifier = "ptr" });
4335 try w.writeByte(';');
4336 try f.newline();
4337
4338 try f.writeCValueMember(w, dest_local, .{ .identifier = "len" });
4339 try w.writeAll(" = ");
4340 try f.writeCValueMember(w, operand, .{ .identifier = "len" });
4341 try w.writeByte(';');
4342 try f.newline();
4343
4344 try reap(f, inst, &.{ty_op.operand});
4345 return dest_local;
4346}
4347
4348fn airSimpleCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4349 const zcu = f.dg.pt.zcu;
4350
4351 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4352 const dest_ty = f.typeOfIndex(inst);
4353 const operand_ty = f.typeOf(ty_op.operand);
4354 const operand = try f.resolveInst(ty_op.operand);
4355
4356 const w = &f.code.writer;
4357 const dest_local = try f.allocLocal(inst, dest_ty);
4358 const v: Vectorize = try .start(f, inst, w, operand_ty);
4359 try f.writeCValue(w, dest_local, .other);
4360 try v.elem(f, w);
4361 try w.writeAll(" = (");
4362 try f.renderType(w, dest_ty.scalarType(zcu));
4363 try w.writeByte(')');
4364 try f.writeCValue(w, operand, .other);
4365 try v.elem(f, w);
4366 try w.writeByte(';');
4367 try f.newline();
4368 try v.end(f, inst, w);
4369
4370 try reap(f, inst, &.{ty_op.operand});
4371 return dest_local;
4372}
42644373
4374fn airNopCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4375 const zcu = f.dg.pt.zcu;
4376
4377 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4378 const dest_ty = f.typeOfIndex(inst);
4379 const operand_ty = f.typeOf(ty_op.operand);
4265 const operand = try f.resolveInst(ty_op.operand);4380 const operand = try f.resolveInst(ty_op.operand);
4381
4382 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
4383 assert(operand_ty.isAbiInt(zcu) == dest_ty.isAbiInt(zcu));
4384
4385 try reap(f, inst, &.{ty_op.operand});
4386 return f.moveCValue(inst, dest_ty, operand);
4387}
4388
4389fn airUnionFromEnum(f: *Function, inst: Air.Inst.Index) Error!CValue {
4390 const zcu = f.dg.pt.zcu;
4391
4392 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4393 const dest_ty = f.typeOfIndex(inst);
4266 const operand_ty = f.typeOf(ty_op.operand);4394 const operand_ty = f.typeOf(ty_op.operand);
4395 const operand = try f.resolveInst(ty_op.operand);
4396
4397 assert(dest_ty.zigTypeTag(zcu) == .@"union");
4398 assert(operand_ty.zigTypeTag(zcu) == .@"enum");
4399
4400 const w = &f.code.writer;
4401 const dest_local = try f.allocLocal(inst, dest_ty);
4402 try f.writeCValueMember(w, dest_local, .{ .identifier = "tag" });
4403 try w.writeAll(" = ");
4404 try f.writeCValue(w, operand, .other);
4405 try w.writeByte(';');
4406 try f.newline();
42674407
4268 const bitcasted = try bitcast(f, inst_ty, operand, operand_ty);
4269 try reap(f, inst, &.{ty_op.operand});4408 try reap(f, inst, &.{ty_op.operand});
4270 return f.moveCValue(inst, inst_ty, bitcasted);4409 return dest_local;
4271}4410}
42724411
4273fn bitcast(f: *Function, dest_ty: Type, operand: CValue, operand_ty: Type) !CValue {4412fn airBitCast(f: *Function, inst: Air.Inst.Index) Error!CValue {
4274 const pt = f.dg.pt;4413 const pt = f.dg.pt;
4275 const zcu = pt.zcu;4414 const zcu = pt.zcu;
4276 const target = &f.dg.mod.resolved_target.result;
4277 const w = &f.code.writer;4415 const w = &f.code.writer;
42784416
4279 if (operand_ty.isAbiInt(zcu) and dest_ty.isAbiInt(zcu)) {4417 const ty_op = f.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4280 const src_info = dest_ty.intInfo(zcu);4418 const dest_ty = f.typeOfIndex(inst);
4281 const dest_info = operand_ty.intInfo(zcu);
4282 if (src_info.signedness == dest_info.signedness and
4283 src_info.bits == dest_info.bits) return operand;
4284 }
42854419
4286 if (dest_ty.isPtrAtRuntime(zcu) or operand_ty.isPtrAtRuntime(zcu)) {4420 const operand = try f.resolveInst(ty_op.operand);
4287 const local = try f.allocLocal(null, dest_ty);4421 const operand_ty = f.typeOf(ty_op.operand);
4288 try f.writeCValue(w, local, .other);4422
4423 const dest_local = try f.allocLocal(inst, dest_ty);
4424
4425 // Because we have `scalarize_bit_cast_array` and `scalarize_bit_cast_vector_non_elementwise`
4426 // enabled, we usually only see scalars here. The only case in which we may see vectors is when
4427 // the operation happens elementwise, which we can handle with `Vectorize`.
4428 var v: Vectorize = try .start(f, inst, w, operand_ty);
4429 const operand_scalar_ty = operand_ty.scalarType(zcu);
4430 const dest_scalar_ty = dest_ty.scalarType(zcu);
4431
4432 // Some cases are handled with a simple cast:
4433 // * float -> float
4434 // * bool -> int
4435 if ((operand_scalar_ty.isRuntimeFloat() and dest_scalar_ty.isRuntimeFloat()) or
4436 (operand_scalar_ty.toIntern() == .bool_type and dest_scalar_ty.isAbiInt(zcu)))
4437 {
4438 try f.writeCValue(w, dest_local, .other);
4439 try v.elem(f, w);
4289 try w.writeAll(" = (");4440 try w.writeAll(" = (");
4290 try f.renderType(w, dest_ty);4441 try f.renderType(w, dest_scalar_ty);
4291 try w.writeByte(')');4442 try w.writeByte(')');
4292 try f.writeCValue(w, operand, .other);4443 try f.writeCValue(w, operand, .other);
4444 try v.elem(f, w);
4293 try w.writeByte(';');4445 try w.writeByte(';');
4294 try f.newline();4446 try f.newline();
4295 return local;4447 } else if (dest_scalar_ty.toIntern() == .bool_type) {
4296 }4448 // If the result is a boolean type, just check if the operand is non-zero.
42974449 assert(operand_scalar_ty.isAbiInt(zcu));
4298 const local = try f.allocLocal(null, dest_ty);4450 try f.writeCValue(w, dest_local, .other);
4299 // On big-endian targets, copying ABI integers with padding bits is awkward, because the padding bits are at the low bytes of the value.4451 try v.elem(f, w);
4300 // We need to offset the source or destination pointer appropriately and copy the right number of bytes.4452 try w.writeAll(" = ");
4301 if (target.cpu.arch.endian() == .big and dest_ty.isAbiInt(zcu) and !operand_ty.isAbiInt(zcu)) {4453 try f.writeCValue(w, operand, .other);
4302 // e.g. [10]u8 -> u80. We need to offset the destination so that we copy to the least significant bits of the integer.4454 try v.elem(f, w);
4303 const offset = dest_ty.abiSize(zcu) - operand_ty.abiSize(zcu);4455 try w.writeAll(" != 0;");
4304 try w.writeAll("memcpy((char *)&");4456 try f.newline();
4305 try f.writeCValue(w, local, .other);4457 } else if (dest_scalar_ty.isRuntimeFloat()) {
4306 try w.print(" + {d}, &", .{offset});4458 // For int->float, just do a memcpy.
4307 switch (operand) {4459 assert(operand_scalar_ty.isAbiInt(zcu));
4308 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4309 else => try f.writeCValue(w, operand, .other),
4310 }
4311 try w.print(", {d});", .{operand_ty.abiSize(zcu)});
4312 } else if (target.cpu.arch.endian() == .big and operand_ty.isAbiInt(zcu) and !dest_ty.isAbiInt(zcu)) {
4313 // e.g. u80 -> [10]u8. We need to offset the source so that we copy from the least significant bits of the integer.
4314 const offset = operand_ty.abiSize(zcu) - dest_ty.abiSize(zcu);
4315 try w.writeAll("memcpy(&");4460 try w.writeAll("memcpy(&");
4316 try f.writeCValue(w, local, .other);4461 try f.writeCValue(w, dest_local, .other);
4317 try w.writeAll(", (const char *)&");4462 try v.elem(f, w);
4463 try w.writeAll(", &");
4318 switch (operand) {4464 switch (operand) {
4319 .constant => |val| try f.dg.renderValueAsLvalue(w, val),4465 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4320 else => try f.writeCValue(w, operand, .other),4466 else => try f.writeCValue(w, operand, .other),
4321 }4467 }
4322 try w.print(" + {d}, {d});", .{ offset, dest_ty.abiSize(zcu) });4468 try v.elem(f, w);
4469 try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
4470 try f.newline();
4323 } else {4471 } else {
4472 // The only remaining possibility is that the result is an integer. We will need to use
4473 // `zig_wrap_*` to correct the "padding" bits after we populate the value bits.
4474 assert(dest_scalar_ty.isAbiInt(zcu));
4475 assert(operand_scalar_ty.isRuntimeFloat() or operand_scalar_ty.isAbiInt(zcu));
4476
4477 // memcpy the value...
4324 try w.writeAll("memcpy(&");4478 try w.writeAll("memcpy(&");
4325 try f.writeCValue(w, local, .other);4479 try f.writeCValue(w, dest_local, .other);
4480 try v.elem(f, w);
4326 try w.writeAll(", &");4481 try w.writeAll(", &");
4327 switch (operand) {4482 switch (operand) {
4328 .constant => |val| try f.dg.renderValueAsLvalue(w, val),4483 .constant => |val| try f.dg.renderValueAsLvalue(w, val),
4329 else => try f.writeCValue(w, operand, .other),4484 else => try f.writeCValue(w, operand, .other),
4330 }4485 }
4331 try w.print(", {d});", .{@min(dest_ty.abiSize(zcu), operand_ty.abiSize(zcu))});4486 try v.elem(f, w);
4332 }4487 try w.print(", {d});", .{@min(operand_scalar_ty.abiSize(zcu), dest_scalar_ty.abiSize(zcu))});
43334488 try f.newline();
4334 try f.newline();
43354489
4336 // Ensure padding bits have the expected value.4490 // ...and ensure padding bits have the correct value.
4337 if (dest_ty.isAbiInt(zcu)) {4491 switch (CType.classifyInt(dest_scalar_ty, zcu)) {
4338 switch (CType.classifyInt(dest_ty, zcu)) {
4339 .void => unreachable, // opv4492 .void => unreachable, // opv
4340 .small => {4493 .small => {
4341 try f.writeCValue(w, local, .other);4494 try f.writeCValue(w, dest_local, .other);
4495 try v.elem(f, w);
4342 try w.writeAll(" = zig_wrap_");4496 try w.writeAll(" = zig_wrap_");
4343 try f.dg.renderTypeForBuiltinFnName(w, dest_ty);4497 try f.dg.renderTypeForBuiltinFnName(w, dest_scalar_ty);
4344 try w.writeByte('(');4498 try w.writeByte('(');
4345 try f.writeCValue(w, local, .other);4499 try f.writeCValue(w, dest_local, .other);
4346 try f.dg.renderBuiltinInfo(w, dest_ty, .bits);4500 try v.elem(f, w);
4501 try f.dg.renderBuiltinInfo(w, dest_scalar_ty, .bits);
4347 try w.writeAll(");");4502 try w.writeAll(");");
4348 try f.newline();4503 try f.newline();
4349 },4504 },
4350 .big => |big| {4505 .big => |big| {
4351 const dest_info = dest_ty.intInfo(zcu);4506 const dest_info = dest_scalar_ty.intInfo(zcu);
4352 const padding_index: u16 = switch (target.cpu.arch.endian()) {4507 const padding_index: u16 = switch (f.dg.mod.resolved_target.result.cpu.arch.endian()) {
4353 .little => big.limbs_len - 1,4508 .little => big.limbs_len - 1,
4354 .big => 0,4509 .big => 0,
4355 };4510 };
4356 const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;4511 const wrap_bits = ((dest_info.bits - 1) % big.limb_size.bits()) + 1;
4357 if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {4512 if (big.limb_size != .@"128" or dest_info.signedness == .unsigned) {
4358 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4513 try f.writeCValue(w, dest_local, .other);
4359 try w.print("[{d}] = zig_wrap_{c}{d}(", .{4514 try v.elem(f, w);
4515 try w.print(".limbs[{d}] = zig_wrap_{c}{d}(", .{
4360 padding_index,4516 padding_index,
4361 signAbbrev(dest_info.signedness),4517 signAbbrev(dest_info.signedness),
4362 big.limb_size.bits(),4518 big.limb_size.bits(),
4363 });4519 });
4364 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4520 try f.writeCValue(w, dest_local, .other);
4365 try w.print("[{d}], {d});", .{ padding_index, wrap_bits });4521 try v.elem(f, w);
4522 try w.print(".limbs[{d}], {d});", .{ padding_index, wrap_bits });
4366 } else {4523 } else {
4367 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4524 try f.writeCValue(w, dest_local, .other);
4368 try w.print("[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{4525 try v.elem(f, w);
4526 try w.print(".limbs[{d}] = zig_bitCast_u128(zig_wrap_i128(zig_bitCast_i128(", .{
4369 padding_index,4527 padding_index,
4370 });4528 });
4371 try f.writeCValueMember(w, local, .{ .identifier = "limbs" });4529 try f.writeCValue(w, dest_local, .other);
4372 try w.print("[{d}]), {d}));", .{ padding_index, wrap_bits });4530 try v.elem(f, w);
4531 try w.print(".limbs[{d}]), {d}));", .{ padding_index, wrap_bits });
4373 try f.newline();4532 try f.newline();
4374 }4533 }
4375 },4534 },
4376 }4535 }
4377 }4536 }
43784537
4379 return local;4538 try v.end(f, inst, w);
4539
4540 try reap(f, inst, &.{ty_op.operand});
4541 return dest_local;
4380}4542}
43814543
4382fn airTrap(f: *Function) !void {4544fn airTrap(f: *Function) !void {
...@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {...@@ -6151,28 +6313,32 @@ fn airMemset(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue {
6151 return .none;6313 return .none;
6152 }6314 }
61536315
6154 if (elem_abi_size == 1 and !dest_ty.isVolatilePtr(zcu)) {6316 if (elem_abi_size == 1 and elem_ty.isAbiInt(zcu) and !dest_ty.isVolatilePtr(zcu)) {
6155 const bitcasted = try bitcast(f, .u8, value, elem_ty);
6156 try w.writeAll("memset(");6317 try w.writeAll("memset(");
6157 switch (dest_ty.ptrSize(zcu)) {6318 switch (dest_ty.ptrSize(zcu)) {
6158 .slice => {6319 .slice => {
6159 try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });6320 try f.writeCValueMember(w, dest_slice, .{ .identifier = "ptr" });
6160 try w.writeAll(", ");6321 try w.writeAll(", *(const char *)&");
6161 try f.writeCValue(w, bitcasted, .other);6322 switch (value) {
6323 .constant => |v| try f.dg.renderValueAsLvalue(w, v),
6324 else => try f.writeCValue(w, value, .other),
6325 }
6162 try w.writeAll(", ");6326 try w.writeAll(", ");
6163 try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });6327 try f.writeCValueMember(w, dest_slice, .{ .identifier = "len" });
6164 },6328 },
6165 .one => {6329 .one => {
6166 try f.writeCValue(w, dest_slice, .other);6330 try f.writeCValue(w, dest_slice, .other);
6167 try w.writeAll(", ");6331 try w.writeAll(", *(const char *)&");
6168 try f.writeCValue(w, bitcasted, .other);6332 switch (value) {
6333 .constant => |v| try f.dg.renderValueAsLvalue(w, v),
6334 else => try f.writeCValue(w, value, .other),
6335 }
6169 try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});6336 try w.print(", {d}", .{dest_ty.childType(zcu).arrayLen(zcu)});
6170 },6337 },
6171 .many, .c => unreachable,6338 .many, .c => unreachable,
6172 }6339 }
6173 try w.writeAll(");");6340 try w.writeAll(");");
6174 try f.newline();6341 try f.newline();
6175 try f.freeCValue(inst, bitcasted);
6176 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });6342 try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs });
6177 return .none;6343 return .none;
6178 }6344 }
src/codegen/c/type.zig+15-19
...@@ -514,40 +514,39 @@ pub const CType = union(enum) {...@@ -514,40 +514,39 @@ pub const CType = union(enum) {
514 }514 }
515 }515 }
516 fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {516 fn classifyBitInt(signedness: std.lang.Signedness, bits: u16, zcu: *const Zcu) IntClass {
517 const is_ez80 = zcu.getTarget().cpu.arch == .ez80;517 const target = zcu.getTarget();
518 return switch (bits) {518 return switch (std.zig.target.intByteSize(target, bits)) {
519 0 => .void,519 0 => .void,
520 1...8 => switch (signedness) {520 1 => switch (signedness) {
521 .unsigned => .{ .small = .uint8_t },521 .unsigned => .{ .small = .uint8_t },
522 .signed => .{ .small = .int8_t },522 .signed => .{ .small = .int8_t },
523 },523 },
524 9...16 => switch (signedness) {524 2 => switch (signedness) {
525 .unsigned => .{ .small = .uint16_t },525 .unsigned => .{ .small = .uint16_t },
526 .signed => .{ .small = .int16_t },526 .signed => .{ .small = .int16_t },
527 },527 },
528 17...24 => switch (signedness) {528 3 => switch (signedness) {
529 .unsigned => .{ .small = if (is_ez80) .uint24_t else .uint32_t },529 .unsigned => .{ .small = .uint24_t },
530 .signed => .{ .small = if (is_ez80) .int24_t else .int32_t },530 .signed => .{ .small = .int24_t },
531 },531 },
532 25...32 => switch (signedness) {532 4 => switch (signedness) {
533 .unsigned => .{ .small = .uint32_t },533 .unsigned => .{ .small = .uint32_t },
534 .signed => .{ .small = .int32_t },534 .signed => .{ .small = .int32_t },
535 },535 },
536 33...48 => switch (signedness) {536 6 => switch (signedness) {
537 .unsigned => .{ .small = if (is_ez80) .uint48_t else .uint64_t },537 .unsigned => .{ .small = .uint48_t },
538 .signed => .{ .small = if (is_ez80) .int48_t else .int64_t },538 .signed => .{ .small = .int48_t },
539 },539 },
540 49...64 => switch (signedness) {540 8 => switch (signedness) {
541 .unsigned => .{ .small = .uint64_t },541 .unsigned => .{ .small = .uint64_t },
542 .signed => .{ .small = .int64_t },542 .signed => .{ .small = .int64_t },
543 },543 },
544 65...128 => switch (signedness) {544 16 => switch (signedness) {
545 .unsigned => .{ .small = .zig_u128 },545 .unsigned => .{ .small = .zig_u128 },
546 .signed => .{ .small = .zig_i128 },546 .signed => .{ .small = .zig_i128 },
547 },547 },
548 else => {548 else => |n| {
549 @branchHint(.unlikely);549 @branchHint(.unlikely);
550 const target = zcu.getTarget();
551 const limb_bytes = std.zig.target.intAlignment(target, bits);550 const limb_bytes = std.zig.target.intAlignment(target, bits);
552 return .{ .big = .{551 return .{ .big = .{
553 .limb_size = switch (limb_bytes) {552 .limb_size = switch (limb_bytes) {
...@@ -558,10 +557,7 @@ pub const CType = union(enum) {...@@ -558,10 +557,7 @@ pub const CType = union(enum) {
558 16 => .@"128",557 16 => .@"128",
559 else => unreachable,558 else => unreachable,
560 },559 },
561 .limbs_len = @divExact(560 .limbs_len = @divExact(n, limb_bytes),
562 std.zig.target.intByteSize(target, bits),
563 limb_bytes,
564 ),
565 } };561 } };
566 },562 },
567 };563 };
src/codegen/llvm.zig+138-115
...@@ -21,8 +21,7 @@ const Zcu = @import("../Zcu.zig");...@@ -21,8 +21,7 @@ const Zcu = @import("../Zcu.zig");
21const aarch64_c_abi = @import("aarch64/abi.zig");21const aarch64_c_abi = @import("aarch64/abi.zig");
22const FuncGen = @import("llvm/FuncGen.zig");22const FuncGen = @import("llvm/FuncGen.zig");
23const isByRef = FuncGen.isByRef;23const isByRef = FuncGen.isByRef;
24const firstParamSRet = FuncGen.firstParamSRet;24const fnReturnStrat = FuncGen.fnReturnStrat;
25const lowerFnRetTy = FuncGen.lowerFnRetTy;
26const iterateParamTypes = FuncGen.iterateParamTypes;25const iterateParamTypes = FuncGen.iterateParamTypes;
27const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;26const ccAbiPromoteInt = FuncGen.ccAbiPromoteInt;
2827
...@@ -37,10 +36,10 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -37,10 +36,10 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
37 .expand_int_from_float_safe,36 .expand_int_from_float_safe,
38 .expand_int_from_float_optimized_safe,37 .expand_int_from_float_optimized_safe,
3938
40 .scalarize_bitcast_array,39 .scalarize_bit_cast_array,
41 // Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest40 // Needed because LLVM's `bitcast` on vectors is endian-specific unless the source and dest
42 // types are vectors with equal length (hence also with equal bits-per-element).41 // types are vectors with equal length (hence also with equal bits-per-element).
43 .scalarize_bitcast_vector_non_elementwise,42 .scalarize_bit_cast_vector_non_elementwise,
44 });43 });
45}44}
4645
...@@ -732,8 +731,8 @@ pub const Object = struct {...@@ -732,8 +731,8 @@ pub const Object = struct {
732731
733 // TODO: Address space732 // TODO: Address space
734 const slice_ty = Type.slice_const_u8_sentinel_0;733 const slice_ty = Type.slice_const_u8_sentinel_0;
735 const llvm_usize_ty = try o.lowerType(.usize);734 const llvm_usize_ty = try o.lowerType(.usize, .in_memory);
736 const llvm_slice_ty = try o.lowerType(slice_ty);735 const llvm_slice_ty = try o.lowerType(slice_ty, .in_memory);
737 const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);736 const llvm_table_ty = try o.builder.arrayType(1 + error_name_list.len, llvm_slice_ty);
738737
739 llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);738 llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty);
...@@ -799,7 +798,7 @@ pub const Object = struct {...@@ -799,7 +798,7 @@ pub const Object = struct {
799 {798 {
800 if (o.errors_len_variable != .none) {799 if (o.errors_len_variable != .none) {
801 const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;800 const errors_len = zcu.intern_pool.global_error_set.getNamesFromMainThread().len;
802 const init_val = try o.builder.intConst(try o.errorIntType(), errors_len);801 const init_val = try o.builder.intConst(try o.errorIntType(.in_memory), errors_len);
803 try o.errors_len_variable.setInitializer(init_val, &o.builder);802 try o.errors_len_variable.setInitializer(init_val, &o.builder);
804 }803 }
805 try o.genErrorNameTable();804 try o.genErrorNameTable();
...@@ -1191,7 +1190,7 @@ pub const Object = struct {...@@ -1191,7 +1190,7 @@ pub const Object = struct {
1191 };1190 };
1192 {1191 {
1193 const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);1192 const global = llvm_function.ptrConst(&o.builder).global.ptr(&o.builder);
1194 global.type = try o.lowerType(fn_ty);1193 global.type = try o.lowerType(fn_ty, .in_memory);
1195 global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);1194 global.addr_space = toLlvmAddressSpace(nav.resolved.?.@"addrspace", target);
1196 global.linkage = if (o.builder.strip) .private else .internal;1195 global.linkage = if (o.builder.strip) .private else .internal;
1197 global.visibility = .default;1196 global.visibility = .default;
...@@ -1440,10 +1439,10 @@ pub const Object = struct {...@@ -1440,10 +1439,10 @@ pub const Object = struct {
1440 // represent (because it doesn't have runtime bits), we instead lower as the zero-size1439 // represent (because it doesn't have runtime bits), we instead lower as the zero-size
1441 // type `[0 x i8]`. I don't think the type on an extern declaration actually does much1440 // type `[0 x i8]`. I don't think the type on an extern declaration actually does much
1442 // anyway.1441 // anyway.
1443 if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty);1442 if (nav_ty.isRuntimeFnOrHasRuntimeBits(zcu)) break :ty try o.lowerType(nav_ty, .in_memory);
1444 break :ty try o.builder.arrayType(0, .i8);1443 break :ty try o.builder.arrayType(0, .i8);
1445 } else if (nav_ty.hasRuntimeBits(zcu)) ty: {1444 } else if (nav_ty.hasRuntimeBits(zcu)) ty: {
1446 break :ty try o.lowerType(nav_ty);1445 break :ty try o.lowerType(nav_ty, .in_memory);
1447 } else {1446 } else {
1448 // This is a non-extern zero-bit `Nav`---we're not interested in it.1447 // This is a non-extern zero-bit `Nav`---we're not interested in it.
1449 // TODO: we might need to rethink this a little under incremental compilation. If a1448 // TODO: we might need to rethink this a little under incremental compilation. If a
...@@ -1536,7 +1535,7 @@ pub const Object = struct {...@@ -1536,7 +1535,7 @@ pub const Object = struct {
1536 llvm_variable.setAlignment(llvm_align, &o.builder);1535 llvm_variable.setAlignment(llvm_align, &o.builder);
1537 llvm_variable.setSection(llvm_section, &o.builder);1536 llvm_variable.setSection(llvm_section, &o.builder);
1538 llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);1537 llvm_variable.setMutability(if (resolved.@"const") .constant else .global, &o.builder);
1539 try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value), &o.builder);1538 try llvm_variable.setInitializer(if (opt_extern != null) .no_init else try o.lowerValue(resolved.value, .in_memory), &o.builder);
1540 llvm_variable.setThreadLocal(tl: {1539 llvm_variable.setThreadLocal(tl: {
1541 if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;1540 if (resolved.@"threadlocal" and !mod.single_threaded) break :tl .generaldynamic;
1542 break :tl .default;1541 break :tl .default;
...@@ -2134,7 +2133,7 @@ pub const Object = struct {...@@ -2134,7 +2133,7 @@ pub const Object = struct {
2134 defer debug_param_types.deinit(gpa);2133 defer debug_param_types.deinit(gpa);
21352134
2136 // Return type goes first.2135 // Return type goes first.
2137 if (firstParamSRet(fn_info, zcu, target)) {2136 if (try fnReturnStrat(o, fn_info) == .sret) {
2138 // Actual return type is void, then first arg is the sret pointer.2137 // Actual return type is void, then first arg is the sret pointer.
2139 const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));2138 const ptr_ty = try pt.singleMutPtrType(.fromInterned(fn_info.return_type));
2140 debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));2139 debug_param_types.appendAssumeCapacity(try o.getDebugType(pt, .void));
...@@ -2682,12 +2681,12 @@ pub const Object = struct {...@@ -2682,12 +2681,12 @@ pub const Object = struct {
2682 if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);2681 if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder);
26832682
2684 var it = iterateParamTypes(o, fn_info);2683 var it = iterateParamTypes(o, fn_info);
2685 if (firstParamSRet(fn_info, zcu, target)) {2684 if (try fnReturnStrat(o, fn_info) == .sret) {
2686 // Sret pointers must not be address 02685 // Sret pointers must not be address 0
2687 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);2686 try attributes.addParamAttr(it.llvm_index, .nonnull, &o.builder);
2688 try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);2687 try attributes.addParamAttr(it.llvm_index, .@"noalias", &o.builder);
26892688
2690 const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type));2689 const raw_llvm_ret_ty = try o.lowerType(.fromInterned(fn_info.return_type), .in_memory);
2691 try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);2690 try attributes.addParamAttr(it.llvm_index, .{ .sret = raw_llvm_ret_ty }, &o.builder);
2692 it.llvm_index += 1;2691 it.llvm_index += 1;
2693 } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {2692 } else if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {
...@@ -2729,9 +2728,7 @@ pub const Object = struct {...@@ -2729,9 +2728,7 @@ pub const Object = struct {
2729 },2728 },
2730 .byref => {2729 .byref => {
2731 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);2730 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
2732 const param_llvm_ty = try o.lowerType(param_ty);2731 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
2733 const alignment = param_ty.abiAlignment(zcu);
2734 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment.toLlvm(), it.byval_attr, param_llvm_ty);
2735 },2732 },
2736 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),2733 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
2737 .slice => {2734 .slice => {
...@@ -2841,14 +2838,31 @@ pub const Object = struct {...@@ -2841,14 +2838,31 @@ pub const Object = struct {
2841 }2838 }
2842 }2839 }
28432840
2844 pub fn errorIntType(o: *Object) Allocator.Error!Builder.Type {2841 pub const TypeRepr = enum {
2845 return o.builder.intType(o.zcu.errorSetBits());2842 /// The representation of the type when it is being manipulated as a value in a function.
2843 /// e.g. Zig `u5` -> LLVM `i5`
2844 by_value,
2845 /// The representation of the type when it is stored in memory.
2846 /// e.g. Zig `u5` -> LLVM `i8`
2847 in_memory,
2848 };
2849
2850 pub fn errorIntType(o: *Object, repr: TypeRepr) Allocator.Error!Builder.Type {
2851 return o.builder.intType(switch (repr) {
2852 .by_value => o.zcu.errorSetBits(),
2853 .in_memory => @intCast(Type.anyerror.abiSize(o.zcu) * 8),
2854 });
2846 }2855 }
28472856
2848 pub fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type {2857 pub fn lowerType(o: *Object, t: Type, repr: TypeRepr) Allocator.Error!Builder.Type {
2849 const zcu = o.zcu;2858 const zcu = o.zcu;
2850 const target = zcu.getTarget();2859 const target = zcu.getTarget();
2851 const ip = &zcu.intern_pool;2860 const ip = &zcu.intern_pool;
2861
2862 if (repr == .by_value) {
2863 assert(!isByRef(t, zcu)); // by-ref types must only be manipulated in memory
2864 }
2865
2852 return switch (t.toIntern()) {2866 return switch (t.toIntern()) {
2853 .u0_type => unreachable, // no runtime bits2867 .u0_type => unreachable, // no runtime bits
2854 inline .u1_type,2868 inline .u1_type,
...@@ -2864,7 +2878,10 @@ pub const Object = struct {...@@ -2864,7 +2878,10 @@ pub const Object = struct {
2864 .u80_type,2878 .u80_type,
2865 .u128_type,2879 .u128_type,
2866 .i128_type,2880 .i128_type,
2867 => |tag| @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),2881 => |tag| switch (repr) {
2882 .by_value => @field(Builder.Type, "i" ++ @tagName(tag)[1 .. @tagName(tag).len - "_type".len]),
2883 .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
2884 },
2868 .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),2885 .usize_type, .isize_type => try o.builder.intType(target.ptrBitWidth()),
2869 inline .c_char_type,2886 inline .c_char_type,
2870 .c_short_type,2887 .c_short_type,
...@@ -2899,7 +2916,7 @@ pub const Object = struct {...@@ -2899,7 +2916,7 @@ pub const Object = struct {
2899 return .i8;2916 return .i8;
2900 },2917 },
2901 .bool_type => .i1,2918 .bool_type => .i1,
2902 .anyerror_type => try o.errorIntType(),2919 .anyerror_type => try o.errorIntType(repr),
2903 .void_type => unreachable, // no runtime bits2920 .void_type => unreachable, // no runtime bits
2904 .type_type => unreachable, // no runtime bits2921 .type_type => unreachable, // no runtime bits
2905 .comptime_int_type => unreachable, // no runtime bits2922 .comptime_int_type => unreachable, // no runtime bits
...@@ -2919,10 +2936,10 @@ pub const Object = struct {...@@ -2919,10 +2936,10 @@ pub const Object = struct {
2919 => .ptr,2936 => .ptr,
2920 .slice_const_u8_type,2937 .slice_const_u8_type,
2921 .slice_const_u8_sentinel_0_type,2938 .slice_const_u8_sentinel_0_type,
2922 => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize) }),2939 => try o.builder.structType(.normal, &.{ .ptr, try o.lowerType(.usize, repr) }),
2923 .anyerror_void_error_union_type,2940 .anyerror_void_error_union_type,
2924 .adhoc_inferred_error_set_type,2941 .adhoc_inferred_error_set_type,
2925 => try o.errorIntType(),2942 => try o.errorIntType(repr),
2926 .generic_poison_type => unreachable,2943 .generic_poison_type => unreachable,
2927 // values, not types2944 // values, not types
2928 .undef,2945 .undef,
...@@ -2948,7 +2965,10 @@ pub const Object = struct {...@@ -2948,7 +2965,10 @@ pub const Object = struct {
2948 .none,2965 .none,
2949 => unreachable,2966 => unreachable,
2950 else => switch (ip.indexToKey(t.toIntern())) {2967 else => switch (ip.indexToKey(t.toIntern())) {
2951 .int_type => |int_type| try o.builder.intType(int_type.bits),2968 .int_type => |int_type| switch (repr) {
2969 .by_value => try o.builder.intType(int_type.bits),
2970 .in_memory => try o.builder.intType(@intCast(t.abiSize(zcu) * 8)),
2971 },
2952 .ptr_type => |ptr_type| type: {2972 .ptr_type => |ptr_type| type: {
2953 const ptr_ty = try o.builder.ptrType(2973 const ptr_ty = try o.builder.ptrType(
2954 toLlvmAddressSpace(ptr_type.flags.address_space, target),2974 toLlvmAddressSpace(ptr_type.flags.address_space, target),
...@@ -2957,18 +2977,18 @@ pub const Object = struct {...@@ -2957,18 +2977,18 @@ pub const Object = struct {
2957 .one, .many, .c => ptr_ty,2977 .one, .many, .c => ptr_ty,
2958 .slice => try o.builder.structType(.normal, &.{2978 .slice => try o.builder.structType(.normal, &.{
2959 ptr_ty,2979 ptr_ty,
2960 try o.lowerType(.usize),2980 try o.lowerType(.usize, repr),
2961 }),2981 }),
2962 };2982 };
2963 },2983 },
2964 .array_type => |array_type| o.builder.arrayType(2984 .array_type => |array_type| o.builder.arrayType(
2965 array_type.lenIncludingSentinel(),2985 array_type.lenIncludingSentinel(),
2966 try o.lowerType(.fromInterned(array_type.child)),2986 try o.lowerType(.fromInterned(array_type.child), repr),
2967 ),2987 ),
2968 .vector_type => |vector_type| o.builder.vectorType(2988 .vector_type => |vector_type| o.builder.vectorType(
2969 .normal,2989 .normal,
2970 vector_type.len,2990 vector_type.len,
2971 try o.lowerType(.fromInterned(vector_type.child)),2991 try o.lowerType(.fromInterned(vector_type.child), .by_value),
2972 ),2992 ),
2973 .opt_type => |child_ty| {2993 .opt_type => |child_ty| {
2974 // Must stay in sync with `opt_payload` logic in `lowerPtr`.2994 // Must stay in sync with `opt_payload` logic in `lowerPtr`.
...@@ -2978,8 +2998,11 @@ pub const Object = struct {...@@ -2978,8 +2998,11 @@ pub const Object = struct {
2978 .runtime, .partially_comptime => {},2998 .runtime, .partially_comptime => {},
2979 }2999 }
29803000
2981 const payload_ty = try o.lowerType(.fromInterned(child_ty));3001 if (t.optionalReprIsPayload(zcu)) {
2982 if (t.optionalReprIsPayload(zcu)) return payload_ty;3002 return o.lowerType(.fromInterned(child_ty), repr);
3003 }
3004
3005 const payload_ty = try o.lowerType(.fromInterned(child_ty), repr);
29833006
2984 comptime assert(optional_layout_version == 3);3007 comptime assert(optional_layout_version == 3);
2985 var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };3008 var fields: [3]Builder.Type = .{ payload_ty, .i8, undefined };
...@@ -2997,7 +3020,7 @@ pub const Object = struct {...@@ -2997,7 +3020,7 @@ pub const Object = struct {
2997 .error_union_type => |error_union_type| {3020 .error_union_type => |error_union_type| {
2998 // Must stay in sync with `codegen.errUnionPayloadOffset`.3021 // Must stay in sync with `codegen.errUnionPayloadOffset`.
2999 // See logic in `lowerPtr`.3022 // See logic in `lowerPtr`.
3000 const error_type = try o.errorIntType();3023 const error_type = try o.errorIntType(repr);
30013024
3002 switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {3025 switch (Type.fromInterned(error_union_type.payload_type).classify(zcu)) {
3003 .fully_comptime => unreachable,3026 .fully_comptime => unreachable,
...@@ -3005,7 +3028,7 @@ pub const Object = struct {...@@ -3005,7 +3028,7 @@ pub const Object = struct {
3005 .runtime, .partially_comptime => {},3028 .runtime, .partially_comptime => {},
3006 }3029 }
30073030
3008 const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type));3031 const payload_type = try o.lowerType(.fromInterned(error_union_type.payload_type), repr);
30093032
3010 const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);3033 const payload_align = Type.fromInterned(error_union_type.payload_type).abiAlignment(zcu);
3011 const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));3034 const error_align: InternPool.Alignment = .fromByteUnits(std.zig.target.intAlignment(target, zcu.errorSetBits()));
...@@ -3040,16 +3063,14 @@ pub const Object = struct {...@@ -3040,16 +3063,14 @@ pub const Object = struct {
3040 },3063 },
3041 .simple_type => unreachable,3064 .simple_type => unreachable,
3042 .struct_type => {3065 .struct_type => {
3043 if (o.type_map.get(t.toIntern())) |value| return value;
3044
3045 const struct_type = ip.loadStructType(t.toIntern());3066 const struct_type = ip.loadStructType(t.toIntern());
30463067
3047 if (struct_type.layout == .@"packed") {3068 if (struct_type.layout == .@"packed") {
3048 const int_ty = try o.lowerType(.fromInterned(struct_type.packed_backing_int_type));3069 return o.lowerType(.fromInterned(struct_type.packed_backing_int_type), repr);
3049 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3050 return int_ty;
3051 }3070 }
30523071
3072 if (o.type_map.get(t.toIntern())) |value| return value;
3073
3053 assert(struct_type.size > 0);3074 assert(struct_type.size > 0);
30543075
3055 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;3076 var llvm_field_types: std.ArrayList(Builder.Type) = .empty;
...@@ -3083,7 +3104,7 @@ pub const Object = struct {...@@ -3083,7 +3104,7 @@ pub const Object = struct {
30833104
3084 if (!field_ty.hasRuntimeBits(zcu)) continue;3105 if (!field_ty.hasRuntimeBits(zcu)) continue;
30853106
3086 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty));3107 try llvm_field_types.append(o.gpa, try o.lowerType(field_ty, repr));
30873108
3088 offset += field_ty.abiSize(zcu);3109 offset += field_ty.abiSize(zcu);
3089 }3110 }
...@@ -3139,7 +3160,7 @@ pub const Object = struct {...@@ -3139,7 +3160,7 @@ pub const Object = struct {
3139 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {3160 if (!Type.fromInterned(field_ty).hasRuntimeBits(zcu)) {
3140 continue;3161 continue;
3141 }3162 }
3142 try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty)));3163 try llvm_field_types.append(o.gpa, try o.lowerType(.fromInterned(field_ty), repr));
31433164
3144 offset += Type.fromInterned(field_ty).abiSize(zcu);3165 offset += Type.fromInterned(field_ty).abiSize(zcu);
3145 }3166 }
...@@ -3156,28 +3177,24 @@ pub const Object = struct {...@@ -3156,28 +3177,24 @@ pub const Object = struct {
3156 return o.builder.structType(.normal, llvm_field_types.items);3177 return o.builder.structType(.normal, llvm_field_types.items);
3157 },3178 },
3158 .union_type => {3179 .union_type => {
3159 if (o.type_map.get(t.toIntern())) |value| return value;
3160
3161 const union_obj = ip.loadUnionType(t.toIntern());3180 const union_obj = ip.loadUnionType(t.toIntern());
31623181
3163 if (union_obj.layout == .@"packed") {3182 if (union_obj.layout == .@"packed") {
3164 const int_ty = try o.lowerType(.fromInterned(union_obj.packed_backing_int_type));3183 return o.lowerType(.fromInterned(union_obj.packed_backing_int_type), repr);
3165 try o.type_map.put(o.gpa, t.toIntern(), int_ty);
3166 return int_ty;
3167 }3184 }
31683185
3169 assert(union_obj.size > 0);
3170
3171 const layout = Type.getUnionLayout(union_obj, zcu);3186 const layout = Type.getUnionLayout(union_obj, zcu);
31723187
3173 if (layout.payload_size == 0) {3188 if (layout.payload_size == 0) {
3174 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));3189 return o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
3175 try o.type_map.put(o.gpa, t.toIntern(), enum_tag_ty);
3176 return enum_tag_ty;
3177 }3190 }
31783191
3192 if (o.type_map.get(t.toIntern())) |value| return value;
3193
3194 assert(union_obj.size > 0);
3195
3179 const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);3196 const aligned_field_ty = Type.fromInterned(union_obj.field_types.get(ip)[layout.most_aligned_field]);
3180 const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty);3197 const aligned_field_llvm_ty = try o.lowerType(aligned_field_ty, repr);
31813198
3182 const payload_ty = ty: {3199 const payload_ty = ty: {
3183 if (layout.most_aligned_field_size == layout.payload_size) {3200 if (layout.most_aligned_field_size == layout.payload_size) {
...@@ -3203,7 +3220,7 @@ pub const Object = struct {...@@ -3203,7 +3220,7 @@ pub const Object = struct {
3203 );3220 );
3204 return ty;3221 return ty;
3205 }3222 }
3206 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type));3223 const enum_tag_ty = try o.lowerType(.fromInterned(union_obj.enum_tag_type), repr);
32073224
3208 // Put the tag before or after the payload depending on which one's3225 // Put the tag before or after the payload depending on which one's
3209 // alignment is greater.3226 // alignment is greater.
...@@ -3232,9 +3249,9 @@ pub const Object = struct {...@@ -3232,9 +3249,9 @@ pub const Object = struct {
3232 return ty;3249 return ty;
3233 },3250 },
3234 .opaque_type, .spirv_type => unreachable, // no runtime bits3251 .opaque_type, .spirv_type => unreachable, // no runtime bits
3235 .enum_type => try o.lowerType(t.intTagType(zcu)),3252 .enum_type => try o.lowerType(t.intTagType(zcu), repr),
3236 .func_type => |func_type| try o.lowerFnType(t, func_type),3253 .func_type => |func_type| try o.lowerFnType(t, func_type),
3237 .error_set_type, .inferred_error_set_type => try o.errorIntType(),3254 .error_set_type, .inferred_error_set_type => try o.errorIntType(repr),
3238 // values, not types3255 // values, not types
3239 .undef,3256 .undef,
3240 .simple_value,3257 .simple_value,
...@@ -3266,12 +3283,12 @@ pub const Object = struct {...@@ -3266,12 +3283,12 @@ pub const Object = struct {
32663283
3267 assert(fn_ty.fnHasRuntimeBits(zcu));3284 assert(fn_ty.fnHasRuntimeBits(zcu));
32683285
3269 const ret_ty = try lowerFnRetTy(o, fn_info);3286 const ret_strat = try fnReturnStrat(o, fn_info);
32703287
3271 var llvm_params: std.ArrayList(Builder.Type) = .empty;3288 var llvm_params: std.ArrayList(Builder.Type) = .empty;
3272 defer llvm_params.deinit(o.gpa);3289 defer llvm_params.deinit(o.gpa);
32733290
3274 if (firstParamSRet(fn_info, zcu, target)) {3291 if (ret_strat == .sret) {
3275 try llvm_params.append(o.gpa, .ptr);3292 try llvm_params.append(o.gpa, .ptr);
3276 }3293 }
32773294
...@@ -3286,7 +3303,7 @@ pub const Object = struct {...@@ -3286,7 +3303,7 @@ pub const Object = struct {
3286 .no_bits => continue,3303 .no_bits => continue,
3287 .byval => {3304 .byval => {
3288 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3305 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3289 try llvm_params.append(o.gpa, try o.lowerType(param_ty));3306 try llvm_params.append(o.gpa, try o.lowerType(param_ty, if (isByRef(param_ty, zcu)) .in_memory else .by_value));
3290 },3307 },
3291 .byref, .byref_mut => {3308 .byref, .byref_mut => {
3292 try llvm_params.append(o.gpa, .ptr);3309 try llvm_params.append(o.gpa, .ptr);
...@@ -3301,7 +3318,7 @@ pub const Object = struct {...@@ -3301,7 +3318,7 @@ pub const Object = struct {
3301 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3318 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3302 try llvm_params.appendSlice(o.gpa, &.{3319 try llvm_params.appendSlice(o.gpa, &.{
3303 try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),3320 try o.builder.ptrType(toLlvmAddressSpace(param_ty.ptrAddressSpace(zcu), target)),
3304 try o.lowerType(.usize),3321 try o.lowerType(.usize, .by_value),
3305 });3322 });
3306 },3323 },
3307 .multiple_llvm_types => {3324 .multiple_llvm_types => {
...@@ -3309,7 +3326,7 @@ pub const Object = struct {...@@ -3309,7 +3326,7 @@ pub const Object = struct {
3309 },3326 },
3310 .float_array => |count| {3327 .float_array => |count| {
3311 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);3328 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[it.zig_index - 1]);
3312 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?);3329 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(param_ty, zcu).?, .in_memory);
3313 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));3330 try llvm_params.append(o.gpa, try o.builder.arrayType(count, float_ty));
3314 },3331 },
3315 .i32_array, .i64_array => |arr_len| {3332 .i32_array, .i64_array => |arr_len| {
...@@ -3321,14 +3338,19 @@ pub const Object = struct {...@@ -3321,14 +3338,19 @@ pub const Object = struct {
3321 },3338 },
3322 };3339 };
33233340
3324 return o.builder.fnType(3341 const llvm_ret_ty: Builder.Type = switch (ret_strat) {
3325 ret_ty,3342 .void, .sret => .void,
3326 llvm_params.items,3343 .by_val => try o.lowerType(.fromInterned(fn_info.return_type), .by_value),
3327 if (fn_info.is_var_args) .vararg else .normal,3344 .mem_cast => |llvm_ret_ty| llvm_ret_ty,
3328 );3345 };
3346 const llvm_fn_kind: Builder.Type.Function.Kind = switch (fn_info.is_var_args) {
3347 true => .vararg,
3348 false => .normal,
3349 };
3350 return o.builder.fnType(llvm_ret_ty, llvm_params.items, llvm_fn_kind);
3329 }3351 }
33303352
3331 pub fn lowerValue(o: *Object, arg_val: InternPool.Index) Allocator.Error!Builder.Constant {3353 pub fn lowerValue(o: *Object, arg_val: InternPool.Index, repr: TypeRepr) Allocator.Error!Builder.Constant {
3332 const zcu = o.zcu;3354 const zcu = o.zcu;
3333 const ip = &zcu.intern_pool;3355 const ip = &zcu.intern_pool;
3334 const target = zcu.getTarget();3356 const target = zcu.getTarget();
...@@ -3360,7 +3382,7 @@ pub const Object = struct {...@@ -3360,7 +3382,7 @@ pub const Object = struct {
3360 .inferred_error_set_type,3382 .inferred_error_set_type,
3361 => unreachable, // types, not values3383 => unreachable, // types, not values
33623384
3363 .undef => return o.builder.undefConst(try o.lowerType(ty)),3385 .undef => return o.builder.undefConst(try o.lowerType(ty, repr)),
3364 .simple_value => |simple_value| switch (simple_value) {3386 .simple_value => |simple_value| switch (simple_value) {
3365 .void => unreachable, // non-runtime value3387 .void => unreachable, // non-runtime value
3366 .null => unreachable, // non-runtime value3388 .null => unreachable, // non-runtime value
...@@ -3375,15 +3397,15 @@ pub const Object = struct {...@@ -3375,15 +3397,15 @@ pub const Object = struct {
3375 .int => {3397 .int => {
3376 var bigint_space: Value.BigIntSpace = undefined;3398 var bigint_space: Value.BigIntSpace = undefined;
3377 const bigint = val.toBigInt(&bigint_space, zcu);3399 const bigint = val.toBigInt(&bigint_space, zcu);
3378 const llvm_int_ty = try o.builder.intType(ty.intInfo(zcu).bits);3400 const llvm_int_ty = try o.lowerType(ty, repr);
3379 return o.builder.bigIntConst(llvm_int_ty, bigint);3401 return o.builder.bigIntConst(llvm_int_ty, bigint);
3380 },3402 },
3381 .err => |err| {3403 .err => |err| {
3382 const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;3404 const int = zcu.intern_pool.getErrorValueIfExists(err.name).?;
3383 return o.builder.intConst(try o.errorIntType(), int);3405 return o.builder.intConst(try o.errorIntType(repr), int);
3384 },3406 },
3385 .error_union => |error_union| {3407 .error_union => |error_union| {
3386 const llvm_error_ty = try o.errorIntType();3408 const llvm_error_ty = try o.errorIntType(repr);
3387 const llvm_error_value = switch (error_union.val) {3409 const llvm_error_value = switch (error_union.val) {
3388 .err_name => |name| try o.builder.intConst(3410 .err_name => |name| try o.builder.intConst(
3389 llvm_error_ty,3411 llvm_error_ty,
...@@ -3401,8 +3423,8 @@ pub const Object = struct {...@@ -3401,8 +3423,8 @@ pub const Object = struct {
3401 const payload_align = payload_type.abiAlignment(zcu);3423 const payload_align = payload_type.abiAlignment(zcu);
3402 const error_align = Type.errorAbiAlignment(zcu);3424 const error_align = Type.errorAbiAlignment(zcu);
3403 const llvm_payload_value = switch (error_union.val) {3425 const llvm_payload_value = switch (error_union.val) {
3404 .err_name => try o.builder.undefConst(try o.lowerType(payload_type)),3426 .err_name => try o.builder.undefConst(try o.lowerType(payload_type, repr)),
3405 .payload => |payload| try o.lowerValue(payload),3427 .payload => |payload| try o.lowerValue(payload, repr),
3406 };3428 };
34073429
3408 var fields: [3]Builder.Type = undefined;3430 var fields: [3]Builder.Type = undefined;
...@@ -3417,7 +3439,7 @@ pub const Object = struct {...@@ -3417,7 +3439,7 @@ pub const Object = struct {
3417 fields[0] = vals[0].typeOf(&o.builder);3439 fields[0] = vals[0].typeOf(&o.builder);
3418 fields[1] = vals[1].typeOf(&o.builder);3440 fields[1] = vals[1].typeOf(&o.builder);
34193441
3420 const llvm_ty = try o.lowerType(ty);3442 const llvm_ty = try o.lowerType(ty, repr);
3421 const llvm_ty_fields = llvm_ty.structFields(&o.builder);3443 const llvm_ty_fields = llvm_ty.structFields(&o.builder);
3422 if (llvm_ty_fields.len > 2) {3444 if (llvm_ty_fields.len > 2) {
3423 assert(llvm_ty_fields.len == 3);3445 assert(llvm_ty_fields.len == 3);
...@@ -3429,7 +3451,7 @@ pub const Object = struct {...@@ -3429,7 +3451,7 @@ pub const Object = struct {
3429 fields[0..llvm_ty_fields.len],3451 fields[0..llvm_ty_fields.len],
3430 ), vals[0..llvm_ty_fields.len]);3452 ), vals[0..llvm_ty_fields.len]);
3431 },3453 },
3432 .enum_tag => |enum_tag| o.lowerValue(enum_tag.int),3454 .enum_tag => |enum_tag| o.lowerValue(enum_tag.int, repr),
3433 .float => switch (ty.floatBits(target)) {3455 .float => switch (ty.floatBits(target)) {
3434 16 => if (backendSupportsF16(target))3456 16 => if (backendSupportsF16(target))
3435 try o.builder.halfConst(val.toFloat(f16, zcu))3457 try o.builder.halfConst(val.toFloat(f16, zcu))
...@@ -3445,9 +3467,9 @@ pub const Object = struct {...@@ -3445,9 +3467,9 @@ pub const Object = struct {
3445 else => unreachable,3467 else => unreachable,
3446 },3468 },
3447 .ptr => try o.lowerPtr(arg_val, 0),3469 .ptr => try o.lowerPtr(arg_val, 0),
3448 .slice => |slice| return o.builder.structConst(try o.lowerType(ty), &.{3470 .slice => |slice| return o.builder.structConst(try o.lowerType(ty, repr), &.{
3449 try o.lowerValue(slice.ptr),3471 try o.lowerValue(slice.ptr, repr),
3450 try o.lowerValue(slice.len),3472 try o.lowerValue(slice.len, repr),
3451 }),3473 }),
3452 .opt => |opt| {3474 .opt => |opt| {
3453 comptime assert(optional_layout_version == 3);3475 comptime assert(optional_layout_version == 3);
...@@ -3457,7 +3479,7 @@ pub const Object = struct {...@@ -3457,7 +3479,7 @@ pub const Object = struct {
3457 if (!payload_ty.hasRuntimeBits(zcu)) {3479 if (!payload_ty.hasRuntimeBits(zcu)) {
3458 return non_null_bit;3480 return non_null_bit;
3459 }3481 }
3460 const llvm_ty = try o.lowerType(ty);3482 const llvm_ty = try o.lowerType(ty, repr);
3461 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {3483 if (ty.optionalReprIsPayload(zcu)) return switch (opt.val) {
3462 .none => switch (llvm_ty.tag(&o.builder)) {3484 .none => switch (llvm_ty.tag(&o.builder)) {
3463 .integer => try o.builder.intConst(llvm_ty, 0),3485 .integer => try o.builder.intConst(llvm_ty, 0),
...@@ -3465,15 +3487,15 @@ pub const Object = struct {...@@ -3465,15 +3487,15 @@ pub const Object = struct {
3465 .structure => try o.builder.zeroInitConst(llvm_ty),3487 .structure => try o.builder.zeroInitConst(llvm_ty),
3466 else => unreachable,3488 else => unreachable,
3467 },3489 },
3468 else => |payload| try o.lowerValue(payload),3490 else => |payload| try o.lowerValue(payload, repr),
3469 };3491 };
3470 assert(payload_ty.zigTypeTag(zcu) != .@"fn");3492 assert(payload_ty.zigTypeTag(zcu) != .@"fn");
34713493
3472 var fields: [3]Builder.Type = undefined;3494 var fields: [3]Builder.Type = undefined;
3473 var vals: [3]Builder.Constant = undefined;3495 var vals: [3]Builder.Constant = undefined;
3474 vals[0] = switch (opt.val) {3496 vals[0] = switch (opt.val) {
3475 .none => try o.builder.undefConst(try o.lowerType(payload_ty)),3497 .none => try o.builder.undefConst(try o.lowerType(payload_ty, repr)),
3476 else => |payload| try o.lowerValue(payload),3498 else => |payload| try o.lowerValue(payload, repr),
3477 };3499 };
3478 vals[1] = non_null_bit;3500 vals[1] = non_null_bit;
3479 fields[0] = vals[0].typeOf(&o.builder);3501 fields[0] = vals[0].typeOf(&o.builder);
...@@ -3490,14 +3512,14 @@ pub const Object = struct {...@@ -3490,14 +3512,14 @@ pub const Object = struct {
3490 fields[0..llvm_ty_fields.len],3512 fields[0..llvm_ty_fields.len],
3491 ), vals[0..llvm_ty_fields.len]);3513 ), vals[0..llvm_ty_fields.len]);
3492 },3514 },
3493 .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val),3515 .bitpack => |bitpack| return o.lowerValue(bitpack.backing_int_val, repr),
3494 .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {3516 .aggregate => |aggregate| switch (ip.indexToKey(ty.toIntern())) {
3495 .array_type => |array_type| switch (aggregate.storage) {3517 .array_type => |array_type| switch (aggregate.storage) {
3496 .bytes => |bytes| try o.builder.stringConst(try o.builder.string(3518 .bytes => |bytes| try o.builder.stringConst(try o.builder.string(
3497 bytes.toSlice(array_type.lenIncludingSentinel(), ip),3519 bytes.toSlice(array_type.lenIncludingSentinel(), ip),
3498 )),3520 )),
3499 .elems => |elems| {3521 .elems => |elems| {
3500 const array_ty = try o.lowerType(ty);3522 const array_ty = try o.lowerType(ty, repr);
3501 const elem_ty = array_ty.childType(&o.builder);3523 const elem_ty = array_ty.childType(&o.builder);
3502 assert(elems.len == array_ty.aggregateLen(&o.builder));3524 assert(elems.len == array_ty.aggregateLen(&o.builder));
35033525
...@@ -3515,7 +3537,7 @@ pub const Object = struct {...@@ -3515,7 +3537,7 @@ pub const Object = struct {
35153537
3516 var need_unnamed = false;3538 var need_unnamed = false;
3517 for (vals, fields, elems) |*result_val, *result_field, elem| {3539 for (vals, fields, elems) |*result_val, *result_field, elem| {
3518 result_val.* = try o.lowerValue(elem);3540 result_val.* = try o.lowerValue(elem, repr);
3519 result_field.* = result_val.typeOf(&o.builder);3541 result_field.* = result_val.typeOf(&o.builder);
3520 if (result_field.* != elem_ty) need_unnamed = true;3542 if (result_field.* != elem_ty) need_unnamed = true;
3521 }3543 }
...@@ -3527,7 +3549,7 @@ pub const Object = struct {...@@ -3527,7 +3549,7 @@ pub const Object = struct {
3527 .repeated_elem => |elem| {3549 .repeated_elem => |elem| {
3528 const len: usize = @intCast(array_type.len);3550 const len: usize = @intCast(array_type.len);
3529 const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());3551 const len_including_sentinel: usize = @intCast(array_type.lenIncludingSentinel());
3530 const array_ty = try o.lowerType(ty);3552 const array_ty = try o.lowerType(ty, repr);
3531 const elem_ty = array_ty.childType(&o.builder);3553 const elem_ty = array_ty.childType(&o.builder);
35323554
3533 const ExpectedContents = extern struct {3555 const ExpectedContents = extern struct {
...@@ -3543,12 +3565,12 @@ pub const Object = struct {...@@ -3543,12 +3565,12 @@ pub const Object = struct {
3543 defer allocator.free(fields);3565 defer allocator.free(fields);
35443566
3545 var need_unnamed = false;3567 var need_unnamed = false;
3546 @memset(vals[0..len], try o.lowerValue(elem));3568 @memset(vals[0..len], try o.lowerValue(elem, repr));
3547 @memset(fields[0..len], vals[0].typeOf(&o.builder));3569 @memset(fields[0..len], vals[0].typeOf(&o.builder));
3548 if (fields[0] != elem_ty) need_unnamed = true;3570 if (fields[0] != elem_ty) need_unnamed = true;
35493571
3550 if (array_type.sentinel != .none) {3572 if (array_type.sentinel != .none) {
3551 vals[len] = try o.lowerValue(array_type.sentinel);3573 vals[len] = try o.lowerValue(array_type.sentinel, repr);
3552 fields[len] = vals[len].typeOf(&o.builder);3574 fields[len] = vals[len].typeOf(&o.builder);
3553 if (fields[len] != elem_ty) need_unnamed = true;3575 if (fields[len] != elem_ty) need_unnamed = true;
3554 }3576 }
...@@ -3560,7 +3582,7 @@ pub const Object = struct {...@@ -3560,7 +3582,7 @@ pub const Object = struct {
3560 },3582 },
3561 },3583 },
3562 .vector_type => |vector_type| {3584 .vector_type => |vector_type| {
3563 const vector_ty = try o.lowerType(ty);3585 const vector_ty = try o.lowerType(ty, repr);
3564 switch (aggregate.storage) {3586 switch (aggregate.storage) {
3565 .bytes, .elems => {3587 .bytes, .elems => {
3566 const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;3588 const ExpectedContents = [Builder.expected_fields_len]Builder.Constant;
...@@ -3575,7 +3597,7 @@ pub const Object = struct {...@@ -3575,7 +3597,7 @@ pub const Object = struct {
3575 result_val.* = try o.builder.intConst(.i8, byte);3597 result_val.* = try o.builder.intConst(.i8, byte);
3576 },3598 },
3577 .elems => |elems| for (vals, elems) |*result_val, elem| {3599 .elems => |elems| for (vals, elems) |*result_val, elem| {
3578 result_val.* = try o.lowerValue(elem);3600 result_val.* = try o.lowerValue(elem, .by_value);
3579 },3601 },
3580 .repeated_elem => unreachable,3602 .repeated_elem => unreachable,
3581 }3603 }
...@@ -3583,12 +3605,12 @@ pub const Object = struct {...@@ -3583,12 +3605,12 @@ pub const Object = struct {
3583 },3605 },
3584 .repeated_elem => |elem| return o.builder.splatConst(3606 .repeated_elem => |elem| return o.builder.splatConst(
3585 vector_ty,3607 vector_ty,
3586 try o.lowerValue(elem),3608 try o.lowerValue(elem, .by_value),
3587 ),3609 ),
3588 }3610 }
3589 },3611 },
3590 .tuple_type => |tuple| {3612 .tuple_type => |tuple| {
3591 const struct_ty = try o.lowerType(ty);3613 const struct_ty = try o.lowerType(ty, repr);
3592 const llvm_len = struct_ty.aggregateLen(&o.builder);3614 const llvm_len = struct_ty.aggregateLen(&o.builder);
35933615
3594 const ExpectedContents = extern struct {3616 const ExpectedContents = extern struct {
...@@ -3633,8 +3655,8 @@ pub const Object = struct {...@@ -3633,8 +3655,8 @@ pub const Object = struct {
36333655
3634 vals[llvm_index] = switch (aggregate.storage) {3656 vals[llvm_index] = switch (aggregate.storage) {
3635 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),3657 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3636 .elems => |elems| try o.lowerValue(elems[field_index]),3658 .elems => |elems| try o.lowerValue(elems[field_index], repr),
3637 .repeated_elem => |elem| try o.lowerValue(elem),3659 .repeated_elem => |elem| try o.lowerValue(elem, repr),
3638 };3660 };
3639 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);3661 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3640 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])3662 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...@@ -3663,7 +3685,7 @@ pub const Object = struct {...@@ -3663,7 +3685,7 @@ pub const Object = struct {
3663 },3685 },
3664 .struct_type => {3686 .struct_type => {
3665 const struct_type = ip.loadStructType(ty.toIntern());3687 const struct_type = ip.loadStructType(ty.toIntern());
3666 const struct_ty = try o.lowerType(ty);3688 const struct_ty = try o.lowerType(ty, repr);
3667 assert(struct_type.layout != .@"packed");3689 assert(struct_type.layout != .@"packed");
3668 const llvm_len = struct_ty.aggregateLen(&o.builder);3690 const llvm_len = struct_ty.aggregateLen(&o.builder);
36693691
...@@ -3707,8 +3729,8 @@ pub const Object = struct {...@@ -3707,8 +3729,8 @@ pub const Object = struct {
37073729
3708 vals[llvm_index] = switch (aggregate.storage) {3730 vals[llvm_index] = switch (aggregate.storage) {
3709 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),3731 .bytes => |bytes| try o.builder.intConst(.i8, bytes.at(field_index, ip)),
3710 .elems => |elems| try o.lowerValue(elems[field_index]),3732 .elems => |elems| try o.lowerValue(elems[field_index], repr),
3711 .repeated_elem => |elem| try o.lowerValue(elem),3733 .repeated_elem => |elem| try o.lowerValue(elem, repr),
3712 };3734 };
3713 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);3735 fields[llvm_index] = vals[llvm_index].typeOf(&o.builder);
3714 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])3736 if (fields[llvm_index] != struct_ty.structFields(&o.builder)[llvm_index])
...@@ -3738,9 +3760,9 @@ pub const Object = struct {...@@ -3738,9 +3760,9 @@ pub const Object = struct {
3738 else => unreachable,3760 else => unreachable,
3739 },3761 },
3740 .un => |un| {3762 .un => |un| {
3741 const union_ty = try o.lowerType(ty);3763 const union_ty = try o.lowerType(ty, repr);
3742 const layout = ty.unionGetLayout(zcu);3764 const layout = ty.unionGetLayout(zcu);
3743 if (layout.payload_size == 0) return o.lowerValue(un.tag);3765 if (layout.payload_size == 0) return o.lowerValue(un.tag, repr);
37443766
3745 const union_obj = zcu.typeToUnion(ty).?;3767 const union_obj = zcu.typeToUnion(ty).?;
3746 const container_layout = union_obj.layout;3768 const container_layout = union_obj.layout;
...@@ -3761,7 +3783,7 @@ pub const Object = struct {...@@ -3761,7 +3783,7 @@ pub const Object = struct {
3761 const padding_len = layout.payload_size;3783 const padding_len = layout.payload_size;
3762 break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));3784 break :p try o.builder.undefConst(try o.builder.arrayType(padding_len, .i8));
3763 }3785 }
3764 const payload = try o.lowerValue(un.val);3786 const payload = try o.lowerValue(un.val, repr);
3765 const payload_ty = payload.typeOf(&o.builder);3787 const payload_ty = payload.typeOf(&o.builder);
3766 if (payload_ty != union_ty.structFields(&o.builder)[3788 if (payload_ty != union_ty.structFields(&o.builder)[
3767 @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))3789 @intFromBool(layout.tag_size > 0 and layout.tag_align.compare(.gte, layout.payload_align))
...@@ -3776,7 +3798,7 @@ pub const Object = struct {...@@ -3776,7 +3798,7 @@ pub const Object = struct {
3776 );3798 );
3777 } else p: {3799 } else p: {
3778 assert(layout.tag_size == 0);3800 assert(layout.tag_size == 0);
3779 const union_val = try o.lowerValue(un.val);3801 const union_val = try o.lowerValue(un.val, repr);
3780 need_unnamed = true;3802 need_unnamed = true;
3781 break :p union_val;3803 break :p union_val;
3782 };3804 };
...@@ -3786,7 +3808,7 @@ pub const Object = struct {...@@ -3786,7 +3808,7 @@ pub const Object = struct {
3786 try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})3808 try o.builder.structType(union_ty.structKind(&o.builder), &.{payload_ty})
3787 else3809 else
3788 union_ty, &.{payload});3810 union_ty, &.{payload});
3789 const tag = try o.lowerValue(un.tag);3811 const tag = try o.lowerValue(un.tag, repr);
3790 const tag_ty = tag.typeOf(&o.builder);3812 const tag_ty = tag.typeOf(&o.builder);
3791 var fields: [3]Builder.Type = undefined;3813 var fields: [3]Builder.Type = undefined;
3792 var vals: [3]Builder.Constant = undefined;3814 var vals: [3]Builder.Constant = undefined;
...@@ -3840,8 +3862,8 @@ pub const Object = struct {...@@ -3840,8 +3862,8 @@ pub const Object = struct {
3840 },3862 },
3841 .int => try o.builder.castConst(3863 .int => try o.builder.castConst(
3842 .inttoptr,3864 .inttoptr,
3843 try o.builder.intConst(try o.lowerType(.usize), offset),3865 try o.builder.intConst(try o.lowerType(.usize, .by_value), offset),
3844 try o.lowerType(.fromInterned(ptr.ty)),3866 try o.lowerType(.fromInterned(ptr.ty), .by_value),
3845 ),3867 ),
3846 .eu_payload => |eu_ptr| try o.lowerPtr(3868 .eu_payload => |eu_ptr| try o.lowerPtr(
3847 eu_ptr,3869 eu_ptr,
...@@ -3888,7 +3910,7 @@ pub const Object = struct {...@@ -3888,7 +3910,7 @@ pub const Object = struct {
3888 @"addrspace": std.lang.AddressSpace,3910 @"addrspace": std.lang.AddressSpace,
3889 ) Allocator.Error!Builder.Constant {3911 ) Allocator.Error!Builder.Constant {
3890 const addr: u64 = @"align".toByteUnits().?;3912 const addr: u64 = @"align".toByteUnits().?;
3891 const llvm_usize = try o.lowerType(.usize);3913 const llvm_usize = try o.lowerType(.usize, .by_value);
3892 const llvm_addr = try o.builder.intConst(llvm_usize, addr);3914 const llvm_addr = try o.builder.intConst(llvm_usize, addr);
3893 const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));3915 const llvm_ptr_ty = try o.builder.ptrType(toLlvmAddressSpace(@"addrspace", o.zcu.getTarget()));
3894 return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);3916 return o.builder.castConst(.inttoptr, llvm_addr, llvm_ptr_ty);
...@@ -3932,11 +3954,11 @@ pub const Object = struct {...@@ -3932,11 +3954,11 @@ pub const Object = struct {
3932 }3954 }
3933 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));3955 errdefer assert(o.uav_map.remove(.{ .val = uav_val, .@"addrspace" = @"addrspace" }));
39343956
3935 const llvm_ty = try o.lowerType(uav_ty);3957 const llvm_ty = try o.lowerType(uav_ty, .in_memory);
3936 const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});3958 const llvm_name = try o.builder.strtabStringFmt("__anon_{d}", .{@intFromEnum(uav_val)});
3937 const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);3959 const llvm_variable = try o.builder.addVariable(llvm_name, llvm_ty, llvm_addrspace);
3938 gop.value_ptr.* = llvm_variable;3960 gop.value_ptr.* = llvm_variable;
3939 try llvm_variable.setInitializer(try o.lowerValue(uav_val), &o.builder);3961 try llvm_variable.setInitializer(try o.lowerValue(uav_val, .in_memory), &o.builder);
3940 llvm_variable.setMutability(.constant, &o.builder);3962 llvm_variable.setMutability(.constant, &o.builder);
3941 llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);3963 llvm_variable.setAlignment(@"align".toLlvm(), &o.builder);
3942 const llvm_global = llvm_variable.ptrConst(&o.builder).global;3964 const llvm_global = llvm_variable.ptrConst(&o.builder).global;
...@@ -4008,7 +4030,7 @@ pub const Object = struct {...@@ -4008,7 +4030,7 @@ pub const Object = struct {
4008 .x86_64_interrupt,4030 .x86_64_interrupt,
4009 .x86_interrupt,4031 .x86_interrupt,
4010 => {4032 => {
4011 const child_type = try lowerType(o, Type.fromInterned(ptr_info.child));4033 const child_type = try o.lowerType(.fromInterned(ptr_info.child), .in_memory);
4012 try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);4034 try attributes.addParamAttr(llvm_arg_i, .{ .byval = child_type }, &o.builder);
4013 },4035 },
4014 }4036 }
...@@ -4030,14 +4052,15 @@ pub const Object = struct {...@@ -4030,14 +4052,15 @@ pub const Object = struct {
4030 o: *Object,4052 o: *Object,
4031 attributes: *Builder.FunctionAttributes.Wip,4053 attributes: *Builder.FunctionAttributes.Wip,
4032 llvm_arg_i: u32,4054 llvm_arg_i: u32,
4033 alignment: Builder.Alignment,
4034 byval: bool,4055 byval: bool,
4035 param_llvm_ty: Builder.Type,4056 param_ty: Type,
4036 ) Allocator.Error!void {4057 ) Allocator.Error!void {
4058 const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
4059 const alignment = param_ty.abiAlignment(o.zcu).toLlvm();
4037 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);4060 try attributes.addParamAttr(llvm_arg_i, .nonnull, &o.builder);
4038 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);4061 try attributes.addParamAttr(llvm_arg_i, .readonly, &o.builder);
4039 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);4062 try attributes.addParamAttr(llvm_arg_i, .{ .@"align" = .wrap(alignment) }, &o.builder);
4040 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = param_llvm_ty }, &o.builder);4063 if (byval) try attributes.addParamAttr(llvm_arg_i, .{ .byval = llvm_param_ty }, &o.builder);
4041 }4064 }
40424065
4043 pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {4066 pub fn getErrorNameTable(o: *Object) Allocator.Error!Builder.Variable.Index {
...@@ -4062,7 +4085,7 @@ pub const Object = struct {...@@ -4062,7 +4085,7 @@ pub const Object = struct {
4062 pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {4085 pub fn getErrorsLen(o: *Object) Allocator.Error!Builder.Variable.Index {
4063 const builder = &o.builder;4086 const builder = &o.builder;
4064 if (o.errors_len_variable == .none) {4087 if (o.errors_len_variable == .none) {
4065 const llvm_err_int_ty = try o.errorIntType();4088 const llvm_err_int_ty = try o.errorIntType(.in_memory);
4066 const name = try builder.strtabString("__zig_errors_len");4089 const name = try builder.strtabString("__zig_errors_len");
4067 const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);4090 const variable_index = try builder.addVariable(name, llvm_err_int_ty, .default);
4068 variable_index.setMutability(.constant, builder);4091 variable_index.setMutability(.constant, builder);
...@@ -4102,9 +4125,9 @@ pub const Object = struct {...@@ -4102,9 +4125,9 @@ pub const Object = struct {
4102 const ip = &zcu.intern_pool;4125 const ip = &zcu.intern_pool;
4103 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());4126 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
41044127
4105 const llvm_usize_ty = try o.lowerType(.usize);4128 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
4106 const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0);4129 const llvm_ret_ty = try o.lowerType(.slice_const_u8_sentinel_0, .by_value);
4107 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));4130 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
41084131
4109 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =4132 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4110 try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);4133 try o.builder.fnType(llvm_ret_ty, &.{llvm_int_ty}, .normal);
...@@ -4153,7 +4176,7 @@ pub const Object = struct {...@@ -4153,7 +4176,7 @@ pub const Object = struct {
4153 const return_block = try wip.block(1, "Name");4176 const return_block = try wip.block(1, "Name");
4154 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {4177 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, field_index)) {
4155 .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered4178 .none => try o.builder.intConst(llvm_int_ty, field_index), // auto-numbered
4156 else => |tag_val_ip| try o.lowerValue(tag_val_ip),4179 else => |tag_val_ip| try o.lowerValue(tag_val_ip, .by_value),
4157 };4180 };
4158 try wip_switch.addCase(llvm_tag_val, return_block, &wip);4181 try wip_switch.addCase(llvm_tag_val, return_block, &wip);
41594182
...@@ -4199,7 +4222,7 @@ pub const Object = struct {...@@ -4199,7 +4222,7 @@ pub const Object = struct {
4199 const ip = &zcu.intern_pool;4222 const ip = &zcu.intern_pool;
4200 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());4223 const loaded_enum = ip.loadEnumType(enum_ty.toIntern());
42014224
4202 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type));4225 const llvm_int_ty = try o.lowerType(.fromInterned(loaded_enum.int_tag_type), .by_value);
4203 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =4226 function_index.ptrConst(&o.builder).global.ptr(&o.builder).type =
4204 try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);4227 try o.builder.fnType(.i1, &.{llvm_int_ty}, .normal);
42054228
...@@ -4226,7 +4249,7 @@ pub const Object = struct {...@@ -4226,7 +4249,7 @@ pub const Object = struct {
42264249
4227 if (loaded_enum.field_values.len > 0) {4250 if (loaded_enum.field_values.len > 0) {
4228 for (loaded_enum.field_values.get(ip)) |tag_val_ip| {4251 for (loaded_enum.field_values.get(ip)) |tag_val_ip| {
4229 const llvm_tag_val = try o.lowerValue(tag_val_ip);4252 const llvm_tag_val = try o.lowerValue(tag_val_ip, .by_value);
4230 try wip_switch.addCase(llvm_tag_val, named_block, &wip);4253 try wip_switch.addCase(llvm_tag_val, named_block, &wip);
4231 }4254 }
4232 } else {4255 } else {
src/codegen/llvm/FuncGen.zig+467-579
...@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {...@@ -164,7 +164,7 @@ fn resolveValue(self: *FuncGen, val: Value) Allocator.Error!Builder.Constant {
164 const zcu = o.zcu;164 const zcu = o.zcu;
165 const ty = val.typeOf(zcu);165 const ty = val.typeOf(zcu);
166 if (!isByRef(ty, zcu)) {166 if (!isByRef(ty, zcu)) {
167 return o.lowerValue(val.toIntern());167 return o.lowerValue(val.toIntern(), .by_value);
168 } else {168 } else {
169 // We need a pointer to a global constant, i.e. a UAV.169 // We need a pointer to a global constant, i.e. a UAV.
170 return o.lowerUavRef(170 return o.lowerUavRef(
...@@ -193,12 +193,13 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {...@@ -193,12 +193,13 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
193 var it = iterateParamTypes(o, fn_info);193 var it = iterateParamTypes(o, fn_info);
194194
195 // Populate `fg.ret_ptr`...195 // Populate `fg.ret_ptr`...
196 if (firstParamSRet(fn_info, zcu, zcu.getTarget())) {196 fg.ret_ptr = switch (try fnReturnStrat(o, fn_info)) {
197 fg.ret_ptr = fg.wip.arg(it.llvm_index);197 .sret => rp: {
198 it.llvm_index += 1;198 defer it.llvm_index += 1;
199 } else {199 break :rp fg.wip.arg(it.llvm_index);
200 fg.ret_ptr = .none;200 },
201 }201 else => .none,
202 };
202 // ...and `fg.err_ret_trace`...203 // ...and `fg.err_ret_trace`...
203 if (fn_info.cc == .auto and comp.config.any_error_tracing) {204 if (fn_info.cc == .auto and comp.config.any_error_tracing) {
204 fg.err_ret_trace = fg.wip.arg(it.llvm_index);205 fg.err_ret_trace = fg.wip.arg(it.llvm_index);
...@@ -224,7 +225,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {...@@ -224,7 +225,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
224225
225 if (isByRef(param_ty, zcu)) {226 if (isByRef(param_ty, zcu)) {
226 const alignment = param_ty.abiAlignment(zcu).toLlvm();227 const alignment = param_ty.abiAlignment(zcu).toLlvm();
227 const arg_ptr = try fg.buildAlloca(try o.lowerType(param_ty), alignment);228 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
228 // We don't need to handle non-ABI-sized integer types in memory here since they229 // We don't need to handle non-ABI-sized integer types in memory here since they
229 // are never by-ref.230 // are never by-ref.
230 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);231 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
...@@ -248,9 +249,8 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {...@@ -248,9 +249,8 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
248 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);249 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
249 const param = fg.wip.arg(it.llvm_index - 1);250 const param = fg.wip.arg(it.llvm_index - 1);
250251
251 const param_llvm_ty = try o.lowerType(param_ty);
252 const alignment = param_ty.abiAlignment(zcu).toLlvm();252 const alignment = param_ty.abiAlignment(zcu).toLlvm();
253 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);253 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
254 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);254 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
255255
256 if (isByRef(param_ty, zcu)) {256 if (isByRef(param_ty, zcu)) {
...@@ -264,7 +264,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {...@@ -264,7 +264,7 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
264 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);264 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
265 assert(!isByRef(param_ty, zcu));265 assert(!isByRef(param_ty, zcu));
266 const slice_val = try fg.wip.buildAggregate(266 const slice_val = try fg.wip.buildAggregate(
267 try o.lowerType(param_ty),267 try o.lowerType(param_ty, .by_value),
268 &.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },268 &.{ fg.wip.arg(it.llvm_index - 2), fg.wip.arg(it.llvm_index - 1) },
269 "",269 "",
270 );270 );
...@@ -291,11 +291,10 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {...@@ -291,11 +291,10 @@ pub fn genMainBody(fg: *FuncGen) TodoError!void {
291 },291 },
292 .float_array => {292 .float_array => {
293 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);293 const param_ty: Type = .fromInterned(param_types[it.zig_index - 1]);
294 const param_llvm_ty = try o.lowerType(param_ty);
295 const param = fg.wip.arg(it.llvm_index - 1);294 const param = fg.wip.arg(it.llvm_index - 1);
296295
297 const alignment = param_ty.abiAlignment(zcu).toLlvm();296 const alignment = param_ty.abiAlignment(zcu).toLlvm();
298 const arg_ptr = try fg.buildAlloca(param_llvm_ty, alignment);297 const arg_ptr = try fg.buildZigAlloca(param_ty, .none);
299 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);298 _ = try fg.wip.store(.normal, param, arg_ptr, alignment);
300299
301 if (isByRef(param_ty, zcu)) {300 if (isByRef(param_ty, zcu)) {
...@@ -349,13 +348,13 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov...@@ -349,13 +348,13 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
349 try fuzz.pcs.append(gpa, pc);348 try fuzz.pcs.append(gpa, pc);
350 },349 },
351 }350 }
352 for (body, 0..) |inst, i| {351 for (body) |inst| {
353 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;352 if (self.liveness.isUnused(inst) and !self.air.mustLower(inst, ip)) continue;
354353
355 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {354 const val: Builder.Value = switch (air_tags[@intFromEnum(inst)]) {
356 // zig fmt: off355 // zig fmt: off
357356
358 // Required due to `.scalarize_bitcast_vector_non_elementwise` being enabled.357 // Required due to `.scalarize_bit_cast_vector_non_elementwise` being enabled.
359 .legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),358 .legalize_vec_elem_val => try self.airLegalizeVecElemVal(inst),
360 .legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),359 .legalize_vec_store_elem => try self.airLegalizeVecStoreElem(inst),
361360
...@@ -461,29 +460,36 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov...@@ -461,29 +460,36 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
461 .is_err => try self.airIsErr(inst, .ne, false),460 .is_err => try self.airIsErr(inst, .ne, false),
462 .is_err_ptr => try self.airIsErr(inst, .ne, true),461 .is_err_ptr => try self.airIsErr(inst, .ne, true),
463462
464 .alloc => try self.airAlloc(inst),463 .alloc => try self.airAlloc(inst),
465 .ret_ptr => try self.airRetPtr(inst),464 .ret_ptr => try self.airRetPtr(inst),
466 .arg => try self.airArg(inst),465 .arg => try self.airArg(inst),
467 .bitcast => try self.airBitCast(inst),466 .bit_cast => try self.airBitCast(inst),
468 .breakpoint => try self.airBreakpoint(inst),467 .ptr_cast => try self.airNopCast(inst),
469 .ret_addr => try self.airRetAddr(inst),468 .ptr_from_int => try self.airPtrFromInt(inst),
470 .frame_addr => try self.airFrameAddress(inst),469 .int_from_ptr => try self.airIntFromPtr(inst),
471 .@"try" => try self.airTry(inst, false),470 .error_cast => try self.airNopCast(inst),
472 .try_cold => try self.airTry(inst, true),471 .error_from_int => try self.airNopCast(inst),
473 .try_ptr => try self.airTryPtr(inst, false),472 .int_from_error => try self.airNopCast(inst),
474 .try_ptr_cold => try self.airTryPtr(inst, true),473 .union_from_enum => try self.airUnionFromEnum(inst),
475 .intcast => try self.airIntCast(inst, false),474 .breakpoint => try self.airBreakpoint(inst),
476 .intcast_safe => try self.airIntCast(inst, true),475 .ret_addr => try self.airRetAddr(inst),
477 .trunc => try self.airTrunc(inst),476 .frame_addr => try self.airFrameAddress(inst),
478 .fptrunc => try self.airFptrunc(inst),477 .@"try" => try self.airTry(inst, false),
479 .fpext => try self.airFpext(inst),478 .try_cold => try self.airTry(inst, true),
480 .load => try self.airLoad(inst),479 .try_ptr => try self.airTryPtr(inst, false),
481 .not => try self.airNot(inst),480 .try_ptr_cold => try self.airTryPtr(inst, true),
482 .store => try self.airStore(inst, false),481 .int_cast => try self.airIntCast(inst, false),
483 .store_safe => try self.airStore(inst, true),482 .int_cast_safe => try self.airIntCast(inst, true),
484 .assembly => try self.airAssembly(inst),483 .trunc => try self.airTrunc(inst),
485 .slice_ptr => try self.airSliceField(inst, 0),484 .fptrunc => try self.airFptrunc(inst),
486 .slice_len => try self.airSliceField(inst, 1),485 .fpext => try self.airFpext(inst),
486 .load => try self.airLoad(inst),
487 .not => try self.airNot(inst),
488 .store => try self.airStore(inst, false),
489 .store_safe => try self.airStore(inst, true),
490 .assembly => try self.airAssembly(inst),
491 .slice_ptr => try self.airSliceField(inst, 0),
492 .slice_len => try self.airSliceField(inst, 1),
487493
488 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),494 .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0),
489 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),495 .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1),
...@@ -561,9 +567,9 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov...@@ -561,9 +567,9 @@ fn genBody(self: *FuncGen, body: []const Air.Inst.Index, coverage_point: Air.Cov
561 .set_err_return_trace => try self.airSetErrReturnTrace(inst),567 .set_err_return_trace => try self.airSetErrReturnTrace(inst),
562 .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),568 .save_err_return_trace_index => try self.airSaveErrReturnTraceIndex(inst),
563569
564 .wrap_optional => try self.airWrapOptional(body[i..]),570 .wrap_optional => try self.airWrapOptional(inst),
565 .wrap_errunion_payload => try self.airWrapErrUnionPayload(body[i..]),571 .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst),
566 .wrap_errunion_err => try self.airWrapErrUnionErr(body[i..]),572 .wrap_errunion_err => try self.airWrapErrUnionErr(inst),
567573
568 .wasm_memory_size => try self.airWasmMemorySize(inst),574 .wasm_memory_size => try self.airWasmMemorySize(inst),
569 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),575 .wasm_memory_grow => try self.airWasmMemoryGrow(inst),
...@@ -746,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -746,7 +752,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
746 break :llvm_fn try self.resolveInst(air_call.callee);752 break :llvm_fn try self.resolveInst(air_call.callee);
747 };753 };
748 const target = zcu.getTarget();754 const target = zcu.getTarget();
749 const sret = firstParamSRet(fn_info, zcu, target);755 const ret_strat = try fnReturnStrat(o, fn_info);
750756
751 var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);757 var llvm_args = std.array_list.Managed(Builder.Value).init(self.gpa);
752 defer llvm_args.deinit();758 defer llvm_args.deinit();
...@@ -764,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -764,20 +770,21 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
764 .no_suspend, .always_inline, .compile_time => unreachable,770 .no_suspend, .always_inline, .compile_time => unreachable,
765 }771 }
766772
767 const ret_ptr = if (sret) ret_ptr: {773 const sret_alloc: ?Builder.Value = switch (ret_strat) {
768 const llvm_ret_ty = try o.lowerType(return_type);774 .sret => sret_alloc: {
769 try attributes.addParamAttr(0, .{ .sret = llvm_ret_ty }, &o.builder);775 try attributes.addParamAttr(0, .{ .sret = try o.lowerType(return_type, .in_memory) }, &o.builder);
770776
771 const alignment = return_type.abiAlignment(zcu).toLlvm();777 const ptr = try self.buildZigAlloca(return_type, .none);
772 const ret_ptr = try self.buildAlloca(llvm_ret_ty, alignment);778 try llvm_args.append(ptr);
773 try llvm_args.append(ret_ptr);779 break :sret_alloc ptr;
774 break :ret_ptr ret_ptr;780 },
775 } else ret_ptr: {781 else => sret_alloc: {
776 if (ccAbiPromoteInt(fn_info.cc, zcu, Type.fromInterned(fn_info.return_type))) |s| switch (s) {782 if (ccAbiPromoteInt(fn_info.cc, zcu, .fromInterned(fn_info.return_type))) |s| switch (s) {
777 .signed => try attributes.addRetAttr(.signext, &o.builder),783 .signed => try attributes.addRetAttr(.signext, &o.builder),
778 .unsigned => try attributes.addRetAttr(.zeroext, &o.builder),784 .unsigned => try attributes.addRetAttr(.zeroext, &o.builder),
779 };785 };
780 break :ret_ptr null;786 break :sret_alloc null;
787 },
781 };788 };
782789
783 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;790 const err_return_tracing = fn_info.cc == .auto and zcu.comp.config.any_error_tracing;
...@@ -793,11 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -793,11 +800,11 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
793 const arg = args[it.zig_index - 1];800 const arg = args[it.zig_index - 1];
794 const param_ty = self.typeOf(arg);801 const param_ty = self.typeOf(arg);
795 const llvm_arg = try self.resolveInst(arg);802 const llvm_arg = try self.resolveInst(arg);
796 const llvm_param_ty = try o.lowerType(param_ty);
797 if (isByRef(param_ty, zcu)) {803 if (isByRef(param_ty, zcu)) {
798 const alignment = param_ty.abiAlignment(zcu).toLlvm();804 const alignment = param_ty.abiAlignment(zcu).toLlvm();
799 // We don't need to handle non-ABI-sized integer types in memory here since they are805 // We don't need to handle non-ABI-sized integer types in memory here since they are
800 // never by-ref.806 // never by-ref.
807 const llvm_param_ty = try o.lowerType(param_ty, .in_memory);
801 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");808 const loaded = try self.wip.load(.normal, llvm_param_ty, llvm_arg, alignment, "");
802 try llvm_args.append(loaded);809 try llvm_args.append(loaded);
803 } else {810 } else {
...@@ -811,9 +818,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -811,9 +818,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
811 if (isByRef(param_ty, zcu)) {818 if (isByRef(param_ty, zcu)) {
812 try llvm_args.append(llvm_arg);819 try llvm_args.append(llvm_arg);
813 } else {820 } else {
814 const alignment = param_ty.abiAlignment(zcu).toLlvm();821 const arg_ptr = try self.buildZigAlloca(param_ty, .none);
815 const param_llvm_ty = llvm_arg.typeOfWip(&self.wip);
816 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
817 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);822 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
818 try llvm_args.append(arg_ptr);823 try llvm_args.append(arg_ptr);
819 }824 }
...@@ -823,9 +828,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -823,9 +828,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
823 const param_ty = self.typeOf(arg);828 const param_ty = self.typeOf(arg);
824 const llvm_arg = try self.resolveInst(arg);829 const llvm_arg = try self.resolveInst(arg);
825830
826 const alignment = param_ty.abiAlignment(zcu).toLlvm();831 const arg_ptr = try self.buildZigAlloca(param_ty, .none);
827 const param_llvm_ty = try o.lowerType(param_ty);
828 const arg_ptr = try self.buildAlloca(param_llvm_ty, alignment);
829 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);832 try self.store(arg_ptr, .none, llvm_arg, param_ty, .normal);
830 try llvm_args.append(arg_ptr);833 try llvm_args.append(arg_ptr);
831 },834 },
...@@ -877,18 +880,16 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -877,18 +880,16 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
877 const arg_ty = self.typeOf(arg);880 const arg_ty = self.typeOf(arg);
878 const arg_val = try self.resolveInst(arg);881 const arg_val = try self.resolveInst(arg);
879882
880 const arg_align = arg_ty.abiAlignment(zcu);
881
882 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {883 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
883 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());884 const ptr = try self.buildZigAlloca(arg_ty, .none);
884 try self.store(ptr, .none, arg_val, arg_ty, .normal);885 try self.store(ptr, .none, arg_val, arg_ty, .normal);
885 break :ptr ptr;886 break :ptr ptr;
886 } else arg_val;887 } else arg_val;
887888
888 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?);889 const float_ty = try o.lowerType(aarch64_c_abi.getFloatArrayType(arg_ty, zcu).?, .in_memory);
889 const array_ty = try o.builder.arrayType(count, float_ty);890 const array_ty = try o.builder.arrayType(count, float_ty);
890891
891 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");892 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
892 try llvm_args.append(loaded);893 try llvm_args.append(loaded);
893 },894 },
894 .i32_array, .i64_array => |arr_len| {895 .i32_array, .i64_array => |arr_len| {
...@@ -897,16 +898,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -897,16 +898,14 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
897 const arg_ty = self.typeOf(arg);898 const arg_ty = self.typeOf(arg);
898 const arg_val = try self.resolveInst(arg);899 const arg_val = try self.resolveInst(arg);
899900
900 const arg_align = arg_ty.abiAlignment(zcu);
901
902 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {901 const arg_ptr: Builder.Value = if (!isByRef(arg_ty, zcu)) ptr: {
903 const ptr = try self.buildAlloca(try o.lowerType(arg_ty), arg_align.toLlvm());902 const ptr = try self.buildZigAlloca(arg_ty, .none);
904 try self.store(ptr, .none, arg_val, arg_ty, .normal);903 try self.store(ptr, .none, arg_val, arg_ty, .normal);
905 break :ptr ptr;904 break :ptr ptr;
906 } else arg_val;905 } else arg_val;
907906
908 const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));907 const array_ty = try o.builder.arrayType(arr_len, try o.builder.intType(@intCast(elem_size)));
909 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_align.toLlvm(), "");908 const loaded = try self.wip.load(.normal, array_ty, arg_ptr, arg_ty.abiAlignment(zcu).toLlvm(), "");
910 try llvm_args.append(loaded);909 try llvm_args.append(loaded);
911 },910 },
912 };911 };
...@@ -916,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -916,7 +915,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
916 {915 {
917 // Add argument attributes.916 // Add argument attributes.
918 it = iterateParamTypes(o, fn_info);917 it = iterateParamTypes(o, fn_info);
919 it.llvm_index += @intFromBool(sret);918 it.llvm_index += @intFromBool(ret_strat == .sret);
920 it.llvm_index += @intFromBool(err_return_tracing);919 it.llvm_index += @intFromBool(err_return_tracing);
921 var remaining_inreg_int = cc_info.inreg_int_params;920 var remaining_inreg_int = cc_info.inreg_int_params;
922 var remaining_inreg_float = cc_info.inreg_float_params;921 var remaining_inreg_float = cc_info.inreg_float_params;
...@@ -945,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -945,10 +944,8 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
945 },944 },
946 .byref => {945 .byref => {
947 const param_index = it.zig_index - 1;946 const param_index = it.zig_index - 1;
948 const param_ty = Type.fromInterned(fn_info.param_types.get(ip)[param_index]);947 const param_ty: Type = .fromInterned(fn_info.param_types.get(ip)[param_index]);
949 const param_llvm_ty = try o.lowerType(param_ty);948 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, it.byval_attr, param_ty);
950 const alignment = param_ty.abiAlignment(zcu).toLlvm();
951 try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty);
952 },949 },
953 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),950 .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder),
954 // No attributes needed for these.951 // No attributes needed for these.
...@@ -998,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -998,7 +995,7 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
998 },995 },
999 cc_info.llvm_cc,996 cc_info.llvm_cc,
1000 try attributes.finish(&o.builder),997 try attributes.finish(&o.builder),
1001 try o.lowerType(zig_fn_ty),998 try o.lowerType(zig_fn_ty, .by_value),
1002 llvm_fn,999 llvm_fn,
1003 llvm_args.items,1000 llvm_args.items,
1004 "",1001 "",
...@@ -1008,48 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier...@@ -1008,48 +1005,27 @@ fn airCall(self: *FuncGen, inst: Air.Inst.Index, modifier: std.lang.CallModifier
1008 return .none;1005 return .none;
1009 }1006 }
10101007
1011 if (self.liveness.isUnused(inst) or !return_type.hasRuntimeBits(zcu)) {1008 if (self.liveness.isUnused(inst)) {
1012 return .none;1009 return .none;
1013 }1010 }
10141011
1015 const llvm_ret_ty = try o.lowerType(return_type);1012 // We exit this `switch` if we have a pointer to the return value.
1016 if (ret_ptr) |rp| {1013 const ret_val_ptr: Builder.Value = switch (ret_strat) {
1017 if (isByRef(return_type, zcu)) {1014 .void => return .none,
1018 return rp;1015 .by_val => return call,
1019 } else {
1020 // our by-ref status disagrees with sret so we must load.
1021 return self.load(rp, .none, return_type, .normal);
1022 }
1023 }
1024
1025 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1026
1027 if (abi_ret_ty != llvm_ret_ty) {
1028 // In this case the function return type is honoring the calling convention by having
1029 // a different LLVM type than the usual one. We solve this here at the callsite
1030 // by using our canonical type, then loading it if necessary.
1031 const alignment = return_type.abiAlignment(zcu).toLlvm();
1032 const rp = try self.buildAlloca(abi_ret_ty, alignment);
1033 // We don't need to handle non-ABI-sized integer types in memory here since they can only be
1034 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal
1035 // `llvm_ret_ty` anyway.
1036 _ = try self.wip.store(.normal, call, rp, alignment);
1037 return if (isByRef(return_type, zcu))
1038 rp
1039 else
1040 try self.load(rp, .none, return_type, .normal);
1041 }
10421016
1017 .sret => sret_alloc.?,
1018 .mem_cast => |llvm_ret_ty| ret_val_ptr: {
1019 const alignment = return_type.abiAlignment(zcu).toLlvm();
1020 const ptr = try self.buildAlloca(llvm_ret_ty, alignment);
1021 _ = try self.wip.store(.normal, call, ptr, alignment);
1022 break :ret_val_ptr ptr;
1023 },
1024 };
1043 if (isByRef(return_type, zcu)) {1025 if (isByRef(return_type, zcu)) {
1044 // our by-ref status disagrees with sret so we must allocate, store,1026 return ret_val_ptr;
1045 // and return the allocation pointer.
1046 const alignment = return_type.abiAlignment(zcu).toLlvm();
1047 const rp = try self.buildAlloca(llvm_ret_ty, alignment);
1048 // We don't need to handle non-ABI-sized integer types here since they are never by-ref.
1049 _ = try self.wip.store(.normal, call, rp, alignment);
1050 return rp;
1051 } else {1027 } else {
1052 return call;1028 return self.load(ret_val_ptr, .none, return_type, .normal);
1053 }1029 }
1054}1030}
10551031
...@@ -1059,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v...@@ -1059,7 +1035,7 @@ fn buildSimplePanic(fg: *FuncGen, panic_id: Zcu.SimplePanicId) Allocator.Error!v
1059 const target = zcu.getTarget();1035 const target = zcu.getTarget();
1060 const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));1036 const panic_func = zcu.funcInfo(zcu.std_lang_decl_values.get(panic_id.toStdLangDecl()));
1061 const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;1037 const fn_info = zcu.typeToFunc(.fromInterned(panic_func.ty)).?;
1062 const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty));1038 const llvm_panic_fn_ty = try o.lowerType(.fromInterned(panic_func.ty), .by_value);
10631039
1064 const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);1040 const llvm_panic_fn_ref = try o.lowerNavRef(panic_func.owner_nav);
10651041
...@@ -1083,64 +1059,21 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -1083,64 +1059,21 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1083 const zcu = o.zcu;1059 const zcu = o.zcu;
1084 const ip = &zcu.intern_pool;1060 const ip = &zcu.intern_pool;
1085 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;1061 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
1086 const ret_ty = self.typeOf(un_op);
1087
1088 if (self.ret_ptr != .none) {
1089 const operand = try self.resolveInst(un_op);
1090 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
1091 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1092 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));
1093 _ = try self.wip.callMemSet(
1094 self.ret_ptr,
1095 ret_ty.abiAlignment(zcu).toLlvm(),
1096 try o.builder.intValue(.i8, 0xaa),
1097 len,
1098 .normal,
1099 self.disable_intrinsics,
1100 );
1101 const owner_mod = self.ownerModule();
1102 if (owner_mod.valgrind) {
1103 try self.valgrindMarkUndef(self.ret_ptr, len);
1104 }
1105 _ = try self.wip.retVoid();
1106 return;
1107 }
1108
1109 const unwrapped_operand = operand.unwrap();
1110 const unwrapped_ret = self.ret_ptr.unwrap();
11111062
1112 // Return value was stored previously1063 const ret_ty = self.typeOf(un_op);
1113 if (unwrapped_operand == .instruction and unwrapped_ret == .instruction and unwrapped_operand.instruction == unwrapped_ret.instruction) {
1114 _ = try self.wip.retVoid();
1115 return;
1116 }
11171064
1118 try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
1119 _ = try self.wip.retVoid();
1120 return;
1121 }
1122 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;1065 const fn_info = zcu.typeToFunc(Type.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1123 if (!ret_ty.hasRuntimeBits(zcu)) {
1124 if (Type.fromInterned(fn_info.return_type).isError(zcu)) {
1125 // Functions with an empty error set are emitted with an error code
1126 // return type and return zero so they can be function pointers coerced
1127 // to functions that return anyerror.
1128 _ = try self.wip.ret(try o.builder.intValue(try o.errorIntType(), 0));
1129 } else {
1130 _ = try self.wip.retVoid();
1131 }
1132 return;
1133 }
11341066
1135 const llvm_ret_ty = try o.lowerType(ret_ty);1067 const ret_strat = try fnReturnStrat(o, fn_info);
1136 const abi_ret_ty = try lowerFnRetTy(o, fn_info);
1137 const operand = try self.resolveInst(un_op);
1138 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;1068 const val_is_undef = if (un_op.toInterned()) |i| Value.fromInterned(i).isUndef(zcu) else false;
1139 const ret_ty_align = ret_ty.abiAlignment(zcu);1069 const ret_ty_align = ret_ty.abiAlignment(zcu);
11401070
1141 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {1071 if (val_is_undef and safety and !self.needMemsetWorkaround(ret_ty.abiSize(zcu))) {
1142 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());1072 const rp = switch (self.ret_ptr) {
1143 const len = try o.builder.intValue(try o.lowerType(.usize), ret_ty.abiSize(zcu));1073 .none => try self.buildZigAlloca(ret_ty, .none),
1074 else => |rp| rp,
1075 };
1076 const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), ret_ty.abiSize(zcu));
1144 _ = try self.wip.callMemSet(1077 _ = try self.wip.callMemSet(
1145 rp,1078 rp,
1146 ret_ty_align.toLlvm(),1079 ret_ty_align.toLlvm(),
...@@ -1153,36 +1086,46 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo...@@ -1153,36 +1086,46 @@ fn airRet(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!vo
1153 if (owner_mod.valgrind) {1086 if (owner_mod.valgrind) {
1154 try self.valgrindMarkUndef(rp, len);1087 try self.valgrindMarkUndef(rp, len);
1155 }1088 }
1156 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {1089 switch (ret_strat) {
1157 assert(!isByRef(ret_ty, zcu));1090 .void => unreachable, // value is undef so return type cannot be OPV
1158 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure1091 .sret => {
1159 // we load it from memory correctly.1092 // We just stored directly to `self.ret_ptr`.
1160 const loaded = try self.load(rp, .none, ret_ty, .normal);1093 _ = try self.wip.retVoid();
1161 _ = try self.wip.ret(loaded);1094 },
1162 } else {1095 .by_val => {
1163 const loaded = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");1096 const loaded = try self.load(rp, .none, ret_ty, .normal);
1164 _ = try self.wip.ret(loaded);1097 _ = try self.wip.ret(loaded);
1098 },
1099 .mem_cast => |llvm_abi_ret_ty| {
1100 const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, rp, ret_ty_align.toLlvm(), "");
1101 _ = try self.wip.ret(loaded);
1102 },
1165 }1103 }
1166 return;1104 return;
1167 }1105 }
11681106
1169 if (isByRef(ret_ty, zcu)) {1107 switch (ret_strat) {
1170 // operand is a pointer however self.ret_ptr is null so that means we need to return a value.1108 .void => _ = try self.wip.retVoid(),
1171 // No need to handle non-ABI-sized integer types in memory here since they are never by-ref.1109 .sret => {
1172 _ = try self.wip.ret(try self.wip.load(.normal, abi_ret_ty, operand, ret_ty_align.toLlvm(), ""));1110 const operand = try self.resolveInst(un_op);
1173 return;1111 try self.store(self.ret_ptr, .none, operand, ret_ty, .normal);
1174 }1112 _ = try self.wip.retVoid();
11751113 },
1176 if (abi_ret_ty == llvm_ret_ty) {1114 .by_val => {
1177 _ = try self.wip.ret(operand);1115 assert(!isByRef(ret_ty, zcu));
1178 } else {1116 const operand = try self.resolveInst(un_op);
1179 const rp = try self.buildAlloca(llvm_ret_ty, ret_ty_align.toLlvm());1117 _ = try self.wip.ret(operand);
1180 try self.store(rp, .none, operand, ret_ty, .normal);1118 },
1181 // No need to handle non-ABI-sized integer types in memory here since they can only be1119 .mem_cast => |llvm_ret_ty| {
1182 // returned from `CallingConvention.auto` functions, in which case `abi_ret_ty` will equal1120 const operand = try self.resolveInst(un_op);
1183 // `llvm_ret_ty` anyway.1121 const ptr: Builder.Value = if (!isByRef(ret_ty, zcu)) ptr: {
1184 const ret_val = try self.wip.load(.normal, abi_ret_ty, rp, ret_ty_align.toLlvm(), "");1122 const ptr = try self.buildZigAlloca(ret_ty, .none);
1185 _ = try self.wip.ret(ret_val);1123 try self.store(ptr, .none, operand, ret_ty, .normal);
1124 break :ptr ptr;
1125 } else operand;
1126 const ret_val = try self.wip.load(.normal, llvm_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1127 _ = try self.wip.ret(ret_val);
1128 },
1186 }1129 }
1187}1130}
11881131
...@@ -1194,23 +1137,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {...@@ -1194,23 +1137,24 @@ fn airRetLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!void {
1194 const ptr_ty = self.typeOf(un_op);1137 const ptr_ty = self.typeOf(un_op);
1195 const ret_ty = ptr_ty.childType(zcu);1138 const ret_ty = ptr_ty.childType(zcu);
1196 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;1139 const fn_info = zcu.typeToFunc(.fromInterned(ip.getNav(self.nav_index).resolved.?.type)).?;
1197 if (!ret_ty.hasRuntimeBits(zcu) or self.ret_ptr != .none) {
1198 _ = try self.wip.retVoid();
1199 return;
1200 }
1201 const ptr = try self.resolveInst(un_op);1140 const ptr = try self.resolveInst(un_op);
1202 const llvm_ret_ty = try o.lowerType(ret_ty);1141 switch (try fnReturnStrat(o, fn_info)) {
1203 const abi_ret_ty = try lowerFnRetTy(o, fn_info);1142 .void => _ = try self.wip.retVoid(),
1204 if (fn_info.cc == .auto and abi_ret_ty == llvm_ret_ty) {1143 .sret => {
1205 assert(!isByRef(ret_ty, zcu));1144 assert(self.ret_ptr != .none);
1206 // The return type could be a non-ABI-sized integer, so use `FuncGen.load` to make sure we1145 _ = try self.wip.retVoid();
1207 // load it from memory correctly.1146 },
1208 const loaded = try self.load(ptr, .none, ret_ty, .normal);1147 .by_val => {
1209 _ = try self.wip.ret(loaded);1148 assert(self.ret_ptr == .none);
1210 } else {1149 const loaded = try self.load(ptr, .none, ret_ty, .normal);
1211 const ret_ty_align = ret_ty.abiAlignment(zcu);1150 _ = try self.wip.ret(loaded);
1212 const loaded = try self.wip.load(.normal, abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");1151 },
1213 _ = try self.wip.ret(loaded);1152 .mem_cast => |llvm_abi_ret_ty| {
1153 assert(self.ret_ptr == .none);
1154 const ret_ty_align = ret_ty.abiAlignment(zcu);
1155 const loaded = try self.wip.load(.normal, llvm_abi_ret_ty, ptr, ret_ty_align.toLlvm(), "");
1156 _ = try self.wip.ret(loaded);
1157 },
1214 }1158 }
1215}1159}
12161160
...@@ -1218,7 +1162,7 @@ fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -1218,7 +1162,7 @@ fn airCVaArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
1218 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1162 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1219 const list = try self.resolveInst(ty_op.operand);1163 const list = try self.resolveInst(ty_op.operand);
1220 const arg_ty = ty_op.ty.toType();1164 const arg_ty = ty_op.ty.toType();
1221 const llvm_arg_ty = try self.object.lowerType(arg_ty);1165 const llvm_arg_ty = try self.object.lowerType(arg_ty, .by_value);
12221166
1223 return self.wip.vaArg(list, llvm_arg_ty, "");1167 return self.wip.vaArg(list, llvm_arg_ty, "");
1224}1168}
...@@ -1229,10 +1173,8 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -1229,10 +1173,8 @@ fn airCVaCopy(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
1229 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1173 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
1230 const src_list = try self.resolveInst(ty_op.operand);1174 const src_list = try self.resolveInst(ty_op.operand);
1231 const va_list_ty = ty_op.ty.toType();1175 const va_list_ty = ty_op.ty.toType();
1232 const llvm_va_list_ty = try o.lowerType(va_list_ty);
12331176
1234 const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();1177 const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1235 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
12361178
1237 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");1179 _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
1238 return if (isByRef(va_list_ty, zcu))1180 return if (isByRef(va_list_ty, zcu))
...@@ -1253,10 +1195,8 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -1253,10 +1195,8 @@ fn airCVaStart(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
1253 const o = self.object;1195 const o = self.object;
1254 const zcu = o.zcu;1196 const zcu = o.zcu;
1255 const va_list_ty = self.typeOfIndex(inst);1197 const va_list_ty = self.typeOfIndex(inst);
1256 const llvm_va_list_ty = try o.lowerType(va_list_ty);
12571198
1258 const result_alignment = va_list_ty.abiAlignment(zcu).toLlvm();1199 const dest_list = try self.buildZigAlloca(va_list_ty, .none);
1259 const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment);
12601200
1261 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");1201 _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
1262 return if (isByRef(va_list_ty, zcu))1202 return if (isByRef(va_list_ty, zcu))
...@@ -1434,18 +1374,10 @@ fn lowerBlock(...@@ -1434,18 +1374,10 @@ fn lowerBlock(
14341374
1435 // Create a phi node only if the block returns a value.1375 // Create a phi node only if the block returns a value.
1436 if (have_block_result) {1376 if (have_block_result) {
1437 const raw_llvm_ty = try o.lowerType(inst_ty);1377 const llvm_ty: Builder.Type = switch (isByRef(inst_ty, zcu)) {
1438 const llvm_ty: Builder.Type = ty: {1378 true => .ptr,
1439 // If the zig tag type is a function, this represents an actual function body; not1379 false => try o.lowerType(inst_ty, .by_value),
1440 // a pointer to it. LLVM IR allows the call instruction to use function bodies instead
1441 // of function pointers, however the phi makes it a runtime value and therefore
1442 // the LLVM type has to be wrapped in a pointer.
1443 if (inst_ty.zigTypeTag(zcu) == .@"fn" or isByRef(inst_ty, zcu)) {
1444 break :ty .ptr;
1445 }
1446 break :ty raw_llvm_ty;
1447 };1380 };
1448
1449 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);1381 parent_bb.ptr(&self.wip).incoming = @intCast(breaks.list.len);
1450 const phi = try self.wip.phi(llvm_ty, "");1382 const phi = try self.wip.phi(llvm_ty, "");
1451 phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);1383 phi.finish(breaks.list.items(.val), breaks.list.items(.bb), &self.wip);
...@@ -1551,7 +1483,7 @@ fn lowerSwitchDispatch(...@@ -1551,7 +1483,7 @@ fn lowerSwitchDispatch(
1551 const table_index = try self.wip.conv(1483 const table_index = try self.wip.conv(
1552 .unsigned,1484 .unsigned,
1553 try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),1485 try self.wip.bin(.@"sub nuw", cond, jmp_table.min.toValue(), ""),
1554 try o.lowerType(.usize),1486 try o.lowerType(.usize, .by_value),
1555 "",1487 "",
1556 );1488 );
1557 const target_ptr_ptr = try self.ptraddScaled(1489 const target_ptr_ptr = try self.ptraddScaled(
...@@ -1576,7 +1508,7 @@ fn lowerSwitchDispatch(...@@ -1576,7 +1508,7 @@ fn lowerSwitchDispatch(
1576 // The switch prongs will correspond to our scalar cases. Ranges will1508 // The switch prongs will correspond to our scalar cases. Ranges will
1577 // be handled by conditional branches in the `else` prong.1509 // be handled by conditional branches in the `else` prong.
15781510
1579 const llvm_usize = try o.lowerType(.usize);1511 const llvm_usize = try o.lowerType(.usize, .by_value);
1580 const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)1512 const cond_int = if (cond_ty.zigTypeTag(zcu) == .pointer)
1581 try self.wip.cast(.ptrtoint, cond, llvm_usize, "")1513 try self.wip.cast(.ptrtoint, cond, llvm_usize, "")
1582 else1514 else
...@@ -1771,7 +1703,6 @@ fn lowerTry(...@@ -1771,7 +1703,6 @@ fn lowerTry(
1771 const zcu = o.zcu;1703 const zcu = o.zcu;
1772 const payload_ty = err_union_ty.errorUnionPayload(zcu);1704 const payload_ty = err_union_ty.errorUnionPayload(zcu);
1773 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);1705 const payload_has_bits = payload_ty.hasRuntimeBits(zcu);
1774 const error_type = try o.errorIntType();
17751706
1776 const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);1707 const operand_align: InternPool.Alignment = if (operand_is_ptr) operand_ptr_align else err_union_ty.abiAlignment(zcu);
17771708
...@@ -1792,7 +1723,7 @@ fn lowerTry(...@@ -1792,7 +1723,7 @@ fn lowerTry(
1792 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,1723 if (err_union_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
1793 );1724 );
1794 };1725 };
1795 const zero = try o.builder.intValue(error_type, 0);1726 const zero = try o.builder.intValue(try o.errorIntType(.by_value), 0);
1796 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");1727 const is_err = try fg.wip.icmp(.ne, loaded, zero, "");
17971728
1798 const return_block = try fg.wip.block(1, "TryRet");1729 const return_block = try fg.wip.block(1, "TryRet");
...@@ -1929,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod...@@ -1929,8 +1860,8 @@ fn airSwitchBr(self: *FuncGen, inst: Air.Inst.Index, is_dispatch_loop: bool) Tod
1929 const table_includes_else = item_count != table_len;1860 const table_includes_else = item_count != table_len;
19301861
1931 break :jmp_table .{1862 break :jmp_table .{
1932 .min = try o.lowerValue(min.toIntern()),1863 .min = try o.lowerValue(min.toIntern(), .by_value),
1933 .max = try o.lowerValue(max.toIntern()),1864 .max = try o.lowerValue(max.toIntern(), .by_value),
1934 .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {1865 .in_bounds_hint = if (table_includes_else) .none else switch (switch_br.getElseHint()) {
1935 .none, .cold => .none,1866 .none, .cold => .none,
1936 .unpredictable => .unpredictable,1867 .unpredictable => .unpredictable,
...@@ -2088,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder...@@ -2088,9 +2019,9 @@ fn airArrayToSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder
2088 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2019 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2089 const operand_ty = self.typeOf(ty_op.operand);2020 const operand_ty = self.typeOf(ty_op.operand);
2090 const array_ty = operand_ty.childType(zcu);2021 const array_ty = operand_ty.childType(zcu);
2091 const llvm_usize = try o.lowerType(.usize);2022 const llvm_usize = try o.lowerType(.usize, .by_value);
2092 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));2023 const len = try o.builder.intValue(llvm_usize, array_ty.arrayLen(zcu));
2093 const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst));2024 const slice_llvm_ty = try o.lowerType(self.typeOfIndex(inst), .by_value);
2094 const operand = try self.resolveInst(ty_op.operand);2025 const operand = try self.resolveInst(ty_op.operand);
2095 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");2026 return self.wip.buildAggregate(slice_llvm_ty, &.{ operand, len }, "");
2096}2027}
...@@ -2107,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value...@@ -2107,7 +2038,7 @@ fn airFloatFromInt(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value
21072038
2108 const dest_ty = self.typeOfIndex(inst);2039 const dest_ty = self.typeOfIndex(inst);
2109 const dest_scalar_ty = dest_ty.scalarType(zcu);2040 const dest_scalar_ty = dest_ty.scalarType(zcu);
2110 const dest_llvm_ty = try o.lowerType(dest_ty);2041 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
2111 const target = zcu.getTarget();2042 const target = zcu.getTarget();
21122043
2113 if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(2044 if (intrinsicsAllowed(dest_scalar_ty, target)) return self.wip.conv(
...@@ -2175,7 +2106,7 @@ fn airIntFromFloat(...@@ -2175,7 +2106,7 @@ fn airIntFromFloat(
21752106
2176 const dest_ty = self.typeOfIndex(inst);2107 const dest_ty = self.typeOfIndex(inst);
2177 const dest_scalar_ty = dest_ty.scalarType(zcu);2108 const dest_scalar_ty = dest_ty.scalarType(zcu);
2178 const dest_llvm_ty = try o.lowerType(dest_ty);2109 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
21792110
2180 if (intrinsicsAllowed(operand_scalar_ty, target)) {2111 if (intrinsicsAllowed(operand_scalar_ty, target)) {
2181 // TODO set fast math flag2112 // TODO set fast math flag
...@@ -2209,7 +2140,7 @@ fn airIntFromFloat(...@@ -2209,7 +2140,7 @@ fn airIntFromFloat(
2209 compiler_rt_dest_abbrev,2140 compiler_rt_dest_abbrev,
2210 });2141 });
22112142
2212 const operand_llvm_ty = try o.lowerType(operand_ty);2143 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
2213 const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);2144 const libc_fn = try o.getLibcFunction(fn_name, &.{operand_llvm_ty}, libc_ret_ty);
2214 var result = try self.wip.call(2145 var result = try self.wip.call(
2215 .normal,2146 .normal,
...@@ -2234,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B...@@ -2234,7 +2165,7 @@ fn sliceOrArrayPtr(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!B
2234fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {2165fn sliceOrArrayLenInBytes(fg: *FuncGen, ptr: Builder.Value, ty: Type) Allocator.Error!Builder.Value {
2235 const o = fg.object;2166 const o = fg.object;
2236 const zcu = o.zcu;2167 const zcu = o.zcu;
2237 const llvm_usize = try o.lowerType(.usize);2168 const llvm_usize = try o.lowerType(.usize, .by_value);
2238 switch (ty.ptrSize(zcu)) {2169 switch (ty.ptrSize(zcu)) {
2239 .slice => {2170 .slice => {
2240 const len = try fg.wip.extractValue(ptr, &.{1}, "");2171 const len = try fg.wip.extractValue(ptr, &.{1}, "");
...@@ -2370,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -2370,9 +2301,6 @@ fn airPtrElemPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
2370 const base_ptr = try self.resolveInst(bin_op.lhs);2301 const base_ptr = try self.resolveInst(bin_op.lhs);
2371 const rhs = try self.resolveInst(bin_op.rhs);2302 const rhs = try self.resolveInst(bin_op.rhs);
23722303
2373 const elem_ptr = ty_pl.ty.toType();
2374 if (elem_ptr.ptrInfo(zcu).flags.vector_index != .none) return base_ptr;
2375
2376 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));2304 return self.ptraddScaled(base_ptr, rhs, elem_ty.abiSize(zcu));
2377}2305}
23782306
...@@ -2435,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2435,7 +2363,7 @@ fn airStructFieldVal(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2435 },2363 },
2436 .float => {2364 .float => {
2437 // bitcast int->float2365 // bitcast int->float
2438 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty), "");2366 return self.wip.cast(.bitcast, field_int_val, try o.lowerType(field_ty, .by_value), "");
2439 },2367 },
2440 }2368 }
2441 }2369 }
...@@ -2465,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -2465,8 +2393,8 @@ fn airFieldParentPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
2465 const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);2393 const field_offset = parent_ty.structFieldOffset(extra.field_index, zcu);
2466 if (field_offset == 0) return field_ptr;2394 if (field_offset == 0) return field_ptr;
24672395
2468 const res_ty = try o.lowerType(ty_pl.ty.toType());2396 const res_ty = try o.lowerType(ty_pl.ty.toType(), .by_value);
2469 const llvm_usize = try o.lowerType(.usize);2397 const llvm_usize = try o.lowerType(.usize, .by_value);
24702398
2471 const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");2399 const field_ptr_int = try self.wip.cast(.ptrtoint, field_ptr, llvm_usize, "");
2472 const base_ptr_int = try self.wip.bin(2400 const base_ptr_int = try self.wip.bin(
...@@ -2590,8 +2518,7 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er...@@ -2590,8 +2518,7 @@ fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index, is_arg: bool) Allocator.Er
2590 // We avoid taking this path for naked functions because there's no guarantee that such2518 // We avoid taking this path for naked functions because there's no guarantee that such
2591 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.2519 // functions even have a valid stack pointer, making the `alloca` + `store` unsafe.
25922520
2593 const alignment = operand_ty.abiAlignment(zcu).toLlvm();2521 const alloca = try self.buildZigAlloca(operand_ty, .none);
2594 const alloca = try self.buildAlloca(try o.lowerType(operand_ty), alignment);
2595 try self.store(alloca, .none, operand, operand_ty, .normal);2522 try self.store(alloca, .none, operand, operand_ty, .normal);
2596 _ = try self.wip.callIntrinsic(2523 _ = try self.wip.callIntrinsic(
2597 .normal,2524 .normal,
...@@ -2683,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2683,7 +2610,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2683 const output_inst = try self.resolveInst(output.operand);2610 const output_inst = try self.resolveInst(output.operand);
2684 const output_ty = self.typeOf(output.operand);2611 const output_ty = self.typeOf(output.operand);
2685 assert(output_ty.zigTypeTag(zcu) == .pointer);2612 assert(output_ty.zigTypeTag(zcu) == .pointer);
2686 const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu));2613 const elem_llvm_ty = try o.lowerType(output_ty.childType(zcu), .by_value);
26872614
2688 switch (constraint[0]) {2615 switch (constraint[0]) {
2689 '=' => {},2616 '=' => {},
...@@ -2721,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2721,7 +2648,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2721 llvm_ret_indirect[output.index] = false;2648 llvm_ret_indirect[output.index] = false;
27222649
2723 const ret_ty = self.typeOfIndex(inst);2650 const ret_ty = self.typeOfIndex(inst);
2724 llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty);2651 llvm_ret_types[llvm_ret_i] = try o.lowerType(ret_ty, .by_value);
2725 llvm_ret_i += 1;2652 llvm_ret_i += 1;
2726 }2653 }
27272654
...@@ -2760,7 +2687,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2760,7 +2687,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2760 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);2687 llvm_param_types[llvm_param_i] = arg_llvm_value.typeOfWip(&self.wip);
2761 } else {2688 } else {
2762 const alignment = arg_ty.abiAlignment(zcu).toLlvm();2689 const alignment = arg_ty.abiAlignment(zcu).toLlvm();
2763 const arg_llvm_ty = try o.lowerType(arg_ty);2690 const arg_llvm_ty = try o.lowerType(arg_ty, .by_value);
2764 const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");2691 const load_inst = try self.wip.load(.normal, arg_llvm_ty, arg_llvm_value, alignment, "");
2765 llvm_param_values[llvm_param_i] = load_inst;2692 llvm_param_values[llvm_param_i] = load_inst;
2766 llvm_param_types[llvm_param_i] = arg_llvm_ty;2693 llvm_param_types[llvm_param_i] = arg_llvm_ty;
...@@ -2800,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2800,7 +2727,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2800 llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {2727 llvm_param_attrs[llvm_param_i] = if (constraint[0] == '*') blk: {
2801 if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));2728 if (!is_by_ref) self.maybeMarkAllowZeroAccess(arg_ty.ptrInfo(zcu));
28022729
2803 break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu));2730 break :blk try o.lowerType(if (is_by_ref) arg_ty else arg_ty.childType(zcu), .by_value);
2804 } else .none;2731 } else .none;
28052732
2806 llvm_param_i += 1;2733 llvm_param_i += 1;
...@@ -2814,7 +2741,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {...@@ -2814,7 +2741,7 @@ fn airAssembly(self: *FuncGen, inst: Air.Inst.Index) TodoError!Builder.Value {
2814 if (constraint[0] != '+') continue;2741 if (constraint[0] != '+') continue;
28152742
2816 const rw_ty = self.typeOf(output.operand);2743 const rw_ty = self.typeOf(output.operand);
2817 const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu));2744 const llvm_elem_ty = try o.lowerType(rw_ty.childType(zcu), .by_value);
2818 if (llvm_ret_indirect[output.index]) {2745 if (llvm_ret_indirect[output.index]) {
2819 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];2746 llvm_param_values[llvm_param_i] = llvm_rw_vals[output.index];
2820 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);2747 llvm_param_types[llvm_param_i] = llvm_rw_vals[output.index].typeOfWip(&self.wip);
...@@ -3028,7 +2955,7 @@ fn airIsNonNull(...@@ -3028,7 +2955,7 @@ fn airIsNonNull(
3028 ));2955 ));
3029 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");2956 return self.wip.icmp(cond, slice_ptr, try o.builder.nullValue(ptr_ty), "");
3030 }2957 }
3031 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty)), "");2958 return self.wip.icmp(cond, loaded, try o.builder.zeroInitValue(try o.lowerType(optional_ty, .by_value)), "");
3032 }2959 }
30332960
3034 comptime assert(optional_layout_version == 3);2961 comptime assert(optional_layout_version == 3);
...@@ -3057,8 +2984,7 @@ fn airIsErr(...@@ -3057,8 +2984,7 @@ fn airIsErr(
3057 const operand_ty = self.typeOf(un_op);2984 const operand_ty = self.typeOf(un_op);
3058 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;2985 const err_union_ty = if (operand_is_ptr) operand_ty.childType(zcu) else operand_ty;
3059 const payload_ty = err_union_ty.errorUnionPayload(zcu);2986 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3060 const error_type = try o.errorIntType();2987 const zero_err = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3061 const zero = try o.builder.intValue(error_type, 0);
30622988
3063 const access_kind: Builder.MemoryAccessKind =2989 const access_kind: Builder.MemoryAccessKind =
3064 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;2990 if (operand_is_ptr and operand_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -3079,7 +3005,7 @@ fn airIsErr(...@@ -3079,7 +3005,7 @@ fn airIsErr(
3079 try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)3005 try self.load(operand, operand_ty.ptrAlignment(zcu), err_union_ty, access_kind)
3080 else3006 else
3081 operand;3007 operand;
3082 return self.wip.icmp(cond, loaded, zero, "");3008 return self.wip.icmp(cond, loaded, zero_err, "");
3083 }3009 }
3084 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref3010 assert(isByRef(err_union_ty, zcu)); // error unions with runtime bits are always by-ref
30853011
...@@ -3089,7 +3015,7 @@ fn airIsErr(...@@ -3089,7 +3015,7 @@ fn airIsErr(
3089 .none;3015 .none;
3090 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));3016 const err_field_ptr = try self.ptraddConst(operand, codegen.errUnionErrorOffset(payload_ty, zcu));
3091 const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);3017 const loaded = try self.load(err_field_ptr, err_align, .anyerror, access_kind);
3092 return self.wip.icmp(cond, loaded, zero, "");3018 return self.wip.icmp(cond, loaded, zero_err, "");
3093}3019}
30943020
3095fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {3021fn airOptionalPayloadPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -3228,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro...@@ -3228,7 +3154,7 @@ fn airErrUnionPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) Allocator.Erro
3228 const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);3154 const err_union_ptr_align = err_union_ptr_ty.ptrAlignment(zcu);
32293155
3230 const payload_ty = err_union_ty.errorUnionPayload(zcu);3156 const payload_ty = err_union_ty.errorUnionPayload(zcu);
3231 const non_error_val = try o.builder.intValue(try o.errorIntType(), 0);3157 const non_error_val = try o.builder.intValue(try o.errorIntType(.by_value), 0);
32323158
3233 const access_kind: Builder.MemoryAccessKind =3159 const access_kind: Builder.MemoryAccessKind =
3234 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;3160 if (err_union_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -3274,33 +3200,9 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er...@@ -3274,33 +3200,9 @@ fn airSaveErrReturnTraceIndex(self: *FuncGen, inst: Air.Inst.Index) Allocator.Er
3274 return self.load(field_ptr, field_align, field_ty, .normal);3200 return self.load(field_ptr, field_align, field_ty, .normal);
3275}3201}
32763202
3277/// As an optimization, we want to avoid unnecessary copies of3203fn airWrapOptional(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3278/// error union/optional types when returning from a function.
3279/// Here, we scan forward in the current block, looking to see
3280/// if the next instruction is a return (ignoring debug instructions).
3281///
3282/// The first instruction of `body_tail` is a wrap instruction.
3283fn isNextRet(
3284 self: *FuncGen,
3285 body_tail: []const Air.Inst.Index,
3286) bool {
3287 const air_tags = self.air.instructions.items(.tag);
3288 for (body_tail[1..]) |body_inst| {
3289 switch (air_tags[@intFromEnum(body_inst)]) {
3290 .ret => return true,
3291 .dbg_stmt => continue,
3292 else => return false,
3293 }
3294 }
3295 // The only way to get here is to hit the end of a loop instruction
3296 // (implicit repeat).
3297 return false;
3298}
3299
3300fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {
3301 const o = self.object;3204 const o = self.object;
3302 const zcu = o.zcu;3205 const zcu = o.zcu;
3303 const inst = body_tail[0];
3304 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3206 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3305 const payload_ty = self.typeOf(ty_op.operand);3207 const payload_ty = self.typeOf(ty_op.operand);
3306 comptime assert(optional_layout_version == 3);3208 comptime assert(optional_layout_version == 3);
...@@ -3309,14 +3211,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator....@@ -3309,14 +3211,7 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3309 const optional_ty = self.typeOfIndex(inst);3211 const optional_ty = self.typeOfIndex(inst);
3310 if (optional_ty.optionalReprIsPayload(zcu)) return operand;3212 if (optional_ty.optionalReprIsPayload(zcu)) return operand;
3311 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`3213 assert(isByRef(optional_ty, zcu)); // optionals with runtime bits are by-ref unless `optionalReprIsPayload`
3312 const llvm_optional_ty = try o.lowerType(optional_ty);3214 const optional_ptr = try self.buildZigAlloca(optional_ty, .none);
3313 const optional_ptr = if (self.isNextRet(body_tail))
3314 self.ret_ptr
3315 else brk: {
3316 const alignment = optional_ty.abiAlignment(zcu).toLlvm();
3317 const optional_ptr = try self.buildAlloca(llvm_optional_ty, alignment);
3318 break :brk optional_ptr;
3319 };
33203215
3321 const payload_ptr = optional_ptr; // payload always at offset 03216 const payload_ptr = optional_ptr; // payload always at offset 0
3322 try self.store(payload_ptr, .none, operand, payload_ty, .normal);3217 try self.store(payload_ptr, .none, operand, payload_ty, .normal);
...@@ -3328,26 +3223,18 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator....@@ -3328,26 +3223,18 @@ fn airWrapOptional(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.
3328 return optional_ptr;3223 return optional_ptr;
3329}3224}
33303225
3331fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {3226fn airWrapErrUnionPayload(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3332 const o = self.object;3227 const o = self.object;
3333 const zcu = o.zcu;3228 const zcu = o.zcu;
3334 const inst = body_tail[0];
3335 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3229 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3336 const err_un_ty = self.typeOfIndex(inst);3230 const err_un_ty = self.typeOfIndex(inst);
3337 const operand = try self.resolveInst(ty_op.operand);3231 const operand = try self.resolveInst(ty_op.operand);
3338 const payload_ty = self.typeOf(ty_op.operand);3232 const payload_ty = self.typeOf(ty_op.operand);
3339 assert(payload_ty.hasRuntimeBits(zcu));3233 assert(payload_ty.hasRuntimeBits(zcu));
3340 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref3234 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3341 const ok_err_code = try o.builder.intValue(try o.errorIntType(), 0);3235 const ok_err_code = try o.builder.intValue(try o.errorIntType(.by_value), 0);
3342 const err_un_llvm_ty = try o.lowerType(err_un_ty);3236
33433237 const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3344 const result_ptr = if (self.isNextRet(body_tail))
3345 self.ret_ptr
3346 else brk: {
3347 const alignment = err_un_ty.abiAlignment(o.zcu).toLlvm();
3348 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3349 break :brk result_ptr;
3350 };
33513238
3352 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3239 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3353 try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);3240 try self.store(err_ptr, .none, ok_err_code, .anyerror, .normal);
...@@ -3358,25 +3245,17 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All...@@ -3358,25 +3245,17 @@ fn airWrapErrUnionPayload(self: *FuncGen, body_tail: []const Air.Inst.Index) All
3358 return result_ptr;3245 return result_ptr;
3359}3246}
33603247
3361fn airWrapErrUnionErr(self: *FuncGen, body_tail: []const Air.Inst.Index) Allocator.Error!Builder.Value {3248fn airWrapErrUnionErr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3362 const o = self.object;3249 const o = self.object;
3363 const zcu = o.zcu;3250 const zcu = o.zcu;
3364 const inst = body_tail[0];
3365 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;3251 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
3366 const err_un_ty = self.typeOfIndex(inst);3252 const err_un_ty = self.typeOfIndex(inst);
3367 const payload_ty = err_un_ty.errorUnionPayload(zcu);3253 const payload_ty = err_un_ty.errorUnionPayload(zcu);
3368 const operand = try self.resolveInst(ty_op.operand);3254 const operand = try self.resolveInst(ty_op.operand);
3369 if (!payload_ty.hasRuntimeBits(zcu)) return operand;3255 if (!payload_ty.hasRuntimeBits(zcu)) return operand;
3370 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref3256 assert(isByRef(err_un_ty, zcu)); // error unions with runtime bits are always by-ref
3371 const err_un_llvm_ty = try o.lowerType(err_un_ty);3257
33723258 const result_ptr = try self.buildZigAlloca(err_un_ty, .none);
3373 const result_ptr = if (self.isNextRet(body_tail))
3374 self.ret_ptr
3375 else brk: {
3376 const alignment = err_un_ty.abiAlignment(zcu).toLlvm();
3377 const result_ptr = try self.buildAlloca(err_un_llvm_ty, alignment);
3378 break :brk result_ptr;
3379 };
33803259
3381 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));3260 const err_ptr = try self.ptraddConst(result_ptr, codegen.errUnionErrorOffset(payload_ty, zcu));
3382 try self.store(err_ptr, .none, operand, .anyerror, .normal);3261 try self.store(err_ptr, .none, operand, .anyerror, .normal);
...@@ -3392,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -3392,7 +3271,7 @@ fn airWasmMemorySize(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3392 const o = self.object;3271 const o = self.object;
3393 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;3272 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3394 const index = pl_op.payload;3273 const index = pl_op.payload;
3395 const llvm_usize = try o.lowerType(.usize);3274 const llvm_usize = try o.lowerType(.usize, .by_value);
3396 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{3275 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{llvm_usize}, &.{
3397 try o.builder.intValue(.i32, index),3276 try o.builder.intValue(.i32, index),
3398 }, "");3277 }, "");
...@@ -3402,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build...@@ -3402,7 +3281,7 @@ fn airWasmMemoryGrow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Build
3402 const o = self.object;3281 const o = self.object;
3403 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;3282 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
3404 const index = pl_op.payload;3283 const index = pl_op.payload;
3405 const llvm_isize = try o.lowerType(.isize);3284 const llvm_isize = try o.lowerType(.isize, .by_value);
3406 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{3285 return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{llvm_isize}, &.{
3407 try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),3286 try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand),
3408 }, "");3287 }, "");
...@@ -3429,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -3429,7 +3308,7 @@ fn airMin(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3429 .normal,3308 .normal,
3430 .none,3309 .none,
3431 if (scalar_ty.isSignedInt(zcu)) .smin else .umin,3310 if (scalar_ty.isSignedInt(zcu)) .smin else .umin,
3432 &.{try o.lowerType(inst_ty)},3311 &.{try o.lowerType(inst_ty, .by_value)},
3433 &.{ lhs, rhs },3312 &.{ lhs, rhs },
3434 "",3313 "",
3435 );3314 );
...@@ -3449,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -3449,7 +3328,7 @@ fn airMax(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
3449 .normal,3328 .normal,
3450 .none,3329 .none,
3451 if (scalar_ty.isSignedInt(zcu)) .smax else .umax,3330 if (scalar_ty.isSignedInt(zcu)) .smax else .umax,
3452 &.{try o.lowerType(inst_ty)},3331 &.{try o.lowerType(inst_ty, .by_value)},
3453 &.{ lhs, rhs },3332 &.{ lhs, rhs },
3454 "",3333 "",
3455 );3334 );
...@@ -3461,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3461,7 +3340,7 @@ fn airSlice(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3461 const ptr = try self.resolveInst(bin_op.lhs);3340 const ptr = try self.resolveInst(bin_op.lhs);
3462 const len = try self.resolveInst(bin_op.rhs);3341 const len = try self.resolveInst(bin_op.rhs);
3463 const inst_ty = self.typeOfIndex(inst);3342 const inst_ty = self.typeOfIndex(inst);
3464 return self.wip.buildAggregate(try self.object.lowerType(inst_ty), &.{ ptr, len }, "");3343 return self.wip.buildAggregate(try self.object.lowerType(inst_ty, .by_value), &.{ ptr, len }, "");
3465}3344}
34663345
3467fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {3346fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...@@ -3492,7 +3371,7 @@ fn airSafeArithmetic(...@@ -3492,7 +3371,7 @@ fn airSafeArithmetic(
3492 const scalar_ty = inst_ty.scalarType(zcu);3371 const scalar_ty = inst_ty.scalarType(zcu);
34933372
3494 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;3373 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3495 const llvm_inst_ty = try o.lowerType(inst_ty);3374 const llvm_inst_ty = try o.lowerType(inst_ty, .by_value);
3496 const results =3375 const results =
3497 try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");3376 try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, "");
34983377
...@@ -3542,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3542,7 +3421,7 @@ fn airAddSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3542 .normal,3421 .normal,
3543 .none,3422 .none,
3544 if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",3423 if (scalar_ty.isSignedInt(zcu)) .@"sadd.sat" else .@"uadd.sat",
3545 &.{try o.lowerType(inst_ty)},3424 &.{try o.lowerType(inst_ty, .by_value)},
3546 &.{ lhs, rhs },3425 &.{ lhs, rhs },
3547 "",3426 "",
3548 );3427 );
...@@ -3581,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3581,7 +3460,7 @@ fn airSubSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3581 .normal,3460 .normal,
3582 .none,3461 .none,
3583 if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",3462 if (scalar_ty.isSignedInt(zcu)) .@"ssub.sat" else .@"usub.sat",
3584 &.{try o.lowerType(inst_ty)},3463 &.{try o.lowerType(inst_ty, .by_value)},
3585 &.{ lhs, rhs },3464 &.{ lhs, rhs },
3586 "",3465 "",
3587 );3466 );
...@@ -3620,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3620,7 +3499,7 @@ fn airMulSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3620 .normal,3499 .normal,
3621 .none,3500 .none,
3622 if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",3501 if (scalar_ty.isSignedInt(zcu)) .@"smul.fix.sat" else .@"umul.fix.sat",
3623 &.{try o.lowerType(inst_ty)},3502 &.{try o.lowerType(inst_ty, .by_value)},
3624 &.{ lhs, rhs, .@"0" },3503 &.{ lhs, rhs, .@"0" },
3625 "",3504 "",
3626 );3505 );
...@@ -3664,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)...@@ -3664,8 +3543,8 @@ fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind)
3664 return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});3543 return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result});
3665 }3544 }
3666 if (scalar_ty.isSignedInt(zcu)) {3545 if (scalar_ty.isSignedInt(zcu)) {
3667 const scalar_llvm_ty = try o.lowerType(scalar_ty);3546 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
3668 const inst_llvm_ty = try o.lowerType(inst_ty);3547 const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
36693548
3670 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;3549 const ExpectedContents = [std.math.big.int.calcTwosCompLimbCount(256)]std.math.big.Limb;
3671 var bfa_buf: ExpectedContents = undefined;3550 var bfa_buf: ExpectedContents = undefined;
...@@ -3739,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo...@@ -3739,7 +3618,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3739 const lhs = try self.resolveInst(bin_op.lhs);3618 const lhs = try self.resolveInst(bin_op.lhs);
3740 const rhs = try self.resolveInst(bin_op.rhs);3619 const rhs = try self.resolveInst(bin_op.rhs);
3741 const inst_ty = self.typeOfIndex(inst);3620 const inst_ty = self.typeOfIndex(inst);
3742 const inst_llvm_ty = try o.lowerType(inst_ty);3621 const inst_llvm_ty = try o.lowerType(inst_ty, .by_value);
3743 const scalar_ty = inst_ty.scalarType(zcu);3622 const scalar_ty = inst_ty.scalarType(zcu);
37443623
3745 if (scalar_ty.isRuntimeFloat()) {3624 if (scalar_ty.isRuntimeFloat()) {
...@@ -3768,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo...@@ -3768,7 +3647,7 @@ fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allo
3768 defer allocator.free(smin_big_int.limbs);3647 defer allocator.free(smin_big_int.limbs);
3769 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);3648 smin_big_int.setTwosCompIntLimit(.min, .signed, scalar_bits);
3770 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(3649 const smin = try o.builder.splatValue(inst_llvm_ty, try o.builder.bigIntConst(
3771 try o.lowerType(scalar_ty),3650 try o.lowerType(scalar_ty, .by_value),
3772 smin_big_int.toConst(),3651 smin_big_int.toConst(),
3773 ));3652 ));
37743653
...@@ -3804,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -3804,7 +3683,7 @@ fn airPtrSub(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
3804 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;3683 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
3805 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;3684 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
3806 const ptr_or_slice = try self.resolveInst(bin_op.lhs);3685 const ptr_or_slice = try self.resolveInst(bin_op.lhs);
3807 const llvm_usize_ty = try o.lowerType(.usize);3686 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
3808 const ptr_ty = self.typeOf(bin_op.lhs);3687 const ptr_ty = self.typeOf(bin_op.lhs);
3809 const elem_ty = ptr_ty.indexableElem(zcu);3688 const elem_ty = ptr_ty.indexableElem(zcu);
3810 const ptr = switch (ptr_ty.ptrSize(zcu)) {3689 const ptr = switch (ptr_ty.ptrSize(zcu)) {
...@@ -3837,8 +3716,7 @@ fn airOverflow(...@@ -3837,8 +3716,7 @@ fn airOverflow(
3837 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref3716 assert(isByRef(inst_ty, zcu)); // auto structs are by-ref
38383717
3839 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;3718 const intrinsic = if (scalar_ty.isSignedInt(zcu)) signed_intrinsic else unsigned_intrinsic;
3840 const llvm_inst_ty = try o.lowerType(inst_ty);3719 const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
3841 const llvm_lhs_ty = try o.lowerType(lhs_ty);
3842 const results =3720 const results =
3843 try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");3721 try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, "");
38443722
...@@ -3846,7 +3724,7 @@ fn airOverflow(...@@ -3846,7 +3724,7 @@ fn airOverflow(
3846 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");3724 const overflow_bit = try self.wip.extractValue(results, &.{1}, "");
38473725
3848 const result_alignment = inst_ty.abiAlignment(zcu);3726 const result_alignment = inst_ty.abiAlignment(zcu);
3849 const alloca_inst = try self.buildAlloca(llvm_inst_ty, result_alignment.toLlvm());3727 const alloca_inst = try self.buildZigAlloca(inst_ty, .none);
38503728
3851 {3729 {
3852 // Store to 'result: IntType' field3730 // Store to 'result: IntType' field
...@@ -3911,7 +3789,7 @@ fn buildFloatCmp(...@@ -3911,7 +3789,7 @@ fn buildFloatCmp(
3911 const zcu = o.zcu;3789 const zcu = o.zcu;
3912 const target = zcu.getTarget();3790 const target = zcu.getTarget();
3913 const scalar_ty = ty.scalarType(zcu);3791 const scalar_ty = ty.scalarType(zcu);
3914 const scalar_llvm_ty = try o.lowerType(scalar_ty);3792 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
39153793
3916 if (intrinsicsAllowed(scalar_ty, target)) {3794 if (intrinsicsAllowed(scalar_ty, target)) {
3917 const cond: Builder.FloatCondition = switch (pred) {3795 const cond: Builder.FloatCondition = switch (pred) {
...@@ -4017,7 +3895,7 @@ fn buildFloatOp(...@@ -4017,7 +3895,7 @@ fn buildFloatOp(
4017 const zcu = o.zcu;3895 const zcu = o.zcu;
4018 const target = zcu.getTarget();3896 const target = zcu.getTarget();
4019 const scalar_ty = ty.scalarType(zcu);3897 const scalar_ty = ty.scalarType(zcu);
4020 const llvm_ty = try o.lowerType(ty);3898 const llvm_ty = try o.lowerType(ty, .by_value);
40213899
4022 if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {3900 if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) {
4023 // Some operations are dedicated LLVM instructions, not available as intrinsics3901 // Some operations are dedicated LLVM instructions, not available as intrinsics
...@@ -4122,7 +4000,7 @@ fn buildFloatOp(...@@ -4122,7 +4000,7 @@ fn buildFloatOp(
4122 }),4000 }),
4123 };4001 };
41244002
4125 const scalar_llvm_ty = try o.lowerType(scalar_ty);4003 const scalar_llvm_ty = try o.lowerType(scalar_ty, .by_value);
4126 const libc_fn = try o.getLibcFunction(4004 const libc_fn = try o.getLibcFunction(
4127 fn_name,4005 fn_name,
4128 @as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],4006 @as([3]Builder.Type, @splat(scalar_llvm_ty))[0..params.len],
...@@ -4176,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -4176,9 +4054,8 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
41764054
4177 const dest_ty = self.typeOfIndex(inst);4055 const dest_ty = self.typeOfIndex(inst);
4178 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref4056 assert(isByRef(dest_ty, zcu)); // auto structs are by-ref
4179 const llvm_dest_ty = try o.lowerType(dest_ty);
41804057
4181 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4058 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
41824059
4183 const result = try self.wip.bin(.shl, lhs, casted_rhs, "");4060 const result = try self.wip.bin(.shl, lhs, casted_rhs, "");
4184 const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))4061 const reconstructed = try self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
...@@ -4189,7 +4066,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil...@@ -4189,7 +4066,7 @@ fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Buil
4189 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");4066 const overflow_bit = try self.wip.icmp(.ne, lhs, reconstructed, "");
41904067
4191 const result_alignment = dest_ty.abiAlignment(zcu);4068 const result_alignment = dest_ty.abiAlignment(zcu);
4192 const alloca_inst = try self.buildAlloca(llvm_dest_ty, result_alignment.toLlvm());4069 const alloca_inst = try self.buildZigAlloca(dest_ty, .none);
41934070
4194 {4071 {
4195 // Store to 'result: IntType' field4072 // Store to 'result: IntType' field
...@@ -4245,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -4245,7 +4122,7 @@ fn airShlExact(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
4245 }4122 }
4246 const lhs_scalar_ty = lhs_ty.scalarType(zcu);4123 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
42474124
4248 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4125 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4249 return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))4126 return self.wip.bin(if (lhs_scalar_ty.isSignedInt(zcu))
4250 .@"shl nsw"4127 .@"shl nsw"
4251 else4128 else
...@@ -4266,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4266,7 +4143,7 @@ fn airShl(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4266 // features which we do not use. Therefore this branch is currently impossible.4143 // features which we do not use. Therefore this branch is currently impossible.
4267 unreachable;4144 unreachable;
4268 }4145 }
4269 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4146 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4270 return self.wip.bin(.shl, lhs, casted_rhs, "");4147 return self.wip.bin(.shl, lhs, casted_rhs, "");
4271}4148}
42724149
...@@ -4280,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4280,8 +4157,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
42804157
4281 const lhs_ty = self.typeOf(bin_op.lhs);4158 const lhs_ty = self.typeOf(bin_op.lhs);
4282 const lhs_info = lhs_ty.intInfo(zcu);4159 const lhs_info = lhs_ty.intInfo(zcu);
4283 const llvm_lhs_ty = try o.lowerType(lhs_ty);4160 const llvm_lhs_ty = try o.lowerType(lhs_ty, .by_value);
4284 const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu));4161 const llvm_lhs_scalar_ty = try o.lowerType(lhs_ty.scalarType(zcu), .by_value);
42854162
4286 const rhs_ty = self.typeOf(bin_op.rhs);4163 const rhs_ty = self.typeOf(bin_op.rhs);
4287 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {4164 if (lhs_ty.isVector(zcu) and !rhs_ty.isVector(zcu)) {
...@@ -4291,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4291,8 +4168,8 @@ fn airShlSat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4291 }4168 }
4292 const rhs_info = rhs_ty.intInfo(zcu);4169 const rhs_info = rhs_ty.intInfo(zcu);
4293 assert(rhs_info.signedness == .unsigned);4170 assert(rhs_info.signedness == .unsigned);
4294 const llvm_rhs_ty = try o.lowerType(rhs_ty);4171 const llvm_rhs_ty = try o.lowerType(rhs_ty, .by_value);
4295 const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu));4172 const llvm_rhs_scalar_ty = try o.lowerType(rhs_ty.scalarType(zcu), .by_value);
42964173
4297 const result = try self.wip.callIntrinsic(4174 const result = try self.wip.callIntrinsic(
4298 .normal,4175 .normal,
...@@ -4368,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!...@@ -4368,7 +4245,7 @@ fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) Allocator.Error!
4368 }4245 }
4369 const lhs_scalar_ty = lhs_ty.scalarType(zcu);4246 const lhs_scalar_ty = lhs_ty.scalarType(zcu);
43704247
4371 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty), "");4248 const casted_rhs = try self.wip.conv(.unsigned, rhs, try o.lowerType(lhs_ty, .by_value), "");
4372 const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);4249 const is_signed_int = lhs_scalar_ty.isSignedInt(zcu);
43734250
4374 return self.wip.bin(if (is_exact)4251 return self.wip.bin(if (is_exact)
...@@ -4389,7 +4266,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4389,7 +4266,7 @@ fn airAbs(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4389 .normal,4266 .normal,
4390 .none,4267 .none,
4391 .abs,4268 .abs,
4392 &.{try o.lowerType(operand_ty)},4269 &.{try o.lowerType(operand_ty, .by_value)},
4393 &.{ operand, .false },4270 &.{ operand, .false },
4394 "",4271 "",
4395 ),4272 ),
...@@ -4403,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4403,7 +4280,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4403 const zcu = o.zcu;4280 const zcu = o.zcu;
4404 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4281 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4405 const dest_ty = fg.typeOfIndex(inst);4282 const dest_ty = fg.typeOfIndex(inst);
4406 const dest_llvm_ty = try o.lowerType(dest_ty);4283 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
4407 const operand = try fg.resolveInst(ty_op.operand);4284 const operand = try fg.resolveInst(ty_op.operand);
4408 const operand_ty = fg.typeOf(ty_op.operand);4285 const operand_ty = fg.typeOf(ty_op.operand);
4409 const operand_info = operand_ty.intInfo(zcu);4286 const operand_info = operand_ty.intInfo(zcu);
...@@ -4431,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4431,8 +4308,8 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
44314308
4432 if (!have_min_check and !have_max_check) break :bounds_check;4309 if (!have_min_check and !have_max_check) break :bounds_check;
44334310
4434 const operand_llvm_ty = try o.lowerType(operand_ty);4311 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4435 const operand_scalar_llvm_ty = try o.lowerType(operand_scalar);4312 const operand_scalar_llvm_ty = try o.lowerType(operand_scalar, .by_value);
44364313
4437 const is_vector = operand_ty.zigTypeTag(zcu) == .vector;4314 const is_vector = operand_ty.zigTypeTag(zcu) == .vector;
4438 assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));4315 assert(is_vector == (dest_ty.zigTypeTag(zcu) == .vector));
...@@ -4510,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!...@@ -4510,7 +4387,7 @@ fn airIntCast(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!
4510fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4387fn airTrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4511 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;4388 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4512 const operand = try self.resolveInst(ty_op.operand);4389 const operand = try self.resolveInst(ty_op.operand);
4513 const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst));4390 const dest_llvm_ty = try self.object.lowerType(self.typeOfIndex(inst), .by_value);
4514 return self.wip.cast(.trunc, operand, dest_llvm_ty, "");4391 return self.wip.cast(.trunc, operand, dest_llvm_ty, "");
4515}4392}
45164393
...@@ -4524,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -4524,10 +4401,10 @@ fn airFptrunc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
4524 const target = zcu.getTarget();4401 const target = zcu.getTarget();
45254402
4526 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {4403 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4527 return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty), "");4404 return self.wip.cast(.fptrunc, operand, try o.lowerType(dest_ty, .by_value), "");
4528 } else {4405 } else {
4529 const operand_llvm_ty = try o.lowerType(operand_ty);4406 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4530 const dest_llvm_ty = try o.lowerType(dest_ty);4407 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
45314408
4532 const dest_bits = dest_ty.floatBits(target);4409 const dest_bits = dest_ty.floatBits(target);
4533 const src_bits = operand_ty.floatBits(target);4410 const src_bits = operand_ty.floatBits(target);
...@@ -4558,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4558,10 +4435,10 @@ fn airFpext(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4558 const target = zcu.getTarget();4435 const target = zcu.getTarget();
45594436
4560 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {4437 if (intrinsicsAllowed(dest_ty, target) and intrinsicsAllowed(operand_ty, target)) {
4561 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty), "");4438 return self.wip.cast(.fpext, operand, try o.lowerType(dest_ty, .by_value), "");
4562 } else {4439 } else {
4563 const operand_llvm_ty = try o.lowerType(operand_ty);4440 const operand_llvm_ty = try o.lowerType(operand_ty, .by_value);
4564 const dest_llvm_ty = try o.lowerType(dest_ty);4441 const dest_llvm_ty = try o.lowerType(dest_ty, .by_value);
45654442
4566 const dest_bits = dest_ty.scalarType(zcu).floatBits(target);4443 const dest_bits = dest_ty.scalarType(zcu).floatBits(target);
4567 const src_bits = operand_ty.scalarType(zcu).floatBits(target);4444 const src_bits = operand_ty.scalarType(zcu).floatBits(target);
...@@ -4599,39 +4476,69 @@ fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4599,39 +4476,69 @@ fn airBitCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
45994476
4600 // We have the following `Air.Legalize` features enabled:4477 // We have the following `Air.Legalize` features enabled:
4601 //4478 //
4602 // * `.scalarize_bitcast_array`4479 // * `.scalarize_bit_cast_array`
4603 // * `.scalarize_bitcast_vector_non_elementwise`4480 // * `.scalarize_bit_cast_vector_non_elementwise`
4604 //4481 //
4605 // That means the set of bitcasts we might see is limited to the following:4482 // That means the `bit_cast` instructions we might see are limited to the following:
4606 //4483 //
4607 // * bool/int/float <-> bool/int/float4484 // * bool/int/float <-> bool/int/float
4608 // * `@Vector(n, A)` <-> `@Vector(n, B)`4485 // * `@Vector(n, A)` <-> `@Vector(n, B)`
4609 // * pointer <-> pointer
4610 // * pointer <-> int
4611 // * slice <-> slice
4612 //4486 //
4613 // Most of these can be handled by LLVM's `bitcast` instruction. We will check for the few cases4487 // All of these cases can be handled by LLVM's `bitcast` instruction.
4614 // that aren't, and otherwise use `bitcast`.
4615
4616 if (operand_ty.isSlice(zcu) and dest_ty.isSlice(zcu)) {
4617 // The slice types are the same type in LLVM IR, so this conversion is a nop.
4618 return operand;
4619 }
46204488
4621 assert(!isByRef(operand_ty, zcu));4489 assert(!isByRef(operand_ty, zcu));
4622 assert(!isByRef(dest_ty, zcu));4490 assert(!isByRef(dest_ty, zcu));
46234491
4624 const llvm_dest_ty = try o.lowerType(dest_ty);4492 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4493 return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");
4494}
46254495
4626 if (operand_ty.scalarType(zcu).zigTypeTag(zcu) == .int and dest_ty.scalarType(zcu).isPtrAtRuntime(zcu)) {4496fn airNopCast(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4627 return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");4497 const zcu = fg.object.zcu;
4628 }4498 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4499 const operand_ty = fg.typeOf(ty_op.operand);
4500 const dest_ty = fg.typeOfIndex(inst);
4501 assert(isByRef(operand_ty, zcu) == isByRef(dest_ty, zcu));
4502 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
4503 return fg.resolveInst(ty_op.operand);
4504}
46294505
4630 if (operand_ty.scalarType(zcu).isPtrAtRuntime(zcu) and dest_ty.scalarType(zcu).zigTypeTag(zcu) == .int) {4506fn airPtrFromInt(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4631 return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");4507 const o = fg.object;
4632 }4508 const zcu = o.zcu;
4509 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4510 const operand_ty = fg.typeOf(ty_op.operand);
4511 const dest_ty = fg.typeOfIndex(inst);
4512 assert(operand_ty.scalarType(zcu).toIntern() == .usize_type);
4513 assert(dest_ty.scalarType(zcu).isPtrAtRuntime(zcu));
46334514
4634 return fg.wip.cast(.bitcast, operand, llvm_dest_ty, "");4515 const operand = try fg.resolveInst(ty_op.operand);
4516 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4517 return fg.wip.cast(.inttoptr, operand, llvm_dest_ty, "");
4518}
4519
4520fn airIntFromPtr(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4521 const o = fg.object;
4522 const zcu = o.zcu;
4523 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4524 const operand_ty = fg.typeOf(ty_op.operand);
4525 const dest_ty = fg.typeOfIndex(inst);
4526 assert(operand_ty.scalarType(zcu).isPtrAtRuntime(zcu));
4527 assert(dest_ty.scalarType(zcu).toIntern() == .usize_type);
4528
4529 const operand = try fg.resolveInst(ty_op.operand);
4530 const llvm_dest_ty = try o.lowerType(dest_ty, .by_value);
4531 return fg.wip.cast(.ptrtoint, operand, llvm_dest_ty, "");
4532}
4533
4534fn airUnionFromEnum(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4535 const ty_op = fg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
4536 const enum_ty = fg.typeOf(ty_op.operand);
4537 const union_ty = fg.typeOfIndex(inst);
4538 const enum_val = try fg.resolveInst(ty_op.operand);
4539 const union_ptr = try fg.buildZigAlloca(union_ty, .none);
4540 try fg.store(union_ptr, .none, enum_val, enum_ty, .normal);
4541 return union_ptr;
4635}4542}
46364543
4637fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4544fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -4690,8 +4597,7 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4690,8 +4597,7 @@ fn airArg(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4690 "",4597 "",
4691 );4598 );
4692 } else if (mod.optimize_mode == .Debug) {4599 } else if (mod.optimize_mode == .Debug) {
4693 const alignment = inst_ty.abiAlignment(zcu).toLlvm();4600 const alloca = try self.buildZigAlloca(inst_ty, .none);
4694 const alloca = try self.buildAlloca(try o.lowerType(inst_ty), alignment);
4695 try self.store(alloca, .none, arg_val, inst_ty, .normal);4601 try self.store(alloca, .none, arg_val, inst_ty, .normal);
4696 _ = try self.wip.callIntrinsic(4602 _ = try self.wip.callIntrinsic(
4697 .normal,4603 .normal,
...@@ -4733,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4733,8 +4639,7 @@ fn airAlloc(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4733 if (!elem_ty.hasRuntimeBits(zcu)) {4639 if (!elem_ty.hasRuntimeBits(zcu)) {
4734 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();4640 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4735 }4641 }
4736 const llvm_elem_ty = try o.lowerType(elem_ty);4642 return self.buildZigAlloca(elem_ty, ptr_align);
4737 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());
4738}4643}
47394644
4740fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4645fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -4747,8 +4652,19 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -4747,8 +4652,19 @@ fn airRetPtr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
4747 if (!elem_ty.hasRuntimeBits(zcu)) {4652 if (!elem_ty.hasRuntimeBits(zcu)) {
4748 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();4653 return (try o.lowerPtrToVoid(ptr_align, ptr_ty.ptrAddressSpace(zcu))).toValue();
4749 }4654 }
4750 const llvm_elem_ty = try o.lowerType(elem_ty);4655 return self.buildZigAlloca(elem_ty, ptr_align);
4751 return self.buildAlloca(llvm_elem_ty, ptr_align.toLlvm());4656}
4657
4658fn buildZigAlloca(fg: *FuncGen, ty: Type, @"align": InternPool.Alignment) Allocator.Error!Builder.Value {
4659 const o = fg.object;
4660 const resolved_align: InternPool.Alignment = switch (@"align") {
4661 .none => ty.abiAlignment(o.zcu),
4662 else => |a| a,
4663 };
4664 return fg.buildAlloca(
4665 try o.lowerType(ty, .in_memory),
4666 resolved_align.toLlvm(),
4667 );
4752}4668}
47534669
4754/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.4670/// Unlike `WipFunction.alloca`, this puts the alloca instruction at the top of the function.
...@@ -4823,7 +4739,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu...@@ -4823,7 +4739,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu
4823 return .none;4739 return .none;
4824 }4740 }
48254741
4826 const len = try o.builder.intValue(try o.lowerType(.usize), elem_ty.abiSize(zcu));4742 const len = try o.builder.intValue(try o.lowerType(.usize, .by_value), elem_ty.abiSize(zcu));
4827 _ = try fg.wip.callMemSet(4743 _ = try fg.wip.callMemSet(
4828 ptr,4744 ptr,
4829 ptr_alignment.toLlvm(),4745 ptr_alignment.toLlvm(),
...@@ -4858,7 +4774,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu...@@ -4858,7 +4774,7 @@ fn airStore(fg: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error!Bu
4858 if (ptr_info.packed_offset.host_size != 0) {4774 if (ptr_info.packed_offset.host_size != 0) {
4859 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.4775 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4860 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));4776 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4861 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);4777 const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
48624778
4863 const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);4779 const backing_int_val = try fg.load(ptr, ptr_alignment, backing_int_ty, access_kind);
48644780
...@@ -4936,14 +4852,14 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {...@@ -4936,14 +4852,14 @@ fn airLoad(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
49364852
4937 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.4853 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate this usage of `pt`.
4938 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));4854 const backing_int_ty = try fg.pt.intType(.unsigned, @intCast(ptr_info.packed_offset.host_size * 8));
4939 const llvm_backing_int_ty = try o.lowerType(backing_int_ty);4855 const llvm_backing_int_ty = try o.lowerType(backing_int_ty, .by_value);
49404856
4941 const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);4857 const backing_int_val = try fg.load(ptr, ptr_align, backing_int_ty, .normal);
49424858
4943 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);4859 const elem_bits = ptr_ty.childType(zcu).bitSize(zcu);
4944 const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);4860 const shift_amt = try o.builder.intValue(llvm_backing_int_ty, ptr_info.packed_offset.bit_offset);
4945 const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");4861 const shifted_value = try fg.wip.bin(.lshr, backing_int_val, shift_amt, "");
4946 const elem_llvm_ty = try o.lowerType(elem_ty);4862 const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
49474863
4948 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {4864 if (elem_ty.zigTypeTag(zcu) == .float or elem_ty.zigTypeTag(zcu) == .vector) {
4949 const same_size_int = try o.builder.intType(@intCast(elem_bits));4865 const same_size_int = try o.builder.intType(@intCast(elem_bits));
...@@ -4993,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -4993,7 +4909,7 @@ fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
4993fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4909fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
4994 _ = inst;4910 _ = inst;
4995 const o = self.object;4911 const o = self.object;
4996 const llvm_usize = try o.lowerType(.usize);4912 const llvm_usize = try o.lowerType(.usize, .by_value);
4997 if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {4913 if (!target_util.supportsReturnAddress(self.object.zcu.getTarget(), self.ownerModule().optimize_mode)) {
4998 // https://github.com/ziglang/zig/issues/119464914 // https://github.com/ziglang/zig/issues/11946
4999 return o.builder.intValue(llvm_usize, 0);4915 return o.builder.intValue(llvm_usize, 0);
...@@ -5005,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu...@@ -5005,7 +4921,7 @@ fn airRetAddr(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Valu
5005fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {4921fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
5006 _ = inst;4922 _ = inst;
5007 const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");4923 const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{.@"0"}, "");
5008 return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize), "");4924 return self.wip.cast(.ptrtoint, result, try self.object.lowerType(.usize, .by_value), "");
5009}4925}
50104926
5011fn airCmpxchg(4927fn airCmpxchg(
...@@ -5022,7 +4938,7 @@ fn airCmpxchg(...@@ -5022,7 +4938,7 @@ fn airCmpxchg(
5022 var expected_value = try self.resolveInst(extra.expected_value);4938 var expected_value = try self.resolveInst(extra.expected_value);
5023 var new_value = try self.resolveInst(extra.new_value);4939 var new_value = try self.resolveInst(extra.new_value);
5024 const operand_ty = ptr_ty.childType(zcu);4940 const operand_ty = ptr_ty.childType(zcu);
5025 const llvm_operand_ty = try o.lowerType(operand_ty);4941 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5026 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);4942 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, false);
5027 if (llvm_abi_ty != .none) {4943 if (llvm_abi_ty != .none) {
5028 // operand needs widening and truncating4944 // operand needs widening and truncating
...@@ -5066,7 +4982,7 @@ fn airCmpxchg(...@@ -5066,7 +4982,7 @@ fn airCmpxchg(
5066 const non_null_bit = try self.wip.not(success_bit, "");4982 const non_null_bit = try self.wip.not(success_bit, "");
50674983
5068 const payload_align = operand_ty.abiAlignment(zcu);4984 const payload_align = operand_ty.abiAlignment(zcu);
5069 const alloca_inst = try self.buildAlloca(try o.lowerType(optional_ty), payload_align.toLlvm());4985 const alloca_inst = try self.buildZigAlloca(optional_ty, .none);
50704986
5071 // Payload is always the first field at offset 0, so address is `alloca_inst`4987 // Payload is always the first field at offset 0, so address is `alloca_inst`
5072 try self.store(alloca_inst, .none, payload, operand_ty, .normal);4988 try self.store(alloca_inst, .none, payload, operand_ty, .normal);
...@@ -5092,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5092,7 +5008,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5092 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);5008 const op = toLlvmAtomicRmwBinOp(extra.op(), is_signed_int, is_float);
5093 const ordering = toLlvmAtomicOrdering(extra.ordering());5009 const ordering = toLlvmAtomicOrdering(extra.ordering());
5094 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);5010 const llvm_abi_ty = try self.getAtomicAbiType(operand_ty, op == .xchg);
5095 const llvm_operand_ty = try o.lowerType(operand_ty);5011 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
50965012
5097 const access_kind: Builder.MemoryAccessKind =5013 const access_kind: Builder.MemoryAccessKind =
5098 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;5014 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal;
...@@ -5121,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5121,7 +5037,7 @@ fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
51215037
5122 // If we are storing a pointer we need to convert to and from a plain old integer.5038 // If we are storing a pointer we need to convert to and from a plain old integer.
5123 const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {5039 const non_ptr_operand = switch (operand_ty.zigTypeTag(zcu)) {
5124 .pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize), ""),5040 .pointer => try self.wip.cast(.ptrtoint, operand, try o.lowerType(.usize, .by_value), ""),
5125 else => operand,5041 else => operand,
5126 };5042 };
51275043
...@@ -5160,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V...@@ -5160,7 +5076,7 @@ fn airAtomicLoad(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.V
5160 Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();5076 Type.fromInterned(info.child).abiAlignment(zcu)).toLlvm();
5161 const access_kind: Builder.MemoryAccessKind =5077 const access_kind: Builder.MemoryAccessKind =
5162 if (info.flags.is_volatile) .@"volatile" else .normal;5078 if (info.flags.is_volatile) .@"volatile" else .normal;
5163 const elem_llvm_ty = try o.lowerType(elem_ty);5079 const elem_llvm_ty = try o.lowerType(elem_ty, .by_value);
51645080
5165 self.maybeMarkAllowZeroAccess(info);5081 self.maybeMarkAllowZeroAccess(info);
51665082
...@@ -5306,10 +5222,13 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error...@@ -5306,10 +5222,13 @@ fn airMemset(self: *FuncGen, inst: Air.Inst.Index, safety: bool) Allocator.Error
5306 }5222 }
5307 if (elem_ty.isAbiInt(zcu)) {5223 if (elem_ty.isAbiInt(zcu)) {
5308 const info = elem_ty.intInfo(zcu);5224 const info = elem_ty.intInfo(zcu);
5309 break :byte self.wip.conv(info.signedness, value, .i8, "");5225 break :byte try self.wip.conv(switch (info.signedness) {
5226 .unsigned => .unsigned,
5227 .signed => .signed,
5228 }, value, .i8, "");
5310 }5229 }
5311 if (elem_ty == .bool) {5230 if (elem_ty.toIntern() == .bool_type) {
5312 break :byte self.wip.cast(.zext, value, .i8, "");5231 break :byte try self.wip.cast(.zext, value, .i8, "");
5313 }5232 }
5314 break :intrinsic;5233 break :intrinsic;
5315 };5234 };
...@@ -5459,15 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder....@@ -5459,15 +5378,9 @@ fn airGetUnionTag(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.
5459 const layout = un_ty.unionGetLayout(zcu);5378 const layout = un_ty.unionGetLayout(zcu);
5460 assert(layout.tag_size != 0);5379 assert(layout.tag_size != 0);
5461 const operand = try self.resolveInst(ty_op.operand);5380 const operand = try self.resolveInst(ty_op.operand);
5462 if (isByRef(un_ty, zcu)) {5381 assert(isByRef(un_ty, zcu));
5463 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());5382 const tag_field_ptr = try self.ptraddConst(operand, layout.tagOffset());
5464 return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);5383 return self.load(tag_field_ptr, .none, un_ty.unionTagTypeRuntime(zcu).?, .normal);
5465 } else {
5466 // This is only possible if all fields are zero-bit, in which case `operand` is already an
5467 // integer value (the union is lowered as its enum tag).
5468 assert(layout.payload_size == 0);
5469 return operand;
5470 }
5471}5384}
54725385
5473fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {5386fn airUnaryOp(self: *FuncGen, inst: Air.Inst.Index, comptime op: FloatOp) Allocator.Error!Builder.Value {
...@@ -5497,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)...@@ -5497,11 +5410,11 @@ fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5497 .normal,5410 .normal,
5498 .none,5411 .none,
5499 intrinsic,5412 intrinsic,
5500 &.{try o.lowerType(operand_ty)},5413 &.{try o.lowerType(operand_ty, .by_value)},
5501 &.{ operand, .false },5414 &.{ operand, .false },
5502 "",5415 "",
5503 );5416 );
5504 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5417 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5505}5418}
55065419
5507fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {5420fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) Allocator.Error!Builder.Value {
...@@ -5515,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)...@@ -5515,11 +5428,11 @@ fn airBitOp(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic)
5515 .normal,5428 .normal,
5516 .none,5429 .none,
5517 intrinsic,5430 intrinsic,
5518 &.{try o.lowerType(operand_ty)},5431 &.{try o.lowerType(operand_ty, .by_value)},
5519 &.{operand},5432 &.{operand},
5520 "",5433 "",
5521 );5434 );
5522 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5435 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5523}5436}
55245437
5525fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5438fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5532,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5532,7 +5445,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
55325445
5533 const inst_ty = self.typeOfIndex(inst);5446 const inst_ty = self.typeOfIndex(inst);
5534 var operand = try self.resolveInst(ty_op.operand);5447 var operand = try self.resolveInst(ty_op.operand);
5535 var llvm_operand_ty = try o.lowerType(operand_ty);5448 var llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
55365449
5537 if (bits % 16 == 8) {5450 if (bits % 16 == 8) {
5538 // If not an even byte-multiple, we need zero-extend + shift-left 1 byte5451 // If not an even byte-multiple, we need zero-extend + shift-left 1 byte
...@@ -5553,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5553,7 +5466,7 @@ fn airByteSwap(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
55535466
5554 const result =5467 const result =
5555 try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");5468 try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, "");
5556 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), "");5469 return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty, .by_value), "");
5557}5470}
55585471
5559fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5472fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5573,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui...@@ -5573,7 +5486,7 @@ fn airErrorSetHasValue(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Bui
55735486
5574 for (0..names.len) |name_index| {5487 for (0..names.len) |name_index| {
5575 const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;5488 const err_int = ip.getErrorValueIfExists(names.get(ip)[name_index]).?;
5576 const this_tag_int_value = try o.builder.intConst(try o.errorIntType(), err_int);5489 const this_tag_int_value = try o.builder.intConst(try o.errorIntType(.by_value), err_int);
5577 try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);5490 try wip_switch.addCase(this_tag_int_value, valid_block, &self.wip);
5578 }5491 }
5579 self.wip.cursor = .{ .block = valid_block };5492 self.wip.cursor = .{ .block = valid_block };
...@@ -5632,7 +5545,7 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -5632,7 +5545,7 @@ fn airErrorName(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
5632 const slice_ty = self.typeOfIndex(inst);5545 const slice_ty = self.typeOfIndex(inst);
56335546
5634 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.5547 // If operand is small (e.g. `u8`), then signedness becomes a problem -- GEP always treats the index as signed.
5635 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize), "");5548 const operand_usize = try self.wip.conv(.unsigned, operand, try o.lowerType(.usize, .by_value), "");
56365549
5637 const error_name_table_ptr = try o.getErrorNameTable();5550 const error_name_table_ptr = try o.getErrorNameTable();
5638 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));5551 const error_name_ptr = try self.ptraddScaled(error_name_table_ptr.toValue(&o.builder), operand_usize, slice_ty.abiSize(zcu));
...@@ -5643,7 +5556,7 @@ fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value...@@ -5643,7 +5556,7 @@ fn airSplat(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value
5643 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5556 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5644 const scalar = try self.resolveInst(ty_op.operand);5557 const scalar = try self.resolveInst(ty_op.operand);
5645 const vector_ty = self.typeOfIndex(inst);5558 const vector_ty = self.typeOfIndex(inst);
5646 return self.wip.splatVector(try self.object.lowerType(vector_ty), scalar, "");5559 return self.wip.splatVector(try self.object.lowerType(vector_ty, .by_value), scalar, "");
5647}5560}
56485561
5649fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {5562fn airSelect(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Value {
...@@ -5666,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5666,9 +5579,9 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5666 const operand = try fg.resolveInst(unwrapped.operand);5579 const operand = try fg.resolveInst(unwrapped.operand);
5667 const mask = unwrapped.mask;5580 const mask = unwrapped.mask;
5668 const operand_ty = fg.typeOf(unwrapped.operand);5581 const operand_ty = fg.typeOf(unwrapped.operand);
5669 const llvm_operand_ty = try o.lowerType(operand_ty);5582 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5670 const llvm_result_ty = try o.lowerType(unwrapped.result_ty);5583 const llvm_result_ty = try o.lowerType(unwrapped.result_ty, .by_value);
5671 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));5584 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5672 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);5585 const llvm_poison_elem = try o.builder.poisonConst(llvm_elem_ty);
5673 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);5586 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
5674 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);5587 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
...@@ -5698,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5698,7 +5611,7 @@ fn airShuffleOne(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5698 .elem => llvm_poison_elem,5611 .elem => llvm_poison_elem,
5699 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {5612 .value => |val| if (!Value.fromInterned(val).isUndef(zcu)) elem: {
5700 any_defined_comptime_value = true;5613 any_defined_comptime_value = true;
5701 break :elem try o.lowerValue(val);5614 break :elem try o.lowerValue(val, .by_value);
5702 } else llvm_poison_elem,5615 } else llvm_poison_elem,
5703 };5616 };
5704 }5617 }
...@@ -5770,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val...@@ -5770,7 +5683,7 @@ fn airShuffleTwo(fg: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Val
5770 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);5683 const unwrapped = fg.air.unwrapShuffleTwo(zcu, inst);
57715684
5772 const mask = unwrapped.mask;5685 const mask = unwrapped.mask;
5773 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu));5686 const llvm_elem_ty = try o.lowerType(unwrapped.result_ty.childType(zcu), .by_value);
5774 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);5687 const llvm_mask_ty = try o.builder.vectorType(.normal, @intCast(mask.len), .i32);
5775 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);5688 const llvm_poison_mask_elem = try o.builder.poisonConst(.i32);
57765689
...@@ -5860,7 +5773,7 @@ fn buildReducedCall(...@@ -5860,7 +5773,7 @@ fn buildReducedCall(
5860 accum_init: Builder.Value,5773 accum_init: Builder.Value,
5861) Allocator.Error!Builder.Value {5774) Allocator.Error!Builder.Value {
5862 const o = self.object;5775 const o = self.object;
5863 const llvm_usize_ty = try o.lowerType(.usize);5776 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
5864 const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);5777 const llvm_vector_len = try o.builder.intValue(llvm_usize_ty, vector_len);
5865 const llvm_result_ty = accum_init.typeOfWip(&self.wip);5778 const llvm_result_ty = accum_init.typeOfWip(&self.wip);
58665779
...@@ -5907,7 +5820,7 @@ fn buildReducedCall(...@@ -5907,7 +5820,7 @@ fn buildReducedCall(
5907 accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);5820 accum.finish(&.{ accum_init, new_accum }, &.{ entry_block, body_block }, &self.wip);
59085821
5909 self.wip.cursor = .{ .block = exit_block };5822 self.wip.cursor = .{ .block = exit_block };
5910 return new_accum;5823 return accum.toValue();
5911}5824}
59125825
5913fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {5826fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) Allocator.Error!Builder.Value {
...@@ -5918,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A...@@ -5918,9 +5831,9 @@ fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) A
5918 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;5831 const reduce = self.air.instructions.items(.data)[@intFromEnum(inst)].reduce;
5919 const operand = try self.resolveInst(reduce.operand);5832 const operand = try self.resolveInst(reduce.operand);
5920 const operand_ty = self.typeOf(reduce.operand);5833 const operand_ty = self.typeOf(reduce.operand);
5921 const llvm_operand_ty = try o.lowerType(operand_ty);5834 const llvm_operand_ty = try o.lowerType(operand_ty, .by_value);
5922 const scalar_ty = self.typeOfIndex(inst);5835 const scalar_ty = self.typeOfIndex(inst);
5923 const llvm_scalar_ty = try o.lowerType(scalar_ty);5836 const llvm_scalar_ty = try o.lowerType(scalar_ty, .by_value);
59245837
5925 switch (reduce.operation) {5838 switch (reduce.operation) {
5926 .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {5839 .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) {
...@@ -6027,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6027,10 +5940,10 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6027 const result_ty = self.typeOfIndex(inst);5940 const result_ty = self.typeOfIndex(inst);
6028 const len: usize = @intCast(result_ty.arrayLen(zcu));5941 const len: usize = @intCast(result_ty.arrayLen(zcu));
6029 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);5942 const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]);
6030 const llvm_result_ty = try o.lowerType(result_ty);
60315943
6032 switch (result_ty.zigTypeTag(zcu)) {5944 switch (result_ty.zigTypeTag(zcu)) {
6033 .vector => {5945 .vector => {
5946 const llvm_result_ty = try o.lowerType(result_ty, .by_value);
6034 var vector = try o.builder.poisonValue(llvm_result_ty);5947 var vector = try o.builder.poisonValue(llvm_result_ty);
6035 for (elements, 0..) |elem, i| {5948 for (elements, 0..) |elem, i| {
6036 const index_u32 = try o.builder.intValue(.i32, i);5949 const index_u32 = try o.builder.intValue(.i32, i);
...@@ -6072,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6072,7 +5985,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6072 // TODO in debug builds init to undef so that the padding will be 0xaa5985 // TODO in debug builds init to undef so that the padding will be 0xaa
6073 // even if we fully populate the fields.5986 // even if we fully populate the fields.
6074 const struct_align = result_ty.abiAlignment(zcu);5987 const struct_align = result_ty.abiAlignment(zcu);
6075 const alloca_inst = try self.buildAlloca(llvm_result_ty, struct_align.toLlvm());5988 const alloca_inst = try self.buildZigAlloca(result_ty, .none);
60765989
6077 for (elements, 0..) |elem, field_index| {5990 for (elements, 0..) |elem, field_index| {
6078 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;5991 if (result_ty.structFieldIsComptime(field_index, zcu)) continue;
...@@ -6093,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6093,8 +6006,7 @@ fn airAggregateInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6093 .array => {6006 .array => {
6094 assert(isByRef(result_ty, zcu));6007 assert(isByRef(result_ty, zcu));
60956008
6096 const alignment = result_ty.abiAlignment(zcu).toLlvm();6009 const alloca_inst = try self.buildZigAlloca(result_ty, .none);
6097 const alloca_inst = try self.buildAlloca(llvm_result_ty, alignment);
60986010
6099 const array_info = result_ty.arrayInfo(zcu);6011 const array_info = result_ty.arrayInfo(zcu);
61006012
...@@ -6124,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6124,7 +6036,6 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6124 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;6036 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
6125 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;6037 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
6126 const union_ty = self.typeOfIndex(inst);6038 const union_ty = self.typeOfIndex(inst);
6127 const union_llvm_ty = try o.lowerType(union_ty);
6128 const union_obj = zcu.typeToUnion(union_ty).?;6039 const union_obj = zcu.typeToUnion(union_ty).?;
61296040
6130 assert(union_obj.layout != .@"packed");6041 assert(union_obj.layout != .@"packed");
...@@ -6134,8 +6045,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6134,8 +6045,7 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6134 assert(layout.payload_size != 0); // otherwise the value would be comptime-known6045 assert(layout.payload_size != 0); // otherwise the value would be comptime-known
6135 assert(isByRef(union_ty, zcu));6046 assert(isByRef(union_ty, zcu));
61366047
6137 const alignment = layout.abi_align.toLlvm();6048 const result_ptr = try self.buildZigAlloca(union_ty, layout.abi_align);
6138 const result_ptr = try self.buildAlloca(union_llvm_ty, alignment);
6139 const llvm_payload = try self.resolveInst(extra.init);6049 const llvm_payload = try self.resolveInst(extra.init);
6140 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);6050 const field_ty = Type.fromInterned(union_obj.field_types.get(ip)[extra.field_index]);
6141 assert(field_ty.hasRuntimeBits(zcu));6051 assert(field_ty.hasRuntimeBits(zcu));
...@@ -6150,10 +6060,10 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va...@@ -6150,10 +6060,10 @@ fn airUnionInit(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builder.Va
6150 const loaded_enum = ip.loadEnumType(tag_ty.toIntern());6060 const loaded_enum = ip.loadEnumType(tag_ty.toIntern());
6151 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {6061 const llvm_tag_val = switch (loaded_enum.field_values.getOrNone(ip, extra.field_index)) {
6152 .none => try o.builder.intConst(6062 .none => try o.builder.intConst(
6153 try o.lowerType(.fromInterned(union_obj.enum_tag_type)),6063 try o.lowerType(.fromInterned(union_obj.enum_tag_type), .by_value),
6154 extra.field_index, // auto-numbered6064 extra.field_index, // auto-numbered
6155 ),6065 ),
6156 else => |tag_val_ip| try o.lowerValue(tag_val_ip),6066 else => |tag_val_ip| try o.lowerValue(tag_val_ip, .by_value),
6157 };6067 };
6158 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());6068 const tag_ptr = try self.ptraddConst(result_ptr, layout.tagOffset());
6159 try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);6069 try self.store(tag_ptr, layout.tag_align, llvm_tag_val.toValue(), tag_ty, .normal);
...@@ -6215,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde...@@ -6215,7 +6125,7 @@ fn airAddrSpaceCast(self: *FuncGen, inst: Air.Inst.Index) Allocator.Error!Builde
6215 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6125 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6216 const inst_ty = self.typeOfIndex(inst);6126 const inst_ty = self.typeOfIndex(inst);
6217 const operand = try self.resolveInst(ty_op.operand);6127 const operand = try self.resolveInst(ty_op.operand);
6218 return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty), "");6128 return self.wip.cast(.addrspacecast, operand, try self.object.lowerType(inst_ty, .by_value), "");
6219}6129}
62206130
6221fn workIntrinsic(6131fn workIntrinsic(
...@@ -6361,15 +6271,14 @@ fn load(...@@ -6361,15 +6271,14 @@ fn load(
6361 const abi_align = load_ty.abiAlignment(zcu);6271 const abi_align = load_ty.abiAlignment(zcu);
6362 const abi_size = load_ty.abiSize(zcu);6272 const abi_size = load_ty.abiSize(zcu);
63636273
6364 const llvm_load_ty = try o.lowerType(load_ty);
6365 const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {6274 const llvm_ptr_align: Builder.Alignment = switch (ptr_align) {
6366 .none => abi_align.toLlvm(),6275 .none => abi_align.toLlvm(),
6367 else => |a| a.toLlvm(),6276 else => |a| a.toLlvm(),
6368 };6277 };
63696278
6370 if (isByRef(load_ty, zcu)) {6279 if (isByRef(load_ty, zcu)) {
6371 const llvm_usize_ty = try o.lowerType(.usize);6280 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6372 const result_ptr = try fg.buildAlloca(llvm_load_ty, abi_align.toLlvm());6281 const result_ptr = try fg.buildZigAlloca(load_ty, .none);
6373 _ = try fg.wip.callMemCpy(6282 _ = try fg.wip.callMemCpy(
6374 result_ptr,6283 result_ptr,
6375 abi_align.toLlvm(),6284 abi_align.toLlvm(),
...@@ -6382,7 +6291,11 @@ fn load(...@@ -6382,7 +6291,11 @@ fn load(
6382 return result_ptr;6291 return result_ptr;
6383 }6292 }
63846293
6385 if (load_ty.isAbiInt(zcu) and load_ty.bitSize(zcu) != abi_size * 8) {6294 const llvm_memory_ty = try o.lowerType(load_ty, .in_memory);
6295 const llvm_value_ty = try o.lowerType(load_ty, .by_value);
6296
6297 if (llvm_memory_ty != llvm_value_ty) {
6298 assert(load_ty.isAbiInt(zcu));
6386 // `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special6299 // `load_ty` is an integer type with padding bits. In theory, we shouldn't need any special
6387 // handling for these, as LLVM's documented semantics are a valid implementation of Zig's6300 // handling for these, as LLVM's documented semantics are a valid implementation of Zig's
6388 // semantics. However:6301 // semantics. However:
...@@ -6395,8 +6308,7 @@ fn load(...@@ -6395,8 +6308,7 @@ fn load(
6395 //6308 //
6396 // Therefore, we handle these memory accesses specially: in this case we will actually load6309 // Therefore, we handle these memory accesses specially: in this case we will actually load
6397 // the next-largest "natural" integer type and then truncate to `load_ty`.6310 // the next-largest "natural" integer type and then truncate to `load_ty`.
6398 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));6311 const loaded = try fg.wip.load(access_kind, llvm_memory_ty, ptr, llvm_ptr_align, "");
6399 const loaded = try fg.wip.load(access_kind, llvm_abi_ty, ptr, llvm_ptr_align, "");
6400 // For packed structs, current Zig semantics don't really allow us to make the padding bits6312 // For packed structs, current Zig semantics don't really allow us to make the padding bits
6401 // well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is6313 // well-defined. This should be solved once https://github.com/ziglang/zig/issues/24061 is
6402 // implemented, but until then, do a normal trunc for packed types.6314 // implemented, but until then, do a normal trunc for packed types.
...@@ -6406,11 +6318,11 @@ fn load(...@@ -6406,11 +6318,11 @@ fn load(
6406 .unsigned => .@"trunc nuw",6318 .unsigned => .@"trunc nuw",
6407 .signed => .@"trunc nsw",6319 .signed => .@"trunc nsw",
6408 },6320 },
6409 }, loaded, llvm_load_ty, "");6321 }, loaded, llvm_value_ty, "");
6410 }6322 }
64116323
6412 // `load_ty` is a simple by-val type which requires no special handling.6324 // `load_ty` is a simple by-val type which requires no special handling.
6413 return fg.wip.load(access_kind, llvm_load_ty, ptr, llvm_ptr_align, "");6325 return fg.wip.load(access_kind, llvm_value_ty, ptr, llvm_ptr_align, "");
6414}6326}
64156327
6416/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.6328/// Non-atomic, non-bitpacked store of `elem` to pointer `ptr`.
...@@ -6438,7 +6350,7 @@ fn store(...@@ -6438,7 +6350,7 @@ fn store(
6438 };6350 };
64396351
6440 if (isByRef(elem_ty, zcu)) {6352 if (isByRef(elem_ty, zcu)) {
6441 const llvm_usize_ty = try o.lowerType(.usize);6353 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
6442 _ = try fg.wip.callMemCpy(6354 _ = try fg.wip.callMemCpy(
6443 ptr,6355 ptr,
6444 llvm_ptr_align,6356 llvm_ptr_align,
...@@ -6451,16 +6363,19 @@ fn store(...@@ -6451,16 +6363,19 @@ fn store(
6451 return;6363 return;
6452 }6364 }
64536365
6454 assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty));6366 assert(elem.typeOfWip(&fg.wip) == try o.lowerType(elem_ty, .by_value));
64556367
6456 if (elem_ty.isAbiInt(zcu) and elem_ty.bitSize(zcu) != abi_size * 8) {6368 const llvm_memory_ty = try o.lowerType(elem_ty, .in_memory);
6369 const llvm_value_ty = try o.lowerType(elem_ty, .by_value);
6370
6371 if (llvm_memory_ty != llvm_value_ty) {
6372 assert(elem_ty.isAbiInt(zcu));
6457 // `elem_ty` is an integer type with padding bits, so we need to handle it specially---see6373 // `elem_ty` is an integer type with padding bits, so we need to handle it specially---see
6458 // the corresponding comment in `FuncGen.load` for more details.6374 // the corresponding comment in `FuncGen.load` for more details.
6459 const llvm_abi_ty = try o.builder.intType(@intCast(abi_size * 8));
6460 const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {6375 const extended = try fg.wip.cast(switch (elem_ty.intInfo(zcu).signedness) {
6461 .unsigned => .zext,6376 .unsigned => .zext,
6462 .signed => .sext,6377 .signed => .sext,
6463 }, elem, llvm_abi_ty, "");6378 }, elem, llvm_memory_ty, "");
6464 _ = try fg.wip.storeAtomic(6379 _ = try fg.wip.storeAtomic(
6465 access_kind,6380 access_kind,
6466 extended,6381 extended,
...@@ -6486,7 +6401,7 @@ fn store(...@@ -6486,7 +6401,7 @@ fn store(
6486fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {6401fn valgrindMarkUndef(fg: *FuncGen, ptr: Builder.Value, len: Builder.Value) Allocator.Error!void {
6487 const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;6402 const VG_USERREQ__MAKE_MEM_UNDEFINED = 1296236545;
6488 const o = fg.object;6403 const o = fg.object;
6489 const usize_ty = try o.lowerType(.usize);6404 const usize_ty = try o.lowerType(.usize, .by_value);
6490 const zero = try o.builder.intValue(usize_ty, 0);6405 const zero = try o.builder.intValue(usize_ty, 0);
6491 const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);6406 const req = try o.builder.intValue(usize_ty, VG_USERREQ__MAKE_MEM_UNDEFINED);
6492 const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");6407 const ptr_as_usize = try fg.wip.cast(.ptrtoint, ptr, usize_ty, "");
...@@ -6508,7 +6423,7 @@ fn valgrindClientRequest(...@@ -6508,7 +6423,7 @@ fn valgrindClientRequest(
6508 const target = zcu.getTarget();6423 const target = zcu.getTarget();
6509 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;6424 if (!target_util.hasValgrindSupport(target, .stage2_llvm)) return default_value;
65106425
6511 const llvm_usize = try o.lowerType(.usize);6426 const llvm_usize = try o.lowerType(.usize, .by_value);
6512 const usize_align = Type.usize.abiAlignment(zcu).toLlvm();6427 const usize_align = Type.usize.abiAlignment(zcu).toLlvm();
65136428
6514 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);6429 const array_llvm_ty = try o.builder.arrayType(6, llvm_usize);
...@@ -6790,7 +6705,7 @@ const ParamTypeIterator = struct {...@@ -6790,7 +6705,7 @@ const ParamTypeIterator = struct {
6790 while (field_it.next()) |field_index| {6705 while (field_it.next()) |field_index| {
6791 const field_ty = ty.fieldType(field_index, zcu);6706 const field_ty = ty.fieldType(field_index, zcu);
6792 if (!field_ty.hasRuntimeBits(zcu)) continue;6707 if (!field_ty.hasRuntimeBits(zcu)) continue;
6793 it.types_buffer[it.types_len] = try it.object.lowerType(field_ty);6708 it.types_buffer[it.types_len] = try it.object.lowerType(field_ty, .by_value);
6794 it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);6709 it.offsets_buffer[it.types_len] = ty.structFieldOffset(field_index, zcu);
6795 it.types_len += 1;6710 it.types_len += 1;
6796 }6711 }
...@@ -6807,7 +6722,7 @@ const ParamTypeIterator = struct {...@@ -6807,7 +6722,7 @@ const ParamTypeIterator = struct {
6807 it.llvm_index += 1;6722 it.llvm_index += 1;
6808 return .byval;6723 return .byval;
6809 } else {6724 } else {
6810 it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty)};6725 it.types_buffer[0..1].* = .{try it.object.lowerType(scalar_ty, .by_value)};
6811 it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };6726 it.offsets_buffer[0..2].* = .{ 0, scalar_ty.abiSize(zcu) };
6812 it.types_len = 1;6727 it.types_len = 1;
6813 it.llvm_index += 1;6728 it.llvm_index += 1;
...@@ -6988,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para...@@ -6988,166 +6903,138 @@ pub fn iterateParamTypes(object: *Object, fn_info: InternPool.Key.FuncType) Para
6988 };6903 };
6989}6904}
69906905
6991fn returnTypeByRef(zcu: *Zcu, target: *const std.Target, ty: Type) bool {6906pub const FnReturnStrat = union(enum) {
6992 if (isByRef(ty, zcu)) {6907 /// The function return type is OPV (zero-bit), so the LLVM function return type is `void`.
6993 return true;6908 void,
6994 } else if (target.cpu.arch.isX86() and6909 /// An sret parameter is used. The LLVM function return type is `void`.
6995 !target.cpu.has(.x86, .avx512f) and6910 sret,
6996 ty.totalVectorBits(zcu) >= 512)6911 /// The function's return type directly corresponds to the LLVM function return type.
6997 {6912 ///
6998 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns6913 /// The return type is by-val, i.e. `isByRef` returns `false`.
6999 // "512-bit vector arguments require 'avx512f' for AVX512"6914 by_val,
7000 return true;6915 /// The LLVM function returns the given `Builder.Type` by reinterpreting memory containing the
7001 } else {6916 /// actual return value. The actual return type may be by-val or by-ref.
7002 return false;6917 mem_cast: Builder.Type,
7003 }
7004}
7005
7006pub fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const std.Target) bool {
7007 const return_type = Type.fromInterned(fn_info.return_type);
7008 if (!return_type.hasRuntimeBits(zcu)) return false;
7009
7010 return switch (fn_info.cc) {
7011 .auto => returnTypeByRef(zcu, target, return_type),
7012 .x86_64_sysv, .x86_64_x32 => firstParamSRetSystemV(return_type, zcu, target),
7013 .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
7014 .x86_sysv, .x86_win => isByRef(return_type, zcu),
7015 .x86_stdcall => !isScalar(zcu, return_type),
7016 .x86_fastcall => firstParamSRetX86Fastcall(zcu, return_type),
7017 .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
7018 .aarch64_aapcs,
7019 .aarch64_aapcs_darwin,
7020 .aarch64_aapcs_win,
7021 => aarch64_c_abi.classifyType(return_type, zcu) == .memory,
7022 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {
7023 .memory, .i64_array => true,
7024 .i32_array => |size| size != 1,
7025 .byval => false,
7026 },
7027 .riscv64_lp64, .riscv32_ilp32 => riscv_c_abi.classifyType(return_type, zcu) == .memory,
7028 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {
7029 .memory, .i32_array => true,
7030 .byval => false,
7031 },
7032 else => false, // TODO: investigate other targets/callconvs
7033 };
7034}
70356918
7036fn firstParamSRetX86Fastcall(zcu: *Zcu, ty: Type) bool {6919 fn forceByVal(o: *Object, ret_ty: Type) Allocator.Error!FnReturnStrat {
7037 if (isScalar(zcu, ty)) {6920 if (!isByRef(ret_ty, o.zcu)) return .by_val;
7038 return false;6921 return .{ .mem_cast = try o.lowerType(ret_ty, .in_memory) };
7039 }
7040 const tag = ty.zigTypeTag(zcu);
7041 if (tag == .@"struct" or tag == .@"union") {
7042 const size = ty.abiSize(zcu);
7043 if (size == 1 or size == 2 or size == 4 or size == 8) {
7044 return false;
7045 }
7046 }6922 }
7047 return true;6923};
7048}
7049
7050fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: *const std.Target) bool {
7051 if (isScalar(zcu, ty)) return false;
7052 const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret);
7053 if (class[0] == .memory) return true;
7054 if (class[0] == .x87 and class[2] != .none) return true;
7055 return false;
7056}
7057
7058/// In order to support the C calling convention, some return types need to be lowered6924/// In order to support the C calling convention, some return types need to be lowered
7059/// completely differently in the function prototype to honor the C ABI, and then6925/// completely differently in the function prototype to honor the C ABI, and then
7060/// be effectively bitcasted to the actual return type.6926/// be effectively bitcasted to the actual return type.
7061pub fn lowerFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {6927pub fn fnReturnStrat(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7062 const zcu = o.zcu;6928 const zcu = o.zcu;
7063 const return_type = Type.fromInterned(fn_info.return_type);6929 const ret_ty: Type = .fromInterned(fn_info.return_type);
7064 if (!return_type.hasRuntimeBits(zcu)) {6930 ret_ty.assertHasLayout(zcu);
7065 assert(!return_type.isError(zcu));6931 if (!ret_ty.hasRuntimeBits(zcu)) return .void;
7066 return .void;
7067 }
7068 const target = zcu.getTarget();
7069 switch (fn_info.cc) {6932 switch (fn_info.cc) {
7070 .@"inline" => unreachable,6933 .@"inline" => unreachable,
7071 .auto => return if (returnTypeByRef(zcu, target, return_type)) .void else o.lowerType(return_type),6934 .auto => {
6935 if (isByRef(ret_ty, zcu)) return .sret;
6936
6937 const target = zcu.getTarget();
6938 if (target.cpu.arch.isX86() and
6939 !target.cpu.has(.x86, .avx512f) and
6940 ret_ty.totalVectorBits(zcu) >= 512)
6941 {
6942 // As of LLVM 18, passing a vector byval with fastcc that is 512 bits or more returns
6943 // "512-bit vector arguments require 'avx512f' for AVX512"
6944 return .sret;
6945 }
6946
6947 return .by_val;
6948 },
7072 .x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),6949 .x86_64_sysv, .x86_64_x32 => return lowerSystemVFnRetTy(o, fn_info),
7073 .x86_64_win => return lowerWin64FnRetTy(o, fn_info),6950 .x86_64_win => return lowerWin64FnRetTy(o, fn_info),
7074 .x86_stdcall => return if (isScalar(zcu, return_type)) o.lowerType(return_type) else .void,6951 .x86_stdcall => if (isScalar(zcu, ret_ty)) {
7075 .x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, return_type),6952 assert(!isByRef(ret_ty, zcu));
7076 .x86_sysv, .x86_win => return if (isByRef(return_type, zcu)) .void else o.lowerType(return_type),6953 return .by_val;
7077 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(return_type, zcu)) {6954 } else return .sret,
7078 .memory => return .void,6955 .x86_fastcall => return lowerX86FastcallFnRetTy(o, zcu, ret_ty),
7079 .float_array => return o.lowerType(return_type),6956 .x86_sysv, .x86_win => return if (isByRef(ret_ty, zcu)) .sret else .by_val,
7080 .byval => return o.lowerType(return_type),6957 .aarch64_aapcs, .aarch64_aapcs_darwin, .aarch64_aapcs_win => switch (aarch64_c_abi.classifyType(ret_ty, zcu)) {
7081 .integer => return .i64,6958 .memory => return .sret,
7082 .double_integer => return o.builder.arrayType(2, .i64),6959 .float_array, .byval => return .forceByVal(o, ret_ty),
6960 .integer => return .{ .mem_cast = .i64 },
6961 .double_integer => return .{ .mem_cast = try o.builder.arrayType(2, .i64) },
7083 },6962 },
7084 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(return_type, zcu, .ret)) {6963 .arm_aapcs, .arm_aapcs_vfp => switch (arm_c_abi.classifyType(ret_ty, zcu, .ret)) {
7085 .memory, .i64_array => return .void,6964 .memory, .i64_array => return .sret,
7086 .i32_array => |len| return if (len == 1) .i32 else .void,6965 .i32_array => |len| return if (len == 1) .{ .mem_cast = .i32 } else .sret,
7087 .byval => return o.lowerType(return_type),6966 .byval => return .forceByVal(o, ret_ty),
7088 },6967 },
7089 .mips_o32 => switch (mips_c_abi.classifyType(return_type, zcu, .ret)) {6968 .mips_o32 => switch (mips_c_abi.classifyType(ret_ty, zcu, .ret)) {
7090 .memory, .i32_array => return .void,6969 .memory, .i32_array => return .sret,
7091 .byval => return o.lowerType(return_type),6970 .byval => return .forceByVal(o, ret_ty),
7092 },6971 },
7093 .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(return_type, zcu)) {6972 .riscv64_lp64, .riscv32_ilp32 => switch (riscv_c_abi.classifyType(ret_ty, zcu)) {
7094 .memory => return .void,6973 .memory => return .sret,
7095 .integer => return o.builder.intType(@intCast(return_type.bitSize(zcu))),6974 .integer => return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) },
7096 .double_integer => {6975 .double_integer => {
7097 const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {6976 const integer: Builder.Type = switch (zcu.getTarget().cpu.arch) {
7098 .riscv64, .riscv64be => .i64,6977 .riscv64, .riscv64be => .i64,
7099 .riscv32, .riscv32be => .i32,6978 .riscv32, .riscv32be => .i32,
7100 else => unreachable,6979 else => unreachable,
7101 };6980 };
7102 return o.builder.structType(.normal, &.{ integer, integer });6981 return .{ .mem_cast = try o.builder.structType(.normal, &.{ integer, integer }) };
7103 },6982 },
7104 .byval => return o.lowerType(return_type),6983 .byval => return .forceByVal(o, ret_ty),
7105 .fields => {6984 .fields => {
7106 var types_len: usize = 0;6985 var types_len: usize = 0;
7107 var types: [8]Builder.Type = undefined;6986 var types: [8]Builder.Type = undefined;
7108 for (0..return_type.structFieldCount(zcu)) |field_index| {6987 for (0..ret_ty.structFieldCount(zcu)) |field_index| {
7109 const field_ty = return_type.fieldType(field_index, zcu);6988 const field_ty = ret_ty.fieldType(field_index, zcu);
7110 if (!field_ty.hasRuntimeBits(zcu)) continue;6989 if (!field_ty.hasRuntimeBits(zcu)) continue;
7111 types[types_len] = try o.lowerType(field_ty);6990 types[types_len] = try o.lowerType(field_ty, .by_value);
7112 types_len += 1;6991 types_len += 1;
7113 }6992 }
7114 return o.builder.structType(.normal, types[0..types_len]);6993 return .{ .mem_cast = try o.builder.structType(.normal, types[0..types_len]) };
7115 },6994 },
7116 },6995 },
7117 .wasm_mvp => switch (wasm_c_abi.classifyType(return_type, zcu)) {6996 .wasm_mvp => switch (wasm_c_abi.classifyType(ret_ty, zcu)) {
7118 .direct => |scalar_ty| return o.lowerType(scalar_ty),6997 .direct => |scalar_ty| if (scalar_ty.toIntern() == ret_ty.toIntern()) {
7119 .indirect => return .void,6998 assert(!isByRef(ret_ty, zcu));
6999 return .by_val;
7000 } else {
7001 return .{ .mem_cast = try o.lowerType(scalar_ty, .by_value) };
7002 },
7003 .indirect => return .sret,
7120 },7004 },
7121 // TODO investigate other callconvs7005 // TODO investigate other callconvs
7122 else => return o.lowerType(return_type),7006 else => return .forceByVal(o, ret_ty),
7123 }7007 }
7124}7008}
71257009
7126fn lowerX86FastcallFnRetTy(o: *Object, zcu: *Zcu, ty: Type) Allocator.Error!Builder.Type {7010fn lowerX86FastcallFnRetTy(o: *Object, zcu: *Zcu, ty: Type) Allocator.Error!FnReturnStrat {
7127 if (isScalar(zcu, ty)) {7011 if (isScalar(zcu, ty)) {
7128 return o.lowerType(ty);7012 assert(!isByRef(ty, zcu));
7013 return .by_val;
7129 }7014 }
7130 const tag = ty.zigTypeTag(zcu);7015 const tag = ty.zigTypeTag(zcu);
7131 if (tag == .@"struct" or tag == .@"union") {7016 if (tag == .@"struct" or tag == .@"union") {
7132 const size = ty.abiSize(zcu);7017 const size = ty.abiSize(zcu);
7133 if (size == 1 or size == 2 or size == 4 or size == 8) {7018 if (size == 1 or size == 2 or size == 4 or size == 8) {
7134 return o.builder.intType(@intCast(size * 8));7019 return .{ .mem_cast = try o.builder.intType(@intCast(size * 8)) };
7135 }7020 }
7136 }7021 }
7137 return .void;7022 return .sret;
7138}7023}
71397024
7140fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {7025fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7141 const zcu = o.zcu;7026 const zcu = o.zcu;
7142 const return_type = Type.fromInterned(fn_info.return_type);7027 const ret_ty = Type.fromInterned(fn_info.return_type);
7143 switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {7028 switch (x86_64_abi.classifyWindows(ret_ty, zcu, zcu.getTarget(), .ret)) {
7144 .integer => {7029 .integer => if (isScalar(zcu, ret_ty)) {
7145 if (isScalar(zcu, return_type)) {7030 assert(!isByRef(ret_ty, zcu));
7146 return o.lowerType(return_type);7031 return .by_val;
7147 } else {7032 } else {
7148 return o.builder.intType(@intCast(return_type.abiSize(zcu) * 8));7033 return .{ .mem_cast = try o.builder.intType(@intCast(ret_ty.abiSize(zcu) * 8)) };
7149 }
7150 },7034 },
7035 .win_i128 => return .{ .mem_cast = try o.builder.vectorType(.normal, 2, .i64) },
7036 .memory => return .sret,
7037
7151 .sse,7038 .sse,
7152 .bool_vector_mask,7039 .bool_vector_mask,
7153 .integer_per_element,7040 .integer_per_element,
...@@ -7156,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err...@@ -7156,7 +7043,10 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7156 .sse_per_xword,7043 .sse_per_xword,
7157 .sse_per_yword,7044 .sse_per_yword,
7158 .sse_per_zword,7045 .sse_per_zword,
7159 => return o.lowerType(return_type),7046 => {
7047 assert(!isByRef(ret_ty, zcu));
7048 return .by_val;
7049 },
7160 .sseup,7050 .sseup,
7161 .x87,7051 .x87,
7162 .x87up,7052 .x87up,
...@@ -7164,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err...@@ -7164,20 +7054,18 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7164 .float,7054 .float,
7165 .float_combine,7055 .float_combine,
7166 => unreachable,7056 => unreachable,
7167 .win_i128 => return o.builder.vectorType(.normal, 2, .i64),
7168 .memory => return .void,
7169 }7057 }
7170}7058}
71717059
7172fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {7060fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Error!FnReturnStrat {
7173 const zcu = o.zcu;7061 const zcu = o.zcu;
7174 const ip = &zcu.intern_pool;7062 const ip = &zcu.intern_pool;
7175 const return_type = Type.fromInterned(fn_info.return_type);7063 const ret_ty = Type.fromInterned(fn_info.return_type);
7176 return_type.assertHasLayout(zcu);7064 if (isScalar(zcu, ret_ty)) {
7177 if (isScalar(zcu, return_type)) {7065 assert(!isByRef(ret_ty, zcu));
7178 return o.lowerType(return_type);7066 return .by_val;
7179 }7067 }
7180 const classes = x86_64_abi.classifySystemV(return_type, zcu, zcu.getTarget(), .ret);7068 const classes = x86_64_abi.classifySystemV(ret_ty, zcu, zcu.getTarget(), .ret);
7181 var types_index: u32 = 0;7069 var types_index: u32 = 0;
7182 var types_buffer: [8]Builder.Type = undefined;7070 var types_buffer: [8]Builder.Type = undefined;
7183 for (classes) |class| {7071 for (classes) |class| {
...@@ -7207,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7207,13 +7095,13 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7207 types_index += 1;7095 types_index += 1;
7208 },7096 },
7209 .x87 => {7097 .x87 => {
7210 if (types_index != 0 or classes[2] != .none) return .void;7098 if (types_index != 0 or classes[2] != .none) return .sret;
7211 types_buffer[types_index] = .x86_fp80;7099 types_buffer[types_index] = .x86_fp80;
7212 types_index += 1;7100 types_index += 1;
7213 },7101 },
7214 .x87up => continue,7102 .x87up => continue,
7215 .none => break,7103 .none => break,
7216 .memory => return .void,7104 .memory => return .sret,
7217 .win_i128 => unreachable, // windows only7105 .win_i128 => unreachable, // windows only
7218 .bool_vector_mask,7106 .bool_vector_mask,
7219 .integer_per_element,7107 .integer_per_element,
...@@ -7228,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7228,9 +7116,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7228 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});7116 const first_non_integer = std.mem.indexOfNone(x86_64_abi.Class, &classes, &.{.integer});
7229 if (first_non_integer == null or classes[first_non_integer.?] == .none) {7117 if (first_non_integer == null or classes[first_non_integer.?] == .none) {
7230 assert(first_non_integer orelse classes.len == types_index);7118 assert(first_non_integer orelse classes.len == types_index);
7231 switch (ip.indexToKey(return_type.toIntern())) {7119 switch (ip.indexToKey(ret_ty.toIntern())) {
7232 .struct_type => {7120 .struct_type => {
7233 const size = return_type.abiSize(zcu);7121 const size = ret_ty.abiSize(zcu);
7234 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);7122 assert((std.math.divCeil(u64, size, 8) catch unreachable) == types_index);
7235 if (size % 8 > 0) {7123 if (size % 8 > 0) {
7236 types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));7124 types_buffer[types_index - 1] = try o.builder.intType(@intCast(size % 8 * 8));
...@@ -7238,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7238,9 +7126,9 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7238 },7126 },
7239 else => {},7127 else => {},
7240 }7128 }
7241 if (types_index == 1) return types_buffer[0];7129 if (types_index == 1) return .{ .mem_cast = types_buffer[0] };
7242 }7130 }
7243 return o.builder.structType(.normal, types_buffer[0..types_index]);7131 return .{ .mem_cast = try o.builder.structType(.normal, types_buffer[0..types_index]) };
7244}7132}
72457133
7246/// This function deliberately does not handle `_BitInt` because it typically7134/// This function deliberately does not handle `_BitInt` because it typically
...@@ -7380,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {...@@ -7380,7 +7268,7 @@ pub fn isByRef(ty: Type, zcu: *const Zcu) bool {
7380 },7268 },
7381 .@"union" => switch (ty.containerLayout(zcu)) {7269 .@"union" => switch (ty.containerLayout(zcu)) {
7382 .@"packed" => false,7270 .@"packed" => false,
7383 else => ty.hasRuntimeBits(zcu) and !ty.unionHasAllZeroBitFieldTypes(zcu),7271 else => ty.hasRuntimeBits(zcu),
7384 },7272 },
7385 };7273 };
7386}7274}
...@@ -7411,7 +7299,7 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E...@@ -7411,7 +7299,7 @@ fn getAtomicAbiType(fg: *const FuncGen, ty: Type, is_rmw_xchg: bool) Allocator.E
7411fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {7299fn ptraddConst(fg: *FuncGen, ptr: Builder.Value, offset: u64) Allocator.Error!Builder.Value {
7412 if (offset == 0) return ptr;7300 if (offset == 0) return ptr;
7413 const o = fg.object;7301 const o = fg.object;
7414 const llvm_usize_ty = try o.lowerType(.usize);7302 const llvm_usize_ty = try o.lowerType(.usize, .by_value);
7415 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);7303 const offset_val = try o.builder.intValue(llvm_usize_ty, offset);
7416 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");7304 return fg.wip.gep(.inbounds, .i8, ptr, &.{offset_val}, "");
7417}7305}
src/codegen/mips/abi.zig+4-5
...@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {...@@ -18,24 +18,23 @@ pub fn classifyType(ty: Type, zcu: *Zcu, ctx: Context) Class {
18 const max_direct_size = target.ptrBitWidth() * 2;18 const max_direct_size = target.ptrBitWidth() * 2;
19 switch (ty.zigTypeTag(zcu)) {19 switch (ty.zigTypeTag(zcu)) {
20 .@"struct" => {20 .@"struct" => {
21 const bit_size = ty.bitSize(zcu);
22 if (ty.containerLayout(zcu) == .@"packed") {21 if (ty.containerLayout(zcu) == .@"packed") {
23 if (bit_size > max_direct_size) return .memory;22 if (ty.bitSize(zcu) > max_direct_size) return .memory;
24 return .byval;23 return .byval;
25 }24 }
25 const bit_size = ty.abiSize(zcu) * 8;
26 if (bit_size > max_direct_size) return .memory;26 if (bit_size > max_direct_size) return .memory;
27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute27 // TODO: for bit_size <= 32 using byval is more correct, but that needs inreg argument attribute
28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));28 const count = @as(u8, @intCast(std.mem.alignForward(u64, bit_size, 32) / 32));
29 return .{ .i32_array = count };29 return .{ .i32_array = count };
30 },30 },
31 .@"union" => {31 .@"union" => {
32 const bit_size = ty.bitSize(zcu);
33 if (ty.containerLayout(zcu) == .@"packed") {32 if (ty.containerLayout(zcu) == .@"packed") {
34 if (bit_size > max_direct_size) return .memory;33 if (ty.bitSize(zcu) > max_direct_size) return .memory;
35 return .byval;34 return .byval;
36 }35 }
36 const bit_size = ty.abiSize(zcu) * 8;
37 if (bit_size > max_direct_size) return .memory;37 if (bit_size > max_direct_size) return .memory;
38
39 return .byval;38 return .byval;
40 },39 },
41 .bool => return .byval,40 .bool => return .byval,
src/codegen/riscv64/CodeGen.zig+12-7
...@@ -51,7 +51,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};...@@ -51,7 +51,7 @@ const InnerError = codegen.Error || error{OutOfRegisters};
5151
52pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {52pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
53 return comptime &.initMany(&.{53 return comptime &.initMany(&.{
54 .expand_intcast_safe,54 .expand_int_cast_safe,
55 .expand_int_from_float_safe,55 .expand_int_from_float_safe,
56 .expand_int_from_float_optimized_safe,56 .expand_int_from_float_optimized_safe,
57 .expand_add_safe,57 .expand_add_safe,
...@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1453,7 +1453,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1453 .add_safe,1453 .add_safe,
1454 .sub_safe,1454 .sub_safe,
1455 .mul_safe,1455 .mul_safe,
1456 .intcast_safe,1456 .int_cast_safe,
1457 .int_from_float_safe,1457 .int_from_float_safe,
1458 .int_from_float_optimized_safe,1458 .int_from_float_optimized_safe,
1459 => return func.fail("TODO implement safety_checked_instructions", .{}),1459 => return func.fail("TODO implement safety_checked_instructions", .{}),
...@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1479,7 +1479,14 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1479 .ret_ptr => try func.airRetPtr(inst),1479 .ret_ptr => try func.airRetPtr(inst),
1480 .arg => try func.airArg(inst),1480 .arg => try func.airArg(inst),
1481 .assembly => try func.airAsm(inst),1481 .assembly => try func.airAsm(inst),
1482 .bitcast => try func.airBitCast(inst),1482 .bit_cast => try func.airBitCast(inst),
1483 .ptr_cast => try func.airBitCast(inst),
1484 .ptr_from_int => try func.airBitCast(inst),
1485 .int_from_ptr => try func.airBitCast(inst),
1486 .error_cast => try func.airBitCast(inst),
1487 .error_from_int => try func.airBitCast(inst),
1488 .int_from_error => try func.airBitCast(inst),
1489 .union_from_enum => try func.airBitCast(inst),
1483 .block => try func.airBlock(inst),1490 .block => try func.airBlock(inst),
1484 .br => try func.airBr(inst),1491 .br => try func.airBr(inst),
1485 .repeat => try func.airRepeat(inst),1492 .repeat => try func.airRepeat(inst),
...@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1493,7 +1500,7 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1493 .dbg_empty_stmt => func.finishAirBookkeeping(),1500 .dbg_empty_stmt => func.finishAirBookkeeping(),
1494 .fptrunc => try func.airFptrunc(inst),1501 .fptrunc => try func.airFptrunc(inst),
1495 .fpext => try func.airFpext(inst),1502 .fpext => try func.airFpext(inst),
1496 .intcast => try func.airIntCast(inst),1503 .int_cast => try func.airIntCast(inst),
1497 .trunc => try func.airTrunc(inst),1504 .trunc => try func.airTrunc(inst),
1498 .is_non_null => try func.airIsNonNull(inst),1505 .is_non_null => try func.airIsNonNull(inst),
1499 .is_non_null_ptr => try func.airIsNonNullPtr(inst),1506 .is_non_null_ptr => try func.airIsNonNullPtr(inst),
...@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {...@@ -3953,9 +3960,7 @@ fn airPtrElemPtr(func: *Func, inst: Air.Inst.Index) !void {
3953 const elem_ptr_ty = func.typeOfIndex(inst);3960 const elem_ptr_ty = func.typeOfIndex(inst);
3954 const base_ptr_ty = func.typeOf(extra.lhs);3961 const base_ptr_ty = func.typeOf(extra.lhs);
39553962
3956 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {3963 assert(elem_ptr_ty.ptrInfo(zcu).flags.vector_index == .none);
3957 @panic("audit");
3958 }
39593964
3960 const base_ptr_mcv = try func.resolveInst(extra.lhs);3965 const base_ptr_mcv = try func.resolveInst(extra.lhs);
3961 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {3966 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
src/codegen/riscv64/abi.zig+5-6
...@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -16,9 +16,8 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
16 const max_byval_size = target.ptrBitWidth() * 2;16 const max_byval_size = target.ptrBitWidth() * 2;
17 switch (ty.zigTypeTag(zcu)) {17 switch (ty.zigTypeTag(zcu)) {
18 .@"struct" => {18 .@"struct" => {
19 const bit_size = ty.bitSize(zcu);
20 if (ty.containerLayout(zcu) == .@"packed") {19 if (ty.containerLayout(zcu) == .@"packed") {
21 if (bit_size > max_byval_size) return .memory;20 if (ty.bitSize(zcu) > max_byval_size) return .memory;
22 return .byval;21 return .byval;
23 }22 }
2423
...@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {...@@ -40,17 +39,18 @@ pub fn classifyType(ty: Type, zcu: *Zcu) Class {
40 }39 }
4140
42 // TODO this doesn't exactly match what clang produces but its better than nothing41 // TODO this doesn't exactly match what clang produces but its better than nothing
42 const bit_size = ty.abiSize(zcu) * 8;
43 if (bit_size > max_byval_size) return .memory;43 if (bit_size > max_byval_size) return .memory;
44 if (bit_size > max_byval_size / 2) return .double_integer;44 if (bit_size > max_byval_size / 2) return .double_integer;
45 return .integer;45 return .integer;
46 },46 },
47 .@"union" => {47 .@"union" => {
48 const bit_size = ty.bitSize(zcu);
49 if (ty.containerLayout(zcu) == .@"packed") {48 if (ty.containerLayout(zcu) == .@"packed") {
50 if (bit_size > max_byval_size) return .memory;49 if (ty.bitSize(zcu) > max_byval_size) return .memory;
51 return .byval;50 return .byval;
52 }51 }
53 // TODO this doesn't exactly match what clang produces but its better than nothing52 // TODO this doesn't exactly match what clang produces but its better than nothing
53 const bit_size = ty.abiSize(zcu) * 8;
54 if (bit_size > max_byval_size) return .memory;54 if (bit_size > max_byval_size) return .memory;
55 if (bit_size > max_byval_size / 2) return .double_integer;55 if (bit_size > max_byval_size / 2) return .double_integer;
56 return .integer;56 return .integer;
...@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {...@@ -153,13 +153,12 @@ pub fn classifySystem(ty: Type, zcu: *Zcu) [8]SystemClass {
153 },153 },
154 .error_union => {154 .error_union => {
155 const payload_ty = ty.errorUnionPayload(zcu);155 const payload_ty = ty.errorUnionPayload(zcu);
156 const payload_bits = payload_ty.bitSize(zcu);
157156
158 // the error union itself157 // the error union itself
159 result[0] = .integer;158 result[0] = .integer;
160159
161 // anyerror!void can fit into one register160 // anyerror!void can fit into one register
162 if (payload_bits == 0) return result;161 if (!payload_ty.hasRuntimeBits(zcu)) return result;
163162
164 return memory_class;163 return memory_class;
165 },164 },
src/codegen/sparc64/CodeGen.zig+11-4
...@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -538,7 +538,14 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
538 .ret_ptr => try self.airRetPtr(inst),538 .ret_ptr => try self.airRetPtr(inst),
539 .arg => try self.airArg(inst),539 .arg => try self.airArg(inst),
540 .assembly => try self.airAsm(inst),540 .assembly => try self.airAsm(inst),
541 .bitcast => try self.airBitCast(inst),541 .bit_cast => try self.airBitCast(inst),
542 .ptr_cast => try self.airBitCast(inst),
543 .ptr_from_int => try self.airBitCast(inst),
544 .int_from_ptr => try self.airBitCast(inst),
545 .error_cast => try self.airBitCast(inst),
546 .error_from_int => try self.airBitCast(inst),
547 .int_from_error => try self.airBitCast(inst),
548 .union_from_enum => try self.airBitCast(inst),
542 .block => try self.airBlock(inst),549 .block => try self.airBlock(inst),
543 .br => try self.airBr(inst),550 .br => try self.airBr(inst),
544 .repeat => return self.fail("TODO implement `repeat`", .{}),551 .repeat => return self.fail("TODO implement `repeat`", .{}),
...@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -550,7 +557,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
550 .cond_br => try self.airCondBr(inst),557 .cond_br => try self.airCondBr(inst),
551 .fptrunc => @panic("TODO try self.airFptrunc(inst)"),558 .fptrunc => @panic("TODO try self.airFptrunc(inst)"),
552 .fpext => @panic("TODO try self.airFpext(inst)"),559 .fpext => @panic("TODO try self.airFpext(inst)"),
553 .intcast => try self.airIntCast(inst),560 .int_cast => try self.airIntCast(inst),
554 .trunc => try self.airTrunc(inst),561 .trunc => try self.airTrunc(inst),
555 .is_non_null => try self.airIsNonNull(inst),562 .is_non_null => try self.airIsNonNull(inst),
556 .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),563 .is_non_null_ptr => @panic("TODO try self.airIsNonNullPtr(inst)"),
...@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -689,7 +696,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
689 .add_safe,696 .add_safe,
690 .sub_safe,697 .sub_safe,
691 .mul_safe,698 .mul_safe,
692 .intcast_safe,699 .int_cast_safe,
693 .int_from_float_safe,700 .int_from_float_safe,
694 .int_from_float_optimized_safe,701 .int_from_float_optimized_safe,
695 => @panic("TODO implement safety_checked_instructions"),702 => @panic("TODO implement safety_checked_instructions"),
...@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {...@@ -1659,7 +1666,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void {
1659 const info_a = operand_ty.intInfo(zcu);1666 const info_a = operand_ty.intInfo(zcu);
1660 const info_b = self.typeOfIndex(inst).intInfo(zcu);1667 const info_b = self.typeOfIndex(inst).intInfo(zcu);
1661 if (info_a.signedness != info_b.signedness)1668 if (info_a.signedness != info_b.signedness)
1662 return self.fail("TODO gen intcast sign safety in semantic analysis", .{});1669 return self.fail("TODO gen int_cast sign safety in semantic analysis", .{});
16631670
1664 if (info_a.bits == info_b.bits)1671 if (info_a.bits == info_b.bits)
1665 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });1672 return self.finishAir(inst, operand, .{ ty_op.operand, .none, .none });
src/codegen/spirv/CodeGen.zig+67-28
...@@ -34,7 +34,7 @@ const CodeGen = @This();...@@ -34,7 +34,7 @@ const CodeGen = @This();
3434
35pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {35pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
36 return comptime &.initMany(&.{36 return comptime &.initMany(&.{
37 .expand_intcast_safe,37 .expand_int_cast_safe,
38 .expand_int_from_float_safe,38 .expand_int_from_float_safe,
39 .expand_int_from_float_optimized_safe,39 .expand_int_from_float_optimized_safe,
40 .expand_add_safe,40 .expand_add_safe,
...@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {...@@ -1848,7 +1848,17 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id {
1848 .pointer => {1848 .pointer => {
1849 const ptr_info = ty.ptrInfo(zcu);1849 const ptr_info = ty.ptrInfo(zcu);
18501850
1851 const child_ty: Type = .fromInterned(ptr_info.child);1851 const child_ty: Type = switch (ptr_info.packed_offset.host_size) {
1852 0 => .fromInterned(ptr_info.child),
1853 else => switch (ptr_info.flags.vector_index) {
1854 // Accepted proposal https://github.com/ziglang/zig/issues/24061 will eliminate these usages of `pt`.
1855 .none => try pt.intType(.unsigned, ptr_info.packed_offset.host_size * 8),
1856 else => try pt.vectorType(.{
1857 .child = ptr_info.child,
1858 .len = ptr_info.packed_offset.host_size,
1859 }),
1860 },
1861 };
1852 const child_ty_id = try cg.resolveType(child_ty, .indirect);1862 const child_ty_id = try cg.resolveType(child_ty, .indirect);
1853 const storage_class = cg.module.storageClass(ptr_info.flags.address_space);1863 const storage_class = cg.module.storageClass(ptr_info.flags.address_space);
1854 const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);1864 const ptr_ty_id = try cg.module.ptrType(child_ty_id, storage_class);
...@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {...@@ -3847,12 +3857,19 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void {
3847 .min => try cg.airMinMax(inst, .min),3857 .min => try cg.airMinMax(inst, .min),
3848 .max => try cg.airMinMax(inst, .max),3858 .max => try cg.airMinMax(inst, .max),
38493859
3850 .bitcast => try cg.airBitCast(inst),3860 .bit_cast => try cg.airBitCast(inst),
3851 .intcast, .trunc => try cg.airIntCast(inst),3861 .ptr_cast => try cg.airBitCast(inst),
3852 .float_from_int => try cg.airFloatFromInt(inst),3862 .ptr_from_int => try cg.airBitCast(inst),
3853 .int_from_float => try cg.airIntFromFloat(inst),3863 .int_from_ptr => try cg.airBitCast(inst),
3854 .fpext, .fptrunc => try cg.airFloatCast(inst),3864 .error_cast => try cg.airBitCast(inst),
3855 .not => try cg.airNot(inst),3865 .error_from_int => try cg.airBitCast(inst),
3866 .int_from_error => try cg.airBitCast(inst),
3867 .union_from_enum => try cg.airBitCast(inst),
3868 .int_cast, .trunc => try cg.airIntCast(inst),
3869 .float_from_int => try cg.airFloatFromInt(inst),
3870 .int_from_float => try cg.airIntFromFloat(inst),
3871 .fpext, .fptrunc => try cg.airFloatCast(inst),
3872 .not => try cg.airNot(inst),
38563873
3857 .array_to_slice => try cg.airArrayToSlice(inst),3874 .array_to_slice => try cg.airArrayToSlice(inst),
3858 .slice => try cg.airSlice(inst),3875 .slice => try cg.airSlice(inst),
...@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6913,13 +6930,15 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6913 const zcu = cg.module.zcu;6930 const zcu = cg.module.zcu;
6914 const pt = cg.pt;6931 const pt = cg.pt;
6915 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6932 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6916 const ptr_ty = cg.typeOf(ty_op.operand);6933
6917 const ptr_info = ptr_ty.ptrInfo(zcu);6934 const ptr_info = cg.typeOf(ty_op.operand).ptrInfo(zcu);
6935
6918 const elem_ty = cg.typeOfIndex(inst);6936 const elem_ty = cg.typeOfIndex(inst);
6919 const operand = try cg.resolve(ty_op.operand);6937 const operand_ptr_id = try cg.resolve(ty_op.operand);
6920 if (!ptr_ty.isVolatilePtr(zcu) and cg.liveness.isUnused(inst)) return null;
69216938
6922 if (cg.virtual_allocas.get(operand)) |stored| return stored.?;6939 assert(ptr_info.child == elem_ty.toIntern());
6940
6941 if (cg.virtual_allocas.get(operand_ptr_id)) |stored| return stored.?;
69236942
6924 if (ptr_info.packed_offset.host_size != 0 and6943 if (ptr_info.packed_offset.host_size != 0 and
6925 ptr_info.flags.vector_index == .none)6944 ptr_info.flags.vector_index == .none)
...@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6927,7 +6946,7 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6927 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;6946 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6928 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));6947 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6929 const host_int_ty = try pt.intType(.unsigned, host_bits);6948 const host_int_ty = try pt.intType(.unsigned, host_bits);
6930 const host_val = try cg.load(host_int_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });6949 const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6931 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;6950 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6932 const field_int_ty = try pt.intType(signedness, elem_bit_size);6951 const field_int_ty = try pt.intType(signedness, elem_bit_size);
6933 const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {6952 const narrowed = if (ptr_info.packed_offset.bit_offset > 0) blk: {
...@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {...@@ -6946,21 +6965,30 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) !?Id {
6946 return try cg.bitCast(elem_ty, field_int_ty, result_id);6965 return try cg.bitCast(elem_ty, field_int_ty, result_id);
6947 }6966 }
69486967
6949 return try cg.load(elem_ty, operand, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });6968 const ptr_id = switch (ptr_info.flags.vector_index) {
6969 .none => operand_ptr_id,
6970 else => |index| ptr_id: {
6971 const elem_ptr_ty_id = try cg.module.ptrType(
6972 try cg.resolveType(elem_ty, .indirect),
6973 cg.module.storageClass(ptr_info.flags.address_space),
6974 );
6975 break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
6976 },
6977 };
6978 return try cg.load(elem_ty, ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6950}6979}
69516980
6952fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {6981fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6953 const zcu = cg.module.zcu;6982 const zcu = cg.module.zcu;
6954 const pt = cg.pt;6983 const pt = cg.pt;
6955 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;6984 const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
6956 const ptr_ty = cg.typeOf(bin_op.lhs);6985 const ptr_info = cg.typeOf(bin_op.lhs).ptrInfo(zcu);
6957 const ptr_info = ptr_ty.ptrInfo(zcu);6986 const elem_ty: Type = .fromInterned(ptr_info.child);
6958 const elem_ty = ptr_ty.childType(zcu);6987 const operand_ptr_id = try cg.resolve(bin_op.lhs);
6959 const ptr = try cg.resolve(bin_op.lhs);6988 const value_id = try cg.resolve(bin_op.rhs);
6960 const value = try cg.resolve(bin_op.rhs);
69616989
6962 if (cg.virtual_allocas.getPtr(ptr)) |slot| {6990 if (cg.virtual_allocas.getPtr(operand_ptr_id)) |slot| {
6963 slot.* = value;6991 slot.* = value_id;
6964 return;6992 return;
6965 }6993 }
69666994
...@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -6969,19 +6997,19 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
6969 {6997 {
6970 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;6998 const host_bits: u16 = ptr_info.packed_offset.host_size * 8;
6971 const host_int_ty = try pt.intType(.unsigned, host_bits);6999 const host_int_ty = try pt.intType(.unsigned, host_bits);
6972 const host_val = try cg.load(host_int_ty, ptr, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7000 const host_val = try cg.load(host_int_ty, operand_ptr_id, .{ .is_volatile = ptr_info.flags.is_volatile });
6973 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));7001 const elem_bit_size: u16 = @intCast(elem_ty.bitSize(zcu));
6974 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;7002 const signedness: Signedness = if (elem_ty.isInt(zcu)) elem_ty.intInfo(zcu).signedness else .unsigned;
6975 const field_int_ty = try pt.intType(signedness, elem_bit_size);7003 const field_int_ty = try pt.intType(signedness, elem_bit_size);
69767004
6977 var value_as_int: Id = undefined;7005 var value_as_int: Id = undefined;
6978 if (elem_ty.ip_index == .bool_type) {7006 if (elem_ty.ip_index == .bool_type) {
6979 value_as_int = try cg.convertToIndirect(.bool, value);7007 value_as_int = try cg.convertToIndirect(.bool, value_id);
6980 value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);7008 value_as_int = try cg.bitCast(field_int_ty, .u1, value_as_int);
6981 } else if (elem_ty.isInt(zcu)) {7009 } else if (elem_ty.isInt(zcu)) {
6982 value_as_int = value;7010 value_as_int = value_id;
6983 } else {7011 } else {
6984 value_as_int = try cg.bitCast(field_int_ty, elem_ty, value);7012 value_as_int = try cg.bitCast(field_int_ty, elem_ty, value_id);
6985 }7013 }
69867014
6987 const extended = blk: {7015 const extended = blk: {
...@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -7002,11 +7030,22 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index) !void {
7002 const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);7030 const combined = try cg.buildBinary(.OpBitwiseOr, cleared, shifted_val);
7003 const combined_id = try combined.materialize(cg);7031 const combined_id = try combined.materialize(cg);
70047032
7005 try cg.store(host_int_ty, ptr, combined_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7033 try cg.store(host_int_ty, operand_ptr_id, combined_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7006 return;7034 return;
7007 }7035 }
70087036
7009 try cg.store(elem_ty, ptr, value, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) });7037 const ptr_id = switch (ptr_info.flags.vector_index) {
7038 .none => operand_ptr_id,
7039 else => |index| ptr_id: {
7040 const elem_ptr_ty_id = try cg.module.ptrType(
7041 try cg.resolveType(elem_ty, .indirect),
7042 cg.module.storageClass(ptr_info.flags.address_space),
7043 );
7044 break :ptr_id try cg.accessChain(elem_ptr_ty_id, operand_ptr_id, &.{@intFromEnum(index)});
7045 },
7046 };
7047
7048 try cg.store(elem_ty, ptr_id, value_id, .{ .is_volatile = ptr_info.flags.is_volatile });
7010}7049}
70117050
7012fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {7051fn airRet(cg: *CodeGen, inst: Air.Inst.Index) !void {
src/codegen/wasm/CodeGen.zig+96-25
...@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;...@@ -32,7 +32,7 @@ const compilerRtIntAbbrev = target_util.compilerRtIntAbbrev;
3232
33pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {33pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
34 return comptime &.initMany(&.{34 return comptime &.initMany(&.{
35 .expand_intcast_safe,35 .expand_int_cast_safe,
36 .expand_int_from_float_safe,36 .expand_int_from_float_safe,
37 .expand_int_from_float_optimized_safe,37 .expand_int_from_float_optimized_safe,
38 .expand_add_safe,38 .expand_add_safe,
...@@ -108,7 +108,10 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -108,7 +108,10 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
108 .scalarize_cmp_vector_optimized,108 .scalarize_cmp_vector_optimized,
109 .scalarize_fptrunc,109 .scalarize_fptrunc,
110 .scalarize_fpext,110 .scalarize_fpext,
111 .scalarize_intcast,111 .scalarize_int_cast,
112 .scalarize_ptr_cast,
113 .scalarize_ptr_from_int,
114 .scalarize_int_from_ptr,
112 .scalarize_trunc,115 .scalarize_trunc,
113 .scalarize_int_from_float,116 .scalarize_int_from_float,
114 .scalarize_int_from_float_optimized,117 .scalarize_int_from_float_optimized,
...@@ -120,7 +123,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -120,7 +123,7 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
120 .scalarize_select,123 .scalarize_select,
121 .scalarize_mul_add,124 .scalarize_mul_add,
122125
123 .scalarize_bitcast_padded_elems,126 .scalarize_bit_cast_padded_elems,
124 });127 });
125}128}
126129
...@@ -1551,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1551,9 +1554,17 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1551 try cg.finishAir(inst, result, &.{ty_op.operand});1554 try cg.finishAir(inst, result, &.{ty_op.operand});
1552 },1555 },
15531556
1554 .bitcast => cg.airBitcast(inst),1557 .ptr_cast => cg.airNopCast(inst),
1558 .error_cast => cg.airNopCast(inst),
1559 .error_from_int => cg.airNopCast(inst),
1560 .int_from_error => cg.airNopCast(inst),
1561 .ptr_from_int => cg.airNopCast(inst),
1562 .int_from_ptr => cg.airIntFromPtr(inst),
15551563
1556 .intcast => {1564 .bit_cast => cg.airBitcast(inst),
1565 .union_from_enum => cg.airBitcast(inst),
1566
1567 .int_cast => {
1557 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;1568 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
15581569
1559 const dest_ty = ty_op.ty.toType();1570 const dest_ty = ty_op.ty.toType();
...@@ -1561,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1561,7 +1572,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1561 const src_ty = cg.typeOf(ty_op.operand);1572 const src_ty = cg.typeOf(ty_op.operand);
15621573
1563 if (dest_ty.zigTypeTag(zcu) == .vector) {1574 if (dest_ty.zigTypeTag(zcu) == .vector) {
1564 return cg.fail("TODO: implement AIR op: intcast for vectors", .{});1575 return cg.fail("TODO: implement AIR op: int_cast for vectors", .{});
1565 }1576 }
15661577
1567 const src_int_ty: IntType = .fromType(cg, src_ty);1578 const src_int_ty: IntType = .fromType(cg, src_ty);
...@@ -1876,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1876,7 +1887,7 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1876 .add_safe,1887 .add_safe,
1877 .sub_safe,1888 .sub_safe,
1878 .mul_safe,1889 .mul_safe,
1879 .intcast_safe,1890 .int_cast_safe,
1880 .int_from_float_safe,1891 .int_from_float_safe,
1881 .int_from_float_optimized_safe,1892 .int_from_float_optimized_safe,
1882 => return cg.fail("TODO implement safety_checked_instructions", .{}),1893 => return cg.fail("TODO implement safety_checked_instructions", .{}),
...@@ -2100,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {...@@ -2100,16 +2111,20 @@ fn airStore(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) InnerError!void {
2100 const rhs = try cg.resolveInst(bin_op.rhs);2111 const rhs = try cg.resolveInst(bin_op.rhs);
2101 const ptr_ty = cg.typeOf(bin_op.lhs);2112 const ptr_ty = cg.typeOf(bin_op.lhs);
2102 const ptr_info = ptr_ty.ptrInfo(zcu);2113 const ptr_info = ptr_ty.ptrInfo(zcu);
2103 const ty = ptr_ty.childType(zcu);2114 const elem_ty = ptr_ty.childType(zcu);
21042115
2105 if (!safety and bin_op.rhs == .undef) {2116 if (!safety and bin_op.rhs == .undef) {
2106 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2117 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2107 }2118 }
21082119
2109 assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_store2120 const offset: u32 = switch (ptr_info.flags.vector_index) {
21102121 .none => offset: {
2111 try cg.store(lhs, rhs, ty, 0);2122 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_store
21122123 break :offset 0;
2124 },
2125 else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
2126 };
2127 try cg.store(lhs, rhs, elem_ty, offset);
2113 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });2128 return cg.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs });
2114}2129}
21152130
...@@ -2122,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2122,7 +2137,16 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2122 if (!ty.hasRuntimeBits(zcu)) return;2137 if (!ty.hasRuntimeBits(zcu)) return;
21232138
2124 if (isByRef(ty, zcu, cg.target)) {2139 if (isByRef(ty, zcu, cg.target)) {
2125 return cg.memcpy(lhs, rhs, .{ .imm32 = @intCast(abi_size) });2140 const offset_ptr: WValue = switch (offset + lhs.offset()) {
2141 0 => lhs,
2142 else => |total_offset| ptr: {
2143 try cg.emitWValue(lhs);
2144 try cg.addImm32(total_offset);
2145 try cg.addTag(.i32_add);
2146 break :ptr .stack;
2147 },
2148 };
2149 return cg.memcpy(offset_ptr, rhs, .{ .imm32 = @intCast(abi_size) });
2126 }2150 }
21272151
2128 if (ty.zigTypeTag(zcu) == .vector) {2152 if (ty.zigTypeTag(zcu) == .vector) {
...@@ -2134,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr...@@ -2134,7 +2158,7 @@ fn store(cg: *CodeGen, lhs: WValue, rhs: WValue, ty: Type, offset: u32) InnerErr
2134 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{2158 try cg.mir_extra.appendSlice(cg.gpa, &[_]u32{
2135 @intFromEnum(std.wasm.SimdOpcode.v128_store),2159 @intFromEnum(std.wasm.SimdOpcode.v128_store),
2136 offset + lhs.offset(),2160 offset + lhs.offset(),
2137 @intCast(ty.abiAlignment(zcu).toByteUnits() orelse 0),2161 @intCast(ty.abiAlignment(zcu).toByteUnits().?),
2138 });2162 });
2139 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });2163 return cg.addInst(.{ .tag = .simd_prefix, .data = .{ .payload = extra_index } });
2140 }2164 }
...@@ -2175,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2175,15 +2199,20 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2175 const zcu = pt.zcu;2199 const zcu = pt.zcu;
2176 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;2200 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
2177 const operand = try cg.resolveInst(ty_op.operand);2201 const operand = try cg.resolveInst(ty_op.operand);
2178 const ty = ty_op.ty.toType();2202 const elem_ty = ty_op.ty.toType();
2179 const ptr_ty = cg.typeOf(ty_op.operand);2203 const ptr_ty = cg.typeOf(ty_op.operand);
2180 const ptr_info = ptr_ty.ptrInfo(zcu);2204 const ptr_info = ptr_ty.ptrInfo(zcu);
21812205
2182 if (!ty.hasRuntimeBits(zcu)) return cg.finishAir(inst, .none, &.{ty_op.operand});2206 assert(elem_ty.hasRuntimeBits(zcu));
2183
2184 assert(!(ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none)); // legalize .expand_packed_load
21852207
2186 const result = try cg.load(operand, ty, 0);2208 const offset: u32 = switch (ptr_info.flags.vector_index) {
2209 .none => offset: {
2210 assert(ptr_info.packed_offset.host_size == 0); // legalize .expand_packed_load
2211 break :offset 0;
2212 },
2213 else => |index| @intCast(@intFromEnum(index) * elem_ty.abiSize(zcu)),
2214 };
2215 const result = try cg.load(operand, elem_ty, offset);
2187 return cg.finishAir(inst, result, &.{ty_op.operand});2216 return cg.finishAir(inst, result, &.{ty_op.operand});
2188}2217}
21892218
...@@ -2192,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -2192,9 +2221,19 @@ fn airLoad(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
2192fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {2221fn load(cg: *CodeGen, operand: WValue, ty: Type, offset: u32) InnerError!WValue {
2193 const zcu = cg.pt.zcu;2222 const zcu = cg.pt.zcu;
2194 if (isByRef(ty, zcu, cg.target)) {2223 if (isByRef(ty, zcu, cg.target)) {
2195 const val = try cg.allocStack(ty);2224 const src_ptr_maybe_stack: WValue = switch (offset + operand.offset()) {
2196 try cg.store(val, try operand.toLocal(cg, .usize), ty, 0);2225 0 => operand,
2197 return val;2226 else => |total_offset| ptr: {
2227 try cg.emitWValue(operand);
2228 try cg.addImm32(total_offset);
2229 try cg.addTag(.i32_add);
2230 break :ptr .stack;
2231 },
2232 };
2233 const src_ptr = try src_ptr_maybe_stack.toLocal(cg, .usize);
2234 const new_ptr = try cg.allocStack(ty);
2235 try cg.store(new_ptr, src_ptr, ty, 0);
2236 return new_ptr;
2198 }2237 }
21992238
2200 // load local's value from memory by its stack position2239 // load local's value from memory by its stack position
...@@ -5236,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -5236,6 +5275,39 @@ fn airUnreachable(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5236 return cg.finishAir(inst, .none, &.{});5275 return cg.finishAir(inst, .none, &.{});
5237}5276}
52385277
5278fn airNopCast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5279 const zcu = cg.pt.zcu;
5280 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5281
5282 const operand_ty = cg.typeOf(ty_op.operand);
5283 const dest_ty = cg.typeOfIndex(inst);
5284 assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5285 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5286 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5287
5288 const operand = try cg.resolveInst(ty_op.operand);
5289 const result = cg.reuseOperand(ty_op.operand, operand);
5290 return cg.finishAir(inst, result, &.{ty_op.operand});
5291}
5292
5293fn airIntFromPtr(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5294 const zcu = cg.pt.zcu;
5295 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5296
5297 const operand_ty = cg.typeOf(ty_op.operand);
5298 const dest_ty = cg.typeOfIndex(inst);
5299 assert(isByRef(operand_ty, zcu, cg.target) == isByRef(dest_ty, zcu, cg.target));
5300 assert(operand_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
5301 assert(operand_ty.abiAlignment(zcu) == dest_ty.abiAlignment(zcu));
5302
5303 const operand = try cg.resolveInst(ty_op.operand);
5304 const result = switch (operand) {
5305 .stack_offset => try cg.buildPointerOffset(operand, 0, .new),
5306 else => cg.reuseOperand(ty_op.operand, operand),
5307 };
5308 return cg.finishAir(inst, result, &.{ty_op.operand});
5309}
5310
5239fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {5311fn airBitcast(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
5240 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5312 const ty_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5241 const operand = try cg.resolveInst(ty_op.operand);5313 const operand = try cg.resolveInst(ty_op.operand);
...@@ -6341,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -6341,12 +6413,11 @@ fn airArrayElemVal(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
6341 }6413 }
6342 }6414 }
63436415
6344 const elem_result = if (isByRef(elem_ty, zcu, cg.target))6416 const result = if (isByRef(elem_ty, zcu, cg.target))
6345 .stack6417 .stack
6346 else6418 else
6347 try cg.load(.stack, elem_ty, 0);6419 try cg.load(.stack, elem_ty, 0);
63486420 return cg.finishAir(inst, result, &.{ bin_op.lhs, bin_op.rhs });
6349 return cg.finishAir(inst, elem_result, &.{ bin_op.lhs, bin_op.rhs });
6350}6421}
63516422
6352fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {6423fn airSplat(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
src/codegen/x86_64/CodeGen.zig+32-19
...@@ -57,13 +57,13 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {...@@ -57,13 +57,13 @@ pub fn legalizeFeatures(_: *const std.Target) *const Air.Legalize.Features {
57 .scalarize_shuffle_two,57 .scalarize_shuffle_two,
58 .scalarize_select,58 .scalarize_select,
5959
60 .scalarize_bitcast_padded_elems,60 .scalarize_bit_cast_padded_elems,
6161
62 //.unsplat_shift_rhs,62 //.unsplat_shift_rhs,
63 .reduce_one_elem_to_bitcast,63 .reduce_one_elem_to_bit_cast,
64 .splat_one_elem_to_bitcast,64 .splat_one_elem_to_bit_cast,
6565
66 .expand_intcast_safe,66 .expand_int_cast_safe,
67 .expand_int_from_float_safe,67 .expand_int_from_float_safe,
68 .expand_int_from_float_optimized_safe,68 .expand_int_from_float_optimized_safe,
69 .expand_add_safe,69 .expand_add_safe,
...@@ -67434,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -67434,7 +67434,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67434 };67434 };
67435 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);67435 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
67436 },67436 },
67437 .bitcast => try cg.airBitCast(inst),67437 .bit_cast,
67438 .ptr_cast,
67439 .ptr_from_int,
67440 .int_from_ptr,
67441 .error_cast,
67442 .error_from_int,
67443 .int_from_error,
67444 .union_from_enum,
67445 => try cg.airBitCast(inst),
67438 .block => {67446 .block => {
67439 const block = cg.air.unwrapBlock(inst);67447 const block = cg.air.unwrapBlock(inst);
67440 if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);67448 if (!cg.mod.strip) try cg.asmPseudo(.pseudo_dbg_enter_block_none);
...@@ -93375,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -93375,7 +93383,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
93375 };93383 };
93376 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);93384 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
93377 },93385 },
93378 .intcast => |air_tag| {93386 .int_cast => |air_tag| {
93379 const ty_op = air_datas[@intFromEnum(inst)].ty_op;93387 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
93380 const dst_ty = ty_op.ty.toType();93388 const dst_ty = ty_op.ty.toType();
93381 const src_ty = cg.typeOf(ty_op.operand);93389 const src_ty = cg.typeOf(ty_op.operand);
...@@ -98133,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -98133,7 +98141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
98133 };98141 };
98134 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);98142 try res[0].finish(inst, &.{ty_op.operand}, &ops, cg);
98135 },98143 },
98136 .intcast_safe => unreachable,98144 .int_cast_safe => unreachable,
98137 .trunc => |air_tag| {98145 .trunc => |air_tag| {
98138 const ty_op = air_datas[@intFromEnum(inst)].ty_op;98146 const ty_op = air_datas[@intFromEnum(inst)].ty_op;
98139 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});98147 var ops = try cg.tempsFromOperands(inst, .{ty_op.operand});
...@@ -104351,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -104351,7 +104359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
104351 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });104359 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
104352 try ops[0].toSlicePtr(cg);104360 try ops[0].toSlicePtr(cg);
104353 const dst_ty = ty_pl.ty.toType();104361 const dst_ty = ty_pl.ty.toType();
104354 if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: {104362 zero_offset: {
104355 const elem_size = dst_ty.childType(zcu).abiSize(zcu);104363 const elem_size = dst_ty.childType(zcu).abiSize(zcu);
104356 if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;104364 if (hack_around_sema_opv_bugs and elem_size == 0) break :zero_offset;
104357 while (true) for (&ops) |*op| {104365 while (true) for (&ops) |*op| {
...@@ -179054,8 +179062,11 @@ fn genSetReg(...@@ -179054,8 +179062,11 @@ fn genSetReg(
179054 const zcu = pt.zcu;179062 const zcu = pt.zcu;
179055 const abi_size: u32 = @intCast(ty.abiSize(zcu));179063 const abi_size: u32 = @intCast(ty.abiSize(zcu));
179056 const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));179064 const dst_alias = registerAlias(dst_reg, @intCast(cg.unalignedSize(ty)));
179057 if (ty.bitSize(zcu) > dst_alias.size().bitSize(cg.target))179065 {
179058 return cg.fail("genSetReg called with a value larger than dst_reg", .{});179066 const ty_bit_size = if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else 8 * abi_size;
179067 if (ty_bit_size > dst_alias.size().bitSize(cg.target))
179068 return cg.fail("genSetReg called with a value larger than dst_reg", .{});
179069 }
179059 switch (src_mcv) {179070 switch (src_mcv) {
179060 .none,179071 .none,
179061 .unreach,179072 .unreach,
...@@ -180128,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {...@@ -180128,14 +180139,19 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
180128 break :dst dst_mcv;180139 break :dst dst_mcv;
180129 };180140 };
180130180141
180131 if (dst_ty.isRuntimeFloat()) break :result dst_mcv;180142 switch (dst_ty.zigTypeTag(zcu)) {
180143 .float, .error_union, .error_set, .vector => break :result dst_mcv,
180144 .@"struct", .@"union" => if (dst_ty.containerLayout(zcu) != .@"packed") break :result dst_mcv,
180145 .optional, .pointer => if (!dst_ty.isPtrAtRuntime(zcu)) break :result dst_mcv,
180146 else => {},
180147 }
180132180148
180133 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and180149 if (dst_ty.isAbiInt(zcu) and src_ty.isAbiInt(zcu) and src_ty.zigTypeTag(zcu) != .@"struct" and
180134 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;180150 dst_ty.intInfo(zcu).signedness == src_ty.intInfo(zcu).signedness) break :result dst_mcv;
180135180151
180136 const abi_size = dst_ty.abiSize(zcu);180152 const abi_size = dst_ty.abiSize(zcu);
180137 const bit_size = dst_ty.bitSize(zcu);180153 const bit_size = dst_ty.bitSize(zcu);
180138 if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv;180154 if (abi_size * 8 <= bit_size) break :result dst_mcv;
180139180155
180140 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;180156 const dst_limbs_len = std.math.divCeil(u31, @intCast(bit_size), 64) catch unreachable;
180141 const high_mcv: MCValue = switch (dst_mcv) {180157 const high_mcv: MCValue = switch (dst_mcv) {
...@@ -182411,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {...@@ -182411,7 +182427,7 @@ fn truncateRegister(self: *CodeGen, ty: Type, reg: Register) !void {
182411 const zcu = pt.zcu;182427 const zcu = pt.zcu;
182412 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{182428 const int_info: InternPool.Key.IntType = if (ty.isAbiInt(zcu)) ty.intInfo(zcu) else .{
182413 .signedness = .unsigned,182429 .signedness = .unsigned,
182414 .bits = @intCast(ty.bitSize(zcu)),182430 .bits = @intCast(if (ty.hasBitRepresentation(zcu)) ty.bitSize(zcu) else ty.abiSize(zcu) * 8),
182415 };182431 };
182416 const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;182432 const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return;
182417 try self.spillEflagsIfOccupied();182433 try self.spillEflagsIfOccupied();
...@@ -182451,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {...@@ -182451,10 +182467,6 @@ fn regBitSize(self: *CodeGen, ty: Type) u64 {
182451 };182467 };
182452}182468}
182453182469
182454fn regExtraBits(self: *CodeGen, ty: Type) u64 {
182455 return self.regBitSize(ty) - ty.bitSize(self.pt.zcu);
182456}
182457
182458fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {182470fn hasFeature(cg: *CodeGen, feature: std.Target.x86.Feature) bool {
182459 return switch (feature) {182471 return switch (feature) {
182460 .@"64bit" => switch (cg.target.cpu.arch) {182472 .@"64bit" => switch (cg.target.cpu.arch) {
...@@ -182570,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {...@@ -182570,7 +182582,7 @@ fn nonBoolScalarBitSize(cg: *CodeGen, ty: Type) u32 {
182570 .bool_type => vector_type.len,182582 .bool_type => vector_type.len,
182571 else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),182583 else => @intCast(Type.fromInterned(vector_type.child).bitSize(zcu)),
182572 },182584 },
182573 else => @intCast(ty.bitSize(zcu)),182585 else => if (ty.hasBitRepresentation(zcu) or ty.isAbiInt(zcu)) @intCast(ty.bitSize(zcu)) else @intCast(ty.abiSize(zcu) * 8),
182574 };182586 };
182575}182587}
182576182588
...@@ -192243,7 +192255,8 @@ const Select = struct {...@@ -192243,7 +192255,8 @@ const Select = struct {
192243 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),192255 .src0_bit_size => @intCast(s.cg.nonBoolScalarBitSize(Select.Operand.Ref.src0.typeOf(s))),
192244 .@"8_size_sub_bit_size" => {192256 .@"8_size_sub_bit_size" => {
192245 const ty = op.flags.base.ref.typeOf(s);192257 const ty = op.flags.base.ref.typeOf(s);
192246 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - ty.bitSize(s.cg.pt.zcu));192258 const bit_size = s.cg.intInfo(ty).?.bits;
192259 break :lhs @intCast(8 * ty.abiSize(s.cg.pt.zcu) - bit_size);
192247 },192260 },
192248 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),192261 .len => @intCast(op.flags.base.ref.typeOf(s).vectorLen(s.cg.pt.zcu)),
192249 .elem_limbs => @intCast(@divExact(192262 .elem_limbs => @intCast(@divExact(