authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-11-15 16:18:40+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-11-15 16:18:40+00:00
logbc589c271a52e3010ca0af1191c527fb0db63a89
treeff027ccc1ca4ae96c42e6dae0b75bb7f8e7fa66d
parent1ebbdf8eef4904b2a12fad04225ee099e68e1dd7
parentcd8fdd252d9ffd5bfb772cef49e4c79041872d38
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25924 from mlugg/legalize-soft-float

Legalize: implement soft-float legalizations

27 files changed, 1442 insertions(+), 99 deletions(-)

build.zig+5-5
...@@ -678,14 +678,14 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {...@@ -678,14 +678,14 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
678 });678 });
679 run_opt.addArtifactArg(exe);679 run_opt.addArtifactArg(exe);
680 run_opt.addArg("-o");680 run_opt.addArg("-o");
681 run_opt.addFileArg(b.path("stage1/zig1.wasm"));681 const optimized_wasm = run_opt.addOutputFileArg("zig1.wasm");
682682
683 const copy_zig_h = b.addUpdateSourceFiles();683 const update_zig1 = b.addUpdateSourceFiles();
684 copy_zig_h.addCopyFileToSource(b.path("lib/zig.h"), "stage1/zig.h");684 update_zig1.addCopyFileToSource(optimized_wasm, "stage1/zig1.wasm");
685 update_zig1.addCopyFileToSource(b.path("lib/zig.h"), "stage1/zig.h");
685686
686 const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");687 const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm");
687 update_zig1_step.dependOn(&run_opt.step);688 update_zig1_step.dependOn(&update_zig1.step);
688 update_zig1_step.dependOn(&copy_zig_h.step);
689}689}
690690
691const AddCompilerModOptions = struct {691const AddCompilerModOptions = struct {
lib/compiler_rt/arm.zig+10-5
...@@ -15,11 +15,16 @@ comptime {...@@ -15,11 +15,16 @@ comptime {
15 @export(&__aeabi_unwind_cpp_pr1, .{ .name = "__aeabi_unwind_cpp_pr1", .linkage = common.linkage, .visibility = common.visibility });15 @export(&__aeabi_unwind_cpp_pr1, .{ .name = "__aeabi_unwind_cpp_pr1", .linkage = common.linkage, .visibility = common.visibility });
16 @export(&__aeabi_unwind_cpp_pr2, .{ .name = "__aeabi_unwind_cpp_pr2", .linkage = common.linkage, .visibility = common.visibility });16 @export(&__aeabi_unwind_cpp_pr2, .{ .name = "__aeabi_unwind_cpp_pr2", .linkage = common.linkage, .visibility = common.visibility });
1717
18 @export(&__aeabi_ldivmod, .{ .name = if (common.want_windows_arm_abi) "__rt_sdiv64" else "__aeabi_ldivmod", .linkage = common.linkage, .visibility = common.visibility });18 if (common.want_windows_arm_abi) {
19 @export(&__aeabi_uldivmod, .{ .name = if (common.want_windows_arm_abi) "__rt_udiv64" else "__aeabi_uldivmod", .linkage = common.linkage, .visibility = common.visibility });19 @export(&__aeabi_ldivmod, .{ .name = "__rt_sdiv64", .linkage = common.linkage, .visibility = common.visibility });
2020 @export(&__aeabi_uldivmod, .{ .name = "__rt_udiv64", .linkage = common.linkage, .visibility = common.visibility });
21 @export(&__aeabi_idivmod, .{ .name = if (common.want_windows_arm_abi) "__rt_sdiv" else "__aeabi_idivmod", .linkage = common.linkage, .visibility = common.visibility });21 @export(&__aeabi_idivmod, .{ .name = "__rt_sdiv", .linkage = common.linkage, .visibility = common.visibility });
22 @export(&__aeabi_uidivmod, .{ .name = if (common.want_windows_arm_abi) "__rt_udiv" else "__aeabi_uidivmod", .linkage = common.linkage, .visibility = common.visibility });22 @export(&__aeabi_uidivmod, .{ .name = "__rt_udiv", .linkage = common.linkage, .visibility = common.visibility });
23 }
24 @export(&__aeabi_ldivmod, .{ .name = "__aeabi_ldivmod", .linkage = common.linkage, .visibility = common.visibility });
25 @export(&__aeabi_uldivmod, .{ .name = "__aeabi_uldivmod", .linkage = common.linkage, .visibility = common.visibility });
26 @export(&__aeabi_idivmod, .{ .name = "__aeabi_idivmod", .linkage = common.linkage, .visibility = common.visibility });
27 @export(&__aeabi_uidivmod, .{ .name = "__aeabi_uidivmod", .linkage = common.linkage, .visibility = common.visibility });
2328
24 @export(&__aeabi_memcpy, .{ .name = "__aeabi_memcpy", .linkage = common.linkage, .visibility = common.visibility });29 @export(&__aeabi_memcpy, .{ .name = "__aeabi_memcpy", .linkage = common.linkage, .visibility = common.visibility });
25 @export(&__aeabi_memcpy4, .{ .name = "__aeabi_memcpy4", .linkage = common.linkage, .visibility = common.visibility });30 @export(&__aeabi_memcpy4, .{ .name = "__aeabi_memcpy4", .linkage = common.linkage, .visibility = common.visibility });
lib/compiler_rt/aulldiv.zig+1-1
...@@ -8,7 +8,7 @@ const common = @import("common.zig");...@@ -8,7 +8,7 @@ const common = @import("common.zig");
8pub const panic = common.panic;8pub const panic = common.panic;
99
10comptime {10comptime {
11 if (arch == .x86 and common.want_windows_msvc_or_itanium_abi and !builtin.link_libc) {11 if (common.want_windows_x86_msvc_abi) {
12 // Don't let LLVM apply the stdcall name mangling on those MSVC builtins12 // Don't let LLVM apply the stdcall name mangling on those MSVC builtins
13 @export(&_alldiv, .{ .name = "\x01__alldiv", .linkage = common.linkage, .visibility = common.visibility });13 @export(&_alldiv, .{ .name = "\x01__alldiv", .linkage = common.linkage, .visibility = common.visibility });
14 @export(&_aulldiv, .{ .name = "\x01__aulldiv", .linkage = common.linkage, .visibility = common.visibility });14 @export(&_aulldiv, .{ .name = "\x01__aulldiv", .linkage = common.linkage, .visibility = common.visibility });
lib/compiler_rt/aullrem.zig+1-1
...@@ -8,7 +8,7 @@ const common = @import("common.zig");...@@ -8,7 +8,7 @@ const common = @import("common.zig");
8pub const panic = common.panic;8pub const panic = common.panic;
99
10comptime {10comptime {
11 if (arch == .x86 and common.want_windows_msvc_or_itanium_abi and !builtin.link_libc) {11 if (common.want_windows_x86_msvc_abi) {
12 // Don't let LLVM apply the stdcall name mangling on those MSVC builtins12 // Don't let LLVM apply the stdcall name mangling on those MSVC builtins
13 @export(&_allrem, .{ .name = "\x01__allrem", .linkage = common.linkage, .visibility = common.visibility });13 @export(&_allrem, .{ .name = "\x01__allrem", .linkage = common.linkage, .visibility = common.visibility });
14 @export(&_aullrem, .{ .name = "\x01__aullrem", .linkage = common.linkage, .visibility = common.visibility });14 @export(&_aullrem, .{ .name = "\x01__aullrem", .linkage = common.linkage, .visibility = common.visibility });
lib/compiler_rt/common.zig+23-6
...@@ -48,13 +48,30 @@ pub const want_aeabi = switch (builtin.abi) {...@@ -48,13 +48,30 @@ pub const want_aeabi = switch (builtin.abi) {
48 else => false,48 else => false,
49};49};
5050
51/// These functions are provided by libc when targeting MSVC, but not MinGW.51/// These functions are required on Windows on ARM. They are provided by MSVC libc, but in libc-less
52// Temporarily used for thumb-uefi until https://github.com/ziglang/zig/issues/21630 is addressed.52/// builds or when linking MinGW libc they are our responsibility.
53pub const want_windows_arm_abi = builtin.cpu.arch.isArm() and (builtin.os.tag == .windows or builtin.os.tag == .uefi) and (builtin.abi.isGnu() or !builtin.link_libc);53/// Temporarily used for thumb-uefi until https://github.com/ziglang/zig/issues/21630 is addressed.
54pub const want_windows_arm_abi = e: {
55 if (!builtin.cpu.arch.isArm()) break :e false;
56 switch (builtin.os.tag) {
57 .windows, .uefi => {},
58 else => break :e false,
59 }
60 // The ABI is needed, but it's only our reponsibility if libc won't provide it.
61 break :e builtin.abi.isGnu() or !builtin.link_libc;
62};
5463
55pub const want_windows_msvc_or_itanium_abi = switch (builtin.abi) {64/// These functions are required by on Windows on x86 on some ABIs. They are provided by MSVC libc,
56 .none, .msvc, .itanium => builtin.os.tag == .windows,65/// but in libc-less builds they are our responsibility.
57 else => false,66pub const want_windows_x86_msvc_abi = e: {
67 if (builtin.cpu.arch != .x86) break :e false;
68 if (builtin.os.tag != .windows) break :e false;
69 switch (builtin.abi) {
70 .none, .msvc, .itanium => {},
71 else => break :e false,
72 }
73 // The ABI is needed, but it's only our responsibility if libc won't provide it.
74 break :e !builtin.link_libc;
58};75};
5976
60pub const want_ppc_abi = builtin.cpu.arch.isPowerPC();77pub const want_ppc_abi = builtin.cpu.arch.isPowerPC();
lib/compiler_rt/fixdfdi.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_d2lz, .{ .name = "__aeabi_d2lz", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_d2lz, .{ .name = "__aeabi_d2lz", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__fixdfdi, .{ .name = if (common.want_windows_arm_abi) "__dtoi64" else "__fixdfdi", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__fixdfdi, .{ .name = "__dtoi64", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__fixdfdi, .{ .name = "__fixdfdi", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/fixsfdi.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_f2lz, .{ .name = "__aeabi_f2lz", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_f2lz, .{ .name = "__aeabi_f2lz", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__fixsfdi, .{ .name = if (common.want_windows_arm_abi) "__stoi64" else "__fixsfdi", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__fixsfdi, .{ .name = "__stoi64", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__fixsfdi, .{ .name = "__fixsfdi", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/fixunsdfdi.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_d2ulz, .{ .name = "__aeabi_d2ulz", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_d2ulz, .{ .name = "__aeabi_d2ulz", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__fixunsdfdi, .{ .name = if (common.want_windows_arm_abi) "__dtou64" else "__fixunsdfdi", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__fixunsdfdi, .{ .name = "__dtou64", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__fixunsdfdi, .{ .name = "__fixunsdfdi", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/fixunssfdi.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_f2ulz, .{ .name = "__aeabi_f2ulz", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_f2ulz, .{ .name = "__aeabi_f2ulz", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__fixunssfdi, .{ .name = if (common.want_windows_arm_abi) "__stou64" else "__fixunssfdi", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__fixunssfdi, .{ .name = "__stou64", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__fixunssfdi, .{ .name = "__fixunssfdi", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/floatdidf.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_l2d, .{ .name = "__aeabi_l2d", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_l2d, .{ .name = "__aeabi_l2d", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__floatdidf, .{ .name = if (common.want_windows_arm_abi) "__i64tod" else "__floatdidf", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__floatdidf, .{ .name = "__i64tod", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__floatdidf, .{ .name = "__floatdidf", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/floatdisf.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_l2f, .{ .name = "__aeabi_l2f", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_l2f, .{ .name = "__aeabi_l2f", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__floatdisf, .{ .name = if (common.want_windows_arm_abi) "__i64tos" else "__floatdisf", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__floatdisf, .{ .name = "__i64tos", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__floatdisf, .{ .name = "__floatdisf", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/floatundidf.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_ul2d, .{ .name = "__aeabi_ul2d", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_ul2d, .{ .name = "__aeabi_ul2d", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__floatundidf, .{ .name = if (common.want_windows_arm_abi) "__u64tod" else "__floatundidf", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__floatundidf, .{ .name = "__u64tod", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__floatundidf, .{ .name = "__floatundidf", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
lib/compiler_rt/floatundisf.zig+4-1
...@@ -8,7 +8,10 @@ comptime {...@@ -8,7 +8,10 @@ comptime {
8 if (common.want_aeabi) {8 if (common.want_aeabi) {
9 @export(&__aeabi_ul2f, .{ .name = "__aeabi_ul2f", .linkage = common.linkage, .visibility = common.visibility });9 @export(&__aeabi_ul2f, .{ .name = "__aeabi_ul2f", .linkage = common.linkage, .visibility = common.visibility });
10 } else {10 } else {
11 @export(&__floatundisf, .{ .name = if (common.want_windows_arm_abi) "__u64tos" else "__floatundisf", .linkage = common.linkage, .visibility = common.visibility });11 if (common.want_windows_arm_abi) {
12 @export(&__floatundisf, .{ .name = "__u64tos", .linkage = common.linkage, .visibility = common.visibility });
13 }
14 @export(&__floatundisf, .{ .name = "__floatundisf", .linkage = common.linkage, .visibility = common.visibility });
12 }15 }
13}16}
1417
src/Air.zig+354
...@@ -935,6 +935,17 @@ pub const Inst = struct {...@@ -935,6 +935,17 @@ pub const Inst = struct {
935 /// type is the vector element type.935 /// type is the vector element type.
936 legalize_vec_elem_val,936 legalize_vec_elem_val,
937937
938 /// A call to a compiler_rt routine. `Legalize` may emit this instruction if any soft-float
939 /// legalizations are enabled.
940 ///
941 /// Uses the `legalize_compiler_rt_call` union field.
942 ///
943 /// The name of the function symbol is given by `func.name(target)`.
944 /// The calling convention is given by `func.@"callconv"(target)`.
945 /// The return type (and hence the result type of this instruction) is `func.returnType()`.
946 /// The parameter types are the types of the arguments given in `Air.Call`.
947 legalize_compiler_rt_call,
948
938 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {949 pub fn fromCmpOp(op: std.math.CompareOperator, optimized: bool) Tag {
939 switch (op) {950 switch (op) {
940 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,951 .lt => return if (optimized) .cmp_lt_optimized else .cmp_lt,
...@@ -1240,6 +1251,11 @@ pub const Inst = struct {...@@ -1240,6 +1251,11 @@ pub const Inst = struct {
1240 ty: InternPool.Index,1251 ty: InternPool.Index,
1241 nav: InternPool.Nav.Index,1252 nav: InternPool.Nav.Index,
1242 },1253 },
1254 legalize_compiler_rt_call: struct {
1255 func: CompilerRtFunc,
1256 /// Index into `extra` to a payload of type `Call`.
1257 payload: u32,
1258 },
1243 inferred_alloc_comptime: InferredAllocComptime,1259 inferred_alloc_comptime: InferredAllocComptime,
1244 inferred_alloc: InferredAlloc,1260 inferred_alloc: InferredAlloc,
12451261
...@@ -1756,6 +1772,8 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)...@@ -1756,6 +1772,8 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
1756 .work_group_id,1772 .work_group_id,
1757 => return .u32,1773 => return .u32,
17581774
1775 .legalize_compiler_rt_call => return datas[@intFromEnum(inst)].legalize_compiler_rt_call.func.returnType(),
1776
1759 .inferred_alloc => unreachable,1777 .inferred_alloc => unreachable,
1760 .inferred_alloc_comptime => unreachable,1778 .inferred_alloc_comptime => unreachable,
1761 }1779 }
...@@ -1879,6 +1897,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {...@@ -1879,6 +1897,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
1879 .int_from_float_safe,1897 .int_from_float_safe,
1880 .int_from_float_optimized_safe,1898 .int_from_float_optimized_safe,
1881 .legalize_vec_store_elem,1899 .legalize_vec_store_elem,
1900 .legalize_compiler_rt_call,
1882 => true,1901 => true,
18831902
1884 .add,1903 .add,
...@@ -2192,3 +2211,338 @@ pub const CoveragePoint = enum(u1) {...@@ -2192,3 +2211,338 @@ pub const CoveragePoint = enum(u1) {
2192 /// a source location used for coverage instrumentation.2211 /// a source location used for coverage instrumentation.
2193 poi,2212 poi,
2194};2213};
2214
2215pub const CompilerRtFunc = enum(u32) {
2216 // zig fmt: off
2217
2218 // float simple arithmetic
2219 __addhf3, __addsf3, __adddf3, __addxf3, __addtf3,
2220 __subhf3, __subsf3, __subdf3, __subxf3, __subtf3,
2221 __mulhf3, __mulsf3, __muldf3, __mulxf3, __multf3,
2222 __divhf3, __divsf3, __divdf3, __divxf3, __divtf3,
2223
2224 // float minmax
2225 __fminh, fminf, fmin, __fminx, fminq,
2226 __fmaxh, fmaxf, fmax, __fmaxx, fmaxq,
2227
2228 // float round
2229 __ceilh, ceilf, ceil, __ceilx, ceilq,
2230 __floorh, floorf, floor, __floorx, floorq,
2231 __trunch, truncf, trunc, __truncx, truncq,
2232 __roundh, roundf, round, __roundx, roundq,
2233
2234 // float log
2235 __logh, logf, log, __logx, logq,
2236 __log2h, log2f, log2, __log2x, log2q,
2237 __log10h, log10f, log10, __log10x, log10q,
2238
2239 // float exp
2240 __exph, expf, exp, __expx, expq,
2241 __exp2h, exp2f, exp2, __exp2x, exp2q,
2242
2243 // float trigonometry
2244 __sinh, sinf, sin, __sinx, sinq,
2245 __cosh, cosf, cos, __cosx, cosq,
2246 __tanh, tanf, tan, __tanx, tanq,
2247
2248 // float misc ops
2249 __fabsh, fabsf, fabs, __fabsx, fabsq,
2250 __sqrth, sqrtf, sqrt, __sqrtx, sqrtq,
2251 __fmodh, fmodf, fmod, __fmodx, fmodq,
2252 __fmah, fmaf, fma, __fmax, fmaq,
2253
2254 // float comparison
2255 __eqhf2, __eqsf2, __eqdf2, __eqxf2, __eqtf2, // == iff return == 0
2256 __nehf2, __nesf2, __nedf2, __nexf2, __netf2, // != iff return != 0
2257 __lthf2, __ltsf2, __ltdf2, __ltxf2, __lttf2, // < iff return < 0
2258 __lehf2, __lesf2, __ledf2, __lexf2, __letf2, // <= iff return <= 0
2259 __gthf2, __gtsf2, __gtdf2, __gtxf2, __gttf2, // > iff return > 0
2260 __gehf2, __gesf2, __gedf2, __gexf2, __getf2, // >= iff return >= 0
2261
2262 // AEABI float comparison. On ARM, the `sf`/`df` functions above are not available,
2263 // and these must be used instead. They are not just aliases for the above functions
2264 // because they have a different (better) ABI.
2265 __aeabi_fcmpeq, __aeabi_dcmpeq, // ==, returns bool
2266 __aeabi_fcmplt, __aeabi_dcmplt, // <, returns bool
2267 __aeabi_fcmple, __aeabi_dcmple, // <=, returns bool
2268 __aeabi_fcmpgt, __aeabi_dcmpgt, // >, returns bool
2269 __aeabi_fcmpge, __aeabi_dcmpge, // >=, returns bool
2270
2271 // float shortening
2272 // to f16 // to f32 // to f64 // to f80
2273 __trunctfhf2, __trunctfsf2, __trunctfdf2, __trunctfxf2, // from f128
2274 __truncxfhf2, __truncxfsf2, __truncxfdf2, // from f80
2275 __truncdfhf2, __truncdfsf2, // from f64
2276 __truncsfhf2, // from f32
2277
2278 // float widening
2279 // to f128 // to f80 // to f64 // to f32
2280 __extendhftf2, __extendhfxf2, __extendhfdf2, __extendhfsf2, // from f16
2281 __extendsftf2, __extendsfxf2, __extendsfdf2, // from f32
2282 __extenddftf2, __extenddfxf2, // from f64
2283 __extendxftf2, // from f80
2284
2285 // int to float
2286 __floatsihf, __floatsisf, __floatsidf, __floatsixf, __floatsitf, // i32 to float
2287 __floatdihf, __floatdisf, __floatdidf, __floatdixf, __floatditf, // i64 to float
2288 __floattihf, __floattisf, __floattidf, __floattixf, __floattitf, // i128 to float
2289 __floateihf, __floateisf, __floateidf, __floateixf, __floateitf, // arbitrary iN to float
2290 __floatunsihf, __floatunsisf, __floatunsidf, __floatunsixf, __floatunsitf, // u32 to float
2291 __floatundihf, __floatundisf, __floatundidf, __floatundixf, __floatunditf, // u64 to float
2292 __floatuntihf, __floatuntisf, __floatuntidf, __floatuntixf, __floatuntitf, // u128 to float
2293 __floatuneihf, __floatuneisf, __floatuneidf, __floatuneixf, __floatuneitf, // arbitrary uN to float
2294
2295 // float to int
2296 __fixhfsi, __fixsfsi, __fixdfsi, __fixxfsi, __fixtfsi, // float to i32
2297 __fixhfdi, __fixsfdi, __fixdfdi, __fixxfdi, __fixtfdi, // float to i64
2298 __fixhfti, __fixsfti, __fixdfti, __fixxfti, __fixtfti, // float to i128
2299 __fixhfei, __fixsfei, __fixdfei, __fixxfei, __fixtfei, // float to arbitray iN
2300 __fixunshfsi, __fixunssfsi, __fixunsdfsi, __fixunsxfsi, __fixunstfsi, // float to u32
2301 __fixunshfdi, __fixunssfdi, __fixunsdfdi, __fixunsxfdi, __fixunstfdi, // float to u64
2302 __fixunshfti, __fixunssfti, __fixunsdfti, __fixunsxfti, __fixunstfti, // float to u128
2303 __fixunshfei, __fixunssfei, __fixunsdfei, __fixunsxfei, __fixunstfei, // float to arbitray uN
2304
2305 // zig fmt: on
2306
2307 /// Usually, the tag names of `CompilerRtFunc` match the corresponding symbol name, but not
2308 /// always; some target triples have slightly different compiler-rt ABIs for one reason or
2309 /// another.
2310 pub fn name(f: CompilerRtFunc, target: *const std.Target) []const u8 {
2311 const use_gnu_f16_abi = switch (target.cpu.arch) {
2312 .wasm32,
2313 .wasm64,
2314 .riscv64,
2315 .riscv64be,
2316 .riscv32,
2317 .riscv32be,
2318 => false,
2319 .x86, .x86_64 => true,
2320 .arm, .armeb, .thumb, .thumbeb => switch (target.abi) {
2321 .eabi, .eabihf => false,
2322 else => true,
2323 },
2324 else => !target.os.tag.isDarwin(),
2325 };
2326 const use_aeabi = target.cpu.arch.isArm() and switch (target.abi) {
2327 .eabi,
2328 .eabihf,
2329 .musleabi,
2330 .musleabihf,
2331 .gnueabi,
2332 .gnueabihf,
2333 .android,
2334 .androideabi,
2335 => true,
2336 else => false,
2337 };
2338
2339 // GNU didn't like the standard names specifically for conversions between f16
2340 // and f32, so decided to make their own naming convention with blackjack and
2341 // hookers (but only use it on a few random targets of course). This overrides
2342 // the ARM EABI in some cases. I don't like GNU.
2343 if (use_gnu_f16_abi) switch (f) {
2344 .__truncsfhf2 => return "__gnu_f2h_ieee",
2345 .__extendhfsf2 => return "__gnu_h2f_ieee",
2346 else => {},
2347 };
2348
2349 if (use_aeabi) return switch (f) {
2350 .__addsf3 => "__aeabi_fadd",
2351 .__adddf3 => "__aeabi_dadd",
2352 .__subsf3 => "__aeabi_fsub",
2353 .__subdf3 => "__aeabi_dsub",
2354 .__mulsf3 => "__aeabi_fmul",
2355 .__muldf3 => "__aeabi_dmul",
2356 .__divsf3 => "__aeabi_fdiv",
2357 .__divdf3 => "__aeabi_ddiv",
2358 .__truncdfhf2 => "__aeabi_d2h",
2359 .__truncdfsf2 => "__aeabi_d2f",
2360 .__truncsfhf2 => "__aeabi_f2h",
2361 .__extendsfdf2 => "__aeabi_f2d",
2362 .__extendhfsf2 => "__aeabi_h2f",
2363 .__floatsisf => "__aeabi_i2f",
2364 .__floatsidf => "__aeabi_i2d",
2365 .__floatdisf => "__aeabi_l2f",
2366 .__floatdidf => "__aeabi_l2d",
2367 .__floatunsisf => "__aeabi_ui2f",
2368 .__floatunsidf => "__aeabi_ui2d",
2369 .__floatundisf => "__aeabi_ul2f",
2370 .__floatundidf => "__aeabi_ul2d",
2371 .__fixsfsi => "__aeabi_f2iz",
2372 .__fixdfsi => "__aeabi_d2iz",
2373 .__fixsfdi => "__aeabi_f2lz",
2374 .__fixdfdi => "__aeabi_d2lz",
2375 .__fixunssfsi => "__aeabi_f2uiz",
2376 .__fixunsdfsi => "__aeabi_d2uiz",
2377 .__fixunssfdi => "__aeabi_f2ulz",
2378 .__fixunsdfdi => "__aeabi_d2ulz",
2379
2380 // These functions are not available on AEABI. The AEABI equivalents are
2381 // separate fields rather than aliases because they have a different ABI.
2382 .__eqsf2, .__eqdf2 => unreachable,
2383 .__nesf2, .__nedf2 => unreachable,
2384 .__ltsf2, .__ltdf2 => unreachable,
2385 .__lesf2, .__ledf2 => unreachable,
2386 .__gtsf2, .__gtdf2 => unreachable,
2387 .__gesf2, .__gedf2 => unreachable,
2388
2389 else => @tagName(f),
2390 };
2391
2392 return switch (f) {
2393 // These functions are only available on AEABI.
2394 .__aeabi_fcmpeq, .__aeabi_dcmpeq => unreachable,
2395 .__aeabi_fcmplt, .__aeabi_dcmplt => unreachable,
2396 .__aeabi_fcmple, .__aeabi_dcmple => unreachable,
2397 .__aeabi_fcmpgt, .__aeabi_dcmpgt => unreachable,
2398 .__aeabi_fcmpge, .__aeabi_dcmpge => unreachable,
2399
2400 else => @tagName(f),
2401 };
2402 }
2403
2404 pub fn @"callconv"(f: CompilerRtFunc, target: *const std.Target) std.builtin.CallingConvention {
2405 const use_gnu_f16_abi = switch (target.cpu.arch) {
2406 .wasm32,
2407 .wasm64,
2408 .riscv64,
2409 .riscv64be,
2410 .riscv32,
2411 .riscv32be,
2412 => false,
2413 .x86, .x86_64 => true,
2414 .arm, .armeb, .thumb, .thumbeb => switch (target.abi) {
2415 .eabi, .eabihf => false,
2416 else => true,
2417 },
2418 else => !target.os.tag.isDarwin(),
2419 };
2420 const use_aeabi = target.cpu.arch.isArm() and switch (target.abi) {
2421 .eabi,
2422 .eabihf,
2423 .musleabi,
2424 .musleabihf,
2425 .gnueabi,
2426 .gnueabihf,
2427 .android,
2428 .androideabi,
2429 => true,
2430 else => false,
2431 };
2432
2433 if (use_gnu_f16_abi) switch (f) {
2434 .__truncsfhf2,
2435 .__extendhfsf2,
2436 => return target.cCallingConvention().?,
2437 else => {},
2438 };
2439
2440 if (use_aeabi) switch (f) {
2441 // zig fmt: off
2442 .__addsf3, .__adddf3, .__subsf3, .__subdf3,
2443 .__mulsf3, .__muldf3, .__divsf3, .__divdf3,
2444 .__truncdfhf2, .__truncdfsf2, .__truncsfhf2,
2445 .__extendsfdf2, .__extendhfsf2,
2446 .__floatsisf, .__floatsidf, .__floatdisf, .__floatdidf,
2447 .__floatunsisf, .__floatunsidf, .__floatundisf, .__floatundidf,
2448 .__fixsfsi, .__fixdfsi, .__fixsfdi, .__fixdfdi,
2449 .__fixunssfsi, .__fixunsdfsi, .__fixunssfdi, .__fixunsdfdi,
2450 => return .{ .arm_aapcs = .{} },
2451 // zig fmt: on
2452 else => {},
2453 };
2454
2455 return target.cCallingConvention().?;
2456 }
2457
2458 pub fn returnType(f: CompilerRtFunc) Type {
2459 return switch (f) {
2460 .__addhf3, .__subhf3, .__mulhf3, .__divhf3 => .f16,
2461 .__addsf3, .__subsf3, .__mulsf3, .__divsf3 => .f32,
2462 .__adddf3, .__subdf3, .__muldf3, .__divdf3 => .f64,
2463 .__addxf3, .__subxf3, .__mulxf3, .__divxf3 => .f80,
2464 .__addtf3, .__subtf3, .__multf3, .__divtf3 => .f128,
2465
2466 // zig fmt: off
2467 .__fminh, .__fmaxh,
2468 .__ceilh, .__floorh, .__trunch, .__roundh,
2469 .__logh, .__log2h, .__log10h,
2470 .__exph, .__exp2h,
2471 .__sinh, .__cosh, .__tanh,
2472 .__fabsh, .__sqrth, .__fmodh, .__fmah,
2473 => .f16,
2474 .fminf, .fmaxf,
2475 .ceilf, .floorf, .truncf, .roundf,
2476 .logf, .log2f, .log10f,
2477 .expf, .exp2f,
2478 .sinf, .cosf, .tanf,
2479 .fabsf, .sqrtf, .fmodf, .fmaf,
2480 => .f32,
2481 .fmin, .fmax,
2482 .ceil, .floor, .trunc, .round,
2483 .log, .log2, .log10,
2484 .exp, .exp2,
2485 .sin, .cos, .tan,
2486 .fabs, .sqrt, .fmod, .fma,
2487 => .f64,
2488 .__fminx, .__fmaxx,
2489 .__ceilx, .__floorx, .__truncx, .__roundx,
2490 .__logx, .__log2x, .__log10x,
2491 .__expx, .__exp2x,
2492 .__sinx, .__cosx, .__tanx,
2493 .__fabsx, .__sqrtx, .__fmodx, .__fmax,
2494 => .f80,
2495 .fminq, .fmaxq,
2496 .ceilq, .floorq, .truncq, .roundq,
2497 .logq, .log2q, .log10q,
2498 .expq, .exp2q,
2499 .sinq, .cosq, .tanq,
2500 .fabsq, .sqrtq, .fmodq, .fmaq,
2501 => .f128,
2502 // zig fmt: on
2503
2504 .__eqhf2, .__eqsf2, .__eqdf2, .__eqxf2, .__eqtf2 => .i32,
2505 .__nehf2, .__nesf2, .__nedf2, .__nexf2, .__netf2 => .i32,
2506 .__lthf2, .__ltsf2, .__ltdf2, .__ltxf2, .__lttf2 => .i32,
2507 .__lehf2, .__lesf2, .__ledf2, .__lexf2, .__letf2 => .i32,
2508 .__gthf2, .__gtsf2, .__gtdf2, .__gtxf2, .__gttf2 => .i32,
2509 .__gehf2, .__gesf2, .__gedf2, .__gexf2, .__getf2 => .i32,
2510
2511 .__aeabi_fcmpeq, .__aeabi_dcmpeq => .i32,
2512 .__aeabi_fcmplt, .__aeabi_dcmplt => .i32,
2513 .__aeabi_fcmple, .__aeabi_dcmple => .i32,
2514 .__aeabi_fcmpgt, .__aeabi_dcmpgt => .i32,
2515 .__aeabi_fcmpge, .__aeabi_dcmpge => .i32,
2516
2517 .__trunctfhf2, .__truncxfhf2, .__truncdfhf2, .__truncsfhf2 => .f16,
2518 .__trunctfsf2, .__truncxfsf2, .__truncdfsf2 => .f32,
2519 .__trunctfdf2, .__truncxfdf2 => .f64,
2520 .__trunctfxf2 => .f80,
2521
2522 .__extendhftf2, .__extendsftf2, .__extenddftf2, .__extendxftf2 => .f128,
2523 .__extendhfxf2, .__extendsfxf2, .__extenddfxf2 => .f80,
2524 .__extendhfdf2, .__extendsfdf2 => .f64,
2525 .__extendhfsf2 => .f32,
2526
2527 .__floatsihf, .__floatdihf, .__floattihf, .__floateihf => .f16,
2528 .__floatsisf, .__floatdisf, .__floattisf, .__floateisf => .f32,
2529 .__floatsidf, .__floatdidf, .__floattidf, .__floateidf => .f64,
2530 .__floatsixf, .__floatdixf, .__floattixf, .__floateixf => .f80,
2531 .__floatsitf, .__floatditf, .__floattitf, .__floateitf => .f128,
2532 .__floatunsihf, .__floatundihf, .__floatuntihf, .__floatuneihf => .f16,
2533 .__floatunsisf, .__floatundisf, .__floatuntisf, .__floatuneisf => .f32,
2534 .__floatunsidf, .__floatundidf, .__floatuntidf, .__floatuneidf => .f64,
2535 .__floatunsixf, .__floatundixf, .__floatuntixf, .__floatuneixf => .f80,
2536 .__floatunsitf, .__floatunditf, .__floatuntitf, .__floatuneitf => .f128,
2537
2538 .__fixhfsi, .__fixsfsi, .__fixdfsi, .__fixxfsi, .__fixtfsi => .i32,
2539 .__fixhfdi, .__fixsfdi, .__fixdfdi, .__fixxfdi, .__fixtfdi => .i64,
2540 .__fixhfti, .__fixsfti, .__fixdfti, .__fixxfti, .__fixtfti => .i128,
2541 .__fixhfei, .__fixsfei, .__fixdfei, .__fixxfei, .__fixtfei => .void,
2542 .__fixunshfsi, .__fixunssfsi, .__fixunsdfsi, .__fixunsxfsi, .__fixunstfsi => .u32,
2543 .__fixunshfdi, .__fixunssfdi, .__fixunsdfdi, .__fixunsxfdi, .__fixunstfdi => .u64,
2544 .__fixunshfti, .__fixunssfti, .__fixunsdfti, .__fixunsxfti, .__fixunstfti => .u128,
2545 .__fixunshfei, .__fixunssfei, .__fixunsdfei, .__fixunsxfei, .__fixunstfei => .void,
2546 };
2547 }
2548};
src/Air/Legalize.zig+944-68
...@@ -115,6 +115,8 @@ pub const Feature = enum {...@@ -115,6 +115,8 @@ pub const Feature = enum {
115 scalarize_int_from_float_safe,115 scalarize_int_from_float_safe,
116 scalarize_int_from_float_optimized_safe,116 scalarize_int_from_float_optimized_safe,
117 scalarize_float_from_int,117 scalarize_float_from_int,
118 scalarize_reduce,
119 scalarize_reduce_optimized,
118 scalarize_shuffle_one,120 scalarize_shuffle_one,
119 scalarize_shuffle_two,121 scalarize_shuffle_two,
120 scalarize_select,122 scalarize_select,
...@@ -159,6 +161,27 @@ pub const Feature = enum {...@@ -159,6 +161,27 @@ pub const Feature = enum {
159 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.161 /// Replace `aggregate_init` of a packed struct with a sequence of `shl_exact`, `bitcast`, `intcast`, and `bit_or`.
160 expand_packed_aggregate_init,162 expand_packed_aggregate_init,
161163
164 /// Replace all arithmetic operations on 16-bit floating-point types with calls to soft-float
165 /// routines in compiler_rt, including `fptrunc`/`fpext`/`float_from_int`/`int_from_float`
166 /// where the operand or target type is a 16-bit floating-point type. This feature implies:
167 ///
168 /// * scalarization of 16-bit float vector operations
169 /// * expansion of safety-checked 16-bit float operations
170 ///
171 /// If this feature is enabled, the following AIR instruction tags may be emitted:
172 /// * `.legalize_vec_elem_val`
173 /// * `.legalize_vec_store_elem`
174 /// * `.legalize_compiler_rt_call`
175 soft_f16,
176 /// Like `soft_f16`, but for 32-bit floating-point types.
177 soft_f32,
178 /// Like `soft_f16`, but for 64-bit floating-point types.
179 soft_f64,
180 /// Like `soft_f16`, but for 80-bit floating-point types.
181 soft_f80,
182 /// Like `soft_f16`, but for 128-bit floating-point types.
183 soft_f128,
184
162 fn scalarize(tag: Air.Inst.Tag) Feature {185 fn scalarize(tag: Air.Inst.Tag) Feature {
163 return switch (tag) {186 return switch (tag) {
164 else => unreachable,187 else => unreachable,
...@@ -238,6 +261,8 @@ pub const Feature = enum {...@@ -238,6 +261,8 @@ pub const Feature = enum {
238 .int_from_float_safe => .scalarize_int_from_float_safe,261 .int_from_float_safe => .scalarize_int_from_float_safe,
239 .int_from_float_optimized_safe => .scalarize_int_from_float_optimized_safe,262 .int_from_float_optimized_safe => .scalarize_int_from_float_optimized_safe,
240 .float_from_int => .scalarize_float_from_int,263 .float_from_int => .scalarize_float_from_int,
264 .reduce => .scalarize_reduce,
265 .reduce_optimized => .scalarize_reduce_optimized,
241 .shuffle_one => .scalarize_shuffle_one,266 .shuffle_one => .scalarize_shuffle_one,
242 .shuffle_two => .scalarize_shuffle_two,267 .shuffle_two => .scalarize_shuffle_two,
243 .select => .scalarize_select,268 .select => .scalarize_select,
...@@ -283,6 +308,10 @@ fn extraData(l: *const Legalize, comptime T: type, index: usize) @TypeOf(Air.ext...@@ -283,6 +308,10 @@ fn extraData(l: *const Legalize, comptime T: type, index: usize) @TypeOf(Air.ext
283}308}
284309
285fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {310fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
311 // In zig1, this function needs a lot of eval branch quota, because all of the inlined feature
312 // checks are comptime-evaluated (to ensure unused features are not included in the binary).
313 @setEvalBranchQuota(4000);
314
286 const zcu = l.pt.zcu;315 const zcu = l.pt.zcu;
287 const ip = &zcu.intern_pool;316 const ip = &zcu.intern_pool;
288 for (0..body_len) |body_index| {317 for (0..body_len) |body_index| {
...@@ -291,30 +320,67 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -291,30 +320,67 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
291 .arg => {},320 .arg => {},
292 inline .add,321 inline .add,
293 .add_optimized,322 .add_optimized,
294 .add_wrap,
295 .add_sat,
296 .sub,323 .sub,
297 .sub_optimized,324 .sub_optimized,
298 .sub_wrap,
299 .sub_sat,
300 .mul,325 .mul,
301 .mul_optimized,326 .mul_optimized,
302 .mul_wrap,
303 .mul_sat,
304 .div_float,327 .div_float,
305 .div_float_optimized,328 .div_float_optimized,
306 .div_trunc,
307 .div_trunc_optimized,
308 .div_floor,
309 .div_floor_optimized,
310 .div_exact,329 .div_exact,
311 .div_exact_optimized,330 .div_exact_optimized,
312 .rem,331 .rem,
313 .rem_optimized,332 .rem_optimized,
314 .mod,
315 .mod_optimized,
316 .max,
317 .min,333 .min,
334 .max,
335 => |air_tag| {
336 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
337 const ty = l.typeOf(bin_op.lhs);
338 switch (l.wantScalarizeOrSoftFloat(air_tag, ty)) {
339 .none => {},
340 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
341 .soft_float => continue :inst try l.compilerRtCall(
342 inst,
343 softFloatFunc(air_tag, ty, zcu),
344 &.{ bin_op.lhs, bin_op.rhs },
345 l.typeOf(bin_op.lhs),
346 ),
347 }
348 },
349 inline .div_trunc,
350 .div_trunc_optimized,
351 .div_floor,
352 .div_floor_optimized,
353 => |air_tag| {
354 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
355 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(bin_op.lhs))) {
356 .none => {},
357 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
358 .soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatDivTruncFloorBlockPayload(
359 inst,
360 bin_op.lhs,
361 bin_op.rhs,
362 air_tag,
363 )),
364 }
365 },
366 inline .mod, .mod_optimized => |air_tag| {
367 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
368 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(bin_op.lhs))) {
369 .none => {},
370 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .bin_op)),
371 .soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatModBlockPayload(
372 inst,
373 bin_op.lhs,
374 bin_op.rhs,
375 )),
376 }
377 },
378 inline .add_wrap,
379 .add_sat,
380 .sub_wrap,
381 .sub_sat,
382 .mul_wrap,
383 .mul_sat,
318 .bit_and,384 .bit_and,
319 .bit_or,385 .bit_or,
320 .xor,386 .xor,
...@@ -408,20 +474,80 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -408,20 +474,80 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
408 .popcount,474 .popcount,
409 .byte_swap,475 .byte_swap,
410 .bit_reverse,476 .bit_reverse,
411 .abs,
412 .fptrunc,
413 .fpext,
414 .intcast,477 .intcast,
415 .trunc,478 .trunc,
416 .int_from_float,
417 .int_from_float_optimized,
418 .float_from_int,
419 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {479 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {
420 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;480 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
421 if (ty_op.ty.toType().isVector(zcu)) {481 if (ty_op.ty.toType().isVector(zcu)) {
422 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));482 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
423 }483 }
424 },484 },
485 .abs => {
486 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
487 switch (l.wantScalarizeOrSoftFloat(.abs, ty_op.ty.toType())) {
488 .none => {},
489 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
490 .soft_float => continue :inst try l.compilerRtCall(
491 inst,
492 softFloatFunc(.abs, ty_op.ty.toType(), zcu),
493 &.{ty_op.operand},
494 ty_op.ty.toType(),
495 ),
496 }
497 },
498 .fptrunc => {
499 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
500 const src_ty = l.typeOf(ty_op.operand);
501 const dest_ty = ty_op.ty.toType();
502 if (src_ty.zigTypeTag(zcu) == .vector) {
503 if (l.features.has(.scalarize_fptrunc) or
504 l.wantSoftFloatScalar(src_ty.childType(zcu)) or
505 l.wantSoftFloatScalar(dest_ty.childType(zcu)))
506 {
507 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
508 }
509 } else if (l.wantSoftFloatScalar(src_ty) or l.wantSoftFloatScalar(dest_ty)) {
510 continue :inst try l.compilerRtCall(inst, l.softFptruncFunc(src_ty, dest_ty), &.{ty_op.operand}, dest_ty);
511 }
512 },
513 .fpext => {
514 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
515 const src_ty = l.typeOf(ty_op.operand);
516 const dest_ty = ty_op.ty.toType();
517 if (src_ty.zigTypeTag(zcu) == .vector) {
518 if (l.features.has(.scalarize_fpext) or
519 l.wantSoftFloatScalar(src_ty.childType(zcu)) or
520 l.wantSoftFloatScalar(dest_ty.childType(zcu)))
521 {
522 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
523 }
524 } else if (l.wantSoftFloatScalar(src_ty) or l.wantSoftFloatScalar(dest_ty)) {
525 continue :inst try l.compilerRtCall(inst, l.softFpextFunc(src_ty, dest_ty), &.{ty_op.operand}, dest_ty);
526 }
527 },
528 inline .int_from_float, .int_from_float_optimized => |air_tag| {
529 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
530 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(ty_op.operand))) {
531 .none => {},
532 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
533 .soft_float => switch (try l.softIntFromFloat(inst)) {
534 .call => |func| continue :inst try l.compilerRtCall(inst, func, &.{ty_op.operand}, ty_op.ty.toType()),
535 .block_payload => |data| continue :inst l.replaceInst(inst, .block, data),
536 },
537 }
538 },
539 .float_from_int => {
540 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
541 const dest_ty = ty_op.ty.toType();
542 switch (l.wantScalarizeOrSoftFloat(.float_from_int, dest_ty)) {
543 .none => {},
544 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
545 .soft_float => switch (try l.softFloatFromInt(inst)) {
546 .call => |func| continue :inst try l.compilerRtCall(inst, func, &.{ty_op.operand}, dest_ty),
547 .block_payload => |data| continue :inst l.replaceInst(inst, .block, data),
548 },
549 }
550 },
425 .bitcast => if (l.features.has(.scalarize_bitcast)) {551 .bitcast => if (l.features.has(.scalarize_bitcast)) {
426 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {552 if (try l.scalarizeBitcastBlockPayload(inst)) |payload| {
427 continue :inst l.replaceInst(inst, .block, payload);553 continue :inst l.replaceInst(inst, .block, payload);
...@@ -436,22 +562,25 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -436,22 +562,25 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
436 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));562 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));
437 }563 }
438 },564 },
439 .int_from_float_safe => if (l.features.has(.expand_int_from_float_safe)) {565 inline .int_from_float_safe,
440 assert(!l.features.has(.scalarize_int_from_float_safe));566 .int_from_float_optimized_safe,
441 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, false));567 => |air_tag| {
442 } else if (l.features.has(.scalarize_int_from_float_safe)) {568 const optimized = air_tag == .int_from_float_optimized_safe;
443 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;569 const expand_feature = switch (air_tag) {
444 if (ty_op.ty.toType().isVector(zcu)) {570 .int_from_float_safe => .expand_int_from_float_safe,
445 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));571 .int_from_float_optimized_safe => .expand_int_from_float_optimized_safe,
572 else => unreachable,
573 };
574 if (l.features.has(expand_feature)) {
575 assert(!l.features.has(.scalarize(air_tag)));
576 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, optimized));
446 }577 }
447 },
448 .int_from_float_optimized_safe => if (l.features.has(.expand_int_from_float_optimized_safe)) {
449 assert(!l.features.has(.scalarize_int_from_float_optimized_safe));
450 continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, true));
451 } else if (l.features.has(.scalarize_int_from_float_optimized_safe)) {
452 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;578 const ty_op = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_op;
453 if (ty_op.ty.toType().isVector(zcu)) {579 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(ty_op.operand))) {
454 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op));580 .none => {},
581 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .ty_op)),
582 // Expand the safety check so that soft-float can rewrite the unchecked operation.
583 .soft_float => continue :inst l.replaceInst(inst, .block, try l.safeIntFromFloatBlockPayload(inst, optimized)),
455 }584 }
456 },585 },
457 .block, .loop => {586 .block, .loop => {
...@@ -483,12 +612,26 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -483,12 +612,26 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
483 .ceil,612 .ceil,
484 .round,613 .round,
485 .trunc_float,614 .trunc_float,
486 .neg,615 => |air_tag| {
487 .neg_optimized,616 const operand = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
488 => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {617 const ty = l.typeOf(operand);
489 const un_op = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;618 switch (l.wantScalarizeOrSoftFloat(air_tag, ty)) {
490 if (l.typeOf(un_op).isVector(zcu)) {619 .none => {},
491 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op));620 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op)),
621 .soft_float => continue :inst try l.compilerRtCall(
622 inst,
623 softFloatFunc(air_tag, ty, zcu),
624 &.{operand},
625 l.typeOf(operand),
626 ),
627 }
628 },
629 inline .neg, .neg_optimized => |air_tag| {
630 const operand = l.air_instructions.items(.data)[@intFromEnum(inst)].un_op;
631 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(operand))) {
632 .none => {},
633 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .un_op)),
634 .soft_float => continue :inst l.replaceInst(inst, .block, try l.softFloatNegBlockPayload(inst, operand)),
492 }635 }
493 },636 },
494 .cmp_lt,637 .cmp_lt,
...@@ -503,11 +646,24 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -503,11 +646,24 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
503 .cmp_gt_optimized,646 .cmp_gt_optimized,
504 .cmp_neq,647 .cmp_neq,
505 .cmp_neq_optimized,648 .cmp_neq_optimized,
506 => {},649 => |air_tag| {
507 inline .cmp_vector, .cmp_vector_optimized => |air_tag| if (l.features.has(comptime .scalarize(air_tag))) {650 const bin_op = l.air_instructions.items(.data)[@intFromEnum(inst)].bin_op;
651 const ty = l.typeOf(bin_op.lhs);
652 if (l.wantSoftFloatScalar(ty)) {
653 continue :inst l.replaceInst(
654 inst,
655 .block,
656 try l.softFloatCmpBlockPayload(inst, ty, air_tag.toCmpOp().?, bin_op.lhs, bin_op.rhs),
657 );
658 }
659 },
660 inline .cmp_vector, .cmp_vector_optimized => |air_tag| {
508 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;661 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
509 if (ty_pl.ty.toType().isVector(zcu)) {662 const payload = l.extraData(Air.VectorCmp, ty_pl.payload).data;
510 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector));663 switch (l.wantScalarizeOrSoftFloat(air_tag, l.typeOf(payload.lhs))) {
664 .none => {},
665 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .cmp_vector)),
666 .soft_float => unreachable, // the operand is not a scalar
511 }667 }
512 },668 },
513 .cond_br => {669 .cond_br => {
...@@ -615,16 +771,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -615,16 +771,27 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
615 .ptr_elem_ptr,771 .ptr_elem_ptr,
616 .array_to_slice,772 .array_to_slice,
617 => {},773 => {},
618 .reduce, .reduce_optimized => if (l.features.has(.reduce_one_elem_to_bitcast)) {774 inline .reduce, .reduce_optimized => |air_tag| {
619 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;775 const reduce = l.air_instructions.items(.data)[@intFromEnum(inst)].reduce;
620 const vector_ty = l.typeOf(reduce.operand);776 const vector_ty = l.typeOf(reduce.operand);
621 switch (vector_ty.vectorLen(zcu)) {777 if (l.features.has(.reduce_one_elem_to_bitcast)) {
622 0 => unreachable,778 switch (vector_ty.vectorLen(zcu)) {
623 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{779 0 => unreachable,
624 .ty = .fromType(vector_ty.childType(zcu)),780 1 => continue :inst l.replaceInst(inst, .bitcast, .{ .ty_op = .{
625 .operand = reduce.operand,781 .ty = .fromType(vector_ty.childType(zcu)),
626 } }),782 .operand = reduce.operand,
627 else => {},783 } }),
784 else => {},
785 }
786 }
787 switch (l.wantScalarizeOrSoftFloat(air_tag, vector_ty)) {
788 .none => {},
789 .scalarize => continue :inst l.replaceInst(
790 inst,
791 .block,
792 try l.scalarizeReduceBlockPayload(inst, air_tag == .reduce_optimized),
793 ),
794 .soft_float => unreachable, // the operand is not a scalar
628 }795 }
629 },796 },
630 .splat => if (l.features.has(.splat_one_elem_to_bitcast)) {797 .splat => if (l.features.has(.splat_one_elem_to_bitcast)) {
...@@ -638,14 +805,30 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -638,14 +805,30 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
638 else => {},805 else => {},
639 }806 }
640 },807 },
641 .shuffle_one => if (l.features.has(.scalarize_shuffle_one)) {808 .shuffle_one => {
642 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst));809 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
810 switch (l.wantScalarizeOrSoftFloat(.shuffle_one, ty_pl.ty.toType())) {
811 .none => {},
812 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleOneBlockPayload(inst)),
813 .soft_float => unreachable, // the operand is not a scalar
814 }
643 },815 },
644 .shuffle_two => if (l.features.has(.scalarize_shuffle_two)) {816 .shuffle_two => {
645 continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst));817 const ty_pl = l.air_instructions.items(.data)[@intFromEnum(inst)].ty_pl;
818 switch (l.wantScalarizeOrSoftFloat(.shuffle_two, ty_pl.ty.toType())) {
819 .none => {},
820 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeShuffleTwoBlockPayload(inst)),
821 .soft_float => unreachable, // the operand is not a scalar
822 }
646 },823 },
647 .select => if (l.features.has(.scalarize_select)) {824 .select => {
648 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select));825 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
826 const bin = l.extraData(Air.Bin, pl_op.payload).data;
827 switch (l.wantScalarizeOrSoftFloat(.select, l.typeOf(bin.lhs))) {
828 .none => {},
829 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .select)),
830 .soft_float => unreachable, // the operand is not a scalar
831 }
649 },832 },
650 .memset,833 .memset,
651 .memset_safe,834 .memset_safe,
...@@ -685,10 +868,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -685,10 +868,17 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
685 }868 }
686 },869 },
687 .union_init, .prefetch => {},870 .union_init, .prefetch => {},
688 .mul_add => if (l.features.has(.scalarize_mul_add)) {871 .mul_add => {
689 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;872 const pl_op = l.air_instructions.items(.data)[@intFromEnum(inst)].pl_op;
690 if (l.typeOf(pl_op.operand).isVector(zcu)) {873 const ty = l.typeOf(pl_op.operand);
691 continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin));874 switch (l.wantScalarizeOrSoftFloat(.mul_add, ty)) {
875 .none => {},
876 .scalarize => continue :inst l.replaceInst(inst, .block, try l.scalarizeBlockPayload(inst, .pl_op_bin)),
877 .soft_float => {
878 const bin = l.extraData(Air.Bin, pl_op.payload).data;
879 const func = softFloatFunc(.mul_add, ty, zcu);
880 continue :inst try l.compilerRtCall(inst, func, &.{ bin.lhs, bin.rhs, pl_op.operand }, ty);
881 },
692 }882 }
693 },883 },
694 .field_parent_ptr,884 .field_parent_ptr,
...@@ -709,6 +899,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {...@@ -709,6 +899,7 @@ fn legalizeBody(l: *Legalize, body_start: usize, body_len: usize) Error!void {
709 .work_group_id,899 .work_group_id,
710 .legalize_vec_elem_val,900 .legalize_vec_elem_val,
711 .legalize_vec_store_elem,901 .legalize_vec_store_elem,
902 .legalize_compiler_rt_call,
712 => {},903 => {},
713 }904 }
714 }905 }
...@@ -1606,6 +1797,128 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!...@@ -1606,6 +1797,128 @@ fn scalarizeOverflowBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!
1606 .payload = try l.addBlockBody(main_block.body()),1797 .payload = try l.addBlockBody(main_block.body()),
1607 } };1798 } };
1608}1799}
1800fn scalarizeReduceBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index, optimized: bool) Error!Air.Inst.Data {
1801 const pt = l.pt;
1802 const zcu = pt.zcu;
1803
1804 const reduce = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].reduce;
1805
1806 const vector_ty = l.typeOf(reduce.operand);
1807 const scalar_ty = vector_ty.childType(zcu);
1808
1809 const ident_val: Value = switch (reduce.operation) {
1810 // identity for add is 0; identity for OR and XOR is all 0 bits
1811 .Or, .Xor, .Add => switch (scalar_ty.zigTypeTag(zcu)) {
1812 .int => try pt.intValue(scalar_ty, 0),
1813 .float => try pt.floatValue(scalar_ty, 0.0),
1814 else => unreachable,
1815 },
1816 // identity for multiplication is 1
1817 .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
1818 .int => try pt.intValue(scalar_ty, 1),
1819 .float => try pt.floatValue(scalar_ty, 1.0),
1820 else => unreachable,
1821 },
1822 // identity for AND is all 1 bits
1823 .And => switch (scalar_ty.intInfo(zcu).signedness) {
1824 .unsigned => try scalar_ty.maxIntScalar(pt, scalar_ty),
1825 .signed => try pt.intValue(scalar_ty, -1),
1826 },
1827 // identity for @min is maximum value
1828 .Min => switch (scalar_ty.zigTypeTag(zcu)) {
1829 .int => try scalar_ty.maxIntScalar(pt, scalar_ty),
1830 .float => try pt.floatValue(scalar_ty, std.math.inf(f32)),
1831 else => unreachable,
1832 },
1833 // identity for @max is minimum value
1834 .Max => switch (scalar_ty.zigTypeTag(zcu)) {
1835 .int => try scalar_ty.minIntScalar(pt, scalar_ty),
1836 .float => try pt.floatValue(scalar_ty, -std.math.inf(f32)),
1837 else => unreachable,
1838 },
1839 };
1840
1841 const op_tag: Air.Inst.Tag = switch (reduce.operation) {
1842 .Or => .bit_or,
1843 .And => .bit_and,
1844 .Xor => .xor,
1845 .Min => .min,
1846 .Max => .max,
1847 .Add => switch (scalar_ty.zigTypeTag(zcu)) {
1848 .int => .add_wrap,
1849 .float => if (optimized) .add_optimized else .add,
1850 else => unreachable,
1851 },
1852 .Mul => switch (scalar_ty.zigTypeTag(zcu)) {
1853 .int => .mul_wrap,
1854 .float => if (optimized) .mul_optimized else .mul,
1855 else => unreachable,
1856 },
1857 };
1858
1859 // %1 = block(Scalar, {
1860 // %2 = alloc(*usize)
1861 // %3 = alloc(*Scalar)
1862 // %4 = store(%2, @zero_usize)
1863 // %5 = store(%3, <Scalar, 0>) // or whatever the identity is for this operator
1864 // %6 = loop({
1865 // %7 = load(%2)
1866 // %8 = legalize_vec_elem_val(orig_operand, %7)
1867 // %9 = load(%3)
1868 // %10 = add(%8, %9) // or whatever the operator is
1869 // %11 = cmp_eq(%7, <usize, N-1>)
1870 // %12 = cond_br(%11, {
1871 // %13 = br(%1, %10)
1872 // }, {
1873 // %14 = store(%3, %10)
1874 // %15 = add(%7, @one_usize)
1875 // %16 = store(%2, %15)
1876 // %17 = repeat(%6)
1877 // })
1878 // })
1879 // })
1880
1881 var inst_buf: [16]Air.Inst.Index = undefined;
1882 var main_block: Block = .init(&inst_buf);
1883 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
1884
1885 const index_ptr = main_block.addTy(l, .alloc, .ptr_usize).toRef();
1886 const accum_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(scalar_ty)).toRef();
1887 _ = main_block.addBinOp(l, .store, index_ptr, .zero_usize);
1888 _ = main_block.addBinOp(l, .store, accum_ptr, .fromValue(ident_val));
1889
1890 var loop: Loop = .init(l, &main_block);
1891 loop.block = .init(main_block.stealRemainingCapacity());
1892
1893 const index_val = loop.block.addTyOp(l, .load, .usize, index_ptr).toRef();
1894 const elem_val = loop.block.addBinOp(l, .legalize_vec_elem_val, reduce.operand, index_val).toRef();
1895 const old_accum = loop.block.addTyOp(l, .load, scalar_ty, accum_ptr).toRef();
1896 const new_accum = loop.block.addBinOp(l, op_tag, old_accum, elem_val).toRef();
1897
1898 const is_end_val = loop.block.addBinOp(l, .cmp_eq, index_val, .fromValue(try pt.intValue(.usize, vector_ty.vectorLen(zcu) - 1))).toRef();
1899
1900 var condbr: CondBr = .init(l, is_end_val, &loop.block, .{});
1901
1902 condbr.then_block = .init(loop.block.stealRemainingCapacity());
1903 condbr.then_block.addBr(l, orig_inst, new_accum);
1904
1905 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
1906 _ = condbr.else_block.addBinOp(l, .store, accum_ptr, new_accum);
1907 const new_index_val = condbr.else_block.addBinOp(l, .add, index_val, .one_usize).toRef();
1908 _ = condbr.else_block.addBinOp(l, .store, index_ptr, new_index_val);
1909 _ = condbr.else_block.add(l, .{
1910 .tag = .repeat,
1911 .data = .{ .repeat = .{ .loop_inst = loop.inst } },
1912 });
1913
1914 try condbr.finish(l);
1915 try loop.finish(l);
1916
1917 return .{ .ty_pl = .{
1918 .ty = .fromType(scalar_ty),
1919 .payload = try l.addBlockBody(main_block.body()),
1920 } };
1921}
16091922
1610fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {1923fn safeIntcastBlockPayload(l: *Legalize, orig_inst: Air.Inst.Index) Error!Air.Inst.Data {
1611 const pt = l.pt;1924 const pt = l.pt;
...@@ -2298,6 +2611,22 @@ const Block = struct {...@@ -2298,6 +2611,22 @@ const Block = struct {
2298 });2611 });
2299 }2612 }
23002613
2614 fn addCompilerRtCall(b: *Block, l: *Legalize, func: Air.CompilerRtFunc, args: []const Air.Inst.Ref) Error!Air.Inst.Index {
2615 return b.add(l, .{
2616 .tag = .legalize_compiler_rt_call,
2617 .data = .{ .legalize_compiler_rt_call = .{
2618 .func = func,
2619 .payload = payload: {
2620 const extra_len = @typeInfo(Air.Call).@"struct".fields.len + args.len;
2621 try l.air_extra.ensureUnusedCapacity(l.pt.zcu.gpa, extra_len);
2622 const index = l.addExtra(Air.Call, .{ .args_len = @intCast(args.len) }) catch unreachable;
2623 l.air_extra.appendSliceAssumeCapacity(@ptrCast(args));
2624 break :payload index;
2625 },
2626 } },
2627 });
2628 }
2629
2301 /// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,2630 /// Adds the code to call the panic handler `panic_id`. This is usually `.call` then `.unreach`,
2302 /// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.2631 /// but if `Zcu.Feature.panic_fn` is unsupported, we lower to `.trap` instead.
2303 fn addPanic(b: *Block, l: *Legalize, panic_id: Zcu.SimplePanicId) Error!void {2632 fn addPanic(b: *Block, l: *Legalize, panic_id: Zcu.SimplePanicId) Error!void {
...@@ -2365,14 +2694,7 @@ const Block = struct {...@@ -2365,14 +2694,7 @@ const Block = struct {
2365 optimized: bool,2694 optimized: bool,
2366 ) Air.Inst.Index {2695 ) Air.Inst.Index {
2367 return b.add(l, .{2696 return b.add(l, .{
2368 .tag = switch (op) {2697 .tag = .fromCmpOp(op, optimized),
2369 .lt => if (optimized) .cmp_lt_optimized else .cmp_lt,
2370 .lte => if (optimized) .cmp_lte_optimized else .cmp_lte,
2371 .eq => if (optimized) .cmp_eq_optimized else .cmp_eq,
2372 .gte => if (optimized) .cmp_gte_optimized else .cmp_gte,
2373 .gt => if (optimized) .cmp_gt_optimized else .cmp_gt,
2374 .neq => if (optimized) .cmp_neq_optimized else .cmp_neq,
2375 },
2376 .data = .{ .bin_op = .{2698 .data = .{ .bin_op = .{
2377 .lhs = lhs,2699 .lhs = lhs,
2378 .rhs = rhs,2700 .rhs = rhs,
...@@ -2399,6 +2721,82 @@ const Block = struct {...@@ -2399,6 +2721,82 @@ const Block = struct {
2399 return operand;2721 return operand;
2400 }2722 }
24012723
2724 /// This function emits *two* instructions.
2725 fn addSoftFloatCmp(
2726 b: *Block,
2727 l: *Legalize,
2728 float_ty: Type,
2729 op: std.math.CompareOperator,
2730 lhs: Air.Inst.Ref,
2731 rhs: Air.Inst.Ref,
2732 ) Error!Air.Inst.Ref {
2733 const pt = l.pt;
2734 const target = pt.zcu.getTarget();
2735 const use_aeabi = target.cpu.arch.isArm() and switch (target.abi) {
2736 .eabi,
2737 .eabihf,
2738 .musleabi,
2739 .musleabihf,
2740 .gnueabi,
2741 .gnueabihf,
2742 .android,
2743 .androideabi,
2744 => true,
2745 else => false,
2746 };
2747 const func: Air.CompilerRtFunc, const ret_cmp_op: std.math.CompareOperator = switch (float_ty.floatBits(target)) {
2748 // zig fmt: off
2749 16 => switch (op) {
2750 .eq => .{ .__eqhf2, .eq },
2751 .neq => .{ .__nehf2, .neq },
2752 .lt => .{ .__lthf2, .lt },
2753 .lte => .{ .__lehf2, .lte },
2754 .gt => .{ .__gthf2, .gt },
2755 .gte => .{ .__gehf2, .gte },
2756 },
2757 32 => switch (op) {
2758 .eq => if (use_aeabi) .{ .__aeabi_fcmpeq, .neq } else .{ .__eqsf2, .eq },
2759 .neq => if (use_aeabi) .{ .__aeabi_fcmpeq, .eq } else .{ .__nesf2, .neq },
2760 .lt => if (use_aeabi) .{ .__aeabi_fcmplt, .neq } else .{ .__ltsf2, .lt },
2761 .lte => if (use_aeabi) .{ .__aeabi_fcmple, .neq } else .{ .__lesf2, .lte },
2762 .gt => if (use_aeabi) .{ .__aeabi_fcmpgt, .neq } else .{ .__gtsf2, .gt },
2763 .gte => if (use_aeabi) .{ .__aeabi_fcmpge, .neq } else .{ .__gesf2, .gte },
2764 },
2765 64 => switch (op) {
2766 .eq => if (use_aeabi) .{ .__aeabi_dcmpeq, .neq } else .{ .__eqdf2, .eq },
2767 .neq => if (use_aeabi) .{ .__aeabi_dcmpeq, .eq } else .{ .__nedf2, .neq },
2768 .lt => if (use_aeabi) .{ .__aeabi_dcmplt, .neq } else .{ .__ltdf2, .lt },
2769 .lte => if (use_aeabi) .{ .__aeabi_dcmple, .neq } else .{ .__ledf2, .lte },
2770 .gt => if (use_aeabi) .{ .__aeabi_dcmpgt, .neq } else .{ .__gtdf2, .gt },
2771 .gte => if (use_aeabi) .{ .__aeabi_dcmpge, .neq } else .{ .__gedf2, .gte },
2772 },
2773 80 => switch (op) {
2774 .eq => .{ .__eqxf2, .eq },
2775 .neq => .{ .__nexf2, .neq },
2776 .lt => .{ .__ltxf2, .lt },
2777 .lte => .{ .__lexf2, .lte },
2778 .gt => .{ .__gtxf2, .gt },
2779 .gte => .{ .__gexf2, .gte },
2780 },
2781 128 => switch (op) {
2782 .eq => .{ .__eqtf2, .eq },
2783 .neq => .{ .__netf2, .neq },
2784 .lt => .{ .__lttf2, .lt },
2785 .lte => .{ .__letf2, .lte },
2786 .gt => .{ .__gttf2, .gt },
2787 .gte => .{ .__getf2, .gte },
2788 },
2789 else => unreachable,
2790 // zig fmt: on
2791 };
2792 const call_inst = try b.addCompilerRtCall(l, func, &.{ lhs, rhs });
2793 const raw_result = call_inst.toRef();
2794 assert(l.typeOf(raw_result).toIntern() == .i32_type);
2795 const zero_i32: Air.Inst.Ref = .fromValue(try pt.intValue(.i32, 0));
2796 const ret_cmp_tag: Air.Inst.Tag = .fromCmpOp(ret_cmp_op, false);
2797 return b.addBinOp(l, ret_cmp_tag, raw_result, zero_i32).toRef();
2798 }
2799
2402 /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`.2800 /// Returns the unused capacity of `b.instructions`, and shrinks `b.instructions` down to `b.len`.
2403 /// This is useful when you've provided a buffer big enough for all your instructions, but you are2801 /// This is useful when you've provided a buffer big enough for all your instructions, but you are
2404 /// now starting a new block and some of them need to live there instead.2802 /// now starting a new block and some of them need to live there instead.
...@@ -2525,6 +2923,484 @@ inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst...@@ -2525,6 +2923,484 @@ inline fn replaceInst(l: *Legalize, inst: Air.Inst.Index, comptime tag: Air.Inst
2525 return tag;2923 return tag;
2526}2924}
25272925
2926fn compilerRtCall(
2927 l: *Legalize,
2928 orig_inst: Air.Inst.Index,
2929 func: Air.CompilerRtFunc,
2930 args: []const Air.Inst.Ref,
2931 result_ty: Type,
2932) Error!Air.Inst.Tag {
2933 const zcu = l.pt.zcu;
2934 const gpa = zcu.gpa;
2935
2936 const func_ret_ty = func.returnType();
2937
2938 if (func_ret_ty.toIntern() == result_ty.toIntern()) {
2939 try l.air_extra.ensureUnusedCapacity(gpa, @typeInfo(Air.Call).@"struct".fields.len + args.len);
2940 const payload = l.addExtra(Air.Call, .{ .args_len = @intCast(args.len) }) catch unreachable;
2941 l.air_extra.appendSliceAssumeCapacity(@ptrCast(args));
2942 return l.replaceInst(orig_inst, .legalize_compiler_rt_call, .{ .legalize_compiler_rt_call = .{
2943 .func = func,
2944 .payload = payload,
2945 } });
2946 }
2947
2948 // We need to bitcast the result to an "alias" type (e.g. c_int/i32, c_longdouble/f128).
2949
2950 assert(func_ret_ty.bitSize(zcu) == result_ty.bitSize(zcu));
2951
2952 var inst_buf: [3]Air.Inst.Index = undefined;
2953 var main_block: Block = .init(&inst_buf);
2954 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
2955
2956 const call_inst = try main_block.addCompilerRtCall(l, func, args);
2957 const casted_result = main_block.addBitCast(l, result_ty, call_inst.toRef());
2958 main_block.addBr(l, orig_inst, casted_result);
2959
2960 return l.replaceInst(orig_inst, .block, .{ .ty_pl = .{
2961 .ty = .fromType(result_ty),
2962 .payload = try l.addBlockBody(main_block.body()),
2963 } });
2964}
2965
2966fn softFptruncFunc(l: *const Legalize, src_ty: Type, dst_ty: Type) Air.CompilerRtFunc {
2967 const target = l.pt.zcu.getTarget();
2968 const src_bits = src_ty.floatBits(target);
2969 const dst_bits = dst_ty.floatBits(target);
2970 assert(dst_bits < src_bits);
2971 const to_f16_func: Air.CompilerRtFunc = switch (src_bits) {
2972 128 => .__trunctfhf2,
2973 80 => .__truncxfhf2,
2974 64 => .__truncdfhf2,
2975 32 => .__truncsfhf2,
2976 else => unreachable,
2977 };
2978 const offset: u8 = switch (dst_bits) {
2979 16 => 0,
2980 32 => 1,
2981 64 => 2,
2982 80 => 3,
2983 else => unreachable,
2984 };
2985 return @enumFromInt(@intFromEnum(to_f16_func) + offset);
2986}
2987fn softFpextFunc(l: *const Legalize, src_ty: Type, dst_ty: Type) Air.CompilerRtFunc {
2988 const target = l.pt.zcu.getTarget();
2989 const src_bits = src_ty.floatBits(target);
2990 const dst_bits = dst_ty.floatBits(target);
2991 assert(dst_bits > src_bits);
2992 const to_f128_func: Air.CompilerRtFunc = switch (src_bits) {
2993 16 => .__extendhftf2,
2994 32 => .__extendsftf2,
2995 64 => .__extenddftf2,
2996 80 => .__extendxftf2,
2997 else => unreachable,
2998 };
2999 const offset: u8 = switch (dst_bits) {
3000 128 => 0,
3001 80 => 1,
3002 64 => 2,
3003 32 => 3,
3004 else => unreachable,
3005 };
3006 return @enumFromInt(@intFromEnum(to_f128_func) + offset);
3007}
3008fn softFloatFromInt(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3009 call: Air.CompilerRtFunc,
3010 block_payload: Air.Inst.Data,
3011} {
3012 const pt = l.pt;
3013 const zcu = pt.zcu;
3014 const target = zcu.getTarget();
3015
3016 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
3017 const dest_ty = ty_op.ty.toType();
3018 const src_ty = l.typeOf(ty_op.operand);
3019
3020 const src_info = src_ty.intInfo(zcu);
3021 const float_off: u32 = switch (dest_ty.floatBits(target)) {
3022 16 => 0,
3023 32 => 1,
3024 64 => 2,
3025 80 => 3,
3026 128 => 4,
3027 else => unreachable,
3028 };
3029 const base: Air.CompilerRtFunc = switch (src_info.signedness) {
3030 .signed => .__floatsihf,
3031 .unsigned => .__floatunsihf,
3032 };
3033 fixed: {
3034 const extended_int_bits: u16, const int_bits_off: u32 = switch (src_info.bits) {
3035 0...32 => .{ 32, 0 },
3036 33...64 => .{ 64, 5 },
3037 65...128 => .{ 128, 10 },
3038 else => break :fixed,
3039 };
3040 // x86_64-windows uses an odd callconv for 128-bit integers, so we use the
3041 // arbitrary-precision routine in that case for simplicity.
3042 if (target.cpu.arch == .x86_64 and target.os.tag == .windows and extended_int_bits == 128) {
3043 break :fixed;
3044 }
3045
3046 const func: Air.CompilerRtFunc = @enumFromInt(@intFromEnum(base) + int_bits_off + float_off);
3047 if (extended_int_bits == src_info.bits) return .{ .call = func };
3048
3049 // We need to emit a block which first sign/zero-extends to the right type and *then* calls
3050 // the required routine.
3051 const extended_ty = try l.pt.intType(src_info.signedness, extended_int_bits);
3052
3053 var inst_buf: [4]Air.Inst.Index = undefined;
3054 var main_block: Block = .init(&inst_buf);
3055 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3056
3057 const extended_val = main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
3058 const call_inst = try main_block.addCompilerRtCall(l, func, &.{extended_val});
3059 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3060 main_block.addBr(l, orig_inst, casted_result);
3061
3062 return .{ .block_payload = .{ .ty_pl = .{
3063 .ty = .fromType(dest_ty),
3064 .payload = try l.addBlockBody(main_block.body()),
3065 } } };
3066 }
3067
3068 // We need to emit a block which puts the integer into an `alloc` (possibly sign/zero-extended)
3069 // and calls an arbitrary-width conversion routine.
3070
3071 const func: Air.CompilerRtFunc = @enumFromInt(@intFromEnum(base) + 15 + float_off);
3072
3073 // The extended integer routines expect the integer representation where the integer is
3074 // effectively zero- or sign-extended to its ABI size. We represent that by intcasting to
3075 // such an integer type and passing a pointer to *that*.
3076 const extended_ty = try pt.intType(src_info.signedness, @intCast(src_ty.abiSize(zcu) * 8));
3077 assert(extended_ty.abiSize(zcu) == src_ty.abiSize(zcu));
3078
3079 var inst_buf: [6]Air.Inst.Index = undefined;
3080 var main_block: Block = .init(&inst_buf);
3081 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3082
3083 const extended_val: Air.Inst.Ref = if (extended_ty.toIntern() != src_ty.toIntern()) ext: {
3084 break :ext main_block.addTyOp(l, .intcast, extended_ty, ty_op.operand).toRef();
3085 } else ext: {
3086 _ = main_block.stealCapacity(1);
3087 break :ext ty_op.operand;
3088 };
3089 const extended_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(extended_ty)).toRef();
3090 _ = main_block.addBinOp(l, .store, extended_ptr, extended_val);
3091 const bits_val = try pt.intValue(.usize, src_info.bits);
3092 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val) });
3093 const casted_result = main_block.addBitCast(l, dest_ty, call_inst.toRef());
3094 main_block.addBr(l, orig_inst, casted_result);
3095
3096 return .{ .block_payload = .{ .ty_pl = .{
3097 .ty = .fromType(dest_ty),
3098 .payload = try l.addBlockBody(main_block.body()),
3099 } } };
3100}
3101fn softIntFromFloat(l: *Legalize, orig_inst: Air.Inst.Index) Error!union(enum) {
3102 call: Air.CompilerRtFunc,
3103 block_payload: Air.Inst.Data,
3104} {
3105 const pt = l.pt;
3106 const zcu = pt.zcu;
3107 const target = zcu.getTarget();
3108
3109 const ty_op = l.air_instructions.items(.data)[@intFromEnum(orig_inst)].ty_op;
3110 const src_ty = l.typeOf(ty_op.operand);
3111 const dest_ty = ty_op.ty.toType();
3112
3113 const dest_info = dest_ty.intInfo(zcu);
3114 const float_off: u32 = switch (src_ty.floatBits(target)) {
3115 16 => 0,
3116 32 => 1,
3117 64 => 2,
3118 80 => 3,
3119 128 => 4,
3120 else => unreachable,
3121 };
3122 const base: Air.CompilerRtFunc = switch (dest_info.signedness) {
3123 .signed => .__fixhfsi,
3124 .unsigned => .__fixunshfsi,
3125 };
3126 fixed: {
3127 const extended_int_bits: u16, const int_bits_off: u32 = switch (dest_info.bits) {
3128 0...32 => .{ 32, 0 },
3129 33...64 => .{ 64, 5 },
3130 65...128 => .{ 128, 10 },
3131 else => break :fixed,
3132 };
3133 // x86_64-windows uses an odd callconv for 128-bit integers, so we use the
3134 // arbitrary-precision routine in that case for simplicity.
3135 if (target.cpu.arch == .x86_64 and target.os.tag == .windows and extended_int_bits == 128) {
3136 break :fixed;
3137 }
3138
3139 const func: Air.CompilerRtFunc = @enumFromInt(@intFromEnum(base) + int_bits_off + float_off);
3140 if (extended_int_bits == dest_info.bits) return .{ .call = func };
3141
3142 // We need to emit a block which calls the routine and then casts to the required type.
3143
3144 var inst_buf: [3]Air.Inst.Index = undefined;
3145 var main_block: Block = .init(&inst_buf);
3146 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3147
3148 const call_inst = try main_block.addCompilerRtCall(l, func, &.{ty_op.operand});
3149 const casted_val = main_block.addTyOp(l, .intcast, dest_ty, call_inst.toRef()).toRef();
3150 main_block.addBr(l, orig_inst, casted_val);
3151
3152 return .{ .block_payload = .{ .ty_pl = .{
3153 .ty = .fromType(dest_ty),
3154 .payload = try l.addBlockBody(main_block.body()),
3155 } } };
3156 }
3157
3158 // We need to emit a block which calls an arbitrary-width conversion routine, then loads the
3159 // integer from an `alloc` and possibly truncates it.
3160 const func: Air.CompilerRtFunc = @enumFromInt(@intFromEnum(base) + 15 + float_off);
3161
3162 const extended_ty = try pt.intType(dest_info.signedness, @intCast(dest_ty.abiSize(zcu) * 8));
3163 assert(extended_ty.abiSize(zcu) == dest_ty.abiSize(zcu));
3164
3165 var inst_buf: [5]Air.Inst.Index = undefined;
3166 var main_block: Block = .init(&inst_buf);
3167 try l.air_instructions.ensureUnusedCapacity(zcu.gpa, inst_buf.len);
3168
3169 const extended_ptr = main_block.addTy(l, .alloc, try pt.singleMutPtrType(extended_ty)).toRef();
3170 const bits_val = try pt.intValue(.usize, dest_info.bits);
3171 _ = try main_block.addCompilerRtCall(l, func, &.{ extended_ptr, .fromValue(bits_val), ty_op.operand });
3172 const extended_val = main_block.addTyOp(l, .load, extended_ty, extended_ptr).toRef();
3173 const result_val = main_block.addTyOp(l, .intcast, dest_ty, extended_val).toRef();
3174 main_block.addBr(l, orig_inst, result_val);
3175
3176 return .{ .block_payload = .{ .ty_pl = .{
3177 .ty = .fromType(dest_ty),
3178 .payload = try l.addBlockBody(main_block.body()),
3179 } } };
3180}
3181fn softFloatFunc(op: Air.Inst.Tag, float_ty: Type, zcu: *const Zcu) Air.CompilerRtFunc {
3182 const f16_func: Air.CompilerRtFunc = switch (op) {
3183 .add, .add_optimized => .__addhf3,
3184 .sub, .sub_optimized => .__subhf3,
3185 .mul, .mul_optimized => .__mulhf3,
3186
3187 .div_float,
3188 .div_float_optimized,
3189 .div_exact,
3190 .div_exact_optimized,
3191 => .__divhf3,
3192
3193 .min => .__fminh,
3194 .max => .__fmaxh,
3195
3196 .ceil => .__ceilh,
3197 .floor => .__floorh,
3198 .trunc_float => .__trunch,
3199 .round => .__roundh,
3200
3201 .log => .__logh,
3202 .log2 => .__log2h,
3203 .log10 => .__log10h,
3204
3205 .exp => .__exph,
3206 .exp2 => .__exp2h,
3207
3208 .sin => .__sinh,
3209 .cos => .__cosh,
3210 .tan => .__tanh,
3211
3212 .abs => .__fabsh,
3213 .sqrt => .__sqrth,
3214 .rem, .rem_optimized => .__fmodh,
3215 .mul_add => .__fmah,
3216
3217 else => unreachable,
3218 };
3219 const offset: u8 = switch (float_ty.floatBits(zcu.getTarget())) {
3220 16 => 0,
3221 32 => 1,
3222 64 => 2,
3223 80 => 3,
3224 128 => 4,
3225 else => unreachable,
3226 };
3227 return @enumFromInt(@intFromEnum(f16_func) + offset);
3228}
3229
3230fn softFloatNegBlockPayload(
3231 l: *Legalize,
3232 orig_inst: Air.Inst.Index,
3233 operand: Air.Inst.Ref,
3234) Error!Air.Inst.Data {
3235 const pt = l.pt;
3236 const zcu = pt.zcu;
3237 const gpa = zcu.gpa;
3238
3239 const float_ty = l.typeOfIndex(orig_inst);
3240
3241 const int_ty: Type, const sign_bit: Value = switch (float_ty.floatBits(zcu.getTarget())) {
3242 16 => .{ .u16, try pt.intValue(.u16, @as(u16, 1) << 15) },
3243 32 => .{ .u32, try pt.intValue(.u32, @as(u32, 1) << 31) },
3244 64 => .{ .u64, try pt.intValue(.u64, @as(u64, 1) << 63) },
3245 80 => .{ .u80, try pt.intValue(.u80, @as(u80, 1) << 79) },
3246 128 => .{ .u128, try pt.intValue(.u128, @as(u128, 1) << 127) },
3247 else => unreachable,
3248 };
3249
3250 const sign_bit_ref: Air.Inst.Ref = .fromValue(sign_bit);
3251
3252 var inst_buf: [4]Air.Inst.Index = undefined;
3253 var main_block: Block = .init(&inst_buf);
3254 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3255
3256 const operand_as_int = main_block.addBitCast(l, int_ty, operand);
3257 const result_as_int = main_block.addBinOp(l, .xor, operand_as_int, sign_bit_ref).toRef();
3258 const result = main_block.addBitCast(l, float_ty, result_as_int);
3259 main_block.addBr(l, orig_inst, result);
3260
3261 return .{ .ty_pl = .{
3262 .ty = .fromType(float_ty),
3263 .payload = try l.addBlockBody(main_block.body()),
3264 } };
3265}
3266
3267fn softFloatDivTruncFloorBlockPayload(
3268 l: *Legalize,
3269 orig_inst: Air.Inst.Index,
3270 lhs: Air.Inst.Ref,
3271 rhs: Air.Inst.Ref,
3272 air_tag: Air.Inst.Tag,
3273) Error!Air.Inst.Data {
3274 const zcu = l.pt.zcu;
3275 const gpa = zcu.gpa;
3276
3277 const float_ty = l.typeOfIndex(orig_inst);
3278
3279 const floor_tag: Air.Inst.Tag = switch (air_tag) {
3280 .div_trunc, .div_trunc_optimized => .trunc_float,
3281 .div_floor, .div_floor_optimized => .floor,
3282 else => unreachable,
3283 };
3284
3285 var inst_buf: [4]Air.Inst.Index = undefined;
3286 var main_block: Block = .init(&inst_buf);
3287 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3288
3289 const div_inst = try main_block.addCompilerRtCall(l, softFloatFunc(.div_float, float_ty, zcu), &.{ lhs, rhs });
3290 const floor_inst = try main_block.addCompilerRtCall(l, softFloatFunc(floor_tag, float_ty, zcu), &.{div_inst.toRef()});
3291 const casted_result = main_block.addBitCast(l, float_ty, floor_inst.toRef());
3292 main_block.addBr(l, orig_inst, casted_result);
3293
3294 return .{ .ty_pl = .{
3295 .ty = .fromType(float_ty),
3296 .payload = try l.addBlockBody(main_block.body()),
3297 } };
3298}
3299fn softFloatModBlockPayload(
3300 l: *Legalize,
3301 orig_inst: Air.Inst.Index,
3302 lhs: Air.Inst.Ref,
3303 rhs: Air.Inst.Ref,
3304) Error!Air.Inst.Data {
3305 const pt = l.pt;
3306 const zcu = pt.zcu;
3307 const gpa = zcu.gpa;
3308
3309 const float_ty = l.typeOfIndex(orig_inst);
3310
3311 var inst_buf: [10]Air.Inst.Index = undefined;
3312 var main_block: Block = .init(&inst_buf);
3313 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3314
3315 const rem = try main_block.addCompilerRtCall(l, softFloatFunc(.rem, float_ty, zcu), &.{ lhs, rhs });
3316 const lhs_lt_zero = try main_block.addSoftFloatCmp(l, float_ty, .lt, lhs, .fromValue(try pt.floatValue(float_ty, 0.0)));
3317
3318 var condbr: CondBr = .init(l, lhs_lt_zero, &main_block, .{});
3319 condbr.then_block = .init(main_block.stealRemainingCapacity());
3320 {
3321 const add = try condbr.then_block.addCompilerRtCall(l, softFloatFunc(.add, float_ty, zcu), &.{ rem.toRef(), rhs });
3322 const inner_rem = try condbr.then_block.addCompilerRtCall(l, softFloatFunc(.rem, float_ty, zcu), &.{ add.toRef(), rhs });
3323 const casted_result = condbr.then_block.addBitCast(l, float_ty, inner_rem.toRef());
3324 condbr.then_block.addBr(l, orig_inst, casted_result);
3325 }
3326 condbr.else_block = .init(condbr.then_block.stealRemainingCapacity());
3327 {
3328 const casted_result = condbr.else_block.addBitCast(l, float_ty, rem.toRef());
3329 condbr.else_block.addBr(l, orig_inst, casted_result);
3330 }
3331
3332 try condbr.finish(l);
3333
3334 return .{ .ty_pl = .{
3335 .ty = .fromType(float_ty),
3336 .payload = try l.addBlockBody(main_block.body()),
3337 } };
3338}
3339fn softFloatCmpBlockPayload(
3340 l: *Legalize,
3341 orig_inst: Air.Inst.Index,
3342 float_ty: Type,
3343 op: std.math.CompareOperator,
3344 lhs: Air.Inst.Ref,
3345 rhs: Air.Inst.Ref,
3346) Error!Air.Inst.Data {
3347 const pt = l.pt;
3348 const gpa = pt.zcu.gpa;
3349
3350 var inst_buf: [3]Air.Inst.Index = undefined;
3351 var main_block: Block = .init(&inst_buf);
3352 try l.air_instructions.ensureUnusedCapacity(gpa, inst_buf.len);
3353
3354 const result = try main_block.addSoftFloatCmp(l, float_ty, op, lhs, rhs);
3355 main_block.addBr(l, orig_inst, result);
3356
3357 return .{ .ty_pl = .{
3358 .ty = .bool_type,
3359 .payload = try l.addBlockBody(main_block.body()),
3360 } };
3361}
3362
3363/// `inline` to propagate potentially comptime-known return value.
3364inline fn wantScalarizeOrSoftFloat(
3365 l: *const Legalize,
3366 comptime air_tag: Air.Inst.Tag,
3367 ty: Type,
3368) enum {
3369 none,
3370 scalarize,
3371 soft_float,
3372} {
3373 const zcu = l.pt.zcu;
3374 const is_vec, const scalar_ty = switch (ty.zigTypeTag(zcu)) {
3375 .vector => .{ true, ty.childType(zcu) },
3376 else => .{ false, ty },
3377 };
3378
3379 if (is_vec and l.features.has(.scalarize(air_tag))) return .scalarize;
3380
3381 if (l.wantSoftFloatScalar(scalar_ty)) {
3382 return if (is_vec) .scalarize else .soft_float;
3383 }
3384 return .none;
3385}
3386
3387/// `inline` to propagate potentially comptime-known return value.
3388inline fn wantSoftFloatScalar(l: *const Legalize, ty: Type) bool {
3389 const zcu = l.pt.zcu;
3390 return switch (ty.zigTypeTag(zcu)) {
3391 .vector => unreachable,
3392 .float => switch (ty.floatBits(zcu.getTarget())) {
3393 16 => l.features.has(.soft_f16),
3394 32 => l.features.has(.soft_f32),
3395 64 => l.features.has(.soft_f64),
3396 80 => l.features.has(.soft_f80),
3397 128 => l.features.has(.soft_f128),
3398 else => unreachable,
3399 },
3400 else => false,
3401 };
3402}
3403
2528const Air = @import("../Air.zig");3404const Air = @import("../Air.zig");
2529const assert = std.debug.assert;3405const assert = std.debug.assert;
2530const dev = @import("../dev.zig");3406const dev = @import("../dev.zig");
src/Air/Liveness.zig+18
...@@ -776,6 +776,24 @@ fn analyzeInst(...@@ -776,6 +776,24 @@ fn analyzeInst(
776 const bin = a.air.extraData(Air.Bin, pl_op.payload).data;776 const bin = a.air.extraData(Air.Bin, pl_op.payload).data;
777 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });777 return analyzeOperands(a, pass, data, inst, .{ pl_op.operand, bin.lhs, bin.rhs });
778 },778 },
779
780 .legalize_compiler_rt_call => {
781 const extra = a.air.extraData(Air.Call, inst_datas[@intFromEnum(inst)].legalize_compiler_rt_call.payload);
782 const args: []const Air.Inst.Ref = @ptrCast(a.air.extra.items[extra.end..][0..extra.data.args_len]);
783 if (args.len <= bpi - 1) {
784 var buf: [bpi - 1]Air.Inst.Ref = @splat(.none);
785 @memcpy(buf[0..args.len], args);
786 return analyzeOperands(a, pass, data, inst, buf);
787 }
788 var big = try AnalyzeBigOperands(pass).init(a, data, inst, args.len + 1);
789 defer big.deinit();
790 var i: usize = args.len;
791 while (i > 0) {
792 i -= 1;
793 try big.feed(args[i]);
794 }
795 return big.finish();
796 },
779 }797 }
780}798}
781799
src/Air/Liveness/Verify.zig+9
...@@ -583,6 +583,15 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {...@@ -583,6 +583,15 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
583 const bin = self.air.extraData(Air.Bin, pl_op.payload).data;583 const bin = self.air.extraData(Air.Bin, pl_op.payload).data;
584 try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });584 try self.verifyInstOperands(inst, .{ pl_op.operand, bin.lhs, bin.rhs });
585 },585 },
586 .legalize_compiler_rt_call => {
587 const extra = self.air.extraData(Air.Call, data[@intFromEnum(inst)].legalize_compiler_rt_call.payload);
588 const args: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[extra.end..][0..extra.data.args_len]);
589 var bt = self.liveness.iterateBigTomb(inst);
590 for (args) |arg| {
591 try self.verifyOperand(inst, arg, bt.feed());
592 }
593 try self.verifyInst(inst);
594 },
586 }595 }
587 }596 }
588}597}
src/Air/print.zig+14
...@@ -333,6 +333,7 @@ const Writer = struct {...@@ -333,6 +333,7 @@ const Writer = struct {
333 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),333 .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst),
334 .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),334 .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst),
335 .legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),335 .legalize_vec_store_elem => try w.writeLegalizeVecStoreElem(s, inst),
336 .legalize_compiler_rt_call => try w.writeLegalizeCompilerRtCall(s, inst),
336337
337 .work_item_id,338 .work_item_id,
338 .work_group_size,339 .work_group_size,
...@@ -522,6 +523,19 @@ const Writer = struct {...@@ -522,6 +523,19 @@ const Writer = struct {
522 try s.writeAll(", ");523 try s.writeAll(", ");
523 }524 }
524525
526 fn writeLegalizeCompilerRtCall(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
527 const inst_data = w.air.instructions.items(.data)[@intFromEnum(inst)].legalize_compiler_rt_call;
528 const extra = w.air.extraData(Air.Call, inst_data.payload);
529 const args: []const Air.Inst.Ref = @ptrCast(w.air.extra.items[extra.end..][0..extra.data.args_len]);
530
531 try s.print("{t}, [", .{inst_data.func});
532 for (args, 0..) |arg, i| {
533 if (i != 0) try s.writeAll(", ");
534 try w.writeOperand(s, inst, i, arg);
535 }
536 try s.writeByte(']');
537 }
538
525 fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {539 fn writeShuffleOne(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void {
526 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);540 const unwrapped = w.air.unwrapShuffleOne(w.pt.zcu, inst);
527 try w.writeType(s, unwrapped.result_ty);541 try w.writeType(s, unwrapped.result_ty);
src/Air/types_resolved.zig+6
...@@ -418,6 +418,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {...@@ -418,6 +418,12 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
418 for (inputs) |input| if (input != .none and !checkRef(input, zcu)) return false;418 for (inputs) |input| if (input != .none and !checkRef(input, zcu)) return false;
419 },419 },
420420
421 .legalize_compiler_rt_call => {
422 const extra = air.extraData(Air.Call, data.legalize_compiler_rt_call.payload);
423 const args: []const Air.Inst.Ref = @ptrCast(air.extra.items[extra.end..][0..extra.data.args_len]);
424 for (args) |arg| if (!checkRef(arg, zcu)) return false;
425 },
426
421 .trap,427 .trap,
422 .breakpoint,428 .breakpoint,
423 .ret_addr,429 .ret_addr,
src/codegen/aarch64/Select.zig+2
...@@ -137,6 +137,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {...@@ -137,6 +137,8 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void {
137 // No "scalarize" legalizations are enabled, so these instructions never appear.137 // No "scalarize" legalizations are enabled, so these instructions never appear.
138 .legalize_vec_elem_val => unreachable,138 .legalize_vec_elem_val => unreachable,
139 .legalize_vec_store_elem => unreachable,139 .legalize_vec_store_elem => unreachable,
140 // No soft float legalizations are enabled.
141 .legalize_compiler_rt_call => unreachable,
140142
141 .arg,143 .arg,
142 .ret_addr,144 .ret_addr,
src/codegen/c.zig+2
...@@ -3328,6 +3328,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {...@@ -3328,6 +3328,8 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void {
3328 // No "scalarize" legalizations are enabled, so these instructions never appear.3328 // No "scalarize" legalizations are enabled, so these instructions never appear.
3329 .legalize_vec_elem_val => unreachable,3329 .legalize_vec_elem_val => unreachable,
3330 .legalize_vec_store_elem => unreachable,3330 .legalize_vec_store_elem => unreachable,
3331 // No soft float legalizations are enabled.
3332 .legalize_compiler_rt_call => unreachable,
33313333
3332 .arg => try airArg(f, inst),3334 .arg => try airArg(f, inst),
33333335
src/codegen/llvm.zig+10-4
...@@ -4889,6 +4889,8 @@ pub const FuncGen = struct {...@@ -4889,6 +4889,8 @@ pub const FuncGen = struct {
4889 // No "scalarize" legalizations are enabled, so these instructions never appear.4889 // No "scalarize" legalizations are enabled, so these instructions never appear.
4890 .legalize_vec_elem_val => unreachable,4890 .legalize_vec_elem_val => unreachable,
4891 .legalize_vec_store_elem => unreachable,4891 .legalize_vec_store_elem => unreachable,
4892 // No soft float legalizations are enabled.
4893 .legalize_compiler_rt_call => unreachable,
48924894
4893 .add => try self.airAdd(inst, .normal),4895 .add => try self.airAdd(inst, .normal),
4894 .add_optimized => try self.airAdd(inst, .fast),4896 .add_optimized => try self.airAdd(inst, .fast),
...@@ -6670,7 +6672,9 @@ pub const FuncGen = struct {...@@ -6670,7 +6672,9 @@ pub const FuncGen = struct {
6670 "",6672 "",
6671 );6673 );
66726674
6673 const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu)));6675 const rt_int_bits = compilerRtIntBits(@intCast(operand_scalar_ty.bitSize(zcu))) orelse {
6676 return self.todo("float_from_int from '{f}' without intrinsics", .{operand_scalar_ty.fmt(pt)});
6677 };
6674 const rt_int_ty = try o.builder.intType(rt_int_bits);6678 const rt_int_ty = try o.builder.intType(rt_int_bits);
6675 var extended = try self.wip.conv(6679 var extended = try self.wip.conv(
6676 if (is_signed_int) .signed else .unsigned,6680 if (is_signed_int) .signed else .unsigned,
...@@ -6739,7 +6743,9 @@ pub const FuncGen = struct {...@@ -6739,7 +6743,9 @@ pub const FuncGen = struct {
6739 );6743 );
6740 }6744 }
67416745
6742 const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu)));6746 const rt_int_bits = compilerRtIntBits(@intCast(dest_scalar_ty.bitSize(zcu))) orelse {
6747 return self.todo("int_from_float to '{f}' without intrinsics", .{dest_scalar_ty.fmt(pt)});
6748 };
6743 const ret_ty = try o.builder.intType(rt_int_bits);6749 const ret_ty = try o.builder.intType(rt_int_bits);
6744 const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {6750 const libc_ret_ty = if (rt_int_bits == 128 and (target.os.tag == .windows and target.cpu.arch == .x86_64)) b: {
6745 // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard6751 // On Windows x86-64, "ti" functions must use Vector(2, u64) instead of the standard
...@@ -12823,13 +12829,13 @@ const optional_layout_version = 3;...@@ -12823,13 +12829,13 @@ const optional_layout_version = 3;
1282312829
12824const lt_errors_fn_name = "__zig_lt_errors_len";12830const lt_errors_fn_name = "__zig_lt_errors_len";
1282512831
12826fn compilerRtIntBits(bits: u16) u16 {12832fn compilerRtIntBits(bits: u16) ?u16 {
12827 inline for (.{ 32, 64, 128 }) |b| {12833 inline for (.{ 32, 64, 128 }) |b| {
12828 if (bits <= b) {12834 if (bits <= b) {
12829 return b;12835 return b;
12830 }12836 }
12831 }12837 }
12832 return bits;12838 return null;
12833}12839}
1283412840
12835fn buildAllocaInner(12841fn buildAllocaInner(
src/codegen/riscv64/CodeGen.zig+2
...@@ -1395,6 +1395,8 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {...@@ -1395,6 +1395,8 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
1395 // No "scalarize" legalizations are enabled, so these instructions never appear.1395 // No "scalarize" legalizations are enabled, so these instructions never appear.
1396 .legalize_vec_elem_val => unreachable,1396 .legalize_vec_elem_val => unreachable,
1397 .legalize_vec_store_elem => unreachable,1397 .legalize_vec_store_elem => unreachable,
1398 // No soft float legalizations are enabled.
1399 .legalize_compiler_rt_call => unreachable,
13981400
1399 .add,1401 .add,
1400 .add_wrap,1402 .add_wrap,
src/codegen/sparc64/CodeGen.zig+2
...@@ -483,6 +483,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {...@@ -483,6 +483,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
483 // No "scalarize" legalizations are enabled, so these instructions never appear.483 // No "scalarize" legalizations are enabled, so these instructions never appear.
484 .legalize_vec_elem_val => unreachable,484 .legalize_vec_elem_val => unreachable,
485 .legalize_vec_store_elem => unreachable,485 .legalize_vec_store_elem => unreachable,
486 // No soft float legalizations are enabled.
487 .legalize_compiler_rt_call => unreachable,
486488
487 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),489 .ptr_add => try self.airPtrArithmetic(inst, .ptr_add),
488 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),490 .ptr_sub => try self.airPtrArithmetic(inst, .ptr_sub),
src/codegen/wasm/CodeGen.zig+2
...@@ -1789,6 +1789,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {...@@ -1789,6 +1789,8 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void {
1789 // No "scalarize" legalizations are enabled, so these instructions never appear.1789 // No "scalarize" legalizations are enabled, so these instructions never appear.
1790 .legalize_vec_elem_val => unreachable,1790 .legalize_vec_elem_val => unreachable,
1791 .legalize_vec_store_elem => unreachable,1791 .legalize_vec_store_elem => unreachable,
1792 // No soft float legalizations are enabled.
1793 .legalize_compiler_rt_call => unreachable,
17921794
1793 .inferred_alloc, .inferred_alloc_comptime => unreachable,1795 .inferred_alloc, .inferred_alloc_comptime => unreachable,
17941796
src/codegen/x86_64/CodeGen.zig+4
...@@ -173689,6 +173689,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -173689,6 +173689,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
173689 };173689 };
173690 for (ops) |op| try op.die(cg);173690 for (ops) |op| try op.die(cg);
173691 },173691 },
173692
173693 // No soft-float `Legalize` features are enabled, so this instruction never appears.
173694 .legalize_compiler_rt_call => unreachable,
173695
173692 .work_item_id, .work_group_size, .work_group_id => unreachable,173696 .work_item_id, .work_group_size, .work_group_id => unreachable,
173693 }173697 }
173694 try cg.resetTemps(@enumFromInt(0));173698 try cg.resetTemps(@enumFromInt(0));
src/target.zig+1-1
...@@ -842,7 +842,7 @@ pub fn compilerRtIntAbbrev(bits: u16) []const u8 {...@@ -842,7 +842,7 @@ pub fn compilerRtIntAbbrev(bits: u16) []const u8 {
842 32 => "s",842 32 => "s",
843 64 => "d",843 64 => "d",
844 128 => "t",844 128 => "t",
845 else => "o", // Non-standard845 else => unreachable,
846 };846 };
847}847}
848848