authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-10-03 05:03:44-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-10-03 05:03:44-04:00
log12ed0ff1efa71d11f6220d9cf94202b888e177fc
tree58679862045c06d8ebe2f1c67bedb5f6f91eb3fa
parent1f083e9ed78f5c3c2d848d0abc58612c4ce88804
parent759e038a44eda0c950f0a5baac37b3a1d7f786b3
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #25430 from jacobly0/x86_64-win

Coff2: create a new linker from scratch

64 files changed, 9274 insertions(+), 14124 deletions(-)

CMakeLists.txt-1
......@@ -561,7 +561,6 @@ set(ZIG_STAGE2_SOURCES
561561 src/libs/libunwind.zig
562562 src/link.zig
563563 src/link/C.zig
564 src/link/Coff.zig
565564 src/link/Dwarf.zig
566565 src/link/Elf.zig
567566 src/link/Elf/Archive.zig
lib/compiler/resinator/cvtres.zig+22-22
......@@ -168,7 +168,7 @@ pub fn parseNameOrOrdinal(allocator: Allocator, reader: *std.Io.Reader) !NameOrO
168168}
169169
170170pub const CoffOptions = struct {
171 target: std.coff.MachineType = .X64,
171 target: std.coff.IMAGE.FILE.MACHINE = .AMD64,
172172 /// If true, zeroes will be written to all timestamp fields
173173 reproducible: bool = true,
174174 /// If true, the MEM_WRITE flag will not be set in the .rsrc section header
......@@ -210,19 +210,19 @@ pub fn writeCoff(allocator: Allocator, writer: *std.Io.Writer, resources: []cons
210210 const lengths = resource_tree.dataLengths();
211211 const byte_size_of_relocation = 10;
212212 const relocations_len: u32 = @intCast(byte_size_of_relocation * resources.len);
213 const pointer_to_rsrc01_data = @sizeOf(std.coff.CoffHeader) + (@sizeOf(std.coff.SectionHeader) * 2);
213 const pointer_to_rsrc01_data = @sizeOf(std.coff.Header) + (@sizeOf(std.coff.SectionHeader) * 2);
214214 const pointer_to_relocations = pointer_to_rsrc01_data + lengths.rsrc01;
215215 const pointer_to_rsrc02_data = pointer_to_relocations + relocations_len;
216216 const pointer_to_symbol_table = pointer_to_rsrc02_data + lengths.rsrc02;
217217
218218 const timestamp: i64 = if (options.reproducible) 0 else std.time.timestamp();
219219 const size_of_optional_header = 0;
220 const machine_type: std.coff.MachineType = options.target;
221 const flags = std.coff.CoffHeaderFlags{
222 .@"32BIT_MACHINE" = 1,
220 const machine_type: std.coff.IMAGE.FILE.MACHINE = options.target;
221 const flags = std.coff.Header.Flags{
222 .@"32BIT_MACHINE" = true,
223223 };
224224 const number_of_symbols = 5 + @as(u32, @intCast(resources.len)) + @intFromBool(options.define_external_symbol != null);
225 const coff_header = std.coff.CoffHeader{
225 const coff_header = std.coff.Header{
226226 .machine = machine_type,
227227 .number_of_sections = 2,
228228 .time_date_stamp = @as(u32, @truncate(@as(u64, @bitCast(timestamp)))),
......@@ -245,9 +245,9 @@ pub fn writeCoff(allocator: Allocator, writer: *std.Io.Writer, resources: []cons
245245 .number_of_relocations = @intCast(resources.len),
246246 .number_of_linenumbers = 0,
247247 .flags = .{
248 .CNT_INITIALIZED_DATA = 1,
249 .MEM_WRITE = @intFromBool(!options.read_only),
250 .MEM_READ = 1,
248 .CNT_INITIALIZED_DATA = true,
249 .MEM_WRITE = !options.read_only,
250 .MEM_READ = true,
251251 },
252252 };
253253 try writer.writeStruct(rsrc01_header, .little);
......@@ -263,9 +263,9 @@ pub fn writeCoff(allocator: Allocator, writer: *std.Io.Writer, resources: []cons
263263 .number_of_relocations = 0,
264264 .number_of_linenumbers = 0,
265265 .flags = .{
266 .CNT_INITIALIZED_DATA = 1,
267 .MEM_WRITE = @intFromBool(!options.read_only),
268 .MEM_READ = 1,
266 .CNT_INITIALIZED_DATA = true,
267 .MEM_WRITE = !options.read_only,
268 .MEM_READ = true,
269269 },
270270 };
271271 try writer.writeStruct(rsrc02_header, .little);
......@@ -1005,9 +1005,9 @@ pub const supported_targets = struct {
10051005 x86_64,
10061006 aarch64,
10071007
1008 pub fn toCoffMachineType(arch: Arch) std.coff.MachineType {
1008 pub fn toCoffMachineType(arch: Arch) std.coff.IMAGE.FILE.MACHINE {
10091009 return switch (arch) {
1010 .x64, .amd64, .x86_64 => .X64,
1010 .x64, .amd64, .x86_64 => .AMD64,
10111011 .x86, .i386 => .I386,
10121012 .arm, .armnt => .ARMNT,
10131013 .arm64, .aarch64 => .ARM64,
......@@ -1079,26 +1079,26 @@ pub const supported_targets = struct {
10791079 };
10801080
10811081 // https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#type-indicators
1082 pub fn rvaRelocationTypeIndicator(target: std.coff.MachineType) ?u16 {
1082 pub fn rvaRelocationTypeIndicator(target: std.coff.IMAGE.FILE.MACHINE) ?u16 {
10831083 return switch (target) {
1084 .X64 => 0x3, // IMAGE_REL_AMD64_ADDR32NB
1085 .I386 => 0x7, // IMAGE_REL_I386_DIR32NB
1086 .ARMNT => 0x2, // IMAGE_REL_ARM_ADDR32NB
1087 .ARM64, .ARM64EC, .ARM64X => 0x2, // IMAGE_REL_ARM64_ADDR32NB
1088 .IA64 => 0x10, // IMAGE_REL_IA64_DIR32NB
1084 .AMD64 => @intFromEnum(std.coff.IMAGE.REL.AMD64.ADDR32NB),
1085 .I386 => @intFromEnum(std.coff.IMAGE.REL.I386.DIR32NB),
1086 .ARMNT => @intFromEnum(std.coff.IMAGE.REL.ARM.ADDR32NB),
1087 .ARM64, .ARM64EC, .ARM64X => @intFromEnum(std.coff.IMAGE.REL.ARM64.ADDR32NB),
1088 .IA64 => @intFromEnum(std.coff.IMAGE.REL.IA64.DIR32NB),
10891089 .EBC => 0x1, // This is what cvtres.exe writes for this target, unsure where it comes from
10901090 else => null,
10911091 };
10921092 }
10931093
1094 pub fn isSupported(target: std.coff.MachineType) bool {
1094 pub fn isSupported(target: std.coff.IMAGE.FILE.MACHINE) bool {
10951095 return rvaRelocationTypeIndicator(target) != null;
10961096 }
10971097
10981098 comptime {
10991099 // Enforce two things:
11001100 // 1. Arch enum field names are all lowercase (necessary for how fromStringIgnoreCase is implemented)
1101 // 2. All enum fields in Arch have an associated RVA relocation type when converted to a coff.MachineType
1101 // 2. All enum fields in Arch have an associated RVA relocation type when converted to a coff.IMAGE.FILE.MACHINE
11021102 for (@typeInfo(Arch).@"enum".fields) |enum_field| {
11031103 const all_lower = all_lower: for (enum_field.name) |c| {
11041104 if (std.ascii.isUpper(c)) break :all_lower false;
lib/compiler/resinator/main.zig+3-3
......@@ -527,7 +527,7 @@ const LazyIncludePaths = struct {
527527 arena: std.mem.Allocator,
528528 auto_includes_option: cli.Options.AutoIncludes,
529529 zig_lib_dir: []const u8,
530 target_machine_type: std.coff.MachineType,
530 target_machine_type: std.coff.IMAGE.FILE.MACHINE,
531531 resolved_include_paths: ?[]const []const u8 = null,
532532
533533 pub fn get(self: *LazyIncludePaths, error_handler: *ErrorHandler) ![]const []const u8 {
......@@ -555,11 +555,11 @@ const LazyIncludePaths = struct {
555555 }
556556};
557557
558fn getIncludePaths(arena: std.mem.Allocator, auto_includes_option: cli.Options.AutoIncludes, zig_lib_dir: []const u8, target_machine_type: std.coff.MachineType) ![]const []const u8 {
558fn getIncludePaths(arena: std.mem.Allocator, auto_includes_option: cli.Options.AutoIncludes, zig_lib_dir: []const u8, target_machine_type: std.coff.IMAGE.FILE.MACHINE) ![]const []const u8 {
559559 if (auto_includes_option == .none) return &[_][]const u8{};
560560
561561 const includes_arch: std.Target.Cpu.Arch = switch (target_machine_type) {
562 .X64 => .x86_64,
562 .AMD64 => .x86_64,
563563 .I386 => .x86,
564564 .ARMNT => .thumb,
565565 .ARM64 => .aarch64,
lib/std/Target.zig+2-2
......@@ -1082,7 +1082,7 @@ pub fn toElfMachine(target: *const Target) std.elf.EM {
10821082 };
10831083}
10841084
1085pub fn toCoffMachine(target: *const Target) std.coff.MachineType {
1085pub fn toCoffMachine(target: *const Target) std.coff.IMAGE.FILE.MACHINE {
10861086 return switch (target.cpu.arch) {
10871087 .arm => .ARM,
10881088 .thumb => .ARMNT,
......@@ -1092,7 +1092,7 @@ pub fn toCoffMachine(target: *const Target) std.coff.MachineType {
10921092 .riscv32 => .RISCV32,
10931093 .riscv64 => .RISCV64,
10941094 .x86 => .I386,
1095 .x86_64 => .X64,
1095 .x86_64 => .AMD64,
10961096
10971097 .amdgcn,
10981098 .arc,
lib/std/array_hash_map.zig+1-1
......@@ -50,7 +50,7 @@ pub fn eqlString(a: []const u8, b: []const u8) bool {
5050}
5151
5252pub fn hashString(s: []const u8) u32 {
53 return @as(u32, @truncate(std.hash.Wyhash.hash(0, s)));
53 return @truncate(std.hash.Wyhash.hash(0, s));
5454}
5555
5656/// Deprecated in favor of `ArrayHashMapWithAllocator` (no code changes needed)
lib/std/coff.zig+771-432
......@@ -2,70 +2,9 @@ const std = @import("std.zig");
22const assert = std.debug.assert;
33const mem = std.mem;
44
5pub const CoffHeaderFlags = packed struct {
6 /// Image only, Windows CE, and Microsoft Windows NT and later.
7 /// This indicates that the file does not contain base relocations
8 /// and must therefore be loaded at its preferred base address.
9 /// If the base address is not available, the loader reports an error.
10 /// The default behavior of the linker is to strip base relocations
11 /// from executable (EXE) files.
12 RELOCS_STRIPPED: u1 = 0,
13
14 /// Image only. This indicates that the image file is valid and can be run.
15 /// If this flag is not set, it indicates a linker error.
16 EXECUTABLE_IMAGE: u1 = 0,
17
18 /// COFF line numbers have been removed. This flag is deprecated and should be zero.
19 LINE_NUMS_STRIPPED: u1 = 0,
20
21 /// COFF symbol table entries for local symbols have been removed.
22 /// This flag is deprecated and should be zero.
23 LOCAL_SYMS_STRIPPED: u1 = 0,
24
25 /// Obsolete. Aggressively trim working set.
26 /// This flag is deprecated for Windows 2000 and later and must be zero.
27 AGGRESSIVE_WS_TRIM: u1 = 0,
28
29 /// Application can handle > 2-GB addresses.
30 LARGE_ADDRESS_AWARE: u1 = 0,
31
32 /// This flag is reserved for future use.
33 RESERVED: u1 = 0,
34
35 /// Little endian: the least significant bit (LSB) precedes the
36 /// most significant bit (MSB) in memory. This flag is deprecated and should be zero.
37 BYTES_REVERSED_LO: u1 = 0,
38
39 /// Machine is based on a 32-bit-word architecture.
40 @"32BIT_MACHINE": u1 = 0,
41
42 /// Debugging information is removed from the image file.
43 DEBUG_STRIPPED: u1 = 0,
44
45 /// If the image is on removable media, fully load it and copy it to the swap file.
46 REMOVABLE_RUN_FROM_SWAP: u1 = 0,
47
48 /// If the image is on network media, fully load it and copy it to the swap file.
49 NET_RUN_FROM_SWAP: u1 = 0,
50
51 /// The image file is a system file, not a user program.
52 SYSTEM: u1 = 0,
53
54 /// The image file is a dynamic-link library (DLL).
55 /// Such files are considered executable files for almost all purposes,
56 /// although they cannot be directly run.
57 DLL: u1 = 0,
58
59 /// The file should be run only on a uniprocessor machine.
60 UP_SYSTEM_ONLY: u1 = 0,
61
62 /// Big endian: the MSB precedes the LSB in memory. This flag is deprecated and should be zero.
63 BYTES_REVERSED_HI: u1 = 0,
64};
65
66pub const CoffHeader = extern struct {
5pub const Header = extern struct {
676 /// The number that identifies the type of target machine.
68 machine: MachineType,
7 machine: IMAGE.FILE.MACHINE,
698
709 /// The number of sections. This indicates the size of the section table, which immediately follows the headers.
7110 number_of_sections: u16,
......@@ -88,49 +27,110 @@ pub const CoffHeader = extern struct {
8827 size_of_optional_header: u16,
8928
9029 /// The flags that indicate the attributes of the file.
91 flags: CoffHeaderFlags,
30 flags: Header.Flags,
31
32 pub const Flags = packed struct(u16) {
33 /// Image only, Windows CE, and Microsoft Windows NT and later.
34 /// This indicates that the file does not contain base relocations
35 /// and must therefore be loaded at its preferred base address.
36 /// If the base address is not available, the loader reports an error.
37 /// The default behavior of the linker is to strip base relocations
38 /// from executable (EXE) files.
39 RELOCS_STRIPPED: bool = false,
40
41 /// Image only. This indicates that the image file is valid and can be run.
42 /// If this flag is not set, it indicates a linker error.
43 EXECUTABLE_IMAGE: bool = false,
44
45 /// COFF line numbers have been removed. This flag is deprecated and should be zero.
46 LINE_NUMS_STRIPPED: bool = false,
47
48 /// COFF symbol table entries for local symbols have been removed.
49 /// This flag is deprecated and should be zero.
50 LOCAL_SYMS_STRIPPED: bool = false,
51
52 /// Obsolete. Aggressively trim working set.
53 /// This flag is deprecated for Windows 2000 and later and must be zero.
54 AGGRESSIVE_WS_TRIM: bool = false,
55
56 /// Application can handle > 2-GB addresses.
57 LARGE_ADDRESS_AWARE: bool = false,
58
59 /// This flag is reserved for future use.
60 RESERVED: bool = false,
61
62 /// Little endian: the least significant bit (LSB) precedes the
63 /// most significant bit (MSB) in memory. This flag is deprecated and should be zero.
64 BYTES_REVERSED_LO: bool = false,
65
66 /// Machine is based on a 32-bit-word architecture.
67 @"32BIT_MACHINE": bool = false,
68
69 /// Debugging information is removed from the image file.
70 DEBUG_STRIPPED: bool = false,
71
72 /// If the image is on removable media, fully load it and copy it to the swap file.
73 REMOVABLE_RUN_FROM_SWAP: bool = false,
74
75 /// If the image is on network media, fully load it and copy it to the swap file.
76 NET_RUN_FROM_SWAP: bool = false,
77
78 /// The image file is a system file, not a user program.
79 SYSTEM: bool = false,
80
81 /// The image file is a dynamic-link library (DLL).
82 /// Such files are considered executable files for almost all purposes,
83 /// although they cannot be directly run.
84 DLL: bool = false,
85
86 /// The file should be run only on a uniprocessor machine.
87 UP_SYSTEM_ONLY: bool = false,
88
89 /// Big endian: the MSB precedes the LSB in memory. This flag is deprecated and should be zero.
90 BYTES_REVERSED_HI: bool = false,
91 };
9292};
9393
9494// OptionalHeader.magic values
9595// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms680339(v=vs.85).aspx
96pub const IMAGE_NT_OPTIONAL_HDR32_MAGIC = 0x10b;
97pub const IMAGE_NT_OPTIONAL_HDR64_MAGIC = 0x20b;
96pub const IMAGE_NT_OPTIONAL_HDR32_MAGIC = @intFromEnum(OptionalHeader.Magic.PE32);
97pub const IMAGE_NT_OPTIONAL_HDR64_MAGIC = @intFromEnum(OptionalHeader.Magic.@"PE32+");
9898
99pub const DllFlags = packed struct {
99pub const DllFlags = packed struct(u16) {
100100 _reserved_0: u5 = 0,
101101
102102 /// Image can handle a high entropy 64-bit virtual address space.
103 HIGH_ENTROPY_VA: u1 = 0,
103 HIGH_ENTROPY_VA: bool = false,
104104
105105 /// DLL can be relocated at load time.
106 DYNAMIC_BASE: u1 = 0,
106 DYNAMIC_BASE: bool = false,
107107
108108 /// Code Integrity checks are enforced.
109 FORCE_INTEGRITY: u1 = 0,
109 FORCE_INTEGRITY: bool = false,
110110
111111 /// Image is NX compatible.
112 NX_COMPAT: u1 = 0,
112 NX_COMPAT: bool = false,
113113
114114 /// Isolation aware, but do not isolate the image.
115 NO_ISOLATION: u1 = 0,
115 NO_ISOLATION: bool = false,
116116
117117 /// Does not use structured exception (SE) handling. No SE handler may be called in this image.
118 NO_SEH: u1 = 0,
118 NO_SEH: bool = false,
119119
120120 /// Do not bind the image.
121 NO_BIND: u1 = 0,
121 NO_BIND: bool = false,
122122
123123 /// Image must execute in an AppContainer.
124 APPCONTAINER: u1 = 0,
124 APPCONTAINER: bool = false,
125125
126126 /// A WDM driver.
127 WDM_DRIVER: u1 = 0,
127 WDM_DRIVER: bool = false,
128128
129129 /// Image supports Control Flow Guard.
130 GUARD_CF: u1 = 0,
130 GUARD_CF: bool = false,
131131
132132 /// Terminal Server aware.
133 TERMINAL_SERVER_AWARE: u1 = 0,
133 TERMINAL_SERVER_AWARE: bool = false,
134134};
135135
136136pub const Subsystem = enum(u16) {
......@@ -180,7 +180,7 @@ pub const Subsystem = enum(u16) {
180180};
181181
182182pub const OptionalHeader = extern struct {
183 magic: u16,
183 magic: OptionalHeader.Magic,
184184 major_linker_version: u8,
185185 minor_linker_version: u8,
186186 size_of_code: u32,
......@@ -188,71 +188,63 @@ pub const OptionalHeader = extern struct {
188188 size_of_uninitialized_data: u32,
189189 address_of_entry_point: u32,
190190 base_of_code: u32,
191};
192191
193pub const OptionalHeaderPE32 = extern struct {
194 magic: u16,
195 major_linker_version: u8,
196 minor_linker_version: u8,
197 size_of_code: u32,
198 size_of_initialized_data: u32,
199 size_of_uninitialized_data: u32,
200 address_of_entry_point: u32,
201 base_of_code: u32,
202 base_of_data: u32,
203 image_base: u32,
204 section_alignment: u32,
205 file_alignment: u32,
206 major_operating_system_version: u16,
207 minor_operating_system_version: u16,
208 major_image_version: u16,
209 minor_image_version: u16,
210 major_subsystem_version: u16,
211 minor_subsystem_version: u16,
212 win32_version_value: u32,
213 size_of_image: u32,
214 size_of_headers: u32,
215 checksum: u32,
216 subsystem: Subsystem,
217 dll_flags: DllFlags,
218 size_of_stack_reserve: u32,
219 size_of_stack_commit: u32,
220 size_of_heap_reserve: u32,
221 size_of_heap_commit: u32,
222 loader_flags: u32,
223 number_of_rva_and_sizes: u32,
224};
192 pub const Magic = enum(u16) {
193 PE32 = 0x10b,
194 @"PE32+" = 0x20b,
195 _,
196 };
225197
226pub const OptionalHeaderPE64 = extern struct {
227 magic: u16,
228 major_linker_version: u8,
229 minor_linker_version: u8,
230 size_of_code: u32,
231 size_of_initialized_data: u32,
232 size_of_uninitialized_data: u32,
233 address_of_entry_point: u32,
234 base_of_code: u32,
235 image_base: u64,
236 section_alignment: u32,
237 file_alignment: u32,
238 major_operating_system_version: u16,
239 minor_operating_system_version: u16,
240 major_image_version: u16,
241 minor_image_version: u16,
242 major_subsystem_version: u16,
243 minor_subsystem_version: u16,
244 win32_version_value: u32,
245 size_of_image: u32,
246 size_of_headers: u32,
247 checksum: u32,
248 subsystem: Subsystem,
249 dll_flags: DllFlags,
250 size_of_stack_reserve: u64,
251 size_of_stack_commit: u64,
252 size_of_heap_reserve: u64,
253 size_of_heap_commit: u64,
254 loader_flags: u32,
255 number_of_rva_and_sizes: u32,
198 pub const PE32 = extern struct {
199 standard: OptionalHeader,
200 base_of_data: u32,
201 image_base: u32,
202 section_alignment: u32,
203 file_alignment: u32,
204 major_operating_system_version: u16,
205 minor_operating_system_version: u16,
206 major_image_version: u16,
207 minor_image_version: u16,
208 major_subsystem_version: u16,
209 minor_subsystem_version: u16,
210 win32_version_value: u32,
211 size_of_image: u32,
212 size_of_headers: u32,
213 checksum: u32,
214 subsystem: Subsystem,
215 dll_flags: DllFlags,
216 size_of_stack_reserve: u32,
217 size_of_stack_commit: u32,
218 size_of_heap_reserve: u32,
219 size_of_heap_commit: u32,
220 loader_flags: u32,
221 number_of_rva_and_sizes: u32,
222 };
223
224 pub const @"PE32+" = extern struct {
225 standard: OptionalHeader,
226 image_base: u64,
227 section_alignment: u32,
228 file_alignment: u32,
229 major_operating_system_version: u16,
230 minor_operating_system_version: u16,
231 major_image_version: u16,
232 minor_image_version: u16,
233 major_subsystem_version: u16,
234 minor_subsystem_version: u16,
235 win32_version_value: u32,
236 size_of_image: u32,
237 size_of_headers: u32,
238 checksum: u32,
239 subsystem: Subsystem,
240 dll_flags: DllFlags,
241 size_of_stack_reserve: u64,
242 size_of_stack_commit: u64,
243 size_of_heap_reserve: u64,
244 size_of_heap_commit: u64,
245 loader_flags: u32,
246 number_of_rva_and_sizes: u32,
247 };
256248};
257249
258250pub const IMAGE_NUMBEROF_DIRECTORY_ENTRIES = 16;
......@@ -319,7 +311,7 @@ pub const BaseRelocationDirectoryEntry = extern struct {
319311 block_size: u32,
320312};
321313
322pub const BaseRelocation = packed struct {
314pub const BaseRelocation = packed struct(u16) {
323315 /// Stored in the remaining 12 bits of the WORD, an offset from the starting address that was specified in the Page RVA field for the block.
324316 /// This offset specifies where the base relocation is to be applied.
325317 offset: u12,
......@@ -447,12 +439,12 @@ pub const ImportDirectoryEntry = extern struct {
447439};
448440
449441pub const ImportLookupEntry32 = struct {
450 pub const ByName = packed struct {
442 pub const ByName = packed struct(u32) {
451443 name_table_rva: u31,
452444 flag: u1 = 0,
453445 };
454446
455 pub const ByOrdinal = packed struct {
447 pub const ByOrdinal = packed struct(u32) {
456448 ordinal_number: u16,
457449 unused: u15 = 0,
458450 flag: u1 = 1,
......@@ -472,13 +464,13 @@ pub const ImportLookupEntry32 = struct {
472464};
473465
474466pub const ImportLookupEntry64 = struct {
475 pub const ByName = packed struct {
467 pub const ByName = packed struct(u64) {
476468 name_table_rva: u31,
477469 unused: u32 = 0,
478470 flag: u1 = 0,
479471 };
480472
481 pub const ByOrdinal = packed struct {
473 pub const ByOrdinal = packed struct(u64) {
482474 ordinal_number: u16,
483475 unused: u47 = 0,
484476 flag: u1 = 1,
......@@ -519,7 +511,7 @@ pub const SectionHeader = extern struct {
519511 pointer_to_linenumbers: u32,
520512 number_of_relocations: u16,
521513 number_of_linenumbers: u16,
522 flags: SectionHeaderFlags,
514 flags: SectionHeader.Flags,
523515
524516 pub fn getName(self: *align(1) const SectionHeader) ?[]const u8 {
525517 if (self.name[0] == '/') return null;
......@@ -546,109 +538,121 @@ pub const SectionHeader = extern struct {
546538 }
547539
548540 pub fn isCode(self: SectionHeader) bool {
549 return self.flags.CNT_CODE == 0b1;
541 return self.flags.CNT_CODE;
550542 }
551543
552544 pub fn isComdat(self: SectionHeader) bool {
553 return self.flags.LNK_COMDAT == 0b1;
545 return self.flags.LNK_COMDAT;
554546 }
555};
556547
557pub const SectionHeaderFlags = packed struct {
558 _reserved_0: u3 = 0,
548 pub const Flags = packed struct(u32) {
549 SCALE_INDEX: bool = false,
550
551 unused1: u2 = 0,
559552
560 /// The section should not be padded to the next boundary.
561 /// This flag is obsolete and is replaced by IMAGE_SCN_ALIGN_1BYTES.
562 /// This is valid only for object files.
563 TYPE_NO_PAD: u1 = 0,
553 /// The section should not be padded to the next boundary.
554 /// This flag is obsolete and is replaced by `.ALIGN = .@"1BYTES"`.
555 /// This is valid only for object files.
556 TYPE_NO_PAD: bool = false,
564557
565 _reserved_1: u1 = 0,
558 unused4: u1 = 0,
566559
567 /// The section contains executable code.
568 CNT_CODE: u1 = 0,
560 /// The section contains executable code.
561 CNT_CODE: bool = false,
569562
570 /// The section contains initialized data.
571 CNT_INITIALIZED_DATA: u1 = 0,
563 /// The section contains initialized data.
564 CNT_INITIALIZED_DATA: bool = false,
572565
573 /// The section contains uninitialized data.
574 CNT_UNINITIALIZED_DATA: u1 = 0,
566 /// The section contains uninitialized data.
567 CNT_UNINITIALIZED_DATA: bool = false,
575568
576 /// Reserved for future use.
577 LNK_OTHER: u1 = 0,
569 /// Reserved for future use.
570 LNK_OTHER: bool = false,
578571
579 /// The section contains comments or other information.
580 /// The .drectve section has this type.
581 /// This is valid for object files only.
582 LNK_INFO: u1 = 0,
572 /// The section contains comments or other information.
573 /// The .drectve section has this type.
574 /// This is valid for object files only.
575 LNK_INFO: bool = false,
583576
584 _reserved_2: u1 = 0,
577 unused10: u1 = 0,
585578
586 /// The section will not become part of the image.
587 /// This is valid only for object files.
588 LNK_REMOVE: u1 = 0,
579 /// The section will not become part of the image.
580 /// This is valid only for object files.
581 LNK_REMOVE: bool = false,
589582
590 /// The section contains COMDAT data.
591 /// For more information, see COMDAT Sections (Object Only).
592 /// This is valid only for object files.
593 LNK_COMDAT: u1 = 0,
583 /// The section contains COMDAT data.
584 /// For more information, see COMDAT Sections (Object Only).
585 /// This is valid only for object files.
586 LNK_COMDAT: bool = false,
594587
595 _reserved_3: u2 = 0,
588 unused13: u2 = 0,
596589
597 /// The section contains data referenced through the global pointer (GP).
598 GPREL: u1 = 0,
590 union14: packed union {
591 mask: u1,
592 /// The section contains data referenced through the global pointer (GP).
593 GPREL: bool,
594 MEM_FARDATA: bool,
595 } = .{ .mask = 0 },
599596
600 /// Reserved for future use.
601 MEM_PURGEABLE: u1 = 0,
597 unused15: u1 = 0,
602598
603 /// Reserved for future use.
604 MEM_16BIT: u1 = 0,
599 union16: packed union {
600 mask: u1,
601 MEM_PURGEABLE: bool,
602 MEM_16BIT: bool,
603 } = .{ .mask = 0 },
605604
606 /// Reserved for future use.
607 MEM_LOCKED: u1 = 0,
605 /// Reserved for future use.
606 MEM_LOCKED: bool = false,
608607
609 /// Reserved for future use.
610 MEM_PRELOAD: u1 = 0,
608 /// Reserved for future use.
609 MEM_PRELOAD: bool = false,
611610
612 /// Takes on multiple values according to flags:
613 /// pub const IMAGE_SCN_ALIGN_1BYTES: u32 = 0x100000;
614 /// pub const IMAGE_SCN_ALIGN_2BYTES: u32 = 0x200000;
615 /// pub const IMAGE_SCN_ALIGN_4BYTES: u32 = 0x300000;
616 /// pub const IMAGE_SCN_ALIGN_8BYTES: u32 = 0x400000;
617 /// pub const IMAGE_SCN_ALIGN_16BYTES: u32 = 0x500000;
618 /// pub const IMAGE_SCN_ALIGN_32BYTES: u32 = 0x600000;
619 /// pub const IMAGE_SCN_ALIGN_64BYTES: u32 = 0x700000;
620 /// pub const IMAGE_SCN_ALIGN_128BYTES: u32 = 0x800000;
621 /// pub const IMAGE_SCN_ALIGN_256BYTES: u32 = 0x900000;
622 /// pub const IMAGE_SCN_ALIGN_512BYTES: u32 = 0xA00000;
623 /// pub const IMAGE_SCN_ALIGN_1024BYTES: u32 = 0xB00000;
624 /// pub const IMAGE_SCN_ALIGN_2048BYTES: u32 = 0xC00000;
625 /// pub const IMAGE_SCN_ALIGN_4096BYTES: u32 = 0xD00000;
626 /// pub const IMAGE_SCN_ALIGN_8192BYTES: u32 = 0xE00000;
627 ALIGN: u4 = 0,
611 ALIGN: SectionHeader.Flags.Align = .NONE,
628612
629 /// The section contains extended relocations.
630 LNK_NRELOC_OVFL: u1 = 0,
613 /// The section contains extended relocations.
614 LNK_NRELOC_OVFL: bool = false,
631615
632 /// The section can be discarded as needed.
633 MEM_DISCARDABLE: u1 = 0,
616 /// The section can be discarded as needed.
617 MEM_DISCARDABLE: bool = false,
634618
635 /// The section cannot be cached.
636 MEM_NOT_CACHED: u1 = 0,
619 /// The section cannot be cached.
620 MEM_NOT_CACHED: bool = false,
637621
638 /// The section is not pageable.
639 MEM_NOT_PAGED: u1 = 0,
622 /// The section is not pageable.
623 MEM_NOT_PAGED: bool = false,
640624
641 /// The section can be shared in memory.
642 MEM_SHARED: u1 = 0,
625 /// The section can be shared in memory.
626 MEM_SHARED: bool = false,
643627
644 /// The section can be executed as code.
645 MEM_EXECUTE: u1 = 0,
628 /// The section can be executed as code.
629 MEM_EXECUTE: bool = false,
646630
647 /// The section can be read.
648 MEM_READ: u1 = 0,
631 /// The section can be read.
632 MEM_READ: bool = false,
649633
650 /// The section can be written to.
651 MEM_WRITE: u1 = 0,
634 /// The section can be written to.
635 MEM_WRITE: bool = false,
636
637 pub const Align = enum(u4) {
638 NONE = 0,
639 @"1BYTES" = 1,
640 @"2BYTES" = 2,
641 @"4BYTES" = 3,
642 @"8BYTES" = 4,
643 @"16BYTES" = 5,
644 @"32BYTES" = 6,
645 @"64BYTES" = 7,
646 @"128BYTES" = 8,
647 @"256BYTES" = 9,
648 @"512BYTES" = 10,
649 @"1024BYTES" = 11,
650 @"2048BYTES" = 12,
651 @"4096BYTES" = 13,
652 @"8192BYTES" = 14,
653 _,
654 };
655 };
652656};
653657
654658pub const Symbol = struct {
......@@ -691,7 +695,7 @@ pub const SectionNumber = enum(u16) {
691695 _,
692696};
693697
694pub const SymType = packed struct {
698pub const SymType = packed struct(u16) {
695699 complex_type: ComplexType,
696700 base_type: BaseType,
697701};
......@@ -982,87 +986,7 @@ pub const DebugInfoDefinition = struct {
982986 unused_3: [2]u8,
983987};
984988
985pub const MachineType = enum(u16) {
986 UNKNOWN = 0x0,
987 /// Alpha AXP, 32-bit address space
988 ALPHA = 0x184,
989 /// Alpha 64, 64-bit address space
990 ALPHA64 = 0x284,
991 /// Matsushita AM33
992 AM33 = 0x1d3,
993 /// x64
994 X64 = 0x8664,
995 /// ARM little endian
996 ARM = 0x1c0,
997 /// ARM64 little endian
998 ARM64 = 0xaa64,
999 /// ARM64EC
1000 ARM64EC = 0xa641,
1001 /// ARM64X
1002 ARM64X = 0xa64e,
1003 /// ARM Thumb-2 little endian
1004 ARMNT = 0x1c4,
1005 /// CEE
1006 CEE = 0xc0ee,
1007 /// CEF
1008 CEF = 0xcef,
1009 /// Hybrid PE
1010 CHPE_X86 = 0x3a64,
1011 /// EFI byte code
1012 EBC = 0xebc,
1013 /// Intel 386 or later processors and compatible processors
1014 I386 = 0x14c,
1015 /// Intel Itanium processor family
1016 IA64 = 0x200,
1017 /// LoongArch32
1018 LOONGARCH32 = 0x6232,
1019 /// LoongArch64
1020 LOONGARCH64 = 0x6264,
1021 /// Mitsubishi M32R little endian
1022 M32R = 0x9041,
1023 /// MIPS16
1024 MIPS16 = 0x266,
1025 /// MIPS with FPU
1026 MIPSFPU = 0x366,
1027 /// MIPS16 with FPU
1028 MIPSFPU16 = 0x466,
1029 /// Power PC little endian
1030 POWERPC = 0x1f0,
1031 /// Power PC with floating point support
1032 POWERPCFP = 0x1f1,
1033 /// MIPS little endian
1034 R3000 = 0x162,
1035 /// MIPS little endian
1036 R4000 = 0x166,
1037 /// MIPS little endian
1038 R10000 = 0x168,
1039 /// RISC-V 32-bit address space
1040 RISCV32 = 0x5032,
1041 /// RISC-V 64-bit address space
1042 RISCV64 = 0x5064,
1043 /// RISC-V 128-bit address space
1044 RISCV128 = 0x5128,
1045 /// Hitachi SH3
1046 SH3 = 0x1a2,
1047 /// Hitachi SH3 DSP
1048 SH3DSP = 0x1a3,
1049 /// SH3E little-endian
1050 SH3E = 0x1a4,
1051 /// Hitachi SH4
1052 SH4 = 0x1a6,
1053 /// Hitachi SH5
1054 SH5 = 0x1a8,
1055 /// Thumb
1056 THUMB = 0x1c2,
1057 /// Infineon
1058 TRICORE = 0x520,
1059 /// MIPS little-endian WCE v2
1060 WCEMIPSV2 = 0x169,
1061
1062 _,
1063};
1064
1065pub const CoffError = error{
989pub const Error = error{
1066990 InvalidPEMagic,
1067991 InvalidPEHeader,
1068992 InvalidMachine,
......@@ -1104,7 +1028,7 @@ pub const Coff = struct {
11041028
11051029 // Do some basic validation upfront
11061030 if (is_image) {
1107 const coff_header = coff.getCoffHeader();
1031 const coff_header = coff.getHeader();
11081032 if (coff_header.size_of_optional_header == 0) return error.MissingPEHeader;
11091033 }
11101034
......@@ -1161,31 +1085,31 @@ pub const Coff = struct {
11611085 return self.data[start .. start + len];
11621086 }
11631087
1164 pub fn getCoffHeader(self: Coff) CoffHeader {
1165 return @as(*align(1) const CoffHeader, @ptrCast(self.data[self.coff_header_offset..][0..@sizeOf(CoffHeader)])).*;
1088 pub fn getHeader(self: Coff) Header {
1089 return @as(*align(1) const Header, @ptrCast(self.data[self.coff_header_offset..][0..@sizeOf(Header)])).*;
11661090 }
11671091
11681092 pub fn getOptionalHeader(self: Coff) OptionalHeader {
11691093 assert(self.is_image);
1170 const offset = self.coff_header_offset + @sizeOf(CoffHeader);
1094 const offset = self.coff_header_offset + @sizeOf(Header);
11711095 return @as(*align(1) const OptionalHeader, @ptrCast(self.data[offset..][0..@sizeOf(OptionalHeader)])).*;
11721096 }
11731097
1174 pub fn getOptionalHeader32(self: Coff) OptionalHeaderPE32 {
1098 pub fn getOptionalHeader32(self: Coff) OptionalHeader.PE32 {
11751099 assert(self.is_image);
1176 const offset = self.coff_header_offset + @sizeOf(CoffHeader);
1177 return @as(*align(1) const OptionalHeaderPE32, @ptrCast(self.data[offset..][0..@sizeOf(OptionalHeaderPE32)])).*;
1100 const offset = self.coff_header_offset + @sizeOf(Header);
1101 return @as(*align(1) const OptionalHeader.PE32, @ptrCast(self.data[offset..][0..@sizeOf(OptionalHeader.PE32)])).*;
11781102 }
11791103
1180 pub fn getOptionalHeader64(self: Coff) OptionalHeaderPE64 {
1104 pub fn getOptionalHeader64(self: Coff) OptionalHeader.@"PE32+" {
11811105 assert(self.is_image);
1182 const offset = self.coff_header_offset + @sizeOf(CoffHeader);
1183 return @as(*align(1) const OptionalHeaderPE64, @ptrCast(self.data[offset..][0..@sizeOf(OptionalHeaderPE64)])).*;
1106 const offset = self.coff_header_offset + @sizeOf(Header);
1107 return @as(*align(1) const OptionalHeader.@"PE32+", @ptrCast(self.data[offset..][0..@sizeOf(OptionalHeader.@"PE32+")])).*;
11841108 }
11851109
11861110 pub fn getImageBase(self: Coff) u64 {
11871111 const hdr = self.getOptionalHeader();
1188 return switch (hdr.magic) {
1112 return switch (@intFromEnum(hdr.magic)) {
11891113 IMAGE_NT_OPTIONAL_HDR32_MAGIC => self.getOptionalHeader32().image_base,
11901114 IMAGE_NT_OPTIONAL_HDR64_MAGIC => self.getOptionalHeader64().image_base,
11911115 else => unreachable, // We assume we have validated the header already
......@@ -1194,7 +1118,7 @@ pub const Coff = struct {
11941118
11951119 pub fn getNumberOfDataDirectories(self: Coff) u32 {
11961120 const hdr = self.getOptionalHeader();
1197 return switch (hdr.magic) {
1121 return switch (@intFromEnum(hdr.magic)) {
11981122 IMAGE_NT_OPTIONAL_HDR32_MAGIC => self.getOptionalHeader32().number_of_rva_and_sizes,
11991123 IMAGE_NT_OPTIONAL_HDR64_MAGIC => self.getOptionalHeader64().number_of_rva_and_sizes,
12001124 else => unreachable, // We assume we have validated the header already
......@@ -1203,17 +1127,17 @@ pub const Coff = struct {
12031127
12041128 pub fn getDataDirectories(self: *const Coff) []align(1) const ImageDataDirectory {
12051129 const hdr = self.getOptionalHeader();
1206 const size: usize = switch (hdr.magic) {
1207 IMAGE_NT_OPTIONAL_HDR32_MAGIC => @sizeOf(OptionalHeaderPE32),
1208 IMAGE_NT_OPTIONAL_HDR64_MAGIC => @sizeOf(OptionalHeaderPE64),
1130 const size: usize = switch (@intFromEnum(hdr.magic)) {
1131 IMAGE_NT_OPTIONAL_HDR32_MAGIC => @sizeOf(OptionalHeader.PE32),
1132 IMAGE_NT_OPTIONAL_HDR64_MAGIC => @sizeOf(OptionalHeader.@"PE32+"),
12091133 else => unreachable, // We assume we have validated the header already
12101134 };
1211 const offset = self.coff_header_offset + @sizeOf(CoffHeader) + size;
1135 const offset = self.coff_header_offset + @sizeOf(Header) + size;
12121136 return @as([*]align(1) const ImageDataDirectory, @ptrCast(self.data[offset..]))[0..self.getNumberOfDataDirectories()];
12131137 }
12141138
12151139 pub fn getSymtab(self: *const Coff) ?Symtab {
1216 const coff_header = self.getCoffHeader();
1140 const coff_header = self.getHeader();
12171141 if (coff_header.pointer_to_symbol_table == 0) return null;
12181142
12191143 const offset = coff_header.pointer_to_symbol_table;
......@@ -1222,7 +1146,7 @@ pub const Coff = struct {
12221146 }
12231147
12241148 pub fn getStrtab(self: *const Coff) error{InvalidStrtabSize}!?Strtab {
1225 const coff_header = self.getCoffHeader();
1149 const coff_header = self.getHeader();
12261150 if (coff_header.pointer_to_symbol_table == 0) return null;
12271151
12281152 const offset = coff_header.pointer_to_symbol_table + Symbol.sizeOf() * coff_header.number_of_symbols;
......@@ -1238,8 +1162,8 @@ pub const Coff = struct {
12381162 }
12391163
12401164 pub fn getSectionHeaders(self: *const Coff) []align(1) const SectionHeader {
1241 const coff_header = self.getCoffHeader();
1242 const offset = self.coff_header_offset + @sizeOf(CoffHeader) + coff_header.size_of_optional_header;
1165 const coff_header = self.getHeader();
1166 const offset = self.coff_header_offset + @sizeOf(Header) + coff_header.size_of_optional_header;
12431167 return @as([*]align(1) const SectionHeader, @ptrCast(self.data.ptr + offset))[0..coff_header.number_of_sections];
12441168 }
12451169
......@@ -1414,14 +1338,14 @@ pub const Strtab = struct {
14141338};
14151339
14161340pub const ImportHeader = extern struct {
1417 sig1: MachineType,
1341 sig1: IMAGE.FILE.MACHINE,
14181342 sig2: u16,
14191343 version: u16,
1420 machine: MachineType,
1344 machine: IMAGE.FILE.MACHINE,
14211345 time_date_stamp: u32,
14221346 size_of_data: u32,
14231347 hint: u16,
1424 types: packed struct {
1348 types: packed struct(u32) {
14251349 type: ImportType,
14261350 name_type: ImportNameType,
14271351 reserved: u11,
......@@ -1461,119 +1385,534 @@ pub const Relocation = extern struct {
14611385 type: u16,
14621386};
14631387
1464pub const ImageRelAmd64 = enum(u16) {
1465 /// The relocation is ignored.
1466 absolute = 0,
1467
1468 /// The 64-bit VA of the relocation target.
1469 addr64 = 1,
1470
1471 /// The 32-bit VA of the relocation target.
1472 addr32 = 2,
1473
1474 /// The 32-bit address without an image base.
1475 addr32nb = 3,
1476
1477 /// The 32-bit relative address from the byte following the relocation.
1478 rel32 = 4,
1479
1480 /// The 32-bit address relative to byte distance 1 from the relocation.
1481 rel32_1 = 5,
1482
1483 /// The 32-bit address relative to byte distance 2 from the relocation.
1484 rel32_2 = 6,
1485
1486 /// The 32-bit address relative to byte distance 3 from the relocation.
1487 rel32_3 = 7,
1488
1489 /// The 32-bit address relative to byte distance 4 from the relocation.
1490 rel32_4 = 8,
1491
1492 /// The 32-bit address relative to byte distance 5 from the relocation.
1493 rel32_5 = 9,
1494
1495 /// The 16-bit section index of the section that contains the target.
1496 /// This is used to support debugging information.
1497 section = 10,
1498
1499 /// The 32-bit offset of the target from the beginning of its section.
1500 /// This is used to support debugging information and static thread local storage.
1501 secrel = 11,
1502
1503 /// A 7-bit unsigned offset from the base of the section that contains the target.
1504 secrel7 = 12,
1505
1506 /// CLR tokens.
1507 token = 13,
1508
1509 /// A 32-bit signed span-dependent value emitted into the object.
1510 srel32 = 14,
1511
1512 /// A pair that must immediately follow every span-dependent value.
1513 pair = 15,
1514
1515 /// A 32-bit signed span-dependent value that is applied at link time.
1516 sspan32 = 16,
1517
1518 _,
1519};
1520
1521pub const ImageRelArm64 = enum(u16) {
1522 /// The relocation is ignored.
1523 absolute = 0,
1524
1525 /// The 32-bit VA of the target.
1526 addr32 = 1,
1527
1528 /// The 32-bit RVA of the target.
1529 addr32nb = 2,
1530
1531 /// The 26-bit relative displacement to the target, for B and BL instructions.
1532 branch26 = 3,
1533
1534 /// The page base of the target, for ADRP instruction.
1535 pagebase_rel21 = 4,
1536
1537 /// The 21-bit relative displacement to the target, for instruction ADR.
1538 rel21 = 5,
1539
1540 /// The 12-bit page offset of the target, for instructions ADD/ADDS (immediate) with zero shift.
1541 pageoffset_12a = 6,
1542
1543 /// The 12-bit page offset of the target, for instruction LDR (indexed, unsigned immediate).
1544 pageoffset_12l = 7,
1545
1546 /// The 32-bit offset of the target from the beginning of its section.
1547 /// This is used to support debugging information and static thread local storage.
1548 secrel = 8,
1549
1550 /// Bit 0:11 of section offset of the target for instructions ADD/ADDS (immediate) with zero shift.
1551 low12a = 9,
1388pub const IMAGE = struct {
1389 pub const FILE = struct {
1390 /// Machine Types
1391 /// The Machine field has one of the following values, which specify the CPU type.
1392 /// An image file can be run only on the specified machine or on a system that emulates the specified machine.
1393 pub const MACHINE = enum(u16) {
1394 /// The content of this field is assumed to be applicable to any machine type
1395 UNKNOWN = 0x0,
1396 /// Alpha AXP, 32-bit address space
1397 ALPHA = 0x184,
1398 /// Alpha 64, 64-bit address space
1399 ALPHA64 = 0x284,
1400 /// Matsushita AM33
1401 AM33 = 0x1d3,
1402 /// x64
1403 AMD64 = 0x8664,
1404 /// ARM little endian
1405 ARM = 0x1c0,
1406 /// ARM64 little endian
1407 ARM64 = 0xaa64,
1408 /// ABI that enables interoperability between native ARM64 and emulated x64 code.
1409 ARM64EC = 0xA641,
1410 /// Binary format that allows both native ARM64 and ARM64EC code to coexist in the same file.
1411 ARM64X = 0xA64E,
1412 /// ARM Thumb-2 little endian
1413 ARMNT = 0x1c4,
1414 /// EFI byte code
1415 EBC = 0xebc,
1416 /// Intel 386 or later processors and compatible processors
1417 I386 = 0x14c,
1418 /// Intel Itanium processor family
1419 IA64 = 0x200,
1420 /// LoongArch 32-bit processor family
1421 LOONGARCH32 = 0x6232,
1422 /// LoongArch 64-bit processor family
1423 LOONGARCH64 = 0x6264,
1424 /// Mitsubishi M32R little endian
1425 M32R = 0x9041,
1426 /// MIPS16
1427 MIPS16 = 0x266,
1428 /// MIPS with FPU
1429 MIPSFPU = 0x366,
1430 /// MIPS16 with FPU
1431 MIPSFPU16 = 0x466,
1432 /// Power PC little endian
1433 POWERPC = 0x1f0,
1434 /// Power PC with floating point support
1435 POWERPCFP = 0x1f1,
1436 /// MIPS I compatible 32-bit big endian
1437 R3000BE = 0x160,
1438 /// MIPS I compatible 32-bit little endian
1439 R3000 = 0x162,
1440 /// MIPS III compatible 64-bit little endian
1441 R4000 = 0x166,
1442 /// MIPS IV compatible 64-bit little endian
1443 R10000 = 0x168,
1444 /// RISC-V 32-bit address space
1445 RISCV32 = 0x5032,
1446 /// RISC-V 64-bit address space
1447 RISCV64 = 0x5064,
1448 /// RISC-V 128-bit address space
1449 RISCV128 = 0x5128,
1450 /// Hitachi SH3
1451 SH3 = 0x1a2,
1452 /// Hitachi SH3 DSP
1453 SH3DSP = 0x1a3,
1454 /// Hitachi SH4
1455 SH4 = 0x1a6,
1456 /// Hitachi SH5
1457 SH5 = 0x1a8,
1458 /// Thumb
1459 THUMB = 0x1c2,
1460 /// MIPS little-endian WCE v2
1461 WCEMIPSV2 = 0x169,
1462 _,
1463 /// AXP 64 (Same as Alpha 64)
1464 pub const AXP64: IMAGE.FILE.MACHINE = .ALPHA64;
1465 };
1466 };
15521467
1553 /// Bit 12:23 of section offset of the target, for instructions ADD/ADDS (immediate) with zero shift.
1554 high12a = 10,
1468 pub const REL = struct {
1469 /// x64 Processors
1470 /// The following relocation type indicators are defined for x64 and compatible processors.
1471 pub const AMD64 = enum(u16) {
1472 /// The relocation is ignored.
1473 ABSOLUTE = 0x0000,
1474 /// The 64-bit VA of the relocation target.
1475 ADDR64 = 0x0001,
1476 /// The 32-bit VA of the relocation target.
1477 ADDR32 = 0x0002,
1478 /// The 32-bit address without an image base (RVA).
1479 ADDR32NB = 0x0003,
1480 /// The 32-bit relative address from the byte following the relocation.
1481 REL32 = 0x0004,
1482 /// The 32-bit address relative to byte distance 1 from the relocation.
1483 REL32_1 = 0x0005,
1484 /// The 32-bit address relative to byte distance 2 from the relocation.
1485 REL32_2 = 0x0006,
1486 /// The 32-bit address relative to byte distance 3 from the relocation.
1487 REL32_3 = 0x0007,
1488 /// The 32-bit address relative to byte distance 4 from the relocation.
1489 REL32_4 = 0x0008,
1490 /// The 32-bit address relative to byte distance 5 from the relocation.
1491 REL32_5 = 0x0009,
1492 /// The 16-bit section index of the section that contains the target.
1493 /// This is used to support debugging information.
1494 SECTION = 0x000A,
1495 /// The 32-bit offset of the target from the beginning of its section.
1496 /// This is used to support debugging information and static thread local storage.
1497 SECREL = 0x000B,
1498 /// A 7-bit unsigned offset from the base of the section that contains the target.
1499 SECREL7 = 0x000C,
1500 /// CLR tokens.
1501 TOKEN = 0x000D,
1502 /// A 32-bit signed span-dependent value emitted into the object.
1503 SREL32 = 0x000E,
1504 /// A pair that must immediately follow every span-dependent value.
1505 PAIR = 0x000F,
1506 /// A 32-bit signed span-dependent value that is applied at link time.
1507 SSPAN32 = 0x0010,
1508 _,
1509 };
15551510
1556 /// Bit 0:11 of section offset of the target, for instruction LDR (indexed, unsigned immediate).
1557 low12l = 11,
1511 /// ARM Processors
1512 /// The following relocation type indicators are defined for ARM processors.
1513 pub const ARM = enum(u16) {
1514 /// The relocation is ignored.
1515 ABSOLUTE = 0x0000,
1516 /// The 32-bit VA of the target.
1517 ADDR32 = 0x0001,
1518 /// The 32-bit RVA of the target.
1519 ADDR32NB = 0x0002,
1520 /// The 24-bit relative displacement to the target.
1521 BRANCH24 = 0x0003,
1522 /// The reference to a subroutine call.
1523 /// The reference consists of two 16-bit instructions with 11-bit offsets.
1524 BRANCH11 = 0x0004,
1525 /// The 32-bit relative address from the byte following the relocation.
1526 REL32 = 0x000A,
1527 /// The 16-bit section index of the section that contains the target.
1528 /// This is used to support debugging information.
1529 SECTION = 0x000E,
1530 /// The 32-bit offset of the target from the beginning of its section.
1531 /// This is used to support debugging information and static thread local storage.
1532 SECREL = 0x000F,
1533 /// The 32-bit VA of the target.
1534 /// This relocation is applied using a MOVW instruction for the low 16 bits followed by a MOVT for the high 16 bits.
1535 MOV32 = 0x0010,
1536 /// The 32-bit VA of the target.
1537 /// This relocation is applied using a MOVW instruction for the low 16 bits followed by a MOVT for the high 16 bits.
1538 THUMB_MOV32 = 0x0011,
1539 /// The instruction is fixed up with the 21-bit relative displacement to the 2-byte aligned target.
1540 /// The least significant bit of the displacement is always zero and is not stored.
1541 /// This relocation corresponds to a Thumb-2 32-bit conditional B instruction.
1542 THUMB_BRANCH20 = 0x0012,
1543 Unused = 0x0013,
1544 /// The instruction is fixed up with the 25-bit relative displacement to the 2-byte aligned target.
1545 /// The least significant bit of the displacement is zero and is not stored.This relocation corresponds to a Thumb-2 B instruction.
1546 THUMB_BRANCH24 = 0x0014,
1547 /// The instruction is fixed up with the 25-bit relative displacement to the 4-byte aligned target.
1548 /// The low 2 bits of the displacement are zero and are not stored.
1549 /// This relocation corresponds to a Thumb-2 BLX instruction.
1550 THUMB_BLX23 = 0x0015,
1551 /// The relocation is valid only when it immediately follows a ARM_REFHI or THUMB_REFHI.
1552 /// Its SymbolTableIndex contains a displacement and not an index into the symbol table.
1553 PAIR = 0x0016,
1554 _,
1555 };
15581556
1559 /// CLR token.
1560 token = 12,
1557 /// ARM64 Processors
1558 /// The following relocation type indicators are defined for ARM64 processors.
1559 pub const ARM64 = enum(u16) {
1560 /// The relocation is ignored.
1561 ABSOLUTE = 0x0000,
1562 /// The 32-bit VA of the target.
1563 ADDR32 = 0x0001,
1564 /// The 32-bit RVA of the target.
1565 ADDR32NB = 0x0002,
1566 /// The 26-bit relative displacement to the target, for B and BL instructions.
1567 BRANCH26 = 0x0003,
1568 /// The page base of the target, for ADRP instruction.
1569 PAGEBASE_REL21 = 0x0004,
1570 /// The 12-bit relative displacement to the target, for instruction ADR
1571 REL21 = 0x0005,
1572 /// The 12-bit page offset of the target, for instructions ADD/ADDS (immediate) with zero shift.
1573 PAGEOFFSET_12A = 0x0006,
1574 /// The 12-bit page offset of the target, for instruction LDR (indexed, unsigned immediate).
1575 PAGEOFFSET_12L = 0x0007,
1576 /// The 32-bit offset of the target from the beginning of its section.
1577 /// This is used to support debugging information and static thread local storage.
1578 SECREL = 0x0008,
1579 /// Bit 0:11 of section offset of the target, for instructions ADD/ADDS (immediate) with zero shift.
1580 SECREL_LOW12A = 0x0009,
1581 /// Bit 12:23 of section offset of the target, for instructions ADD/ADDS (immediate) with zero shift.
1582 SECREL_HIGH12A = 0x000A,
1583 /// Bit 0:11 of section offset of the target, for instruction LDR (indexed, unsigned immediate).
1584 SECREL_LOW12L = 0x000B,
1585 /// CLR token.
1586 TOKEN = 0x000C,
1587 /// The 16-bit section index of the section that contains the target.
1588 /// This is used to support debugging information.
1589 SECTION = 0x000D,
1590 /// The 64-bit VA of the relocation target.
1591 ADDR64 = 0x000E,
1592 /// The 19-bit offset to the relocation target, for conditional B instruction.
1593 BRANCH19 = 0x000F,
1594 /// The 14-bit offset to the relocation target, for instructions TBZ and TBNZ.
1595 BRANCH14 = 0x0010,
1596 /// The 32-bit relative address from the byte following the relocation.
1597 REL32 = 0x0011,
1598 _,
1599 };
15611600
1562 /// The 16-bit section index of the section that contains the target.
1563 /// This is used to support debugging information.
1564 section = 13,
1601 /// Hitachi SuperH Processors
1602 /// The following relocation type indicators are defined for SH3 and SH4 processors.
1603 /// SH5-specific relocations are noted as SHM (SH Media).
1604 pub const SH = enum(u16) {
1605 /// The relocation is ignored.
1606 @"3_ABSOLUTE" = 0x0000,
1607 /// A reference to the 16-bit location that contains the VA of the target symbol.
1608 @"3_DIRECT16" = 0x0001,
1609 /// The 32-bit VA of the target symbol.
1610 @"3_DIRECT32" = 0x0002,
1611 /// A reference to the 8-bit location that contains the VA of the target symbol.
1612 @"3_DIRECT8" = 0x0003,
1613 /// A reference to the 8-bit instruction that contains the effective 16-bit VA of the target symbol.
1614 @"3_DIRECT8_WORD" = 0x0004,
1615 /// A reference to the 8-bit instruction that contains the effective 32-bit VA of the target symbol.
1616 @"3_DIRECT8_LONG" = 0x0005,
1617 /// A reference to the 8-bit location whose low 4 bits contain the VA of the target symbol.
1618 @"3_DIRECT4" = 0x0006,
1619 /// A reference to the 8-bit instruction whose low 4 bits contain the effective 16-bit VA of the target symbol.
1620 @"3_DIRECT4_WORD" = 0x0007,
1621 /// A reference to the 8-bit instruction whose low 4 bits contain the effective 32-bit VA of the target symbol.
1622 @"3_DIRECT4_LONG" = 0x0008,
1623 /// A reference to the 8-bit instruction that contains the effective 16-bit relative offset of the target symbol.
1624 @"3_PCREL8_WORD" = 0x0009,
1625 /// A reference to the 8-bit instruction that contains the effective 32-bit relative offset of the target symbol.
1626 @"3_PCREL8_LONG" = 0x000A,
1627 /// A reference to the 16-bit instruction whose low 12 bits contain the effective 16-bit relative offset of the target symbol.
1628 @"3_PCREL12_WORD" = 0x000B,
1629 /// A reference to a 32-bit location that is the VA of the section that contains the target symbol.
1630 @"3_STARTOF_SECTION" = 0x000C,
1631 /// A reference to the 32-bit location that is the size of the section that contains the target symbol.
1632 @"3_SIZEOF_SECTION" = 0x000D,
1633 /// The 16-bit section index of the section that contains the target.
1634 /// This is used to support debugging information.
1635 @"3_SECTION" = 0x000E,
1636 /// The 32-bit offset of the target from the beginning of its section.
1637 /// This is used to support debugging information and static thread local storage.
1638 @"3_SECREL" = 0x000F,
1639 /// The 32-bit RVA of the target symbol.
1640 @"3_DIRECT32_NB" = 0x0010,
1641 /// GP relative.
1642 @"3_GPREL4_LONG" = 0x0011,
1643 /// CLR token.
1644 @"3_TOKEN" = 0x0012,
1645 /// The offset from the current instruction in longwords.
1646 /// If the NOMODE bit is not set, insert the inverse of the low bit at bit 32 to select PTA or PTB.
1647 M_PCRELPT = 0x0013,
1648 /// The low 16 bits of the 32-bit address.
1649 M_REFLO = 0x0014,
1650 /// The high 16 bits of the 32-bit address.
1651 M_REFHALF = 0x0015,
1652 /// The low 16 bits of the relative address.
1653 M_RELLO = 0x0016,
1654 /// The high 16 bits of the relative address.
1655 M_RELHALF = 0x0017,
1656 /// The relocation is valid only when it immediately follows a REFHALF, RELHALF, or RELLO relocation.
1657 /// The SymbolTableIndex field of the relocation contains a displacement and not an index into the symbol table.
1658 M_PAIR = 0x0018,
1659 /// The relocation ignores section mode.
1660 M_NOMODE = 0x8000,
1661 _,
1662 };
15651663
1566 /// The 64-bit VA of the relocation target.
1567 addr64 = 14,
1664 /// IBM PowerPC Processors
1665 /// The following relocation type indicators are defined for PowerPC processors.
1666 pub const PPC = enum(u16) {
1667 /// The relocation is ignored.
1668 ABSOLUTE = 0x0000,
1669 /// The 64-bit VA of the target.
1670 ADDR64 = 0x0001,
1671 /// The 32-bit VA of the target.
1672 ADDR32 = 0x0002,
1673 /// The low 24 bits of the VA of the target.
1674 /// This is valid only when the target symbol is absolute and can be sign-extended to its original value.
1675 ADDR24 = 0x0003,
1676 /// The low 16 bits of the target's VA.
1677 ADDR16 = 0x0004,
1678 /// The low 14 bits of the target's VA.
1679 /// This is valid only when the target symbol is absolute and can be sign-extended to its original value.
1680 ADDR14 = 0x0005,
1681 /// A 24-bit PC-relative offset to the symbol's location.
1682 REL24 = 0x0006,
1683 /// A 14-bit PC-relative offset to the symbol's location.
1684 REL14 = 0x0007,
1685 /// The 32-bit RVA of the target.
1686 ADDR32NB = 0x000A,
1687 /// The 32-bit offset of the target from the beginning of its section.
1688 /// This is used to support debugging information and static thread local storage.
1689 SECREL = 0x000B,
1690 /// The 16-bit section index of the section that contains the target.
1691 /// This is used to support debugging information.
1692 SECTION = 0x000C,
1693 /// The 16-bit offset of the target from the beginning of its section.
1694 /// This is used to support debugging information and static thread local storage.
1695 SECREL16 = 0x000F,
1696 /// The high 16 bits of the target's 32-bit VA.
1697 /// This is used for the first instruction in a two-instruction sequence that loads a full address.
1698 /// This relocation must be immediately followed by a PAIR relocation whose SymbolTableIndex contains a signed 16-bit displacement that is added to the upper 16 bits that was taken from the location that is being relocated.
1699 REFHI = 0x0010,
1700 /// The low 16 bits of the target's VA.
1701 REFLO = 0x0011,
1702 /// A relocation that is valid only when it immediately follows a REFHI or SECRELHI relocation.
1703 /// Its SymbolTableIndex contains a displacement and not an index into the symbol table.
1704 PAIR = 0x0012,
1705 /// The low 16 bits of the 32-bit offset of the target from the beginning of its section.
1706 SECRELLO = 0x0013,
1707 /// The 16-bit signed displacement of the target relative to the GP register.
1708 GPREL = 0x0015,
1709 /// The CLR token.
1710 TOKEN = 0x0016,
1711 _,
1712 };
15681713
1569 /// The 19-bit offset to the relocation target, for conditional B instruction.
1570 branch19 = 15,
1714 /// Intel 386 Processors
1715 /// The following relocation type indicators are defined for Intel 386 and compatible processors.
1716 pub const I386 = enum(u16) {
1717 /// The relocation is ignored.
1718 ABSOLUTE = 0x0000,
1719 /// Not supported.
1720 DIR16 = 0x0001,
1721 /// Not supported.
1722 REL16 = 0x0002,
1723 /// The target's 32-bit VA.
1724 DIR32 = 0x0006,
1725 /// The target's 32-bit RVA.
1726 DIR32NB = 0x0007,
1727 /// Not supported.
1728 SEG12 = 0x0009,
1729 /// The 16-bit section index of the section that contains the target.
1730 /// This is used to support debugging information.
1731 SECTION = 0x000A,
1732 /// The 32-bit offset of the target from the beginning of its section.
1733 /// This is used to support debugging information and static thread local storage.
1734 SECREL = 0x000B,
1735 /// The CLR token.
1736 TOKEN = 0x000C,
1737 /// A 7-bit offset from the base of the section that contains the target.
1738 SECREL7 = 0x000D,
1739 /// The 32-bit relative displacement to the target.
1740 /// This supports the x86 relative branch and call instructions.
1741 REL32 = 0x0014,
1742 _,
1743 };
15711744
1572 /// The 14-bit offset to the relocation target, for instructions TBZ and TBNZ.
1573 branch14 = 16,
1745 /// Intel Itanium Processor Family (IPF)
1746 /// The following relocation type indicators are defined for the Intel Itanium processor family and compatible processors.
1747 /// Note that relocations on instructions use the bundle's offset and slot number for the relocation offset.
1748 pub const IA64 = enum(u16) {
1749 /// The relocation is ignored.
1750 ABSOLUTE = 0x0000,
1751 /// The instruction relocation can be followed by an ADDEND relocation whose value is added to the target address before it is inserted into the specified slot in the IMM14 bundle.
1752 /// The relocation target must be absolute or the image must be fixed.
1753 IMM14 = 0x0001,
1754 /// The instruction relocation can be followed by an ADDEND relocation whose value is added to the target address before it is inserted into the specified slot in the IMM22 bundle.
1755 /// The relocation target must be absolute or the image must be fixed.
1756 IMM22 = 0x0002,
1757 /// The slot number of this relocation must be one (1).
1758 /// The relocation can be followed by an ADDEND relocation whose value is added to the target address before it is stored in all three slots of the IMM64 bundle.
1759 IMM64 = 0x0003,
1760 /// The target's 32-bit VA.
1761 /// This is supported only for /LARGEADDRESSAWARE:NO images.
1762 DIR32 = 0x0004,
1763 /// The target's 64-bit VA.
1764 DIR64 = 0x0005,
1765 /// The instruction is fixed up with the 25-bit relative displacement to the 16-bit aligned target.
1766 /// The low 4 bits of the displacement are zero and are not stored.
1767 PCREL21B = 0x0006,
1768 /// The instruction is fixed up with the 25-bit relative displacement to the 16-bit aligned target.
1769 /// The low 4 bits of the displacement, which are zero, are not stored.
1770 PCREL21M = 0x0007,
1771 /// The LSBs of this relocation's offset must contain the slot number whereas the rest is the bundle address.
1772 /// The bundle is fixed up with the 25-bit relative displacement to the 16-bit aligned target.
1773 /// The low 4 bits of the displacement are zero and are not stored.
1774 PCREL21F = 0x0008,
1775 /// The instruction relocation can be followed by an ADDEND relocation whose value is added to the target address and then a 22-bit GP-relative offset that is calculated and applied to the GPREL22 bundle.
1776 GPREL22 = 0x0009,
1777 /// The instruction is fixed up with the 22-bit GP-relative offset to the target symbol's literal table entry.
1778 /// The linker creates this literal table entry based on this relocation and the ADDEND relocation that might follow.
1779 LTOFF22 = 0x000A,
1780 /// The 16-bit section index of the section contains the target.
1781 /// This is used to support debugging information.
1782 SECTION = 0x000B,
1783 /// The instruction is fixed up with the 22-bit offset of the target from the beginning of its section.
1784 /// This relocation can be followed immediately by an ADDEND relocation, whose Value field contains the 32-bit unsigned offset of the target from the beginning of the section.
1785 SECREL22 = 0x000C,
1786 /// The slot number for this relocation must be one (1).
1787 /// The instruction is fixed up with the 64-bit offset of the target from the beginning of its section.
1788 /// This relocation can be followed immediately by an ADDEND relocation whose Value field contains the 32-bit unsigned offset of the target from the beginning of the section.
1789 SECREL64I = 0x000D,
1790 /// The address of data to be fixed up with the 32-bit offset of the target from the beginning of its section.
1791 SECREL32 = 0x000E,
1792 /// The target's 32-bit RVA.
1793 DIR32NB = 0x0010,
1794 /// This is applied to a signed 14-bit immediate that contains the difference between two relocatable targets.
1795 /// This is a declarative field for the linker that indicates that the compiler has already emitted this value.
1796 SREL14 = 0x0011,
1797 /// This is applied to a signed 22-bit immediate that contains the difference between two relocatable targets.
1798 /// This is a declarative field for the linker that indicates that the compiler has already emitted this value.
1799 SREL22 = 0x0012,
1800 /// This is applied to a signed 32-bit immediate that contains the difference between two relocatable values.
1801 /// This is a declarative field for the linker that indicates that the compiler has already emitted this value.
1802 SREL32 = 0x0013,
1803 /// This is applied to an unsigned 32-bit immediate that contains the difference between two relocatable values.
1804 /// This is a declarative field for the linker that indicates that the compiler has already emitted this value.
1805 UREL32 = 0x0014,
1806 /// A 60-bit PC-relative fixup that always stays as a BRL instruction of an MLX bundle.
1807 PCREL60X = 0x0015,
1808 /// A 60-bit PC-relative fixup.
1809 /// If the target displacement fits in a signed 25-bit field, convert the entire bundle to an MBB bundle with NOP.B in slot 1 and a 25-bit BR instruction (with the 4 lowest bits all zero and dropped) in slot 2.
1810 PCREL60B = 0x0016,
1811 /// A 60-bit PC-relative fixup.
1812 /// If the target displacement fits in a signed 25-bit field, convert the entire bundle to an MFB bundle with NOP.F in slot 1 and a 25-bit (4 lowest bits all zero and dropped) BR instruction in slot 2.
1813 PCREL60F = 0x0017,
1814 /// A 60-bit PC-relative fixup.
1815 /// If the target displacement fits in a signed 25-bit field, convert the entire bundle to an MIB bundle with NOP.I in slot 1 and a 25-bit (4 lowest bits all zero and dropped) BR instruction in slot 2.
1816 PCREL60I = 0x0018,
1817 /// A 60-bit PC-relative fixup.
1818 /// If the target displacement fits in a signed 25-bit field, convert the entire bundle to an MMB bundle with NOP.M in slot 1 and a 25-bit (4 lowest bits all zero and dropped) BR instruction in slot 2.
1819 PCREL60M = 0x0019,
1820 /// A 64-bit GP-relative fixup.
1821 IMMGPREL64 = 0x001a,
1822 /// A CLR token.
1823 TOKEN = 0x001b,
1824 /// A 32-bit GP-relative fixup.
1825 GPREL32 = 0x001c,
1826 /// The relocation is valid only when it immediately follows one of the following relocations: IMM14, IMM22, IMM64, GPREL22, LTOFF22, LTOFF64, SECREL22, SECREL64I, or SECREL32.
1827 /// Its value contains the addend to apply to instructions within a bundle, not for data.
1828 ADDEND = 0x001F,
1829 _,
1830 };
15741831
1575 /// The 32-bit relative address from the byte following the relocation.
1576 rel32 = 17,
1832 /// MIPS Processors
1833 /// The following relocation type indicators are defined for MIPS processors.
1834 pub const MIPS = enum(u16) {
1835 /// The relocation is ignored.
1836 ABSOLUTE = 0x0000,
1837 /// The high 16 bits of the target's 32-bit VA.
1838 REFHALF = 0x0001,
1839 /// The target's 32-bit VA.
1840 REFWORD = 0x0002,
1841 /// The low 26 bits of the target's VA.
1842 /// This supports the MIPS J and JAL instructions.
1843 JMPADDR = 0x0003,
1844 /// The high 16 bits of the target's 32-bit VA.
1845 /// This is used for the first instruction in a two-instruction sequence that loads a full address.
1846 /// This relocation must be immediately followed by a PAIR relocation whose SymbolTableIndex contains a signed 16-bit displacement that is added to the upper 16 bits that are taken from the location that is being relocated.
1847 REFHI = 0x0004,
1848 /// The low 16 bits of the target's VA.
1849 REFLO = 0x0005,
1850 /// A 16-bit signed displacement of the target relative to the GP register.
1851 GPREL = 0x0006,
1852 /// The same as IMAGE_REL_MIPS_GPREL.
1853 LITERAL = 0x0007,
1854 /// The 16-bit section index of the section contains the target.
1855 /// This is used to support debugging information.
1856 SECTION = 0x000A,
1857 /// The 32-bit offset of the target from the beginning of its section.
1858 /// This is used to support debugging information and static thread local storage.
1859 SECREL = 0x000B,
1860 /// The low 16 bits of the 32-bit offset of the target from the beginning of its section.
1861 SECRELLO = 0x000C,
1862 /// The high 16 bits of the 32-bit offset of the target from the beginning of its section.
1863 /// An IMAGE_REL_MIPS_PAIR relocation must immediately follow this one.
1864 /// The SymbolTableIndex of the PAIR relocation contains a signed 16-bit displacement that is added to the upper 16 bits that are taken from the location that is being relocated.
1865 SECRELHI = 0x000D,
1866 /// The low 26 bits of the target's VA.
1867 /// This supports the MIPS16 JAL instruction.
1868 JMPADDR16 = 0x0010,
1869 /// The target's 32-bit RVA.
1870 REFWORDNB = 0x0022,
1871 /// The relocation is valid only when it immediately follows a REFHI or SECRELHI relocation.
1872 /// Its SymbolTableIndex contains a displacement and not an index into the symbol table.
1873 PAIR = 0x0025,
1874 _,
1875 };
15771876
1578 _,
1877 /// Mitsubishi M32R
1878 /// The following relocation type indicators are defined for the Mitsubishi M32R processors.
1879 pub const M32R = enum(u16) {
1880 /// The relocation is ignored.
1881 ABSOLUTE = 0x0000,
1882 /// The target's 32-bit VA.
1883 ADDR32 = 0x0001,
1884 /// The target's 32-bit RVA.
1885 ADDR32NB = 0x0002,
1886 /// The target's 24-bit VA.
1887 ADDR24 = 0x0003,
1888 /// The target's 16-bit offset from the GP register.
1889 GPREL16 = 0x0004,
1890 /// The target's 24-bit offset from the program counter (PC), shifted left by 2 bits and sign-extended
1891 PCREL24 = 0x0005,
1892 /// The target's 16-bit offset from the PC, shifted left by 2 bits and sign-extended
1893 PCREL16 = 0x0006,
1894 /// The target's 8-bit offset from the PC, shifted left by 2 bits and sign-extended
1895 PCREL8 = 0x0007,
1896 /// The 16 MSBs of the target VA.
1897 REFHALF = 0x0008,
1898 /// The 16 MSBs of the target VA, adjusted for LSB sign extension.
1899 /// This is used for the first instruction in a two-instruction sequence that loads a full 32-bit address.
1900 /// This relocation must be immediately followed by a PAIR relocation whose SymbolTableIndex contains a signed 16-bit displacement that is added to the upper 16 bits that are taken from the location that is being relocated.
1901 REFHI = 0x0009,
1902 /// The 16 LSBs of the target VA.
1903 REFLO = 0x000A,
1904 /// The relocation must follow the REFHI relocation.
1905 /// Its SymbolTableIndex contains a displacement and not an index into the symbol table.
1906 PAIR = 0x000B,
1907 /// The 16-bit section index of the section that contains the target.
1908 /// This is used to support debugging information.
1909 SECTION = 0x000C,
1910 /// The 32-bit offset of the target from the beginning of its section.
1911 /// This is used to support debugging information and static thread local storage.
1912 SECREL = 0x000D,
1913 /// The CLR token.
1914 TOKEN = 0x000E,
1915 _,
1916 };
1917 };
15791918};
lib/std/heap.zig+21-12
......@@ -78,13 +78,15 @@ pub fn defaultQueryPageSize() usize {
7878 };
7979 var size = global.cached_result.load(.unordered);
8080 if (size > 0) return size;
81 size = switch (builtin.os.tag) {
82 .linux => if (builtin.link_libc) @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE))) else std.os.linux.getauxval(std.elf.AT_PAGESZ),
83 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => blk: {
81 size = size: switch (builtin.os.tag) {
82 .linux => if (builtin.link_libc)
83 @max(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE)), 0)
84 else
85 std.os.linux.getauxval(std.elf.AT_PAGESZ),
86 .driverkit, .ios, .macos, .tvos, .visionos, .watchos => {
8487 const task_port = std.c.mach_task_self();
8588 // mach_task_self may fail "if there are any resource failures or other errors".
86 if (task_port == std.c.TASK.NULL)
87 break :blk 0;
89 if (task_port == std.c.TASK.NULL) break :size 0;
8890 var info_count = std.c.TASK.VM.INFO_COUNT;
8991 var vm_info: std.c.task_vm_info_data_t = undefined;
9092 vm_info.page_size = 0;
......@@ -94,21 +96,28 @@ pub fn defaultQueryPageSize() usize {
9496 @as(std.c.task_info_t, @ptrCast(&vm_info)),
9597 &info_count,
9698 );
97 assert(vm_info.page_size != 0);
98 break :blk @intCast(vm_info.page_size);
99 break :size @intCast(vm_info.page_size);
99100 },
100 .windows => blk: {
101 var info: std.os.windows.SYSTEM_INFO = undefined;
102 std.os.windows.kernel32.GetSystemInfo(&info);
103 break :blk info.dwPageSize;
101 .windows => {
102 var sbi: windows.SYSTEM_BASIC_INFORMATION = undefined;
103 switch (windows.ntdll.NtQuerySystemInformation(
104 .SystemBasicInformation,
105 &sbi,
106 @sizeOf(windows.SYSTEM_BASIC_INFORMATION),
107 null,
108 )) {
109 .SUCCESS => break :size sbi.PageSize,
110 else => break :size 0,
111 }
104112 },
105113 else => if (builtin.link_libc)
106 @intCast(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE)))
114 @max(std.c.sysconf(@intFromEnum(std.c._SC.PAGESIZE)), 0)
107115 else if (builtin.os.tag == .freestanding or builtin.os.tag == .other)
108116 @compileError("unsupported target: freestanding/other")
109117 else
110118 @compileError("pageSize on " ++ @tagName(builtin.cpu.arch) ++ "-" ++ @tagName(builtin.os.tag) ++ " is not supported without linking libc, using the default implementation"),
111119 };
120 if (size == 0) size = page_size_max;
112121
113122 assert(size >= page_size_min);
114123 assert(size <= page_size_max);
src/Compilation.zig+18-10
......@@ -256,8 +256,8 @@ test_filters: []const []const u8,
256256
257257link_task_wait_group: WaitGroup = .{},
258258link_prog_node: std.Progress.Node = .none,
259link_uav_prog_node: std.Progress.Node = .none,
260link_lazy_prog_node: std.Progress.Node = .none,
259link_const_prog_node: std.Progress.Node = .none,
260link_synth_prog_node: std.Progress.Node = .none,
261261
262262llvm_opt_bisect_limit: c_int,
263263
......@@ -1982,13 +1982,13 @@ pub fn create(gpa: Allocator, arena: Allocator, diag: *CreateDiagnostic, options
19821982 };
19831983 if (have_zcu and (!need_llvm or use_llvm)) {
19841984 if (output_mode == .Obj) break :s .zcu;
1985 if (options.config.use_new_linker) break :s .zcu;
19861985 switch (target_util.zigBackend(target, use_llvm)) {
19871986 else => {},
19881987 .stage2_aarch64, .stage2_x86_64 => if (target.ofmt == .coff) {
19891988 break :s if (is_exe_or_dyn_lib) .dyn_lib else .zcu;
19901989 },
19911990 }
1991 if (options.config.use_new_linker) break :s .zcu;
19921992 }
19931993 if (need_llvm and !build_options.have_llvm) break :s .none; // impossible to build without llvm
19941994 if (is_exe_or_dyn_lib) break :s .lib;
......@@ -3081,22 +3081,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
30813081 comp.link_prog_node = main_progress_node.start("Linking", 0);
30823082 if (lf.cast(.elf2)) |elf| {
30833083 comp.link_prog_node.increaseEstimatedTotalItems(3);
3084 comp.link_uav_prog_node = comp.link_prog_node.start("Constants", 0);
3085 comp.link_lazy_prog_node = comp.link_prog_node.start("Synthetics", 0);
3084 comp.link_const_prog_node = comp.link_prog_node.start("Constants", 0);
3085 comp.link_synth_prog_node = comp.link_prog_node.start("Synthetics", 0);
30863086 elf.mf.update_prog_node = comp.link_prog_node.start("Relocations", elf.mf.updates.items.len);
3087 } else if (lf.cast(.coff2)) |coff| {
3088 comp.link_prog_node.increaseEstimatedTotalItems(3);
3089 comp.link_const_prog_node = comp.link_prog_node.start("Constants", 0);
3090 comp.link_synth_prog_node = comp.link_prog_node.start("Synthetics", 0);
3091 coff.mf.update_prog_node = comp.link_prog_node.start("Relocations", coff.mf.updates.items.len);
30873092 }
30883093 }
30893094 defer {
30903095 comp.link_prog_node.end();
30913096 comp.link_prog_node = .none;
3092 comp.link_uav_prog_node.end();
3093 comp.link_uav_prog_node = .none;
3094 comp.link_lazy_prog_node.end();
3095 comp.link_lazy_prog_node = .none;
3097 comp.link_const_prog_node.end();
3098 comp.link_const_prog_node = .none;
3099 comp.link_synth_prog_node.end();
3100 comp.link_synth_prog_node = .none;
30963101 if (comp.bin_file) |lf| {
30973102 if (lf.cast(.elf2)) |elf| {
30983103 elf.mf.update_prog_node.end();
30993104 elf.mf.update_prog_node = .none;
3105 } else if (lf.cast(.coff2)) |coff| {
3106 coff.mf.update_prog_node.end();
3107 coff.mf.update_prog_node = .none;
31003108 }
31013109 }
31023110 }
......@@ -3218,7 +3226,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
32183226 .root_dir = comp.dirs.local_cache,
32193227 .sub_path = try fs.path.join(arena, &.{ o_sub_path, comp.emit_bin.? }),
32203228 };
3221 const result: link.File.OpenError!void = switch (need_writable_dance) {
3229 const result: (link.File.OpenError || error{HotSwapUnavailableOnHostOperatingSystem})!void = switch (need_writable_dance) {
32223230 .no => {},
32233231 .lf_only => lf.makeWritable(),
32243232 .lf_and_debug => res: {
src/InternPool.zig+3-3
......@@ -11919,10 +11919,10 @@ pub fn getString(ip: *InternPool, key: []const u8) OptionalNullTerminatedString
1191911919 var map_index = hash;
1192011920 while (true) : (map_index += 1) {
1192111921 map_index &= map_mask;
11922 const entry = map.at(map_index);
11923 const index = entry.acquire().unwrap() orelse return null;
11922 const entry = &map.entries[map_index];
11923 const index = entry.value.unwrap() orelse return .none;
1192411924 if (entry.hash != hash) continue;
11925 if (index.eqlSlice(key, ip)) return index;
11925 if (index.eqlSlice(key, ip)) return index.toOptional();
1192611926 }
1192711927}
1192811928
src/codegen.zig+2-15
......@@ -978,21 +978,8 @@ pub fn genNavRef(
978978 },
979979 .link_once => unreachable,
980980 }
981 } else if (lf.cast(.coff)) |coff_file| {
982 // TODO audit this
983 switch (linkage) {
984 .internal => {
985 const atom_index = try coff_file.getOrCreateAtomForNav(nav_index);
986 const sym_index = coff_file.getAtom(atom_index).getSymbolIndex().?;
987 return .{ .sym_index = sym_index };
988 },
989 .strong, .weak => {
990 const global_index = try coff_file.getGlobalSymbol(nav.name.toSlice(ip), lib_name.toSlice(ip));
991 try coff_file.need_got_table.put(zcu.gpa, global_index, {}); // needs GOT
992 return .{ .sym_index = global_index };
993 },
994 .link_once => unreachable,
995 }
981 } else if (lf.cast(.coff2)) |coff| {
982 return .{ .sym_index = @intFromEnum(try coff.navSymbol(zcu, nav_index)) };
996983 } else {
997984 const msg = try ErrorMsg.create(zcu.gpa, src_loc, "TODO genNavRef for target {}", .{target});
998985 return .{ .fail = msg };
src/codegen/aarch64/Mir.zig-7
......@@ -135,11 +135,6 @@ pub fn emit(
135135 else if (lf.cast(.macho)) |mf|
136136 mf.getZigObject().?.getOrCreateMetadataForLazySymbol(mf, pt, lazy_reloc.symbol) catch |err|
137137 return zcu.codegenFail(func.owner_nav, "{s} creating lazy symbol", .{@errorName(err)})
138 else if (lf.cast(.coff)) |cf|
139 if (cf.getOrCreateAtomForLazySymbol(pt, lazy_reloc.symbol)) |atom|
140 cf.getAtom(atom).getSymbolIndex().?
141 else |err|
142 return zcu.codegenFail(func.owner_nav, "{s} creating lazy symbol", .{@errorName(err)})
143138 else
144139 return zcu.codegenFail(func.owner_nav, "external symbols unimplemented for {s}", .{@tagName(lf.tag)}),
145140 mir.body[lazy_reloc.reloc.label],
......@@ -154,8 +149,6 @@ pub fn emit(
154149 try ef.getGlobalSymbol(std.mem.span(global_reloc.name), null)
155150 else if (lf.cast(.macho)) |mf|
156151 try mf.getGlobalSymbol(std.mem.span(global_reloc.name), null)
157 else if (lf.cast(.coff)) |cf|
158 try cf.getGlobalSymbol(std.mem.span(global_reloc.name), "compiler_rt")
159152 else
160153 return zcu.codegenFail(func.owner_nav, "external symbols unimplemented for {s}", .{@tagName(lf.tag)}),
161154 mir.body[global_reloc.reloc.label],
src/codegen/llvm.zig+3-3
......@@ -12103,7 +12103,7 @@ fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: *const st
1210312103 return switch (fn_info.cc) {
1210412104 .auto => returnTypeByRef(zcu, target, return_type),
1210512105 .x86_64_sysv => firstParamSRetSystemV(return_type, zcu, target),
12106 .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target) == .memory,
12106 .x86_64_win => x86_64_abi.classifyWindows(return_type, zcu, target, .ret) == .memory,
1210712107 .x86_sysv, .x86_win => isByRef(return_type, zcu),
1210812108 .x86_stdcall => !isScalar(zcu, return_type),
1210912109 .wasm_mvp => wasm_c_abi.classifyType(return_type, zcu) == .indirect,
......@@ -12205,7 +12205,7 @@ fn lowerFnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType)
1220512205fn lowerWin64FnRetTy(o: *Object, pt: Zcu.PerThread, fn_info: InternPool.Key.FuncType) Allocator.Error!Builder.Type {
1220612206 const zcu = pt.zcu;
1220712207 const return_type = Type.fromInterned(fn_info.return_type);
12208 switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget())) {
12208 switch (x86_64_abi.classifyWindows(return_type, zcu, zcu.getTarget(), .ret)) {
1220912209 .integer => {
1221012210 if (isScalar(zcu, return_type)) {
1221112211 return o.lowerType(pt, return_type);
......@@ -12476,7 +12476,7 @@ const ParamTypeIterator = struct {
1247612476
1247712477 fn nextWin64(it: *ParamTypeIterator, ty: Type) ?Lowering {
1247812478 const zcu = it.pt.zcu;
12479 switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget())) {
12479 switch (x86_64_abi.classifyWindows(ty, zcu, zcu.getTarget(), .arg)) {
1248012480 .integer => {
1248112481 if (isScalar(zcu, ty)) {
1248212482 it.zig_index += 1;
src/codegen/x86_64/CodeGen.zig+5740-9955
......@@ -2292,7 +2292,7 @@ fn genBodyBlock(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
22922292}
22932293
22942294fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2295 @setEvalBranchQuota(29_600);
2295 @setEvalBranchQuota(31_000);
22962296 const pt = cg.pt;
22972297 const zcu = pt.zcu;
22982298 const ip = &zcu.intern_pool;
......@@ -4168,6 +4168,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
41684168 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
41694169 } },
41704170 }, .{
4171 .required_cc_abi = .sysv64,
41714172 .required_features = .{ .sse, null, null, null },
41724173 .src_constraints = .{
41734174 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -4201,6 +4202,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42014202 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
42024203 } },
42034204 }, .{
4205 .required_cc_abi = .win64,
4206 .required_features = .{ .sse, null, null, null },
4207 .src_constraints = .{
4208 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
4209 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
4210 .any,
4211 },
4212 .patterns = &.{
4213 .{ .src = .{ .to_mem, .to_mem, .none } },
4214 },
4215 .call_frame = .{ .alignment = .@"16" },
4216 .extra_temps = .{
4217 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4218 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4219 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4220 .unused,
4221 .unused,
4222 .unused,
4223 .unused,
4224 .unused,
4225 .unused,
4226 .unused,
4227 .unused,
4228 },
4229 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
4230 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4231 .each = .{ .once = &.{
4232 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
4233 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
4234 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
4235 } },
4236 }, .{
4237 .required_cc_abi = .sysv64,
42044238 .required_features = .{ .avx, null, null, null },
42054239 .src_constraints = .{
42064240 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -4212,7 +4246,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42124246 },
42134247 .call_frame = .{ .alignment = .@"16" },
42144248 .extra_temps = .{
4215 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4249 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42164250 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42174251 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42184252 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
......@@ -4227,15 +4261,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42274261 .dst_temps = .{ .mem, .unused },
42284262 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42294263 .each = .{ .once = &.{
4230 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4231 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
4232 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
4264 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4265 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
4266 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
42334267 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4234 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
4235 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
4236 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4268 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
4269 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4270 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
42374271 } },
42384272 }, .{
4273 .required_cc_abi = .sysv64,
42394274 .required_features = .{ .sse2, null, null, null },
42404275 .src_constraints = .{
42414276 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -4247,7 +4282,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42474282 },
42484283 .call_frame = .{ .alignment = .@"16" },
42494284 .extra_temps = .{
4250 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4285 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42514286 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42524287 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42534288 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
......@@ -4262,15 +4297,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42624297 .dst_temps = .{ .mem, .unused },
42634298 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42644299 .each = .{ .once = &.{
4265 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4266 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
4267 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
4300 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4301 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
4302 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
42684303 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4269 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
4270 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
4271 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4304 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
4305 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4306 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
42724307 } },
42734308 }, .{
4309 .required_cc_abi = .sysv64,
42744310 .required_features = .{ .sse, null, null, null },
42754311 .src_constraints = .{
42764312 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -4282,7 +4318,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42824318 },
42834319 .call_frame = .{ .alignment = .@"16" },
42844320 .extra_temps = .{
4285 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
4321 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42864322 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42874323 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42884324 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
......@@ -4297,13 +4333,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
42974333 .dst_temps = .{ .mem, .unused },
42984334 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42994335 .each = .{ .once = &.{
4300 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4301 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
4302 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
4336 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4337 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
4338 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
43034339 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4304 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
4305 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
4306 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4340 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
4341 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4342 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4343 } },
4344 }, .{
4345 .required_cc_abi = .win64,
4346 .required_features = .{ .avx, null, null, null },
4347 .src_constraints = .{
4348 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4349 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4350 .any,
4351 },
4352 .patterns = &.{
4353 .{ .src = .{ .to_mem, .to_mem, .none } },
4354 },
4355 .call_frame = .{ .alignment = .@"16" },
4356 .extra_temps = .{
4357 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4358 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4359 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4360 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4361 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4362 .unused,
4363 .unused,
4364 .unused,
4365 .unused,
4366 .unused,
4367 .unused,
4368 },
4369 .dst_temps = .{ .mem, .unused },
4370 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4371 .each = .{ .once = &.{
4372 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4373 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
4374 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
4375 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4376 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
4377 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4378 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4379 } },
4380 }, .{
4381 .required_cc_abi = .win64,
4382 .required_features = .{ .sse2, null, null, null },
4383 .src_constraints = .{
4384 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4385 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4386 .any,
4387 },
4388 .patterns = &.{
4389 .{ .src = .{ .to_mem, .to_mem, .none } },
4390 },
4391 .call_frame = .{ .alignment = .@"16" },
4392 .extra_temps = .{
4393 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4394 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4395 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4396 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4397 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4398 .unused,
4399 .unused,
4400 .unused,
4401 .unused,
4402 .unused,
4403 .unused,
4404 },
4405 .dst_temps = .{ .mem, .unused },
4406 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4407 .each = .{ .once = &.{
4408 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4409 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
4410 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
4411 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4412 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
4413 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4414 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4415 } },
4416 }, .{
4417 .required_cc_abi = .win64,
4418 .required_features = .{ .sse, null, null, null },
4419 .src_constraints = .{
4420 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4421 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
4422 .any,
4423 },
4424 .patterns = &.{
4425 .{ .src = .{ .to_mem, .to_mem, .none } },
4426 },
4427 .call_frame = .{ .alignment = .@"16" },
4428 .extra_temps = .{
4429 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4430 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4431 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4432 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4433 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4434 .unused,
4435 .unused,
4436 .unused,
4437 .unused,
4438 .unused,
4439 .unused,
4440 },
4441 .dst_temps = .{ .mem, .unused },
4442 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4443 .each = .{ .once = &.{
4444 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
4445 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
4446 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
4447 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
4448 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
4449 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
4450 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
43074451 } },
43084452 } }) catch |err| switch (err) {
43094453 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -14775,6 +14919,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1477514919 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
1477614920 } },
1477714921 }, .{
14922 .required_cc_abi = .sysv64,
1477814923 .required_features = .{ .sse, null, null, null },
1477914924 .src_constraints = .{
1478014925 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -14808,6 +14953,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1480814953 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
1480914954 } },
1481014955 }, .{
14956 .required_cc_abi = .win64,
14957 .required_features = .{ .sse, null, null, null },
14958 .src_constraints = .{
14959 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
14960 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
14961 .any,
14962 },
14963 .patterns = &.{
14964 .{ .src = .{ .to_mem, .to_mem, .none } },
14965 },
14966 .call_frame = .{ .alignment = .@"16" },
14967 .extra_temps = .{
14968 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
14969 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
14970 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
14971 .unused,
14972 .unused,
14973 .unused,
14974 .unused,
14975 .unused,
14976 .unused,
14977 .unused,
14978 .unused,
14979 },
14980 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
14981 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
14982 .each = .{ .once = &.{
14983 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
14984 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
14985 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
14986 } },
14987 }, .{
14988 .required_cc_abi = .sysv64,
1481114989 .required_features = .{ .avx, null, null, null },
1481214990 .src_constraints = .{
1481314991 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -14819,7 +14997,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1481914997 },
1482014998 .call_frame = .{ .alignment = .@"16" },
1482114999 .extra_temps = .{
14822 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
15000 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
1482315001 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
1482415002 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
1482515003 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
......@@ -14834,15 +15012,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1483415012 .dst_temps = .{ .mem, .unused },
1483515013 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
1483615014 .each = .{ .once = &.{
14837 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
14838 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
14839 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
15015 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15016 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
15017 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
1484015018 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
14841 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
14842 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
14843 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
15019 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
15020 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15021 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
1484415022 } },
1484515023 }, .{
15024 .required_cc_abi = .sysv64,
1484615025 .required_features = .{ .sse2, null, null, null },
1484715026 .src_constraints = .{
1484815027 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -14854,7 +15033,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1485415033 },
1485515034 .call_frame = .{ .alignment = .@"16" },
1485615035 .extra_temps = .{
14857 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
15036 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
1485815037 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
1485915038 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
1486015039 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
......@@ -14869,15 +15048,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1486915048 .dst_temps = .{ .mem, .unused },
1487015049 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
1487115050 .each = .{ .once = &.{
14872 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
14873 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
14874 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
15051 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15052 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
15053 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
1487515054 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
14876 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
14877 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
14878 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
15055 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
15056 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15057 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
1487915058 } },
1488015059 }, .{
15060 .required_cc_abi = .sysv64,
1488115061 .required_features = .{ .sse, null, null, null },
1488215062 .src_constraints = .{
1488315063 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -14889,7 +15069,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1488915069 },
1489015070 .call_frame = .{ .alignment = .@"16" },
1489115071 .extra_temps = .{
14892 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
15072 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
1489315073 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
1489415074 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
1489515075 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
......@@ -14904,13 +15084,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
1490415084 .dst_temps = .{ .mem, .unused },
1490515085 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
1490615086 .each = .{ .once = &.{
14907 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
14908 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
14909 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
15087 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15088 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
15089 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
1491015090 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
14911 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
14912 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
14913 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
15091 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
15092 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15093 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
15094 } },
15095 }, .{
15096 .required_cc_abi = .win64,
15097 .required_features = .{ .avx, null, null, null },
15098 .src_constraints = .{
15099 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15100 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15101 .any,
15102 },
15103 .patterns = &.{
15104 .{ .src = .{ .to_mem, .to_mem, .none } },
15105 },
15106 .call_frame = .{ .alignment = .@"16" },
15107 .extra_temps = .{
15108 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
15109 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15110 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
15111 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
15112 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15113 .unused,
15114 .unused,
15115 .unused,
15116 .unused,
15117 .unused,
15118 .unused,
15119 },
15120 .dst_temps = .{ .mem, .unused },
15121 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
15122 .each = .{ .once = &.{
15123 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15124 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
15125 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
15126 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
15127 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
15128 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15129 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
15130 } },
15131 }, .{
15132 .required_cc_abi = .win64,
15133 .required_features = .{ .sse2, null, null, null },
15134 .src_constraints = .{
15135 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15136 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15137 .any,
15138 },
15139 .patterns = &.{
15140 .{ .src = .{ .to_mem, .to_mem, .none } },
15141 },
15142 .call_frame = .{ .alignment = .@"16" },
15143 .extra_temps = .{
15144 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
15145 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15146 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
15147 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
15148 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15149 .unused,
15150 .unused,
15151 .unused,
15152 .unused,
15153 .unused,
15154 .unused,
15155 },
15156 .dst_temps = .{ .mem, .unused },
15157 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
15158 .each = .{ .once = &.{
15159 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15160 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
15161 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
15162 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
15163 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
15164 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15165 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
15166 } },
15167 }, .{
15168 .required_cc_abi = .win64,
15169 .required_features = .{ .sse, null, null, null },
15170 .src_constraints = .{
15171 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15172 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
15173 .any,
15174 },
15175 .patterns = &.{
15176 .{ .src = .{ .to_mem, .to_mem, .none } },
15177 },
15178 .call_frame = .{ .alignment = .@"16" },
15179 .extra_temps = .{
15180 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
15181 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15182 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
15183 .{ .type = .usize, .kind = .{ .extern_func = "__subtf3" } },
15184 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
15185 .unused,
15186 .unused,
15187 .unused,
15188 .unused,
15189 .unused,
15190 .unused,
15191 },
15192 .dst_temps = .{ .mem, .unused },
15193 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
15194 .each = .{ .once = &.{
15195 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
15196 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
15197 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
15198 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
15199 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
15200 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
15201 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
1491415202 } },
1491515203 } }) catch |err| switch (err) {
1491615204 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -24415,6 +24703,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2441524703 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
2441624704 } },
2441724705 }, .{
24706 .required_cc_abi = .sysv64,
2441824707 .required_features = .{ .sse, null, null, null },
2441924708 .src_constraints = .{
2442024709 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -24448,6 +24737,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2444824737 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
2444924738 } },
2445024739 }, .{
24740 .required_cc_abi = .win64,
24741 .required_features = .{ .sse, null, null, null },
24742 .src_constraints = .{
24743 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
24744 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
24745 .any,
24746 },
24747 .patterns = &.{
24748 .{ .src = .{ .to_mem, .to_mem, .none } },
24749 },
24750 .call_frame = .{ .alignment = .@"16" },
24751 .extra_temps = .{
24752 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24753 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
24754 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
24755 .unused,
24756 .unused,
24757 .unused,
24758 .unused,
24759 .unused,
24760 .unused,
24761 .unused,
24762 .unused,
24763 },
24764 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
24765 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
24766 .each = .{ .once = &.{
24767 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
24768 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
24769 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
24770 } },
24771 }, .{
24772 .required_cc_abi = .sysv64,
2445124773 .required_features = .{ .avx, null, null, null },
2445224774 .src_constraints = .{
2445324775 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -24459,7 +24781,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2445924781 },
2446024782 .call_frame = .{ .alignment = .@"16" },
2446124783 .extra_temps = .{
24462 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24784 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2446324785 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
2446424786 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
2446524787 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
......@@ -24474,15 +24796,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2447424796 .dst_temps = .{ .mem, .unused },
2447524797 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
2447624798 .each = .{ .once = &.{
24477 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
24478 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
24479 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
24799 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24800 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
24801 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
2448024802 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24481 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
24482 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
24483 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24803 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
24804 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24805 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
2448424806 } },
2448524807 }, .{
24808 .required_cc_abi = .sysv64,
2448624809 .required_features = .{ .sse2, null, null, null },
2448724810 .src_constraints = .{
2448824811 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -24494,7 +24817,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2449424817 },
2449524818 .call_frame = .{ .alignment = .@"16" },
2449624819 .extra_temps = .{
24497 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24820 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2449824821 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
2449924822 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
2450024823 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
......@@ -24509,15 +24832,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2450924832 .dst_temps = .{ .mem, .unused },
2451024833 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
2451124834 .each = .{ .once = &.{
24512 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
24513 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
24514 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
24835 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24836 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
24837 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
2451524838 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24516 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
24517 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
24518 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24839 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
24840 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24841 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
2451924842 } },
2452024843 }, .{
24844 .required_cc_abi = .sysv64,
2452124845 .required_features = .{ .sse, null, null, null },
2452224846 .src_constraints = .{
2452324847 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -24529,7 +24853,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2452924853 },
2453024854 .call_frame = .{ .alignment = .@"16" },
2453124855 .extra_temps = .{
24532 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
24856 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
2453324857 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
2453424858 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
2453524859 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
......@@ -24544,13 +24868,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2454424868 .dst_temps = .{ .mem, .unused },
2454524869 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
2454624870 .each = .{ .once = &.{
24547 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
24548 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
24549 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
24871 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24872 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
24873 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
2455024874 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24551 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
24552 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
24553 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
24875 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
24876 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24877 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
24878 } },
24879 }, .{
24880 .required_cc_abi = .win64,
24881 .required_features = .{ .avx, null, null, null },
24882 .src_constraints = .{
24883 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24884 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24885 .any,
24886 },
24887 .patterns = &.{
24888 .{ .src = .{ .to_mem, .to_mem, .none } },
24889 },
24890 .call_frame = .{ .alignment = .@"16" },
24891 .extra_temps = .{
24892 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
24893 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24894 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
24895 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
24896 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24897 .unused,
24898 .unused,
24899 .unused,
24900 .unused,
24901 .unused,
24902 .unused,
24903 },
24904 .dst_temps = .{ .mem, .unused },
24905 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
24906 .each = .{ .once = &.{
24907 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24908 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
24909 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
24910 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24911 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
24912 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24913 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
24914 } },
24915 }, .{
24916 .required_cc_abi = .win64,
24917 .required_features = .{ .sse2, null, null, null },
24918 .src_constraints = .{
24919 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24920 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24921 .any,
24922 },
24923 .patterns = &.{
24924 .{ .src = .{ .to_mem, .to_mem, .none } },
24925 },
24926 .call_frame = .{ .alignment = .@"16" },
24927 .extra_temps = .{
24928 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
24929 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24930 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
24931 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
24932 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24933 .unused,
24934 .unused,
24935 .unused,
24936 .unused,
24937 .unused,
24938 .unused,
24939 },
24940 .dst_temps = .{ .mem, .unused },
24941 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
24942 .each = .{ .once = &.{
24943 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24944 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
24945 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
24946 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24947 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
24948 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24949 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
24950 } },
24951 }, .{
24952 .required_cc_abi = .win64,
24953 .required_features = .{ .sse, null, null, null },
24954 .src_constraints = .{
24955 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24956 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
24957 .any,
24958 },
24959 .patterns = &.{
24960 .{ .src = .{ .to_mem, .to_mem, .none } },
24961 },
24962 .call_frame = .{ .alignment = .@"16" },
24963 .extra_temps = .{
24964 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
24965 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24966 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
24967 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
24968 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
24969 .unused,
24970 .unused,
24971 .unused,
24972 .unused,
24973 .unused,
24974 .unused,
24975 },
24976 .dst_temps = .{ .mem, .unused },
24977 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
24978 .each = .{ .once = &.{
24979 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
24980 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
24981 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
24982 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
24983 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
24984 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
24985 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
2455424986 } },
2455524987 } }) catch |err| switch (err) {
2455624988 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -26350,18 +26782,53 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2635026782 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
2635126783 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
2635226784 } },
26353 }, .{
26354 .required_features = .{ .f16c, null, null, null },
26355 .src_constraints = .{
26356 .{ .scalar_float = .{ .of = .word, .is = .word } },
26357 .{ .scalar_float = .{ .of = .word, .is = .word } },
26358 .any,
26785 } }) catch |err| switch (err) {
26786 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
26787 @tagName(air_tag),
26788 ty.fmt(pt),
26789 ops[0].tracking(cg),
26790 ops[1].tracking(cg),
26791 }),
26792 else => |e| return e,
26793 };
26794 res[0].wrapInt(cg) catch |err| switch (err) {
26795 error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{
26796 @tagName(air_tag),
26797 cg.typeOf(bin_op.lhs).fmt(pt),
26798 res[0].tracking(cg),
26799 }),
26800 else => |e| return e,
26801 };
26802 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
26803 },
26804 .mul_sat => |air_tag| {
26805 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
26806 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
26807 var res: [1]Temp = undefined;
26808 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{
26809 .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any },
26810 .patterns = &.{
26811 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
26812 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
26813 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
2635926814 },
26815 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26816 .clobbers = .{ .eflags = true },
26817 .each = .{ .once = &.{
26818 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
26819 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
26820 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
26821 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ },
26822 } },
26823 }, .{
26824 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
2636026825 .patterns = &.{
26361 .{ .src = .{ .to_sse, .to_sse, .none } },
26826 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
26827 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
26828 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
2636226829 },
2636326830 .extra_temps = .{
26364 .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
26831 .{ .type = .i8, .kind = .{ .rc = .gphi } },
2636526832 .unused,
2636626833 .unused,
2636726834 .unused,
......@@ -26373,30 +26840,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2637326840 .unused,
2637426841 .unused,
2637526842 },
26376 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26843 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26844 .clobbers = .{ .eflags = true },
2637726845 .each = .{ .once = &.{
26378 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
26379 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
26380 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ },
26381 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
26846 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
26847 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
26848 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
26849 .{ ._, ._r, .sa, .tmp0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
26850 .{ ._, ._, .cmp, .tmp0b, .dst0h, ._, ._ },
26851 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
26852 .{ .@"1:", ._r, .sa, .dst0w, .ui(15), ._, ._ },
26853 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ },
2638226854 } },
2638326855 }, .{
26384 .required_features = .{ .sse, null, null, null },
26385 .src_constraints = .{
26386 .{ .scalar_float = .{ .of = .word, .is = .word } },
26387 .{ .scalar_float = .{ .of = .word, .is = .word } },
26388 .any,
26389 },
26856 .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any },
2639026857 .patterns = &.{
26391 .{ .src = .{
26392 .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } },
26393 .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } },
26394 .none,
26395 } },
26858 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
26859 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
26860 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
2639626861 },
26397 .call_frame = .{ .alignment = .@"16" },
2639826862 .extra_temps = .{
26399 .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } },
26863 .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2640026864 .unused,
2640126865 .unused,
2640226866 .unused,
......@@ -26409,25 +26873,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2640926873 .unused,
2641026874 },
2641126875 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26412 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
26876 .clobbers = .{ .eflags = true },
2641326877 .each = .{ .once = &.{
26414 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
26878 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
26879 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
26880 .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ },
2641526881 } },
2641626882 }, .{
26417 .required_features = .{ .f16c, null, null, null },
26418 .src_constraints = .{
26419 .{ .scalar_float = .{ .of = .qword, .is = .word } },
26420 .{ .scalar_float = .{ .of = .qword, .is = .word } },
26421 .any,
26422 },
26883 .required_features = .{ .cmov, null, null, null },
26884 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
2642326885 .patterns = &.{
26424 .{ .src = .{ .mem, .mem, .none } },
26425 .{ .src = .{ .to_sse, .mem, .none } },
26426 .{ .src = .{ .mem, .to_sse, .none } },
26427 .{ .src = .{ .to_sse, .to_sse, .none } },
26886 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
26887 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
26888 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
2642826889 },
2642926890 .extra_temps = .{
26430 .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
26891 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2643126892 .unused,
2643226893 .unused,
2643326894 .unused,
......@@ -26439,28 +26900,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2643926900 .unused,
2644026901 .unused,
2644126902 },
26442 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26903 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26904 .clobbers = .{ .eflags = true },
2644326905 .each = .{ .once = &.{
26444 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
26445 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
26446 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
26447 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
26906 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
26907 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
26908 .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ },
26909 .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ },
2644826910 } },
2644926911 }, .{
26450 .required_features = .{ .f16c, null, null, null },
26451 .src_constraints = .{
26452 .{ .scalar_float = .{ .of = .xword, .is = .word } },
26453 .{ .scalar_float = .{ .of = .xword, .is = .word } },
26454 .any,
26455 },
26912 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
2645626913 .patterns = &.{
26457 .{ .src = .{ .mem, .mem, .none } },
26458 .{ .src = .{ .to_sse, .mem, .none } },
26459 .{ .src = .{ .mem, .to_sse, .none } },
26460 .{ .src = .{ .to_sse, .to_sse, .none } },
26914 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
26915 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
26916 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
2646126917 },
2646226918 .extra_temps = .{
26463 .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
26919 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2646426920 .unused,
2646526921 .unused,
2646626922 .unused,
......@@ -26472,27 +26928,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2647226928 .unused,
2647326929 .unused,
2647426930 },
26475 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26931 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26932 .clobbers = .{ .eflags = true },
2647626933 .each = .{ .once = &.{
26477 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
26478 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },
26479 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ },
26480 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },
26934 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
26935 .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ },
26936 .{ ._, ._na, .j, .@"0f", ._, ._, ._ },
26937 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
2648126938 } },
2648226939 }, .{
26483 .required_features = .{ .f16c, null, null, null },
26484 .src_constraints = .{
26485 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
26486 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
26487 .any,
26488 },
26940 .required_features = .{ .fast_imm16, null, null, null },
26941 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
2648926942 .patterns = &.{
26490 .{ .src = .{ .to_mem, .to_mem, .none } },
26943 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
26944 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
26945 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2649126946 },
2649226947 .extra_temps = .{
26493 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26494 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
26495 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
26948 .{ .type = .i16, .kind = .{ .reg = .dx } },
26949 .unused,
26950 .unused,
2649626951 .unused,
2649726952 .unused,
2649826953 .unused,
......@@ -26502,33 +26957,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2650226957 .unused,
2650326958 .unused,
2650426959 },
26505 .dst_temps = .{ .mem, .unused },
26960 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2650626961 .clobbers = .{ .eflags = true },
2650726962 .each = .{ .once = &.{
26508 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26509 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
26510 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
26511 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ },
26512 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },
26513 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
26514 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26963 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
26964 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
26965 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
26966 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
26967 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
26968 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
2651526969 } },
2651626970 }, .{
26517 .required_features = .{ .avx, null, null, null },
26518 .src_constraints = .{
26519 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26520 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26521 .any,
26522 },
26971 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
2652326972 .patterns = &.{
26524 .{ .src = .{ .to_mem, .to_mem, .none } },
26973 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
26974 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
26975 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2652526976 },
26526 .call_frame = .{ .alignment = .@"16" },
2652726977 .extra_temps = .{
26528 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26529 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
26530 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
26531 .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } },
26978 .{ .type = .i16, .kind = .{ .reg = .dx } },
26979 .unused,
26980 .unused,
26981 .unused,
2653226982 .unused,
2653326983 .unused,
2653426984 .unused,
......@@ -26537,34 +26987,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2653726987 .unused,
2653826988 .unused,
2653926989 },
26540 .dst_temps = .{ .mem, .unused },
26541 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
26990 .dst_temps = .{ .{ .ref = .src0 }, .unused },
26991 .clobbers = .{ .eflags = true },
2654226992 .each = .{ .once = &.{
26543 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26544 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
26545 .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) },
26546 .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) },
26547 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
26548 .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
26549 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
26550 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26993 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
26994 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
26995 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
26996 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
26997 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
26998 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
2655126999 } },
2655227000 }, .{
26553 .required_features = .{ .sse4_1, null, null, null },
26554 .src_constraints = .{
26555 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26556 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26557 .any,
26558 },
27001 .required_features = .{ .fast_imm16, null, null, null },
27002 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any },
2655927003 .patterns = &.{
26560 .{ .src = .{ .to_mem, .to_mem, .none } },
27004 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27005 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27006 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2656127007 },
26562 .call_frame = .{ .alignment = .@"16" },
2656327008 .extra_temps = .{
26564 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26565 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
26566 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
26567 .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } },
27009 .{ .type = .i16, .kind = .{ .reg = .dx } },
27010 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27011 .unused,
27012 .unused,
2656827013 .unused,
2656927014 .unused,
2657027015 .unused,
......@@ -26573,36 +27018,33 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2657327018 .unused,
2657427019 .unused,
2657527020 },
26576 .dst_temps = .{ .mem, .unused },
26577 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27021 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27022 .clobbers = .{ .eflags = true },
2657827023 .each = .{ .once = &.{
26579 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26580 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
26581 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
26582 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
26583 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
26584 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
26585 .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
26586 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
26587 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27024 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27025 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27026 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27027 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27028 .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ },
27029 .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ },
27030 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27031 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27032 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27033 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
2658827034 } },
2658927035 }, .{
26590 .required_features = .{ .sse2, null, null, null },
26591 .src_constraints = .{
26592 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26593 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26594 .any,
26595 },
27036 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any },
2659627037 .patterns = &.{
26597 .{ .src = .{ .to_mem, .to_mem, .none } },
27038 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27039 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27040 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2659827041 },
26599 .call_frame = .{ .alignment = .@"16" },
2660027042 .extra_temps = .{
26601 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26602 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
26603 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
26604 .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } },
26605 .{ .type = .f16, .kind = .{ .reg = .ax } },
27043 .{ .type = .i16, .kind = .{ .reg = .dx } },
27044 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27045 .unused,
27046 .unused,
27047 .unused,
2660627048 .unused,
2660727049 .unused,
2660827050 .unused,
......@@ -26610,154 +27052,90 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2661027052 .unused,
2661127053 .unused,
2661227054 },
26613 .dst_temps = .{ .mem, .unused },
26614 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27055 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27056 .clobbers = .{ .eflags = true },
2661527057 .each = .{ .once = &.{
26616 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26617 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
26618 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
26619 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
26620 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
26621 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
26622 .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ },
26623 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ },
26624 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
26625 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27058 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27059 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27060 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27061 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27062 .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ },
27063 .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ },
27064 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27065 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27066 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27067 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
2662627068 } },
2662727069 }, .{
26628 .required_features = .{ .sse, null, null, null },
26629 .src_constraints = .{
26630 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26631 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
26632 .any,
26633 },
27070 .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any },
2663427071 .patterns = &.{
26635 .{ .src = .{ .to_mem, .to_mem, .none } },
27072 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27073 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27074 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2663627075 },
26637 .call_frame = .{ .alignment = .@"16" },
2663827076 .extra_temps = .{
26639 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26640 .{ .type = .f16, .kind = .{ .reg = .ax } },
26641 .{ .type = .f32, .kind = .mem },
26642 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
26643 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
26644 .{ .type = .usize, .kind = .{ .extern_func = "__mulhf3" } },
27077 .{ .type = .u16, .kind = .{ .reg = .dx } },
27078 .unused,
27079 .unused,
27080 .unused,
27081 .unused,
27082 .unused,
2664527083 .unused,
2664627084 .unused,
2664727085 .unused,
2664827086 .unused,
2664927087 .unused,
26650 },
26651 .dst_temps = .{ .mem, .unused },
26652 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
26653 .each = .{ .once = &.{
26654 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26655 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
26656 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
26657 .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ },
26658 .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
26659 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
26660 .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ },
26661 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
26662 .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ },
26663 .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ },
26664 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
26665 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
26666 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26667 } },
26668 }, .{
26669 .required_features = .{ .avx, null, null, null },
26670 .src_constraints = .{
26671 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
26672 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
26673 .any,
26674 },
26675 .patterns = &.{
26676 .{ .src = .{ .to_sse, .mem, .none } },
26677 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26678 .{ .src = .{ .to_sse, .to_sse, .none } },
26679 },
26680 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26681 .each = .{ .once = &.{
26682 .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ },
26683 } },
26684 }, .{
26685 .required_features = .{ .sse, null, null, null },
26686 .src_constraints = .{
26687 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
26688 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
26689 .any,
26690 },
26691 .patterns = &.{
26692 .{ .src = .{ .to_mut_sse, .mem, .none } },
26693 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
26694 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
2669527088 },
2669627089 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27090 .clobbers = .{ .eflags = true },
2669727091 .each = .{ .once = &.{
26698 .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ },
27092 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27093 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27094 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
27095 .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ },
2669927096 } },
2670027097 }, .{
26701 .required_features = .{ .avx, null, null, null },
26702 .src_constraints = .{
26703 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
26704 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
26705 .any,
26706 },
27098 .required_features = .{ .bmi, .cmov, null, null },
27099 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
2670727100 .patterns = &.{
26708 .{ .src = .{ .to_sse, .mem, .none } },
26709 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26710 .{ .src = .{ .to_sse, .to_sse, .none } },
26711 },
26712 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26713 .each = .{ .once = &.{
26714 .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ },
26715 } },
26716 }, .{
26717 .required_features = .{ .sse, null, null, null },
26718 .src_constraints = .{
26719 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
26720 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
26721 .any,
27101 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27102 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27103 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2672227104 },
26723 .patterns = &.{
26724 .{ .src = .{ .to_mut_sse, .mem, .none } },
26725 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
26726 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
27105 .extra_temps = .{
27106 .{ .type = .u16, .kind = .{ .reg = .dx } },
27107 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
27108 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27109 .unused,
27110 .unused,
27111 .unused,
27112 .unused,
27113 .unused,
27114 .unused,
27115 .unused,
27116 .unused,
2672727117 },
2672827118 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27119 .clobbers = .{ .eflags = true },
2672927120 .each = .{ .once = &.{
26730 .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ },
26731 } },
26732 }, .{
26733 .required_features = .{ .avx, null, null, null },
26734 .src_constraints = .{
26735 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
26736 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
26737 .any,
26738 },
26739 .patterns = &.{
26740 .{ .src = .{ .to_sse, .mem, .none } },
26741 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26742 .{ .src = .{ .to_sse, .to_sse, .none } },
26743 },
26744 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26745 .each = .{ .once = &.{
26746 .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ },
27121 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27122 .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ },
27123 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27124 .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ },
27125 .{ ._, ._, .@"or", .tmp2w, .tmp0w, ._, ._ },
27126 .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ },
2674727127 } },
2674827128 }, .{
26749 .required_features = .{ .avx, null, null, null },
26750 .src_constraints = .{
26751 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
26752 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
26753 .any,
26754 },
27129 .required_features = .{ .cmov, .fast_imm16, null, null },
27130 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
2675527131 .patterns = &.{
26756 .{ .src = .{ .to_mem, .to_mem, .none } },
27132 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27133 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27134 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2675727135 },
2675827136 .extra_temps = .{
26759 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26760 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
27137 .{ .type = .u16, .kind = .{ .reg = .dx } },
27138 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2676127139 .unused,
2676227140 .unused,
2676327141 .unused,
......@@ -26768,29 +27146,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2676827146 .unused,
2676927147 .unused,
2677027148 },
26771 .dst_temps = .{ .mem, .unused },
27149 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2677227150 .clobbers = .{ .eflags = true },
2677327151 .each = .{ .once = &.{
26774 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26775 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
26776 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
26777 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
26778 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
26779 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27152 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27153 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27154 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27155 .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ },
27156 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27157 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27158 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
2678027159 } },
2678127160 }, .{
26782 .required_features = .{ .sse, null, null, null },
26783 .src_constraints = .{
26784 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
26785 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
26786 .any,
26787 },
27161 .required_features = .{ .cmov, null, null, null },
27162 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
2678827163 .patterns = &.{
26789 .{ .src = .{ .to_mem, .to_mem, .none } },
27164 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27165 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27166 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2679027167 },
2679127168 .extra_temps = .{
26792 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26793 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
27169 .{ .type = .u16, .kind = .{ .reg = .dx } },
27170 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2679427171 .unused,
2679527172 .unused,
2679627173 .unused,
......@@ -26801,61 +27178,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2680127178 .unused,
2680227179 .unused,
2680327180 },
26804 .dst_temps = .{ .mem, .unused },
26805 .clobbers = .{ .eflags = true },
26806 .each = .{ .once = &.{
26807 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26808 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
26809 .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
26810 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
26811 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
26812 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
26813 } },
26814 }, .{
26815 .required_features = .{ .avx, null, null, null },
26816 .src_constraints = .{
26817 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26818 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26819 .any,
26820 },
26821 .patterns = &.{
26822 .{ .src = .{ .to_sse, .mem, .none } },
26823 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26824 .{ .src = .{ .to_sse, .to_sse, .none } },
26825 },
26826 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26827 .each = .{ .once = &.{
26828 .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ },
26829 } },
26830 }, .{
26831 .required_features = .{ .sse2, null, null, null },
26832 .src_constraints = .{
26833 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26834 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26835 .any,
26836 },
26837 .patterns = &.{
26838 .{ .src = .{ .to_mut_sse, .mem, .none } },
26839 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
26840 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
26841 },
2684227181 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27182 .clobbers = .{ .eflags = true },
2684327183 .each = .{ .once = &.{
26844 .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ },
27184 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27185 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27186 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27187 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27188 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27189 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27190 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
2684527191 } },
2684627192 }, .{
26847 .required_features = .{ .x87, null, null, null },
26848 .src_constraints = .{
26849 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26850 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
26851 .any,
26852 },
27193 .required_features = .{ .fast_imm16, null, null, null },
27194 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
2685327195 .patterns = &.{
26854 .{ .src = .{ .mem, .mem, .none } },
27196 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27197 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27198 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2685527199 },
2685627200 .extra_temps = .{
26857 .{ .type = .f64, .kind = .{ .reg = .st6 } },
26858 .{ .type = .f64, .kind = .{ .reg = .st7 } },
27201 .{ .type = .u16, .kind = .{ .reg = .dx } },
27202 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2685927203 .unused,
2686027204 .unused,
2686127205 .unused,
......@@ -26866,73 +27210,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2686627210 .unused,
2686727211 .unused,
2686827212 },
26869 .dst_temps = .{ .mem, .unused },
26870 .each = .{ .once = &.{
26871 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
26872 .{ ._, .f_, .mul, .src1q, ._, ._, ._ },
26873 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
26874 } },
26875 }, .{
26876 .required_features = .{ .avx, null, null, null },
26877 .src_constraints = .{
26878 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
26879 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
26880 .any,
26881 },
26882 .patterns = &.{
26883 .{ .src = .{ .to_sse, .mem, .none } },
26884 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26885 .{ .src = .{ .to_sse, .to_sse, .none } },
26886 },
26887 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26888 .each = .{ .once = &.{
26889 .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ },
26890 } },
26891 }, .{
26892 .required_features = .{ .sse2, null, null, null },
26893 .src_constraints = .{
26894 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
26895 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
26896 .any,
26897 },
26898 .patterns = &.{
26899 .{ .src = .{ .to_mut_sse, .mem, .none } },
26900 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
26901 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
26902 },
2690327213 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27214 .clobbers = .{ .eflags = true },
2690427215 .each = .{ .once = &.{
26905 .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ },
26906 } },
26907 }, .{
26908 .required_features = .{ .avx, null, null, null },
26909 .src_constraints = .{
26910 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
26911 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
26912 .any,
26913 },
26914 .patterns = &.{
26915 .{ .src = .{ .to_sse, .mem, .none } },
26916 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
26917 .{ .src = .{ .to_sse, .to_sse, .none } },
26918 },
26919 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
26920 .each = .{ .once = &.{
26921 .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ },
27216 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27217 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27218 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27219 .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ },
27220 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27221 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27222 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
2692227223 } },
2692327224 }, .{
26924 .required_features = .{ .avx, null, null, null },
26925 .src_constraints = .{
26926 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
26927 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
26928 .any,
26929 },
27225 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
2693027226 .patterns = &.{
26931 .{ .src = .{ .to_mem, .to_mem, .none } },
27227 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27228 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27229 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
2693227230 },
2693327231 .extra_temps = .{
26934 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26935 .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } },
27232 .{ .type = .u16, .kind = .{ .reg = .dx } },
27233 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2693627234 .unused,
2693727235 .unused,
2693827236 .unused,
......@@ -26943,29 +27241,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2694327241 .unused,
2694427242 .unused,
2694527243 },
26946 .dst_temps = .{ .mem, .unused },
27244 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2694727245 .clobbers = .{ .eflags = true },
2694827246 .each = .{ .once = &.{
26949 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26950 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
26951 .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
26952 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
26953 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
26954 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27247 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27248 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27249 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27250 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27251 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27252 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27253 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
2695527254 } },
2695627255 }, .{
26957 .required_features = .{ .sse2, null, null, null },
26958 .src_constraints = .{
26959 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
26960 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
26961 .any,
26962 },
27256 .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any },
2696327257 .patterns = &.{
26964 .{ .src = .{ .to_mem, .to_mem, .none } },
27258 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27259 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27260 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2696527261 },
2696627262 .extra_temps = .{
26967 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
26968 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
27263 .{ .type = .i32, .kind = .{ .reg = .edx } },
27264 .unused,
2696927265 .unused,
2697027266 .unused,
2697127267 .unused,
......@@ -26976,30 +27272,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2697627272 .unused,
2697727273 .unused,
2697827274 },
26979 .dst_temps = .{ .mem, .unused },
27275 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2698027276 .clobbers = .{ .eflags = true },
2698127277 .each = .{ .once = &.{
26982 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
26983 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
26984 .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
26985 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
26986 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
26987 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27278 .{ ._, .i_, .mul, .src1d, ._, ._, ._ },
27279 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
27280 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
27281 .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ },
27282 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
2698827283 } },
2698927284 }, .{
26990 .required_features = .{ .x87, null, null, null },
26991 .src_constraints = .{
26992 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
26993 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
26994 .any,
26995 },
27285 .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any },
2699627286 .patterns = &.{
26997 .{ .src = .{ .to_mem, .to_mem, .none } },
27287 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27288 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27289 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2699827290 },
2699927291 .extra_temps = .{
27000 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27001 .{ .type = .f64, .kind = .{ .reg = .st6 } },
27002 .{ .type = .f64, .kind = .{ .reg = .st7 } },
27292 .{ .type = .i32, .kind = .{ .reg = .edx } },
27293 .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27294 .unused,
2700327295 .unused,
2700427296 .unused,
2700527297 .unused,
......@@ -27009,29 +27301,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2700927301 .unused,
2701027302 .unused,
2701127303 },
27012 .dst_temps = .{ .mem, .unused },
27304 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2701327305 .clobbers = .{ .eflags = true },
2701427306 .each = .{ .once = &.{
27015 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27016 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
27017 .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
27018 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
27019 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
27020 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27307 .{ ._, .i_, .mul, .src1d, ._, ._, ._ },
27308 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27309 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27310 .{ ._, ._r, .sa, .tmp1d, .sia(-1, .src0, .add_bit_size), ._, ._ },
27311 .{ ._, ._, .cmp, .tmp1d, .tmp0d, ._, ._ },
27312 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27313 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27314 .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ },
27315 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
2702127316 } },
2702227317 }, .{
27023 .required_features = .{ .x87, null, null, null },
27024 .src_constraints = .{
27025 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
27026 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
27027 .any,
27028 },
27318 .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any },
2702927319 .patterns = &.{
27030 .{ .src = .{ .mem, .mem, .none } },
27320 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27321 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27322 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2703127323 },
2703227324 .extra_temps = .{
27033 .{ .type = .f80, .kind = .{ .reg = .st6 } },
27034 .{ .type = .f80, .kind = .{ .reg = .st7 } },
27325 .{ .type = .u32, .kind = .{ .reg = .edx } },
27326 .unused,
2703527327 .unused,
2703627328 .unused,
2703727329 .unused,
......@@ -27042,29 +27334,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2704227334 .unused,
2704327335 .unused,
2704427336 },
27045 .dst_temps = .{ .{ .rc = .x87 }, .unused },
27337 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27338 .clobbers = .{ .eflags = true },
2704627339 .each = .{ .once = &.{
27047 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
27048 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
27049 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
27050 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
27340 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27341 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
27342 .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ },
2705127343 } },
2705227344 }, .{
27053 .required_features = .{ .x87, null, null, null },
27054 .src_constraints = .{
27055 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
27056 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
27057 .any,
27058 },
27345 .required_features = .{ .bmi, .cmov, null, null },
27346 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
2705927347 .patterns = &.{
27060 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },
27061 .{ .src = .{ .mem, .to_x87, .none } },
27062 .{ .src = .{ .to_x87, .to_x87, .none } },
27348 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27349 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27350 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2706327351 },
2706427352 .extra_temps = .{
27065 .{ .type = .f80, .kind = .{ .reg = .st7 } },
27066 .unused,
27067 .unused,
27353 .{ .type = .u32, .kind = .{ .reg = .edx } },
27354 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
27355 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2706827356 .unused,
2706927357 .unused,
2707027358 .unused,
......@@ -27074,26 +27362,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2707427362 .unused,
2707527363 .unused,
2707627364 },
27077 .dst_temps = .{ .{ .rc = .x87 }, .unused },
27365 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27366 .clobbers = .{ .eflags = true },
2707827367 .each = .{ .once = &.{
27079 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
27080 .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ },
27081 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
27368 .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ },
27369 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27370 .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ },
27371 .{ ._, ._, .@"or", .tmp2d, .tmp0d, ._, ._ },
27372 .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ },
2708227373 } },
2708327374 }, .{
27084 .required_features = .{ .x87, null, null, null },
27085 .src_constraints = .{
27086 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
27087 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
27088 .any,
27089 },
27375 .required_features = .{ .cmov, null, null, null },
27376 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
2709027377 .patterns = &.{
27091 .{ .src = .{ .to_mem, .to_mem, .none } },
27378 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27379 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27380 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2709227381 },
2709327382 .extra_temps = .{
27094 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27095 .{ .type = .f80, .kind = .{ .reg = .st6 } },
27096 .{ .type = .f80, .kind = .{ .reg = .st7 } },
27383 .{ .type = .u32, .kind = .{ .reg = .edx } },
27384 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27385 .unused,
2709727386 .unused,
2709827387 .unused,
2709927388 .unused,
......@@ -27103,35 +27392,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2710327392 .unused,
2710427393 .unused,
2710527394 },
27106 .dst_temps = .{ .mem, .unused },
27395 .dst_temps = .{ .{ .ref = .src0 }, .unused },
2710727396 .clobbers = .{ .eflags = true },
2710827397 .each = .{ .once = &.{
27109 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27110 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
27111 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },
27112 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
27113 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
27114 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27115 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27398 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27399 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27400 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27401 .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ },
27402 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27403 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
2711627404 } },
2711727405 }, .{
27118 .required_features = .{ .sse, null, null, null },
27119 .src_constraints = .{
27120 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
27121 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
27122 .any,
27123 },
27406 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
2712427407 .patterns = &.{
27125 .{ .src = .{
27126 .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } },
27127 .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } },
27128 .none,
27129 } },
27408 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27409 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27410 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
2713027411 },
27131 .call_frame = .{ .alignment = .@"16" },
2713227412 .extra_temps = .{
27133 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
27134 .unused,
27413 .{ .type = .u32, .kind = .{ .reg = .edx } },
27414 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
2713527415 .unused,
2713627416 .unused,
2713727417 .unused,
......@@ -27143,775 +27423,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2714327423 .unused,
2714427424 },
2714527425 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27146 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27426 .clobbers = .{ .eflags = true },
2714727427 .each = .{ .once = &.{
27148 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
27428 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27429 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27430 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27431 .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ },
27432 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27433 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
2714927434 } },
2715027435 }, .{
27151 .required_features = .{ .avx, null, null, null },
27152 .src_constraints = .{
27153 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27154 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27155 .any,
27156 },
27436 .required_features = .{ .@"64bit", null, null, null },
27437 .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any },
2715727438 .patterns = &.{
27158 .{ .src = .{ .to_mem, .to_mem, .none } },
27439 .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } },
27440 .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } },
27441 .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } },
2715927442 },
27160 .call_frame = .{ .alignment = .@"16" },
2716127443 .extra_temps = .{
27162 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27163 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
27164 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
27165 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
27166 .unused,
27167 .unused,
27168 .unused,
27169 .unused,
27170 .unused,
27171 .unused,
27172 .unused,
27173 },
27174 .dst_temps = .{ .mem, .unused },
27175 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27176 .each = .{ .once = &.{
27177 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27178 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
27179 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
27180 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
27181 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
27182 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27183 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27184 } },
27185 }, .{
27186 .required_features = .{ .sse2, null, null, null },
27187 .src_constraints = .{
27188 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27189 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27190 .any,
27191 },
27192 .patterns = &.{
27193 .{ .src = .{ .to_mem, .to_mem, .none } },
27194 },
27195 .call_frame = .{ .alignment = .@"16" },
27196 .extra_temps = .{
27197 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27198 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
27199 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
27200 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
27201 .unused,
27202 .unused,
27203 .unused,
27204 .unused,
27205 .unused,
27206 .unused,
27207 .unused,
27208 },
27209 .dst_temps = .{ .mem, .unused },
27210 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27211 .each = .{ .once = &.{
27212 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27213 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
27214 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
27215 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
27216 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
27217 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27218 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27219 } },
27220 }, .{
27221 .required_features = .{ .sse, null, null, null },
27222 .src_constraints = .{
27223 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27224 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
27225 .any,
27226 },
27227 .patterns = &.{
27228 .{ .src = .{ .to_mem, .to_mem, .none } },
27229 },
27230 .call_frame = .{ .alignment = .@"16" },
27231 .extra_temps = .{
27232 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
27233 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
27234 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
27235 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
27236 .unused,
27237 .unused,
27238 .unused,
27239 .unused,
27240 .unused,
27241 .unused,
27242 .unused,
27243 },
27244 .dst_temps = .{ .mem, .unused },
27245 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
27246 .each = .{ .once = &.{
27247 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
27248 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
27249 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
27250 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
27251 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
27252 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
27253 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
27254 } },
27255 } }) catch |err| switch (err) {
27256 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
27257 @tagName(air_tag),
27258 ty.fmt(pt),
27259 ops[0].tracking(cg),
27260 ops[1].tracking(cg),
27261 }),
27262 else => |e| return e,
27263 };
27264 res[0].wrapInt(cg) catch |err| switch (err) {
27265 error.SelectFailed => return cg.fail("failed to select {s} wrap {f} {f}", .{
27266 @tagName(air_tag),
27267 cg.typeOf(bin_op.lhs).fmt(pt),
27268 res[0].tracking(cg),
27269 }),
27270 else => |e| return e,
27271 };
27272 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
27273 },
27274 .mul_sat => |air_tag| {
27275 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
27276 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
27277 var res: [1]Temp = undefined;
27278 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{
27279 .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any },
27280 .patterns = &.{
27281 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
27282 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
27283 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
27284 },
27285 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27286 .clobbers = .{ .eflags = true },
27287 .each = .{ .once = &.{
27288 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
27289 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
27290 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27291 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ },
27292 } },
27293 }, .{
27294 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
27295 .patterns = &.{
27296 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
27297 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
27298 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
27299 },
27300 .extra_temps = .{
27301 .{ .type = .i8, .kind = .{ .rc = .gphi } },
27302 .unused,
27303 .unused,
27304 .unused,
27305 .unused,
27306 .unused,
27307 .unused,
27308 .unused,
27309 .unused,
27310 .unused,
27311 .unused,
27312 },
27313 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27314 .clobbers = .{ .eflags = true },
27315 .each = .{ .once = &.{
27316 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
27317 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27318 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
27319 .{ ._, ._r, .sa, .tmp0b, .sia(-1, .src0, .add_bit_size), ._, ._ },
27320 .{ ._, ._, .cmp, .tmp0b, .dst0h, ._, ._ },
27321 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27322 .{ .@"1:", ._r, .sa, .dst0w, .ui(15), ._, ._ },
27323 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ },
27324 } },
27325 }, .{
27326 .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any },
27327 .patterns = &.{
27328 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
27329 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
27330 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
27331 },
27332 .extra_temps = .{
27333 .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27334 .unused,
27335 .unused,
27336 .unused,
27337 .unused,
27338 .unused,
27339 .unused,
27340 .unused,
27341 .unused,
27342 .unused,
27343 .unused,
27344 },
27345 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27346 .clobbers = .{ .eflags = true },
27347 .each = .{ .once = &.{
27348 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
27349 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
27350 .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ },
27351 } },
27352 }, .{
27353 .required_features = .{ .cmov, null, null, null },
27354 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
27355 .patterns = &.{
27356 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
27357 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
27358 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
27359 },
27360 .extra_temps = .{
27361 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27362 .unused,
27363 .unused,
27364 .unused,
27365 .unused,
27366 .unused,
27367 .unused,
27368 .unused,
27369 .unused,
27370 .unused,
27371 .unused,
27372 },
27373 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27374 .clobbers = .{ .eflags = true },
27375 .each = .{ .once = &.{
27376 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
27377 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27378 .{ ._, ._, .cmp, .dst0w, .tmp0w, ._, ._ },
27379 .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ },
27380 } },
27381 }, .{
27382 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
27383 .patterns = &.{
27384 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
27385 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
27386 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
27387 },
27388 .extra_temps = .{
27389 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27390 .unused,
27391 .unused,
27392 .unused,
27393 .unused,
27394 .unused,
27395 .unused,
27396 .unused,
27397 .unused,
27398 .unused,
27399 .unused,
27400 },
27401 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27402 .clobbers = .{ .eflags = true },
27403 .each = .{ .once = &.{
27404 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
27405 .{ ._, ._, .cmp, .dst0w, .ua(.src0, .add_umax), ._, ._ },
27406 .{ ._, ._na, .j, .@"0f", ._, ._, ._ },
27407 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
27408 } },
27409 }, .{
27410 .required_features = .{ .fast_imm16, null, null, null },
27411 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
27412 .patterns = &.{
27413 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27414 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27415 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27416 },
27417 .extra_temps = .{
27418 .{ .type = .i16, .kind = .{ .reg = .dx } },
27419 .unused,
27420 .unused,
27421 .unused,
27422 .unused,
27423 .unused,
27424 .unused,
27425 .unused,
27426 .unused,
27427 .unused,
27428 .unused,
27429 },
27430 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27431 .clobbers = .{ .eflags = true },
27432 .each = .{ .once = &.{
27433 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27434 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27435 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
27436 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
27437 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27438 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
27439 } },
27440 }, .{
27441 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
27442 .patterns = &.{
27443 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27444 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27445 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27446 },
27447 .extra_temps = .{
27448 .{ .type = .i16, .kind = .{ .reg = .dx } },
27449 .unused,
27450 .unused,
27451 .unused,
27452 .unused,
27453 .unused,
27454 .unused,
27455 .unused,
27456 .unused,
27457 .unused,
27458 .unused,
27459 },
27460 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27461 .clobbers = .{ .eflags = true },
27462 .each = .{ .once = &.{
27463 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27464 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27465 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
27466 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
27467 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27468 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
27469 } },
27470 }, .{
27471 .required_features = .{ .fast_imm16, null, null, null },
27472 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any },
27473 .patterns = &.{
27474 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27475 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27476 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27477 },
27478 .extra_temps = .{
27479 .{ .type = .i16, .kind = .{ .reg = .dx } },
27480 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27481 .unused,
27482 .unused,
27483 .unused,
27484 .unused,
27485 .unused,
27486 .unused,
27487 .unused,
27488 .unused,
27489 .unused,
27490 },
27491 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27492 .clobbers = .{ .eflags = true },
27493 .each = .{ .once = &.{
27494 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27495 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27496 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27497 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27498 .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ },
27499 .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ },
27500 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27501 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27502 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27503 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
27504 } },
27505 }, .{
27506 .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any },
27507 .patterns = &.{
27508 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27509 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27510 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27511 },
27512 .extra_temps = .{
27513 .{ .type = .i16, .kind = .{ .reg = .dx } },
27514 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27515 .unused,
27516 .unused,
27517 .unused,
27518 .unused,
27519 .unused,
27520 .unused,
27521 .unused,
27522 .unused,
27523 .unused,
27524 },
27525 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27526 .clobbers = .{ .eflags = true },
27527 .each = .{ .once = &.{
27528 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27529 .{ ._, .i_, .mul, .src1w, ._, ._, ._ },
27530 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27531 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27532 .{ ._, ._r, .sa, .tmp1w, .sia(-1, .src0, .add_bit_size), ._, ._ },
27533 .{ ._, ._, .cmp, .tmp1w, .tmp0w, ._, ._ },
27534 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27535 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27536 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
27537 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
27538 } },
27539 }, .{
27540 .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any },
27541 .patterns = &.{
27542 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27543 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27544 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27545 },
27546 .extra_temps = .{
27547 .{ .type = .u16, .kind = .{ .reg = .dx } },
27548 .unused,
27549 .unused,
27550 .unused,
27551 .unused,
27552 .unused,
27553 .unused,
27554 .unused,
27555 .unused,
27556 .unused,
27557 .unused,
27558 },
27559 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27560 .clobbers = .{ .eflags = true },
27561 .each = .{ .once = &.{
27562 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27563 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27564 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
27565 .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ },
27566 } },
27567 }, .{
27568 .required_features = .{ .bmi, .cmov, null, null },
27569 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
27570 .patterns = &.{
27571 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27572 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27573 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27574 },
27575 .extra_temps = .{
27576 .{ .type = .u16, .kind = .{ .reg = .dx } },
27577 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
27578 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27579 .unused,
27580 .unused,
27581 .unused,
27582 .unused,
27583 .unused,
27584 .unused,
27585 .unused,
27586 .unused,
27587 },
27588 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27589 .clobbers = .{ .eflags = true },
27590 .each = .{ .once = &.{
27591 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27592 .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ },
27593 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27594 .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ },
27595 .{ ._, ._, .@"or", .tmp2w, .tmp0w, ._, ._ },
27596 .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ },
27597 } },
27598 }, .{
27599 .required_features = .{ .cmov, .fast_imm16, null, null },
27600 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
27601 .patterns = &.{
27602 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27603 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27604 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27605 },
27606 .extra_temps = .{
27607 .{ .type = .u16, .kind = .{ .reg = .dx } },
27608 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27609 .unused,
27610 .unused,
27611 .unused,
27612 .unused,
27613 .unused,
27614 .unused,
27615 .unused,
27616 .unused,
27617 .unused,
27618 },
27619 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27620 .clobbers = .{ .eflags = true },
27621 .each = .{ .once = &.{
27622 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27623 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27624 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27625 .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ },
27626 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27627 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27628 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
27629 } },
27630 }, .{
27631 .required_features = .{ .cmov, null, null, null },
27632 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
27633 .patterns = &.{
27634 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27635 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27636 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27637 },
27638 .extra_temps = .{
27639 .{ .type = .u16, .kind = .{ .reg = .dx } },
27640 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27641 .unused,
27642 .unused,
27643 .unused,
27644 .unused,
27645 .unused,
27646 .unused,
27647 .unused,
27648 .unused,
27649 .unused,
27650 },
27651 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27652 .clobbers = .{ .eflags = true },
27653 .each = .{ .once = &.{
27654 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27655 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27656 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27657 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27658 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27659 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27660 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
27661 } },
27662 }, .{
27663 .required_features = .{ .fast_imm16, null, null, null },
27664 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
27665 .patterns = &.{
27666 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27667 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27668 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27669 },
27670 .extra_temps = .{
27671 .{ .type = .u16, .kind = .{ .reg = .dx } },
27672 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27673 .unused,
27674 .unused,
27675 .unused,
27676 .unused,
27677 .unused,
27678 .unused,
27679 .unused,
27680 .unused,
27681 .unused,
27682 },
27683 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27684 .clobbers = .{ .eflags = true },
27685 .each = .{ .once = &.{
27686 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27687 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27688 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27689 .{ ._, ._, .@"and", .tmp1w, .sa(.src0, .add_2_smin), ._, ._ },
27690 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27691 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27692 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
27693 } },
27694 }, .{
27695 .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any },
27696 .patterns = &.{
27697 .{ .src = .{ .{ .to_reg = .ax }, .mem, .none } },
27698 .{ .src = .{ .mem, .{ .to_reg = .ax }, .none }, .commute = .{ 0, 1 } },
27699 .{ .src = .{ .{ .to_reg = .ax }, .to_gpr, .none } },
27700 },
27701 .extra_temps = .{
27702 .{ .type = .u16, .kind = .{ .reg = .dx } },
27703 .{ .type = .u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27704 .unused,
27705 .unused,
27706 .unused,
27707 .unused,
27708 .unused,
27709 .unused,
27710 .unused,
27711 .unused,
27712 .unused,
27713 },
27714 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27715 .clobbers = .{ .eflags = true },
27716 .each = .{ .once = &.{
27717 .{ ._, ._, .xor, .tmp0d, .tmp0d, ._, ._ },
27718 .{ ._, ._, .mul, .src1w, ._, ._, ._ },
27719 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27720 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27721 .{ ._, ._, .@"or", .tmp1w, .tmp0w, ._, ._ },
27722 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27723 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
27724 } },
27725 }, .{
27726 .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any },
27727 .patterns = &.{
27728 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27729 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27730 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27731 },
27732 .extra_temps = .{
27733 .{ .type = .i32, .kind = .{ .reg = .edx } },
27734 .unused,
27735 .unused,
27736 .unused,
27737 .unused,
27738 .unused,
27739 .unused,
27740 .unused,
27741 .unused,
27742 .unused,
27743 .unused,
27744 },
27745 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27746 .clobbers = .{ .eflags = true },
27747 .each = .{ .once = &.{
27748 .{ ._, .i_, .mul, .src1d, ._, ._, ._ },
27749 .{ ._, ._nc, .j, .@"0f", ._, ._, ._ },
27750 .{ ._, ._, .mov, .dst0d, .tmp0d, ._, ._ },
27751 .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ },
27752 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
27753 } },
27754 }, .{
27755 .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any },
27756 .patterns = &.{
27757 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27758 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27759 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27760 },
27761 .extra_temps = .{
27762 .{ .type = .i32, .kind = .{ .reg = .edx } },
27763 .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27764 .unused,
27765 .unused,
27766 .unused,
27767 .unused,
27768 .unused,
27769 .unused,
27770 .unused,
27771 .unused,
27772 .unused,
27773 },
27774 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27775 .clobbers = .{ .eflags = true },
27776 .each = .{ .once = &.{
27777 .{ ._, .i_, .mul, .src1d, ._, ._, ._ },
27778 .{ ._, ._c, .j, .@"1f", ._, ._, ._ },
27779 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27780 .{ ._, ._r, .sa, .tmp1d, .sia(-1, .src0, .add_bit_size), ._, ._ },
27781 .{ ._, ._, .cmp, .tmp1d, .tmp0d, ._, ._ },
27782 .{ ._, ._e, .j, .@"0f", ._, ._, ._ },
27783 .{ .@"1:", ._, .mov, .dst0d, .tmp0d, ._, ._ },
27784 .{ ._, ._r, .sa, .dst0d, .ui(31), ._, ._ },
27785 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
27786 } },
27787 }, .{
27788 .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any },
27789 .patterns = &.{
27790 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27791 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27792 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27793 },
27794 .extra_temps = .{
27795 .{ .type = .u32, .kind = .{ .reg = .edx } },
27796 .unused,
27797 .unused,
27798 .unused,
27799 .unused,
27800 .unused,
27801 .unused,
27802 .unused,
27803 .unused,
27804 .unused,
27805 .unused,
27806 },
27807 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27808 .clobbers = .{ .eflags = true },
27809 .each = .{ .once = &.{
27810 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27811 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
27812 .{ ._, ._, .@"or", .dst0d, .tmp0d, ._, ._ },
27813 } },
27814 }, .{
27815 .required_features = .{ .bmi, .cmov, null, null },
27816 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
27817 .patterns = &.{
27818 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27819 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27820 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27821 },
27822 .extra_temps = .{
27823 .{ .type = .u32, .kind = .{ .reg = .edx } },
27824 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
27825 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27826 .unused,
27827 .unused,
27828 .unused,
27829 .unused,
27830 .unused,
27831 .unused,
27832 .unused,
27833 .unused,
27834 },
27835 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27836 .clobbers = .{ .eflags = true },
27837 .each = .{ .once = &.{
27838 .{ ._, ._, .mov, .tmp1d, .ua(.src0, .add_umax), ._, ._ },
27839 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27840 .{ ._, ._, .andn, .tmp2d, .tmp1d, .dst0d, ._ },
27841 .{ ._, ._, .@"or", .tmp2d, .tmp0d, ._, ._ },
27842 .{ ._, ._nz, .cmov, .dst0d, .tmp1d, ._, ._ },
27843 } },
27844 }, .{
27845 .required_features = .{ .cmov, null, null, null },
27846 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
27847 .patterns = &.{
27848 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27849 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27850 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27851 },
27852 .extra_temps = .{
27853 .{ .type = .u32, .kind = .{ .reg = .edx } },
27854 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27855 .unused,
27856 .unused,
27857 .unused,
27858 .unused,
27859 .unused,
27860 .unused,
27861 .unused,
27862 .unused,
27863 .unused,
27864 },
27865 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27866 .clobbers = .{ .eflags = true },
27867 .each = .{ .once = &.{
27868 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27869 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27870 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27871 .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ },
27872 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
27873 .{ ._, ._nz, .cmov, .dst0d, .tmp0d, ._, ._ },
27874 } },
27875 }, .{
27876 .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any },
27877 .patterns = &.{
27878 .{ .src = .{ .{ .to_reg = .eax }, .mem, .none } },
27879 .{ .src = .{ .mem, .{ .to_reg = .eax }, .none }, .commute = .{ 0, 1 } },
27880 .{ .src = .{ .{ .to_reg = .eax }, .to_gpr, .none } },
27881 },
27882 .extra_temps = .{
27883 .{ .type = .u32, .kind = .{ .reg = .edx } },
27884 .{ .type = .u32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
27885 .unused,
27886 .unused,
27887 .unused,
27888 .unused,
27889 .unused,
27890 .unused,
27891 .unused,
27892 .unused,
27893 .unused,
27894 },
27895 .dst_temps = .{ .{ .ref = .src0 }, .unused },
27896 .clobbers = .{ .eflags = true },
27897 .each = .{ .once = &.{
27898 .{ ._, ._, .mul, .src1d, ._, ._, ._ },
27899 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
27900 .{ ._, ._, .@"and", .tmp1d, .sa(.src0, .add_2_smin), ._, ._ },
27901 .{ ._, ._, .@"or", .tmp1d, .tmp0d, ._, ._ },
27902 .{ ._, ._z, .j, .@"0f", ._, ._, ._ },
27903 .{ ._, ._, .mov, .dst0d, .ua(.src0, .add_umax), ._, ._ },
27904 } },
27905 }, .{
27906 .required_features = .{ .@"64bit", null, null, null },
27907 .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any },
27908 .patterns = &.{
27909 .{ .src = .{ .{ .to_reg = .rax }, .mem, .none } },
27910 .{ .src = .{ .mem, .{ .to_reg = .rax }, .none }, .commute = .{ 0, 1 } },
27911 .{ .src = .{ .{ .to_reg = .rax }, .to_gpr, .none } },
27912 },
27913 .extra_temps = .{
27914 .{ .type = .i64, .kind = .{ .reg = .rdx } },
27444 .{ .type = .i64, .kind = .{ .reg = .rdx } },
2791527445 .unused,
2791627446 .unused,
2791727447 .unused,
......@@ -33431,6 +32961,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3343132961 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
3343232962 } },
3343332963 }, .{
32964 .required_cc_abi = .sysv64,
3343432965 .required_features = .{ .sse, null, null, null },
3343532966 .src_constraints = .{
3343632967 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -33464,6 +32995,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3346432995 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
3346532996 } },
3346632997 }, .{
32998 .required_cc_abi = .win64,
32999 .required_features = .{ .sse, null, null, null },
33000 .src_constraints = .{
33001 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
33002 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
33003 .any,
33004 },
33005 .patterns = &.{
33006 .{ .src = .{ .to_mem, .to_mem, .none } },
33007 },
33008 .call_frame = .{ .alignment = .@"16" },
33009 .extra_temps = .{
33010 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33011 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
33012 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
33013 .unused,
33014 .unused,
33015 .unused,
33016 .unused,
33017 .unused,
33018 .unused,
33019 .unused,
33020 .unused,
33021 },
33022 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
33023 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
33024 .each = .{ .once = &.{
33025 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
33026 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
33027 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
33028 } },
33029 }, .{
33030 .required_cc_abi = .sysv64,
3346733031 .required_features = .{ .avx, null, null, null },
3346833032 .src_constraints = .{
3346933033 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -33475,7 +33039,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3347533039 },
3347633040 .call_frame = .{ .alignment = .@"16" },
3347733041 .extra_temps = .{
33478 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
33042 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3347933043 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3348033044 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3348133045 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -33490,15 +33054,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3349033054 .dst_temps = .{ .mem, .unused },
3349133055 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3349233056 .each = .{ .once = &.{
33493 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
33494 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
33495 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
33057 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33058 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
33059 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3349633060 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33497 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
33498 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
33499 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
33061 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
33062 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33063 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3350033064 } },
3350133065 }, .{
33066 .required_cc_abi = .sysv64,
3350233067 .required_features = .{ .sse2, null, null, null },
3350333068 .src_constraints = .{
3350433069 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -33510,7 +33075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3351033075 },
3351133076 .call_frame = .{ .alignment = .@"16" },
3351233077 .extra_temps = .{
33513 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
33078 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3351433079 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3351533080 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3351633081 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -33525,15 +33090,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3352533090 .dst_temps = .{ .mem, .unused },
3352633091 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3352733092 .each = .{ .once = &.{
33528 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
33529 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
33530 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
33093 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33094 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
33095 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3353133096 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33532 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
33533 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
33534 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
33097 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
33098 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33099 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3353533100 } },
3353633101 }, .{
33102 .required_cc_abi = .sysv64,
3353733103 .required_features = .{ .sse, null, null, null },
3353833104 .src_constraints = .{
3353933105 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -33545,7 +33111,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3354533111 },
3354633112 .call_frame = .{ .alignment = .@"16" },
3354733113 .extra_temps = .{
33548 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
33114 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3354933115 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3355033116 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3355133117 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -33560,13 +33126,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3356033126 .dst_temps = .{ .mem, .unused },
3356133127 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3356233128 .each = .{ .once = &.{
33563 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
33564 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
33565 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
33129 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33130 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
33131 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3356633132 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33567 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
33568 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
33569 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
33133 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
33134 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33135 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
33136 } },
33137 }, .{
33138 .required_cc_abi = .win64,
33139 .required_features = .{ .avx, null, null, null },
33140 .src_constraints = .{
33141 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33142 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33143 .any,
33144 },
33145 .patterns = &.{
33146 .{ .src = .{ .to_mem, .to_mem, .none } },
33147 },
33148 .call_frame = .{ .alignment = .@"16" },
33149 .extra_temps = .{
33150 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
33151 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33152 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
33153 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
33154 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33155 .unused,
33156 .unused,
33157 .unused,
33158 .unused,
33159 .unused,
33160 .unused,
33161 },
33162 .dst_temps = .{ .mem, .unused },
33163 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
33164 .each = .{ .once = &.{
33165 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33166 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
33167 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
33168 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33169 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
33170 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33171 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
33172 } },
33173 }, .{
33174 .required_cc_abi = .win64,
33175 .required_features = .{ .sse2, null, null, null },
33176 .src_constraints = .{
33177 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33178 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33179 .any,
33180 },
33181 .patterns = &.{
33182 .{ .src = .{ .to_mem, .to_mem, .none } },
33183 },
33184 .call_frame = .{ .alignment = .@"16" },
33185 .extra_temps = .{
33186 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
33187 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33188 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
33189 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
33190 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33191 .unused,
33192 .unused,
33193 .unused,
33194 .unused,
33195 .unused,
33196 .unused,
33197 },
33198 .dst_temps = .{ .mem, .unused },
33199 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
33200 .each = .{ .once = &.{
33201 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33202 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
33203 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
33204 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33205 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
33206 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33207 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
33208 } },
33209 }, .{
33210 .required_cc_abi = .win64,
33211 .required_features = .{ .sse, null, null, null },
33212 .src_constraints = .{
33213 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33214 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
33215 .any,
33216 },
33217 .patterns = &.{
33218 .{ .src = .{ .to_mem, .to_mem, .none } },
33219 },
33220 .call_frame = .{ .alignment = .@"16" },
33221 .extra_temps = .{
33222 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
33223 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33224 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
33225 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
33226 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
33227 .unused,
33228 .unused,
33229 .unused,
33230 .unused,
33231 .unused,
33232 .unused,
33233 },
33234 .dst_temps = .{ .mem, .unused },
33235 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
33236 .each = .{ .once = &.{
33237 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
33238 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
33239 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
33240 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
33241 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
33242 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
33243 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3357033244 } },
3357133245 } }) else err: {
3357233246 assert(air_tag == .div_exact);
......@@ -34659,6 +34333,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3465934333 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
3466034334 } },
3466134335 }, .{
34336 .required_cc_abi = .sysv64,
3466234337 .required_features = .{ .sse, null, null, null },
3466334338 .src_constraints = .{
3466434339 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -34693,6 +34368,112 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3469334368 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
3469434369 } },
3469534370 }, .{
34371 .required_cc_abi = .win64,
34372 .required_features = .{ .avx, null, null, null },
34373 .src_constraints = .{
34374 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34375 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34376 .any,
34377 },
34378 .patterns = &.{
34379 .{ .src = .{ .to_mem, .to_mem, .none } },
34380 },
34381 .call_frame = .{ .alignment = .@"16" },
34382 .extra_temps = .{
34383 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34384 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34385 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34386 .{ .type = .f128, .kind = .mem },
34387 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34388 .unused,
34389 .unused,
34390 .unused,
34391 .unused,
34392 .unused,
34393 .unused,
34394 },
34395 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
34396 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34397 .each = .{ .once = &.{
34398 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
34399 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
34400 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
34401 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
34402 .{ ._, .v_dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
34403 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34404 } },
34405 }, .{
34406 .required_cc_abi = .win64,
34407 .required_features = .{ .sse2, null, null, null },
34408 .src_constraints = .{
34409 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34410 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34411 .any,
34412 },
34413 .patterns = &.{
34414 .{ .src = .{ .to_mem, .to_mem, .none } },
34415 },
34416 .call_frame = .{ .alignment = .@"16" },
34417 .extra_temps = .{
34418 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34419 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34420 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34421 .{ .type = .f128, .kind = .mem },
34422 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34423 .unused,
34424 .unused,
34425 .unused,
34426 .unused,
34427 .unused,
34428 .unused,
34429 },
34430 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
34431 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34432 .each = .{ .once = &.{
34433 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
34434 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
34435 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
34436 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
34437 .{ ._, ._dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
34438 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34439 } },
34440 }, .{
34441 .required_cc_abi = .win64,
34442 .required_features = .{ .sse, null, null, null },
34443 .src_constraints = .{
34444 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34445 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
34446 .any,
34447 },
34448 .patterns = &.{
34449 .{ .src = .{ .to_mem, .to_mem, .none } },
34450 },
34451 .call_frame = .{ .alignment = .@"16" },
34452 .extra_temps = .{
34453 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34454 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34455 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34456 .{ .type = .f128, .kind = .mem },
34457 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34458 .unused,
34459 .unused,
34460 .unused,
34461 .unused,
34462 .unused,
34463 .unused,
34464 },
34465 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
34466 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34467 .each = .{ .once = &.{
34468 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
34469 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
34470 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
34471 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
34472 .{ ._, ._ps, .mova, .lea(.tmp0x), .dst0x, ._, ._ },
34473 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34474 } },
34475 }, .{
34476 .required_cc_abi = .sysv64,
3469634477 .required_features = .{ .avx, null, null, null },
3469734478 .src_constraints = .{
3469834479 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -34704,7 +34485,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3470434485 },
3470534486 .call_frame = .{ .alignment = .@"16" },
3470634487 .extra_temps = .{
34707 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34488 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3470834489 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3470934490 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3471034491 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -34719,16 +34500,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3471934500 .dst_temps = .{ .mem, .unused },
3472034501 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3472134502 .each = .{ .once = &.{
34722 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
34723 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
34724 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
34503 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34504 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
34505 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3472534506 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3472634507 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34727 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
34728 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
34729 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
34508 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
34509 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34510 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3473034511 } },
3473134512 }, .{
34513 .required_cc_abi = .sysv64,
3473234514 .required_features = .{ .sse2, null, null, null },
3473334515 .src_constraints = .{
3473434516 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -34740,7 +34522,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3474034522 },
3474134523 .call_frame = .{ .alignment = .@"16" },
3474234524 .extra_temps = .{
34743 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34525 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3474434526 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3474534527 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3474634528 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -34755,16 +34537,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3475534537 .dst_temps = .{ .mem, .unused },
3475634538 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3475734539 .each = .{ .once = &.{
34758 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
34759 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
34760 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
34540 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34541 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
34542 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3476134543 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3476234544 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34763 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
34764 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
34765 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
34545 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
34546 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34547 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3476634548 } },
3476734549 }, .{
34550 .required_cc_abi = .sysv64,
3476834551 .required_features = .{ .sse, null, null, null },
3476934552 .src_constraints = .{
3477034553 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -34776,7 +34559,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3477634559 },
3477734560 .call_frame = .{ .alignment = .@"16" },
3477834561 .extra_temps = .{
34779 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34562 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3478034563 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3478134564 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3478234565 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -34791,14 +34574,131 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3479134574 .dst_temps = .{ .mem, .unused },
3479234575 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3479334576 .each = .{ .once = &.{
34794 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
34795 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
34796 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
34577 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34578 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
34579 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3479734580 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3479834581 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
34799 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
34800 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
34801 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
34582 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
34583 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34584 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
34585 } },
34586 }, .{
34587 .required_cc_abi = .win64,
34588 .required_features = .{ .avx, null, null, null },
34589 .src_constraints = .{
34590 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34591 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34592 .any,
34593 },
34594 .patterns = &.{
34595 .{ .src = .{ .to_mem, .to_mem, .none } },
34596 },
34597 .call_frame = .{ .alignment = .@"16" },
34598 .extra_temps = .{
34599 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
34600 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34601 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34602 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34603 .{ .type = .f128, .kind = .mem },
34604 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34605 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34606 .unused,
34607 .unused,
34608 .unused,
34609 .unused,
34610 },
34611 .dst_temps = .{ .mem, .unused },
34612 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34613 .each = .{ .once = &.{
34614 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34615 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
34616 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
34617 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
34618 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
34619 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
34620 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
34621 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
34622 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34623 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
34624 } },
34625 }, .{
34626 .required_cc_abi = .win64,
34627 .required_features = .{ .sse2, null, null, null },
34628 .src_constraints = .{
34629 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34630 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34631 .any,
34632 },
34633 .patterns = &.{
34634 .{ .src = .{ .to_mem, .to_mem, .none } },
34635 },
34636 .call_frame = .{ .alignment = .@"16" },
34637 .extra_temps = .{
34638 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
34639 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34640 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34641 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34642 .{ .type = .f128, .kind = .mem },
34643 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34644 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34645 .unused,
34646 .unused,
34647 .unused,
34648 .unused,
34649 },
34650 .dst_temps = .{ .mem, .unused },
34651 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34652 .each = .{ .once = &.{
34653 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34654 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
34655 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
34656 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
34657 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
34658 .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
34659 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
34660 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
34661 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34662 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
34663 } },
34664 }, .{
34665 .required_cc_abi = .win64,
34666 .required_features = .{ .sse, null, null, null },
34667 .src_constraints = .{
34668 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34669 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
34670 .any,
34671 },
34672 .patterns = &.{
34673 .{ .src = .{ .to_mem, .to_mem, .none } },
34674 },
34675 .call_frame = .{ .alignment = .@"16" },
34676 .extra_temps = .{
34677 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
34678 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34679 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
34680 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
34681 .{ .type = .f128, .kind = .mem },
34682 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
34683 .{ .type = .usize, .kind = .{ .extern_func = "truncq" } },
34684 .unused,
34685 .unused,
34686 .unused,
34687 .unused,
34688 },
34689 .dst_temps = .{ .mem, .unused },
34690 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
34691 .each = .{ .once = &.{
34692 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
34693 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
34694 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
34695 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
34696 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
34697 .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp5x, ._, ._ },
34698 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
34699 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
34700 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
34701 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3480234702 } },
3480334703 } }) else err: {
3480434704 res[0] = ops[0].divTruncInts(&ops[1], cg) catch |err| break :err err;
......@@ -35955,6 +35855,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3595535855 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
3595635856 } },
3595735857 }, .{
35858 .required_cc_abi = .sysv64,
3595835859 .required_features = .{ .sse, null, null, null },
3595935860 .src_constraints = .{
3596035861 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -35993,6 +35894,124 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3599335894 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
3599435895 } },
3599535896 }, .{
35897 .required_cc_abi = .win64,
35898 .required_features = .{ .avx, null, null, null },
35899 .src_constraints = .{
35900 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35901 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35902 .any,
35903 },
35904 .patterns = &.{
35905 .{ .src = .{ .to_mem, .to_mem, .none } },
35906 },
35907 .call_frame = .{ .alignment = .@"16" },
35908 .extra_temps = .{
35909 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
35910 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
35911 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
35912 .{ .type = .f128, .kind = .mem },
35913 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
35914 else => unreachable,
35915 .zero => "truncq",
35916 .down => "floorq",
35917 } } },
35918 .unused,
35919 .unused,
35920 .unused,
35921 .unused,
35922 .unused,
35923 .unused,
35924 },
35925 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
35926 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
35927 .each = .{ .once = &.{
35928 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
35929 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
35930 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
35931 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
35932 .{ ._, .v_dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
35933 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
35934 } },
35935 }, .{
35936 .required_cc_abi = .win64,
35937 .required_features = .{ .sse2, null, null, null },
35938 .src_constraints = .{
35939 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35940 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35941 .any,
35942 },
35943 .patterns = &.{
35944 .{ .src = .{ .to_mem, .to_mem, .none } },
35945 },
35946 .call_frame = .{ .alignment = .@"16" },
35947 .extra_temps = .{
35948 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
35949 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
35950 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
35951 .{ .type = .f128, .kind = .mem },
35952 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
35953 else => unreachable,
35954 .zero => "truncq",
35955 .down => "floorq",
35956 } } },
35957 .unused,
35958 .unused,
35959 .unused,
35960 .unused,
35961 .unused,
35962 .unused,
35963 },
35964 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
35965 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
35966 .each = .{ .once = &.{
35967 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
35968 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
35969 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
35970 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
35971 .{ ._, ._dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
35972 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
35973 } },
35974 }, .{
35975 .required_cc_abi = .win64,
35976 .required_features = .{ .sse, null, null, null },
35977 .src_constraints = .{
35978 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35979 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
35980 .any,
35981 },
35982 .patterns = &.{
35983 .{ .src = .{ .to_mem, .to_mem, .none } },
35984 },
35985 .call_frame = .{ .alignment = .@"16" },
35986 .extra_temps = .{
35987 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
35988 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
35989 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
35990 .{ .type = .f128, .kind = .mem },
35991 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
35992 else => unreachable,
35993 .zero => "truncq",
35994 .down => "floorq",
35995 } } },
35996 .unused,
35997 .unused,
35998 .unused,
35999 .unused,
36000 .unused,
36001 .unused,
36002 },
36003 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
36004 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
36005 .each = .{ .once = &.{
36006 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
36007 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
36008 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
36009 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
36010 .{ ._, ._ps, .mova, .lea(.tmp0x), .dst0x, ._, ._ },
36011 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
36012 } },
36013 }, .{
36014 .required_cc_abi = .sysv64,
3599636015 .required_features = .{ .avx, null, null, null },
3599736016 .src_constraints = .{
3599836017 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -36004,7 +36023,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3600436023 },
3600536024 .call_frame = .{ .alignment = .@"16" },
3600636025 .extra_temps = .{
36007 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
36026 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3600836027 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3600936028 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3601036029 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -36023,16 +36042,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3602336042 .dst_temps = .{ .mem, .unused },
3602436043 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3602536044 .each = .{ .once = &.{
36026 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
36027 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
36028 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
36045 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36046 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
36047 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3602936048 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3603036049 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
36031 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
36032 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
36033 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
36050 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
36051 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36052 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3603436053 } },
3603536054 }, .{
36055 .required_cc_abi = .sysv64,
3603636056 .required_features = .{ .sse2, null, null, null },
3603736057 .src_constraints = .{
3603836058 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -36044,7 +36064,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3604436064 },
3604536065 .call_frame = .{ .alignment = .@"16" },
3604636066 .extra_temps = .{
36047 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
36067 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3604836068 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3604936069 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3605036070 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -36063,16 +36083,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3606336083 .dst_temps = .{ .mem, .unused },
3606436084 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3606536085 .each = .{ .once = &.{
36066 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
36067 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
36068 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
36086 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36087 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
36088 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3606936089 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3607036090 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
36071 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
36072 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
36073 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
36091 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
36092 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36093 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3607436094 } },
3607536095 }, .{
36096 .required_cc_abi = .sysv64,
3607636097 .required_features = .{ .sse, null, null, null },
3607736098 .src_constraints = .{
3607836099 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -36084,7 +36105,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3608436105 },
3608536106 .call_frame = .{ .alignment = .@"16" },
3608636107 .extra_temps = .{
36087 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
36108 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3608836109 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3608936110 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3609036111 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -36103,14 +36124,143 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3610336124 .dst_temps = .{ .mem, .unused },
3610436125 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3610536126 .each = .{ .once = &.{
36106 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
36107 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
36108 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
36127 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36128 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
36129 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3610936130 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3611036131 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
36111 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
36112 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
36113 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
36132 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
36133 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36134 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
36135 } },
36136 }, .{
36137 .required_cc_abi = .win64,
36138 .required_features = .{ .avx, null, null, null },
36139 .src_constraints = .{
36140 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36141 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36142 .any,
36143 },
36144 .patterns = &.{
36145 .{ .src = .{ .to_mem, .to_mem, .none } },
36146 },
36147 .call_frame = .{ .alignment = .@"16" },
36148 .extra_temps = .{
36149 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
36150 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36151 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
36152 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
36153 .{ .type = .f128, .kind = .mem },
36154 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36155 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
36156 else => unreachable,
36157 .zero => "truncq",
36158 .down => "floorq",
36159 } } },
36160 .unused,
36161 .unused,
36162 .unused,
36163 .unused,
36164 },
36165 .dst_temps = .{ .mem, .unused },
36166 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
36167 .each = .{ .once = &.{
36168 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36169 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
36170 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
36171 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
36172 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
36173 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
36174 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
36175 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
36176 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36177 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
36178 } },
36179 }, .{
36180 .required_cc_abi = .win64,
36181 .required_features = .{ .sse2, null, null, null },
36182 .src_constraints = .{
36183 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36184 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36185 .any,
36186 },
36187 .patterns = &.{
36188 .{ .src = .{ .to_mem, .to_mem, .none } },
36189 },
36190 .call_frame = .{ .alignment = .@"16" },
36191 .extra_temps = .{
36192 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
36193 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36194 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
36195 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
36196 .{ .type = .f128, .kind = .mem },
36197 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36198 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
36199 else => unreachable,
36200 .zero => "truncq",
36201 .down => "floorq",
36202 } } },
36203 .unused,
36204 .unused,
36205 .unused,
36206 .unused,
36207 },
36208 .dst_temps = .{ .mem, .unused },
36209 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
36210 .each = .{ .once = &.{
36211 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36212 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
36213 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
36214 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
36215 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
36216 .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
36217 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
36218 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
36219 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36220 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
36221 } },
36222 }, .{
36223 .required_cc_abi = .win64,
36224 .required_features = .{ .sse, null, null, null },
36225 .src_constraints = .{
36226 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36227 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
36228 .any,
36229 },
36230 .patterns = &.{
36231 .{ .src = .{ .to_mem, .to_mem, .none } },
36232 },
36233 .call_frame = .{ .alignment = .@"16" },
36234 .extra_temps = .{
36235 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
36236 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36237 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
36238 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
36239 .{ .type = .f128, .kind = .mem },
36240 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
36241 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
36242 else => unreachable,
36243 .zero => "truncq",
36244 .down => "floorq",
36245 } } },
36246 .unused,
36247 .unused,
36248 .unused,
36249 .unused,
36250 },
36251 .dst_temps = .{ .mem, .unused },
36252 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
36253 .each = .{ .once = &.{
36254 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
36255 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
36256 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
36257 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
36258 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
36259 .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp5x, ._, ._ },
36260 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
36261 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
36262 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
36263 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3611436264 } },
3611536265 } },
3611636266 }) catch |err| switch (err) {
......@@ -37438,6 +37588,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3743837588 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
3743937589 } },
3744037590 }, .{
37591 .required_cc_abi = .sysv64,
3744137592 .required_features = .{ .sse, null, null, null },
3744237593 .src_constraints = .{
3744337594 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -37472,6 +37623,112 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3747237623 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
3747337624 } },
3747437625 }, .{
37626 .required_cc_abi = .win64,
37627 .required_features = .{ .avx, null, null, null },
37628 .src_constraints = .{
37629 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37630 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37631 .any,
37632 },
37633 .patterns = &.{
37634 .{ .src = .{ .to_mem, .to_mem, .none } },
37635 },
37636 .call_frame = .{ .alignment = .@"16" },
37637 .extra_temps = .{
37638 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37639 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37640 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37641 .{ .type = .f128, .kind = .mem },
37642 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37643 .unused,
37644 .unused,
37645 .unused,
37646 .unused,
37647 .unused,
37648 .unused,
37649 },
37650 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
37651 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37652 .each = .{ .once = &.{
37653 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
37654 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
37655 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
37656 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
37657 .{ ._, .v_dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
37658 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37659 } },
37660 }, .{
37661 .required_cc_abi = .win64,
37662 .required_features = .{ .sse2, null, null, null },
37663 .src_constraints = .{
37664 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37665 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37666 .any,
37667 },
37668 .patterns = &.{
37669 .{ .src = .{ .to_mem, .to_mem, .none } },
37670 },
37671 .call_frame = .{ .alignment = .@"16" },
37672 .extra_temps = .{
37673 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37674 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37675 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37676 .{ .type = .f128, .kind = .mem },
37677 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37678 .unused,
37679 .unused,
37680 .unused,
37681 .unused,
37682 .unused,
37683 .unused,
37684 },
37685 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
37686 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37687 .each = .{ .once = &.{
37688 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
37689 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
37690 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
37691 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
37692 .{ ._, ._dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
37693 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37694 } },
37695 }, .{
37696 .required_cc_abi = .win64,
37697 .required_features = .{ .sse, null, null, null },
37698 .src_constraints = .{
37699 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37700 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
37701 .any,
37702 },
37703 .patterns = &.{
37704 .{ .src = .{ .to_mem, .to_mem, .none } },
37705 },
37706 .call_frame = .{ .alignment = .@"16" },
37707 .extra_temps = .{
37708 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37709 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37710 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37711 .{ .type = .f128, .kind = .mem },
37712 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37713 .unused,
37714 .unused,
37715 .unused,
37716 .unused,
37717 .unused,
37718 .unused,
37719 },
37720 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
37721 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37722 .each = .{ .once = &.{
37723 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
37724 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
37725 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
37726 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
37727 .{ ._, ._ps, .mova, .lea(.tmp0x), .dst0x, ._, ._ },
37728 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37729 } },
37730 }, .{
37731 .required_cc_abi = .sysv64,
3747537732 .required_features = .{ .avx, null, null, null },
3747637733 .src_constraints = .{
3747737734 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -37483,7 +37740,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3748337740 },
3748437741 .call_frame = .{ .alignment = .@"16" },
3748537742 .extra_temps = .{
37486 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
37743 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3748737744 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3748837745 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3748937746 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -37498,16 +37755,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3749837755 .dst_temps = .{ .mem, .unused },
3749937756 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3750037757 .each = .{ .once = &.{
37501 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
37502 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
37503 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
37758 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37759 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
37760 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3750437761 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3750537762 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37506 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
37507 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
37508 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
37763 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
37764 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37765 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3750937766 } },
3751037767 }, .{
37768 .required_cc_abi = .sysv64,
3751137769 .required_features = .{ .sse2, null, null, null },
3751237770 .src_constraints = .{
3751337771 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -37519,7 +37777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3751937777 },
3752037778 .call_frame = .{ .alignment = .@"16" },
3752137779 .extra_temps = .{
37522 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
37780 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3752337781 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3752437782 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3752537783 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -37534,16 +37792,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3753437792 .dst_temps = .{ .mem, .unused },
3753537793 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3753637794 .each = .{ .once = &.{
37537 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
37538 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
37539 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
37795 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37796 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
37797 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3754037798 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3754137799 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37542 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
37543 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
37544 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
37800 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
37801 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37802 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3754537803 } },
3754637804 }, .{
37805 .required_cc_abi = .sysv64,
3754737806 .required_features = .{ .sse, null, null, null },
3754837807 .src_constraints = .{
3754937808 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -37555,7 +37814,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3755537814 },
3755637815 .call_frame = .{ .alignment = .@"16" },
3755737816 .extra_temps = .{
37558 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
37817 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3755937818 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3756037819 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3756137820 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
......@@ -37570,14 +37829,131 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3757037829 .dst_temps = .{ .mem, .unused },
3757137830 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3757237831 .each = .{ .once = &.{
37573 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
37574 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
37575 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
37832 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37833 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
37834 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3757637835 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
3757737836 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
37578 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
37579 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
37580 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
37837 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
37838 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37839 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
37840 } },
37841 }, .{
37842 .required_cc_abi = .win64,
37843 .required_features = .{ .avx, null, null, null },
37844 .src_constraints = .{
37845 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37846 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37847 .any,
37848 },
37849 .patterns = &.{
37850 .{ .src = .{ .to_mem, .to_mem, .none } },
37851 },
37852 .call_frame = .{ .alignment = .@"16" },
37853 .extra_temps = .{
37854 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
37855 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37856 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37857 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37858 .{ .type = .f128, .kind = .mem },
37859 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37860 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37861 .unused,
37862 .unused,
37863 .unused,
37864 .unused,
37865 },
37866 .dst_temps = .{ .mem, .unused },
37867 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37868 .each = .{ .once = &.{
37869 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37870 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
37871 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
37872 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
37873 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
37874 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
37875 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
37876 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
37877 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37878 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
37879 } },
37880 }, .{
37881 .required_cc_abi = .win64,
37882 .required_features = .{ .sse2, null, null, null },
37883 .src_constraints = .{
37884 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37885 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37886 .any,
37887 },
37888 .patterns = &.{
37889 .{ .src = .{ .to_mem, .to_mem, .none } },
37890 },
37891 .call_frame = .{ .alignment = .@"16" },
37892 .extra_temps = .{
37893 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
37894 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37895 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37896 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37897 .{ .type = .f128, .kind = .mem },
37898 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37899 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37900 .unused,
37901 .unused,
37902 .unused,
37903 .unused,
37904 },
37905 .dst_temps = .{ .mem, .unused },
37906 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37907 .each = .{ .once = &.{
37908 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37909 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
37910 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
37911 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
37912 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
37913 .{ ._, ._dqa, .mov, .lea(.tmp1x), .tmp5x, ._, ._ },
37914 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
37915 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
37916 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37917 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
37918 } },
37919 }, .{
37920 .required_cc_abi = .win64,
37921 .required_features = .{ .sse, null, null, null },
37922 .src_constraints = .{
37923 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37924 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
37925 .any,
37926 },
37927 .patterns = &.{
37928 .{ .src = .{ .to_mem, .to_mem, .none } },
37929 },
37930 .call_frame = .{ .alignment = .@"16" },
37931 .extra_temps = .{
37932 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
37933 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37934 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
37935 .{ .type = .usize, .kind = .{ .extern_func = "__divtf3" } },
37936 .{ .type = .f128, .kind = .mem },
37937 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
37938 .{ .type = .usize, .kind = .{ .extern_func = "floorq" } },
37939 .unused,
37940 .unused,
37941 .unused,
37942 .unused,
37943 },
37944 .dst_temps = .{ .mem, .unused },
37945 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
37946 .each = .{ .once = &.{
37947 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
37948 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0x, .tmp0), ._, ._ },
37949 .{ ._, ._, .lea, .tmp2p, .memi(.src1x, .tmp0), ._, ._ },
37950 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
37951 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
37952 .{ ._, ._ps, .mova, .lea(.tmp1x), .tmp5x, ._, ._ },
37953 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
37954 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
37955 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
37956 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3758137957 } },
3758237958 } })) catch |err| switch (err) {
3758337959 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -39080,6 +39456,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3908039456 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
3908139457 } },
3908239458 }, .{
39459 .required_cc_abi = .sysv64,
3908339460 .required_features = .{ .sse, null, null, null },
3908439461 .src_constraints = .{
3908539462 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -39113,6 +39490,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3911339490 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
3911439491 } },
3911539492 }, .{
39493 .required_cc_abi = .win64,
39494 .required_features = .{ .sse, null, null, null },
39495 .src_constraints = .{
39496 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
39497 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
39498 .any,
39499 },
39500 .patterns = &.{
39501 .{ .src = .{ .to_mem, .to_mem, .none } },
39502 },
39503 .call_frame = .{ .alignment = .@"16" },
39504 .extra_temps = .{
39505 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39506 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
39507 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
39508 .unused,
39509 .unused,
39510 .unused,
39511 .unused,
39512 .unused,
39513 .unused,
39514 .unused,
39515 .unused,
39516 },
39517 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
39518 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
39519 .each = .{ .once = &.{
39520 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
39521 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
39522 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
39523 } },
39524 }, .{
39525 .required_cc_abi = .sysv64,
3911639526 .required_features = .{ .avx, null, null, null },
3911739527 .src_constraints = .{
3911839528 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -39124,7 +39534,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3912439534 },
3912539535 .call_frame = .{ .alignment = .@"16" },
3912639536 .extra_temps = .{
39127 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
39537 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3912839538 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3912939539 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3913039540 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
......@@ -39139,15 +39549,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3913939549 .dst_temps = .{ .mem, .unused },
3914039550 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3914139551 .each = .{ .once = &.{
39142 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
39143 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
39144 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
39552 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39553 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
39554 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3914539555 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39146 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
39147 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39148 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
39556 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
39557 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39558 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3914939559 } },
3915039560 }, .{
39561 .required_cc_abi = .sysv64,
3915139562 .required_features = .{ .sse2, null, null, null },
3915239563 .src_constraints = .{
3915339564 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -39159,7 +39570,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3915939570 },
3916039571 .call_frame = .{ .alignment = .@"16" },
3916139572 .extra_temps = .{
39162 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
39573 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3916339574 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3916439575 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3916539576 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
......@@ -39174,15 +39585,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3917439585 .dst_temps = .{ .mem, .unused },
3917539586 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3917639587 .each = .{ .once = &.{
39177 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
39178 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
39179 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
39588 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39589 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
39590 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3918039591 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39181 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
39182 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39183 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
39592 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
39593 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39594 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3918439595 } },
3918539596 }, .{
39597 .required_cc_abi = .sysv64,
3918639598 .required_features = .{ .sse, null, null, null },
3918739599 .src_constraints = .{
3918839600 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -39194,7 +39606,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3919439606 },
3919539607 .call_frame = .{ .alignment = .@"16" },
3919639608 .extra_temps = .{
39197 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
39609 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
3919839610 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
3919939611 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
3920039612 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
......@@ -39209,13 +39621,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3920939621 .dst_temps = .{ .mem, .unused },
3921039622 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3921139623 .each = .{ .once = &.{
39212 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
39213 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
39214 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
39624 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39625 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
39626 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
3921539627 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39216 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
39217 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
39218 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
39628 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
39629 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39630 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
39631 } },
39632 }, .{
39633 .required_cc_abi = .win64,
39634 .required_features = .{ .avx, null, null, null },
39635 .src_constraints = .{
39636 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39637 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39638 .any,
39639 },
39640 .patterns = &.{
39641 .{ .src = .{ .to_mem, .to_mem, .none } },
39642 },
39643 .call_frame = .{ .alignment = .@"16" },
39644 .extra_temps = .{
39645 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
39646 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39647 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
39648 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
39649 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39650 .unused,
39651 .unused,
39652 .unused,
39653 .unused,
39654 .unused,
39655 .unused,
39656 },
39657 .dst_temps = .{ .mem, .unused },
39658 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
39659 .each = .{ .once = &.{
39660 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39661 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
39662 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
39663 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39664 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
39665 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39666 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
39667 } },
39668 }, .{
39669 .required_cc_abi = .win64,
39670 .required_features = .{ .sse2, null, null, null },
39671 .src_constraints = .{
39672 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39673 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39674 .any,
39675 },
39676 .patterns = &.{
39677 .{ .src = .{ .to_mem, .to_mem, .none } },
39678 },
39679 .call_frame = .{ .alignment = .@"16" },
39680 .extra_temps = .{
39681 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
39682 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39683 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
39684 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
39685 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39686 .unused,
39687 .unused,
39688 .unused,
39689 .unused,
39690 .unused,
39691 .unused,
39692 },
39693 .dst_temps = .{ .mem, .unused },
39694 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
39695 .each = .{ .once = &.{
39696 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39697 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
39698 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
39699 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39700 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
39701 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39702 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
39703 } },
39704 }, .{
39705 .required_cc_abi = .win64,
39706 .required_features = .{ .sse, null, null, null },
39707 .src_constraints = .{
39708 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39709 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
39710 .any,
39711 },
39712 .patterns = &.{
39713 .{ .src = .{ .to_mem, .to_mem, .none } },
39714 },
39715 .call_frame = .{ .alignment = .@"16" },
39716 .extra_temps = .{
39717 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
39718 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39719 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
39720 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
39721 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39722 .unused,
39723 .unused,
39724 .unused,
39725 .unused,
39726 .unused,
39727 .unused,
39728 },
39729 .dst_temps = .{ .mem, .unused },
39730 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
39731 .each = .{ .once = &.{
39732 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
39733 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
39734 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
39735 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
39736 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
39737 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
39738 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
3921939739 } },
3922039740 } }) catch |err| switch (err) {
3922139741 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -39525,7 +40045,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3952540045 } },
3952640046 }, .{
3952740047 .required_cc_abi = .sysv64,
39528 .required_features = .{ .cmov, null, null, null },
40048 .required_features = .{ .@"64bit", .cmov, null, null },
3952940049 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
3953040050 .patterns = &.{
3953140051 .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } },
......@@ -39565,6 +40085,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3956540085 } },
3956640086 }, .{
3956740087 .required_cc_abi = .sysv64,
40088 .required_features = .{ .@"64bit", null, null, null },
3956840089 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
3956940090 .patterns = &.{
3957040091 .{ .src = .{ .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .to_mem, .none } },
......@@ -39601,70 +40122,344 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3960140122 .{ ._, ._, .adc, .dst0q1, .src0q0, ._, ._ },
3960240123 } },
3960340124 }, .{
39604 .required_cc_abi = .sysv64,
39605 .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any },
40125 .required_cc_abi = .win64,
40126 .required_features = .{ .@"64bit", .cmov, .avx, null },
40127 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
3960640128 .patterns = &.{
39607 .{ .src = .{
39608 .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } },
39609 .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } },
39610 .none,
39611 } },
40129 .{ .src = .{ .to_mem, .to_mem, .none } },
3961240130 },
3961340131 .call_frame = .{ .alignment = .@"16" },
3961440132 .extra_temps = .{
39615 .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } },
40133 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40134 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40135 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40136 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40137 .{ .type = .u64, .kind = .{ .reg = .rax } },
40138 .{ .type = .u64, .kind = .{ .reg = .r10 } },
3961640139 .unused,
3961740140 .unused,
3961840141 .unused,
3961940142 .unused,
3962040143 .unused,
40144 },
40145 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40146 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40147 .each = .{ .once = &.{
40148 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40149 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40150 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40151 .{ ._, .v_q, .mov, .dst0q0, .tmp1x, ._, ._ },
40152 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40153 .{ ._, .vp_q, .extr, .dst0q1, .tmp1x, .ui(1), ._ },
40154 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40155 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40156 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40157 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40158 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
40159 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40160 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40161 .{ ._, ._nae, .cmov, .tmp2q, .tmp5q, ._, ._ },
40162 .{ ._, ._ae, .cmov, .tmp5q, .mem(.src1q), ._, ._ },
40163 .{ ._, ._, .add, .dst0q0, .tmp5q, ._, ._ },
40164 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
40165 } },
40166 }, .{
40167 .required_cc_abi = .win64,
40168 .required_features = .{ .@"64bit", .cmov, .sse4_1, null },
40169 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40170 .patterns = &.{
40171 .{ .src = .{ .to_mem, .to_mem, .none } },
40172 },
40173 .call_frame = .{ .alignment = .@"16" },
40174 .extra_temps = .{
40175 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40176 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40177 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40178 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40179 .{ .type = .u64, .kind = .{ .reg = .rax } },
40180 .{ .type = .u64, .kind = .{ .reg = .r10 } },
3962140181 .unused,
3962240182 .unused,
3962340183 .unused,
3962440184 .unused,
3962540185 .unused,
3962640186 },
39627 .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40187 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
3962840188 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3962940189 .each = .{ .once = &.{
40190 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40191 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
3963040192 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40193 .{ ._, ._q, .mov, .dst0q0, .tmp1x, ._, ._ },
40194 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40195 .{ ._, .p_q, .extr, .dst0q1, .tmp1x, .ui(1), ._ },
40196 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40197 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40198 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40199 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40200 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
40201 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40202 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40203 .{ ._, ._nae, .cmov, .tmp2q, .tmp5q, ._, ._ },
40204 .{ ._, ._ae, .cmov, .tmp5q, .mem(.src1q), ._, ._ },
40205 .{ ._, ._, .add, .dst0q0, .tmp5q, ._, ._ },
40206 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
3963140207 } },
3963240208 }, .{
3963340209 .required_cc_abi = .win64,
39634 .required_features = .{ .sse, null, null, null },
39635 .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any },
40210 .required_features = .{ .@"64bit", .cmov, .sse2, null },
40211 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
3963640212 .patterns = &.{
3963740213 .{ .src = .{ .to_mem, .to_mem, .none } },
3963840214 },
3963940215 .call_frame = .{ .alignment = .@"16" },
3964040216 .extra_temps = .{
39641 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
39642 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
39643 .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } },
40217 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40218 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40219 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40220 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40221 .{ .type = .u64, .kind = .{ .reg = .rax } },
40222 .{ .type = .u64, .kind = .{ .reg = .r10 } },
3964440223 .unused,
3964540224 .unused,
3964640225 .unused,
3964740226 .unused,
3964840227 .unused,
40228 },
40229 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40230 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40231 .each = .{ .once = &.{
40232 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40233 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40234 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40235 .{ ._, ._q, .mov, .dst0q0, .tmp1x, ._, ._ },
40236 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40237 .{ ._, .p_d, .shuf, .tmp1x, .tmp1x, .ui(0b11_10_11_10), ._ },
40238 .{ ._, ._q, .mov, .dst0q1, .tmp1x, ._, ._ },
40239 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40240 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40241 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40242 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40243 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
40244 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40245 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40246 .{ ._, ._nae, .cmov, .tmp2q, .tmp5q, ._, ._ },
40247 .{ ._, ._ae, .cmov, .tmp5q, .mem(.src1q), ._, ._ },
40248 .{ ._, ._, .add, .dst0q0, .tmp5q, ._, ._ },
40249 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
40250 } },
40251 }, .{
40252 .required_cc_abi = .win64,
40253 .required_features = .{ .@"64bit", .cmov, .sse, null },
40254 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40255 .patterns = &.{
40256 .{ .src = .{ .to_mem, .to_mem, .none } },
40257 },
40258 .call_frame = .{ .alignment = .@"16" },
40259 .extra_temps = .{
40260 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40261 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40262 .{ .type = .i128, .kind = .mem },
40263 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40264 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40265 .{ .type = .u64, .kind = .{ .reg = .rax } },
40266 .{ .type = .u64, .kind = .{ .reg = .r10 } },
40267 .unused,
3964940268 .unused,
3965040269 .unused,
3965140270 .unused,
3965240271 },
39653 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40272 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
3965440273 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3965540274 .each = .{ .once = &.{
39656 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
39657 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
39658 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
40275 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40276 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40277 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40278 .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ },
40279 .{ ._, ._, .mov, .tmp3q, .memd(.src1q, 8), ._, ._ },
40280 .{ ._, ._, .mov, .dst0q0, .mem(.tmp2q), ._, ._ },
40281 .{ ._, ._, .mov, .dst0q1, .memd(.tmp2q, 8), ._, ._ },
40282 .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ },
40283 .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ },
40284 .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ },
40285 .{ ._, ._, .xor, .tmp5q, .dst0q1, ._, ._ },
40286 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
40287 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40288 .{ ._, ._, .sbb, .tmp5q, .tmp4q, ._, ._ },
40289 .{ ._, ._nae, .cmov, .tmp3q, .tmp6q, ._, ._ },
40290 .{ ._, ._ae, .cmov, .tmp6q, .mem(.src1q), ._, ._ },
40291 .{ ._, ._, .add, .dst0q0, .tmp6q, ._, ._ },
40292 .{ ._, ._, .adc, .dst0q1, .tmp3q, ._, ._ },
3965940293 } },
3966040294 }, .{
3966140295 .required_cc_abi = .win64,
39662 .required_features = .{ .sse, null, null, null },
40296 .required_features = .{ .@"64bit", .avx, null, null },
40297 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40298 .patterns = &.{
40299 .{ .src = .{ .to_mem, .to_mem, .none } },
40300 },
40301 .call_frame = .{ .alignment = .@"16" },
40302 .extra_temps = .{
40303 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40304 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40305 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40306 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40307 .{ .type = .u64, .kind = .{ .reg = .rax } },
40308 .unused,
40309 .unused,
40310 .unused,
40311 .unused,
40312 .unused,
40313 .unused,
40314 },
40315 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40316 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40317 .each = .{ .once = &.{
40318 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40319 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40320 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40321 .{ ._, .v_q, .mov, .dst0q0, .tmp1x, ._, ._ },
40322 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40323 .{ ._, .vp_q, .extr, .dst0q1, .tmp1x, .ui(1), ._ },
40324 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40325 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40326 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40327 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40328 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40329 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40330 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
40331 .{ ._, ._, .add, .dst0q0, .mem(.src1x), ._, ._ },
40332 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
40333 } },
40334 }, .{
40335 .required_cc_abi = .win64,
40336 .required_features = .{ .@"64bit", .sse4_1, null, null },
40337 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40338 .patterns = &.{
40339 .{ .src = .{ .to_mem, .to_mem, .none } },
40340 },
40341 .call_frame = .{ .alignment = .@"16" },
40342 .extra_temps = .{
40343 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40344 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40345 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40346 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40347 .{ .type = .u64, .kind = .{ .reg = .rax } },
40348 .unused,
40349 .unused,
40350 .unused,
40351 .unused,
40352 .unused,
40353 .unused,
40354 },
40355 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40356 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40357 .each = .{ .once = &.{
40358 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40359 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40360 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40361 .{ ._, ._q, .mov, .dst0q0, .tmp1x, ._, ._ },
40362 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40363 .{ ._, .p_q, .extr, .dst0q1, .tmp1x, .ui(1), ._ },
40364 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40365 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40366 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40367 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40368 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40369 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40370 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
40371 .{ ._, ._, .add, .dst0q0, .mem(.src1x), ._, ._ },
40372 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
40373 } },
40374 }, .{
40375 .required_cc_abi = .win64,
40376 .required_features = .{ .@"64bit", .sse2, null, null },
40377 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40378 .patterns = &.{
40379 .{ .src = .{ .to_mem, .to_mem, .none } },
40380 },
40381 .call_frame = .{ .alignment = .@"16" },
40382 .extra_temps = .{
40383 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40384 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40385 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40386 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40387 .{ .type = .u64, .kind = .{ .reg = .rax } },
40388 .unused,
40389 .unused,
40390 .unused,
40391 .unused,
40392 .unused,
40393 .unused,
40394 },
40395 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40396 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40397 .each = .{ .once = &.{
40398 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40399 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40400 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40401 .{ ._, ._q, .mov, .dst0q0, .tmp1x, ._, ._ },
40402 .{ ._, ._, .mov, .tmp2q, .memd(.src1q, 8), ._, ._ },
40403 .{ ._, .p_d, .shuf, .tmp1x, .tmp1x, .ui(0b11_10_11_10), ._ },
40404 .{ ._, ._q, .mov, .dst0q1, .tmp1x, ._, ._ },
40405 .{ ._, ._, .mov, .tmp3q, .ua(.src0, .add_smin), ._, ._ },
40406 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
40407 .{ ._, ._, .@"and", .tmp4q, .tmp3q, ._, ._ },
40408 .{ ._, ._, .xor, .tmp4q, .dst0q1, ._, ._ },
40409 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40410 .{ ._, ._, .sbb, .tmp4q, .tmp3q, ._, ._ },
40411 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
40412 .{ ._, ._, .add, .dst0q0, .mem(.src1x), ._, ._ },
40413 .{ ._, ._, .adc, .dst0q1, .tmp2q, ._, ._ },
40414 } },
40415 }, .{
40416 .required_cc_abi = .win64,
40417 .required_features = .{ .@"64bit", .sse, null, null },
40418 .src_constraints = .{ .{ .signed_int = .xword }, .{ .signed_int = .xword }, .any },
40419 .patterns = &.{
40420 .{ .src = .{ .to_mem, .to_mem, .none } },
40421 },
40422 .call_frame = .{ .alignment = .@"16" },
40423 .extra_temps = .{
40424 .{ .type = .usize, .kind = .{ .extern_func = "__modti3" } },
40425 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40426 .{ .type = .i128, .kind = .mem },
40427 .{ .type = .u64, .kind = .{ .reg = .r8 } },
40428 .{ .type = .u64, .kind = .{ .reg = .r9 } },
40429 .{ .type = .u64, .kind = .{ .reg = .rax } },
40430 .unused,
40431 .unused,
40432 .unused,
40433 .unused,
40434 .unused,
40435 },
40436 .dst_temps = .{ .{ .param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40437 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40438 .each = .{ .once = &.{
40439 .{ ._, ._, .lea, .dst0q0, .mem(.src0), ._, ._ },
40440 .{ ._, ._, .lea, .dst0q1, .mem(.src1), ._, ._ },
40441 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
40442 .{ ._, ._ps, .mova, .mem(.tmp2x), .tmp1x, ._, ._ },
40443 .{ ._, ._, .mov, .tmp3q, .memd(.src1q, 8), ._, ._ },
40444 .{ ._, ._, .mov, .dst0q0, .mem(.tmp2q), ._, ._ },
40445 .{ ._, ._, .mov, .dst0q1, .memd(.tmp2q, 8), ._, ._ },
40446 .{ ._, ._, .mov, .tmp4q, .ua(.src0, .add_smin), ._, ._ },
40447 .{ ._, ._, .mov, .tmp5q, .tmp3q, ._, ._ },
40448 .{ ._, ._, .@"and", .tmp5q, .tmp4q, ._, ._ },
40449 .{ ._, ._, .xor, .tmp5q, .dst0q1, ._, ._ },
40450 .{ ._, ._, .cmp, .dst0q0, .si(1), ._, ._ },
40451 .{ ._, ._, .sbb, .tmp5q, .tmp4q, ._, ._ },
40452 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
40453 .{ ._, ._, .add, .dst0q0, .mem(.src1x), ._, ._ },
40454 .{ ._, ._, .adc, .dst0q1, .tmp3q, ._, ._ },
40455 } },
40456 }, .{
40457 .required_cc_abi = .sysv64,
3966340458 .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any },
3966440459 .patterns = &.{
3966540460 .{ .src = .{
39666 .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } },
39667 .{ .to_param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } },
40461 .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } },
40462 .{ .to_param_gpr_pair = .{ .cc = .ccc, .after = 2, .at = 2 } },
3966840463 .none,
3966940464 } },
3967040465 },
......@@ -39682,11 +40477,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3968240477 .unused,
3968340478 .unused,
3968440479 },
39685 .dst_temps = .{ .{ .ref = .src0 }, .unused },
40480 .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
3968640481 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
3968740482 .each = .{ .once = &.{
3968840483 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
3968940484 } },
40485 }, .{
40486 .required_cc_abi = .win64,
40487 .required_features = .{ .sse, null, null, null },
40488 .src_constraints = .{ .{ .unsigned_int = .xword }, .{ .unsigned_int = .xword }, .any },
40489 .patterns = &.{
40490 .{ .src = .{ .to_mem, .to_mem, .none } },
40491 },
40492 .call_frame = .{ .alignment = .@"16" },
40493 .extra_temps = .{
40494 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
40495 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
40496 .{ .type = .usize, .kind = .{ .extern_func = "__umodti3" } },
40497 .unused,
40498 .unused,
40499 .unused,
40500 .unused,
40501 .unused,
40502 .unused,
40503 .unused,
40504 .unused,
40505 },
40506 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
40507 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
40508 .each = .{ .once = &.{
40509 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
40510 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
40511 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
40512 } },
3969040513 }, .{
3969140514 .required_features = .{ .@"64bit", null, null, null },
3969240515 .src_constraints = .{
......@@ -41082,8 +41905,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4108241905 .{ .type = .f64, .kind = .{ .reg = .rdx } },
4108341906 .{ .type = .f64, .kind = .mem },
4108441907 .{ .type = .f64, .kind = .{ .reg = .rax } },
41085 .{ .type = .f64, .kind = .{ .reg = .st6 } },
4108641908 .{ .type = .f64, .kind = .{ .reg = .st7 } },
41909 .{ .type = .f64, .kind = .{ .reg = .st6 } },
4108741910 .unused,
4108841911 .unused,
4108941912 },
......@@ -41130,13 +41953,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4113041953 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4113141954 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4113241955 .{ .type = .f80, .kind = .{ .reg = .st7 } },
41956 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4113341957 .{ .type = .f80, .kind = .{ .reg = .rax } },
4113441958 .unused,
4113541959 .unused,
4113641960 .unused,
4113741961 .unused,
4113841962 .unused,
41139 .unused,
4114041963 },
4114141964 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4114241965 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41145,17 +41968,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4114541968 .{ ._, .v_dqa, .mov, .mem(.tmp1x), .tmp0x, ._, ._ },
4114641969 .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ },
4114741970 .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
41971 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4114841972 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41149 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
41973 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4115041974 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41151 .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ },
41152 .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ },
41153 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
41975 .{ ._, ._, .movzx, .tmp5d, .memd(.src1w, 8), ._, ._ },
41976 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
41977 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4115441978 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41155 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
41979 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4115641980 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41157 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
41981 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
4115841982 .{ ._, .f_p, .add, ._, ._, ._, ._ },
41983 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4115941984 } },
4116041985 }, .{
4116141986 .required_abi = .gnu,
......@@ -41175,13 +42000,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4117542000 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4117642001 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4117742002 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42003 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4117842004 .{ .type = .f80, .kind = .{ .reg = .rax } },
4117942005 .unused,
4118042006 .unused,
4118142007 .unused,
4118242008 .unused,
4118342009 .unused,
41184 .unused,
4118542010 },
4118642011 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4118742012 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41191,16 +42016,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4119142016 .{ ._, .v_dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ },
4119242017 .{ ._, .v_dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
4119342018 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41194 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
42019 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
42020 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4119542021 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41196 .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ },
41197 .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ },
41198 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
42022 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
42023 .{ ._, ._, .@"and", .tmp5w, .memd(.src1w, 8), ._, ._ },
42024 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4119942025 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41200 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
42026 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4120142027 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41202 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
42028 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
4120342029 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42030 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4120442031 } },
4120542032 }, .{
4120642033 .required_abi = .gnu,
......@@ -41220,13 +42047,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4122042047 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4122142048 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4122242049 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42050 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4122342051 .{ .type = .f80, .kind = .{ .reg = .rax } },
4122442052 .unused,
4122542053 .unused,
4122642054 .unused,
4122742055 .unused,
4122842056 .unused,
41229 .unused,
4123042057 },
4123142058 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4123242059 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41236,16 +42063,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4123642063 .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ },
4123742064 .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
4123842065 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41239 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
42066 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
42067 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4124042068 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41241 .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ },
41242 .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ },
41243 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
42069 .{ ._, ._, .movzx, .tmp5d, .memd(.src1w, 8), ._, ._ },
42070 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42071 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4124442072 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41245 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
42073 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4124642074 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41247 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
42075 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
4124842076 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42077 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4124942078 } },
4125042079 }, .{
4125142080 .required_abi = .gnu,
......@@ -41265,13 +42094,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4126542094 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4126642095 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4126742096 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42097 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4126842098 .{ .type = .f80, .kind = .{ .reg = .rax } },
4126942099 .unused,
4127042100 .unused,
4127142101 .unused,
4127242102 .unused,
4127342103 .unused,
41274 .unused,
4127542104 },
4127642105 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4127742106 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41281,16 +42110,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4128142110 .{ ._, ._dqa, .mov, .tmp0x, .mem(.src1x), ._, ._ },
4128242111 .{ ._, ._dqa, .mov, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
4128342112 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41284 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
42113 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
42114 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4128542115 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41286 .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ },
41287 .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ },
41288 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
42116 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
42117 .{ ._, ._, .@"and", .tmp5w, .memd(.src1w, 8), ._, ._ },
42118 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4128942119 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41290 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
42120 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4129142121 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41292 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
42122 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
4129342123 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42124 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4129442125 } },
4129542126 }, .{
4129642127 .required_abi = .gnu,
......@@ -41310,13 +42141,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4131042141 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4131142142 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4131242143 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42144 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4131342145 .{ .type = .f80, .kind = .{ .reg = .rax } },
4131442146 .unused,
4131542147 .unused,
4131642148 .unused,
4131742149 .unused,
4131842150 .unused,
41319 .unused,
4132042151 },
4132142152 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4132242153 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41326,16 +42157,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4132642157 .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ },
4132742158 .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
4132842159 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41329 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
42160 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
42161 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4133042162 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41331 .{ ._, ._, .movzx, .tmp4d, .memd(.tmp1w, 16 + 8), ._, ._ },
41332 .{ ._, ._, .@"and", .tmp4w, .sa(.src0, .add_smin), ._, ._ },
41333 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
42163 .{ ._, ._, .movzx, .tmp5d, .memd(.src1w, 8), ._, ._ },
42164 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42165 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4133442166 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41335 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
42167 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4133642168 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41337 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
42169 .{ ._, .f_, .ld, .memd(.src1t, 16), ._, ._, ._ },
4133842170 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42171 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4133942172 } },
4134042173 }, .{
4134142174 .required_abi = .gnu,
......@@ -41355,13 +42188,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4135542188 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4135642189 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4135742190 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42191 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4135842192 .{ .type = .f80, .kind = .{ .reg = .rax } },
4135942193 .unused,
4136042194 .unused,
4136142195 .unused,
4136242196 .unused,
4136342197 .unused,
41364 .unused,
4136542198 },
4136642199 .dst_temps = .{ .{ .reg = .st0 }, .unused },
4136742200 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41371,16 +42204,106 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4137142204 .{ ._, ._ps, .mova, .tmp0x, .mem(.src1x), ._, ._ },
4137242205 .{ ._, ._ps, .mova, .memd(.tmp1x, 16), .tmp0x, ._, ._ },
4137342206 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
41374 .{ ._, .f_, .ld, .dst0t, ._, ._, ._ },
42207 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
42208 .{ ._, .f_, .ld, .tmp3t, ._, ._, ._ },
4137542209 .{ ._, .f_p, .st, .mem(.tmp1t), ._, ._, ._ },
41376 .{ ._, ._, .mov, .tmp4d, .sa(.src0, .add_smin), ._, ._ },
41377 .{ ._, ._, .@"and", .tmp4w, .memd(.tmp1w, 16 + 8), ._, ._ },
41378 .{ ._, ._, .xor, .tmp4w, .memd(.tmp1w, 8), ._, ._ },
42210 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
42211 .{ ._, ._, .@"and", .tmp5w, .memd(.src1w, 8), ._, ._ },
42212 .{ ._, ._, .xor, .tmp5w, .memd(.tmp1w, 8), ._, ._ },
4137942213 .{ ._, ._, .cmp, .mem(.tmp1q), .si(1), ._, ._ },
41380 .{ ._, ._, .sbb, .tmp4w, .sa(.src0, .add_smin), ._, ._ },
42214 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
4138142215 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
41382 .{ ._, .f_, .ld, .memd(.tmp1t, 16), ._, ._, ._ },
42216 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
4138342217 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42218 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
42219 } },
42220 }, .{
42221 .required_abi = .gnu,
42222 .required_cc_abi = .win64,
42223 .required_features = .{ .@"64bit", .x87, .fast_imm16, null },
42224 .src_constraints = .{
42225 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
42226 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
42227 .any,
42228 },
42229 .patterns = &.{
42230 .{ .src = .{ .to_mem, .to_mem, .none } },
42231 },
42232 .call_frame = .{ .alignment = .@"16" },
42233 .extra_temps = .{
42234 .{ .type = .f80, .kind = .mem },
42235 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42236 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42237 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
42238 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
42239 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42240 .{ .type = .f80, .kind = .{ .reg = .st6 } },
42241 .{ .type = .f80, .kind = .{ .reg = .rax } },
42242 .unused,
42243 .unused,
42244 .unused,
42245 },
42246 .dst_temps = .{ .{ .reg = .st0 }, .unused },
42247 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42248 .each = .{ .once = &.{
42249 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
42250 .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ },
42251 .{ ._, ._, .lea, .tmp3p, .mem(.src1), ._, ._ },
42252 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
42253 .{ ._, .f_, .ld, .mem(.tmp0t), ._, ._, ._ },
42254 .{ ._, ._, .movzx, .tmp7d, .memd(.src1w, 8), ._, ._ },
42255 .{ ._, ._, .@"and", .tmp7w, .sa(.src0, .add_smin), ._, ._ },
42256 .{ ._, ._, .xor, .tmp7w, .memd(.tmp0w, 8), ._, ._ },
42257 .{ ._, ._, .cmp, .mem(.tmp0q), .si(1), ._, ._ },
42258 .{ ._, ._, .sbb, .tmp7w, .sa(.src0, .add_smin), ._, ._ },
42259 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
42260 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
42261 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42262 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
42263 } },
42264 }, .{
42265 .required_abi = .gnu,
42266 .required_cc_abi = .win64,
42267 .required_features = .{ .@"64bit", .x87, null, null },
42268 .src_constraints = .{
42269 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
42270 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
42271 .any,
42272 },
42273 .patterns = &.{
42274 .{ .src = .{ .to_mem, .to_mem, .none } },
42275 },
42276 .call_frame = .{ .alignment = .@"16" },
42277 .extra_temps = .{
42278 .{ .type = .f80, .kind = .mem },
42279 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42280 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42281 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
42282 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
42283 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42284 .{ .type = .f80, .kind = .{ .reg = .st6 } },
42285 .{ .type = .f80, .kind = .{ .reg = .rax } },
42286 .unused,
42287 .unused,
42288 .unused,
42289 },
42290 .dst_temps = .{ .{ .reg = .st0 }, .unused },
42291 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42292 .each = .{ .once = &.{
42293 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
42294 .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ },
42295 .{ ._, ._, .lea, .tmp3p, .mem(.src1), ._, ._ },
42296 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
42297 .{ ._, .f_, .ld, .mem(.tmp0t), ._, ._, ._ },
42298 .{ ._, ._, .mov, .tmp7d, .sa(.src0, .add_smin), ._, ._ },
42299 .{ ._, ._, .@"and", .tmp7w, .memd(.src1w, 8), ._, ._ },
42300 .{ ._, ._, .xor, .tmp7w, .memd(.tmp0w, 8), ._, ._ },
42301 .{ ._, ._, .cmp, .mem(.tmp0q), .si(1), ._, ._ },
42302 .{ ._, ._, .sbb, .tmp7w, .sa(.src0, .add_smin), ._, ._ },
42303 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
42304 .{ ._, .f_, .ld, .mem(.src1t), ._, ._, ._ },
42305 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42306 .{ .pseudo, .f_cstp, .in, ._, ._, ._, ._ },
4138442307 } },
4138542308 }, .{
4138642309 .required_abi = .gnu,
......@@ -41401,12 +42324,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4140142324 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4140242325 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4140342326 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42327 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4140442328 .{ .type = .f80, .kind = .{ .reg = .rax } },
4140542329 .unused,
4140642330 .unused,
4140742331 .unused,
4140842332 .unused,
41409 .unused,
4141042333 },
4141142334 .dst_temps = .{ .mem, .unused },
4141242335 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41420,13 +42343,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4142042343 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4142142344 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4142242345 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41423 .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ },
41424 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
41425 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42346 .{ ._, ._, .movzx, .tmp6d, .memid(.src1w, .tmp0, 8), ._, ._ },
42347 .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ },
42348 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4142642349 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41427 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42350 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4142842351 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41429 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42352 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4143042353 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4143142354 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4143242355 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
......@@ -41451,12 +42374,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4145142374 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4145242375 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4145342376 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42377 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4145442378 .{ .type = .f80, .kind = .{ .reg = .rax } },
4145542379 .unused,
4145642380 .unused,
4145742381 .unused,
4145842382 .unused,
41459 .unused,
4146042383 },
4146142384 .dst_temps = .{ .mem, .unused },
4146242385 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41470,13 +42393,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4147042393 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4147142394 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4147242395 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41473 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
41474 .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ },
41475 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42396 .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_smin), ._, ._ },
42397 .{ ._, ._, .@"and", .tmp6w, .memid(.src1w, .tmp0, 8), ._, ._ },
42398 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4147642399 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41477 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42400 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4147842401 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41479 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42402 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4148042403 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4148142404 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4148242405 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
......@@ -41501,12 +42424,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4150142424 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4150242425 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4150342426 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42427 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4150442428 .{ .type = .f80, .kind = .{ .reg = .rax } },
4150542429 .unused,
4150642430 .unused,
4150742431 .unused,
4150842432 .unused,
41509 .unused,
4151042433 },
4151142434 .dst_temps = .{ .mem, .unused },
4151242435 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41520,13 +42443,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4152042443 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4152142444 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4152242445 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41523 .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ },
41524 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
41525 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42446 .{ ._, ._, .movzx, .tmp6d, .memid(.src1w, .tmp0, 8), ._, ._ },
42447 .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ },
42448 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4152642449 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41527 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42450 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4152842451 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41529 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42452 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4153042453 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4153142454 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4153242455 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
......@@ -41551,12 +42474,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4155142474 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4155242475 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4155342476 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42477 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4155442478 .{ .type = .f80, .kind = .{ .reg = .rax } },
4155542479 .unused,
4155642480 .unused,
4155742481 .unused,
4155842482 .unused,
41559 .unused,
4156042483 },
4156142484 .dst_temps = .{ .mem, .unused },
4156242485 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41570,13 +42493,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4157042493 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4157142494 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4157242495 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41573 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
41574 .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ },
41575 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42496 .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_smin), ._, ._ },
42497 .{ ._, ._, .@"and", .tmp6w, .memid(.src1w, .tmp0, 8), ._, ._ },
42498 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4157642499 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41577 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42500 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4157842501 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41579 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42502 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4158042503 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4158142504 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4158242505 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
......@@ -41601,12 +42524,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4160142524 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4160242525 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4160342526 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42527 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4160442528 .{ .type = .f80, .kind = .{ .reg = .rax } },
4160542529 .unused,
4160642530 .unused,
4160742531 .unused,
4160842532 .unused,
41609 .unused,
4161042533 },
4161142534 .dst_temps = .{ .mem, .unused },
4161242535 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41620,13 +42543,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4162042543 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4162142544 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4162242545 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41623 .{ ._, ._, .movzx, .tmp5d, .memd(.tmp2w, 16 + 8), ._, ._ },
41624 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
41625 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42546 .{ ._, ._, .movzx, .tmp6d, .memid(.src1w, .tmp0, 8), ._, ._ },
42547 .{ ._, ._, .@"and", .tmp6w, .sa(.src0, .add_smin), ._, ._ },
42548 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4162642549 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41627 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42550 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4162842551 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41629 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42552 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4163042553 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4163142554 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4163242555 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
......@@ -41651,12 +42574,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4165142574 .{ .type = .f80, .kind = .{ .frame = .call_frame } },
4165242575 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
4165342576 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42577 .{ .type = .f80, .kind = .{ .reg = .st6 } },
4165442578 .{ .type = .f80, .kind = .{ .reg = .rax } },
4165542579 .unused,
4165642580 .unused,
4165742581 .unused,
4165842582 .unused,
41659 .unused,
4166042583 },
4166142584 .dst_temps = .{ .mem, .unused },
4166242585 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
......@@ -41670,19 +42593,114 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4167042593 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
4167142594 .{ ._, .f_, .ld, .tmp4t, ._, ._, ._ },
4167242595 .{ ._, .f_p, .st, .mem(.tmp2t), ._, ._, ._ },
41673 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
41674 .{ ._, ._, .@"and", .tmp5w, .memd(.tmp2w, 16 + 8), ._, ._ },
41675 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42596 .{ ._, ._, .mov, .tmp6d, .sa(.src0, .add_smin), ._, ._ },
42597 .{ ._, ._, .@"and", .tmp6w, .memid(.src1w, .tmp0, 8), ._, ._ },
42598 .{ ._, ._, .xor, .tmp6w, .memd(.tmp2w, 8), ._, ._ },
4167642599 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
41677 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42600 .{ ._, ._, .sbb, .tmp6w, .sa(.src0, .add_smin), ._, ._ },
4167842601 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
41679 .{ ._, .f_, .ld, .memd(.tmp2t, 16), ._, ._, ._ },
42602 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
4168042603 .{ ._, .f_p, .add, ._, ._, ._, ._ },
4168142604 .{ .@"1:", .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
4168242605 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
4168342606 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4168442607 } },
4168542608 }, .{
42609 .required_abi = .gnu,
42610 .required_cc_abi = .win64,
42611 .required_features = .{ .@"64bit", .sse, .x87, .fast_imm16 },
42612 .src_constraints = .{
42613 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
42614 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
42615 .any,
42616 },
42617 .patterns = &.{
42618 .{ .src = .{ .to_mem, .to_mem, .none } },
42619 },
42620 .call_frame = .{ .alignment = .@"16" },
42621 .extra_temps = .{
42622 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42623 .{ .type = .usize, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42624 .{ .type = .usize, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42625 .{ .type = .usize, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 2, .at = 2 } } },
42626 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
42627 .{ .type = .f80, .kind = .{ .reg = .rax } },
42628 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42629 .{ .type = .f80, .kind = .{ .reg = .st6 } },
42630 .unused,
42631 .unused,
42632 .unused,
42633 },
42634 .dst_temps = .{ .mem, .unused },
42635 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42636 .each = .{ .once = &.{
42637 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
42638 .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ },
42639 .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
42640 .{ ._, ._, .lea, .tmp3p, .memi(.src1, .tmp0), ._, ._ },
42641 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
42642 .{ ._, ._, .movzx, .tmp5d, .memid(.src1w, .tmp0, 8), ._, ._ },
42643 .{ ._, ._, .@"and", .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42644 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42645 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
42646 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42647 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
42648 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
42649 .{ ._, .f_, .ld, .memi(.dst0t, .tmp0), ._, ._, ._ },
42650 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42651 .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ },
42652 .{ .@"1:", ._, .sub, .tmp0d, .si(16), ._, ._ },
42653 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
42654 } },
42655 }, .{
42656 .required_abi = .gnu,
42657 .required_cc_abi = .win64,
42658 .required_features = .{ .@"64bit", .sse, .x87, null },
42659 .src_constraints = .{
42660 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
42661 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
42662 .any,
42663 },
42664 .patterns = &.{
42665 .{ .src = .{ .to_mem, .to_mem, .none } },
42666 },
42667 .call_frame = .{ .alignment = .@"16" },
42668 .extra_temps = .{
42669 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
42670 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42671 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42672 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
42673 .{ .type = .usize, .kind = .{ .extern_func = "__fmodx" } },
42674 .{ .type = .f80, .kind = .{ .reg = .rax } },
42675 .{ .type = .f80, .kind = .{ .reg = .st7 } },
42676 .{ .type = .f80, .kind = .{ .reg = .st6 } },
42677 .unused,
42678 .unused,
42679 .unused,
42680 },
42681 .dst_temps = .{ .mem, .unused },
42682 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42683 .each = .{ .once = &.{
42684 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
42685 .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ },
42686 .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
42687 .{ ._, ._, .lea, .tmp3p, .memi(.src1, .tmp0), ._, ._ },
42688 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
42689 .{ ._, ._, .mov, .tmp5d, .sa(.src0, .add_smin), ._, ._ },
42690 .{ ._, ._, .@"and", .tmp5w, .memid(.src1w, .tmp0, 8), ._, ._ },
42691 .{ ._, ._, .xor, .tmp5w, .memd(.tmp2w, 8), ._, ._ },
42692 .{ ._, ._, .cmp, .mem(.tmp2q), .si(1), ._, ._ },
42693 .{ ._, ._, .sbb, .tmp5w, .sa(.src0, .add_smin), ._, ._ },
42694 .{ ._, ._nae, .j, .@"1f", ._, ._, ._ },
42695 .{ ._, .f_, .ld, .memi(.src1t, .tmp0), ._, ._, ._ },
42696 .{ ._, .f_, .ld, .memi(.dst0t, .tmp0), ._, ._, ._ },
42697 .{ ._, .f_p, .add, ._, ._, ._, ._ },
42698 .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ },
42699 .{ .@"1:", ._, .sub, .tmp0d, .si(16), ._, ._ },
42700 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
42701 } },
42702 }, .{
42703 .required_cc_abi = .sysv64,
4168642704 .required_features = .{ .@"64bit", .avx, null, null },
4168742705 .src_constraints = .{
4168842706 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41700,9 +42718,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4170042718 .extra_temps = .{
4170142719 .{ .type = .f128, .kind = .mem },
4170242720 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
41703 .{ .type = .f128, .kind = .{ .reg = .rcx } },
41704 .{ .type = .f128, .kind = .{ .reg = .rdx } },
41705 .{ .type = .f128, .kind = .{ .reg = .rax } },
42721 .{ .type = .u64, .kind = .{ .reg = .rcx } },
42722 .{ .type = .u64, .kind = .{ .reg = .rdx } },
42723 .{ .type = .u64, .kind = .{ .reg = .rax } },
4170642724 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4170742725 .unused,
4170842726 .unused,
......@@ -41728,6 +42746,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4172842746 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
4172942747 } },
4173042748 }, .{
42749 .required_cc_abi = .sysv64,
4173142750 .required_features = .{ .@"64bit", .sse4_1, null, null },
4173242751 .src_constraints = .{
4173342752 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41745,9 +42764,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4174542764 .extra_temps = .{
4174642765 .{ .type = .f128, .kind = .mem },
4174742766 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
41748 .{ .type = .f128, .kind = .{ .reg = .rcx } },
41749 .{ .type = .f128, .kind = .{ .reg = .rdx } },
41750 .{ .type = .f128, .kind = .{ .reg = .rax } },
42767 .{ .type = .u64, .kind = .{ .reg = .rcx } },
42768 .{ .type = .u64, .kind = .{ .reg = .rdx } },
42769 .{ .type = .u64, .kind = .{ .reg = .rax } },
4175142770 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4175242771 .unused,
4175342772 .unused,
......@@ -41773,6 +42792,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4177342792 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
4177442793 } },
4177542794 }, .{
42795 .required_cc_abi = .sysv64,
4177642796 .required_features = .{ .@"64bit", .sse2, null, null },
4177742797 .src_constraints = .{
4177842798 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41790,9 +42810,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4179042810 .extra_temps = .{
4179142811 .{ .type = .f128, .kind = .mem },
4179242812 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
41793 .{ .type = .f128, .kind = .{ .reg = .rcx } },
41794 .{ .type = .f128, .kind = .{ .reg = .rdx } },
41795 .{ .type = .f128, .kind = .{ .reg = .rax } },
42813 .{ .type = .u64, .kind = .{ .reg = .rcx } },
42814 .{ .type = .u64, .kind = .{ .reg = .rdx } },
42815 .{ .type = .u64, .kind = .{ .reg = .rax } },
4179642816 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4179742817 .unused,
4179842818 .unused,
......@@ -41805,8 +42825,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4180542825 .each = .{ .once = &.{
4180642826 .{ ._, ._dqa, .mov, .mem(.tmp0x), .src1x, ._, ._ },
4180742827 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
41808 .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ },
4180942828 .{ ._, .p_d, .shuf, .src1x, .dst0x, .ui(0b11_10_11_10), ._ },
42829 .{ ._, ._, .mov, .tmp2q, .ua(.src0, .add_smin), ._, ._ },
4181042830 .{ ._, ._q, .mov, .tmp3q, .src1x, ._, ._ },
4181142831 .{ ._, ._, .mov, .tmp4q, .tmp2q, ._, ._ },
4181242832 .{ ._, ._, .@"and", .tmp4q, .memd(.tmp0q, 8), ._, ._ },
......@@ -41819,6 +42839,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4181942839 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
4182042840 } },
4182142841 }, .{
42842 .required_cc_abi = .sysv64,
4182242843 .required_features = .{ .@"64bit", .sse, null, null },
4182342844 .src_constraints = .{
4182442845 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41836,9 +42857,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4183642857 .extra_temps = .{
4183742858 .{ .type = .f128, .kind = .mem },
4183842859 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
41839 .{ .type = .f128, .kind = .{ .reg = .rdx } },
42860 .{ .type = .u64, .kind = .{ .reg = .rdx } },
4184042861 .{ .type = .f128, .kind = .mem },
41841 .{ .type = .f128, .kind = .{ .reg = .rax } },
42862 .{ .type = .u64, .kind = .{ .reg = .rax } },
4184242863 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
4184342864 .unused,
4184442865 .unused,
......@@ -41863,6 +42884,186 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4186342884 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
4186442885 } },
4186542886 }, .{
42887 .required_cc_abi = .win64,
42888 .required_features = .{ .@"64bit", .avx, null, null },
42889 .src_constraints = .{
42890 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42891 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42892 .any,
42893 },
42894 .patterns = &.{
42895 .{ .src = .{ .to_mem, .to_mem, .none } },
42896 },
42897 .call_frame = .{ .alignment = .@"16" },
42898 .extra_temps = .{
42899 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42900 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42901 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
42902 .{ .type = .f128, .kind = .mem },
42903 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
42904 .{ .type = .u64, .kind = .{ .reg = .rax } },
42905 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
42906 .unused,
42907 .unused,
42908 .unused,
42909 .unused,
42910 },
42911 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
42912 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42913 .each = .{ .once = &.{
42914 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
42915 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
42916 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
42917 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ },
42918 .{ ._, .vp_q, .extr, .tmp1q, .dst0x, .ui(1), ._ },
42919 .{ ._, ._, .mov, .tmp5q, .tmp0q, ._, ._ },
42920 .{ ._, ._, .@"and", .tmp5q, .memd(.src1q, 8), ._, ._ },
42921 .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ },
42922 .{ ._, .v_q, .mov, .tmp1q, .dst0x, ._, ._ },
42923 .{ ._, ._, .cmp, .tmp1q, .si(1), ._, ._ },
42924 .{ ._, ._, .sbb, .tmp5q, .tmp0q, ._, ._ },
42925 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
42926 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
42927 .{ ._, .v_dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
42928 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
42929 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
42930 } },
42931 }, .{
42932 .required_cc_abi = .win64,
42933 .required_features = .{ .@"64bit", .sse4_1, null, null },
42934 .src_constraints = .{
42935 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42936 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42937 .any,
42938 },
42939 .patterns = &.{
42940 .{ .src = .{ .to_mem, .to_mem, .none } },
42941 },
42942 .call_frame = .{ .alignment = .@"16" },
42943 .extra_temps = .{
42944 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42945 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42946 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
42947 .{ .type = .f128, .kind = .mem },
42948 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
42949 .{ .type = .u64, .kind = .{ .reg = .rax } },
42950 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
42951 .unused,
42952 .unused,
42953 .unused,
42954 .unused,
42955 },
42956 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
42957 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
42958 .each = .{ .once = &.{
42959 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
42960 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
42961 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
42962 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ },
42963 .{ ._, .p_q, .extr, .tmp1q, .dst0x, .ui(1), ._ },
42964 .{ ._, ._, .mov, .tmp5q, .tmp0q, ._, ._ },
42965 .{ ._, ._, .@"and", .tmp5q, .memd(.src1q, 8), ._, ._ },
42966 .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ },
42967 .{ ._, ._q, .mov, .tmp1q, .dst0x, ._, ._ },
42968 .{ ._, ._, .cmp, .tmp1q, .si(1), ._, ._ },
42969 .{ ._, ._, .sbb, .tmp5q, .tmp0q, ._, ._ },
42970 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
42971 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
42972 .{ ._, ._dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
42973 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
42974 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
42975 } },
42976 }, .{
42977 .required_cc_abi = .win64,
42978 .required_features = .{ .@"64bit", .sse2, null, null },
42979 .src_constraints = .{
42980 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42981 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
42982 .any,
42983 },
42984 .patterns = &.{
42985 .{ .src = .{ .to_mem, .to_mem, .none } },
42986 },
42987 .call_frame = .{ .alignment = .@"16" },
42988 .extra_temps = .{
42989 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
42990 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
42991 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
42992 .{ .type = .f128, .kind = .mem },
42993 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
42994 .{ .type = .u64, .kind = .{ .reg = .rax } },
42995 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
42996 .unused,
42997 .unused,
42998 .unused,
42999 .unused,
43000 },
43001 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
43002 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
43003 .each = .{ .once = &.{
43004 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
43005 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
43006 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
43007 .{ ._, .p_d, .shuf, .tmp4x, .dst0x, .ui(0b11_10_11_10), ._ },
43008 .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_smin), ._, ._ },
43009 .{ ._, ._q, .mov, .tmp1q, .tmp4x, ._, ._ },
43010 .{ ._, ._, .mov, .tmp5q, .tmp0q, ._, ._ },
43011 .{ ._, ._, .@"and", .tmp5q, .memd(.src1q, 8), ._, ._ },
43012 .{ ._, ._, .xor, .tmp5q, .tmp1q, ._, ._ },
43013 .{ ._, ._q, .mov, .tmp1q, .dst0x, ._, ._ },
43014 .{ ._, ._, .cmp, .tmp1q, .si(1), ._, ._ },
43015 .{ ._, ._, .sbb, .tmp5q, .tmp0q, ._, ._ },
43016 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
43017 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
43018 .{ ._, ._dqa, .mov, .lea(.tmp0x), .dst0x, ._, ._ },
43019 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
43020 .{ ._, ._, .call, .tmp6d, ._, ._, ._ },
43021 } },
43022 }, .{
43023 .required_cc_abi = .win64,
43024 .required_features = .{ .@"64bit", .sse, null, null },
43025 .src_constraints = .{
43026 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
43027 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
43028 .any,
43029 },
43030 .patterns = &.{
43031 .{ .src = .{ .to_mem, .to_mem, .none } },
43032 },
43033 .call_frame = .{ .alignment = .@"16" },
43034 .extra_temps = .{
43035 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
43036 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
43037 .{ .type = .usize, .kind = .{ .extern_func = "fmodq" } },
43038 .{ .type = .f128, .kind = .mem },
43039 .{ .type = .usize, .kind = .{ .reg = .rax } },
43040 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
43041 .unused,
43042 .unused,
43043 .unused,
43044 .unused,
43045 .unused,
43046 },
43047 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
43048 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
43049 .each = .{ .once = &.{
43050 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
43051 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
43052 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
43053 .{ ._, ._, .lea, .tmp0p, .mem(.tmp3), ._, ._ },
43054 .{ ._, ._, .mov, .tmp1q, .ua(.src0, .add_smin), ._, ._ },
43055 .{ ._, ._ps, .mova, .lea(.tmp0x), .dst0x, ._, ._ },
43056 .{ ._, ._, .mov, .tmp4q, .tmp1q, ._, ._ },
43057 .{ ._, ._, .@"and", .tmp4q, .memd(.src1q, 8), ._, ._ },
43058 .{ ._, ._, .xor, .tmp4q, .lead(.tmp0q, 8), ._, ._ },
43059 .{ ._, ._, .cmp, .lea(.tmp0q), .si(1), ._, ._ },
43060 .{ ._, ._, .sbb, .tmp4q, .tmp1q, ._, ._ },
43061 .{ ._, ._nae, .j, .@"0f", ._, ._, ._ },
43062 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
43063 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
43064 } },
43065 }, .{
43066 .required_cc_abi = .sysv64,
4186643067 .required_features = .{ .@"64bit", .avx, null, null },
4186743068 .src_constraints = .{
4186843069 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41909,6 +43110,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4190943110 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4191043111 } },
4191143112 }, .{
43113 .required_cc_abi = .sysv64,
4191243114 .required_features = .{ .@"64bit", .sse4_1, null, null },
4191343115 .src_constraints = .{
4191443116 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -41955,6 +43157,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4195543157 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4195643158 } },
4195743159 }, .{
43160 .required_cc_abi = .sysv64,
4195843161 .required_features = .{ .@"64bit", .sse2, null, null },
4195943162 .src_constraints = .{
4196043163 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -42002,6 +43205,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4200243205 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4200343206 } },
4200443207 }, .{
43208 .required_cc_abi = .sysv64,
4200543209 .required_features = .{ .@"64bit", .sse, null, null },
4200643210 .src_constraints = .{
4200743211 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -46317,6 +47521,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4631747521 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4631847522 } },
4631947523 }, .{
47524 .required_cc_abi = .sysv64,
4632047525 .required_features = .{ .sse, null, null, null },
4632147526 .src_constraints = .{
4632247527 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -46350,6 +47555,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4635047555 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
4635147556 } },
4635247557 }, .{
47558 .required_cc_abi = .win64,
47559 .required_features = .{ .sse, null, null, null },
47560 .src_constraints = .{
47561 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
47562 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
47563 .any,
47564 },
47565 .patterns = &.{
47566 .{ .src = .{ .to_mem, .to_mem, .none } },
47567 },
47568 .call_frame = .{ .alignment = .@"16" },
47569 .extra_temps = .{
47570 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47571 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
47572 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
47573 .unused,
47574 .unused,
47575 .unused,
47576 .unused,
47577 .unused,
47578 .unused,
47579 .unused,
47580 .unused,
47581 },
47582 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
47583 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
47584 .each = .{ .once = &.{
47585 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
47586 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
47587 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
47588 } },
47589 }, .{
47590 .required_cc_abi = .sysv64,
4635347591 .required_features = .{ .avx, null, null, null },
4635447592 .src_constraints = .{
4635547593 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -46361,7 +47599,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4636147599 },
4636247600 .call_frame = .{ .alignment = .@"16" },
4636347601 .extra_temps = .{
46364 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
47602 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4636547603 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4636647604 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4636747605 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
......@@ -46376,15 +47614,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4637647614 .dst_temps = .{ .mem, .unused },
4637747615 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4637847616 .each = .{ .once = &.{
46379 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
46380 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
46381 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
47617 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47618 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
47619 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
4638247620 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
46383 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
46384 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
46385 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
47621 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
47622 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47623 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4638647624 } },
4638747625 }, .{
47626 .required_cc_abi = .sysv64,
4638847627 .required_features = .{ .sse2, null, null, null },
4638947628 .src_constraints = .{
4639047629 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -46396,7 +47635,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4639647635 },
4639747636 .call_frame = .{ .alignment = .@"16" },
4639847637 .extra_temps = .{
46399 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
47638 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4640047639 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4640147640 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4640247641 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
......@@ -46411,15 +47650,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4641147650 .dst_temps = .{ .mem, .unused },
4641247651 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4641347652 .each = .{ .once = &.{
46414 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
46415 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
46416 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
47653 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47654 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
47655 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
4641747656 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
46418 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
46419 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
46420 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
47657 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
47658 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47659 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4642147660 } },
4642247661 }, .{
47662 .required_cc_abi = .sysv64,
4642347663 .required_features = .{ .sse, null, null, null },
4642447664 .src_constraints = .{
4642547665 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -46431,7 +47671,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4643147671 },
4643247672 .call_frame = .{ .alignment = .@"16" },
4643347673 .extra_temps = .{
46434 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
47674 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
4643547675 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
4643647676 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
4643747677 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
......@@ -46446,13 +47686,121 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4644647686 .dst_temps = .{ .mem, .unused },
4644747687 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
4644847688 .each = .{ .once = &.{
46449 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
46450 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_size), ._, ._ },
46451 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_size), ._, ._ },
47689 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47690 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
47691 .{ ._, ._ps, .mova, .tmp2x, .memi(.src1x, .tmp0), ._, ._ },
4645247692 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
46453 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_size), .tmp1x, ._, ._ },
46454 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
46455 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
47693 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
47694 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47695 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
47696 } },
47697 }, .{
47698 .required_cc_abi = .win64,
47699 .required_features = .{ .avx, null, null, null },
47700 .src_constraints = .{
47701 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47702 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47703 .any,
47704 },
47705 .patterns = &.{
47706 .{ .src = .{ .to_mem, .to_mem, .none } },
47707 },
47708 .call_frame = .{ .alignment = .@"16" },
47709 .extra_temps = .{
47710 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
47711 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47712 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
47713 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
47714 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47715 .unused,
47716 .unused,
47717 .unused,
47718 .unused,
47719 .unused,
47720 .unused,
47721 },
47722 .dst_temps = .{ .mem, .unused },
47723 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
47724 .each = .{ .once = &.{
47725 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47726 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
47727 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
47728 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
47729 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
47730 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47731 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
47732 } },
47733 }, .{
47734 .required_cc_abi = .win64,
47735 .required_features = .{ .sse2, null, null, null },
47736 .src_constraints = .{
47737 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47738 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47739 .any,
47740 },
47741 .patterns = &.{
47742 .{ .src = .{ .to_mem, .to_mem, .none } },
47743 },
47744 .call_frame = .{ .alignment = .@"16" },
47745 .extra_temps = .{
47746 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
47747 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47748 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
47749 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
47750 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47751 .unused,
47752 .unused,
47753 .unused,
47754 .unused,
47755 .unused,
47756 .unused,
47757 },
47758 .dst_temps = .{ .mem, .unused },
47759 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
47760 .each = .{ .once = &.{
47761 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47762 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
47763 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
47764 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
47765 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
47766 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47767 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
47768 } },
47769 }, .{
47770 .required_cc_abi = .win64,
47771 .required_features = .{ .sse, null, null, null },
47772 .src_constraints = .{
47773 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47774 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
47775 .any,
47776 },
47777 .patterns = &.{
47778 .{ .src = .{ .to_mem, .to_mem, .none } },
47779 },
47780 .call_frame = .{ .alignment = .@"16" },
47781 .extra_temps = .{
47782 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
47783 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47784 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
47785 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
47786 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
47787 .unused,
47788 .unused,
47789 .unused,
47790 .unused,
47791 .unused,
47792 .unused,
47793 },
47794 .dst_temps = .{ .mem, .unused },
47795 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
47796 .each = .{ .once = &.{
47797 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
47798 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
47799 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
47800 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
47801 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
47802 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
47803 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
4645647804 } },
4645747805 } }) catch |err| switch (err) {
4645847806 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
......@@ -50476,6 +51824,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5047651824 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5047751825 } },
5047851826 }, .{
51827 .required_cc_abi = .sysv64,
5047951828 .required_features = .{ .sse, null, null, null },
5048051829 .src_constraints = .{
5048151830 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -50509,6 +51858,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5050951858 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
5051051859 } },
5051151860 }, .{
51861 .required_cc_abi = .win64,
51862 .required_features = .{ .sse, null, null, null },
51863 .src_constraints = .{
51864 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
51865 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
51866 .any,
51867 },
51868 .patterns = &.{
51869 .{ .src = .{ .to_mem, .to_mem, .none } },
51870 },
51871 .call_frame = .{ .alignment = .@"16" },
51872 .extra_temps = .{
51873 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
51874 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
51875 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
51876 .unused,
51877 .unused,
51878 .unused,
51879 .unused,
51880 .unused,
51881 .unused,
51882 .unused,
51883 .unused,
51884 },
51885 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
51886 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
51887 .each = .{ .once = &.{
51888 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
51889 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
51890 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
51891 } },
51892 }, .{
51893 .required_cc_abi = .sysv64,
5051251894 .required_features = .{ .avx, null, null, null },
5051351895 .src_constraints = .{
5051451896 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -50544,6 +51926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5054451926 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5054551927 } },
5054651928 }, .{
51929 .required_cc_abi = .sysv64,
5054751930 .required_features = .{ .sse2, null, null, null },
5054851931 .src_constraints = .{
5054951932 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -50579,6 +51962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5057951962 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5058051963 } },
5058151964 }, .{
51965 .required_cc_abi = .sysv64,
5058251966 .required_features = .{ .sse, null, null, null },
5058351967 .src_constraints = .{
5058451968 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -50613,6 +51997,114 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
5061351997 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5061451998 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5061551999 } },
52000 }, .{
52001 .required_cc_abi = .win64,
52002 .required_features = .{ .avx, null, null, null },
52003 .src_constraints = .{
52004 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52005 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52006 .any,
52007 },
52008 .patterns = &.{
52009 .{ .src = .{ .to_mem, .to_mem, .none } },
52010 },
52011 .call_frame = .{ .alignment = .@"16" },
52012 .extra_temps = .{
52013 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
52014 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52015 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
52016 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
52017 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52018 .unused,
52019 .unused,
52020 .unused,
52021 .unused,
52022 .unused,
52023 .unused,
52024 },
52025 .dst_temps = .{ .mem, .unused },
52026 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
52027 .each = .{ .once = &.{
52028 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
52029 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
52030 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
52031 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
52032 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
52033 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
52034 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
52035 } },
52036 }, .{
52037 .required_cc_abi = .win64,
52038 .required_features = .{ .sse2, null, null, null },
52039 .src_constraints = .{
52040 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52041 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52042 .any,
52043 },
52044 .patterns = &.{
52045 .{ .src = .{ .to_mem, .to_mem, .none } },
52046 },
52047 .call_frame = .{ .alignment = .@"16" },
52048 .extra_temps = .{
52049 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
52050 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52051 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
52052 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
52053 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52054 .unused,
52055 .unused,
52056 .unused,
52057 .unused,
52058 .unused,
52059 .unused,
52060 },
52061 .dst_temps = .{ .mem, .unused },
52062 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
52063 .each = .{ .once = &.{
52064 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
52065 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
52066 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
52067 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
52068 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
52069 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
52070 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
52071 } },
52072 }, .{
52073 .required_cc_abi = .win64,
52074 .required_features = .{ .sse, null, null, null },
52075 .src_constraints = .{
52076 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52077 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
52078 .any,
52079 },
52080 .patterns = &.{
52081 .{ .src = .{ .to_mem, .to_mem, .none } },
52082 },
52083 .call_frame = .{ .alignment = .@"16" },
52084 .extra_temps = .{
52085 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
52086 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52087 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
52088 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
52089 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
52090 .unused,
52091 .unused,
52092 .unused,
52093 .unused,
52094 .unused,
52095 .unused,
52096 },
52097 .dst_temps = .{ .mem, .unused },
52098 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
52099 .each = .{ .once = &.{
52100 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
52101 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
52102 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
52103 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
52104 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp4x, ._, ._ },
52105 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
52106 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
52107 } },
5061652108 } }) catch |err| switch (err) {
5061752109 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
5061852110 @tagName(air_tag),
......@@ -74864,6 +76356,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7486476356 .{ ._, .f_cw, .ld, .tmp0w, ._, ._, ._ },
7486576357 } },
7486676358 }, .{
76359 .required_cc_abi = .sysv64,
7486776360 .required_features = .{ .sse, null, null, null },
7486876361 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7486976362 .patterns = &.{
......@@ -74889,6 +76382,34 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7488976382 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
7489076383 } },
7489176384 }, .{
76385 .required_cc_abi = .win64,
76386 .required_features = .{ .sse, null, null, null },
76387 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
76388 .patterns = &.{
76389 .{ .src = .{ .to_mem, .none, .none } },
76390 },
76391 .call_frame = .{ .alignment = .@"16" },
76392 .extra_temps = .{
76393 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76394 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
76395 .unused,
76396 .unused,
76397 .unused,
76398 .unused,
76399 .unused,
76400 .unused,
76401 .unused,
76402 .unused,
76403 .unused,
76404 },
76405 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
76406 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
76407 .each = .{ .once = &.{
76408 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
76409 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
76410 } },
76411 }, .{
76412 .required_cc_abi = .sysv64,
7489276413 .required_features = .{ .avx, null, null, null },
7489376414 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7489476415 .patterns = &.{
......@@ -74896,7 +76417,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7489676417 },
7489776418 .call_frame = .{ .alignment = .@"16" },
7489876419 .extra_temps = .{
74899 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
76420 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7490076421 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7490176422 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
7490276423 .unused,
......@@ -74911,14 +76432,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7491176432 .dst_temps = .{ .mem, .unused },
7491276433 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7491376434 .each = .{ .once = &.{
74914 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
74915 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
76435 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76436 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7491676437 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
74917 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
74918 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
74919 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
76438 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
76439 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76440 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7492076441 } },
7492176442 }, .{
76443 .required_cc_abi = .sysv64,
7492276444 .required_features = .{ .sse2, null, null, null },
7492376445 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7492476446 .patterns = &.{
......@@ -74926,7 +76448,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7492676448 },
7492776449 .call_frame = .{ .alignment = .@"16" },
7492876450 .extra_temps = .{
74929 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
76451 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7493076452 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7493176453 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
7493276454 .unused,
......@@ -74941,14 +76463,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7494176463 .dst_temps = .{ .mem, .unused },
7494276464 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7494376465 .each = .{ .once = &.{
74944 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
74945 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
76466 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76467 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7494676468 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
74947 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
74948 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
74949 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
76469 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
76470 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76471 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7495076472 } },
7495176473 }, .{
76474 .required_cc_abi = .sysv64,
7495276475 .required_features = .{ .sse, null, null, null },
7495376476 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7495476477 .patterns = &.{
......@@ -74956,7 +76479,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7495676479 },
7495776480 .call_frame = .{ .alignment = .@"16" },
7495876481 .extra_temps = .{
74959 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
76482 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7496076483 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7496176484 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
7496276485 .unused,
......@@ -74971,12 +76494,105 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7497176494 .dst_temps = .{ .mem, .unused },
7497276495 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7497376496 .each = .{ .once = &.{
74974 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
74975 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
76497 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76498 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7497676499 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
74977 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
74978 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
74979 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
76500 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
76501 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76502 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
76503 } },
76504 }, .{
76505 .required_cc_abi = .win64,
76506 .required_features = .{ .avx, null, null, null },
76507 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
76508 .patterns = &.{
76509 .{ .src = .{ .to_mem, .none, .none } },
76510 },
76511 .call_frame = .{ .alignment = .@"16" },
76512 .extra_temps = .{
76513 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
76514 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76515 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
76516 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76517 .unused,
76518 .unused,
76519 .unused,
76520 .unused,
76521 .unused,
76522 .unused,
76523 .unused,
76524 },
76525 .dst_temps = .{ .mem, .unused },
76526 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
76527 .each = .{ .once = &.{
76528 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76529 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
76530 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
76531 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
76532 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76533 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
76534 } },
76535 }, .{
76536 .required_cc_abi = .win64,
76537 .required_features = .{ .sse2, null, null, null },
76538 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
76539 .patterns = &.{
76540 .{ .src = .{ .to_mem, .none, .none } },
76541 },
76542 .call_frame = .{ .alignment = .@"16" },
76543 .extra_temps = .{
76544 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
76545 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76546 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
76547 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76548 .unused,
76549 .unused,
76550 .unused,
76551 .unused,
76552 .unused,
76553 .unused,
76554 .unused,
76555 },
76556 .dst_temps = .{ .mem, .unused },
76557 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
76558 .each = .{ .once = &.{
76559 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76560 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
76561 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
76562 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
76563 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76564 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
76565 } },
76566 }, .{
76567 .required_cc_abi = .win64,
76568 .required_features = .{ .sse, null, null, null },
76569 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
76570 .patterns = &.{
76571 .{ .src = .{ .to_mem, .none, .none } },
76572 },
76573 .call_frame = .{ .alignment = .@"16" },
76574 .extra_temps = .{
76575 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
76576 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76577 .{ .type = .usize, .kind = .{ .extern_func = "sqrtq" } },
76578 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
76579 .unused,
76580 .unused,
76581 .unused,
76582 .unused,
76583 .unused,
76584 .unused,
76585 .unused,
76586 },
76587 .dst_temps = .{ .mem, .unused },
76588 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
76589 .each = .{ .once = &.{
76590 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
76591 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
76592 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
76593 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
76594 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
76595 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7498076596 } },
7498176597 } }) catch |err| switch (err) {
7498276598 error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{
......@@ -75589,6 +77205,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7558977205 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7559077206 } },
7559177207 }, .{
77208 .required_cc_abi = .sysv64,
7559277209 .required_features = .{ .sse, null, null, null },
7559377210 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7559477211 .patterns = &.{
......@@ -75614,6 +77231,34 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7561477231 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
7561577232 } },
7561677233 }, .{
77234 .required_cc_abi = .win64,
77235 .required_features = .{ .sse, null, null, null },
77236 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
77237 .patterns = &.{
77238 .{ .src = .{ .to_mem, .none, .none } },
77239 },
77240 .call_frame = .{ .alignment = .@"16" },
77241 .extra_temps = .{
77242 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77243 .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } },
77244 .unused,
77245 .unused,
77246 .unused,
77247 .unused,
77248 .unused,
77249 .unused,
77250 .unused,
77251 .unused,
77252 .unused,
77253 },
77254 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
77255 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
77256 .each = .{ .once = &.{
77257 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
77258 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
77259 } },
77260 }, .{
77261 .required_cc_abi = .sysv64,
7561777262 .required_features = .{ .avx, null, null, null },
7561877263 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7561977264 .patterns = &.{
......@@ -75644,6 +77289,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7564477289 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7564577290 } },
7564677291 }, .{
77292 .required_cc_abi = .sysv64,
7564777293 .required_features = .{ .sse2, null, null, null },
7564877294 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7564977295 .patterns = &.{
......@@ -75674,6 +77320,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7567477320 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7567577321 } },
7567677322 }, .{
77323 .required_cc_abi = .sysv64,
7567777324 .required_features = .{ .sse, null, null, null },
7567877325 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7567977326 .patterns = &.{
......@@ -75703,6 +77350,99 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7570377350 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
7570477351 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7570577352 } },
77353 }, .{
77354 .required_cc_abi = .win64,
77355 .required_features = .{ .avx, null, null, null },
77356 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
77357 .patterns = &.{
77358 .{ .src = .{ .to_mem, .none, .none } },
77359 },
77360 .call_frame = .{ .alignment = .@"16" },
77361 .extra_temps = .{
77362 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
77363 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77364 .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } },
77365 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77366 .unused,
77367 .unused,
77368 .unused,
77369 .unused,
77370 .unused,
77371 .unused,
77372 .unused,
77373 },
77374 .dst_temps = .{ .mem, .unused },
77375 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
77376 .each = .{ .once = &.{
77377 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
77378 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
77379 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
77380 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
77381 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
77382 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
77383 } },
77384 }, .{
77385 .required_cc_abi = .win64,
77386 .required_features = .{ .sse2, null, null, null },
77387 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
77388 .patterns = &.{
77389 .{ .src = .{ .to_mem, .none, .none } },
77390 },
77391 .call_frame = .{ .alignment = .@"16" },
77392 .extra_temps = .{
77393 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
77394 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77395 .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } },
77396 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77397 .unused,
77398 .unused,
77399 .unused,
77400 .unused,
77401 .unused,
77402 .unused,
77403 .unused,
77404 },
77405 .dst_temps = .{ .mem, .unused },
77406 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
77407 .each = .{ .once = &.{
77408 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
77409 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
77410 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
77411 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
77412 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
77413 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
77414 } },
77415 }, .{
77416 .required_cc_abi = .win64,
77417 .required_features = .{ .sse, null, null, null },
77418 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
77419 .patterns = &.{
77420 .{ .src = .{ .to_mem, .none, .none } },
77421 },
77422 .call_frame = .{ .alignment = .@"16" },
77423 .extra_temps = .{
77424 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
77425 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77426 .{ .type = .usize, .kind = .{ .extern_func = @tagName(name) ++ "q" } },
77427 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
77428 .unused,
77429 .unused,
77430 .unused,
77431 .unused,
77432 .unused,
77433 .unused,
77434 .unused,
77435 },
77436 .dst_temps = .{ .mem, .unused },
77437 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
77438 .each = .{ .once = &.{
77439 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
77440 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
77441 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
77442 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
77443 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
77444 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
77445 } },
7570677446 } },
7570777447 }) catch |err| switch (err) {
7570877448 error.SelectFailed => return cg.fail("failed to select {s} {f} {f}", .{
......@@ -78312,6 +80052,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7831280052 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7831380053 } },
7831480054 }, .{
80055 .required_cc_abi = .sysv64,
7831580056 .required_features = .{ .sse, null, null, null },
7831680057 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7831780058 .patterns = &.{
......@@ -78342,6 +80083,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7834280083 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
7834380084 } },
7834480085 }, .{
80086 .required_cc_abi = .win64,
80087 .required_features = .{ .sse, null, null, null },
80088 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
80089 .patterns = &.{
80090 .{ .src = .{ .to_mem, .none, .none } },
80091 },
80092 .call_frame = .{ .alignment = .@"16" },
80093 .extra_temps = .{
80094 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80095 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
80096 else => unreachable,
80097 .down => "floorq",
80098 .up => "ceilq",
80099 .zero => "truncq",
80100 } } },
80101 .unused,
80102 .unused,
80103 .unused,
80104 .unused,
80105 .unused,
80106 .unused,
80107 .unused,
80108 .unused,
80109 .unused,
80110 },
80111 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
80112 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
80113 .each = .{ .once = &.{
80114 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
80115 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
80116 } },
80117 }, .{
80118 .required_cc_abi = .sysv64,
7834580119 .required_features = .{ .avx, null, null, null },
7834680120 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7834780121 .patterns = &.{
......@@ -78349,7 +80123,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7834980123 },
7835080124 .call_frame = .{ .alignment = .@"16" },
7835180125 .extra_temps = .{
78352 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
80126 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7835380127 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7835480128 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
7835580129 else => unreachable,
......@@ -78369,14 +80143,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7836980143 .dst_temps = .{ .mem, .unused },
7837080144 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7837180145 .each = .{ .once = &.{
78372 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
78373 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
80146 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80147 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7837480148 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
78375 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
78376 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
78377 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
80149 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
80150 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80151 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7837880152 } },
7837980153 }, .{
80154 .required_cc_abi = .sysv64,
7838080155 .required_features = .{ .sse2, null, null, null },
7838180156 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7838280157 .patterns = &.{
......@@ -78384,7 +80159,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7838480159 },
7838580160 .call_frame = .{ .alignment = .@"16" },
7838680161 .extra_temps = .{
78387 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
80162 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7838880163 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7838980164 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
7839080165 else => unreachable,
......@@ -78404,14 +80179,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7840480179 .dst_temps = .{ .mem, .unused },
7840580180 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7840680181 .each = .{ .once = &.{
78407 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
78408 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
80182 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80183 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7840980184 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
78410 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
78411 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
78412 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
80185 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
80186 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80187 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7841380188 } },
7841480189 }, .{
80190 .required_cc_abi = .sysv64,
7841580191 .required_features = .{ .sse, null, null, null },
7841680192 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
7841780193 .patterns = &.{
......@@ -78419,7 +80195,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7841980195 },
7842080196 .call_frame = .{ .alignment = .@"16" },
7842180197 .extra_temps = .{
78422 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
80198 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7842380199 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7842480200 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
7842580201 else => unreachable,
......@@ -78439,12 +80215,120 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7843980215 .dst_temps = .{ .mem, .unused },
7844080216 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
7844180217 .each = .{ .once = &.{
78442 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
78443 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
80218 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80219 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
7844480220 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
78445 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
78446 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
78447 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
80221 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
80222 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80223 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
80224 } },
80225 }, .{
80226 .required_cc_abi = .win64,
80227 .required_features = .{ .avx, null, null, null },
80228 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
80229 .patterns = &.{
80230 .{ .src = .{ .to_mem, .none, .none } },
80231 },
80232 .call_frame = .{ .alignment = .@"16" },
80233 .extra_temps = .{
80234 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
80235 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80236 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
80237 else => unreachable,
80238 .down => "floorq",
80239 .up => "ceilq",
80240 .zero => "truncq",
80241 } } },
80242 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80243 .unused,
80244 .unused,
80245 .unused,
80246 .unused,
80247 .unused,
80248 .unused,
80249 .unused,
80250 },
80251 .dst_temps = .{ .mem, .unused },
80252 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
80253 .each = .{ .once = &.{
80254 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80255 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
80256 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
80257 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
80258 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80259 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
80260 } },
80261 }, .{
80262 .required_cc_abi = .win64,
80263 .required_features = .{ .sse2, null, null, null },
80264 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
80265 .patterns = &.{
80266 .{ .src = .{ .to_mem, .none, .none } },
80267 },
80268 .call_frame = .{ .alignment = .@"16" },
80269 .extra_temps = .{
80270 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
80271 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80272 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
80273 else => unreachable,
80274 .down => "floorq",
80275 .up => "ceilq",
80276 .zero => "truncq",
80277 } } },
80278 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80279 .unused,
80280 .unused,
80281 .unused,
80282 .unused,
80283 .unused,
80284 .unused,
80285 .unused,
80286 },
80287 .dst_temps = .{ .mem, .unused },
80288 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
80289 .each = .{ .once = &.{
80290 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80291 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
80292 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
80293 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
80294 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80295 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
80296 } },
80297 }, .{
80298 .required_cc_abi = .win64,
80299 .required_features = .{ .sse, null, null, null },
80300 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
80301 .patterns = &.{
80302 .{ .src = .{ .to_mem, .none, .none } },
80303 },
80304 .call_frame = .{ .alignment = .@"16" },
80305 .extra_temps = .{
80306 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
80307 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80308 .{ .type = .usize, .kind = .{ .extern_func = switch (direction) {
80309 else => unreachable,
80310 .down => "floorq",
80311 .up => "ceilq",
80312 .zero => "truncq",
80313 } } },
80314 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
80315 .unused,
80316 .unused,
80317 .unused,
80318 .unused,
80319 .unused,
80320 .unused,
80321 .unused,
80322 },
80323 .dst_temps = .{ .mem, .unused },
80324 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
80325 .each = .{ .once = &.{
80326 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
80327 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
80328 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
80329 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
80330 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
80331 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
7844880332 } },
7844980333 } },
7845080334 }) catch |err| switch (err) {
......@@ -79063,7 +80947,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7906380947 .call_frame = .{ .alignment = .@"16" },
7906480948 .extra_temps = .{
7906580949 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
79066 .{ .type = .i32, .kind = .{ .reg = .eax } },
80950 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7906780951 .unused,
7906880952 .unused,
7906980953 .unused,
......@@ -79398,6 +81282,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7939881282 },
7939981283 } },
7940081284 }, .{
81285 .required_cc_abi = .sysv64,
7940181286 .required_features = .{ .sse, null, null, null },
7940281287 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any },
7940381288 .patterns = &.{
......@@ -79410,7 +81295,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7941081295 .call_frame = .{ .alignment = .@"16" },
7941181296 .extra_temps = .{
7941281297 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
79413 .{ .type = .i32, .kind = .{ .reg = .eax } },
81298 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7941481299 .unused,
7941581300 .unused,
7941681301 .unused,
......@@ -79430,6 +81315,38 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7943081315 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
7943181316 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
7943281317 } },
81318 }, .{
81319 .required_cc_abi = .win64,
81320 .required_features = .{ .sse, null, null, null },
81321 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any },
81322 .patterns = &.{
81323 .{ .src = .{ .to_mem, .to_mem, .none } },
81324 },
81325 .call_frame = .{ .alignment = .@"16" },
81326 .extra_temps = .{
81327 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
81328 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
81329 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
81330 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
81331 .unused,
81332 .unused,
81333 .unused,
81334 .unused,
81335 .unused,
81336 .unused,
81337 .unused,
81338 },
81339 .dst_temps = .{ .{ .cc = switch (strict) {
81340 true => .l,
81341 false => .le,
81342 } }, .unused },
81343 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
81344 .each = .{ .once = &.{
81345 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
81346 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
81347 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
81348 .{ ._, ._, .@"test", .tmp3d, .tmp3d, ._, ._ },
81349 } },
7943381350 } },
7943481351 });
7943581352 } else err: {
......@@ -79575,7 +81492,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7957581492 .call_frame = .{ .alignment = .@"16" },
7957681493 .extra_temps = .{
7957781494 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
79578 .{ .type = .i32, .kind = .{ .reg = .eax } },
81495 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7957981496 .unused,
7958081497 .unused,
7958181498 .unused,
......@@ -79934,6 +81851,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7993481851 },
7993581852 } },
7993681853 }, .{
81854 .required_cc_abi = .sysv64,
7993781855 .required_features = .{ .sse, null, null, null },
7993881856 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any },
7993981857 .patterns = &.{
......@@ -79946,7 +81864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7994681864 .call_frame = .{ .alignment = .@"16" },
7994781865 .extra_temps = .{
7994881866 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
79949 .{ .type = .i32, .kind = .{ .reg = .eax } },
81867 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
7995081868 .unused,
7995181869 .unused,
7995281870 .unused,
......@@ -79963,6 +81881,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
7996381881 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
7996481882 .{ ._, ._, .@"test", .tmp1d, .tmp1d, ._, ._ },
7996581883 } },
81884 }, .{
81885 .required_cc_abi = .win64,
81886 .required_features = .{ .sse, null, null, null },
81887 .src_constraints = .{ .{ .float = .xword }, .{ .float = .xword }, .any },
81888 .patterns = &.{
81889 .{ .src = .{ .to_mem, .to_mem, .none } },
81890 },
81891 .call_frame = .{ .alignment = .@"16" },
81892 .extra_temps = .{
81893 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
81894 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
81895 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
81896 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
81897 .unused,
81898 .unused,
81899 .unused,
81900 .unused,
81901 .unused,
81902 .unused,
81903 .unused,
81904 },
81905 .dst_temps = .{ .{ .cc = .z }, .unused },
81906 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
81907 .each = .{ .once = &.{
81908 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
81909 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
81910 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
81911 .{ ._, ._, .@"test", .tmp3d, .tmp3d, ._, ._ },
81912 } },
7996681913 } },
7996781914 }) catch |err| break :err err;
7996881915 switch (cmp_op) {
......@@ -80018,14 +81965,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8001881965 }
8001981966 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
8002081967 },
80021 .cmp_vector, .cmp_vector_optimized => |air_tag| fallback: {
81968 .cmp_vector, .cmp_vector_optimized => |air_tag| {
8002281969 const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
8002381970 const vector_cmp = cg.air.extraData(Air.VectorCmp, ty_pl.payload).data;
80024 switch (vector_cmp.compareOperator()) {
80025 .eq, .neq => {},
80026 .lt, .lte, .gte, .gt => if (cg.floatBits(cg.typeOf(vector_cmp.lhs).childType(zcu)) == null)
80027 break :fallback try cg.airCmpVector(inst),
80028 }
8002981971 var ops = try cg.tempsFromOperands(inst, .{ vector_cmp.lhs, vector_cmp.rhs });
8003081972 var res: [1]Temp = undefined;
8003181973 (err: switch (vector_cmp.compareOperator()) {
......@@ -80615,7 +82557,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8061582557 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8061682558 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8061782559 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80618 .{ .type = .i32, .kind = .{ .reg = .eax } },
82560 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8061982561 .{ .type = .u8, .kind = .{ .reg = .cl } },
8062082562 .{ .type = .u32, .kind = .{ .reg = .edx } },
8062182563 .unused,
......@@ -80659,7 +82601,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8065982601 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8066082602 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8066182603 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80662 .{ .type = .i32, .kind = .{ .reg = .eax } },
82604 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8066382605 .{ .type = .u8, .kind = .{ .reg = .cl } },
8066482606 .{ .type = .u32, .kind = .{ .reg = .edx } },
8066582607 .unused,
......@@ -80703,7 +82645,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8070382645 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8070482646 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8070582647 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80706 .{ .type = .i32, .kind = .{ .reg = .eax } },
82648 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8070782649 .{ .type = .u8, .kind = .{ .reg = .cl } },
8070882650 .{ .type = .u32, .kind = .{ .reg = .edx } },
8070982651 .unused,
......@@ -80748,7 +82690,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8074882690 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8074982691 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8075082692 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80751 .{ .type = .i32, .kind = .{ .reg = .eax } },
82693 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8075282694 .{ .type = .u8, .kind = .{ .reg = .cl } },
8075382695 .{ .type = .u32, .kind = .{ .reg = .edx } },
8075482696 .unused,
......@@ -80793,7 +82735,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8079382735 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8079482736 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8079582737 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80796 .{ .type = .i32, .kind = .{ .reg = .eax } },
82738 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8079782739 .{ .type = .u8, .kind = .{ .reg = .cl } },
8079882740 .{ .type = .u32, .kind = .{ .reg = .edx } },
8079982741 .{ .type = .f32, .kind = .mem },
......@@ -80840,7 +82782,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8084082782 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8084182783 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8084282784 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80843 .{ .type = .i32, .kind = .{ .reg = .eax } },
82785 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8084482786 .{ .type = .u8, .kind = .{ .reg = .cl } },
8084582787 .{ .type = .u32, .kind = .{ .reg = .edx } },
8084682788 .{ .type = .f32, .kind = .mem },
......@@ -80887,7 +82829,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8088782829 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8088882830 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8088982831 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80890 .{ .type = .i32, .kind = .{ .reg = .eax } },
82832 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8089182833 .{ .type = .u8, .kind = .{ .reg = .cl } },
8089282834 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8089382835 .unused,
......@@ -80940,7 +82882,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8094082882 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8094182883 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8094282884 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80943 .{ .type = .i32, .kind = .{ .reg = .eax } },
82885 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8094482886 .{ .type = .u8, .kind = .{ .reg = .cl } },
8094582887 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8094682888 .unused,
......@@ -80993,7 +82935,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8099382935 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8099482936 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8099582937 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
80996 .{ .type = .i32, .kind = .{ .reg = .eax } },
82938 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8099782939 .{ .type = .u8, .kind = .{ .reg = .cl } },
8099882940 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8099982941 .unused,
......@@ -81047,7 +82989,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8104782989 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8104882990 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8104982991 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
81050 .{ .type = .i32, .kind = .{ .reg = .eax } },
82992 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8105182993 .{ .type = .u8, .kind = .{ .reg = .cl } },
8105282994 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8105382995 .unused,
......@@ -81101,7 +83043,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8110183043 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8110283044 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8110383045 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
81104 .{ .type = .i32, .kind = .{ .reg = .eax } },
83046 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8110583047 .{ .type = .u8, .kind = .{ .reg = .cl } },
8110683048 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8110783049 .{ .type = .f32, .kind = .mem },
......@@ -81157,7 +83099,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8115783099 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8115883100 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8115983101 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
81160 .{ .type = .i32, .kind = .{ .reg = .eax } },
83102 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8116183103 .{ .type = .u8, .kind = .{ .reg = .cl } },
8116283104 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8116383105 .{ .type = .f32, .kind = .mem },
......@@ -81984,7 +83926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8198483926 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8198583927 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8198683928 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
81987 .{ .type = .i32, .kind = .{ .reg = .eax } },
83929 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8198883930 .{ .type = .u8, .kind = .{ .reg = .cl } },
8198983931 .{ .type = .u32, .kind = .{ .reg = .edx } },
8199083932 .unused,
......@@ -82028,7 +83970,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8202883970 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8202983971 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8203083972 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82031 .{ .type = .i32, .kind = .{ .reg = .eax } },
83973 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8203283974 .{ .type = .u8, .kind = .{ .reg = .cl } },
8203383975 .{ .type = .u32, .kind = .{ .reg = .edx } },
8203483976 .unused,
......@@ -82072,7 +84014,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8207284014 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8207384015 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8207484016 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82075 .{ .type = .i32, .kind = .{ .reg = .eax } },
84017 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8207684018 .{ .type = .u8, .kind = .{ .reg = .cl } },
8207784019 .{ .type = .u32, .kind = .{ .reg = .edx } },
8207884020 .unused,
......@@ -82116,7 +84058,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8211684058 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8211784059 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8211884060 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82119 .{ .type = .i32, .kind = .{ .reg = .eax } },
84061 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8212084062 .{ .type = .u8, .kind = .{ .reg = .cl } },
8212184063 .{ .type = .u32, .kind = .{ .reg = .edx } },
8212284064 .unused,
......@@ -82160,7 +84102,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8216084102 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8216184103 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8216284104 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82163 .{ .type = .i32, .kind = .{ .reg = .eax } },
84105 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8216484106 .{ .type = .u8, .kind = .{ .reg = .cl } },
8216584107 .{ .type = .u32, .kind = .{ .reg = .edx } },
8216684108 .unused,
......@@ -82204,7 +84146,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8220484146 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8220584147 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8220684148 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82207 .{ .type = .i32, .kind = .{ .reg = .eax } },
84149 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8220884150 .{ .type = .u8, .kind = .{ .reg = .cl } },
8220984151 .{ .type = .u32, .kind = .{ .reg = .edx } },
8221084152 .unused,
......@@ -82248,7 +84190,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8224884190 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8224984191 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8225084192 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82251 .{ .type = .i32, .kind = .{ .reg = .eax } },
84193 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8225284194 .{ .type = .u8, .kind = .{ .reg = .cl } },
8225384195 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8225484196 .unused,
......@@ -82301,7 +84243,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8230184243 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8230284244 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8230384245 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82304 .{ .type = .i32, .kind = .{ .reg = .eax } },
84246 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8230584247 .{ .type = .u8, .kind = .{ .reg = .cl } },
8230684248 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8230784249 .unused,
......@@ -82354,7 +84296,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8235484296 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8235584297 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8235684298 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82357 .{ .type = .i32, .kind = .{ .reg = .eax } },
84299 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8235884300 .{ .type = .u8, .kind = .{ .reg = .cl } },
8235984301 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8236084302 .unused,
......@@ -82407,7 +84349,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8240784349 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8240884350 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8240984351 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82410 .{ .type = .i32, .kind = .{ .reg = .eax } },
84352 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8241184353 .{ .type = .u8, .kind = .{ .reg = .cl } },
8241284354 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8241384355 .unused,
......@@ -82460,7 +84402,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8246084402 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8246184403 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8246284404 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82463 .{ .type = .i32, .kind = .{ .reg = .eax } },
84405 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8246484406 .{ .type = .u8, .kind = .{ .reg = .cl } },
8246584407 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8246684408 .unused,
......@@ -82513,7 +84455,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8251384455 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8251484456 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8251584457 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
82516 .{ .type = .i32, .kind = .{ .reg = .eax } },
84458 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8251784459 .{ .type = .u8, .kind = .{ .reg = .cl } },
8251884460 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8251984461 .unused,
......@@ -85125,7 +87067,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8512587067 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8512687068 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8512787069 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85128 .{ .type = .i32, .kind = .{ .reg = .eax } },
87070 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8512987071 .{ .type = .u8, .kind = .{ .reg = .cl } },
8513087072 .{ .type = .u32, .kind = .{ .reg = .edx } },
8513187073 .unused,
......@@ -85169,7 +87111,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8516987111 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8517087112 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8517187113 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85172 .{ .type = .i32, .kind = .{ .reg = .eax } },
87114 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8517387115 .{ .type = .u8, .kind = .{ .reg = .cl } },
8517487116 .{ .type = .u32, .kind = .{ .reg = .edx } },
8517587117 .unused,
......@@ -85213,7 +87155,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8521387155 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8521487156 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8521587157 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85216 .{ .type = .i32, .kind = .{ .reg = .eax } },
87158 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8521787159 .{ .type = .u8, .kind = .{ .reg = .cl } },
8521887160 .{ .type = .u32, .kind = .{ .reg = .edx } },
8521987161 .unused,
......@@ -85258,7 +87200,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8525887200 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8525987201 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8526087202 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85261 .{ .type = .i32, .kind = .{ .reg = .eax } },
87203 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8526287204 .{ .type = .u8, .kind = .{ .reg = .cl } },
8526387205 .{ .type = .u32, .kind = .{ .reg = .edx } },
8526487206 .unused,
......@@ -85303,7 +87245,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8530387245 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8530487246 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8530587247 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85306 .{ .type = .i32, .kind = .{ .reg = .eax } },
87248 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8530787249 .{ .type = .u8, .kind = .{ .reg = .cl } },
8530887250 .{ .type = .u32, .kind = .{ .reg = .edx } },
8530987251 .{ .type = .f32, .kind = .mem },
......@@ -85350,7 +87292,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8535087292 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8535187293 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8535287294 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85353 .{ .type = .i32, .kind = .{ .reg = .eax } },
87295 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8535487296 .{ .type = .u8, .kind = .{ .reg = .cl } },
8535587297 .{ .type = .u32, .kind = .{ .reg = .edx } },
8535687298 .{ .type = .f32, .kind = .mem },
......@@ -85397,7 +87339,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8539787339 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8539887340 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8539987341 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85400 .{ .type = .i32, .kind = .{ .reg = .eax } },
87342 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8540187343 .{ .type = .u8, .kind = .{ .reg = .cl } },
8540287344 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8540387345 .unused,
......@@ -85450,7 +87392,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8545087392 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8545187393 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8545287394 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85453 .{ .type = .i32, .kind = .{ .reg = .eax } },
87395 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8545487396 .{ .type = .u8, .kind = .{ .reg = .cl } },
8545587397 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8545687398 .unused,
......@@ -85503,7 +87445,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8550387445 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8550487446 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8550587447 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85506 .{ .type = .i32, .kind = .{ .reg = .eax } },
87448 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8550787449 .{ .type = .u8, .kind = .{ .reg = .cl } },
8550887450 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8550987451 .unused,
......@@ -85557,7 +87499,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8555787499 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8555887500 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8555987501 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85560 .{ .type = .i32, .kind = .{ .reg = .eax } },
87502 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8556187503 .{ .type = .u8, .kind = .{ .reg = .cl } },
8556287504 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8556387505 .unused,
......@@ -85611,7 +87553,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8561187553 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8561287554 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8561387555 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85614 .{ .type = .i32, .kind = .{ .reg = .eax } },
87556 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8561587557 .{ .type = .u8, .kind = .{ .reg = .cl } },
8561687558 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8561787559 .{ .type = .f32, .kind = .mem },
......@@ -85667,7 +87609,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8566787609 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8566887610 .{ .type = .f16, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8566987611 .{ .type = .usize, .kind = .{ .extern_func = "__cmphf2" } },
85670 .{ .type = .i32, .kind = .{ .reg = .eax } },
87612 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8567187613 .{ .type = .u8, .kind = .{ .reg = .cl } },
8567287614 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8567387615 .{ .type = .f32, .kind = .mem },
......@@ -86508,7 +88450,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8650888450 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8650988451 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8651088452 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86511 .{ .type = .i32, .kind = .{ .reg = .eax } },
88453 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8651288454 .{ .type = .u8, .kind = .{ .reg = .cl } },
8651388455 .{ .type = .u32, .kind = .{ .reg = .edx } },
8651488456 .unused,
......@@ -86552,7 +88494,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8655288494 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8655388495 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8655488496 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86555 .{ .type = .i32, .kind = .{ .reg = .eax } },
88497 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8655688498 .{ .type = .u8, .kind = .{ .reg = .cl } },
8655788499 .{ .type = .u32, .kind = .{ .reg = .edx } },
8655888500 .unused,
......@@ -86596,7 +88538,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8659688538 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8659788539 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8659888540 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86599 .{ .type = .i32, .kind = .{ .reg = .eax } },
88541 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8660088542 .{ .type = .u8, .kind = .{ .reg = .cl } },
8660188543 .{ .type = .u32, .kind = .{ .reg = .edx } },
8660288544 .unused,
......@@ -86640,7 +88582,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8664088582 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8664188583 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8664288584 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86643 .{ .type = .i32, .kind = .{ .reg = .eax } },
88585 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8664488586 .{ .type = .u8, .kind = .{ .reg = .cl } },
8664588587 .{ .type = .u32, .kind = .{ .reg = .edx } },
8664688588 .unused,
......@@ -86684,7 +88626,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8668488626 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8668588627 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8668688628 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86687 .{ .type = .i32, .kind = .{ .reg = .eax } },
88629 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8668888630 .{ .type = .u8, .kind = .{ .reg = .cl } },
8668988631 .{ .type = .u32, .kind = .{ .reg = .edx } },
8669088632 .unused,
......@@ -86728,7 +88670,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8672888670 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8672988671 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8673088672 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86731 .{ .type = .i32, .kind = .{ .reg = .eax } },
88673 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8673288674 .{ .type = .u8, .kind = .{ .reg = .cl } },
8673388675 .{ .type = .u32, .kind = .{ .reg = .edx } },
8673488676 .unused,
......@@ -86772,7 +88714,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8677288714 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8677388715 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8677488716 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86775 .{ .type = .i32, .kind = .{ .reg = .eax } },
88717 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8677688718 .{ .type = .u8, .kind = .{ .reg = .cl } },
8677788719 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8677888720 .unused,
......@@ -86825,7 +88767,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8682588767 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8682688768 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8682788769 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86828 .{ .type = .i32, .kind = .{ .reg = .eax } },
88770 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8682988771 .{ .type = .u8, .kind = .{ .reg = .cl } },
8683088772 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8683188773 .unused,
......@@ -86878,7 +88820,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8687888820 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8687988821 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8688088822 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86881 .{ .type = .i32, .kind = .{ .reg = .eax } },
88823 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8688288824 .{ .type = .u8, .kind = .{ .reg = .cl } },
8688388825 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8688488826 .unused,
......@@ -86931,7 +88873,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8693188873 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8693288874 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8693388875 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86934 .{ .type = .i32, .kind = .{ .reg = .eax } },
88876 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8693588877 .{ .type = .u8, .kind = .{ .reg = .cl } },
8693688878 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8693788879 .unused,
......@@ -86984,7 +88926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8698488926 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8698588927 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8698688928 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
86987 .{ .type = .i32, .kind = .{ .reg = .eax } },
88929 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8698888930 .{ .type = .u8, .kind = .{ .reg = .cl } },
8698988931 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8699088932 .unused,
......@@ -87037,7 +88979,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8703788979 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8703888980 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8703988981 .{ .type = .usize, .kind = .{ .extern_func = "__cmptf2" } },
87040 .{ .type = .i32, .kind = .{ .reg = .eax } },
88982 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8704188983 .{ .type = .u8, .kind = .{ .reg = .cl } },
8704288984 .{ .type = .u64, .kind = .{ .reg = .rdx } },
8704388985 .unused,
......@@ -88690,6 +90632,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8869090632 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8869190633 } },
8869290634 }, .{
90635 .required_cc_abi = .sysv64,
8869390636 .required_features = .{ .sse, null, null, null },
8869490637 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8869590638 .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any },
......@@ -88716,6 +90659,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8871690659 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
8871790660 } },
8871890661 }, .{
90662 .required_cc_abi = .win64,
90663 .required_features = .{ .sse, null, null, null },
90664 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90665 .dst_constraints = .{ .{ .scalar_float = .{ .of = .word, .is = .word } }, .any },
90666 .patterns = &.{
90667 .{ .src = .{ .to_mem, .none, .none } },
90668 },
90669 .call_frame = .{ .alignment = .@"16" },
90670 .extra_temps = .{
90671 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90672 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
90673 .unused,
90674 .unused,
90675 .unused,
90676 .unused,
90677 .unused,
90678 .unused,
90679 .unused,
90680 .unused,
90681 .unused,
90682 },
90683 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
90684 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
90685 .each = .{ .once = &.{
90686 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
90687 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
90688 } },
90689 }, .{
90690 .required_cc_abi = .sysv64,
8871990691 .required_features = .{ .avx, null, null, null },
8872090692 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8872190693 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
......@@ -88747,6 +90719,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8874790719 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8874890720 } },
8874990721 }, .{
90722 .required_cc_abi = .sysv64,
8875090723 .required_features = .{ .sse4_1, null, null, null },
8875190724 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8875290725 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
......@@ -88778,6 +90751,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8877890751 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8877990752 } },
8878090753 }, .{
90754 .required_cc_abi = .sysv64,
8878190755 .required_features = .{ .sse2, null, null, null },
8878290756 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8878390757 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
......@@ -88810,6 +90784,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8881090784 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8881190785 } },
8881290786 }, .{
90787 .required_cc_abi = .sysv64,
8881390788 .required_features = .{ .sse, null, null, null },
8881490789 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8881590790 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
......@@ -88819,7 +90794,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8881990794 .call_frame = .{ .alignment = .@"16" },
8882090795 .extra_temps = .{
8882190796 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
88822 .{ .type = .f64, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90797 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8882390798 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
8882490799 .{ .type = .f32, .kind = .mem },
8882590800 .{ .type = .f16, .kind = .{ .reg = .ax } },
......@@ -88843,6 +90818,138 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8884390818 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8884490819 } },
8884590820 }, .{
90821 .required_cc_abi = .win64,
90822 .required_features = .{ .avx, null, null, null },
90823 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90824 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
90825 .patterns = &.{
90826 .{ .src = .{ .to_mem, .none, .none } },
90827 },
90828 .call_frame = .{ .alignment = .@"16" },
90829 .extra_temps = .{
90830 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
90831 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90832 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
90833 .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90834 .unused,
90835 .unused,
90836 .unused,
90837 .unused,
90838 .unused,
90839 .unused,
90840 .unused,
90841 },
90842 .dst_temps = .{ .mem, .unused },
90843 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
90844 .each = .{ .once = &.{
90845 .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ },
90846 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ },
90847 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
90848 .{ ._, .vp_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ },
90849 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
90850 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
90851 } },
90852 }, .{
90853 .required_cc_abi = .win64,
90854 .required_features = .{ .sse4_1, null, null, null },
90855 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90856 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
90857 .patterns = &.{
90858 .{ .src = .{ .to_mem, .none, .none } },
90859 },
90860 .call_frame = .{ .alignment = .@"16" },
90861 .extra_temps = .{
90862 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
90863 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90864 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
90865 .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90866 .unused,
90867 .unused,
90868 .unused,
90869 .unused,
90870 .unused,
90871 .unused,
90872 .unused,
90873 },
90874 .dst_temps = .{ .mem, .unused },
90875 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
90876 .each = .{ .once = &.{
90877 .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ },
90878 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ },
90879 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
90880 .{ ._, .p_w, .extr, .memi(.dst0w, .tmp0), .tmp3x, .ui(0), ._ },
90881 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
90882 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
90883 } },
90884 }, .{
90885 .required_cc_abi = .win64,
90886 .required_features = .{ .sse2, null, null, null },
90887 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90888 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
90889 .patterns = &.{
90890 .{ .src = .{ .to_mem, .none, .none } },
90891 },
90892 .call_frame = .{ .alignment = .@"16" },
90893 .extra_temps = .{
90894 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
90895 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90896 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
90897 .{ .type = .f16, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90898 .{ .type = .f16, .kind = .{ .reg = .ax } },
90899 .unused,
90900 .unused,
90901 .unused,
90902 .unused,
90903 .unused,
90904 .unused,
90905 },
90906 .dst_temps = .{ .mem, .unused },
90907 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
90908 .each = .{ .once = &.{
90909 .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ },
90910 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ },
90911 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
90912 .{ ._, .p_w, .extr, .tmp4d, .tmp3x, .ui(0), ._ },
90913 .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp4w, ._, ._ },
90914 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
90915 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
90916 } },
90917 }, .{
90918 .required_cc_abi = .win64,
90919 .required_features = .{ .sse, null, null, null },
90920 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90921 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .word, .is = .word } }, .any },
90922 .patterns = &.{
90923 .{ .src = .{ .to_mem, .none, .none } },
90924 },
90925 .call_frame = .{ .alignment = .@"16" },
90926 .extra_temps = .{
90927 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
90928 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90929 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfhf2" } },
90930 .{ .type = .f32, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90931 .{ .type = .f32, .kind = .mem },
90932 .{ .type = .f16, .kind = .{ .reg = .ax } },
90933 .unused,
90934 .unused,
90935 .unused,
90936 .unused,
90937 .unused,
90938 },
90939 .dst_temps = .{ .mem, .unused },
90940 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
90941 .each = .{ .once = &.{
90942 .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ },
90943 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ },
90944 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
90945 .{ ._, ._ss, .mov, .mem(.tmp4d), .tmp3x, ._, ._ },
90946 .{ ._, ._, .mov, .tmp5d, .mem(.tmp4d), ._, ._ },
90947 .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp5w, ._, ._ },
90948 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
90949 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
90950 } },
90951 }, .{
90952 .required_cc_abi = .sysv64,
8884690953 .required_features = .{ .sse, null, null, null },
8884790954 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8884890955 .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any },
......@@ -88869,6 +90976,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8886990976 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
8887090977 } },
8887190978 }, .{
90979 .required_cc_abi = .win64,
90980 .required_features = .{ .sse, null, null, null },
90981 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
90982 .dst_constraints = .{ .{ .scalar_float = .{ .of = .dword, .is = .dword } }, .any },
90983 .patterns = &.{
90984 .{ .src = .{ .to_mem, .none, .none } },
90985 },
90986 .call_frame = .{ .alignment = .@"16" },
90987 .extra_temps = .{
90988 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
90989 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } },
90990 .unused,
90991 .unused,
90992 .unused,
90993 .unused,
90994 .unused,
90995 .unused,
90996 .unused,
90997 .unused,
90998 .unused,
90999 },
91000 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
91001 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
91002 .each = .{ .once = &.{
91003 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
91004 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
91005 } },
91006 }, .{
91007 .required_cc_abi = .sysv64,
8887291008 .required_features = .{ .avx, null, null, null },
8887391009 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8887491010 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any },
......@@ -88900,6 +91036,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8890091036 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8890191037 } },
8890291038 }, .{
91039 .required_cc_abi = .sysv64,
8890391040 .required_features = .{ .sse2, null, null, null },
8890491041 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8890591042 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any },
......@@ -88931,6 +91068,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8893191068 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8893291069 } },
8893391070 }, .{
91071 .required_cc_abi = .sysv64,
8893491072 .required_features = .{ .sse, null, null, null },
8893591073 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8893691074 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any },
......@@ -88962,6 +91100,71 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8896291100 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8896391101 } },
8896491102 }, .{
91103 .required_cc_abi = .win64,
91104 .required_features = .{ .avx, null, null, null },
91105 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91106 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any },
91107 .patterns = &.{
91108 .{ .src = .{ .to_mem, .none, .none } },
91109 },
91110 .call_frame = .{ .alignment = .@"16" },
91111 .extra_temps = .{
91112 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
91113 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91114 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } },
91115 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91116 .unused,
91117 .unused,
91118 .unused,
91119 .unused,
91120 .unused,
91121 .unused,
91122 .unused,
91123 },
91124 .dst_temps = .{ .mem, .unused },
91125 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
91126 .each = .{ .once = &.{
91127 .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ },
91128 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ },
91129 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91130 .{ ._, .v_ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ },
91131 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
91132 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
91133 } },
91134 }, .{
91135 .required_cc_abi = .win64,
91136 .required_features = .{ .sse, null, null, null },
91137 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91138 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .dword, .is = .dword } }, .any },
91139 .patterns = &.{
91140 .{ .src = .{ .to_mem, .none, .none } },
91141 },
91142 .call_frame = .{ .alignment = .@"16" },
91143 .extra_temps = .{
91144 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
91145 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91146 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfsf2" } },
91147 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91148 .unused,
91149 .unused,
91150 .unused,
91151 .unused,
91152 .unused,
91153 .unused,
91154 .unused,
91155 },
91156 .dst_temps = .{ .mem, .unused },
91157 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
91158 .each = .{ .once = &.{
91159 .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ },
91160 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ },
91161 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91162 .{ ._, ._ss, .mov, .memi(.dst0d, .tmp0), .tmp3x, ._, ._ },
91163 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
91164 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
91165 } },
91166 }, .{
91167 .required_cc_abi = .sysv64,
8896591168 .required_features = .{ .sse, null, null, null },
8896691169 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8896791170 .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any },
......@@ -88988,6 +91191,35 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8898891191 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
8898991192 } },
8899091193 }, .{
91194 .required_cc_abi = .win64,
91195 .required_features = .{ .sse, null, null, null },
91196 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91197 .dst_constraints = .{ .{ .scalar_float = .{ .of = .qword, .is = .qword } }, .any },
91198 .patterns = &.{
91199 .{ .src = .{ .to_mem, .none, .none } },
91200 },
91201 .call_frame = .{ .alignment = .@"16" },
91202 .extra_temps = .{
91203 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91204 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } },
91205 .unused,
91206 .unused,
91207 .unused,
91208 .unused,
91209 .unused,
91210 .unused,
91211 .unused,
91212 .unused,
91213 .unused,
91214 },
91215 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
91216 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
91217 .each = .{ .once = &.{
91218 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
91219 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
91220 } },
91221 }, .{
91222 .required_cc_abi = .sysv64,
8899191223 .required_features = .{ .avx, null, null, null },
8899291224 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8899391225 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
......@@ -89019,6 +91251,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8901991251 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8902091252 } },
8902191253 }, .{
91254 .required_cc_abi = .sysv64,
8902291255 .required_features = .{ .sse2, null, null, null },
8902391256 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8902491257 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
......@@ -89050,6 +91283,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8905091283 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8905191284 } },
8905291285 }, .{
91286 .required_cc_abi = .sysv64,
8905391287 .required_features = .{ .sse, null, null, null },
8905491288 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8905591289 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
......@@ -89081,46 +91315,83 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8908191315 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8908291316 } },
8908391317 }, .{
89084 .required_cc_abi = .sysv64,
89085 .required_features = .{ .sse, .x87, null, null },
89086 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
89087 .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
91318 .required_cc_abi = .win64,
91319 .required_features = .{ .avx, null, null, null },
91320 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91321 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
8908891322 .patterns = &.{
89089 .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } },
91323 .{ .src = .{ .to_mem, .none, .none } },
8909091324 },
8909191325 .call_frame = .{ .alignment = .@"16" },
8909291326 .extra_temps = .{
89093 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
91327 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
91328 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91329 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } },
91330 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91331 .unused,
91332 .unused,
8909491333 .unused,
8909591334 .unused,
8909691335 .unused,
8909791336 .unused,
8909891337 .unused,
91338 },
91339 .dst_temps = .{ .mem, .unused },
91340 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
91341 .each = .{ .once = &.{
91342 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ },
91343 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ },
91344 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91345 .{ ._, .v_sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ },
91346 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
91347 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
91348 } },
91349 }, .{
91350 .required_cc_abi = .win64,
91351 .required_features = .{ .sse2, null, null, null },
91352 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91353 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
91354 .patterns = &.{
91355 .{ .src = .{ .to_mem, .none, .none } },
91356 },
91357 .call_frame = .{ .alignment = .@"16" },
91358 .extra_temps = .{
91359 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
91360 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91361 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } },
91362 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
91363 .unused,
91364 .unused,
8909991365 .unused,
8910091366 .unused,
8910191367 .unused,
8910291368 .unused,
8910391369 .unused,
8910491370 },
89105 .dst_temps = .{ .{ .reg = .st0 }, .unused },
91371 .dst_temps = .{ .mem, .unused },
8910691372 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8910791373 .each = .{ .once = &.{
89108 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
91374 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ },
91375 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ },
91376 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91377 .{ ._, ._sd, .mov, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ },
91378 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
91379 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8910991380 } },
8911091381 }, .{
8911191382 .required_cc_abi = .win64,
8911291383 .required_features = .{ .sse, null, null, null },
89113 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
89114 .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
91384 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91385 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } }, .any },
8911591386 .patterns = &.{
89116 .{ .src = .{ .{ .to_reg = .xmm1 }, .none, .none } },
91387 .{ .src = .{ .to_mem, .none, .none } },
8911791388 },
8911891389 .call_frame = .{ .alignment = .@"16" },
8911991390 .extra_temps = .{
91391 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8912091392 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
89121 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
89122 .unused,
89123 .unused,
91393 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfdf2" } },
91394 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8912491395 .unused,
8912591396 .unused,
8912691397 .unused,
......@@ -89132,21 +91403,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8913291403 .dst_temps = .{ .mem, .unused },
8913391404 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8913491405 .each = .{ .once = &.{
89135 .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ },
89136 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
91406 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ },
91407 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ },
91408 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91409 .{ ._, ._ps, .movl, .memi(.dst0q, .tmp0), .tmp3x, ._, ._ },
91410 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
91411 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8913791412 } },
8913891413 }, .{
8913991414 .required_cc_abi = .sysv64,
89140 .required_features = .{ .avx, null, null, null },
89141 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
89142 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
91415 .required_features = .{ .sse, .x87, null, null },
91416 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91417 .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
8914391418 .patterns = &.{
89144 .{ .src = .{ .to_mem, .none, .none } },
91419 .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } },
8914591420 },
8914691421 .call_frame = .{ .alignment = .@"16" },
8914791422 .extra_temps = .{
89148 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
89149 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8915091423 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
8915191424 .unused,
8915291425 .unused,
......@@ -89156,31 +91429,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8915691429 .unused,
8915791430 .unused,
8915891431 .unused,
91432 .unused,
91433 .unused,
8915991434 },
89160 .dst_temps = .{ .mem, .unused },
91435 .dst_temps = .{ .{ .reg = .st0 }, .unused },
8916191436 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8916291437 .each = .{ .once = &.{
89163 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ },
89164 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
89165 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
89166 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
89167 .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ },
89168 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
89169 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
91438 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
8917091439 } },
8917191440 }, .{
8917291441 .required_cc_abi = .win64,
89173 .required_features = .{ .avx, null, null, null },
89174 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
89175 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
91442 .required_features = .{ .sse, null, null, null },
91443 .src_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
91444 .dst_constraints = .{ .{ .scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
8917691445 .patterns = &.{
8917791446 .{ .src = .{ .to_mem, .none, .none } },
8917891447 },
8917991448 .call_frame = .{ .alignment = .@"16" },
8918091449 .extra_temps = .{
89181 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8918291450 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
89183 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
91451 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8918491452 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
8918591453 .unused,
8918691454 .unused,
......@@ -89189,20 +91457,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8918991457 .unused,
8919091458 .unused,
8919191459 .unused,
91460 .unused,
8919291461 },
8919391462 .dst_temps = .{ .mem, .unused },
8919491463 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8919591464 .each = .{ .once = &.{
89196 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ },
89197 .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ },
89198 .{ ._, .v_dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ },
89199 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
89200 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
89201 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
91465 .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ },
91466 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
91467 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
8920291468 } },
8920391469 }, .{
8920491470 .required_cc_abi = .sysv64,
89205 .required_features = .{ .sse2, null, null, null },
91471 .required_features = .{ .avx, null, null, null },
8920691472 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8920791473 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
8920891474 .patterns = &.{
......@@ -89226,7 +91492,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8922691492 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8922791493 .each = .{ .once = &.{
8922891494 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ },
89229 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
91495 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
8923091496 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
8923191497 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
8923291498 .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ },
......@@ -89234,7 +91500,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8923491500 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8923591501 } },
8923691502 }, .{
89237 .required_cc_abi = .win64,
91503 .required_cc_abi = .sysv64,
8923891504 .required_features = .{ .sse2, null, null, null },
8923991505 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
8924091506 .dst_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } }, .any },
......@@ -89244,8 +91510,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8924491510 .call_frame = .{ .alignment = .@"16" },
8924591511 .extra_temps = .{
8924691512 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
89247 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
89248 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
91513 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
8924991514 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
8925091515 .unused,
8925191516 .unused,
......@@ -89254,14 +91519,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8925491519 .unused,
8925591520 .unused,
8925691521 .unused,
91522 .unused,
8925791523 },
8925891524 .dst_temps = .{ .mem, .unused },
8925991525 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8926091526 .each = .{ .once = &.{
8926191527 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ },
89262 .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ },
89263 .{ ._, ._dqa, .mov, .tmp2x, .memi(.src0x, .tmp0), ._, ._ },
89264 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
91528 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
91529 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
91530 .{ .pseudo, .f_cstp, .de, ._, ._, ._, ._ },
91531 .{ ._, .f_p, .st, .memi(.dst0t, .tmp0), ._, ._, ._ },
8926591532 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
8926691533 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
8926791534 } },
......@@ -89310,7 +91577,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8931091577 .extra_temps = .{
8931191578 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
8931291579 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
89313 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 1, .at = 1 } } },
91580 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
8931491581 .{ .type = .usize, .kind = .{ .extern_func = "__trunctfxf2" } },
8931591582 .unused,
8931691583 .unused,
......@@ -89323,9 +91590,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
8932391590 .dst_temps = .{ .mem, .unused },
8932491591 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
8932591592 .each = .{ .once = &.{
89326 .{ ._, ._, .mov, .tmp0d, .sa(.dst0, .add_unaligned_size), ._, ._ },
91593 .{ ._, ._, .mov, .tmp0d, .sia(-16, .dst0, .add_unaligned_size), ._, ._ },
8932791594 .{ .@"0:", ._, .lea, .tmp1p, .memi(.dst0, .tmp0), ._, ._ },
89328 .{ ._, ._ps, .mova, .tmp2x, .memi(.src0x, .tmp0), ._, ._ },
91595 .{ ._, ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
8932991596 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
8933091597 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
8933191598 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
......@@ -110769,6 +113036,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110769113036 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
110770113037 } },
110771113038 }, .{
113039 .required_cc_abi = .sysv64,
110772113040 .required_features = .{ .avx, .slow_incdec, null, null },
110773113041 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110774113042 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110802,6 +113070,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110802113070 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
110803113071 } },
110804113072 }, .{
113073 .required_cc_abi = .sysv64,
110805113074 .required_features = .{ .avx, null, null, null },
110806113075 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110807113076 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110835,6 +113104,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110835113104 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
110836113105 } },
110837113106 }, .{
113107 .required_cc_abi = .sysv64,
110838113108 .required_features = .{ .sse2, .slow_incdec, null, null },
110839113109 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110840113110 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110868,6 +113138,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110868113138 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
110869113139 } },
110870113140 }, .{
113141 .required_cc_abi = .sysv64,
110871113142 .required_features = .{ .sse2, null, null, null },
110872113143 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110873113144 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110901,6 +113172,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110901113172 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
110902113173 } },
110903113174 }, .{
113175 .required_cc_abi = .sysv64,
110904113176 .required_features = .{ .sse, .slow_incdec, null, null },
110905113177 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110906113178 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110934,6 +113206,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110934113206 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
110935113207 } },
110936113208 }, .{
113209 .required_cc_abi = .sysv64,
110937113210 .required_features = .{ .sse, null, null, null },
110938113211 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110939113212 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
......@@ -110967,6 +113240,75 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110967113240 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
110968113241 } },
110969113242 }, .{
113243 .required_cc_abi = .win64,
113244 .required_features = .{ .sse, .slow_incdec, null, null },
113245 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
113246 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
113247 .patterns = &.{
113248 .{ .src = .{ .to_mem, .none, .none } },
113249 },
113250 .call_frame = .{ .alignment = .@"16" },
113251 .extra_temps = .{
113252 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
113253 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
113254 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113255 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } },
113256 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113257 .unused,
113258 .unused,
113259 .unused,
113260 .unused,
113261 .unused,
113262 .unused,
113263 },
113264 .dst_temps = .{ .mem, .unused },
113265 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113266 .each = .{ .once = &.{
113267 .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
113268 .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ },
113269 .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ },
113270 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
113271 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ },
113272 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ },
113273 .{ ._, ._, .sub, .tmp1d, .si(1), ._, ._ },
113274 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
113275 } },
113276 }, .{
113277 .required_cc_abi = .win64,
113278 .required_features = .{ .sse, null, null, null },
113279 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
113280 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } }, .any },
113281 .patterns = &.{
113282 .{ .src = .{ .to_mem, .none, .none } },
113283 },
113284 .call_frame = .{ .alignment = .@"16" },
113285 .extra_temps = .{
113286 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
113287 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
113288 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113289 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } },
113290 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113291 .unused,
113292 .unused,
113293 .unused,
113294 .unused,
113295 .unused,
113296 .unused,
113297 },
113298 .dst_temps = .{ .mem, .unused },
113299 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113300 .each = .{ .once = &.{
113301 .{ ._, ._, .lea, .tmp0p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
113302 .{ ._, ._, .mov, .tmp1d, .sia(-1, .dst0, .add_unaligned_size), ._, ._ },
113303 .{ .@"0:", ._, .mov, .tmp2p, .tmp0p, ._, ._ },
113304 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
113305 .{ ._, ._, .mov, .memi(.dst0b, .tmp1), .tmp4b, ._, ._ },
113306 .{ ._, ._, .lea, .tmp0p, .lead(.tmp0, -16), ._, ._ },
113307 .{ ._, ._c, .de, .tmp1d, ._, ._, ._ },
113308 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
113309 } },
113310 }, .{
113311 .required_cc_abi = .sysv64,
110970113312 .required_features = .{ .avx, null, null, null },
110971113313 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
110972113314 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
......@@ -110998,6 +113340,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
110998113340 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
110999113341 } },
111000113342 }, .{
113343 .required_cc_abi = .sysv64,
111001113344 .required_features = .{ .sse2, null, null, null },
111002113345 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111003113346 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
......@@ -111029,6 +113372,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111029113372 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111030113373 } },
111031113374 }, .{
113375 .required_cc_abi = .sysv64,
111032113376 .required_features = .{ .sse, null, null, null },
111033113377 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111034113378 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
......@@ -111060,6 +113404,39 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111060113404 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111061113405 } },
111062113406 }, .{
113407 .required_cc_abi = .win64,
113408 .required_features = .{ .sse, null, null, null },
113409 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
113410 .dst_constraints = .{ .{ .multiple_scalar_int = .{ .of = .word, .is = .word } }, .any },
113411 .patterns = &.{
113412 .{ .src = .{ .to_mem, .none, .none } },
113413 },
113414 .call_frame = .{ .alignment = .@"16" },
113415 .extra_temps = .{
113416 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
113417 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113418 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } },
113419 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113420 .unused,
113421 .unused,
113422 .unused,
113423 .unused,
113424 .unused,
113425 .unused,
113426 .unused,
113427 },
113428 .dst_temps = .{ .mem, .unused },
113429 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113430 .each = .{ .once = &.{
113431 .{ ._, ._, .mov, .tmp0d, .sia(-2, .dst0, .add_unaligned_size), ._, ._ },
113432 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"8", .tmp0), ._, ._ },
113433 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
113434 .{ ._, ._, .mov, .memi(.dst0w, .tmp0), .tmp3w, ._, ._ },
113435 .{ ._, ._, .sub, .tmp0d, .si(2), ._, ._ },
113436 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
113437 } },
113438 }, .{
113439 .required_cc_abi = .sysv64,
111063113440 .required_features = .{ .sse, null, null, null },
111064113441 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111065113442 .dst_constraints = .{ .{ .signed_int = .dword }, .any },
......@@ -111086,6 +113463,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111086113463 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
111087113464 } },
111088113465 }, .{
113466 .required_cc_abi = .sysv64,
111089113467 .required_features = .{ .sse, null, null, null },
111090113468 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111091113469 .dst_constraints = .{ .{ .unsigned_int = .dword }, .any },
......@@ -111112,6 +113490,63 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111112113490 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
111113113491 } },
111114113492 }, .{
113493 .required_cc_abi = .win64,
113494 .required_features = .{ .sse, null, null, null },
113495 .src_constraints = .{ .{ .float = .xword }, .any, .any },
113496 .dst_constraints = .{ .{ .signed_int = .dword }, .any },
113497 .patterns = &.{
113498 .{ .src = .{ .to_mem, .none, .none } },
113499 },
113500 .call_frame = .{ .alignment = .@"16" },
113501 .extra_temps = .{
113502 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113503 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } },
113504 .unused,
113505 .unused,
113506 .unused,
113507 .unused,
113508 .unused,
113509 .unused,
113510 .unused,
113511 .unused,
113512 .unused,
113513 },
113514 .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
113515 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113516 .each = .{ .once = &.{
113517 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
113518 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
113519 } },
113520 }, .{
113521 .required_cc_abi = .win64,
113522 .required_features = .{ .sse, null, null, null },
113523 .src_constraints = .{ .{ .float = .xword }, .any, .any },
113524 .dst_constraints = .{ .{ .unsigned_int = .dword }, .any },
113525 .patterns = &.{
113526 .{ .src = .{ .to_mem, .none, .none } },
113527 },
113528 .call_frame = .{ .alignment = .@"16" },
113529 .extra_temps = .{
113530 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113531 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } },
113532 .unused,
113533 .unused,
113534 .unused,
113535 .unused,
113536 .unused,
113537 .unused,
113538 .unused,
113539 .unused,
113540 .unused,
113541 },
113542 .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
113543 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113544 .each = .{ .once = &.{
113545 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
113546 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
113547 } },
113548 }, .{
113549 .required_cc_abi = .sysv64,
111115113550 .required_features = .{ .avx, null, null, null },
111116113551 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111117113552 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111143,6 +113578,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111143113578 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111144113579 } },
111145113580 }, .{
113581 .required_cc_abi = .sysv64,
111146113582 .required_features = .{ .avx, null, null, null },
111147113583 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111148113584 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111174,6 +113610,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111174113610 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111175113611 } },
111176113612 }, .{
113613 .required_cc_abi = .sysv64,
111177113614 .required_features = .{ .sse2, null, null, null },
111178113615 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111179113616 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111205,6 +113642,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111205113642 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111206113643 } },
111207113644 }, .{
113645 .required_cc_abi = .sysv64,
111208113646 .required_features = .{ .sse2, null, null, null },
111209113647 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111210113648 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111236,6 +113674,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111236113674 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111237113675 } },
111238113676 }, .{
113677 .required_cc_abi = .sysv64,
111239113678 .required_features = .{ .sse, null, null, null },
111240113679 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111241113680 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111267,6 +113706,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111267113706 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111268113707 } },
111269113708 }, .{
113709 .required_cc_abi = .sysv64,
111270113710 .required_features = .{ .sse, null, null, null },
111271113711 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111272113712 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -111298,6 +113738,71 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111298113738 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111299113739 } },
111300113740 }, .{
113741 .required_cc_abi = .win64,
113742 .required_features = .{ .sse, null, null, null },
113743 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
113744 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .dword, .is = .dword } }, .any },
113745 .patterns = &.{
113746 .{ .src = .{ .to_mem, .none, .none } },
113747 },
113748 .call_frame = .{ .alignment = .@"16" },
113749 .extra_temps = .{
113750 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
113751 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113752 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfsi" } },
113753 .{ .type = .i32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113754 .unused,
113755 .unused,
113756 .unused,
113757 .unused,
113758 .unused,
113759 .unused,
113760 .unused,
113761 },
113762 .dst_temps = .{ .mem, .unused },
113763 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113764 .each = .{ .once = &.{
113765 .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ },
113766 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ },
113767 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
113768 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ },
113769 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
113770 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
113771 } },
113772 }, .{
113773 .required_cc_abi = .win64,
113774 .required_features = .{ .sse, null, null, null },
113775 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
113776 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
113777 .patterns = &.{
113778 .{ .src = .{ .to_mem, .none, .none } },
113779 },
113780 .call_frame = .{ .alignment = .@"16" },
113781 .extra_temps = .{
113782 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
113783 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113784 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfsi" } },
113785 .{ .type = .u32, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113786 .unused,
113787 .unused,
113788 .unused,
113789 .unused,
113790 .unused,
113791 .unused,
113792 .unused,
113793 },
113794 .dst_temps = .{ .mem, .unused },
113795 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113796 .each = .{ .once = &.{
113797 .{ ._, ._, .mov, .tmp0d, .sia(-4, .dst0, .add_unaligned_size), ._, ._ },
113798 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"4", .tmp0), ._, ._ },
113799 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
113800 .{ ._, ._, .mov, .memi(.dst0d, .tmp0), .tmp3d, ._, ._ },
113801 .{ ._, ._, .sub, .tmp0d, .si(4), ._, ._ },
113802 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
113803 } },
113804 }, .{
113805 .required_cc_abi = .sysv64,
111301113806 .required_features = .{ .@"64bit", .sse, null, null },
111302113807 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111303113808 .dst_constraints = .{ .{ .signed_int = .qword }, .any },
......@@ -111324,6 +113829,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111324113829 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
111325113830 } },
111326113831 }, .{
113832 .required_cc_abi = .sysv64,
111327113833 .required_features = .{ .@"64bit", .sse, null, null },
111328113834 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111329113835 .dst_constraints = .{ .{ .unsigned_int = .qword }, .any },
......@@ -111350,6 +113856,63 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111350113856 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
111351113857 } },
111352113858 }, .{
113859 .required_cc_abi = .win64,
113860 .required_features = .{ .@"64bit", .sse, null, null },
113861 .src_constraints = .{ .{ .float = .xword }, .any, .any },
113862 .dst_constraints = .{ .{ .signed_int = .qword }, .any },
113863 .patterns = &.{
113864 .{ .src = .{ .to_mem, .none, .none } },
113865 },
113866 .call_frame = .{ .alignment = .@"16" },
113867 .extra_temps = .{
113868 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113869 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } },
113870 .unused,
113871 .unused,
113872 .unused,
113873 .unused,
113874 .unused,
113875 .unused,
113876 .unused,
113877 .unused,
113878 .unused,
113879 },
113880 .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
113881 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113882 .each = .{ .once = &.{
113883 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
113884 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
113885 } },
113886 }, .{
113887 .required_cc_abi = .win64,
113888 .required_features = .{ .@"64bit", .sse, null, null },
113889 .src_constraints = .{ .{ .float = .xword }, .any, .any },
113890 .dst_constraints = .{ .{ .unsigned_int = .qword }, .any },
113891 .patterns = &.{
113892 .{ .src = .{ .to_mem, .none, .none } },
113893 },
113894 .call_frame = .{ .alignment = .@"16" },
113895 .extra_temps = .{
113896 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
113897 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } },
113898 .unused,
113899 .unused,
113900 .unused,
113901 .unused,
113902 .unused,
113903 .unused,
113904 .unused,
113905 .unused,
113906 .unused,
113907 },
113908 .dst_temps = .{ .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
113909 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
113910 .each = .{ .once = &.{
113911 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
113912 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
113913 } },
113914 }, .{
113915 .required_cc_abi = .sysv64,
111353113916 .required_features = .{ .@"64bit", .avx, null, null },
111354113917 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111355113918 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111381,6 +113944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111381113944 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111382113945 } },
111383113946 }, .{
113947 .required_cc_abi = .sysv64,
111384113948 .required_features = .{ .@"64bit", .avx, null, null },
111385113949 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111386113950 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111412,6 +113976,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111412113976 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111413113977 } },
111414113978 }, .{
113979 .required_cc_abi = .sysv64,
111415113980 .required_features = .{ .@"64bit", .sse2, null, null },
111416113981 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111417113982 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111443,6 +114008,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111443114008 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111444114009 } },
111445114010 }, .{
114011 .required_cc_abi = .sysv64,
111446114012 .required_features = .{ .@"64bit", .sse2, null, null },
111447114013 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111448114014 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111474,6 +114040,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111474114040 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111475114041 } },
111476114042 }, .{
114043 .required_cc_abi = .sysv64,
111477114044 .required_features = .{ .@"64bit", .sse, null, null },
111478114045 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111479114046 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111505,6 +114072,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111505114072 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111506114073 } },
111507114074 }, .{
114075 .required_cc_abi = .sysv64,
111508114076 .required_features = .{ .@"64bit", .sse, null, null },
111509114077 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111510114078 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any },
......@@ -111536,16 +114104,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111536114104 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111537114105 } },
111538114106 }, .{
111539 .required_cc_abi = .sysv64,
111540 .required_features = .{ .sse, null, null, null },
111541 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111542 .dst_constraints = .{ .{ .signed_int = .xword }, .any },
114107 .required_cc_abi = .win64,
114108 .required_features = .{ .@"64bit", .sse, null, null },
114109 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
114110 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .qword } }, .any },
111543114111 .patterns = &.{
111544 .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } },
114112 .{ .src = .{ .to_mem, .none, .none } },
111545114113 },
111546114114 .call_frame = .{ .alignment = .@"16" },
111547114115 .extra_temps = .{
111548 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
114116 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
114117 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114118 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfdi" } },
114119 .{ .type = .i64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111549114120 .unused,
111550114121 .unused,
111551114122 .unused,
......@@ -111553,17 +114124,51 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111553114124 .unused,
111554114125 .unused,
111555114126 .unused,
114127 },
114128 .dst_temps = .{ .mem, .unused },
114129 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
114130 .each = .{ .once = &.{
114131 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ },
114132 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ },
114133 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
114134 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ },
114135 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
114136 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
114137 } },
114138 }, .{
114139 .required_cc_abi = .win64,
114140 .required_features = .{ .@"64bit", .sse, null, null },
114141 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
114142 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .qword } }, .any },
114143 .patterns = &.{
114144 .{ .src = .{ .to_mem, .none, .none } },
114145 },
114146 .call_frame = .{ .alignment = .@"16" },
114147 .extra_temps = .{
114148 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
114149 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114150 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfdi" } },
114151 .{ .type = .u64, .kind = .{ .ret_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114152 .unused,
114153 .unused,
114154 .unused,
114155 .unused,
111556114156 .unused,
111557114157 .unused,
111558114158 .unused,
111559114159 },
111560 .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
114160 .dst_temps = .{ .mem, .unused },
111561114161 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111562114162 .each = .{ .once = &.{
111563 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
114163 .{ ._, ._, .mov, .tmp0d, .sia(-8, .dst0, .add_unaligned_size), ._, ._ },
114164 .{ .@"0:", ._, .lea, .tmp1p, .memsi(.src0, .@"2", .tmp0), ._, ._ },
114165 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
114166 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q, ._, ._ },
114167 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
114168 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111564114169 } },
111565114170 }, .{
111566 .required_cc_abi = .win64,
114171 .required_cc_abi = .sysv64,
111567114172 .required_features = .{ .sse, null, null, null },
111568114173 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111569114174 .dst_constraints = .{ .{ .signed_int = .xword }, .any },
......@@ -111584,7 +114189,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111584114189 .unused,
111585114190 .unused,
111586114191 },
111587 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
114192 .dst_temps = .{ .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
111588114193 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111589114194 .each = .{ .once = &.{
111590114195 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
......@@ -111616,16 +114221,45 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111616114221 .each = .{ .once = &.{
111617114222 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
111618114223 } },
114224 }, .{
114225 .required_cc_abi = .win64,
114226 .required_features = .{ .sse, null, null, null },
114227 .src_constraints = .{ .{ .float = .xword }, .any, .any },
114228 .dst_constraints = .{ .{ .signed_int = .xword }, .any },
114229 .patterns = &.{
114230 .{ .src = .{ .to_mem, .none, .none } },
114231 },
114232 .call_frame = .{ .alignment = .@"16" },
114233 .extra_temps = .{
114234 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114235 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
114236 .unused,
114237 .unused,
114238 .unused,
114239 .unused,
114240 .unused,
114241 .unused,
114242 .unused,
114243 .unused,
114244 .unused,
114245 },
114246 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
114247 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
114248 .each = .{ .once = &.{
114249 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
114250 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
114251 } },
111619114252 }, .{
111620114253 .required_cc_abi = .win64,
111621114254 .required_features = .{ .sse, null, null, null },
111622114255 .src_constraints = .{ .{ .float = .xword }, .any, .any },
111623114256 .dst_constraints = .{ .{ .unsigned_int = .xword }, .any },
111624114257 .patterns = &.{
111625 .{ .src = .{ .{ .to_param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .none, .none } },
114258 .{ .src = .{ .to_mem, .none, .none } },
111626114259 },
111627114260 .call_frame = .{ .alignment = .@"16" },
111628114261 .extra_temps = .{
114262 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111629114263 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111630114264 .unused,
111631114265 .unused,
......@@ -111636,12 +114270,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111636114270 .unused,
111637114271 .unused,
111638114272 .unused,
111639 .unused,
111640114273 },
111641114274 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
111642114275 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111643114276 .each = .{ .once = &.{
111644 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
114277 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
114278 .{ ._, ._, .call, .tmp1d, ._, ._, ._ },
111645114279 } },
111646114280 }, .{
111647114281 .required_cc_abi = .sysv64,
......@@ -111677,10 +114311,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111677114311 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111678114312 } },
111679114313 }, .{
111680 .required_cc_abi = .win64,
114314 .required_cc_abi = .sysv64,
111681114315 .required_features = .{ .avx, null, null, null },
111682114316 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111683 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
114317 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
111684114318 .patterns = &.{
111685114319 .{ .src = .{ .to_mem, .none, .none } },
111686114320 },
......@@ -111688,8 +114322,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111688114322 .extra_temps = .{
111689114323 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111690114324 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111691 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
111692 .unused,
114325 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
114326 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111693114327 .unused,
111694114328 .unused,
111695114329 .unused,
......@@ -111704,15 +114338,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111704114338 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111705114339 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
111706114340 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111707 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
114341 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
114342 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
111708114343 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111709114344 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111710114345 } },
111711114346 }, .{
111712114347 .required_cc_abi = .sysv64,
111713 .required_features = .{ .avx, null, null, null },
114348 .required_features = .{ .sse2, null, null, null },
111714114349 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111715 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
114350 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
111716114351 .patterns = &.{
111717114352 .{ .src = .{ .to_mem, .none, .none } },
111718114353 },
......@@ -111720,8 +114355,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111720114355 .extra_temps = .{
111721114356 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111722114357 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111723 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111724 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114358 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
114359 .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111725114360 .unused,
111726114361 .unused,
111727114362 .unused,
......@@ -111734,7 +114369,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111734114369 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111735114370 .each = .{ .once = &.{
111736114371 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111737 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114372 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
111738114373 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111739114374 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
111740114375 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
......@@ -111742,8 +114377,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111742114377 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111743114378 } },
111744114379 }, .{
111745 .required_cc_abi = .win64,
111746 .required_features = .{ .avx, null, null, null },
114380 .required_cc_abi = .sysv64,
114381 .required_features = .{ .sse2, null, null, null },
111747114382 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111748114383 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
111749114384 .patterns = &.{
......@@ -111754,7 +114389,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111754114389 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111755114390 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111756114391 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111757 .unused,
114392 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111758114393 .unused,
111759114394 .unused,
111760114395 .unused,
......@@ -111767,15 +114402,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111767114402 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111768114403 .each = .{ .once = &.{
111769114404 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111770 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114405 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
111771114406 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111772 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
114407 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
114408 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
111773114409 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111774114410 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111775114411 } },
111776114412 }, .{
111777114413 .required_cc_abi = .sysv64,
111778 .required_features = .{ .sse2, null, null, null },
114414 .required_features = .{ .sse, null, null, null },
111779114415 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111780114416 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
111781114417 .patterns = &.{
......@@ -111799,7 +114435,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111799114435 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111800114436 .each = .{ .once = &.{
111801114437 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111802 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114438 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
111803114439 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111804114440 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
111805114441 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
......@@ -111807,10 +114443,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111807114443 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111808114444 } },
111809114445 }, .{
111810 .required_cc_abi = .win64,
111811 .required_features = .{ .sse2, null, null, null },
114446 .required_cc_abi = .sysv64,
114447 .required_features = .{ .sse, null, null, null },
111812114448 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111813 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
114449 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
111814114450 .patterns = &.{
111815114451 .{ .src = .{ .to_mem, .none, .none } },
111816114452 },
......@@ -111818,8 +114454,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111818114454 .extra_temps = .{
111819114455 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111820114456 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111821 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
111822 .unused,
114457 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
114458 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111823114459 .unused,
111824114460 .unused,
111825114461 .unused,
......@@ -111832,26 +114468,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111832114468 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111833114469 .each = .{ .once = &.{
111834114470 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111835 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114471 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
111836114472 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111837 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
114473 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
114474 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
111838114475 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111839114476 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111840114477 } },
111841114478 }, .{
111842 .required_cc_abi = .sysv64,
111843 .required_features = .{ .sse2, null, null, null },
114479 .required_cc_abi = .win64,
114480 .required_features = .{ .avx, null, null, null },
111844114481 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111845 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
114482 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
111846114483 .patterns = &.{
111847114484 .{ .src = .{ .to_mem, .none, .none } },
111848114485 },
111849114486 .call_frame = .{ .alignment = .@"16" },
111850114487 .extra_temps = .{
111851114488 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111852 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111853 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111854 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114489 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114490 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
114491 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111855114492 .unused,
111856114493 .unused,
111857114494 .unused,
......@@ -111864,16 +114501,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111864114501 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111865114502 .each = .{ .once = &.{
111866114503 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111867 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114504 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
111868114505 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111869 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
111870 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
114506 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
111871114507 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111872114508 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111873114509 } },
111874114510 }, .{
111875114511 .required_cc_abi = .win64,
111876 .required_features = .{ .sse2, null, null, null },
114512 .required_features = .{ .avx, null, null, null },
111877114513 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111878114514 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
111879114515 .patterns = &.{
......@@ -111882,9 +114518,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111882114518 .call_frame = .{ .alignment = .@"16" },
111883114519 .extra_temps = .{
111884114520 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111885 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114521 .{ .type = .f128, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111886114522 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111887 .unused,
114523 .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111888114524 .unused,
111889114525 .unused,
111890114526 .unused,
......@@ -111897,15 +114533,15 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111897114533 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111898114534 .each = .{ .once = &.{
111899114535 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111900 .{ .@"0:", ._dqa, .mov, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114536 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
111901114537 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111902 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
114538 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
111903114539 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111904114540 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111905114541 } },
111906114542 }, .{
111907 .required_cc_abi = .sysv64,
111908 .required_features = .{ .sse, null, null, null },
114543 .required_cc_abi = .win64,
114544 .required_features = .{ .sse2, null, null, null },
111909114545 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111910114546 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
111911114547 .patterns = &.{
......@@ -111914,9 +114550,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111914114550 .call_frame = .{ .alignment = .@"16" },
111915114551 .extra_temps = .{
111916114552 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111917 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114553 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111918114554 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
111919 .{ .type = .i128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114555 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111920114556 .unused,
111921114557 .unused,
111922114558 .unused,
......@@ -111929,27 +114565,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111929114565 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111930114566 .each = .{ .once = &.{
111931114567 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111932 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114568 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
111933114569 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111934 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
111935 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
114570 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
111936114571 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111937114572 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111938114573 } },
111939114574 }, .{
111940114575 .required_cc_abi = .win64,
111941 .required_features = .{ .sse, null, null, null },
114576 .required_features = .{ .sse2, null, null, null },
111942114577 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111943 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
114578 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
111944114579 .patterns = &.{
111945114580 .{ .src = .{ .to_mem, .none, .none } },
111946114581 },
111947114582 .call_frame = .{ .alignment = .@"16" },
111948114583 .extra_temps = .{
111949114584 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111950 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111951 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
111952 .unused,
114585 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114586 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
114587 .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111953114588 .unused,
111954114589 .unused,
111955114590 .unused,
......@@ -111962,26 +114597,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111962114597 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111963114598 .each = .{ .once = &.{
111964114599 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111965 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114600 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
111966114601 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111967 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
114602 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
111968114603 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
111969114604 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
111970114605 } },
111971114606 }, .{
111972 .required_cc_abi = .sysv64,
114607 .required_cc_abi = .win64,
111973114608 .required_features = .{ .sse, null, null, null },
111974114609 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
111975 .dst_constraints = .{ .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .xword } }, .any },
114610 .dst_constraints = .{ .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .xword } }, .any },
111976114611 .patterns = &.{
111977114612 .{ .src = .{ .to_mem, .none, .none } },
111978114613 },
111979114614 .call_frame = .{ .alignment = .@"16" },
111980114615 .extra_temps = .{
111981114616 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
111982 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111983 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
111984 .{ .type = .u128, .kind = .{ .ret_gpr_pair = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114617 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114618 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfti" } },
114619 .{ .type = .i128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
111985114620 .unused,
111986114621 .unused,
111987114622 .unused,
......@@ -111994,10 +114629,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
111994114629 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
111995114630 .each = .{ .once = &.{
111996114631 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
111997 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114632 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
111998114633 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
111999 .{ ._, ._, .mov, .memi(.dst0q, .tmp0), .tmp3q0, ._, ._ },
112000 .{ ._, ._, .mov, .memid(.dst0q, .tmp0, 8), .tmp3q1, ._, ._ },
114634 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp3x, ._, ._ },
112001114635 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
112002114636 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112003114637 } },
......@@ -112012,9 +114646,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112012114646 .call_frame = .{ .alignment = .@"16" },
112013114647 .extra_temps = .{
112014114648 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
112015 .{ .type = .f128, .kind = .{ .param_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114649 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
112016114650 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfti" } },
112017 .unused,
114651 .{ .type = .u128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
112018114652 .unused,
112019114653 .unused,
112020114654 .unused,
......@@ -112027,13 +114661,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112027114661 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
112028114662 .each = .{ .once = &.{
112029114663 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
112030 .{ .@"0:", ._ps, .mova, .tmp1x, .memi(.src0x, .tmp0), ._, ._ },
114664 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
112031114665 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
112032114666 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp1x, ._, ._ },
112033114667 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
112034114668 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112035114669 } },
112036114670 }, .{
114671 .required_cc_abi = .sysv64,
112037114672 .required_features = .{ .@"64bit", .sse, null, null },
112038114673 .src_constraints = .{ .{ .float = .xword }, .any, .any },
112039114674 .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112062,6 +114697,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112062114697 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
112063114698 } },
112064114699 }, .{
114700 .required_cc_abi = .sysv64,
112065114701 .required_features = .{ .@"64bit", .sse, null, null },
112066114702 .src_constraints = .{ .{ .float = .xword }, .any, .any },
112067114703 .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112090,6 +114726,67 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112090114726 .{ ._, ._, .call, .tmp2d, ._, ._, ._ },
112091114727 } },
112092114728 }, .{
114729 .required_cc_abi = .win64,
114730 .required_features = .{ .@"64bit", null, null, null },
114731 .src_constraints = .{ .{ .float = .xword }, .any, .any },
114732 .dst_constraints = .{ .{ .remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
114733 .patterns = &.{
114734 .{ .src = .{ .to_mem, .none, .none } },
114735 },
114736 .call_frame = .{ .alignment = .@"16" },
114737 .extra_temps = .{
114738 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114739 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
114740 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
114741 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } },
114742 .unused,
114743 .unused,
114744 .unused,
114745 .unused,
114746 .unused,
114747 .unused,
114748 .unused,
114749 },
114750 .dst_temps = .{ .mem, .unused },
114751 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
114752 .each = .{ .once = &.{
114753 .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ },
114754 .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ },
114755 .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ },
114756 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
114757 } },
114758 }, .{
114759 .required_cc_abi = .win64,
114760 .required_features = .{ .@"64bit", null, null, null },
114761 .src_constraints = .{ .{ .float = .xword }, .any, .any },
114762 .dst_constraints = .{ .{ .remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
114763 .patterns = &.{
114764 .{ .src = .{ .to_mem, .none, .none } },
114765 },
114766 .call_frame = .{ .alignment = .@"16" },
114767 .extra_temps = .{
114768 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
114769 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
114770 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
114771 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } },
114772 .unused,
114773 .unused,
114774 .unused,
114775 .unused,
114776 .unused,
114777 .unused,
114778 .unused,
114779 },
114780 .dst_temps = .{ .mem, .unused },
114781 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
114782 .each = .{ .once = &.{
114783 .{ ._, ._, .lea, .tmp0p, .mem(.dst0), ._, ._ },
114784 .{ ._, ._, .mov, .tmp1d, .sa(.dst0, .add_bit_size), ._, ._ },
114785 .{ ._, ._, .lea, .tmp2p, .mem(.src0), ._, ._ },
114786 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
114787 } },
114788 }, .{
114789 .required_cc_abi = .sysv64,
112093114790 .required_features = .{ .@"64bit", .avx, null, null },
112094114791 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112095114792 .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112124,6 +114821,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112124114821 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112125114822 } },
112126114823 }, .{
114824 .required_cc_abi = .sysv64,
112127114825 .required_features = .{ .@"64bit", .avx, null, null },
112128114826 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112129114827 .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112158,6 +114856,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112158114856 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112159114857 } },
112160114858 }, .{
114859 .required_cc_abi = .sysv64,
112161114860 .required_features = .{ .@"64bit", .sse2, null, null },
112162114861 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112163114862 .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112192,6 +114891,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112192114891 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112193114892 } },
112194114893 }, .{
114894 .required_cc_abi = .sysv64,
112195114895 .required_features = .{ .@"64bit", .sse2, null, null },
112196114896 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112197114897 .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112226,6 +114926,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112226114926 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112227114927 } },
112228114928 }, .{
114929 .required_cc_abi = .sysv64,
112229114930 .required_features = .{ .@"64bit", .sse, null, null },
112230114931 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112231114932 .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112260,6 +114961,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112260114961 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112261114962 } },
112262114963 }, .{
114964 .required_cc_abi = .sysv64,
112263114965 .required_features = .{ .@"64bit", .sse, null, null },
112264114966 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
112265114967 .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
......@@ -112293,6 +114995,76 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
112293114995 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
112294114996 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
112295114997 } },
114998 }, .{
114999 .required_cc_abi = .win64,
115000 .required_features = .{ .@"64bit", null, null, null },
115001 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
115002 .dst_constraints = .{ .{ .scalar_remainder_signed_int = .{ .of = .dword, .is = .dword } }, .any },
115003 .patterns = &.{
115004 .{ .src = .{ .to_mem, .none, .none } },
115005 },
115006 .call_frame = .{ .alignment = .@"16" },
115007 .extra_temps = .{
115008 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
115009 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
115010 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
115011 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
115012 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
115013 .{ .type = .usize, .kind = .{ .extern_func = "__fixtfei" } },
115014 .unused,
115015 .unused,
115016 .unused,
115017 .unused,
115018 .unused,
115019 },
115020 .dst_temps = .{ .mem, .unused },
115021 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
115022 .each = .{ .once = &.{
115023 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
115024 .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ },
115025 .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ },
115026 .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ },
115027 .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ },
115028 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
115029 .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ },
115030 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
115031 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
115032 } },
115033 }, .{
115034 .required_cc_abi = .win64,
115035 .required_features = .{ .@"64bit", null, null, null },
115036 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
115037 .dst_constraints = .{ .{ .scalar_remainder_unsigned_int = .{ .of = .dword, .is = .dword } }, .any },
115038 .patterns = &.{
115039 .{ .src = .{ .to_mem, .none, .none } },
115040 },
115041 .call_frame = .{ .alignment = .@"16" },
115042 .extra_temps = .{
115043 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
115044 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
115045 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
115046 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
115047 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
115048 .{ .type = .usize, .kind = .{ .extern_func = "__fixunstfei" } },
115049 .unused,
115050 .unused,
115051 .unused,
115052 .unused,
115053 .unused,
115054 },
115055 .dst_temps = .{ .mem, .unused },
115056 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
115057 .each = .{ .once = &.{
115058 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
115059 .{ ._, ._, .lea, .tmp1p, .mema(.dst0, .add_unaligned_size_sub_elem_size), ._, ._ },
115060 .{ .@"0:", ._, .mov, .tmp2p, .tmp1p, ._, ._ },
115061 .{ ._, ._, .mov, .tmp3d, .sa(.dst0, .add_bit_size), ._, ._ },
115062 .{ ._, ._, .lea, .tmp4p, .memi(.src0, .tmp0), ._, ._ },
115063 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
115064 .{ ._, ._, .lea, .tmp1p, .leaa(.tmp1, .sub_dst0_elem_size), ._, ._ },
115065 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
115066 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
115067 } },
112296115068 } }) catch |err| switch (err) {
112297115069 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f}", .{
112298115070 @tagName(air_tag),
......@@ -139664,6 +142436,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
139664142436 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
139665142437 } },
139666142438 }, .{
142439 .required_cc_abi = .sysv64,
139667142440 .required_features = .{ .avx, null, null, null },
139668142441 .dst_constraints = .{ .{ .float = .xword }, .any },
139669142442 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -139695,6 +142468,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
139695142468 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
139696142469 } },
139697142470 }, .{
142471 .required_cc_abi = .sysv64,
139698142472 .required_features = .{ .sse2, null, null, null },
139699142473 .dst_constraints = .{ .{ .float = .xword }, .any },
139700142474 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -139726,6 +142500,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
139726142500 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
139727142501 } },
139728142502 }, .{
142503 .required_cc_abi = .sysv64,
139729142504 .required_features = .{ .sse, null, null, null },
139730142505 .dst_constraints = .{ .{ .float = .xword }, .any },
139731142506 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -139756,6 +142531,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
139756142531 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
139757142532 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
139758142533 } },
142534 }, .{
142535 .required_cc_abi = .win64,
142536 .required_features = .{ .avx, null, null, null },
142537 .dst_constraints = .{ .{ .float = .xword }, .any },
142538 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
142539 .patterns = &.{
142540 .{ .src = .{ .to_mem, .none, .none } },
142541 },
142542 .call_frame = .{ .alignment = .@"16" },
142543 .extra_temps = .{
142544 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
142545 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
142546 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
142547 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
142548 .{ .type = .f128, .kind = .mem },
142549 .unused,
142550 .unused,
142551 .unused,
142552 .unused,
142553 .unused,
142554 .unused,
142555 },
142556 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
142557 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
142558 .each = .{ .once = &.{
142559 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
142560 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
142561 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
142562 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
142563 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
142564 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
142565 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
142566 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
142567 } },
142568 }, .{
142569 .required_cc_abi = .win64,
142570 .required_features = .{ .sse2, null, null, null },
142571 .dst_constraints = .{ .{ .float = .xword }, .any },
142572 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
142573 .patterns = &.{
142574 .{ .src = .{ .to_mem, .none, .none } },
142575 },
142576 .call_frame = .{ .alignment = .@"16" },
142577 .extra_temps = .{
142578 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
142579 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
142580 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
142581 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
142582 .{ .type = .f128, .kind = .mem },
142583 .unused,
142584 .unused,
142585 .unused,
142586 .unused,
142587 .unused,
142588 .unused,
142589 },
142590 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
142591 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
142592 .each = .{ .once = &.{
142593 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
142594 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
142595 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
142596 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
142597 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
142598 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
142599 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
142600 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
142601 } },
142602 }, .{
142603 .required_cc_abi = .win64,
142604 .required_features = .{ .sse2, null, null, null },
142605 .dst_constraints = .{ .{ .float = .xword }, .any },
142606 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
142607 .patterns = &.{
142608 .{ .src = .{ .to_mem, .none, .none } },
142609 },
142610 .call_frame = .{ .alignment = .@"16" },
142611 .extra_temps = .{
142612 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
142613 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
142614 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
142615 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
142616 .{ .type = .f128, .kind = .mem },
142617 .unused,
142618 .unused,
142619 .unused,
142620 .unused,
142621 .unused,
142622 .unused,
142623 },
142624 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
142625 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
142626 .each = .{ .once = &.{
142627 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
142628 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
142629 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
142630 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
142631 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
142632 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
142633 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
142634 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
142635 } },
139759142636 } },
139760142637 .Max => comptime &.{ .{
139761142638 .required_features = .{ .avx, null, null, null },
......@@ -149792,6 +152669,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
149792152669 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
149793152670 } },
149794152671 }, .{
152672 .required_cc_abi = .sysv64,
149795152673 .required_features = .{ .avx, null, null, null },
149796152674 .dst_constraints = .{ .{ .float = .xword }, .any },
149797152675 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -149823,6 +152701,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
149823152701 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
149824152702 } },
149825152703 }, .{
152704 .required_cc_abi = .sysv64,
149826152705 .required_features = .{ .sse2, null, null, null },
149827152706 .dst_constraints = .{ .{ .float = .xword }, .any },
149828152707 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -149854,6 +152733,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
149854152733 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
149855152734 } },
149856152735 }, .{
152736 .required_cc_abi = .sysv64,
149857152737 .required_features = .{ .sse, null, null, null },
149858152738 .dst_constraints = .{ .{ .float = .xword }, .any },
149859152739 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -149884,6 +152764,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
149884152764 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
149885152765 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
149886152766 } },
152767 }, .{
152768 .required_cc_abi = .win64,
152769 .required_features = .{ .avx, null, null, null },
152770 .dst_constraints = .{ .{ .float = .xword }, .any },
152771 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
152772 .patterns = &.{
152773 .{ .src = .{ .to_mem, .none, .none } },
152774 },
152775 .call_frame = .{ .alignment = .@"16" },
152776 .extra_temps = .{
152777 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
152778 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
152779 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
152780 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
152781 .{ .type = .f128, .kind = .mem },
152782 .unused,
152783 .unused,
152784 .unused,
152785 .unused,
152786 .unused,
152787 .unused,
152788 },
152789 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
152790 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
152791 .each = .{ .once = &.{
152792 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
152793 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
152794 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
152795 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
152796 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
152797 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
152798 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
152799 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
152800 } },
152801 }, .{
152802 .required_cc_abi = .win64,
152803 .required_features = .{ .sse2, null, null, null },
152804 .dst_constraints = .{ .{ .float = .xword }, .any },
152805 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
152806 .patterns = &.{
152807 .{ .src = .{ .to_mem, .none, .none } },
152808 },
152809 .call_frame = .{ .alignment = .@"16" },
152810 .extra_temps = .{
152811 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
152812 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
152813 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
152814 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
152815 .{ .type = .f128, .kind = .mem },
152816 .unused,
152817 .unused,
152818 .unused,
152819 .unused,
152820 .unused,
152821 .unused,
152822 },
152823 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
152824 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
152825 .each = .{ .once = &.{
152826 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
152827 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
152828 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
152829 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
152830 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
152831 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
152832 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
152833 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
152834 } },
152835 }, .{
152836 .required_cc_abi = .win64,
152837 .required_features = .{ .sse2, null, null, null },
152838 .dst_constraints = .{ .{ .float = .xword }, .any },
152839 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
152840 .patterns = &.{
152841 .{ .src = .{ .to_mem, .none, .none } },
152842 },
152843 .call_frame = .{ .alignment = .@"16" },
152844 .extra_temps = .{
152845 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
152846 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
152847 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
152848 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
152849 .{ .type = .f128, .kind = .mem },
152850 .unused,
152851 .unused,
152852 .unused,
152853 .unused,
152854 .unused,
152855 .unused,
152856 },
152857 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
152858 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
152859 .each = .{ .once = &.{
152860 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
152861 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
152862 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
152863 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
152864 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
152865 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
152866 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
152867 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
152868 } },
149887152869 } },
149888152870 .Add => comptime &.{ .{
149889152871 .required_features = .{ .avx, null, null, null },
......@@ -154411,6 +157393,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
154411157393 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
154412157394 } },
154413157395 }, .{
157396 .required_cc_abi = .sysv64,
154414157397 .required_features = .{ .avx, null, null, null },
154415157398 .dst_constraints = .{ .{ .float = .xword }, .any },
154416157399 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -154442,6 +157425,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
154442157425 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
154443157426 } },
154444157427 }, .{
157428 .required_cc_abi = .sysv64,
154445157429 .required_features = .{ .sse2, null, null, null },
154446157430 .dst_constraints = .{ .{ .float = .xword }, .any },
154447157431 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -154473,6 +157457,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
154473157457 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
154474157458 } },
154475157459 }, .{
157460 .required_cc_abi = .sysv64,
154476157461 .required_features = .{ .sse, null, null, null },
154477157462 .dst_constraints = .{ .{ .float = .xword }, .any },
154478157463 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -154503,6 +157488,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
154503157488 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
154504157489 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
154505157490 } },
157491 }, .{
157492 .required_cc_abi = .win64,
157493 .required_features = .{ .avx, null, null, null },
157494 .dst_constraints = .{ .{ .float = .xword }, .any },
157495 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
157496 .patterns = &.{
157497 .{ .src = .{ .to_mem, .none, .none } },
157498 },
157499 .call_frame = .{ .alignment = .@"16" },
157500 .extra_temps = .{
157501 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
157502 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
157503 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
157504 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
157505 .{ .type = .f128, .kind = .mem },
157506 .unused,
157507 .unused,
157508 .unused,
157509 .unused,
157510 .unused,
157511 .unused,
157512 },
157513 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
157514 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
157515 .each = .{ .once = &.{
157516 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
157517 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
157518 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
157519 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
157520 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
157521 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
157522 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
157523 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
157524 } },
157525 }, .{
157526 .required_cc_abi = .win64,
157527 .required_features = .{ .sse2, null, null, null },
157528 .dst_constraints = .{ .{ .float = .xword }, .any },
157529 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
157530 .patterns = &.{
157531 .{ .src = .{ .to_mem, .none, .none } },
157532 },
157533 .call_frame = .{ .alignment = .@"16" },
157534 .extra_temps = .{
157535 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
157536 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
157537 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
157538 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
157539 .{ .type = .f128, .kind = .mem },
157540 .unused,
157541 .unused,
157542 .unused,
157543 .unused,
157544 .unused,
157545 .unused,
157546 },
157547 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
157548 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
157549 .each = .{ .once = &.{
157550 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
157551 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
157552 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
157553 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
157554 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
157555 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
157556 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
157557 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
157558 } },
157559 }, .{
157560 .required_cc_abi = .win64,
157561 .required_features = .{ .sse, null, null, null },
157562 .dst_constraints = .{ .{ .float = .xword }, .any },
157563 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
157564 .patterns = &.{
157565 .{ .src = .{ .to_mem, .none, .none } },
157566 },
157567 .call_frame = .{ .alignment = .@"16" },
157568 .extra_temps = .{
157569 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
157570 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
157571 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
157572 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
157573 .{ .type = .f128, .kind = .mem },
157574 .unused,
157575 .unused,
157576 .unused,
157577 .unused,
157578 .unused,
157579 .unused,
157580 },
157581 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
157582 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
157583 .each = .{ .once = &.{
157584 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
157585 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
157586 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
157587 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
157588 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
157589 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
157590 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
157591 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
157592 } },
154506157593 } },
154507157594 .Mul => comptime &.{ .{
154508157595 .required_features = .{ .avx, null, null, null },
......@@ -157989,6 +161076,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
157989161076 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
157990161077 } },
157991161078 }, .{
161079 .required_cc_abi = .sysv64,
157992161080 .required_features = .{ .avx, null, null, null },
157993161081 .dst_constraints = .{ .{ .float = .xword }, .any },
157994161082 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -158020,6 +161108,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
158020161108 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
158021161109 } },
158022161110 }, .{
161111 .required_cc_abi = .sysv64,
158023161112 .required_features = .{ .sse2, null, null, null },
158024161113 .dst_constraints = .{ .{ .float = .xword }, .any },
158025161114 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -158051,6 +161140,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
158051161140 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
158052161141 } },
158053161142 }, .{
161143 .required_cc_abi = .sysv64,
158054161144 .required_features = .{ .sse, null, null, null },
158055161145 .dst_constraints = .{ .{ .float = .xword }, .any },
158056161146 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -158081,6 +161171,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
158081161171 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
158082161172 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
158083161173 } },
161174 }, .{
161175 .required_cc_abi = .win64,
161176 .required_features = .{ .avx, null, null, null },
161177 .dst_constraints = .{ .{ .float = .xword }, .any },
161178 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
161179 .patterns = &.{
161180 .{ .src = .{ .to_mem, .none, .none } },
161181 },
161182 .call_frame = .{ .alignment = .@"16" },
161183 .extra_temps = .{
161184 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
161185 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
161186 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
161187 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
161188 .{ .type = .f128, .kind = .mem },
161189 .unused,
161190 .unused,
161191 .unused,
161192 .unused,
161193 .unused,
161194 .unused,
161195 },
161196 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
161197 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
161198 .each = .{ .once = &.{
161199 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
161200 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
161201 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
161202 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
161203 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
161204 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
161205 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
161206 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
161207 } },
161208 }, .{
161209 .required_cc_abi = .win64,
161210 .required_features = .{ .sse2, null, null, null },
161211 .dst_constraints = .{ .{ .float = .xword }, .any },
161212 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
161213 .patterns = &.{
161214 .{ .src = .{ .to_mem, .none, .none } },
161215 },
161216 .call_frame = .{ .alignment = .@"16" },
161217 .extra_temps = .{
161218 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
161219 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
161220 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
161221 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
161222 .{ .type = .f128, .kind = .mem },
161223 .unused,
161224 .unused,
161225 .unused,
161226 .unused,
161227 .unused,
161228 .unused,
161229 },
161230 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
161231 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
161232 .each = .{ .once = &.{
161233 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
161234 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
161235 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
161236 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
161237 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
161238 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
161239 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
161240 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
161241 } },
161242 }, .{
161243 .required_cc_abi = .win64,
161244 .required_features = .{ .sse, null, null, null },
161245 .dst_constraints = .{ .{ .float = .xword }, .any },
161246 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
161247 .patterns = &.{
161248 .{ .src = .{ .to_mem, .none, .none } },
161249 },
161250 .call_frame = .{ .alignment = .@"16" },
161251 .extra_temps = .{
161252 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
161253 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
161254 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
161255 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
161256 .{ .type = .f128, .kind = .mem },
161257 .unused,
161258 .unused,
161259 .unused,
161260 .unused,
161261 .unused,
161262 .unused,
161263 },
161264 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
161265 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
161266 .each = .{ .once = &.{
161267 .{ ._, ._, .mov, .tmp0p, .sia(16, .src0, .sub_unaligned_size), ._, ._ },
161268 .{ ._, ._, .lea, .tmp1p, .mem(.src0), ._, ._ },
161269 .{ .@"0:", ._, .lea, .tmp2p, .memia(.src0, .tmp0, .add_unaligned_size), ._, ._ },
161270 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
161271 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
161272 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
161273 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
161274 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
161275 } },
158084161276 } },
158085161277 }) catch |err| switch (err) {
158086161278 error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{
......@@ -159711,6 +162903,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
159711162903 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
159712162904 } },
159713162905 }, .{
162906 .required_cc_abi = .sysv64,
159714162907 .required_features = .{ .avx, null, null, null },
159715162908 .dst_constraints = .{ .{ .float = .xword }, .any },
159716162909 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -159742,6 +162935,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
159742162935 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
159743162936 } },
159744162937 }, .{
162938 .required_cc_abi = .sysv64,
159745162939 .required_features = .{ .sse2, null, null, null },
159746162940 .dst_constraints = .{ .{ .float = .xword }, .any },
159747162941 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -159773,6 +162967,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
159773162967 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
159774162968 } },
159775162969 }, .{
162970 .required_cc_abi = .sysv64,
159776162971 .required_features = .{ .sse, null, null, null },
159777162972 .dst_constraints = .{ .{ .float = .xword }, .any },
159778162973 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -159803,6 +162998,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
159803162998 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
159804162999 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
159805163000 } },
163001 }, .{
163002 .required_cc_abi = .win64,
163003 .required_features = .{ .avx, null, null, null },
163004 .dst_constraints = .{ .{ .float = .xword }, .any },
163005 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
163006 .patterns = &.{
163007 .{ .src = .{ .to_mem, .none, .none } },
163008 },
163009 .call_frame = .{ .alignment = .@"16" },
163010 .extra_temps = .{
163011 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
163012 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
163013 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
163014 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
163015 .{ .type = .f128, .kind = .mem },
163016 .unused,
163017 .unused,
163018 .unused,
163019 .unused,
163020 .unused,
163021 .unused,
163022 },
163023 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
163024 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
163025 .each = .{ .once = &.{
163026 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
163027 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
163028 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
163029 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
163030 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
163031 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
163032 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
163033 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163034 } },
163035 }, .{
163036 .required_cc_abi = .win64,
163037 .required_features = .{ .sse2, null, null, null },
163038 .dst_constraints = .{ .{ .float = .xword }, .any },
163039 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
163040 .patterns = &.{
163041 .{ .src = .{ .to_mem, .none, .none } },
163042 },
163043 .call_frame = .{ .alignment = .@"16" },
163044 .extra_temps = .{
163045 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
163046 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
163047 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
163048 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
163049 .{ .type = .f128, .kind = .mem },
163050 .unused,
163051 .unused,
163052 .unused,
163053 .unused,
163054 .unused,
163055 .unused,
163056 },
163057 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
163058 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
163059 .each = .{ .once = &.{
163060 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
163061 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
163062 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
163063 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
163064 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
163065 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
163066 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
163067 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163068 } },
163069 }, .{
163070 .required_cc_abi = .win64,
163071 .required_features = .{ .sse2, null, null, null },
163072 .dst_constraints = .{ .{ .float = .xword }, .any },
163073 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
163074 .patterns = &.{
163075 .{ .src = .{ .to_mem, .none, .none } },
163076 },
163077 .call_frame = .{ .alignment = .@"16" },
163078 .extra_temps = .{
163079 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
163080 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
163081 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
163082 .{ .type = .usize, .kind = .{ .extern_func = "fminq" } },
163083 .{ .type = .f128, .kind = .mem },
163084 .unused,
163085 .unused,
163086 .unused,
163087 .unused,
163088 .unused,
163089 .unused,
163090 },
163091 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
163092 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
163093 .each = .{ .once = &.{
163094 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
163095 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
163096 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
163097 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
163098 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
163099 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
163100 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
163101 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163102 } },
159806163103 } },
159807163104 .Max => comptime &.{ .{
159808163105 .required_features = .{ .f16c, null, null, null },
......@@ -161403,6 +164700,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
161403164700 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
161404164701 } },
161405164702 }, .{
164703 .required_cc_abi = .sysv64,
161406164704 .required_features = .{ .avx, null, null, null },
161407164705 .dst_constraints = .{ .{ .float = .xword }, .any },
161408164706 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -161434,6 +164732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
161434164732 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
161435164733 } },
161436164734 }, .{
164735 .required_cc_abi = .sysv64,
161437164736 .required_features = .{ .sse2, null, null, null },
161438164737 .dst_constraints = .{ .{ .float = .xword }, .any },
161439164738 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -161465,6 +164764,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
161465164764 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
161466164765 } },
161467164766 }, .{
164767 .required_cc_abi = .sysv64,
161468164768 .required_features = .{ .sse, null, null, null },
161469164769 .dst_constraints = .{ .{ .float = .xword }, .any },
161470164770 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -161495,6 +164795,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
161495164795 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
161496164796 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
161497164797 } },
164798 }, .{
164799 .required_cc_abi = .win64,
164800 .required_features = .{ .avx, null, null, null },
164801 .dst_constraints = .{ .{ .float = .xword }, .any },
164802 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
164803 .patterns = &.{
164804 .{ .src = .{ .to_mem, .none, .none } },
164805 },
164806 .call_frame = .{ .alignment = .@"16" },
164807 .extra_temps = .{
164808 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
164809 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
164810 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
164811 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
164812 .{ .type = .f128, .kind = .mem },
164813 .unused,
164814 .unused,
164815 .unused,
164816 .unused,
164817 .unused,
164818 .unused,
164819 },
164820 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
164821 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
164822 .each = .{ .once = &.{
164823 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
164824 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
164825 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
164826 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
164827 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
164828 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
164829 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
164830 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
164831 } },
164832 }, .{
164833 .required_cc_abi = .win64,
164834 .required_features = .{ .sse2, null, null, null },
164835 .dst_constraints = .{ .{ .float = .xword }, .any },
164836 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
164837 .patterns = &.{
164838 .{ .src = .{ .to_mem, .none, .none } },
164839 },
164840 .call_frame = .{ .alignment = .@"16" },
164841 .extra_temps = .{
164842 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
164843 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
164844 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
164845 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
164846 .{ .type = .f128, .kind = .mem },
164847 .unused,
164848 .unused,
164849 .unused,
164850 .unused,
164851 .unused,
164852 .unused,
164853 },
164854 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
164855 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
164856 .each = .{ .once = &.{
164857 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
164858 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
164859 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
164860 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
164861 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
164862 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
164863 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
164864 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
164865 } },
164866 }, .{
164867 .required_cc_abi = .win64,
164868 .required_features = .{ .sse2, null, null, null },
164869 .dst_constraints = .{ .{ .float = .xword }, .any },
164870 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
164871 .patterns = &.{
164872 .{ .src = .{ .to_mem, .none, .none } },
164873 },
164874 .call_frame = .{ .alignment = .@"16" },
164875 .extra_temps = .{
164876 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
164877 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
164878 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
164879 .{ .type = .usize, .kind = .{ .extern_func = "fmaxq" } },
164880 .{ .type = .f128, .kind = .mem },
164881 .unused,
164882 .unused,
164883 .unused,
164884 .unused,
164885 .unused,
164886 .unused,
164887 },
164888 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
164889 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
164890 .each = .{ .once = &.{
164891 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
164892 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
164893 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
164894 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
164895 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
164896 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
164897 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
164898 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
164899 } },
161498164900 } },
161499164901 .Add => comptime &.{ .{
161500164902 .required_features = .{ .f16c, .fast_hops, null, null },
......@@ -163701,6 +167103,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
163701167103 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
163702167104 } },
163703167105 }, .{
167106 .required_cc_abi = .sysv64,
163704167107 .required_features = .{ .avx, null, null, null },
163705167108 .dst_constraints = .{ .{ .float = .xword }, .any },
163706167109 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -163732,6 +167135,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
163732167135 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163733167136 } },
163734167137 }, .{
167138 .required_cc_abi = .sysv64,
163735167139 .required_features = .{ .sse2, null, null, null },
163736167140 .dst_constraints = .{ .{ .float = .xword }, .any },
163737167141 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -163763,6 +167167,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
163763167167 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163764167168 } },
163765167169 }, .{
167170 .required_cc_abi = .sysv64,
163766167171 .required_features = .{ .sse, null, null, null },
163767167172 .dst_constraints = .{ .{ .float = .xword }, .any },
163768167173 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -163793,6 +167198,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
163793167198 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
163794167199 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
163795167200 } },
167201 }, .{
167202 .required_cc_abi = .win64,
167203 .required_features = .{ .avx, null, null, null },
167204 .dst_constraints = .{ .{ .float = .xword }, .any },
167205 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
167206 .patterns = &.{
167207 .{ .src = .{ .to_mem, .none, .none } },
167208 },
167209 .call_frame = .{ .alignment = .@"16" },
167210 .extra_temps = .{
167211 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
167212 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
167213 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
167214 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
167215 .{ .type = .f128, .kind = .mem },
167216 .unused,
167217 .unused,
167218 .unused,
167219 .unused,
167220 .unused,
167221 .unused,
167222 },
167223 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
167224 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
167225 .each = .{ .once = &.{
167226 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
167227 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
167228 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
167229 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
167230 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
167231 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
167232 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
167233 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
167234 } },
167235 }, .{
167236 .required_cc_abi = .win64,
167237 .required_features = .{ .sse2, null, null, null },
167238 .dst_constraints = .{ .{ .float = .xword }, .any },
167239 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
167240 .patterns = &.{
167241 .{ .src = .{ .to_mem, .none, .none } },
167242 },
167243 .call_frame = .{ .alignment = .@"16" },
167244 .extra_temps = .{
167245 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
167246 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
167247 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
167248 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
167249 .{ .type = .f128, .kind = .mem },
167250 .unused,
167251 .unused,
167252 .unused,
167253 .unused,
167254 .unused,
167255 .unused,
167256 },
167257 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
167258 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
167259 .each = .{ .once = &.{
167260 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
167261 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
167262 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
167263 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
167264 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
167265 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
167266 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
167267 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
167268 } },
167269 }, .{
167270 .required_cc_abi = .win64,
167271 .required_features = .{ .sse, null, null, null },
167272 .dst_constraints = .{ .{ .float = .xword }, .any },
167273 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
167274 .patterns = &.{
167275 .{ .src = .{ .to_mem, .none, .none } },
167276 },
167277 .call_frame = .{ .alignment = .@"16" },
167278 .extra_temps = .{
167279 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
167280 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
167281 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
167282 .{ .type = .usize, .kind = .{ .extern_func = "__addtf3" } },
167283 .{ .type = .f128, .kind = .mem },
167284 .unused,
167285 .unused,
167286 .unused,
167287 .unused,
167288 .unused,
167289 .unused,
167290 },
167291 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
167292 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
167293 .each = .{ .once = &.{
167294 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
167295 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
167296 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
167297 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
167298 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
167299 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
167300 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
167301 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
167302 } },
163796167303 } },
163797167304 .Mul => comptime &.{ .{
163798167305 .required_features = .{ .f16c, null, null, null },
......@@ -165283,6 +168790,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
165283168790 .{ ._, .f_cw, .ld, .tmp1w, ._, ._, ._ },
165284168791 } },
165285168792 }, .{
168793 .required_cc_abi = .sysv64,
165286168794 .required_features = .{ .avx, null, null, null },
165287168795 .dst_constraints = .{ .{ .float = .xword }, .any },
165288168796 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -165314,6 +168822,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
165314168822 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
165315168823 } },
165316168824 }, .{
168825 .required_cc_abi = .sysv64,
165317168826 .required_features = .{ .sse2, null, null, null },
165318168827 .dst_constraints = .{ .{ .float = .xword }, .any },
165319168828 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -165345,6 +168854,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
165345168854 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
165346168855 } },
165347168856 }, .{
168857 .required_cc_abi = .sysv64,
165348168858 .required_features = .{ .sse, null, null, null },
165349168859 .dst_constraints = .{ .{ .float = .xword }, .any },
165350168860 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
......@@ -165375,6 +168885,108 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
165375168885 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
165376168886 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
165377168887 } },
168888 }, .{
168889 .required_cc_abi = .win64,
168890 .required_features = .{ .avx, null, null, null },
168891 .dst_constraints = .{ .{ .float = .xword }, .any },
168892 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
168893 .patterns = &.{
168894 .{ .src = .{ .to_mem, .none, .none } },
168895 },
168896 .call_frame = .{ .alignment = .@"16" },
168897 .extra_temps = .{
168898 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
168899 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
168900 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
168901 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
168902 .{ .type = .f128, .kind = .mem },
168903 .unused,
168904 .unused,
168905 .unused,
168906 .unused,
168907 .unused,
168908 .unused,
168909 },
168910 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
168911 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
168912 .each = .{ .once = &.{
168913 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
168914 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
168915 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
168916 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
168917 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
168918 .{ ._, .v_dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
168919 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
168920 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
168921 } },
168922 }, .{
168923 .required_cc_abi = .win64,
168924 .required_features = .{ .sse2, null, null, null },
168925 .dst_constraints = .{ .{ .float = .xword }, .any },
168926 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
168927 .patterns = &.{
168928 .{ .src = .{ .to_mem, .none, .none } },
168929 },
168930 .call_frame = .{ .alignment = .@"16" },
168931 .extra_temps = .{
168932 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
168933 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
168934 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
168935 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
168936 .{ .type = .f128, .kind = .mem },
168937 .unused,
168938 .unused,
168939 .unused,
168940 .unused,
168941 .unused,
168942 .unused,
168943 },
168944 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
168945 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
168946 .each = .{ .once = &.{
168947 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
168948 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
168949 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
168950 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
168951 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
168952 .{ ._, ._dqa, .mov, .lea(.tmp1x), .dst0x, ._, ._ },
168953 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
168954 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
168955 } },
168956 }, .{
168957 .required_cc_abi = .win64,
168958 .required_features = .{ .sse, null, null, null },
168959 .dst_constraints = .{ .{ .float = .xword }, .any },
168960 .src_constraints = .{ .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } }, .any, .any },
168961 .patterns = &.{
168962 .{ .src = .{ .to_mem, .none, .none } },
168963 },
168964 .call_frame = .{ .alignment = .@"16" },
168965 .extra_temps = .{
168966 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
168967 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
168968 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
168969 .{ .type = .usize, .kind = .{ .extern_func = "__multf3" } },
168970 .{ .type = .f128, .kind = .mem },
168971 .unused,
168972 .unused,
168973 .unused,
168974 .unused,
168975 .unused,
168976 .unused,
168977 },
168978 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
168979 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
168980 .each = .{ .once = &.{
168981 .{ ._, ._, .mov, .tmp0d, .sia(-32, .src0, .add_unaligned_size), ._, ._ },
168982 .{ ._, ._, .lea, .tmp1p, .memad(.src0, .add_unaligned_size, -16), ._, ._ },
168983 .{ .@"0:", ._, .lea, .tmp2p, .memi(.src0, .tmp0), ._, ._ },
168984 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
168985 .{ ._, ._, .lea, .tmp1p, .mem(.tmp4), ._, ._ },
168986 .{ ._, ._ps, .mova, .lea(.tmp1x), .dst0x, ._, ._ },
168987 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
168988 .{ ._, ._nb, .j, .@"0b", ._, ._, ._ },
168989 } },
165378168990 } },
165379168991 }) catch |err| switch (err) {
165380168992 error.SelectFailed => return cg.fail("failed to select {s}.{s} {f} {f}", .{
......@@ -169007,6 +172619,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169007172619 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
169008172620 } },
169009172621 }, .{
172622 .required_cc_abi = .sysv64,
169010172623 .required_features = .{ .sse, null, null, null },
169011172624 .src_constraints = .{
169012172625 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -169040,6 +172653,40 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169040172653 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
169041172654 } },
169042172655 }, .{
172656 .required_cc_abi = .win64,
172657 .required_features = .{ .sse, null, null, null },
172658 .src_constraints = .{
172659 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
172660 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
172661 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
172662 },
172663 .patterns = &.{
172664 .{ .src = .{ .to_mem, .to_mem, .to_mem } },
172665 },
172666 .call_frame = .{ .alignment = .@"16" },
172667 .extra_temps = .{
172668 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172669 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
172670 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
172671 .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } },
172672 .unused,
172673 .unused,
172674 .unused,
172675 .unused,
172676 .unused,
172677 .unused,
172678 .unused,
172679 },
172680 .dst_temps = .{ .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } }, .unused },
172681 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
172682 .each = .{ .once = &.{
172683 .{ ._, ._, .lea, .tmp0p, .mem(.src0), ._, ._ },
172684 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
172685 .{ ._, ._, .lea, .tmp2p, .mem(.src2), ._, ._ },
172686 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
172687 } },
172688 }, .{
172689 .required_cc_abi = .sysv64,
169043172690 .required_features = .{ .avx, null, null, null },
169044172691 .src_constraints = .{
169045172692 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -169076,6 +172723,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169076172723 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
169077172724 } },
169078172725 }, .{
172726 .required_cc_abi = .sysv64,
169079172727 .required_features = .{ .sse2, null, null, null },
169080172728 .src_constraints = .{
169081172729 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -169112,6 +172760,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169112172760 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
169113172761 } },
169114172762 }, .{
172763 .required_cc_abi = .sysv64,
169115172764 .required_features = .{ .sse, null, null, null },
169116172765 .src_constraints = .{
169117172766 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
......@@ -169147,6 +172796,117 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
169147172796 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
169148172797 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
169149172798 } },
172799 }, .{
172800 .required_cc_abi = .win64,
172801 .required_features = .{ .avx, null, null, null },
172802 .src_constraints = .{
172803 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172804 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172805 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172806 },
172807 .patterns = &.{
172808 .{ .src = .{ .to_mem, .to_mem, .to_mem } },
172809 },
172810 .call_frame = .{ .alignment = .@"16" },
172811 .extra_temps = .{
172812 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
172813 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172814 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
172815 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
172816 .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } },
172817 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172818 .unused,
172819 .unused,
172820 .unused,
172821 .unused,
172822 .unused,
172823 },
172824 .dst_temps = .{ .mem, .unused },
172825 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
172826 .each = .{ .once = &.{
172827 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
172828 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
172829 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
172830 .{ ._, ._, .lea, .tmp3p, .memi(.src2, .tmp0), ._, ._ },
172831 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
172832 .{ ._, .v_dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
172833 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
172834 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
172835 } },
172836 }, .{
172837 .required_cc_abi = .win64,
172838 .required_features = .{ .sse2, null, null, null },
172839 .src_constraints = .{
172840 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172841 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172842 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172843 },
172844 .patterns = &.{
172845 .{ .src = .{ .to_mem, .to_mem, .to_mem } },
172846 },
172847 .call_frame = .{ .alignment = .@"16" },
172848 .extra_temps = .{
172849 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
172850 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172851 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
172852 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
172853 .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } },
172854 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172855 .unused,
172856 .unused,
172857 .unused,
172858 .unused,
172859 .unused,
172860 },
172861 .dst_temps = .{ .mem, .unused },
172862 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
172863 .each = .{ .once = &.{
172864 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
172865 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
172866 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
172867 .{ ._, ._, .lea, .tmp3p, .memi(.src2, .tmp0), ._, ._ },
172868 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
172869 .{ ._, ._dqa, .mov, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
172870 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
172871 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
172872 } },
172873 }, .{
172874 .required_cc_abi = .win64,
172875 .required_features = .{ .sse, null, null, null },
172876 .src_constraints = .{
172877 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172878 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172879 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
172880 },
172881 .patterns = &.{
172882 .{ .src = .{ .to_mem, .to_mem, .to_mem } },
172883 },
172884 .call_frame = .{ .alignment = .@"16" },
172885 .extra_temps = .{
172886 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
172887 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172888 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 1, .at = 1 } } },
172889 .{ .type = .usize, .kind = .{ .param_gpr = .{ .cc = .ccc, .after = 2, .at = 2 } } },
172890 .{ .type = .usize, .kind = .{ .extern_func = "fmaq" } },
172891 .{ .type = .f128, .kind = .{ .ret_sse = .{ .cc = .ccc, .after = 0, .at = 0 } } },
172892 .unused,
172893 .unused,
172894 .unused,
172895 .unused,
172896 .unused,
172897 },
172898 .dst_temps = .{ .mem, .unused },
172899 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
172900 .each = .{ .once = &.{
172901 .{ ._, ._, .mov, .tmp0d, .sia(-16, .src0, .add_unaligned_size), ._, ._ },
172902 .{ .@"0:", ._, .lea, .tmp1p, .memi(.src0, .tmp0), ._, ._ },
172903 .{ ._, ._, .lea, .tmp2p, .memi(.src1, .tmp0), ._, ._ },
172904 .{ ._, ._, .lea, .tmp3p, .memi(.src2, .tmp0), ._, ._ },
172905 .{ ._, ._, .call, .tmp4d, ._, ._, ._ },
172906 .{ ._, ._ps, .mova, .memi(.dst0x, .tmp0), .tmp5x, ._, ._ },
172907 .{ ._, ._, .sub, .tmp0d, .si(16), ._, ._ },
172908 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
172909 } },
169150172910 } }) catch |err| switch (err) {
169151172911 error.SelectFailed => return cg.fail("failed to select {s} {f} {f} {f} {f}", .{
169152172912 @tagName(air_tag),
......@@ -170541,4887 +174301,78 @@ fn copyToRegisterWithInstTracking(
170541174301 return MCValue{ .register = reg };
170542174302}
170543174303
170544fn airAlloc(self: *CodeGen, inst: Air.Inst.Index) !void {
170545 const result = MCValue{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } };
170546 return self.finishAir(inst, result, .{ .none, .none, .none });
170547}
170548
170549fn airRetPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
170550 const result: MCValue = switch (self.ret_mcv.long) {
170551 else => unreachable,
170552 .none => .{ .lea_frame = .{ .index = try self.allocMemPtr(inst) } },
170553 .load_frame => .{ .register_offset = .{
170554 .reg = (try self.copyToRegisterWithInstTracking(
170555 inst,
170556 self.typeOfIndex(inst),
170557 self.ret_mcv.long,
170558 )).register,
170559 .off = self.ret_mcv.short.indirect.off,
170560 } },
170561 };
170562 return self.finishAir(inst, result, .{ .none, .none, .none });
170563}
170564
170565fn airFptrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
170566 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
170567 const dst_ty = self.typeOfIndex(inst);
170568 const dst_bits = dst_ty.floatBits(self.target);
170569 const src_ty = self.typeOf(ty_op.operand);
170570 const src_bits = src_ty.floatBits(self.target);
170571
170572 const result = result: {
170573 if (switch (dst_bits) {
170574 16 => switch (src_bits) {
170575 32 => !self.hasFeature(.f16c),
170576 64, 80, 128 => true,
170577 else => unreachable,
170578 },
170579 32 => switch (src_bits) {
170580 64 => false,
170581 80, 128 => true,
170582 else => unreachable,
170583 },
170584 64 => switch (src_bits) {
170585 80, 128 => true,
170586 else => unreachable,
170587 },
170588 80 => switch (src_bits) {
170589 128 => true,
170590 else => unreachable,
170591 },
170592 else => unreachable,
170593 }) {
170594 var sym_buf: ["__trunc?f?f2".len]u8 = undefined;
170595 break :result try self.genCall(.{ .extern_func = .{
170596 .return_type = self.floatCompilerRtAbiType(dst_ty, src_ty).toIntern(),
170597 .param_types = &.{self.floatCompilerRtAbiType(src_ty, dst_ty).toIntern()},
170598 .sym = std.fmt.bufPrint(&sym_buf, "__trunc{c}f{c}f2", .{
170599 floatCompilerRtAbiName(src_bits),
170600 floatCompilerRtAbiName(dst_bits),
170601 }) catch unreachable,
170602 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
170603 }
170604
170605 const src_mcv = try self.resolveInst(ty_op.operand);
170606 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170607 src_mcv
170608 else
170609 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
170610 const dst_reg = dst_mcv.getReg().?.to128();
170611 const dst_lock = self.register_manager.lockReg(dst_reg);
170612 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
170613
170614 if (dst_bits == 16) {
170615 assert(self.hasFeature(.f16c));
170616 switch (src_bits) {
170617 32 => {
170618 const mat_src_reg = if (src_mcv.isRegister())
170619 src_mcv.getReg().?
170620 else
170621 try self.copyToTmpRegister(src_ty, src_mcv);
170622 try self.asmRegisterRegisterImmediate(
170623 .{ .v_, .cvtps2ph },
170624 dst_reg,
170625 mat_src_reg.to128(),
170626 bits.RoundMode.imm(.{}),
170627 );
170628 },
170629 else => unreachable,
170630 }
170631 } else {
170632 assert(src_bits == 64 and dst_bits == 32);
170633 if (self.hasFeature(.avx)) if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
170634 .{ .v_ss, .cvtsd2 },
170635 dst_reg,
170636 dst_reg,
170637 try src_mcv.mem(self, .{ .size = .qword }),
170638 ) else try self.asmRegisterRegisterRegister(
170639 .{ .v_ss, .cvtsd2 },
170640 dst_reg,
170641 dst_reg,
170642 (if (src_mcv.isRegister())
170643 src_mcv.getReg().?
170644 else
170645 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
170646 ) else if (src_mcv.isBase()) try self.asmRegisterMemory(
170647 .{ ._ss, .cvtsd2 },
170648 dst_reg,
170649 try src_mcv.mem(self, .{ .size = .qword }),
170650 ) else try self.asmRegisterRegister(
170651 .{ ._ss, .cvtsd2 },
170652 dst_reg,
170653 (if (src_mcv.isRegister())
170654 src_mcv.getReg().?
170655 else
170656 try self.copyToTmpRegister(src_ty, src_mcv)).to128(),
170657 );
170658 }
170659 break :result dst_mcv;
170660 };
170661 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
170662}
170663
170664fn airFpext(self: *CodeGen, inst: Air.Inst.Index) !void {
170665 const pt = self.pt;
170666 const zcu = pt.zcu;
170667 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
170668 const dst_ty = self.typeOfIndex(inst);
170669 const dst_scalar_ty = dst_ty.scalarType(zcu);
170670 const dst_bits = dst_scalar_ty.floatBits(self.target);
170671 const src_ty = self.typeOf(ty_op.operand);
170672 const src_scalar_ty = src_ty.scalarType(zcu);
170673 const src_bits = src_scalar_ty.floatBits(self.target);
170674
170675 const result = result: {
170676 if (switch (src_bits) {
170677 16 => switch (dst_bits) {
170678 32, 64 => !self.hasFeature(.f16c),
170679 80, 128 => true,
170680 else => unreachable,
170681 },
170682 32 => switch (dst_bits) {
170683 64 => false,
170684 80, 128 => true,
170685 else => unreachable,
170686 },
170687 64 => switch (dst_bits) {
170688 80, 128 => true,
170689 else => unreachable,
170690 },
170691 80 => switch (dst_bits) {
170692 128 => true,
170693 else => unreachable,
170694 },
170695 else => unreachable,
170696 }) {
170697 if (dst_ty.isVector(zcu)) break :result null;
170698 var sym_buf: ["__extend?f?f2".len]u8 = undefined;
170699 break :result try self.genCall(.{ .extern_func = .{
170700 .return_type = self.floatCompilerRtAbiType(dst_scalar_ty, src_scalar_ty).toIntern(),
170701 .param_types = &.{self.floatCompilerRtAbiType(src_scalar_ty, dst_scalar_ty).toIntern()},
170702 .sym = std.fmt.bufPrint(&sym_buf, "__extend{c}f{c}f2", .{
170703 floatCompilerRtAbiName(src_bits),
170704 floatCompilerRtAbiName(dst_bits),
170705 }) catch unreachable,
170706 } }, &.{src_scalar_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
170707 }
170708
170709 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
170710 const src_mcv = try self.resolveInst(ty_op.operand);
170711 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170712 src_mcv
170713 else
170714 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
170715 const dst_reg = dst_mcv.getReg().?;
170716 const dst_alias = registerAlias(dst_reg, @intCast(@max(dst_ty.abiSize(zcu), 16)));
170717 const dst_lock = self.register_manager.lockReg(dst_reg);
170718 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
170719
170720 const vec_len = if (dst_ty.isVector(zcu)) dst_ty.vectorLen(zcu) else 1;
170721 if (src_bits == 16) {
170722 assert(self.hasFeature(.f16c));
170723 const mat_src_reg = if (src_mcv.isRegister())
170724 src_mcv.getReg().?
170725 else
170726 try self.copyToTmpRegister(src_ty, src_mcv);
170727 try self.asmRegisterRegister(
170728 .{ .v_ps, .cvtph2 },
170729 dst_alias,
170730 registerAlias(mat_src_reg, src_abi_size),
170731 );
170732 switch (dst_bits) {
170733 32 => {},
170734 64 => try self.asmRegisterRegisterRegister(
170735 .{ .v_sd, .cvtss2 },
170736 dst_alias,
170737 dst_alias,
170738 dst_alias,
170739 ),
170740 else => unreachable,
170741 }
170742 } else {
170743 assert(src_bits == 32 and dst_bits == 64);
170744 if (self.hasFeature(.avx)) switch (vec_len) {
170745 1 => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
170746 .{ .v_sd, .cvtss2 },
170747 dst_alias,
170748 dst_alias,
170749 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
170750 ) else try self.asmRegisterRegisterRegister(
170751 .{ .v_sd, .cvtss2 },
170752 dst_alias,
170753 dst_alias,
170754 registerAlias(if (src_mcv.isRegister())
170755 src_mcv.getReg().?
170756 else
170757 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
170758 ),
170759 2...4 => if (src_mcv.isBase()) try self.asmRegisterMemory(
170760 .{ .v_pd, .cvtps2 },
170761 dst_alias,
170762 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
170763 ) else try self.asmRegisterRegister(
170764 .{ .v_pd, .cvtps2 },
170765 dst_alias,
170766 registerAlias(if (src_mcv.isRegister())
170767 src_mcv.getReg().?
170768 else
170769 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
170770 ),
170771 else => break :result null,
170772 } else if (src_mcv.isBase()) try self.asmRegisterMemory(
170773 switch (vec_len) {
170774 1 => .{ ._sd, .cvtss2 },
170775 2 => .{ ._pd, .cvtps2 },
170776 else => break :result null,
170777 },
170778 dst_alias,
170779 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
170780 ) else try self.asmRegisterRegister(
170781 switch (vec_len) {
170782 1 => .{ ._sd, .cvtss2 },
170783 2 => .{ ._pd, .cvtps2 },
170784 else => break :result null,
170785 },
170786 dst_alias,
170787 registerAlias(if (src_mcv.isRegister())
170788 src_mcv.getReg().?
170789 else
170790 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
170791 );
170792 }
170793 break :result dst_mcv;
170794 } orelse return self.fail("TODO implement airFpext from {f} to {f}", .{
170795 src_ty.fmt(pt), dst_ty.fmt(pt),
170796 });
170797 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
170798}
170799
170800fn airIntCast(self: *CodeGen, inst: Air.Inst.Index) !void {
170801 const pt = self.pt;
170802 const zcu = pt.zcu;
170803 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
170804 const src_ty = self.typeOf(ty_op.operand);
170805 const dst_ty = self.typeOfIndex(inst);
170806
170807 const result = @as(?MCValue, result: {
170808 const src_abi_size: u31 = @intCast(src_ty.abiSize(zcu));
170809 const dst_abi_size: u31 = @intCast(dst_ty.abiSize(zcu));
170810
170811 const src_int_info = src_ty.intInfo(zcu);
170812 const dst_int_info = dst_ty.intInfo(zcu);
170813 const extend = switch (src_int_info.signedness) {
170814 .signed => dst_int_info,
170815 .unsigned => src_int_info,
170816 }.signedness;
170817
170818 const src_mcv = try self.resolveInst(ty_op.operand);
170819 if (dst_ty.isVector(zcu)) {
170820 const max_abi_size = @max(dst_abi_size, src_abi_size);
170821 const has_avx = self.hasFeature(.avx);
170822
170823 const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu);
170824 const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu);
170825 switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) {
170826 .lt => {
170827 if (max_abi_size > self.vectorSize(.int)) break :result null;
170828 const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
170829 else => break :result null,
170830 1 => switch (src_elem_abi_size) {
170831 else => break :result null,
170832 2 => switch (dst_int_info.signedness) {
170833 .signed => if (has_avx) .{ .vp_b, .ackssw } else .{ .p_b, .ackssw },
170834 .unsigned => if (has_avx) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
170835 },
170836 },
170837 2 => switch (src_elem_abi_size) {
170838 else => break :result null,
170839 4 => switch (dst_int_info.signedness) {
170840 .signed => if (has_avx) .{ .vp_w, .ackssd } else .{ .p_w, .ackssd },
170841 .unsigned => if (has_avx)
170842 .{ .vp_w, .ackusd }
170843 else if (self.hasFeature(.sse4_1))
170844 .{ .p_w, .ackusd }
170845 else
170846 break :result null,
170847 },
170848 },
170849 };
170850
170851 const dst_mcv: MCValue = if (src_mcv.isRegister() and
170852 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170853 src_mcv
170854 else if (has_avx and src_mcv.isRegister())
170855 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
170856 else
170857 try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv);
170858 const dst_reg = dst_mcv.getReg().?;
170859 const dst_alias = registerAlias(dst_reg, dst_abi_size);
170860
170861 if (has_avx) try self.asmRegisterRegisterRegister(
170862 mir_tag,
170863 dst_alias,
170864 registerAlias(if (src_mcv.isRegister())
170865 src_mcv.getReg().?
170866 else
170867 dst_reg, src_abi_size),
170868 dst_alias,
170869 ) else try self.asmRegisterRegister(
170870 mir_tag,
170871 dst_alias,
170872 dst_alias,
170873 );
170874 break :result dst_mcv;
170875 },
170876 .eq => if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170877 break :result src_mcv
170878 else {
170879 const dst_mcv = try self.allocRegOrMem(inst, true);
170880 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
170881 break :result dst_mcv;
170882 },
170883 .gt => if (self.hasFeature(.sse4_1)) {
170884 if (max_abi_size > self.vectorSize(.int)) break :result null;
170885 const mir_tag: Mir.Inst.FixedTag = .{ switch (dst_elem_abi_size) {
170886 else => break :result null,
170887 2 => if (has_avx) .vp_w else .p_w,
170888 4 => if (has_avx) .vp_d else .p_d,
170889 8 => if (has_avx) .vp_q else .p_q,
170890 }, switch (src_elem_abi_size) {
170891 else => break :result null,
170892 1 => switch (extend) {
170893 .signed => .movsxb,
170894 .unsigned => .movzxb,
170895 },
170896 2 => switch (extend) {
170897 .signed => .movsxw,
170898 .unsigned => .movzxw,
170899 },
170900 4 => switch (extend) {
170901 .signed => .movsxd,
170902 .unsigned => .movzxd,
170903 },
170904 } };
170905
170906 const dst_mcv: MCValue = if (src_mcv.isRegister() and
170907 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170908 src_mcv
170909 else
170910 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) };
170911 const dst_reg = dst_mcv.getReg().?;
170912 const dst_alias = registerAlias(dst_reg, dst_abi_size);
170913
170914 if (src_mcv.isBase()) try self.asmRegisterMemory(
170915 mir_tag,
170916 dst_alias,
170917 try src_mcv.mem(self, .{ .size = self.memSize(src_ty) }),
170918 ) else try self.asmRegisterRegister(
170919 mir_tag,
170920 dst_alias,
170921 registerAlias(if (src_mcv.isRegister())
170922 src_mcv.getReg().?
170923 else
170924 try self.copyToTmpRegister(src_ty, src_mcv), src_abi_size),
170925 );
170926 break :result dst_mcv;
170927 } else {
170928 const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) {
170929 else => break :result null,
170930 2 => switch (src_elem_abi_size) {
170931 else => break :result null,
170932 1 => .{ .p_, .unpcklbw },
170933 },
170934 4 => switch (src_elem_abi_size) {
170935 else => break :result null,
170936 2 => .{ .p_, .unpcklwd },
170937 },
170938 8 => switch (src_elem_abi_size) {
170939 else => break :result null,
170940 2 => .{ .p_, .unpckldq },
170941 },
170942 };
170943
170944 const dst_mcv: MCValue = if (src_mcv.isRegister() and
170945 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
170946 src_mcv
170947 else
170948 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
170949 const dst_reg = dst_mcv.getReg().?;
170950
170951 const ext_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
170952 const ext_alias = registerAlias(ext_reg, src_abi_size);
170953 const ext_lock = self.register_manager.lockRegAssumeUnused(ext_reg);
170954 defer self.register_manager.unlockReg(ext_lock);
170955
170956 try self.asmRegisterRegister(.{ .p_, .xor }, ext_alias, ext_alias);
170957 switch (extend) {
170958 .signed => try self.asmRegisterRegister(
170959 .{ switch (src_elem_abi_size) {
170960 else => unreachable,
170961 1 => .p_b,
170962 2 => .p_w,
170963 4 => .p_d,
170964 }, .cmpgt },
170965 ext_alias,
170966 registerAlias(dst_reg, src_abi_size),
170967 ),
170968 .unsigned => {},
170969 }
170970 try self.asmRegisterRegister(
170971 mir_tag,
170972 registerAlias(dst_reg, dst_abi_size),
170973 registerAlias(ext_reg, dst_abi_size),
170974 );
170975 break :result dst_mcv;
170976 },
170977 }
170978 @compileError("unreachable");
170979 }
170980
170981 const min_ty = if (dst_int_info.bits < src_int_info.bits) dst_ty else src_ty;
170982
170983 const src_storage_bits: u16 = switch (src_mcv) {
170984 .register, .register_offset => 64,
170985 .register_pair => 128,
170986 .load_frame => |frame_addr| @intCast(self.getFrameAddrSize(frame_addr) * 8),
170987 else => src_int_info.bits,
170988 };
170989
170990 const dst_mcv = if ((if (src_mcv.getReg()) |src_reg| src_reg.isClass(.general_purpose) else src_abi_size > 8) and
170991 dst_int_info.bits <= src_storage_bits and
170992 std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable ==
170993 std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and
170994 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
170995 const dst_mcv = try self.allocRegOrMem(inst, true);
170996 try self.genCopy(min_ty, dst_mcv, src_mcv, .{});
170997 break :dst dst_mcv;
170998 };
170999
171000 if (dst_int_info.bits <= src_int_info.bits) break :result if (dst_mcv.isRegister())
171001 .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }
171002 else
171003 dst_mcv;
171004
171005 if (dst_mcv.isRegister()) {
171006 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
171007 break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) };
171008 }
171009
171010 const src_limbs_len = std.math.divCeil(u31, src_abi_size, 8) catch unreachable;
171011 const dst_limbs_len = @divExact(dst_abi_size, 8);
171012
171013 const high_mcv: MCValue = if (dst_mcv.isBase())
171014 dst_mcv.address().offset((src_limbs_len - 1) * 8).deref()
171015 else
171016 .{ .register = dst_mcv.register_pair[1] };
171017 const high_reg = if (high_mcv.isRegister())
171018 high_mcv.getReg().?
171019 else
171020 try self.copyToTmpRegister(switch (src_int_info.signedness) {
171021 .signed => .isize,
171022 .unsigned => .usize,
171023 }, high_mcv);
171024 const high_lock = self.register_manager.lockRegAssumeUnused(high_reg);
171025 defer self.register_manager.unlockReg(high_lock);
171026
171027 const high_bits = src_int_info.bits % 64;
171028 if (high_bits > 0) {
171029 try self.truncateRegister(src_ty, high_reg);
171030 const high_ty: Type = if (dst_int_info.bits >= 64) .usize else dst_ty;
171031 try self.genCopy(high_ty, high_mcv, .{ .register = high_reg }, .{});
171032 }
171033
171034 if (dst_limbs_len > src_limbs_len) try self.genInlineMemset(
171035 dst_mcv.address().offset(src_limbs_len * 8),
171036 switch (extend) {
171037 .signed => extend: {
171038 const extend_mcv = MCValue{ .register = high_reg };
171039 try self.genShiftBinOpMir(.{ ._r, .sa }, .isize, extend_mcv, .u8, .{ .immediate = 63 });
171040 break :extend extend_mcv;
171041 },
171042 .unsigned => .{ .immediate = 0 },
171043 },
171044 .{ .immediate = (dst_limbs_len - src_limbs_len) * 8 },
171045 .{},
171046 );
171047
171048 break :result dst_mcv;
171049 }) orelse return self.fail("TODO implement airIntCast from {f} to {f}", .{
171050 src_ty.fmt(pt), dst_ty.fmt(pt),
171051 });
171052 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
171053}
171054
171055fn airTrunc(self: *CodeGen, inst: Air.Inst.Index) !void {
171056 const pt = self.pt;
171057 const zcu = pt.zcu;
171058 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
171059
171060 const dst_ty = self.typeOfIndex(inst);
171061 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
171062 const src_ty = self.typeOf(ty_op.operand);
171063 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
171064
171065 const result = result: {
171066 const src_mcv = try self.resolveInst(ty_op.operand);
171067 const src_lock =
171068 if (src_mcv.getReg()) |reg| self.register_manager.lockRegAssumeUnused(reg) else null;
171069 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
171070
171071 const dst_mcv = if (src_mcv.isRegister() and src_mcv.getReg().?.isClass(self.regClassForType(dst_ty)) and
171072 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
171073 src_mcv
171074 else if (dst_abi_size <= 8)
171075 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv)
171076 else if (dst_abi_size <= 16 and !dst_ty.isVector(zcu)) dst: {
171077 const dst_regs =
171078 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
171079 const dst_mcv: MCValue = .{ .register_pair = dst_regs };
171080 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
171081 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
171082
171083 try self.genCopy(dst_ty, dst_mcv, src_mcv, .{});
171084 break :dst dst_mcv;
171085 } else dst: {
171086 const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true);
171087 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
171088 break :dst dst_mcv;
171089 };
171090
171091 if (dst_ty.zigTypeTag(zcu) == .vector) {
171092 assert(src_ty.zigTypeTag(zcu) == .vector and dst_ty.vectorLen(zcu) == src_ty.vectorLen(zcu));
171093 const dst_elem_ty = dst_ty.childType(zcu);
171094 const dst_elem_abi_size: u32 = @intCast(dst_elem_ty.abiSize(zcu));
171095 const src_elem_ty = src_ty.childType(zcu);
171096 const src_elem_abi_size: u32 = @intCast(src_elem_ty.abiSize(zcu));
171097
171098 const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_elem_abi_size) {
171099 1 => switch (src_elem_abi_size) {
171100 2 => switch (dst_ty.vectorLen(zcu)) {
171101 1...8 => if (self.hasFeature(.avx)) .{ .vp_b, .ackusw } else .{ .p_b, .ackusw },
171102 9...16 => if (self.hasFeature(.avx2)) .{ .vp_b, .ackusw } else null,
171103 else => null,
171104 },
171105 else => null,
171106 },
171107 2 => switch (src_elem_abi_size) {
171108 4 => switch (dst_ty.vectorLen(zcu)) {
171109 1...4 => if (self.hasFeature(.avx))
171110 .{ .vp_w, .ackusd }
171111 else if (self.hasFeature(.sse4_1))
171112 .{ .p_w, .ackusd }
171113 else
171114 null,
171115 5...8 => if (self.hasFeature(.avx2)) .{ .vp_w, .ackusd } else null,
171116 else => null,
171117 },
171118 else => null,
171119 },
171120 else => null,
171121 }) orelse return self.fail("TODO implement airTrunc for {f}", .{dst_ty.fmt(pt)});
171122
171123 const dst_info = dst_elem_ty.intInfo(zcu);
171124 const src_info = src_elem_ty.intInfo(zcu);
171125
171126 const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits));
171127
171128 const splat_ty = try pt.vectorType(.{
171129 .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)),
171130 .child = src_elem_ty.ip_index,
171131 });
171132 const splat_abi_size: u32 = @intCast(splat_ty.abiSize(zcu));
171133
171134 const splat_val = try pt.aggregateSplatValue(splat_ty, mask_val);
171135
171136 const splat_mcv = try self.lowerValue(splat_val);
171137 const splat_addr_mcv: MCValue = switch (splat_mcv) {
171138 .memory, .indirect, .load_frame => splat_mcv.address(),
171139 else => .{ .register = try self.copyToTmpRegister(.usize, splat_mcv.address()) },
171140 };
171141
171142 const dst_reg = dst_mcv.getReg().?;
171143 const dst_alias = registerAlias(dst_reg, src_abi_size);
171144 if (self.hasFeature(.avx)) {
171145 try self.asmRegisterRegisterMemory(
171146 .{ .vp_, .@"and" },
171147 dst_alias,
171148 dst_alias,
171149 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
171150 );
171151 if (src_abi_size > 16) {
171152 const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse);
171153 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
171154 defer self.register_manager.unlockReg(temp_lock);
171155
171156 try self.asmRegisterRegisterImmediate(
171157 .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract },
171158 registerAlias(temp_reg, dst_abi_size),
171159 dst_alias,
171160 .u(1),
171161 );
171162 try self.asmRegisterRegisterRegister(
171163 mir_tag,
171164 registerAlias(dst_reg, dst_abi_size),
171165 registerAlias(dst_reg, dst_abi_size),
171166 registerAlias(temp_reg, dst_abi_size),
171167 );
171168 } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias);
171169 } else {
171170 try self.asmRegisterMemory(
171171 .{ .p_, .@"and" },
171172 dst_alias,
171173 try splat_addr_mcv.deref().mem(self, .{ .size = .fromSize(splat_abi_size) }),
171174 );
171175 try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias);
171176 }
171177 break :result dst_mcv;
171178 }
171179
171180 // when truncating a `u16` to `u5`, for example, those top 3 bits in the result
171181 // have to be removed. this only happens if the dst if not a power-of-two size.
171182 if (dst_abi_size <= 8) {
171183 if (self.regExtraBits(dst_ty) > 0) {
171184 try self.truncateRegister(dst_ty, dst_mcv.register.to64());
171185 }
171186 } else if (dst_abi_size <= 16) {
171187 const dst_info = dst_ty.intInfo(zcu);
171188 const high_ty = try pt.intType(dst_info.signedness, dst_info.bits - 64);
171189 if (self.regExtraBits(high_ty) > 0) {
171190 try self.truncateRegister(high_ty, dst_mcv.register_pair[1].to64());
171191 }
171192 }
171193
171194 break :result dst_mcv;
171195 };
171196 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
171197}
171198
171199fn airSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
171200 const zcu = self.pt.zcu;
171201 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
171202 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
171203
171204 const slice_ty = self.typeOfIndex(inst);
171205 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
171206
171207 const ptr_ty = self.typeOf(bin_op.lhs);
171208 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, .{ .air_ref = bin_op.lhs }, .{});
171209
171210 const len_ty = self.typeOf(bin_op.rhs);
171211 try self.genSetMem(
171212 .{ .frame = frame_index },
171213 @intCast(ptr_ty.abiSize(zcu)),
171214 len_ty,
171215 .{ .air_ref = bin_op.rhs },
171216 .{},
171217 );
171218
171219 const result = MCValue{ .load_frame = .{ .index = frame_index } };
171220 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
171221}
171222
171223fn airUnOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
171224 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
171225 const dst_mcv = try self.genUnOp(inst, tag, ty_op.operand);
171226 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
171227}
171228
171229fn airBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
171230 const pt = self.pt;
171231 const zcu = pt.zcu;
171232 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
171233 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
171234
171235 const dst_ty = self.typeOfIndex(inst);
171236 if (dst_ty.isAbiInt(zcu)) {
171237 const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
171238 const bit_size: u32 = @intCast(dst_ty.bitSize(zcu));
171239 if (abi_size * 8 > bit_size) {
171240 const dst_lock = switch (dst_mcv) {
171241 .register => |dst_reg| self.register_manager.lockRegAssumeUnused(dst_reg),
171242 else => null,
171243 };
171244 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
171245
171246 if (dst_mcv.isRegister()) {
171247 try self.truncateRegister(dst_ty, dst_mcv.getReg().?);
171248 } else {
171249 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171250 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
171251 defer self.register_manager.unlockReg(tmp_lock);
171252
171253 const hi_ty = try pt.intType(.unsigned, @intCast((dst_ty.bitSize(zcu) - 1) % 64 + 1));
171254 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
171255 try self.genSetReg(tmp_reg, hi_ty, hi_mcv, .{});
171256 try self.truncateRegister(dst_ty, tmp_reg);
171257 try self.genCopy(hi_ty, hi_mcv, .{ .register = tmp_reg }, .{});
171258 }
171259 }
171260 }
171261 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
171262}
171263
171264fn airPtrArithmetic(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
171265 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
171266 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
171267 const dst_mcv = try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs);
171268 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
171269}
171270
171271fn activeIntBits(self: *CodeGen, dst_air: Air.Inst.Ref) u16 {
171272 const pt = self.pt;
171273 const zcu = pt.zcu;
171274 const air_tag = self.air.instructions.items(.tag);
171275 const air_data = self.air.instructions.items(.data);
171276
171277 const dst_ty = self.typeOf(dst_air);
171278 const dst_info = dst_ty.intInfo(zcu);
171279 if (dst_air.toIndex()) |inst| {
171280 switch (air_tag[@intFromEnum(inst)]) {
171281 .intcast => {
171282 const src_ty = self.typeOf(air_data[@intFromEnum(inst)].ty_op.operand);
171283 const src_info = src_ty.intInfo(zcu);
171284 return @min(switch (src_info.signedness) {
171285 .signed => switch (dst_info.signedness) {
171286 .signed => src_info.bits,
171287 .unsigned => src_info.bits - 1,
171288 },
171289 .unsigned => switch (dst_info.signedness) {
171290 .signed => src_info.bits + 1,
171291 .unsigned => src_info.bits,
171292 },
171293 }, dst_info.bits);
171294 },
171295 else => {},
171296 }
171297 } else if (dst_air.toInterned()) |ip_index| {
171298 var space: Value.BigIntSpace = undefined;
171299 const src_int = Value.fromInterned(ip_index).toBigInt(&space, zcu);
171300 return @as(u16, @intCast(src_int.bitCountTwosComp())) +
171301 @intFromBool(src_int.positive and dst_info.signedness == .signed);
171302 }
171303 return dst_info.bits;
171304}
171305
171306fn airMulDivBinOp(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
171307 const pt = self.pt;
171308 const zcu = pt.zcu;
171309 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
171310 const result = result: {
171311 const dst_ty = self.typeOfIndex(inst);
171312 switch (dst_ty.zigTypeTag(zcu)) {
171313 .float, .vector => break :result try self.genBinOp(inst, tag, bin_op.lhs, bin_op.rhs),
171314 else => {},
171315 }
171316 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
171317
171318 const dst_info = dst_ty.intInfo(zcu);
171319 const src_ty = try pt.intType(dst_info.signedness, switch (tag) {
171320 else => unreachable,
171321 .mul, .mul_wrap => @max(
171322 self.activeIntBits(bin_op.lhs),
171323 self.activeIntBits(bin_op.rhs),
171324 dst_info.bits / 2,
171325 ),
171326 .div_trunc, .div_floor, .div_exact, .rem, .mod => dst_info.bits,
171327 });
171328 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
171329
171330 if (dst_abi_size == 16 and src_abi_size == 16) switch (tag) {
171331 else => unreachable,
171332 .mul, .mul_wrap => {},
171333 .div_trunc, .div_floor, .div_exact, .rem, .mod => {
171334 const signed = dst_ty.isSignedInt(zcu);
171335 var sym_buf: ["__udiv?i3".len]u8 = undefined;
171336 const signed_div_floor_state: struct {
171337 frame_index: FrameIndex,
171338 state: State,
171339 reloc: Mir.Inst.Index,
171340 } = if (signed and tag == .div_floor) state: {
171341 const frame_index = try self.allocFrameIndex(.initType(.usize, zcu));
171342 try self.asmMemoryImmediate(
171343 .{ ._, .mov },
171344 .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } },
171345 .u(0),
171346 );
171347
171348 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171349 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
171350 defer self.register_manager.unlockReg(tmp_lock);
171351
171352 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171353 const mat_lhs_mcv = switch (lhs_mcv) {
171354 .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: {
171355 // TODO clean this up!
171356 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
171357 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
171358 },
171359 else => lhs_mcv,
171360 };
171361 const mat_lhs_lock = switch (mat_lhs_mcv) {
171362 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
171363 else => null,
171364 };
171365 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
171366 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
171367 .{ ._, .mov },
171368 tmp_reg,
171369 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
171370 ) else try self.asmRegisterRegister(
171371 .{ ._, .mov },
171372 tmp_reg,
171373 mat_lhs_mcv.register_pair[1],
171374 );
171375
171376 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171377 const mat_rhs_mcv = switch (rhs_mcv) {
171378 .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: {
171379 // TODO clean this up!
171380 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
171381 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
171382 },
171383 else => rhs_mcv,
171384 };
171385 const mat_rhs_lock = switch (mat_rhs_mcv) {
171386 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
171387 else => null,
171388 };
171389 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
171390 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
171391 .{ ._, .xor },
171392 tmp_reg,
171393 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
171394 ) else try self.asmRegisterRegister(
171395 .{ ._, .xor },
171396 tmp_reg,
171397 mat_rhs_mcv.register_pair[1],
171398 );
171399 const state = try self.saveState();
171400 const reloc = try self.asmJccReloc(.ns, undefined);
171401
171402 break :state .{ .frame_index = frame_index, .state = state, .reloc = reloc };
171403 } else undefined;
171404 const call_mcv = try self.genCall(
171405 .{ .extern_func = .{
171406 .return_type = dst_ty.toIntern(),
171407 .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() },
171408 .sym = std.fmt.bufPrint(&sym_buf, "__{s}{s}{c}i3", .{
171409 if (signed) "" else "u",
171410 switch (tag) {
171411 .div_trunc, .div_exact => "div",
171412 .div_floor => if (signed) "mod" else "div",
171413 .rem, .mod => "mod",
171414 else => unreachable,
171415 },
171416 intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))),
171417 }) catch unreachable,
171418 } },
171419 &.{ src_ty, src_ty },
171420 &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
171421 .{},
171422 );
171423 break :result if (signed) switch (tag) {
171424 .div_floor => {
171425 try self.asmRegisterRegister(
171426 .{ ._, .@"or" },
171427 call_mcv.register_pair[0],
171428 call_mcv.register_pair[1],
171429 );
171430 try self.asmSetccMemory(.nz, .{
171431 .base = .{ .frame = signed_div_floor_state.frame_index },
171432 .mod = .{ .rm = .{ .size = .byte } },
171433 });
171434 try self.restoreState(signed_div_floor_state.state, &.{}, .{
171435 .emit_instructions = true,
171436 .update_tracking = true,
171437 .resurrect = true,
171438 .close_scope = true,
171439 });
171440 self.performReloc(signed_div_floor_state.reloc);
171441 const dst_mcv = try self.genCall(
171442 .{ .extern_func = .{
171443 .return_type = dst_ty.toIntern(),
171444 .param_types = &.{ src_ty.toIntern(), src_ty.toIntern() },
171445 .sym = std.fmt.bufPrint(&sym_buf, "__div{c}i3", .{
171446 intCompilerRtAbiName(@intCast(dst_ty.bitSize(zcu))),
171447 }) catch unreachable,
171448 } },
171449 &.{ src_ty, src_ty },
171450 &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } },
171451 .{},
171452 );
171453 try self.asmRegisterMemory(
171454 .{ ._, .sub },
171455 dst_mcv.register_pair[0],
171456 .{
171457 .base = .{ .frame = signed_div_floor_state.frame_index },
171458 .mod = .{ .rm = .{ .size = .qword } },
171459 },
171460 );
171461 try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0));
171462 try self.freeValue(
171463 .{ .load_frame = .{ .index = signed_div_floor_state.frame_index } },
171464 );
171465 break :result dst_mcv;
171466 },
171467 .mod => {
171468 const dst_regs = call_mcv.register_pair;
171469 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
171470 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
171471
171472 const tmp_regs =
171473 try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
171474 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
171475 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
171476
171477 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171478 const mat_rhs_mcv = switch (rhs_mcv) {
171479 .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: {
171480 // TODO clean this up!
171481 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
171482 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
171483 },
171484 else => rhs_mcv,
171485 };
171486 const mat_rhs_lock = switch (mat_rhs_mcv) {
171487 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
171488 else => null,
171489 };
171490 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
171491
171492 for (tmp_regs, dst_regs) |tmp_reg, dst_reg|
171493 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_reg);
171494 if (mat_rhs_mcv.isBase()) {
171495 try self.asmRegisterMemory(
171496 .{ ._, .add },
171497 tmp_regs[0],
171498 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
171499 );
171500 try self.asmRegisterMemory(
171501 .{ ._, .adc },
171502 tmp_regs[1],
171503 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
171504 );
171505 } else for (
171506 [_]Mir.Inst.Tag{ .add, .adc },
171507 tmp_regs,
171508 mat_rhs_mcv.register_pair,
171509 ) |op, tmp_reg, rhs_reg|
171510 try self.asmRegisterRegister(.{ ._, op }, tmp_reg, rhs_reg);
171511 try self.asmRegisterRegister(.{ ._, .@"test" }, dst_regs[1], dst_regs[1]);
171512 for (dst_regs, tmp_regs) |dst_reg, tmp_reg|
171513 try self.asmCmovccRegisterRegister(.s, dst_reg, tmp_reg);
171514 break :result call_mcv;
171515 },
171516 else => call_mcv,
171517 } else call_mcv;
171518 },
171519 };
171520
171521 try self.spillEflagsIfOccupied();
171522 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
171523 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
171524 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
171525
171526 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171527 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171528 break :result try self.genMulDivBinOp(tag, inst, dst_ty, src_ty, lhs_mcv, rhs_mcv);
171529 };
171530 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
171531}
171532
171533fn airAddSat(self: *CodeGen, inst: Air.Inst.Index) !void {
171534 const pt = self.pt;
171535 const zcu = pt.zcu;
171536 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
171537 const ty = self.typeOf(bin_op.lhs);
171538 if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail(
171539 "TODO implement airAddSat for {f}",
171540 .{ty.fmt(pt)},
171541 );
171542
171543 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171544 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
171545 lhs_mcv
171546 else
171547 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
171548 const dst_reg = dst_mcv.register;
171549 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
171550 defer self.register_manager.unlockReg(dst_lock);
171551
171552 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171553 const rhs_lock = switch (rhs_mcv) {
171554 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
171555 else => null,
171556 };
171557 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
171558
171559 const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171560 const limit_mcv = MCValue{ .register = limit_reg };
171561 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
171562 defer self.register_manager.unlockReg(limit_lock);
171563
171564 const reg_bits = self.regBitSize(ty);
171565 const reg_extra_bits = self.regExtraBits(ty);
171566 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
171567 if (reg_extra_bits > 0) {
171568 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
171569 }
171570 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
171571 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
171572 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
171573 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
171574 });
171575 if (reg_extra_bits > 0) {
171576 const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
171577 const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg };
171578 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
171579 defer self.register_manager.unlockReg(shifted_rhs_lock);
171580
171581 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
171582 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, shifted_rhs_mcv);
171583 } else try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
171584 break :cc .o;
171585 } else cc: {
171586 try self.genSetReg(limit_reg, ty, .{
171587 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)),
171588 }, .{});
171589
171590 try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv);
171591 if (reg_extra_bits > 0) {
171592 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, limit_mcv);
171593 break :cc .a;
171594 }
171595 break :cc .c;
171596 };
171597
171598 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2);
171599 try self.asmCmovccRegisterRegister(
171600 cc,
171601 registerAlias(dst_reg, cmov_abi_size),
171602 registerAlias(limit_reg, cmov_abi_size),
171603 );
171604
171605 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
171606 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
171607
171608 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
171609}
171610
171611fn airSubSat(self: *CodeGen, inst: Air.Inst.Index) !void {
171612 const pt = self.pt;
171613 const zcu = pt.zcu;
171614 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
171615 const ty = self.typeOf(bin_op.lhs);
171616 if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail(
171617 "TODO implement airSubSat for {f}",
171618 .{ty.fmt(pt)},
171619 );
171620
171621 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171622 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
171623 lhs_mcv
171624 else
171625 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
171626 const dst_reg = dst_mcv.register;
171627 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
171628 defer self.register_manager.unlockReg(dst_lock);
171629
171630 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171631 const rhs_lock = switch (rhs_mcv) {
171632 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
171633 else => null,
171634 };
171635 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
171636
171637 const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171638 const limit_mcv = MCValue{ .register = limit_reg };
171639 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
171640 defer self.register_manager.unlockReg(limit_lock);
171641
171642 const reg_bits = self.regBitSize(ty);
171643 const reg_extra_bits = self.regExtraBits(ty);
171644 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
171645 if (reg_extra_bits > 0) {
171646 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
171647 }
171648 try self.genSetReg(limit_reg, ty, dst_mcv, .{});
171649 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
171650 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
171651 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
171652 });
171653 if (reg_extra_bits > 0) {
171654 const shifted_rhs_reg = try self.copyToTmpRegister(ty, rhs_mcv);
171655 const shifted_rhs_mcv = MCValue{ .register = shifted_rhs_reg };
171656 const shifted_rhs_lock = self.register_manager.lockRegAssumeUnused(shifted_rhs_reg);
171657 defer self.register_manager.unlockReg(shifted_rhs_lock);
171658
171659 try self.genShiftBinOpMir(.{ ._l, .sa }, ty, shifted_rhs_mcv, .u8, .{ .immediate = reg_extra_bits });
171660 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, shifted_rhs_mcv);
171661 } else try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
171662 break :cc .o;
171663 } else cc: {
171664 try self.genSetReg(limit_reg, ty, .{ .immediate = 0 }, .{});
171665 try self.genBinOpMir(.{ ._, .sub }, ty, dst_mcv, rhs_mcv);
171666 break :cc .c;
171667 };
171668
171669 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2);
171670 try self.asmCmovccRegisterRegister(
171671 cc,
171672 registerAlias(dst_reg, cmov_abi_size),
171673 registerAlias(limit_reg, cmov_abi_size),
171674 );
171675
171676 if (reg_extra_bits > 0 and ty.isSignedInt(zcu))
171677 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, dst_mcv, .u8, .{ .immediate = reg_extra_bits });
171678
171679 return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none });
171680}
171681
171682fn airMulSat(self: *CodeGen, inst: Air.Inst.Index) !void {
171683 const pt = self.pt;
171684 const zcu = pt.zcu;
171685 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
171686 const ty = self.typeOf(bin_op.lhs);
171687
171688 const result = result: {
171689 if (ty.toIntern() == .i128_type) {
171690 const ptr_c_int = try pt.singleMutPtrType(.c_int);
171691 const overflow = try self.allocTempRegOrMem(.c_int, false);
171692
171693 const dst_mcv = try self.genCall(.{ .extern_func = .{
171694 .return_type = .i128_type,
171695 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
171696 .sym = "__muloti4",
171697 } }, &.{ .i128, .i128, ptr_c_int }, &.{
171698 .{ .air_ref = bin_op.lhs },
171699 .{ .air_ref = bin_op.rhs },
171700 overflow.address(),
171701 }, .{});
171702 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_mcv.register_pair);
171703 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
171704
171705 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171706 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
171707 defer self.register_manager.unlockReg(tmp_lock);
171708
171709 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171710 const mat_lhs_mcv = switch (lhs_mcv) {
171711 .load_nav, .load_uav, .load_lazy_sym => mat_lhs_mcv: {
171712 // TODO clean this up!
171713 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
171714 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
171715 },
171716 else => lhs_mcv,
171717 };
171718 const mat_lhs_lock = switch (mat_lhs_mcv) {
171719 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
171720 else => null,
171721 };
171722 defer if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
171723 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
171724 .{ ._, .mov },
171725 tmp_reg,
171726 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
171727 ) else try self.asmRegisterRegister(
171728 .{ ._, .mov },
171729 tmp_reg,
171730 mat_lhs_mcv.register_pair[1],
171731 );
171732
171733 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171734 const mat_rhs_mcv = switch (rhs_mcv) {
171735 .load_nav, .load_uav, .load_lazy_sym => mat_rhs_mcv: {
171736 // TODO clean this up!
171737 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
171738 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
171739 },
171740 else => rhs_mcv,
171741 };
171742 const mat_rhs_lock = switch (mat_rhs_mcv) {
171743 .indirect => |reg_off| self.register_manager.lockReg(reg_off.reg),
171744 else => null,
171745 };
171746 defer if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
171747 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
171748 .{ ._, .xor },
171749 tmp_reg,
171750 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
171751 ) else try self.asmRegisterRegister(
171752 .{ ._, .xor },
171753 tmp_reg,
171754 mat_rhs_mcv.register_pair[1],
171755 );
171756
171757 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
171758 try self.asmRegister(.{ ._, .not }, tmp_reg);
171759 try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .{ .size = .dword }), .s(0));
171760 try self.freeValue(overflow);
171761 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg);
171762 try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63));
171763 try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg);
171764 break :result dst_mcv;
171765 }
171766
171767 if (ty.zigTypeTag(zcu) == .vector or ty.abiSize(zcu) > 8) return self.fail(
171768 "TODO implement airMulSat for {f}",
171769 .{ty.fmt(pt)},
171770 );
171771
171772 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
171773 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
171774 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
171775
171776 const lhs_mcv = try self.resolveInst(bin_op.lhs);
171777 const lhs_lock = switch (lhs_mcv) {
171778 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
171779 else => null,
171780 };
171781 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
171782
171783 const rhs_mcv = try self.resolveInst(bin_op.rhs);
171784 const rhs_lock = switch (rhs_mcv) {
171785 .register => |reg| self.register_manager.lockReg(reg),
171786 else => null,
171787 };
171788 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
171789
171790 const limit_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
171791 const limit_mcv = MCValue{ .register = limit_reg };
171792 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
171793 defer self.register_manager.unlockReg(limit_lock);
171794
171795 const reg_bits = self.regBitSize(ty);
171796 const cc: Condition = if (ty.isSignedInt(zcu)) cc: {
171797 try self.genSetReg(limit_reg, ty, lhs_mcv, .{});
171798 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, rhs_mcv);
171799 try self.genShiftBinOpMir(.{ ._r, .sa }, ty, limit_mcv, .u8, .{ .immediate = reg_bits - 1 });
171800 try self.genBinOpMir(.{ ._, .xor }, ty, limit_mcv, .{
171801 .immediate = (@as(u64, 1) << @intCast(reg_bits - 1)) - 1,
171802 });
171803 break :cc .o;
171804 } else cc: {
171805 try self.genSetReg(limit_reg, ty, .{
171806 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits),
171807 }, .{});
171808 break :cc .c;
171809 };
171810
171811 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, ty, lhs_mcv, rhs_mcv);
171812 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2);
171813 try self.asmCmovccRegisterRegister(
171814 cc,
171815 registerAlias(dst_mcv.register, cmov_abi_size),
171816 registerAlias(limit_reg, cmov_abi_size),
171817 );
171818 break :result dst_mcv;
171819 };
171820 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
171821}
171822
171823fn airAddSubWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
171824 const pt = self.pt;
171825 const zcu = pt.zcu;
171826 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
171827 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
171828 const result: MCValue = result: {
171829 const tag = self.air.instructions.items(.tag)[@intFromEnum(inst)];
171830 const ty = self.typeOf(bin_op.lhs);
171831 switch (ty.zigTypeTag(zcu)) {
171832 .vector => return self.fail("TODO implement add/sub with overflow for Vector type", .{}),
171833 .int => {
171834 try self.spillEflagsIfOccupied();
171835 try self.spillRegisters(&.{ .rcx, .rdi, .rsi });
171836 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi });
171837 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
171838
171839 const partial_mcv = try self.genBinOp(null, switch (tag) {
171840 .add_with_overflow => .add,
171841 .sub_with_overflow => .sub,
171842 else => unreachable,
171843 }, bin_op.lhs, bin_op.rhs);
171844 const int_info = ty.intInfo(zcu);
171845 const cc: Condition = switch (int_info.signedness) {
171846 .unsigned => .c,
171847 .signed => .o,
171848 };
171849
171850 const tuple_ty = self.typeOfIndex(inst);
171851 if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
171852 switch (partial_mcv) {
171853 .register => |reg| {
171854 self.eflags_inst = inst;
171855 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
171856 },
171857 else => {},
171858 }
171859
171860 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
171861 try self.genSetMem(
171862 .{ .frame = frame_index },
171863 @intCast(tuple_ty.structFieldOffset(1, zcu)),
171864 .u1,
171865 .{ .eflags = cc },
171866 .{},
171867 );
171868 try self.genSetMem(
171869 .{ .frame = frame_index },
171870 @intCast(tuple_ty.structFieldOffset(0, zcu)),
171871 ty,
171872 partial_mcv,
171873 .{},
171874 );
171875 break :result .{ .load_frame = .{ .index = frame_index } };
171876 }
171877
171878 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
171879 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
171880 break :result .{ .load_frame = .{ .index = frame_index } };
171881 },
171882 else => unreachable,
171883 }
171884 };
171885 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
171886}
171887
171888fn airShlWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
171889 const pt = self.pt;
171890 const zcu = pt.zcu;
171891 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
171892 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
171893 const result: MCValue = result: {
171894 const lhs_ty = self.typeOf(bin_op.lhs);
171895 const rhs_ty = self.typeOf(bin_op.rhs);
171896 switch (lhs_ty.zigTypeTag(zcu)) {
171897 .vector => return self.fail("TODO implement shl with overflow for Vector type", .{}),
171898 .int => {
171899 try self.spillEflagsIfOccupied();
171900 try self.spillRegisters(&.{ .rcx, .rdi, .rsi });
171901 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rcx, .rdi, .rsi });
171902 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
171903
171904 const lhs = try self.resolveInst(bin_op.lhs);
171905 const rhs = try self.resolveInst(bin_op.rhs);
171906
171907 const int_info = lhs_ty.intInfo(zcu);
171908
171909 const partial_mcv = try self.genShiftBinOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty);
171910 const partial_lock = switch (partial_mcv) {
171911 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
171912 else => null,
171913 };
171914 defer if (partial_lock) |lock| self.register_manager.unlockReg(lock);
171915
171916 const tmp_mcv = try self.genShiftBinOp(.shr, null, partial_mcv, rhs, lhs_ty, rhs_ty);
171917 const tmp_lock = switch (tmp_mcv) {
171918 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
171919 else => null,
171920 };
171921 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
171922
171923 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, tmp_mcv, lhs);
171924 const cc = Condition.ne;
171925
171926 const tuple_ty = self.typeOfIndex(inst);
171927 if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) {
171928 switch (partial_mcv) {
171929 .register => |reg| {
171930 self.eflags_inst = inst;
171931 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
171932 },
171933 else => {},
171934 }
171935
171936 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
171937 try self.genSetMem(
171938 .{ .frame = frame_index },
171939 @intCast(tuple_ty.structFieldOffset(1, zcu)),
171940 tuple_ty.fieldType(1, zcu),
171941 .{ .eflags = cc },
171942 .{},
171943 );
171944 try self.genSetMem(
171945 .{ .frame = frame_index },
171946 @intCast(tuple_ty.structFieldOffset(0, zcu)),
171947 tuple_ty.fieldType(0, zcu),
171948 partial_mcv,
171949 .{},
171950 );
171951 break :result .{ .load_frame = .{ .index = frame_index } };
171952 }
171953
171954 const frame_index =
171955 try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
171956 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
171957 break :result .{ .load_frame = .{ .index = frame_index } };
171958 },
171959 else => unreachable,
171960 }
171961 };
171962 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
171963}
171964
171965fn genSetFrameTruncatedOverflowCompare(
171966 self: *CodeGen,
171967 tuple_ty: Type,
171968 frame_index: FrameIndex,
171969 src_mcv: MCValue,
171970 overflow_cc: ?Condition,
171971) !void {
171972 const pt = self.pt;
171973 const zcu = pt.zcu;
171974 const src_lock = switch (src_mcv) {
171975 .register => |reg| self.register_manager.lockReg(reg),
171976 else => null,
171977 };
171978 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
171979
171980 const ty = tuple_ty.fieldType(0, zcu);
171981 const ty_size = ty.abiSize(zcu);
171982 const int_info = ty.intInfo(zcu);
171983
171984 const hi_bits = (int_info.bits - 1) % 64 + 1;
171985 const hi_ty = try pt.intType(int_info.signedness, hi_bits);
171986
171987 const limb_bits: u16 = @intCast(if (int_info.bits <= 64) self.regBitSize(ty) else 64);
171988 const limb_ty = try pt.intType(int_info.signedness, limb_bits);
171989
171990 const rest_ty = try pt.intType(.unsigned, int_info.bits - hi_bits);
171991
171992 const temp_regs =
171993 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
171994 const temp_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs);
171995 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
171996
171997 const overflow_reg = temp_regs[0];
171998 if (overflow_cc) |cc| try self.asmSetccRegister(cc, overflow_reg.to8());
171999
172000 const scratch_reg = temp_regs[1];
172001 const hi_limb_off = if (int_info.bits <= 64) 0 else (int_info.bits - 1) / 64 * 8;
172002 const hi_limb_mcv = if (hi_limb_off > 0)
172003 src_mcv.address().offset(int_info.bits / 64 * 8).deref()
172004 else
172005 src_mcv;
172006 try self.genSetReg(scratch_reg, limb_ty, hi_limb_mcv, .{});
172007 try self.truncateRegister(hi_ty, scratch_reg);
172008 try self.genBinOpMir(.{ ._, .cmp }, limb_ty, .{ .register = scratch_reg }, hi_limb_mcv);
172009
172010 const eq_reg = temp_regs[2];
172011 if (overflow_cc) |_| {
172012 try self.asmSetccRegister(.ne, eq_reg.to8());
172013 try self.genBinOpMir(.{ ._, .@"or" }, .u8, .{ .register = overflow_reg }, .{ .register = eq_reg });
172014 }
172015 try self.genSetMem(
172016 .{ .frame = frame_index },
172017 @intCast(tuple_ty.structFieldOffset(1, zcu)),
172018 tuple_ty.fieldType(1, zcu),
172019 if (overflow_cc) |_| .{ .register = overflow_reg.to8() } else .{ .eflags = .ne },
172020 .{},
172021 );
172022
172023 const payload_off: i32 = @intCast(tuple_ty.structFieldOffset(0, zcu));
172024 if (hi_limb_off > 0) try self.genSetMem(
172025 .{ .frame = frame_index },
172026 payload_off,
172027 rest_ty,
172028 src_mcv,
172029 .{},
172030 );
172031 try self.genSetMem(
172032 .{ .frame = frame_index },
172033 payload_off + hi_limb_off,
172034 limb_ty,
172035 .{ .register = scratch_reg },
172036 .{},
172037 );
172038 var ext_off: i32 = hi_limb_off + 8;
172039 if (ext_off < ty_size) {
172040 switch (int_info.signedness) {
172041 .signed => try self.asmRegisterImmediate(.{ ._r, .sa }, scratch_reg.to64(), .s(63)),
172042 .unsigned => try self.asmRegisterRegister(.{ ._, .xor }, scratch_reg.to32(), scratch_reg.to32()),
172043 }
172044 while (ext_off < ty_size) : (ext_off += 8) try self.genSetMem(
172045 .{ .frame = frame_index },
172046 payload_off + ext_off,
172047 limb_ty,
172048 .{ .register = scratch_reg },
172049 .{},
172050 );
172051 }
172052}
172053
172054fn airMulWithOverflow(self: *CodeGen, inst: Air.Inst.Index) !void {
172055 const pt = self.pt;
172056 const zcu = pt.zcu;
172057 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
172058 const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data;
172059 const tuple_ty = self.typeOfIndex(inst);
172060 const dst_ty = self.typeOf(bin_op.lhs);
172061 const result: MCValue = switch (dst_ty.zigTypeTag(zcu)) {
172062 .vector => return self.fail("TODO implement airMulWithOverflow for {f}", .{dst_ty.fmt(pt)}),
172063 .int => result: {
172064 const dst_info = dst_ty.intInfo(zcu);
172065 if (dst_info.bits > 128 and dst_info.signedness == .unsigned) {
172066 const slow_inc = self.hasFeature(.slow_incdec);
172067 const abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
172068 const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
172069
172070 try self.spillRegisters(&.{ .rax, .rcx, .rdx });
172071 const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx });
172072 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
172073
172074 const dst_mcv = try self.allocRegOrMem(inst, false);
172075 try self.genInlineMemset(
172076 dst_mcv.address(),
172077 .{ .immediate = 0 },
172078 .{ .immediate = tuple_ty.abiSize(zcu) },
172079 .{},
172080 );
172081 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172082 const rhs_mcv = try self.resolveInst(bin_op.rhs);
172083
172084 const temp_regs =
172085 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
172086 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
172087 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
172088
172089 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
172090
172091 const outer_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
172092 try self.asmRegisterMemory(.{ ._, .mov }, temp_regs[1].to64(), .{
172093 .base = .{ .frame = rhs_mcv.load_frame.index },
172094 .mod = .{ .rm = .{
172095 .size = .qword,
172096 .index = temp_regs[0].to64(),
172097 .scale = .@"8",
172098 .disp = rhs_mcv.load_frame.off,
172099 } },
172100 });
172101 try self.asmRegisterRegister(.{ ._, .@"test" }, temp_regs[1].to64(), temp_regs[1].to64());
172102 const skip_inner = try self.asmJccReloc(.z, undefined);
172103
172104 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[2].to32(), temp_regs[2].to32());
172105 try self.asmRegisterRegister(.{ ._, .mov }, temp_regs[3].to32(), temp_regs[0].to32());
172106 try self.asmRegisterRegister(.{ ._, .xor }, .ecx, .ecx);
172107 try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx);
172108
172109 const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
172110 try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1));
172111 try self.asmMemoryRegister(.{ ._, .adc }, .{
172112 .base = .{ .frame = dst_mcv.load_frame.index },
172113 .mod = .{ .rm = .{
172114 .size = .qword,
172115 .index = temp_regs[3].to64(),
172116 .scale = .@"8",
172117 .disp = dst_mcv.load_frame.off +
172118 @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
172119 } },
172120 }, .rdx);
172121 try self.asmSetccRegister(.c, .cl);
172122
172123 try self.asmRegisterMemory(.{ ._, .mov }, .rax, .{
172124 .base = .{ .frame = lhs_mcv.load_frame.index },
172125 .mod = .{ .rm = .{
172126 .size = .qword,
172127 .index = temp_regs[2].to64(),
172128 .scale = .@"8",
172129 .disp = lhs_mcv.load_frame.off,
172130 } },
172131 });
172132 try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64());
172133
172134 try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1));
172135 try self.asmMemoryRegister(.{ ._, .adc }, .{
172136 .base = .{ .frame = dst_mcv.load_frame.index },
172137 .mod = .{ .rm = .{
172138 .size = .qword,
172139 .index = temp_regs[3].to64(),
172140 .scale = .@"8",
172141 .disp = dst_mcv.load_frame.off +
172142 @as(i32, @intCast(tuple_ty.structFieldOffset(0, zcu))),
172143 } },
172144 }, .rax);
172145 try self.asmSetccRegister(.c, .ch);
172146
172147 if (slow_inc) {
172148 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
172149 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1));
172150 } else {
172151 try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32());
172152 try self.asmRegister(.{ ._c, .in }, temp_regs[3].to32());
172153 }
172154 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len));
172155 _ = try self.asmJccReloc(.b, inner_loop);
172156
172157 try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx);
172158 const overflow = try self.asmJccReloc(.nz, undefined);
172159 const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
172160 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len));
172161 const no_overflow = try self.asmJccReloc(.nb, undefined);
172162 if (slow_inc) {
172163 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1));
172164 } else {
172165 try self.asmRegister(.{ ._c, .in }, temp_regs[2].to32());
172166 }
172167 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
172168 .base = .{ .frame = lhs_mcv.load_frame.index },
172169 .mod = .{ .rm = .{
172170 .size = .qword,
172171 .index = temp_regs[2].to64(),
172172 .scale = .@"8",
172173 .disp = lhs_mcv.load_frame.off - 8,
172174 } },
172175 }, .u(0));
172176 _ = try self.asmJccReloc(.z, overflow_loop);
172177 self.performReloc(overflow);
172178 try self.asmMemoryImmediate(.{ ._, .mov }, .{
172179 .base = .{ .frame = dst_mcv.load_frame.index },
172180 .mod = .{ .rm = .{
172181 .size = .byte,
172182 .disp = dst_mcv.load_frame.off +
172183 @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))),
172184 } },
172185 }, .u(1));
172186 self.performReloc(no_overflow);
172187
172188 self.performReloc(skip_inner);
172189 if (slow_inc) {
172190 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
172191 } else {
172192 try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32());
172193 }
172194 try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len));
172195 _ = try self.asmJccReloc(.b, outer_loop);
172196
172197 break :result dst_mcv;
172198 }
172199
172200 const lhs_active_bits = self.activeIntBits(bin_op.lhs);
172201 const rhs_active_bits = self.activeIntBits(bin_op.rhs);
172202 const src_bits = @max(lhs_active_bits, rhs_active_bits, dst_info.bits / 2);
172203 const src_ty = try pt.intType(dst_info.signedness, src_bits);
172204 if (src_bits > 64 and src_bits <= 128 and
172205 dst_info.bits > 64 and dst_info.bits <= 128) switch (dst_info.signedness) {
172206 .signed => {
172207 const ptr_c_int = try pt.singleMutPtrType(.c_int);
172208 const overflow = try self.allocTempRegOrMem(.c_int, false);
172209 const result = try self.genCall(.{ .extern_func = .{
172210 .return_type = .i128_type,
172211 .param_types = &.{ .i128_type, .i128_type, ptr_c_int.toIntern() },
172212 .sym = "__muloti4",
172213 } }, &.{ .i128, .i128, ptr_c_int }, &.{
172214 .{ .air_ref = bin_op.lhs },
172215 .{ .air_ref = bin_op.rhs },
172216 overflow.address(),
172217 }, .{});
172218
172219 const dst_mcv = try self.allocRegOrMem(inst, false);
172220 try self.genSetMem(
172221 .{ .frame = dst_mcv.load_frame.index },
172222 @intCast(tuple_ty.structFieldOffset(0, zcu)),
172223 tuple_ty.fieldType(0, zcu),
172224 result,
172225 .{},
172226 );
172227 try self.asmMemoryImmediate(
172228 .{ ._, .cmp },
172229 try overflow.mem(self, .{ .size = self.memSize(.c_int) }),
172230 .s(0),
172231 );
172232 try self.genSetMem(
172233 .{ .frame = dst_mcv.load_frame.index },
172234 @intCast(tuple_ty.structFieldOffset(1, zcu)),
172235 tuple_ty.fieldType(1, zcu),
172236 .{ .eflags = .ne },
172237 .{},
172238 );
172239 try self.freeValue(overflow);
172240 break :result dst_mcv;
172241 },
172242 .unsigned => {
172243 try self.spillEflagsIfOccupied();
172244 try self.spillRegisters(&.{ .rax, .rdx });
172245 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
172246 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
172247
172248 const tmp_regs =
172249 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
172250 const tmp_locks = self.register_manager.lockRegsAssumeUnused(4, tmp_regs);
172251 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
172252
172253 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172254 const rhs_mcv = try self.resolveInst(bin_op.rhs);
172255 const mat_lhs_mcv = mat_lhs_mcv: switch (lhs_mcv) {
172256 .register => |lhs_reg| switch (lhs_reg.class()) {
172257 else => lhs_mcv,
172258 .sse => {
172259 const mat_lhs_mcv: MCValue = .{
172260 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
172261 };
172262 try self.genCopy(dst_ty, mat_lhs_mcv, lhs_mcv, .{});
172263 break :mat_lhs_mcv mat_lhs_mcv;
172264 },
172265 },
172266 .load_nav, .load_uav, .load_lazy_sym => {
172267 // TODO clean this up!
172268 const addr_reg = try self.copyToTmpRegister(.usize, lhs_mcv.address());
172269 break :mat_lhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
172270 },
172271 else => lhs_mcv,
172272 };
172273 const mat_lhs_locks: [2]?RegisterLock = switch (mat_lhs_mcv) {
172274 .register_pair => |mat_lhs_regs| self.register_manager.lockRegs(2, mat_lhs_regs),
172275 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
172276 else => @splat(null),
172277 };
172278 defer for (mat_lhs_locks) |mat_lhs_lock| if (mat_lhs_lock) |lock| self.register_manager.unlockReg(lock);
172279 const mat_rhs_mcv = mat_rhs_mcv: switch (rhs_mcv) {
172280 .register => |rhs_reg| switch (rhs_reg.class()) {
172281 else => rhs_mcv,
172282 .sse => {
172283 const mat_rhs_mcv: MCValue = .{
172284 .register_pair = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp),
172285 };
172286 try self.genCopy(dst_ty, mat_rhs_mcv, rhs_mcv, .{});
172287 break :mat_rhs_mcv mat_rhs_mcv;
172288 },
172289 },
172290 .load_nav, .load_uav, .load_lazy_sym => {
172291 // TODO clean this up!
172292 const addr_reg = try self.copyToTmpRegister(.usize, rhs_mcv.address());
172293 break :mat_rhs_mcv MCValue{ .indirect = .{ .reg = addr_reg } };
172294 },
172295 else => rhs_mcv,
172296 };
172297 const mat_rhs_locks: [2]?RegisterLock = switch (mat_rhs_mcv) {
172298 .register_pair => |mat_rhs_regs| self.register_manager.lockRegs(2, mat_rhs_regs),
172299 .indirect => |reg_off| .{ self.register_manager.lockReg(reg_off.reg), null },
172300 else => @splat(null),
172301 };
172302 defer for (mat_rhs_locks) |mat_rhs_lock| if (mat_rhs_lock) |lock| self.register_manager.unlockReg(lock);
172303
172304 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
172305 .{ ._, .mov },
172306 .rax,
172307 try mat_lhs_mcv.mem(self, .{ .size = .qword }),
172308 ) else try self.asmRegisterRegister(
172309 .{ ._, .mov },
172310 .rax,
172311 mat_lhs_mcv.register_pair[0],
172312 );
172313 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
172314 .{ ._, .mov },
172315 tmp_regs[0],
172316 try mat_rhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
172317 ) else try self.asmRegisterRegister(
172318 .{ ._, .mov },
172319 tmp_regs[0],
172320 mat_rhs_mcv.register_pair[1],
172321 );
172322 try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]);
172323 try self.asmSetccRegister(.nz, tmp_regs[1].to8());
172324 try self.asmRegisterRegister(.{ .i_, .mul }, tmp_regs[0], .rax);
172325 try self.asmSetccRegister(.o, tmp_regs[2].to8());
172326 if (mat_rhs_mcv.isBase())
172327 try self.asmMemory(.{ ._, .mul }, try mat_rhs_mcv.mem(self, .{ .size = .qword }))
172328 else
172329 try self.asmRegister(.{ ._, .mul }, mat_rhs_mcv.register_pair[0]);
172330 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
172331 try self.asmSetccRegister(.c, tmp_regs[3].to8());
172332 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[2].to8(), tmp_regs[3].to8());
172333 if (mat_lhs_mcv.isBase()) try self.asmRegisterMemory(
172334 .{ ._, .mov },
172335 tmp_regs[0],
172336 try mat_lhs_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
172337 ) else try self.asmRegisterRegister(
172338 .{ ._, .mov },
172339 tmp_regs[0],
172340 mat_lhs_mcv.register_pair[1],
172341 );
172342 try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_regs[0], tmp_regs[0]);
172343 try self.asmSetccRegister(.nz, tmp_regs[3].to8());
172344 try self.asmRegisterRegister(
172345 .{ ._, .@"and" },
172346 tmp_regs[1].to8(),
172347 tmp_regs[3].to8(),
172348 );
172349 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
172350 if (mat_rhs_mcv.isBase()) try self.asmRegisterMemory(
172351 .{ .i_, .mul },
172352 tmp_regs[0],
172353 try mat_rhs_mcv.mem(self, .{ .size = .qword }),
172354 ) else try self.asmRegisterRegister(
172355 .{ .i_, .mul },
172356 tmp_regs[0],
172357 mat_rhs_mcv.register_pair[0],
172358 );
172359 try self.asmSetccRegister(.o, tmp_regs[2].to8());
172360 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
172361 try self.asmRegisterRegister(.{ ._, .add }, .rdx, tmp_regs[0]);
172362 try self.asmSetccRegister(.c, tmp_regs[2].to8());
172363 try self.asmRegisterRegister(.{ ._, .@"or" }, tmp_regs[1].to8(), tmp_regs[2].to8());
172364
172365 const dst_mcv = try self.allocRegOrMem(inst, false);
172366 try self.genSetMem(
172367 .{ .frame = dst_mcv.load_frame.index },
172368 @intCast(tuple_ty.structFieldOffset(0, zcu)),
172369 tuple_ty.fieldType(0, zcu),
172370 .{ .register_pair = .{ .rax, .rdx } },
172371 .{},
172372 );
172373 try self.genSetMem(
172374 .{ .frame = dst_mcv.load_frame.index },
172375 @intCast(tuple_ty.structFieldOffset(1, zcu)),
172376 tuple_ty.fieldType(1, zcu),
172377 .{ .register = tmp_regs[1] },
172378 .{},
172379 );
172380 break :result dst_mcv;
172381 },
172382 };
172383
172384 try self.spillEflagsIfOccupied();
172385 try self.spillRegisters(&.{ .rax, .rcx, .rdx, .rdi, .rsi });
172386 const reg_locks = self.register_manager.lockRegsAssumeUnused(5, .{ .rax, .rcx, .rdx, .rdi, .rsi });
172387 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
172388
172389 const cc: Condition = switch (dst_info.signedness) {
172390 .unsigned => .c,
172391 .signed => .o,
172392 };
172393
172394 const lhs = try self.resolveInst(bin_op.lhs);
172395 const rhs = try self.resolveInst(bin_op.rhs);
172396
172397 const extra_bits = if (dst_info.bits <= 64)
172398 self.regExtraBits(dst_ty)
172399 else
172400 dst_info.bits % 64;
172401 const partial_mcv = try self.genMulDivBinOp(.mul, null, dst_ty, src_ty, lhs, rhs);
172402
172403 switch (partial_mcv) {
172404 .register => |reg| if (extra_bits == 0) {
172405 self.eflags_inst = inst;
172406 break :result .{ .register_overflow = .{ .reg = reg, .eflags = cc } };
172407 } else {
172408 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
172409 try self.genSetFrameTruncatedOverflowCompare(tuple_ty, frame_index, partial_mcv, cc);
172410 break :result .{ .load_frame = .{ .index = frame_index } };
172411 },
172412 else => {
172413 // For now, this is the only supported multiply that doesn't fit in a register.
172414 if (dst_info.bits > 128 or src_bits != 64)
172415 return self.fail("TODO implement airWithOverflow from {f} to {f}", .{
172416 src_ty.fmt(pt), dst_ty.fmt(pt),
172417 });
172418
172419 const frame_index = try self.allocFrameIndex(.initSpill(tuple_ty, zcu));
172420 if (dst_info.bits >= lhs_active_bits + rhs_active_bits) {
172421 try self.genSetMem(
172422 .{ .frame = frame_index },
172423 @intCast(tuple_ty.structFieldOffset(0, zcu)),
172424 tuple_ty.fieldType(0, zcu),
172425 partial_mcv,
172426 .{},
172427 );
172428 try self.genSetMem(
172429 .{ .frame = frame_index },
172430 @intCast(tuple_ty.structFieldOffset(1, zcu)),
172431 tuple_ty.fieldType(1, zcu),
172432 .{ .immediate = 0 }, // cc being set is impossible
172433 .{},
172434 );
172435 } else try self.genSetFrameTruncatedOverflowCompare(
172436 tuple_ty,
172437 frame_index,
172438 partial_mcv,
172439 null,
172440 );
172441 break :result .{ .load_frame = .{ .index = frame_index } };
172442 },
172443 }
172444 },
172445 else => unreachable,
172446 };
172447 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
172448}
172449
172450/// Generates signed or unsigned integer multiplication/division.
172451/// Clobbers .rax and .rdx registers.
172452/// Quotient is saved in .rax and remainder in .rdx.
172453fn genIntMulDivOpMir(self: *CodeGen, tag: Mir.Inst.FixedTag, ty: Type, lhs: MCValue, rhs: MCValue) !void {
172454 const pt = self.pt;
172455 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
172456 const bit_size: u32 = @intCast(self.regBitSize(ty));
172457 if (abi_size > 8) {
172458 return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{});
172459 }
172460
172461 try self.genSetReg(.rax, ty, lhs, .{});
172462 switch (tag[1]) {
172463 else => unreachable,
172464 .mul => {},
172465 .div => switch (tag[0]) {
172466 ._ => {
172467 const hi_reg: Register =
172468 switch (bit_size) {
172469 8 => .ah,
172470 16, 32, 64 => .edx,
172471 else => unreachable,
172472 };
172473 try self.asmRegisterRegister(.{ ._, .xor }, hi_reg, hi_reg);
172474 },
172475 .i_ => try self.asmOpOnly(.{ ._, switch (bit_size) {
172476 8 => .cbw,
172477 16 => .cwd,
172478 32 => .cdq,
172479 64 => .cqo,
172480 else => unreachable,
172481 } }),
172482 else => unreachable,
172483 },
172484 }
172485
172486 const mat_rhs: MCValue = switch (rhs) {
172487 .register, .indirect, .load_frame => rhs,
172488 else => .{ .register = try self.copyToTmpRegister(ty, rhs) },
172489 };
172490 switch (mat_rhs) {
172491 .register => |reg| try self.asmRegister(tag, registerAlias(reg, abi_size)),
172492 .memory, .indirect, .load_frame => try self.asmMemory(
172493 tag,
172494 try mat_rhs.mem(self, .{ .size = .fromSize(abi_size) }),
172495 ),
172496 else => unreachable,
172497 }
172498 if (tag[1] == .div and bit_size == 8) try self.asmRegisterRegister(.{ ._, .mov }, .dl, .ah);
172499}
172500
172501/// Always returns a register.
172502/// Clobbers .rax and .rdx registers.
172503fn genInlineIntDivFloor(self: *CodeGen, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue {
172504 const pt = self.pt;
172505 const zcu = pt.zcu;
172506 const abi_size: u32 = @intCast(ty.abiSize(zcu));
172507 const int_info = ty.intInfo(zcu);
172508 const dividend = switch (lhs) {
172509 .register => |reg| reg,
172510 else => try self.copyToTmpRegister(ty, lhs),
172511 };
172512 const dividend_lock = self.register_manager.lockReg(dividend);
172513 defer if (dividend_lock) |lock| self.register_manager.unlockReg(lock);
172514
172515 const divisor = switch (rhs) {
172516 .register => |reg| reg,
172517 else => try self.copyToTmpRegister(ty, rhs),
172518 };
172519 const divisor_lock = self.register_manager.lockReg(divisor);
172520 defer if (divisor_lock) |lock| self.register_manager.unlockReg(lock);
172521
172522 try self.genIntMulDivOpMir(
172523 switch (int_info.signedness) {
172524 .signed => .{ .i_, .div },
172525 .unsigned => .{ ._, .div },
172526 },
172527 ty,
172528 .{ .register = dividend },
172529 .{ .register = divisor },
172530 );
172531
172532 try self.asmRegisterRegister(
172533 .{ ._, .xor },
172534 registerAlias(divisor, abi_size),
172535 registerAlias(dividend, abi_size),
172536 );
172537 try self.asmRegisterImmediate(
172538 .{ ._r, .sa },
172539 registerAlias(divisor, abi_size),
172540 .u(int_info.bits - 1),
172541 );
172542 try self.asmRegisterRegister(
172543 .{ ._, .@"test" },
172544 registerAlias(.rdx, abi_size),
172545 registerAlias(.rdx, abi_size),
172546 );
172547 try self.asmCmovccRegisterRegister(
172548 .z,
172549 registerAlias(divisor, @max(abi_size, 2)),
172550 registerAlias(.rdx, @max(abi_size, 2)),
172551 );
172552 try self.genBinOpMir(.{ ._, .add }, ty, .{ .register = divisor }, .{ .register = .rax });
172553 return MCValue{ .register = divisor };
172554}
172555
172556fn airShlShrBinOp(self: *CodeGen, inst: Air.Inst.Index) !void {
172557 const pt = self.pt;
172558 const zcu = pt.zcu;
172559 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
172560
172561 const air_tags = self.air.instructions.items(.tag);
172562 const tag = air_tags[@intFromEnum(inst)];
172563 const lhs_ty = self.typeOf(bin_op.lhs);
172564 const rhs_ty = self.typeOf(bin_op.rhs);
172565 const result: MCValue = result: {
172566 switch (lhs_ty.zigTypeTag(zcu)) {
172567 .int => {
172568 try self.spillRegisters(&.{.rcx});
172569 try self.register_manager.getKnownReg(.rcx, null);
172570 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172571 const rhs_mcv = try self.resolveInst(bin_op.rhs);
172572
172573 const dst_mcv = try self.genShiftBinOp(tag, inst, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
172574 switch (tag) {
172575 .shr, .shr_exact, .shl_exact => {},
172576 .shl => switch (dst_mcv) {
172577 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),
172578 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),
172579 .load_frame => |frame_addr| {
172580 const tmp_reg =
172581 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
172582 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
172583 defer self.register_manager.unlockReg(tmp_lock);
172584
172585 const lhs_bits: u31 = @intCast(lhs_ty.bitSize(zcu));
172586 const tmp_ty: Type = if (lhs_bits > 64) .usize else lhs_ty;
172587 const off = frame_addr.off + (lhs_bits - 1) / 64 * 8;
172588 try self.genSetReg(
172589 tmp_reg,
172590 tmp_ty,
172591 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },
172592 .{},
172593 );
172594 try self.truncateRegister(lhs_ty, tmp_reg);
172595 try self.genSetMem(
172596 .{ .frame = frame_addr.index },
172597 off,
172598 tmp_ty,
172599 .{ .register = tmp_reg },
172600 .{},
172601 );
172602 },
172603 else => {},
172604 },
172605 else => unreachable,
172606 }
172607 break :result dst_mcv;
172608 },
172609 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
172610 .int => if (@as(?Mir.Inst.FixedTag, switch (lhs_ty.childType(zcu).intInfo(zcu).bits) {
172611 else => null,
172612 16 => switch (lhs_ty.vectorLen(zcu)) {
172613 else => null,
172614 1...8 => switch (tag) {
172615 else => unreachable,
172616 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172617 .signed => if (self.hasFeature(.avx))
172618 .{ .vp_w, .sra }
172619 else
172620 .{ .p_w, .sra },
172621 .unsigned => if (self.hasFeature(.avx))
172622 .{ .vp_w, .srl }
172623 else
172624 .{ .p_w, .srl },
172625 },
172626 .shl, .shl_exact => if (self.hasFeature(.avx))
172627 .{ .vp_w, .sll }
172628 else
172629 .{ .p_w, .sll },
172630 },
172631 9...16 => switch (tag) {
172632 else => unreachable,
172633 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172634 .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .sra } else null,
172635 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .srl } else null,
172636 },
172637 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_w, .sll } else null,
172638 },
172639 },
172640 32 => switch (lhs_ty.vectorLen(zcu)) {
172641 else => null,
172642 1...4 => switch (tag) {
172643 else => unreachable,
172644 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172645 .signed => if (self.hasFeature(.avx))
172646 .{ .vp_d, .sra }
172647 else
172648 .{ .p_d, .sra },
172649 .unsigned => if (self.hasFeature(.avx))
172650 .{ .vp_d, .srl }
172651 else
172652 .{ .p_d, .srl },
172653 },
172654 .shl, .shl_exact => if (self.hasFeature(.avx))
172655 .{ .vp_d, .sll }
172656 else
172657 .{ .p_d, .sll },
172658 },
172659 5...8 => switch (tag) {
172660 else => unreachable,
172661 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172662 .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .sra } else null,
172663 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .srl } else null,
172664 },
172665 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_d, .sll } else null,
172666 },
172667 },
172668 64 => switch (lhs_ty.vectorLen(zcu)) {
172669 else => null,
172670 1...2 => switch (tag) {
172671 else => unreachable,
172672 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172673 .signed => if (self.hasFeature(.avx))
172674 .{ .vp_q, .sra }
172675 else
172676 .{ .p_q, .sra },
172677 .unsigned => if (self.hasFeature(.avx))
172678 .{ .vp_q, .srl }
172679 else
172680 .{ .p_q, .srl },
172681 },
172682 .shl, .shl_exact => if (self.hasFeature(.avx))
172683 .{ .vp_q, .sll }
172684 else
172685 .{ .p_q, .sll },
172686 },
172687 3...4 => switch (tag) {
172688 else => unreachable,
172689 .shr, .shr_exact => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
172690 .signed => if (self.hasFeature(.avx2)) .{ .vp_q, .sra } else null,
172691 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_q, .srl } else null,
172692 },
172693 .shl, .shl_exact => if (self.hasFeature(.avx2)) .{ .vp_q, .sll } else null,
172694 },
172695 },
172696 })) |mir_tag| if (try self.air.value(bin_op.rhs, pt)) |rhs_val| {
172697 switch (zcu.intern_pool.indexToKey(rhs_val.toIntern())) {
172698 .aggregate => |rhs_aggregate| switch (rhs_aggregate.storage) {
172699 .repeated_elem => |rhs_elem| {
172700 const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu));
172701
172702 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172703 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
172704 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
172705 .{lhs_mcv.getReg().?} ** 2
172706 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
172707 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
172708 lhs_mcv.getReg().?,
172709 } else .{(try self.copyToRegisterWithInstTracking(
172710 inst,
172711 lhs_ty,
172712 lhs_mcv,
172713 )).register} ** 2;
172714 const reg_locks =
172715 self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
172716 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
172717 self.register_manager.unlockReg(lock);
172718
172719 const shift_imm: Immediate =
172720 .u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu)));
172721 if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate(
172722 mir_tag,
172723 registerAlias(dst_reg, abi_size),
172724 registerAlias(lhs_reg, abi_size),
172725 shift_imm,
172726 ) else {
172727 assert(dst_reg.id() == lhs_reg.id());
172728 try self.asmRegisterImmediate(
172729 mir_tag,
172730 registerAlias(dst_reg, abi_size),
172731 shift_imm,
172732 );
172733 }
172734 break :result .{ .register = dst_reg };
172735 },
172736 else => {},
172737 },
172738 else => {},
172739 }
172740 } else if (bin_op.rhs.toIndex()) |rhs_inst| switch (air_tags[@intFromEnum(rhs_inst)]) {
172741 .splat => {
172742 const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu));
172743
172744 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172745 const dst_reg, const lhs_reg = if (lhs_mcv.isRegister() and
172746 self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
172747 .{lhs_mcv.getReg().?} ** 2
172748 else if (lhs_mcv.isRegister() and self.hasFeature(.avx)) .{
172749 try self.register_manager.allocReg(inst, abi.RegisterClass.sse),
172750 lhs_mcv.getReg().?,
172751 } else .{(try self.copyToRegisterWithInstTracking(
172752 inst,
172753 lhs_ty,
172754 lhs_mcv,
172755 )).register} ** 2;
172756 const reg_locks = self.register_manager.lockRegs(2, .{ dst_reg, lhs_reg });
172757 defer for (reg_locks) |reg_lock| if (reg_lock) |lock|
172758 self.register_manager.unlockReg(lock);
172759
172760 const shift_reg =
172761 try self.copyToTmpRegister(rhs_ty, .{ .air_ref = bin_op.rhs });
172762 const shift_lock = self.register_manager.lockRegAssumeUnused(shift_reg);
172763 defer self.register_manager.unlockReg(shift_lock);
172764
172765 const mask_ty = try pt.vectorType(.{ .len = 16, .child = .u8_type });
172766 const mask_mcv = try self.lowerValue(try pt.aggregateValue(
172767 mask_ty,
172768 &([1]InternPool.Index{
172769 (try rhs_ty.childType(zcu).maxIntScalar(pt, .u8)).toIntern(),
172770 } ++ [1]InternPool.Index{.zero_u8} ** 15),
172771 ));
172772 const mask_addr_reg = try self.copyToTmpRegister(.usize, mask_mcv.address());
172773 const mask_addr_lock = self.register_manager.lockRegAssumeUnused(mask_addr_reg);
172774 defer self.register_manager.unlockReg(mask_addr_lock);
172775
172776 if (self.hasFeature(.avx)) {
172777 try self.asmRegisterRegisterMemory(
172778 .{ .vp_, .@"and" },
172779 shift_reg.to128(),
172780 shift_reg.to128(),
172781 .{
172782 .base = .{ .reg = mask_addr_reg },
172783 .mod = .{ .rm = .{ .size = .xword } },
172784 },
172785 );
172786 try self.asmRegisterRegisterRegister(
172787 mir_tag,
172788 registerAlias(dst_reg, abi_size),
172789 registerAlias(lhs_reg, abi_size),
172790 shift_reg.to128(),
172791 );
172792 } else {
172793 try self.asmRegisterMemory(
172794 .{ .p_, .@"and" },
172795 shift_reg.to128(),
172796 .{
172797 .base = .{ .reg = mask_addr_reg },
172798 .mod = .{ .rm = .{ .size = .xword } },
172799 },
172800 );
172801 assert(dst_reg.id() == lhs_reg.id());
172802 try self.asmRegisterRegister(
172803 mir_tag,
172804 registerAlias(dst_reg, abi_size),
172805 shift_reg.to128(),
172806 );
172807 }
172808 break :result .{ .register = dst_reg };
172809 },
172810 else => {},
172811 },
172812 else => {},
172813 },
172814 else => {},
172815 }
172816 return self.fail("TODO implement airShlShrBinOp for {f}", .{lhs_ty.fmt(pt)});
172817 };
172818 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
172819}
172820
172821fn airShlSat(self: *CodeGen, inst: Air.Inst.Index) !void {
172822 const zcu = self.pt.zcu;
172823 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
172824 const lhs_ty = self.typeOf(bin_op.lhs);
172825 const rhs_ty = self.typeOf(bin_op.rhs);
172826
172827 const result: MCValue = result: {
172828 switch (lhs_ty.zigTypeTag(zcu)) {
172829 .int => {
172830 const lhs_bits = lhs_ty.bitSize(zcu);
172831 const rhs_bits = rhs_ty.bitSize(zcu);
172832 if (!(lhs_bits <= 32 and rhs_bits <= 5) and !(lhs_bits > 32 and lhs_bits <= 64 and rhs_bits <= 6) and !(rhs_bits <= std.math.log2(lhs_bits))) {
172833 return self.fail("TODO implement shl_sat for {} with lhs bits {}, rhs bits {}", .{ self.target.cpu.arch, lhs_bits, rhs_bits });
172834 }
172835
172836 // clobberred by genShiftBinOp
172837 try self.spillRegisters(&.{.rcx});
172838
172839 const lhs_mcv = try self.resolveInst(bin_op.lhs);
172840 var lhs_temp1 = try self.tempInit(lhs_ty, lhs_mcv);
172841 const rhs_mcv = try self.resolveInst(bin_op.rhs);
172842
172843 const lhs_lock = switch (lhs_mcv) {
172844 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
172845 else => null,
172846 };
172847 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
172848
172849 // shift left
172850 const dst_mcv = try self.genShiftBinOp(.shl, null, lhs_mcv, rhs_mcv, lhs_ty, rhs_ty);
172851 switch (dst_mcv) {
172852 .register => |dst_reg| try self.truncateRegister(lhs_ty, dst_reg),
172853 .register_pair => |dst_regs| try self.truncateRegister(lhs_ty, dst_regs[1]),
172854 .load_frame => |frame_addr| {
172855 const tmp_reg =
172856 try self.register_manager.allocReg(null, abi.RegisterClass.gp);
172857 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
172858 defer self.register_manager.unlockReg(tmp_lock);
172859
172860 const lhs_bits_u31: u31 = @intCast(lhs_bits);
172861 const tmp_ty: Type = if (lhs_bits_u31 > 64) .usize else lhs_ty;
172862 const off = frame_addr.off + (lhs_bits_u31 - 1) / 64 * 8;
172863 try self.genSetReg(
172864 tmp_reg,
172865 tmp_ty,
172866 .{ .load_frame = .{ .index = frame_addr.index, .off = off } },
172867 .{},
172868 );
172869 try self.truncateRegister(lhs_ty, tmp_reg);
172870 try self.genSetMem(
172871 .{ .frame = frame_addr.index },
172872 off,
172873 tmp_ty,
172874 .{ .register = tmp_reg },
172875 .{},
172876 );
172877 },
172878 else => {},
172879 }
172880 const dst_lock = switch (dst_mcv) {
172881 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
172882 else => null,
172883 };
172884 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
172885
172886 // shift right
172887 const tmp_mcv = try self.genShiftBinOp(.shr, null, dst_mcv, rhs_mcv, lhs_ty, rhs_ty);
172888 var tmp_temp = try self.tempInit(lhs_ty, tmp_mcv);
172889
172890 // check if overflow happens
172891 const cc_temp = lhs_temp1.cmpInts(.neq, &tmp_temp, self) catch |err| switch (err) {
172892 error.SelectFailed => unreachable,
172893 else => |e| return e,
172894 };
172895 try lhs_temp1.die(self);
172896 try tmp_temp.die(self);
172897 const overflow_reloc = try self.genCondBrMir(lhs_ty, cc_temp.tracking(self).short);
172898 try cc_temp.die(self);
172899
172900 // if overflow,
172901 // for unsigned integers, the saturating result is just its max
172902 // for signed integers,
172903 // if lhs is positive, the result is its max
172904 // if lhs is negative, it is min
172905 switch (lhs_ty.intInfo(zcu).signedness) {
172906 .unsigned => {
172907 const bound_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty));
172908 try self.genCopy(lhs_ty, dst_mcv, bound_mcv, .{});
172909 },
172910 .signed => {
172911 // check the sign of lhs
172912 // TODO: optimize this.
172913 // we only need the highest bit so shifting the highest part of lhs_mcv
172914 // is enough to check the signedness. other parts can be skipped here.
172915 var lhs_temp2 = try self.tempInit(lhs_ty, lhs_mcv);
172916 var zero_temp = try self.tempInit(lhs_ty, try self.lowerValue(try self.pt.intValue(lhs_ty, 0)));
172917 const sign_cc_temp = lhs_temp2.cmpInts(.lt, &zero_temp, self) catch |err| switch (err) {
172918 error.SelectFailed => unreachable,
172919 else => |e| return e,
172920 };
172921 try lhs_temp2.die(self);
172922 try zero_temp.die(self);
172923 const sign_reloc_condbr = try self.genCondBrMir(lhs_ty, sign_cc_temp.tracking(self).short);
172924 try sign_cc_temp.die(self);
172925
172926 // if it is negative
172927 const min_mcv = try self.lowerValue(try lhs_ty.minIntScalar(self.pt, lhs_ty));
172928 try self.genCopy(lhs_ty, dst_mcv, min_mcv, .{});
172929 const sign_reloc_br = try self.asmJmpReloc(undefined);
172930 self.performReloc(sign_reloc_condbr);
172931
172932 // if it is positive
172933 const max_mcv = try self.lowerValue(try lhs_ty.maxIntScalar(self.pt, lhs_ty));
172934 try self.genCopy(lhs_ty, dst_mcv, max_mcv, .{});
172935 self.performReloc(sign_reloc_br);
172936 },
172937 }
172938
172939 self.performReloc(overflow_reloc);
172940 break :result dst_mcv;
172941 },
172942 else => {
172943 return self.fail("TODO implement shl_sat for {} op type {}", .{ self.target.cpu.arch, lhs_ty.zigTypeTag(zcu) });
172944 },
172945 }
172946 };
172947 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
172948}
172949
172950fn airOptionalPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
172951 const zcu = self.pt.zcu;
172952 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
172953 const result: MCValue = result: {
172954 const pl_ty = self.typeOfIndex(inst);
172955 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
172956
172957 const opt_mcv = try self.resolveInst(ty_op.operand);
172958 if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv)) {
172959 const pl_mcv: MCValue = switch (opt_mcv) {
172960 .register_overflow => |ro| pl: {
172961 self.eflags_inst = null; // actually stop tracking the overflow part
172962 break :pl .{ .register = ro.reg };
172963 },
172964 else => opt_mcv,
172965 };
172966 switch (pl_mcv) {
172967 .register => |pl_reg| try self.truncateRegister(pl_ty, pl_reg),
172968 else => {},
172969 }
172970 break :result pl_mcv;
172971 }
172972
172973 const pl_mcv = try self.allocRegOrMem(inst, true);
172974 try self.genCopy(pl_ty, pl_mcv, switch (opt_mcv) {
172975 else => opt_mcv,
172976 .register_overflow => |ro| .{ .register = ro.reg },
172977 }, .{});
172978 break :result pl_mcv;
172979 };
172980 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
172981}
172982
172983fn airOptionalPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
172984 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
172985
172986 const dst_ty = self.typeOfIndex(inst);
172987 const opt_mcv = try self.resolveInst(ty_op.operand);
172988
172989 const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
172990 opt_mcv
172991 else
172992 try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
172993 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
172994}
172995
172996fn airOptionalPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
172997 const pt = self.pt;
172998 const zcu = pt.zcu;
172999 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173000 const result = result: {
173001 const dst_ty = self.typeOfIndex(inst);
173002 const src_ty = self.typeOf(ty_op.operand);
173003 const opt_ty = src_ty.childType(zcu);
173004 const src_mcv = try self.resolveInst(ty_op.operand);
173005
173006 if (opt_ty.optionalReprIsPayload(zcu)) {
173007 break :result if (self.liveness.isUnused(inst))
173008 .unreach
173009 else if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
173010 src_mcv
173011 else
173012 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
173013 }
173014
173015 const dst_mcv: MCValue = if (src_mcv.isRegister() and
173016 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
173017 src_mcv
173018 else if (self.liveness.isUnused(inst))
173019 .{ .register = try self.copyToTmpRegister(dst_ty, src_mcv) }
173020 else
173021 try self.copyToRegisterWithInstTracking(inst, dst_ty, src_mcv);
173022
173023 const pl_ty = dst_ty.childType(zcu);
173024 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
173025 try self.genSetMem(
173026 .{ .reg = dst_mcv.getReg().? },
173027 pl_abi_size,
173028 .bool,
173029 .{ .immediate = 1 },
173030 .{},
173031 );
173032 break :result if (self.liveness.isUnused(inst)) .unreach else dst_mcv;
173033 };
173034 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173035}
173036
173037fn airUnwrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
173038 const pt = self.pt;
173039 const zcu = pt.zcu;
173040 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173041 const err_union_ty = self.typeOf(ty_op.operand);
173042 const err_ty = err_union_ty.errorUnionSet(zcu);
173043 const payload_ty = err_union_ty.errorUnionPayload(zcu);
173044 const operand = try self.resolveInst(ty_op.operand);
173045
173046 const result: MCValue = result: {
173047 if (err_ty.errorSetIsEmpty(zcu)) {
173048 break :result MCValue{ .immediate = 0 };
173049 }
173050
173051 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) {
173052 break :result try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
173053 }
173054
173055 const err_off = codegen.errUnionErrorOffset(payload_ty, zcu);
173056 switch (operand) {
173057 .register => |reg| {
173058 // TODO reuse operand
173059 const eu_lock = self.register_manager.lockReg(reg);
173060 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
173061
173062 const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand);
173063 if (err_off > 0) try self.genShiftBinOpMir(
173064 .{ ._r, .sh },
173065 err_union_ty,
173066 result,
173067 .u8,
173068 .{ .immediate = @as(u6, @intCast(err_off * 8)) },
173069 ) else try self.truncateRegister(.anyerror, result.register);
173070 break :result result;
173071 },
173072 .load_frame => |frame_addr| break :result .{ .load_frame = .{
173073 .index = frame_addr.index,
173074 .off = frame_addr.off + @as(i32, @intCast(err_off)),
173075 } },
173076 else => return self.fail("TODO implement unwrap_err_err for {f}", .{operand}),
173077 }
173078 };
173079 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173080}
173081
173082fn airUnwrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
173083 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173084 const operand_ty = self.typeOf(ty_op.operand);
173085 const operand = try self.resolveInst(ty_op.operand);
173086 const result = try self.genUnwrapErrUnionPayloadMir(inst, operand_ty, operand);
173087 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173088}
173089
173090// *(E!T) -> E
173091fn airUnwrapErrUnionErrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173092 const pt = self.pt;
173093 const zcu = pt.zcu;
173094 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173095
173096 const src_ty = self.typeOf(ty_op.operand);
173097 const src_mcv = try self.resolveInst(ty_op.operand);
173098 const src_reg = switch (src_mcv) {
173099 .register => |reg| reg,
173100 else => try self.copyToTmpRegister(src_ty, src_mcv),
173101 };
173102 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
173103 defer self.register_manager.unlockReg(src_lock);
173104
173105 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
173106 const dst_mcv = MCValue{ .register = dst_reg };
173107 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
173108 defer self.register_manager.unlockReg(dst_lock);
173109
173110 const eu_ty = src_ty.childType(zcu);
173111 const pl_ty = eu_ty.errorUnionPayload(zcu);
173112 const err_ty = eu_ty.errorUnionSet(zcu);
173113 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
173114 const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
173115 try self.asmRegisterMemory(
173116 .{ ._, .mov },
173117 registerAlias(dst_reg, err_abi_size),
173118 .{
173119 .base = .{ .reg = src_reg },
173120 .mod = .{ .rm = .{
173121 .size = .fromSize(err_abi_size),
173122 .disp = err_off,
173123 } },
173124 },
173125 );
173126
173127 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
173128}
173129
173130// *(E!T) -> *T
173131fn airUnwrapErrUnionPayloadPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173132 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173133 const operand_ty = self.typeOf(ty_op.operand);
173134 const operand = try self.resolveInst(ty_op.operand);
173135 const result = try self.genUnwrapErrUnionPayloadPtrMir(inst, operand_ty, operand);
173136 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173137}
173138
173139fn airErrUnionPayloadPtrSet(self: *CodeGen, inst: Air.Inst.Index) !void {
173140 const pt = self.pt;
173141 const zcu = pt.zcu;
173142 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173143 const result: MCValue = result: {
173144 const src_ty = self.typeOf(ty_op.operand);
173145 const src_mcv = try self.resolveInst(ty_op.operand);
173146 const src_reg = switch (src_mcv) {
173147 .register => |reg| reg,
173148 else => try self.copyToTmpRegister(src_ty, src_mcv),
173149 };
173150 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
173151 defer self.register_manager.unlockReg(src_lock);
173152
173153 const eu_ty = src_ty.childType(zcu);
173154 const pl_ty = eu_ty.errorUnionPayload(zcu);
173155 const err_ty = eu_ty.errorUnionSet(zcu);
173156 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
173157 const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu));
173158 try self.asmMemoryImmediate(
173159 .{ ._, .mov },
173160 .{
173161 .base = .{ .reg = src_reg },
173162 .mod = .{ .rm = .{
173163 .size = .fromSize(err_abi_size),
173164 .disp = err_off,
173165 } },
173166 },
173167 .u(0),
173168 );
173169
173170 if (self.liveness.isUnused(inst)) break :result .unreach;
173171
173172 const dst_ty = self.typeOfIndex(inst);
173173 const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
173174 src_reg
173175 else
173176 try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
173177 const dst_lock = self.register_manager.lockReg(dst_reg);
173178 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
173179
173180 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
173181 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu));
173182 try self.asmRegisterMemory(
173183 .{ ._, .lea },
173184 registerAlias(dst_reg, dst_abi_size),
173185 .{
173186 .base = .{ .reg = src_reg },
173187 .mod = .{ .rm = .{ .disp = pl_off } },
173188 },
173189 );
173190 break :result .{ .register = dst_reg };
173191 };
173192 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173193}
173194
173195fn genUnwrapErrUnionPayloadMir(
173196 self: *CodeGen,
173197 maybe_inst: ?Air.Inst.Index,
173198 err_union_ty: Type,
173199 err_union: MCValue,
173200) !MCValue {
173201 const pt = self.pt;
173202 const zcu = pt.zcu;
173203 const payload_ty = err_union_ty.errorUnionPayload(zcu);
173204
173205 const result: MCValue = result: {
173206 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
173207
173208 const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
173209 switch (err_union) {
173210 .load_frame => |frame_addr| break :result .{ .load_frame = .{
173211 .index = frame_addr.index,
173212 .off = frame_addr.off + payload_off,
173213 } },
173214 .register => |reg| {
173215 // TODO reuse operand
173216 const eu_lock = self.register_manager.lockReg(reg);
173217 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
173218
173219 const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
173220 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
173221 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
173222 else
173223 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
173224 if (payload_off > 0) try self.genShiftBinOpMir(
173225 .{ ._r, .sh },
173226 err_union_ty,
173227 result_mcv,
173228 .u8,
173229 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
173230 ) else try self.truncateRegister(payload_ty, result_mcv.register);
173231 break :result if (payload_in_gp)
173232 result_mcv
173233 else if (maybe_inst) |inst|
173234 try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
173235 else
173236 .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
173237 },
173238 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {f}", .{err_union}),
173239 }
173240 };
173241
173242 return result;
173243}
173244
173245fn genUnwrapErrUnionPayloadPtrMir(
173246 self: *CodeGen,
173247 maybe_inst: ?Air.Inst.Index,
173248 ptr_ty: Type,
173249 ptr_mcv: MCValue,
173250) !MCValue {
173251 const pt = self.pt;
173252 const zcu = pt.zcu;
173253 const err_union_ty = ptr_ty.childType(zcu);
173254 const payload_ty = err_union_ty.errorUnionPayload(zcu);
173255
173256 const result: MCValue = result: {
173257 const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
173258 const result_mcv: MCValue = if (maybe_inst) |inst|
173259 try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
173260 else
173261 .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) };
173262 try self.genBinOpMir(.{ ._, .add }, ptr_ty, result_mcv, .{ .immediate = payload_off });
173263 break :result result_mcv;
173264 };
173265
173266 return result;
173267}
173268
173269fn airWrapOptional(self: *CodeGen, inst: Air.Inst.Index) !void {
173270 const pt = self.pt;
173271 const zcu = pt.zcu;
173272 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173273 const result: MCValue = result: {
173274 const pl_ty = self.typeOf(ty_op.operand);
173275 if (!pl_ty.hasRuntimeBits(zcu)) break :result .{ .immediate = 1 };
173276
173277 const opt_ty = self.typeOfIndex(inst);
173278 const pl_mcv = try self.resolveInst(ty_op.operand);
173279 const same_repr = opt_ty.optionalReprIsPayload(zcu);
173280 if (same_repr and self.reuseOperand(inst, ty_op.operand, 0, pl_mcv)) break :result pl_mcv;
173281
173282 const pl_lock: ?RegisterLock = switch (pl_mcv) {
173283 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173284 else => null,
173285 };
173286 defer if (pl_lock) |lock| self.register_manager.unlockReg(lock);
173287
173288 const opt_mcv = try self.allocRegOrMem(inst, true);
173289 try self.genCopy(pl_ty, opt_mcv, pl_mcv, .{});
173290
173291 if (!same_repr) {
173292 const pl_abi_size: i32 = @intCast(pl_ty.abiSize(zcu));
173293 switch (opt_mcv) {
173294 else => unreachable,
173295
173296 .register => |opt_reg| {
173297 try self.truncateRegister(pl_ty, opt_reg);
173298 try self.asmRegisterImmediate(
173299 .{ ._s, .bt },
173300 opt_reg,
173301 .u(@as(u6, @intCast(pl_abi_size * 8))),
173302 );
173303 },
173304
173305 .load_frame => |frame_addr| try self.asmMemoryImmediate(
173306 .{ ._, .mov },
173307 .{
173308 .base = .{ .frame = frame_addr.index },
173309 .mod = .{ .rm = .{
173310 .size = .byte,
173311 .disp = frame_addr.off + pl_abi_size,
173312 } },
173313 },
173314 .u(1),
173315 ),
173316 }
173317 }
173318 break :result opt_mcv;
173319 };
173320 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173321}
173322
173323/// T to E!T
173324fn airWrapErrUnionPayload(self: *CodeGen, inst: Air.Inst.Index) !void {
173325 const pt = self.pt;
173326 const zcu = pt.zcu;
173327 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173328
173329 const eu_ty = ty_op.ty.toType();
173330 const pl_ty = eu_ty.errorUnionPayload(zcu);
173331 const err_ty = eu_ty.errorUnionSet(zcu);
173332 const operand = try self.resolveInst(ty_op.operand);
173333
173334 const result: MCValue = result: {
173335 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 };
173336
173337 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
173338 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
173339 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
173340 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{});
173341 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{});
173342 break :result .{ .load_frame = .{ .index = frame_index } };
173343 };
173344 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173345}
173346
173347/// E to E!T
173348fn airWrapErrUnionErr(self: *CodeGen, inst: Air.Inst.Index) !void {
173349 const pt = self.pt;
173350 const zcu = pt.zcu;
173351 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173352
173353 const eu_ty = ty_op.ty.toType();
173354 const pl_ty = eu_ty.errorUnionPayload(zcu);
173355 const err_ty = eu_ty.errorUnionSet(zcu);
173356
173357 const result: MCValue = result: {
173358 if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand);
173359
173360 const frame_index = try self.allocFrameIndex(.initSpill(eu_ty, zcu));
173361 const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu));
173362 const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu));
173363 try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{});
173364 const operand = try self.resolveInst(ty_op.operand);
173365 try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{});
173366 break :result .{ .load_frame = .{ .index = frame_index } };
173367 };
173368 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173369}
173370
173371fn airSlicePtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173372 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173373 const result = result: {
173374 const src_mcv = try self.resolveInst(ty_op.operand);
173375 const ptr_mcv: MCValue = switch (src_mcv) {
173376 .register_pair => |regs| .{ .register = regs[0] },
173377 else => src_mcv,
173378 };
173379 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
173380 switch (src_mcv) {
173381 .register_pair => |regs| try self.freeValue(.{ .register = regs[1] }),
173382 else => {},
173383 }
173384 break :result ptr_mcv;
173385 }
173386
173387 const dst_mcv = try self.allocRegOrMem(inst, true);
173388 try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{});
173389 break :result dst_mcv;
173390 };
173391 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173392}
173393
173394fn airSliceLen(self: *CodeGen, inst: Air.Inst.Index) !void {
173395 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173396 const result = result: {
173397 const src_mcv = try self.resolveInst(ty_op.operand);
173398 const len_mcv: MCValue = switch (src_mcv) {
173399 .register_pair => |regs| .{ .register = regs[1] },
173400 .load_frame => |frame_addr| .{ .load_frame = .{
173401 .index = frame_addr.index,
173402 .off = frame_addr.off + 8,
173403 } },
173404 else => return self.fail("TODO implement slice_len for {f}", .{src_mcv}),
173405 };
173406 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
173407 switch (src_mcv) {
173408 .register_pair => |regs| try self.freeValue(.{ .register = regs[0] }),
173409 .load_frame => {},
173410 else => unreachable,
173411 }
173412 break :result len_mcv;
173413 }
173414
173415 const dst_mcv = try self.allocRegOrMem(inst, true);
173416 try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{});
173417 break :result dst_mcv;
173418 };
173419 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
173420}
173421
173422fn airPtrSliceLenPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173423 const pt = self.pt;
173424 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173425
173426 const src_ty = self.typeOf(ty_op.operand);
173427 const src_mcv = try self.resolveInst(ty_op.operand);
173428 const src_reg = switch (src_mcv) {
173429 .register => |reg| reg,
173430 else => try self.copyToTmpRegister(src_ty, src_mcv),
173431 };
173432 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
173433 defer self.register_manager.unlockReg(src_lock);
173434
173435 const dst_ty = self.typeOfIndex(inst);
173436 const dst_reg = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
173437 src_reg
173438 else
173439 try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
173440 const dst_mcv = MCValue{ .register = dst_reg };
173441 const dst_lock = self.register_manager.lockReg(dst_reg);
173442 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
173443
173444 const dst_abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
173445 try self.asmRegisterMemory(
173446 .{ ._, .lea },
173447 registerAlias(dst_reg, dst_abi_size),
173448 .{
173449 .base = .{ .reg = src_reg },
173450 .mod = .{ .rm = .{ .disp = 8 } },
173451 },
173452 );
173453
173454 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
173455}
173456
173457fn airPtrSlicePtrPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173458 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173459
173460 const dst_ty = self.typeOfIndex(inst);
173461 const opt_mcv = try self.resolveInst(ty_op.operand);
173462
173463 const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, opt_mcv))
173464 opt_mcv
173465 else
173466 try self.copyToRegisterWithInstTracking(inst, dst_ty, opt_mcv);
173467 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
173468}
173469
173470fn elemOffset(self: *CodeGen, index_ty: Type, index: MCValue, elem_size: u64) !Register {
173471 const reg: Register = blk: {
173472 switch (index) {
173473 .immediate => |imm| {
173474 // Optimisation: if index MCValue is an immediate, we can multiply in `comptime`
173475 // and set the register directly to the scaled offset as an immediate.
173476 const reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
173477 try self.genSetReg(reg, index_ty, .{ .immediate = imm * elem_size }, .{});
173478 break :blk reg;
173479 },
173480 else => {
173481 const reg = try self.copyToTmpRegister(index_ty, index);
173482 try self.genIntMulComplexOpMir(index_ty, .{ .register = reg }, .{ .immediate = elem_size });
173483 break :blk reg;
173484 },
173485 }
173486 };
173487 return reg;
173488}
173489
173490fn genSliceElemPtr(self: *CodeGen, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue {
173491 const pt = self.pt;
173492 const zcu = pt.zcu;
173493 const slice_ty = self.typeOf(lhs);
173494 const slice_mcv = try self.resolveInst(lhs);
173495 const slice_mcv_lock: ?RegisterLock = switch (slice_mcv) {
173496 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173497 else => null,
173498 };
173499 defer if (slice_mcv_lock) |lock| self.register_manager.unlockReg(lock);
173500
173501 const elem_ty = slice_ty.childType(zcu);
173502 const elem_size = elem_ty.abiSize(zcu);
173503 const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
173504
173505 const index_ty = self.typeOf(rhs);
173506 const index_mcv = try self.resolveInst(rhs);
173507 const index_mcv_lock: ?RegisterLock = switch (index_mcv) {
173508 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173509 else => null,
173510 };
173511 defer if (index_mcv_lock) |lock| self.register_manager.unlockReg(lock);
173512
173513 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_size);
173514 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
173515 defer self.register_manager.unlockReg(offset_reg_lock);
173516
173517 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
173518 try self.genSetReg(addr_reg, .usize, slice_mcv, .{});
173519 // TODO we could allocate register here, but need to expect addr register and potentially
173520 // offset register.
173521 try self.genBinOpMir(.{ ._, .add }, slice_ptr_field_type, .{ .register = addr_reg }, .{
173522 .register = offset_reg,
173523 });
173524 return MCValue{ .register = addr_reg.to64() };
173525}
173526
173527fn airSliceElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
173528 const pt = self.pt;
173529 const zcu = pt.zcu;
173530 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
173531
173532 const result: MCValue = result: {
173533 const elem_ty = self.typeOfIndex(inst);
173534 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
173535
173536 const slice_ty = self.typeOf(bin_op.lhs);
173537 const slice_ptr_field_type = slice_ty.slicePtrFieldType(zcu);
173538 const elem_ptr = try self.genSliceElemPtr(bin_op.lhs, bin_op.rhs);
173539 const dst_mcv = try self.allocRegOrMem(inst, false);
173540 try self.load(dst_mcv, slice_ptr_field_type, elem_ptr);
173541 break :result dst_mcv;
173542 };
173543 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
173544}
173545
173546fn airSliceElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173547 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
173548 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
173549 const dst_mcv = try self.genSliceElemPtr(extra.lhs, extra.rhs);
173550 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
173551}
173552
173553fn airArrayElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
173554 const pt = self.pt;
173555 const zcu = pt.zcu;
173556 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
173557
173558 const result: MCValue = result: {
173559 const array_ty = self.typeOf(bin_op.lhs);
173560 const elem_ty = array_ty.childType(zcu);
173561
173562 const array_mcv = try self.resolveInst(bin_op.lhs);
173563 const array_lock: ?RegisterLock = switch (array_mcv) {
173564 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173565 else => null,
173566 };
173567 defer if (array_lock) |lock| self.register_manager.unlockReg(lock);
173568
173569 const index_ty = self.typeOf(bin_op.rhs);
173570 const index_mcv = try self.resolveInst(bin_op.rhs);
173571 const index_lock = switch (index_mcv) {
173572 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173573 else => null,
173574 };
173575 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
173576
173577 try self.spillEflagsIfOccupied();
173578 if (array_ty.isVector(zcu) and elem_ty.bitSize(zcu) == 1) {
173579 const array_mat_mcv: MCValue = switch (array_mcv) {
173580 else => array_mcv,
173581 .register_mask => .{ .register = try self.copyToTmpRegister(array_ty, array_mcv) },
173582 };
173583 const array_mat_lock = switch (array_mat_mcv) {
173584 .register => |reg| self.register_manager.lockReg(reg),
173585 else => null,
173586 };
173587 defer if (array_mat_lock) |lock| self.register_manager.unlockReg(lock);
173588
173589 switch (array_mat_mcv) {
173590 .register => |array_reg| switch (array_reg.class()) {
173591 .general_purpose => switch (index_mcv) {
173592 .immediate => |index_imm| try self.asmRegisterImmediate(
173593 .{ ._, .bt },
173594 array_reg.to64(),
173595 .u(index_imm),
173596 ),
173597 else => try self.asmRegisterRegister(
173598 .{ ._, .bt },
173599 array_reg.to64(),
173600 switch (index_mcv) {
173601 .register => |index_reg| index_reg,
173602 else => try self.copyToTmpRegister(index_ty, index_mcv),
173603 }.to64(),
173604 ),
173605 },
173606 .sse => {
173607 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
173608 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mat_mcv, .{});
173609 switch (index_mcv) {
173610 .immediate => |index_imm| try self.asmMemoryImmediate(
173611 .{ ._, .bt },
173612 .{
173613 .base = .{ .frame = frame_index },
173614 .mod = .{ .rm = .{
173615 .size = .qword,
173616 .disp = @intCast(index_imm / 64 * 8),
173617 } },
173618 },
173619 .u(index_imm % 64),
173620 ),
173621 else => try self.asmMemoryRegister(
173622 .{ ._, .bt },
173623 .{
173624 .base = .{ .frame = frame_index },
173625 .mod = .{ .rm = .{ .size = .qword } },
173626 },
173627 switch (index_mcv) {
173628 .register => |index_reg| index_reg,
173629 else => try self.copyToTmpRegister(index_ty, index_mcv),
173630 }.to64(),
173631 ),
173632 }
173633 },
173634 else => unreachable,
173635 },
173636 .load_frame => switch (index_mcv) {
173637 .immediate => |index_imm| try self.asmMemoryImmediate(
173638 .{ ._, .bt },
173639 try array_mat_mcv.mem(self, .{
173640 .size = .qword,
173641 .disp = @intCast(index_imm / 64 * 8),
173642 }),
173643 .u(index_imm % 64),
173644 ),
173645 else => try self.asmMemoryRegister(
173646 .{ ._, .bt },
173647 try array_mat_mcv.mem(self, .{ .size = .qword }),
173648 switch (index_mcv) {
173649 .register => |index_reg| index_reg,
173650 else => try self.copyToTmpRegister(index_ty, index_mcv),
173651 }.to64(),
173652 ),
173653 },
173654 .memory,
173655 .load_nav,
173656 .load_uav,
173657 .load_lazy_sym,
173658 .load_extern_func,
173659 => switch (index_mcv) {
173660 .immediate => |index_imm| try self.asmMemoryImmediate(
173661 .{ ._, .bt },
173662 .{
173663 .base = .{
173664 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
173665 },
173666 .mod = .{ .rm = .{
173667 .size = .qword,
173668 .disp = @intCast(index_imm / 64 * 8),
173669 } },
173670 },
173671 .u(index_imm % 64),
173672 ),
173673 else => try self.asmMemoryRegister(
173674 .{ ._, .bt },
173675 .{
173676 .base = .{
173677 .reg = try self.copyToTmpRegister(.usize, array_mat_mcv.address()),
173678 },
173679 .mod = .{ .rm = .{ .size = .qword } },
173680 },
173681 switch (index_mcv) {
173682 .register => |index_reg| index_reg,
173683 else => try self.copyToTmpRegister(index_ty, index_mcv),
173684 }.to64(),
173685 ),
173686 },
173687 else => return self.fail("TODO airArrayElemVal for {s} of {f}", .{
173688 @tagName(array_mat_mcv), array_ty.fmt(pt),
173689 }),
173690 }
173691
173692 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
173693 try self.asmSetccRegister(.c, dst_reg.to8());
173694 break :result .{ .register = dst_reg };
173695 }
173696
173697 const elem_abi_size = elem_ty.abiSize(zcu);
173698 const addr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
173699 const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg);
173700 defer self.register_manager.unlockReg(addr_lock);
173701
173702 switch (array_mcv) {
173703 .register => {
173704 const frame_index = try self.allocFrameIndex(.initType(array_ty, zcu));
173705 try self.genSetMem(.{ .frame = frame_index }, 0, array_ty, array_mcv, .{});
173706 try self.asmRegisterMemory(
173707 .{ ._, .lea },
173708 addr_reg,
173709 .{ .base = .{ .frame = frame_index } },
173710 );
173711 },
173712 .load_frame => |frame_addr| try self.asmRegisterMemory(
173713 .{ ._, .lea },
173714 addr_reg,
173715 .{
173716 .base = .{ .frame = frame_addr.index },
173717 .mod = .{ .rm = .{ .disp = frame_addr.off } },
173718 },
173719 ),
173720 .memory,
173721 .load_nav,
173722 .lea_nav,
173723 .load_uav,
173724 .lea_uav,
173725 .load_lazy_sym,
173726 .lea_lazy_sym,
173727 .load_extern_func,
173728 .lea_extern_func,
173729 => try self.genSetReg(addr_reg, .usize, array_mcv.address(), .{}),
173730 else => return self.fail("TODO airArrayElemVal_val for {s} of {f}", .{
173731 @tagName(array_mcv), array_ty.fmt(pt),
173732 }),
173733 }
173734
173735 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size);
173736 const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
173737 defer self.register_manager.unlockReg(offset_lock);
173738
173739 // TODO we could allocate register here, but need to expect addr register and potentially
173740 // offset register.
173741 const dst_mcv = try self.allocRegOrMem(inst, false);
173742 try self.genBinOpMir(.{ ._, .add }, .usize, .{ .register = addr_reg }, .{ .register = offset_reg });
173743 try self.genCopy(elem_ty, dst_mcv, .{ .indirect = .{ .reg = addr_reg } }, .{});
173744 break :result dst_mcv;
173745 };
173746 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
173747}
173748
173749fn airPtrElemVal(self: *CodeGen, inst: Air.Inst.Index) !void {
173750 const pt = self.pt;
173751 const zcu = pt.zcu;
173752 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
173753 const ptr_ty = self.typeOf(bin_op.lhs);
173754
173755 // this is identical to the `airPtrElemPtr` codegen expect here an
173756 // additional `mov` is needed at the end to get the actual value
173757
173758 const result = result: {
173759 const elem_ty = ptr_ty.elemType2(zcu);
173760 if (!elem_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
173761
173762 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
173763 const index_ty = self.typeOf(bin_op.rhs);
173764 const index_mcv = try self.resolveInst(bin_op.rhs);
173765 const index_lock = switch (index_mcv) {
173766 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173767 else => null,
173768 };
173769 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
173770
173771 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size);
173772 const offset_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
173773 defer self.register_manager.unlockReg(offset_lock);
173774
173775 const ptr_mcv = try self.resolveInst(bin_op.lhs);
173776 const elem_ptr_reg = if (ptr_mcv.isRegister() and self.liveness.operandDies(inst, 0))
173777 ptr_mcv.register
173778 else
173779 try self.copyToTmpRegister(ptr_ty, ptr_mcv);
173780 const elem_ptr_lock = self.register_manager.lockRegAssumeUnused(elem_ptr_reg);
173781 defer self.register_manager.unlockReg(elem_ptr_lock);
173782 try self.asmRegisterRegister(
173783 .{ ._, .add },
173784 elem_ptr_reg,
173785 offset_reg,
173786 );
173787
173788 const dst_mcv = try self.allocRegOrMem(inst, true);
173789 const dst_lock = switch (dst_mcv) {
173790 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173791 else => null,
173792 };
173793 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
173794 try self.load(dst_mcv, ptr_ty, .{ .register = elem_ptr_reg });
173795 break :result dst_mcv;
173796 };
173797 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
173798}
173799
173800fn airPtrElemPtr(self: *CodeGen, inst: Air.Inst.Index) !void {
173801 const pt = self.pt;
173802 const zcu = pt.zcu;
173803 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
173804 const extra = self.air.extraData(Air.Bin, ty_pl.payload).data;
173805
173806 const result = result: {
173807 const elem_ptr_ty = self.typeOfIndex(inst);
173808 const base_ptr_ty = self.typeOf(extra.lhs);
173809
173810 const base_ptr_mcv = try self.resolveInst(extra.lhs);
173811 const base_ptr_lock: ?RegisterLock = switch (base_ptr_mcv) {
173812 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173813 else => null,
173814 };
173815 defer if (base_ptr_lock) |lock| self.register_manager.unlockReg(lock);
173816
173817 if (elem_ptr_ty.ptrInfo(zcu).flags.vector_index != .none) {
173818 break :result if (self.reuseOperand(inst, extra.lhs, 0, base_ptr_mcv))
173819 base_ptr_mcv
173820 else
173821 try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv);
173822 }
173823
173824 const elem_ty = base_ptr_ty.elemType2(zcu);
173825 const elem_abi_size = elem_ty.abiSize(zcu);
173826 const index_ty = self.typeOf(extra.rhs);
173827 const index_mcv = try self.resolveInst(extra.rhs);
173828 const index_lock: ?RegisterLock = switch (index_mcv) {
173829 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173830 else => null,
173831 };
173832 defer if (index_lock) |lock| self.register_manager.unlockReg(lock);
173833
173834 const offset_reg = try self.elemOffset(index_ty, index_mcv, elem_abi_size);
173835 const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg);
173836 defer self.register_manager.unlockReg(offset_reg_lock);
173837
173838 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, elem_ptr_ty, base_ptr_mcv);
173839 try self.genBinOpMir(.{ ._, .add }, elem_ptr_ty, dst_mcv, .{ .register = offset_reg });
173840
173841 break :result dst_mcv;
173842 };
173843 return self.finishAir(inst, result, .{ extra.lhs, extra.rhs, .none });
173844}
173845
173846fn airSetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
173847 const pt = self.pt;
173848 const zcu = pt.zcu;
173849 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
173850 const ptr_union_ty = self.typeOf(bin_op.lhs);
173851 const union_ty = ptr_union_ty.childType(zcu);
173852 const tag_ty = self.typeOf(bin_op.rhs);
173853 const layout = union_ty.unionGetLayout(zcu);
173854
173855 if (layout.tag_size == 0) {
173856 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
173857 }
173858
173859 const ptr = try self.resolveInst(bin_op.lhs);
173860 const ptr_lock: ?RegisterLock = switch (ptr) {
173861 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173862 else => null,
173863 };
173864 defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock);
173865
173866 const tag = try self.resolveInst(bin_op.rhs);
173867 const tag_lock: ?RegisterLock = switch (tag) {
173868 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173869 else => null,
173870 };
173871 defer if (tag_lock) |lock| self.register_manager.unlockReg(lock);
173872
173873 const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align.compare(.lt, layout.payload_align)) blk: {
173874 // TODO reusing the operand
173875 const reg = try self.copyToTmpRegister(ptr_union_ty, ptr);
173876 try self.genBinOpMir(
173877 .{ ._, .add },
173878 ptr_union_ty,
173879 .{ .register = reg },
173880 .{ .immediate = layout.payload_size },
173881 );
173882 break :blk MCValue{ .register = reg };
173883 } else ptr;
173884
173885 const ptr_tag_ty = try pt.adjustPtrTypeChild(ptr_union_ty, tag_ty);
173886 try self.store(ptr_tag_ty, adjusted_ptr, tag, .{});
173887
173888 return self.finishAir(inst, .none, .{ bin_op.lhs, bin_op.rhs, .none });
173889}
173890
173891fn airGetUnionTag(self: *CodeGen, inst: Air.Inst.Index) !void {
173892 const zcu = self.pt.zcu;
173893 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173894
173895 const tag_ty = self.typeOfIndex(inst);
173896 const union_ty = self.typeOf(ty_op.operand);
173897 const layout = union_ty.unionGetLayout(zcu);
173898
173899 if (layout.tag_size == 0) {
173900 return self.finishAir(inst, .none, .{ ty_op.operand, .none, .none });
173901 }
173902
173903 // TODO reusing the operand
173904 const operand = try self.resolveInst(ty_op.operand);
173905 const operand_lock: ?RegisterLock = switch (operand) {
173906 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
173907 else => null,
173908 };
173909 defer if (operand_lock) |lock| self.register_manager.unlockReg(lock);
173910
173911 const tag_abi_size = tag_ty.abiSize(zcu);
173912 const dst_mcv: MCValue = blk: {
173913 switch (operand) {
173914 .load_frame => |frame_addr| {
173915 if (tag_abi_size <= 8) {
173916 const off: i32 = @intCast(layout.tagOffset());
173917 break :blk try self.copyToRegisterWithInstTracking(inst, tag_ty, .{
173918 .load_frame = .{ .index = frame_addr.index, .off = frame_addr.off + off },
173919 });
173920 }
173921
173922 return self.fail(
173923 "TODO implement get_union_tag for ABI larger than 8 bytes and operand {f}",
173924 .{operand},
173925 );
173926 },
173927 .register => {
173928 const shift: u6 = @intCast(layout.tagOffset() * 8);
173929 const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand);
173930 try self.genShiftBinOpMir(.{ ._r, .sh }, .usize, result, .u8, .{ .immediate = shift });
173931 break :blk MCValue{
173932 .register = registerAlias(result.register, @intCast(layout.tag_size)),
173933 };
173934 },
173935 else => return self.fail("TODO implement get_union_tag for {f}", .{operand}),
173936 }
173937 };
173938
173939 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
173940}
173941
173942fn airClz(self: *CodeGen, inst: Air.Inst.Index) !void {
173943 const pt = self.pt;
173944 const zcu = pt.zcu;
173945 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
173946 const result = result: {
173947 try self.spillEflagsIfOccupied();
173948
173949 const dst_ty = self.typeOfIndex(inst);
173950 const src_ty = self.typeOf(ty_op.operand);
173951 if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airClz for {f}", .{
173952 src_ty.fmt(pt),
173953 });
173954
173955 const src_mcv = try self.resolveInst(ty_op.operand);
173956 const mat_src_mcv = switch (src_mcv) {
173957 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
173958 else => src_mcv,
173959 };
173960 const mat_src_lock = switch (mat_src_mcv) {
173961 .register => |reg| self.register_manager.lockReg(reg),
173962 else => null,
173963 };
173964 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
173965
173966 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
173967 const dst_mcv = MCValue{ .register = dst_reg };
173968 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
173969 defer self.register_manager.unlockReg(dst_lock);
173970
173971 const abi_size: u31 = @intCast(src_ty.abiSize(zcu));
173972 const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
173973 const has_lzcnt = self.hasFeature(.lzcnt);
173974 if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
173975 const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
173976 .load_frame => |src_frame_addr| src_frame_addr,
173977 else => {
173978 const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
173979 try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
173980 break :src_frame_addr .{ .index = src_frame_addr };
173981 },
173982 };
173983
173984 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
173985 const extra_bits = abi_size * 8 - src_bits;
173986
173987 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
173988 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
173989 defer self.register_manager.unlockReg(index_lock);
173990
173991 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len));
173992 switch (extra_bits) {
173993 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
173994 else => try self.asmRegisterImmediate(
173995 .{ ._, .mov },
173996 dst_reg.to32(),
173997 .s(@as(i32, extra_bits) - 1),
173998 ),
173999 }
174000 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
174001 try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
174002 const zero = try self.asmJccReloc(.z, undefined);
174003 if (self.hasFeature(.slow_incdec)) {
174004 try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1));
174005 } else {
174006 try self.asmRegister(.{ ._c, .de }, index_reg.to32());
174007 }
174008 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
174009 .base = .{ .frame = src_frame_addr.index },
174010 .mod = .{ .rm = .{
174011 .size = .qword,
174012 .index = index_reg.to64(),
174013 .scale = .@"8",
174014 .disp = src_frame_addr.off,
174015 } },
174016 }, .u(0));
174017 _ = try self.asmJccReloc(.e, loop);
174018 try self.asmRegisterMemory(.{ ._r, .bs }, dst_reg.to64(), .{
174019 .base = .{ .frame = src_frame_addr.index },
174020 .mod = .{ .rm = .{
174021 .size = .qword,
174022 .index = index_reg.to64(),
174023 .scale = .@"8",
174024 .disp = src_frame_addr.off,
174025 } },
174026 });
174027 self.performReloc(zero);
174028 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
174029 try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
174030 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1));
174031 try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
174032 break :result dst_mcv;
174033 }
174034
174035 if (has_lzcnt) {
174036 if (src_bits <= 8) {
174037 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
174038 try self.truncateRegister(src_ty, wide_reg);
174039 try self.genBinOpMir(.{ ._, .lzcnt }, .u32, dst_mcv, .{ .register = wide_reg });
174040 try self.genBinOpMir(
174041 .{ ._, .sub },
174042 dst_ty,
174043 dst_mcv,
174044 .{ .immediate = 32 - src_bits },
174045 );
174046 } else if (src_bits <= 64) {
174047 try self.genBinOpMir(.{ ._, .lzcnt }, src_ty, dst_mcv, mat_src_mcv);
174048 const extra_bits = self.regExtraBits(src_ty);
174049 if (extra_bits > 0) {
174050 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
174051 }
174052 } else {
174053 assert(src_bits <= 128);
174054 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
174055 const tmp_mcv = MCValue{ .register = tmp_reg };
174056 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
174057 defer self.register_manager.unlockReg(tmp_lock);
174058
174059 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, dst_mcv, if (mat_src_mcv.isBase())
174060 mat_src_mcv
174061 else
174062 .{ .register = mat_src_mcv.register_pair[0] });
174063 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
174064 try self.genBinOpMir(.{ ._, .lzcnt }, .u64, tmp_mcv, if (mat_src_mcv.isBase())
174065 mat_src_mcv.address().offset(8).deref()
174066 else
174067 .{ .register = mat_src_mcv.register_pair[1] });
174068 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
174069
174070 if (src_bits < 128) try self.genBinOpMir(
174071 .{ ._, .sub },
174072 dst_ty,
174073 dst_mcv,
174074 .{ .immediate = 128 - src_bits },
174075 );
174076 }
174077 break :result dst_mcv;
174078 }
174079
174080 assert(src_bits <= 64);
174081 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
174082 if (std.math.isPowerOfTwo(src_bits)) {
174083 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
174084 .immediate = src_bits ^ (src_bits - 1),
174085 });
174086 const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
174087 defer self.register_manager.unlockReg(imm_lock);
174088
174089 if (src_bits <= 8) {
174090 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
174091 const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
174092 defer self.register_manager.unlockReg(wide_lock);
174093
174094 try self.truncateRegister(src_ty, wide_reg);
174095 try self.genBinOpMir(.{ ._r, .bs }, .u16, dst_mcv, .{ .register = wide_reg });
174096 } else try self.genBinOpMir(.{ ._r, .bs }, src_ty, dst_mcv, mat_src_mcv);
174097
174098 try self.asmCmovccRegisterRegister(
174099 .z,
174100 registerAlias(dst_reg, cmov_abi_size),
174101 registerAlias(imm_reg, cmov_abi_size),
174102 );
174103
174104 try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 });
174105 } else {
174106 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
174107 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)),
174108 });
174109 const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg);
174110 defer self.register_manager.unlockReg(imm_lock);
174111
174112 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
174113 const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
174114 defer self.register_manager.unlockReg(wide_lock);
174115
174116 try self.truncateRegister(src_ty, wide_reg);
174117 try self.genBinOpMir(
174118 .{ ._r, .bs },
174119 if (src_bits <= 8) .u16 else src_ty,
174120 dst_mcv,
174121 .{ .register = wide_reg },
174122 );
174123
174124 try self.asmCmovccRegisterRegister(
174125 .nz,
174126 registerAlias(imm_reg, cmov_abi_size),
174127 registerAlias(dst_reg, cmov_abi_size),
174128 );
174129
174130 try self.genSetReg(dst_reg, dst_ty, .{ .immediate = src_bits - 1 }, .{});
174131 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .register = imm_reg });
174132 }
174133 break :result dst_mcv;
174134 };
174135 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
174136}
174137
174138fn airCtz(self: *CodeGen, inst: Air.Inst.Index) !void {
174139 const pt = self.pt;
174140 const zcu = pt.zcu;
174141 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
174142 const result = result: {
174143 try self.spillEflagsIfOccupied();
174144
174145 const dst_ty = self.typeOfIndex(inst);
174146 const src_ty = self.typeOf(ty_op.operand);
174147 if (src_ty.zigTypeTag(zcu) == .vector) return self.fail("TODO implement airCtz for {f}", .{
174148 src_ty.fmt(pt),
174149 });
174150
174151 const src_mcv = try self.resolveInst(ty_op.operand);
174152 const mat_src_mcv = switch (src_mcv) {
174153 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
174154 else => src_mcv,
174155 };
174156 const mat_src_lock = switch (mat_src_mcv) {
174157 .register => |reg| self.register_manager.lockReg(reg),
174158 else => null,
174159 };
174160 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
174161
174162 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
174163 const dst_mcv = MCValue{ .register = dst_reg };
174164 const dst_lock = self.register_manager.lockReg(dst_reg);
174165 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
174166
174167 const abi_size: u31 = @intCast(src_ty.abiSize(zcu));
174168 const src_bits: u31 = @intCast(src_ty.bitSize(zcu));
174169 const has_bmi = self.hasFeature(.bmi);
174170 if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
174171 const src_frame_addr: bits.FrameAddr = src_frame_addr: switch (src_mcv) {
174172 .load_frame => |src_frame_addr| src_frame_addr,
174173 else => {
174174 const src_frame_addr = try self.allocFrameIndex(.initSpill(src_ty, zcu));
174175 try self.genSetMem(.{ .frame = src_frame_addr }, 0, src_ty, src_mcv, .{});
174176 break :src_frame_addr .{ .index = src_frame_addr };
174177 },
174178 };
174179
174180 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
174181 const extra_bits = abi_size * 8 - src_bits;
174182
174183 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
174184 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
174185 defer self.register_manager.unlockReg(index_lock);
174186
174187 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1));
174188 switch (extra_bits) {
174189 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
174190 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
174191 else => try self.asmRegisterImmediate(
174192 .{ ._, .mov },
174193 dst_reg.to32(),
174194 .s(-@as(i32, extra_bits)),
174195 ),
174196 }
174197 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
174198 if (self.hasFeature(.slow_incdec)) {
174199 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1));
174200 } else {
174201 try self.asmRegister(.{ ._c, .in }, index_reg.to32());
174202 }
174203 try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len));
174204 const zero = try self.asmJccReloc(.nb, undefined);
174205 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
174206 .base = .{ .frame = src_frame_addr.index },
174207 .mod = .{ .rm = .{
174208 .size = .qword,
174209 .index = index_reg.to64(),
174210 .scale = .@"8",
174211 .disp = src_frame_addr.off,
174212 } },
174213 }, .u(0));
174214 _ = try self.asmJccReloc(.e, loop);
174215 try self.asmRegisterMemory(.{ ._f, .bs }, dst_reg.to64(), .{
174216 .base = .{ .frame = src_frame_addr.index },
174217 .mod = .{ .rm = .{
174218 .size = .qword,
174219 .index = index_reg.to64(),
174220 .scale = .@"8",
174221 .disp = src_frame_addr.off,
174222 } },
174223 });
174224 self.performReloc(zero);
174225 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6));
174226 try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
174227 break :result dst_mcv;
174228 }
174229
174230 const wide_ty: Type = if (src_bits <= 8) .u16 else src_ty;
174231 if (has_bmi) {
174232 if (src_bits <= 64) {
174233 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
174234 const masked_mcv = if (extra_bits > 0) masked: {
174235 const tmp_mcv = tmp: {
174236 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
174237 break :tmp src_mcv;
174238 try self.genSetReg(dst_reg, wide_ty, src_mcv, .{});
174239 break :tmp dst_mcv;
174240 };
174241 try self.genBinOpMir(
174242 .{ ._, .@"or" },
174243 wide_ty,
174244 tmp_mcv,
174245 .{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) <<
174246 @intCast(src_bits) },
174247 );
174248 break :masked tmp_mcv;
174249 } else mat_src_mcv;
174250 try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);
174251 } else {
174252 assert(src_bits <= 128);
174253 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
174254 const tmp_mcv = MCValue{ .register = tmp_reg };
174255 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
174256 defer self.register_manager.unlockReg(tmp_lock);
174257
174258 const lo_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
174259 mat_src_mcv
174260 else
174261 .{ .register = mat_src_mcv.register_pair[0] };
174262 const hi_mat_src_mcv: MCValue = if (mat_src_mcv.isBase())
174263 mat_src_mcv.address().offset(8).deref()
174264 else
174265 .{ .register = mat_src_mcv.register_pair[1] };
174266 const masked_mcv = if (src_bits < 128) masked: {
174267 try self.genCopy(.u64, dst_mcv, hi_mat_src_mcv, .{});
174268 try self.genBinOpMir(
174269 .{ ._, .@"or" },
174270 .u64,
174271 dst_mcv,
174272 .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) },
174273 );
174274 break :masked dst_mcv;
174275 } else hi_mat_src_mcv;
174276 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, dst_mcv, masked_mcv);
174277 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
174278 try self.genBinOpMir(.{ ._, .tzcnt }, .u64, tmp_mcv, lo_mat_src_mcv);
174279 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
174280 }
174281 break :result dst_mcv;
174282 }
174283
174284 assert(src_bits <= 64);
174285 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
174286 const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
174287 defer self.register_manager.unlockReg(width_lock);
174288
174289 if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) {
174290 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
174291 const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg);
174292 defer self.register_manager.unlockReg(wide_lock);
174293
174294 try self.truncateRegister(src_ty, wide_reg);
174295 try self.genBinOpMir(.{ ._f, .bs }, wide_ty, dst_mcv, .{ .register = wide_reg });
174296 } else try self.genBinOpMir(.{ ._f, .bs }, src_ty, dst_mcv, mat_src_mcv);
174297
174298 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2);
174299 try self.asmCmovccRegisterRegister(
174300 .z,
174301 registerAlias(dst_reg, cmov_abi_size),
174302 registerAlias(width_reg, cmov_abi_size),
174303 );
174304 break :result dst_mcv;
174305 };
174306 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
174307}
174308
174309fn airPopCount(self: *CodeGen, inst: Air.Inst.Index) !void {
174310 const pt = self.pt;
174311 const zcu = pt.zcu;
174312 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
174313 const result: MCValue = result: {
174314 try self.spillEflagsIfOccupied();
174315
174316 const src_ty = self.typeOf(ty_op.operand);
174317 const src_abi_size: u32 = @intCast(src_ty.abiSize(zcu));
174318 if (src_ty.zigTypeTag(zcu) == .vector or src_abi_size > 16)
174319 return self.fail("TODO implement airPopCount for {f}", .{src_ty.fmt(pt)});
174320 const src_mcv = try self.resolveInst(ty_op.operand);
174321
174322 const mat_src_mcv = switch (src_mcv) {
174323 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
174324 else => src_mcv,
174325 };
174326 const mat_src_lock = switch (mat_src_mcv) {
174327 .register => |reg| self.register_manager.lockReg(reg),
174328 else => null,
174329 };
174330 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
174331
174332 if (src_abi_size <= 8) {
174333 const dst_contains_src =
174334 src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv);
174335 const dst_reg = if (dst_contains_src)
174336 src_mcv.getReg().?
174337 else
174338 try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
174339 const dst_lock = self.register_manager.lockReg(dst_reg);
174340 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
174341
174342 try self.genPopCount(dst_reg, src_ty, mat_src_mcv, dst_contains_src);
174343 break :result .{ .register = dst_reg };
174344 }
174345
174346 assert(src_abi_size > 8 and src_abi_size <= 16);
174347 const tmp_regs = try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
174348 const tmp_locks = self.register_manager.lockRegsAssumeUnused(2, tmp_regs);
174349 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
174350
174351 try self.genPopCount(tmp_regs[0], .usize, if (mat_src_mcv.isBase())
174352 mat_src_mcv
174353 else
174354 .{ .register = mat_src_mcv.register_pair[0] }, false);
174355 const src_info = src_ty.intInfo(zcu);
174356 const hi_ty = try pt.intType(src_info.signedness, (src_info.bits - 1) % 64 + 1);
174357 try self.genPopCount(tmp_regs[1], hi_ty, if (mat_src_mcv.isBase())
174358 mat_src_mcv.address().offset(8).deref()
174359 else
174360 .{ .register = mat_src_mcv.register_pair[1] }, false);
174361 try self.asmRegisterRegister(.{ ._, .add }, tmp_regs[0].to8(), tmp_regs[1].to8());
174362 break :result .{ .register = tmp_regs[0] };
174363 };
174364 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
174365}
174366
174367fn genPopCount(
174368 self: *CodeGen,
174369 dst_reg: Register,
174370 src_ty: Type,
174371 src_mcv: MCValue,
174372 dst_contains_src: bool,
174373) !void {
174374 const pt = self.pt;
174375
174376 const src_abi_size: u32 = @intCast(src_ty.abiSize(pt.zcu));
174377 if (self.hasFeature(.popcnt)) return self.genBinOpMir(
174378 .{ ._, .popcnt },
174379 if (src_abi_size > 1) src_ty else .u32,
174380 .{ .register = dst_reg },
174381 if (src_abi_size > 1) src_mcv else src: {
174382 if (!dst_contains_src) try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
174383 try self.truncateRegister(try src_ty.toUnsigned(pt), dst_reg);
174384 break :src .{ .register = dst_reg };
174385 },
174386 );
174387
174388 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8);
174389 const imm_0_1: Immediate = .u(mask / 0b1_1);
174390 const imm_00_11: Immediate = .u(mask / 0b01_01);
174391 const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
174392 const imm_0000_0001: Immediate = .u(mask / 0b1111_1111);
174393
174394 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
174395 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
174396 defer self.register_manager.unlockReg(tmp_lock);
174397
174398 const dst = registerAlias(dst_reg, src_abi_size);
174399 const tmp = registerAlias(tmp_reg, src_abi_size);
174400 const imm = if (src_abi_size > 4)
174401 try self.register_manager.allocReg(null, abi.RegisterClass.gp)
174402 else
174403 undefined;
174404
174405 if (!dst_contains_src) try self.genSetReg(dst, src_ty, src_mcv, .{});
174406 // dst = operand
174407 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
174408 // tmp = operand
174409 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
174410 // tmp = operand >> 1
174411 if (src_abi_size > 4) {
174412 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
174413 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
174414 } else try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
174415 // tmp = (operand >> 1) & 0x55...55
174416 try self.asmRegisterRegister(.{ ._, .sub }, dst, tmp);
174417 // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
174418 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
174419 // tmp = temp1
174420 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
174421 // dst = temp1 >> 2
174422 if (src_abi_size > 4) {
174423 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
174424 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
174425 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
174426 } else {
174427 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
174428 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
174429 }
174430 // tmp = temp1 & 0x33...33
174431 // dst = (temp1 >> 2) & 0x33...33
174432 try self.asmRegisterRegister(.{ ._, .add }, tmp, dst);
174433 // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
174434 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
174435 // dst = temp2
174436 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4));
174437 // tmp = temp2 >> 4
174438 try self.asmRegisterRegister(.{ ._, .add }, dst, tmp);
174439 // dst = temp2 + (temp2 >> 4)
174440 if (src_abi_size > 4) {
174441 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
174442 try self.asmRegisterImmediate(.{ ._, .mov }, tmp, imm_0000_0001);
174443 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
174444 try self.asmRegisterRegister(.{ .i_, .mul }, dst, tmp);
174445 } else {
174446 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
174447 if (src_abi_size > 1) {
174448 try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst, dst, imm_0000_0001);
174449 }
174450 }
174451 // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
174452 // dst = temp3 * 0x01...01
174453 if (src_abi_size > 1) {
174454 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8));
174455 }
174456 // dst = (temp3 * 0x01...01) >> (bits - 8)
174457}
174458
174459fn genByteSwap(
174304fn genUnwrapErrUnionPayloadMir(
174460174305 self: *CodeGen,
174461 inst: Air.Inst.Index,
174462 src_ty: Type,
174463 src_mcv: MCValue,
174464 mem_ok: bool,
174306 maybe_inst: ?Air.Inst.Index,
174307 err_union_ty: Type,
174308 err_union: MCValue,
174465174309) !MCValue {
174466174310 const pt = self.pt;
174467174311 const zcu = pt.zcu;
174468 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
174469 const has_movbe = self.hasFeature(.movbe);
174470
174471 if (src_ty.zigTypeTag(zcu) == .vector) return self.fail(
174472 "TODO implement genByteSwap for {f}",
174473 .{src_ty.fmt(pt)},
174474 );
174475
174476 const src_lock = switch (src_mcv) {
174477 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
174478 else => null,
174479 };
174480 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
174481
174482 const abi_size: u32 = @intCast(src_ty.abiSize(zcu));
174483 switch (abi_size) {
174484 0 => unreachable,
174485 1 => return if ((mem_ok or src_mcv.isRegister()) and
174486 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
174487 src_mcv
174488 else
174489 try self.copyToRegisterWithInstTracking(inst, src_ty, src_mcv),
174490 2 => if ((mem_ok or src_mcv.isRegister()) and
174491 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
174492 {
174493 try self.genBinOpMir(.{ ._l, .ro }, src_ty, src_mcv, .{ .immediate = 8 });
174494 return src_mcv;
174495 },
174496 3...8 => if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
174497 try self.genUnOpMir(.{ .b_, .swap }, src_ty, src_mcv);
174498 return src_mcv;
174499 },
174500 9...16 => {
174501 const mat_src_mcv: MCValue = mat_src_mcv: switch (src_mcv) {
174502 .register => {
174503 const frame_index = try self.allocFrameIndex(.initSpill(src_ty, zcu));
174504 try self.genSetMem(.{ .frame = frame_index }, 0, src_ty, src_mcv, .{});
174505 break :mat_src_mcv .{ .load_frame = .{ .index = frame_index } };
174506 },
174507 .register_pair => |src_regs| if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) {
174508 for (src_regs) |src_reg| try self.asmRegister(.{ .b_, .swap }, src_reg.to64());
174509 return .{ .register_pair = .{ src_regs[1], src_regs[0] } };
174510 } else src_mcv,
174511 else => src_mcv,
174512 };
174513
174514 const dst_regs =
174515 try self.register_manager.allocRegs(2, .{ inst, inst }, abi.RegisterClass.gp);
174516 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
174517 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
174518
174519 for (dst_regs, 0..) |dst_reg, limb_index| {
174520 if (mat_src_mcv.isBase()) {
174521 try self.asmRegisterMemory(
174522 .{ if (has_movbe) ._be else ._, .mov },
174523 dst_reg.to64(),
174524 try mat_src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .{ .size = .qword }),
174525 );
174526 if (!has_movbe) try self.asmRegister(.{ .b_, .swap }, dst_reg.to64());
174527 } else {
174528 try self.asmRegisterRegister(
174529 .{ ._, .mov },
174530 dst_reg.to64(),
174531 mat_src_mcv.register_pair[limb_index].to64(),
174532 );
174533 try self.asmRegister(.{ .b_, .swap }, dst_reg.to64());
174534 }
174535 }
174536 return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } };
174537 },
174538 else => {
174539 const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable;
174540
174541 const temp_regs =
174542 try self.register_manager.allocRegs(4, @splat(null), abi.RegisterClass.gp);
174543 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
174544 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
174545
174546 const dst_mcv = try self.allocRegOrMem(inst, false);
174547 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
174548 try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1));
174549
174550 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
174551 try self.asmRegisterMemory(
174552 .{ if (has_movbe) ._be else ._, .mov },
174553 temp_regs[2].to64(),
174554 .{
174555 .base = .{ .frame = dst_mcv.load_frame.index },
174556 .mod = .{ .rm = .{
174557 .size = .qword,
174558 .index = temp_regs[0].to64(),
174559 .scale = .@"8",
174560 .disp = dst_mcv.load_frame.off,
174561 } },
174562 },
174563 );
174564 try self.asmRegisterMemory(
174565 .{ if (has_movbe) ._be else ._, .mov },
174566 temp_regs[3].to64(),
174567 .{
174568 .base = .{ .frame = dst_mcv.load_frame.index },
174569 .mod = .{ .rm = .{
174570 .size = .qword,
174571 .index = temp_regs[1].to64(),
174572 .scale = .@"8",
174573 .disp = dst_mcv.load_frame.off,
174574 } },
174575 },
174576 );
174577 if (!has_movbe) {
174578 try self.asmRegister(.{ .b_, .swap }, temp_regs[2].to64());
174579 try self.asmRegister(.{ .b_, .swap }, temp_regs[3].to64());
174580 }
174581 try self.asmMemoryRegister(.{ ._, .mov }, .{
174582 .base = .{ .frame = dst_mcv.load_frame.index },
174583 .mod = .{ .rm = .{
174584 .size = .qword,
174585 .index = temp_regs[0].to64(),
174586 .scale = .@"8",
174587 .disp = dst_mcv.load_frame.off,
174588 } },
174589 }, temp_regs[3].to64());
174590 try self.asmMemoryRegister(.{ ._, .mov }, .{
174591 .base = .{ .frame = dst_mcv.load_frame.index },
174592 .mod = .{ .rm = .{
174593 .size = .qword,
174594 .index = temp_regs[1].to64(),
174595 .scale = .@"8",
174596 .disp = dst_mcv.load_frame.off,
174597 } },
174598 }, temp_regs[2].to64());
174599 if (self.hasFeature(.slow_incdec)) {
174600 try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1));
174601 try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1));
174602 } else {
174603 try self.asmRegister(.{ ._c, .in }, temp_regs[0].to32());
174604 try self.asmRegister(.{ ._c, .de }, temp_regs[1].to32());
174605 }
174606 try self.asmRegisterRegister(.{ ._, .cmp }, temp_regs[0].to32(), temp_regs[1].to32());
174607 _ = try self.asmJccReloc(.be, loop);
174608 return dst_mcv;
174609 },
174610 }
174611
174612 const dst_mcv: MCValue = if (mem_ok and has_movbe and src_mcv.isRegister())
174613 try self.allocRegOrMem(inst, true)
174614 else
174615 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };
174616 if (dst_mcv.getReg()) |dst_reg| {
174617 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_mcv.register);
174618 defer self.register_manager.unlockReg(dst_lock);
174619
174620 try self.genSetReg(dst_reg, src_ty, src_mcv, .{});
174621 switch (abi_size) {
174622 else => unreachable,
174623 2 => try self.genBinOpMir(.{ ._l, .ro }, src_ty, dst_mcv, .{ .immediate = 8 }),
174624 3...8 => try self.genUnOpMir(.{ .b_, .swap }, src_ty, dst_mcv),
174625 }
174626 } else try self.genBinOpMir(.{ ._be, .mov }, src_ty, dst_mcv, src_mcv);
174627 return dst_mcv;
174628}
174629
174630fn airByteSwap(self: *CodeGen, inst: Air.Inst.Index) !void {
174631 const pt = self.pt;
174632 const zcu = pt.zcu;
174633 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
174634
174635 const src_ty = self.typeOf(ty_op.operand);
174636 const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
174637 const src_mcv = try self.resolveInst(ty_op.operand);
174638
174639 const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, true);
174640 try self.genShiftBinOpMir(
174641 .{ ._r, switch (if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned) {
174642 .signed => .sa,
174643 .unsigned => .sh,
174644 } },
174645 src_ty,
174646 dst_mcv,
174647 if (src_bits > 256) .u16 else .u8,
174648 .{ .immediate = src_ty.abiSize(zcu) * 8 - src_bits },
174649 );
174650 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
174651}
174652
174653fn airBitReverse(self: *CodeGen, inst: Air.Inst.Index) !void {
174654 const pt = self.pt;
174655 const zcu = pt.zcu;
174656 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
174657
174658 const src_ty = self.typeOf(ty_op.operand);
174659 const abi_size: u32 = @intCast(src_ty.abiSize(zcu));
174660 const bit_size: u32 = @intCast(src_ty.bitSize(zcu));
174661 const src_mcv = try self.resolveInst(ty_op.operand);
174662
174663 const dst_mcv = try self.genByteSwap(inst, src_ty, src_mcv, false);
174664 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
174665 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
174666 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
174667 else => unreachable,
174668 };
174669 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
174670
174671 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
174672 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
174673 defer self.register_manager.unlockReg(tmp_lock);
174674
174675 const limb_abi_size: u32 = @min(abi_size, 8);
174676 const tmp = registerAlias(tmp_reg, limb_abi_size);
174677 const imm = if (limb_abi_size > 4)
174678 try self.register_manager.allocReg(null, abi.RegisterClass.gp)
174679 else
174680 undefined;
174681
174682 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8);
174683 const imm_0000_1111: Immediate = .u(mask / 0b0001_0001);
174684 const imm_00_11: Immediate = .u(mask / 0b01_01);
174685 const imm_0_1: Immediate = .u(mask / 0b1_1);
174686
174687 for (dst_mcv.getRegs()) |dst_reg| {
174688 const dst = registerAlias(dst_reg, limb_abi_size);
174689
174690 // dst = temp1 = bswap(operand)
174691 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
174692 // tmp = temp1
174693 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4));
174694 // dst = temp1 >> 4
174695 if (limb_abi_size > 4) {
174696 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111);
174697 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
174698 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
174699 } else {
174700 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0000_1111);
174701 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0000_1111);
174702 }
174703 // tmp = temp1 & 0x0f...0f
174704 // dst = (temp1 >> 4) & 0x0f...0f
174705 try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4));
174706 // tmp = (temp1 & 0x0f...0f) << 4
174707 try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp);
174708 // dst = temp2 = ((temp1 >> 4) & 0x0f...0f) | ((temp1 & 0x0f...0f) << 4)
174709 try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst);
174710 // tmp = temp2
174711 try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2));
174712 // dst = temp2 >> 2
174713 if (limb_abi_size > 4) {
174714 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11);
174715 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
174716 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
174717 } else {
174718 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_00_11);
174719 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_00_11);
174720 }
174721 // tmp = temp2 & 0x33...33
174722 // dst = (temp2 >> 2) & 0x33...33
174723 try self.asmRegisterMemory(
174724 .{ ._, .lea },
174725 if (limb_abi_size > 4) tmp.to64() else tmp.to32(),
174726 .{
174727 .base = .{ .reg = dst.to64() },
174728 .mod = .{ .rm = .{
174729 .index = tmp.to64(),
174730 .scale = .@"4",
174731 } },
174732 },
174733 );
174734 // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2)
174735 try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp);
174736 // dst = temp3
174737 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1));
174738 // tmp = temp3 >> 1
174739 if (limb_abi_size > 4) {
174740 try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1);
174741 try self.asmRegisterRegister(.{ ._, .@"and" }, dst, imm);
174742 try self.asmRegisterRegister(.{ ._, .@"and" }, tmp, imm);
174743 } else {
174744 try self.asmRegisterImmediate(.{ ._, .@"and" }, dst, imm_0_1);
174745 try self.asmRegisterImmediate(.{ ._, .@"and" }, tmp, imm_0_1);
174746 }
174747 // dst = temp3 & 0x55...55
174748 // tmp = (temp3 >> 1) & 0x55...55
174749 try self.asmRegisterMemory(
174750 .{ ._, .lea },
174751 if (limb_abi_size > 4) dst.to64() else dst.to32(),
174752 .{
174753 .base = .{ .reg = tmp.to64() },
174754 .mod = .{ .rm = .{
174755 .index = dst.to64(),
174756 .scale = .@"2",
174757 } },
174758 },
174759 );
174760 // dst = ((temp3 >> 1) & 0x55...55) + ((temp3 & 0x55...55) << 1)
174761 }
174762
174763 const extra_bits = abi_size * 8 - bit_size;
174764 const signedness: std.builtin.Signedness =
174765 if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
174766 if (extra_bits > 0) try self.genShiftBinOpMir(switch (signedness) {
174767 .signed => .{ ._r, .sa },
174768 .unsigned => .{ ._r, .sh },
174769 }, src_ty, dst_mcv, .u8, .{ .immediate = extra_bits });
174770
174771 return self.finishAir(inst, dst_mcv, .{ ty_op.operand, .none, .none });
174772}
174773
174774fn floatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag, operand: Air.Inst.Ref, ty: Type) !void {
174775 const pt = self.pt;
174776 const zcu = pt.zcu;
174777
174778 const result = result: {
174779 const scalar_bits = ty.scalarType(zcu).floatBits(self.target);
174780 if (scalar_bits == 80) {
174781 if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement floatSign for {f}", .{
174782 ty.fmt(pt),
174783 });
174784
174785 const src_mcv = try self.resolveInst(operand);
174786 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
174787 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
174788
174789 const dst_mcv: MCValue = .{ .register = .st0 };
174790 if (!std.meta.eql(src_mcv, dst_mcv) or !self.reuseOperand(inst, operand, 0, src_mcv))
174791 try self.register_manager.getKnownReg(.st0, inst);
174792
174793 try self.genCopy(ty, dst_mcv, src_mcv, .{});
174794 switch (tag) {
174795 .neg => try self.asmOpOnly(.{ .f_, .chs }),
174796 .abs => try self.asmOpOnly(.{ .f_, .abs }),
174797 else => unreachable,
174798 }
174799 break :result dst_mcv;
174800 }
174801
174802 const abi_size: u32 = switch (ty.abiSize(zcu)) {
174803 1...16 => 16,
174804 17...32 => 32,
174805 else => return self.fail("TODO implement floatSign for {f}", .{
174806 ty.fmt(pt),
174807 }),
174808 };
174809
174810 const src_mcv = try self.resolveInst(operand);
174811 const src_lock = if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
174812 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
174813
174814 const dst_mcv: MCValue = if (src_mcv.isRegister() and
174815 self.reuseOperand(inst, operand, 0, src_mcv))
174816 src_mcv
174817 else if (self.hasFeature(.avx))
174818 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }
174819 else
174820 try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
174821 const dst_reg = dst_mcv.getReg().?;
174822 const dst_lock = self.register_manager.lockReg(dst_reg);
174823 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
174312 const payload_ty = err_union_ty.errorUnionPayload(zcu);
174824174313
174825 const vec_ty = try pt.vectorType(.{
174826 .len = @divExact(abi_size * 8, scalar_bits),
174827 .child = (try pt.intType(.signed, scalar_bits)).ip_index,
174828 });
174314 const result: MCValue = result: {
174315 if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none;
174829174316
174830 const sign_mcv = try self.lowerValue(switch (tag) {
174831 .neg => try vec_ty.minInt(pt, vec_ty),
174832 .abs => try vec_ty.maxInt(pt, vec_ty),
174833 else => unreachable,
174834 });
174835 const sign_mem: Memory = if (sign_mcv.isBase())
174836 try sign_mcv.mem(self, .{ .size = .fromSize(abi_size) })
174837 else
174838 .{
174839 .base = .{ .reg = try self.copyToTmpRegister(.usize, sign_mcv.address()) },
174840 .mod = .{ .rm = .{ .size = .fromSize(abi_size) } },
174841 };
174317 const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu));
174318 switch (err_union) {
174319 .load_frame => |frame_addr| break :result .{ .load_frame = .{
174320 .index = frame_addr.index,
174321 .off = frame_addr.off + payload_off,
174322 } },
174323 .register => |reg| {
174324 // TODO reuse operand
174325 const eu_lock = self.register_manager.lockReg(reg);
174326 defer if (eu_lock) |lock| self.register_manager.unlockReg(lock);
174842174327
174843 if (self.hasFeature(.avx)) try self.asmRegisterRegisterMemory(
174844 switch (scalar_bits) {
174845 16, 128 => if (abi_size <= 16 or self.hasFeature(.avx2)) switch (tag) {
174846 .neg => .{ .vp_, .xor },
174847 .abs => .{ .vp_, .@"and" },
174848 else => unreachable,
174849 } else switch (tag) {
174850 .neg => .{ .v_ps, .xor },
174851 .abs => .{ .v_ps, .@"and" },
174852 else => unreachable,
174853 },
174854 32 => switch (tag) {
174855 .neg => .{ .v_ps, .xor },
174856 .abs => .{ .v_ps, .@"and" },
174857 else => unreachable,
174858 },
174859 64 => switch (tag) {
174860 .neg => .{ .v_pd, .xor },
174861 .abs => .{ .v_pd, .@"and" },
174862 else => unreachable,
174863 },
174864 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}),
174865 else => unreachable,
174866 },
174867 registerAlias(dst_reg, abi_size),
174868 registerAlias(if (src_mcv.isRegister())
174869 src_mcv.getReg().?
174870 else
174871 try self.copyToTmpRegister(ty, src_mcv), abi_size),
174872 sign_mem,
174873 ) else try self.asmRegisterMemory(
174874 switch (scalar_bits) {
174875 16, 128 => switch (tag) {
174876 .neg => .{ .p_, .xor },
174877 .abs => .{ .p_, .@"and" },
174878 else => unreachable,
174879 },
174880 32 => switch (tag) {
174881 .neg => .{ ._ps, .xor },
174882 .abs => .{ ._ps, .@"and" },
174883 else => unreachable,
174884 },
174885 64 => switch (tag) {
174886 .neg => .{ ._pd, .xor },
174887 .abs => .{ ._pd, .@"and" },
174888 else => unreachable,
174889 },
174890 80 => return self.fail("TODO implement floatSign for {f}", .{ty.fmt(pt)}),
174891 else => unreachable,
174328 const payload_in_gp = self.regSetForType(payload_ty).supersetOf(abi.RegisterClass.gp);
174329 const result_mcv: MCValue = if (payload_in_gp and maybe_inst != null)
174330 try self.copyToRegisterWithInstTracking(maybe_inst.?, err_union_ty, err_union)
174331 else
174332 .{ .register = try self.copyToTmpRegister(err_union_ty, err_union) };
174333 if (payload_off > 0) try self.genShiftBinOpMir(
174334 .{ ._r, .sh },
174335 err_union_ty,
174336 result_mcv,
174337 .u8,
174338 .{ .immediate = @as(u6, @intCast(payload_off * 8)) },
174339 ) else try self.truncateRegister(payload_ty, result_mcv.register);
174340 break :result if (payload_in_gp)
174341 result_mcv
174342 else if (maybe_inst) |inst|
174343 try self.copyToRegisterWithInstTracking(inst, payload_ty, result_mcv)
174344 else
174345 .{ .register = try self.copyToTmpRegister(payload_ty, result_mcv) };
174892174346 },
174893 registerAlias(dst_reg, abi_size),
174894 sign_mem,
174895 );
174896 break :result dst_mcv;
174897 };
174898 return self.finishAir(inst, result, .{ operand, .none, .none });
174899}
174900
174901fn airFloatSign(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
174902 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
174903 const ty = self.typeOf(un_op);
174904 return self.floatSign(inst, tag, un_op, ty);
174905}
174906
174907fn airRound(self: *CodeGen, inst: Air.Inst.Index, mode: bits.RoundMode) !void {
174908 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
174909 const ty = self.typeOf(un_op);
174910
174911 const result = result: {
174912 switch (try self.genRoundLibcall(ty, .{ .air_ref = un_op }, mode)) {
174913 .none => {},
174914 else => |dst_mcv| break :result dst_mcv,
174347 else => return self.fail("TODO implement genUnwrapErrUnionPayloadMir for {f}", .{err_union}),
174915174348 }
174916
174917 const src_mcv = try self.resolveInst(un_op);
174918 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
174919 src_mcv
174920 else
174921 try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
174922 const dst_reg = dst_mcv.getReg().?;
174923 const dst_lock = self.register_manager.lockReg(dst_reg);
174924 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
174925 try self.genRound(ty, dst_reg, src_mcv, mode);
174926 break :result dst_mcv;
174927 };
174928 return self.finishAir(inst, result, .{ un_op, .none, .none });
174929}
174930
174931fn getRoundTag(self: *CodeGen, ty: Type) ?Mir.Inst.FixedTag {
174932 const pt = self.pt;
174933 const zcu = pt.zcu;
174934 return if (self.hasFeature(.sse4_1)) switch (ty.zigTypeTag(zcu)) {
174935 .float => switch (ty.floatBits(self.target)) {
174936 32 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
174937 64 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
174938 16, 80, 128 => null,
174939 else => unreachable,
174940 },
174941 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
174942 .float => switch (ty.childType(zcu).floatBits(self.target)) {
174943 32 => switch (ty.vectorLen(zcu)) {
174944 1 => if (self.hasFeature(.avx)) .{ .v_ss, .round } else .{ ._ss, .round },
174945 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else .{ ._ps, .round },
174946 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .round } else null,
174947 else => null,
174948 },
174949 64 => switch (ty.vectorLen(zcu)) {
174950 1 => if (self.hasFeature(.avx)) .{ .v_sd, .round } else .{ ._sd, .round },
174951 2 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else .{ ._pd, .round },
174952 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .round } else null,
174953 else => null,
174954 },
174955 16, 80, 128 => null,
174956 else => unreachable,
174957 },
174958 else => null,
174959 },
174960 else => unreachable,
174961 } else null;
174962}
174963
174964fn genRoundLibcall(self: *CodeGen, ty: Type, src_mcv: MCValue, mode: bits.RoundMode) !MCValue {
174965 const pt = self.pt;
174966 const zcu = pt.zcu;
174967 if (self.getRoundTag(ty)) |_| return .none;
174968
174969 if (ty.zigTypeTag(zcu) != .float)
174970 return self.fail("TODO implement genRound for {f}", .{ty.fmt(pt)});
174971
174972 var sym_buf: ["__trunc?".len]u8 = undefined;
174973 return try self.genCall(.{ .extern_func = .{
174974 .return_type = ty.toIntern(),
174975 .param_types = &.{ty.toIntern()},
174976 .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{
174977 floatLibcAbiPrefix(ty),
174978 switch (mode.direction) {
174979 .down => "floor",
174980 .up => "ceil",
174981 .zero => "trunc",
174982 else => unreachable,
174983 },
174984 floatLibcAbiSuffix(ty),
174985 }) catch unreachable,
174986 } }, &.{ty}, &.{src_mcv}, .{});
174987}
174988
174989fn genRound(self: *CodeGen, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: bits.RoundMode) !void {
174990 const pt = self.pt;
174991 const mir_tag = self.getRoundTag(ty) orelse {
174992 const result = try self.genRoundLibcall(ty, src_mcv, mode);
174993 return self.genSetReg(dst_reg, ty, result, .{});
174994174349 };
174995 const abi_size: u32 = @intCast(ty.abiSize(pt.zcu));
174996 const dst_alias = registerAlias(dst_reg, abi_size);
174997 switch (mir_tag[0]) {
174998 .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
174999 mir_tag,
175000 dst_alias,
175001 dst_alias,
175002 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
175003 mode.imm(),
175004 ) else try self.asmRegisterRegisterRegisterImmediate(
175005 mir_tag,
175006 dst_alias,
175007 dst_alias,
175008 registerAlias(if (src_mcv.isRegister())
175009 src_mcv.getReg().?
175010 else
175011 try self.copyToTmpRegister(ty, src_mcv), abi_size),
175012 mode.imm(),
175013 ),
175014 else => if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
175015 mir_tag,
175016 dst_alias,
175017 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
175018 mode.imm(),
175019 ) else try self.asmRegisterRegisterImmediate(
175020 mir_tag,
175021 dst_alias,
175022 registerAlias(if (src_mcv.isRegister())
175023 src_mcv.getReg().?
175024 else
175025 try self.copyToTmpRegister(ty, src_mcv), abi_size),
175026 mode.imm(),
175027 ),
175028 }
175029}
175030
175031fn airAbs(self: *CodeGen, inst: Air.Inst.Index) !void {
175032 const pt = self.pt;
175033 const zcu = pt.zcu;
175034 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
175035 const ty = self.typeOf(ty_op.operand);
175036
175037 const result: MCValue = result: {
175038 const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) {
175039 else => null,
175040 .int => switch (ty.abiSize(zcu)) {
175041 0 => unreachable,
175042 1...8 => {
175043 try self.spillEflagsIfOccupied();
175044 const src_mcv = try self.resolveInst(ty_op.operand);
175045 const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
175046
175047 try self.genUnOpMir(.{ ._, .neg }, ty, dst_mcv);
175048
175049 const cmov_abi_size = @max(@as(u32, @intCast(ty.abiSize(zcu))), 2);
175050 switch (src_mcv) {
175051 .register => |val_reg| try self.asmCmovccRegisterRegister(
175052 .l,
175053 registerAlias(dst_mcv.register, cmov_abi_size),
175054 registerAlias(val_reg, cmov_abi_size),
175055 ),
175056 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
175057 .l,
175058 registerAlias(dst_mcv.register, cmov_abi_size),
175059 try src_mcv.mem(self, .{ .size = .fromSize(cmov_abi_size) }),
175060 ),
175061 else => {
175062 const val_reg = try self.copyToTmpRegister(ty, src_mcv);
175063 try self.asmCmovccRegisterRegister(
175064 .l,
175065 registerAlias(dst_mcv.register, cmov_abi_size),
175066 registerAlias(val_reg, cmov_abi_size),
175067 );
175068 },
175069 }
175070 break :result dst_mcv;
175071 },
175072 9...16 => {
175073 try self.spillEflagsIfOccupied();
175074 const src_mcv = try self.resolveInst(ty_op.operand);
175075 const dst_mcv = if (src_mcv == .register_pair and
175076 self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: {
175077 const dst_regs = try self.register_manager.allocRegs(
175078 2,
175079 .{ inst, inst },
175080 abi.RegisterClass.gp,
175081 );
175082 const dst_mcv: MCValue = .{ .register_pair = dst_regs };
175083 const dst_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
175084 defer for (dst_locks) |lock| self.register_manager.unlockReg(lock);
175085
175086 try self.genCopy(ty, dst_mcv, src_mcv, .{});
175087 break :dst dst_mcv;
175088 };
175089 const dst_regs = dst_mcv.register_pair;
175090 const dst_locks = self.register_manager.lockRegs(2, dst_regs);
175091 defer for (dst_locks) |dst_lock| if (dst_lock) |lock|
175092 self.register_manager.unlockReg(lock);
175093
175094 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
175095 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
175096 defer self.register_manager.unlockReg(tmp_lock);
175097
175098 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
175099 try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63));
175100 try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg);
175101 try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg);
175102 try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg);
175103 try self.asmRegisterRegister(.{ ._, .sbb }, dst_regs[1], tmp_reg);
175104
175105 break :result dst_mcv;
175106 },
175107 else => {
175108 const abi_size: u31 = @intCast(ty.abiSize(zcu));
175109 const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable;
175110
175111 const tmp_regs =
175112 try self.register_manager.allocRegs(3, @splat(null), abi.RegisterClass.gp);
175113 const tmp_locks = self.register_manager.lockRegsAssumeUnused(3, tmp_regs);
175114 defer for (tmp_locks) |lock| self.register_manager.unlockReg(lock);
175115
175116 try self.spillEflagsIfOccupied();
175117 const src_mcv = try self.resolveInst(ty_op.operand);
175118 const dst_mcv = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
175119 src_mcv
175120 else
175121 try self.allocRegOrMem(inst, false);
175122
175123 try self.asmMemoryImmediate(
175124 .{ ._, .cmp },
175125 try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .{ .size = .qword }),
175126 .u(0),
175127 );
175128 const positive = try self.asmJccReloc(.ns, undefined);
175129
175130 try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[0].to32(), tmp_regs[0].to32());
175131 try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[1].to8(), tmp_regs[1].to8());
175132
175133 const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
175134 try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32());
175135 try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1));
175136 try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{
175137 .base = .{ .frame = dst_mcv.load_frame.index },
175138 .mod = .{ .rm = .{
175139 .size = .qword,
175140 .index = tmp_regs[0].to64(),
175141 .scale = .@"8",
175142 .disp = dst_mcv.load_frame.off,
175143 } },
175144 });
175145 try self.asmSetccRegister(.c, tmp_regs[1].to8());
175146 try self.asmMemoryRegister(.{ ._, .mov }, .{
175147 .base = .{ .frame = dst_mcv.load_frame.index },
175148 .mod = .{ .rm = .{
175149 .size = .qword,
175150 .index = tmp_regs[0].to64(),
175151 .scale = .@"8",
175152 .disp = dst_mcv.load_frame.off,
175153 } },
175154 }, tmp_regs[2].to64());
175155
175156 if (self.hasFeature(.slow_incdec)) {
175157 try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1));
175158 } else {
175159 try self.asmRegister(.{ ._c, .in }, tmp_regs[0].to32());
175160 }
175161 try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len));
175162 _ = try self.asmJccReloc(.b, neg_loop);
175163
175164 self.performReloc(positive);
175165 break :result dst_mcv;
175166 },
175167 },
175168 .float => return self.floatSign(inst, .abs, ty_op.operand, ty),
175169 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
175170 else => null,
175171 .int => switch (ty.childType(zcu).intInfo(zcu).bits) {
175172 else => null,
175173 8 => switch (ty.vectorLen(zcu)) {
175174 else => null,
175175 1...16 => if (self.hasFeature(.avx))
175176 .{ .vp_b, .abs }
175177 else if (self.hasFeature(.ssse3))
175178 .{ .p_b, .abs }
175179 else
175180 null,
175181 17...32 => if (self.hasFeature(.avx2)) .{ .vp_b, .abs } else null,
175182 },
175183 16 => switch (ty.vectorLen(zcu)) {
175184 else => null,
175185 1...8 => if (self.hasFeature(.avx))
175186 .{ .vp_w, .abs }
175187 else if (self.hasFeature(.ssse3))
175188 .{ .p_w, .abs }
175189 else
175190 null,
175191 9...16 => if (self.hasFeature(.avx2)) .{ .vp_w, .abs } else null,
175192 },
175193 32 => switch (ty.vectorLen(zcu)) {
175194 else => null,
175195 1...4 => if (self.hasFeature(.avx))
175196 .{ .vp_d, .abs }
175197 else if (self.hasFeature(.ssse3))
175198 .{ .p_d, .abs }
175199 else
175200 null,
175201 5...8 => if (self.hasFeature(.avx2)) .{ .vp_d, .abs } else null,
175202 },
175203 },
175204 .float => return self.floatSign(inst, .abs, ty_op.operand, ty),
175205 },
175206 }) orelse return self.fail("TODO implement airAbs for {f}", .{ty.fmt(pt)});
175207174350
175208 const abi_size: u32 = @intCast(ty.abiSize(zcu));
175209 const src_mcv = try self.resolveInst(ty_op.operand);
175210 const dst_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
175211 src_mcv.getReg().?
175212 else
175213 try self.register_manager.allocReg(inst, self.regSetForType(ty));
175214 const dst_alias = registerAlias(dst_reg, abi_size);
175215 if (src_mcv.isBase()) try self.asmRegisterMemory(
175216 mir_tag,
175217 dst_alias,
175218 try src_mcv.mem(self, .{ .size = self.memSize(ty) }),
175219 ) else try self.asmRegisterRegister(
175220 mir_tag,
175221 dst_alias,
175222 registerAlias(if (src_mcv.isRegister())
175223 src_mcv.getReg().?
175224 else
175225 try self.copyToTmpRegister(ty, src_mcv), abi_size),
175226 );
175227 break :result .{ .register = dst_reg };
175228 };
175229 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
174351 return result;
175230174352}
175231174353
175232fn airSqrt(self: *CodeGen, inst: Air.Inst.Index) !void {
174354fn genUnwrapErrUnionPayloadPtrMir(
174355 self: *CodeGen,
174356 maybe_inst: ?Air.Inst.Index,
174357 ptr_ty: Type,
174358 ptr_mcv: MCValue,
174359) !MCValue {
175233174360 const pt = self.pt;
175234174361 const zcu = pt.zcu;
175235 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
175236 const ty = self.typeOf(un_op);
175237 const abi_size: u32 = @intCast(ty.abiSize(zcu));
174362 const err_union_ty = ptr_ty.childType(zcu);
174363 const payload_ty = err_union_ty.errorUnionPayload(zcu);
175238174364
175239174365 const result: MCValue = result: {
175240 switch (ty.zigTypeTag(zcu)) {
175241 .float => {
175242 const float_bits = ty.floatBits(self.target);
175243 if (switch (float_bits) {
175244 16 => !self.hasFeature(.f16c),
175245 32, 64 => false,
175246 80, 128 => true,
175247 else => unreachable,
175248 }) {
175249 var sym_buf: ["__sqrt?".len]u8 = undefined;
175250 break :result try self.genCall(.{ .extern_func = .{
175251 .return_type = ty.toIntern(),
175252 .param_types = &.{ty.toIntern()},
175253 .sym = std.fmt.bufPrint(&sym_buf, "{s}sqrt{s}", .{
175254 floatLibcAbiPrefix(ty),
175255 floatLibcAbiSuffix(ty),
175256 }) catch unreachable,
175257 } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
175258 }
175259 },
175260 else => {},
175261 }
175262
175263 const src_mcv = try self.resolveInst(un_op);
175264 const dst_mcv = if (src_mcv.isRegister() and self.reuseOperand(inst, un_op, 0, src_mcv))
175265 src_mcv
174366 const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu);
174367 const result_mcv: MCValue = if (maybe_inst) |inst|
174368 try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv)
175266174369 else
175267 try self.copyToRegisterWithInstTracking(inst, ty, src_mcv);
175268 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
175269 const dst_lock = self.register_manager.lockReg(dst_reg);
175270 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
175271
175272 const mir_tag = @as(?Mir.Inst.FixedTag, switch (ty.zigTypeTag(zcu)) {
175273 .float => switch (ty.floatBits(self.target)) {
175274 16 => {
175275 assert(self.hasFeature(.f16c));
175276 const mat_src_reg = if (src_mcv.isRegister())
175277 src_mcv.getReg().?
175278 else
175279 try self.copyToTmpRegister(ty, src_mcv);
175280 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, mat_src_reg.to128());
175281 try self.asmRegisterRegisterRegister(.{ .v_ss, .sqrt }, dst_reg, dst_reg, dst_reg);
175282 try self.asmRegisterRegisterImmediate(
175283 .{ .v_, .cvtps2ph },
175284 dst_reg,
175285 dst_reg,
175286 bits.RoundMode.imm(.{}),
175287 );
175288 break :result dst_mcv;
175289 },
175290 32 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
175291 64 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
175292 else => unreachable,
175293 },
175294 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
175295 .float => switch (ty.childType(zcu).floatBits(self.target)) {
175296 16 => if (self.hasFeature(.f16c)) switch (ty.vectorLen(zcu)) {
175297 1 => {
175298 try self.asmRegisterRegister(
175299 .{ .v_ps, .cvtph2 },
175300 dst_reg,
175301 (if (src_mcv.isRegister())
175302 src_mcv.getReg().?
175303 else
175304 try self.copyToTmpRegister(ty, src_mcv)).to128(),
175305 );
175306 try self.asmRegisterRegisterRegister(
175307 .{ .v_ss, .sqrt },
175308 dst_reg,
175309 dst_reg,
175310 dst_reg,
175311 );
175312 try self.asmRegisterRegisterImmediate(
175313 .{ .v_, .cvtps2ph },
175314 dst_reg,
175315 dst_reg,
175316 bits.RoundMode.imm(.{}),
175317 );
175318 break :result dst_mcv;
175319 },
175320 2...8 => {
175321 const wide_reg = registerAlias(dst_reg, abi_size * 2);
175322 if (src_mcv.isBase()) try self.asmRegisterMemory(
175323 .{ .v_ps, .cvtph2 },
175324 wide_reg,
175325 try src_mcv.mem(self, .{ .size = .fromSize(
175326 @intCast(@divExact(wide_reg.bitSize(), 16)),
175327 ) }),
175328 ) else try self.asmRegisterRegister(
175329 .{ .v_ps, .cvtph2 },
175330 wide_reg,
175331 (if (src_mcv.isRegister())
175332 src_mcv.getReg().?
175333 else
175334 try self.copyToTmpRegister(ty, src_mcv)).to128(),
175335 );
175336 try self.asmRegisterRegister(.{ .v_ps, .sqrt }, wide_reg, wide_reg);
175337 try self.asmRegisterRegisterImmediate(
175338 .{ .v_, .cvtps2ph },
175339 dst_reg,
175340 wide_reg,
175341 bits.RoundMode.imm(.{}),
175342 );
175343 break :result dst_mcv;
175344 },
175345 else => null,
175346 } else null,
175347 32 => switch (ty.vectorLen(zcu)) {
175348 1 => if (self.hasFeature(.avx)) .{ .v_ss, .sqrt } else .{ ._ss, .sqrt },
175349 2...4 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else .{ ._ps, .sqrt },
175350 5...8 => if (self.hasFeature(.avx)) .{ .v_ps, .sqrt } else null,
175351 else => null,
175352 },
175353 64 => switch (ty.vectorLen(zcu)) {
175354 1 => if (self.hasFeature(.avx)) .{ .v_sd, .sqrt } else .{ ._sd, .sqrt },
175355 2 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else .{ ._pd, .sqrt },
175356 3...4 => if (self.hasFeature(.avx)) .{ .v_pd, .sqrt } else null,
175357 else => null,
175358 },
175359 80, 128 => null,
175360 else => unreachable,
175361 },
175362 else => unreachable,
175363 },
175364 else => unreachable,
175365 }) orelse return self.fail("TODO implement airSqrt for {f}", .{ty.fmt(pt)});
175366 switch (mir_tag[0]) {
175367 .v_ss, .v_sd => if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
175368 mir_tag,
175369 dst_reg,
175370 dst_reg,
175371 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
175372 ) else try self.asmRegisterRegisterRegister(
175373 mir_tag,
175374 dst_reg,
175375 dst_reg,
175376 registerAlias(if (src_mcv.isRegister())
175377 src_mcv.getReg().?
175378 else
175379 try self.copyToTmpRegister(ty, src_mcv), abi_size),
175380 ),
175381 else => if (src_mcv.isBase()) try self.asmRegisterMemory(
175382 mir_tag,
175383 dst_reg,
175384 try src_mcv.mem(self, .{ .size = .fromSize(abi_size) }),
175385 ) else try self.asmRegisterRegister(
175386 mir_tag,
175387 dst_reg,
175388 registerAlias(if (src_mcv.isRegister())
175389 src_mcv.getReg().?
175390 else
175391 try self.copyToTmpRegister(ty, src_mcv), abi_size),
175392 ),
175393 }
175394 break :result dst_mcv;
174370 .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) };
174371 try self.genBinOpMir(.{ ._, .add }, ptr_ty, result_mcv, .{ .immediate = payload_off });
174372 break :result result_mcv;
175395174373 };
175396 return self.finishAir(inst, result, .{ un_op, .none, .none });
175397}
175398174374
175399fn airUnaryMath(self: *CodeGen, inst: Air.Inst.Index, tag: Air.Inst.Tag) !void {
175400 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
175401 const ty = self.typeOf(un_op);
175402 var sym_buf: ["__round?".len]u8 = undefined;
175403 const result = try self.genCall(.{ .extern_func = .{
175404 .return_type = ty.toIntern(),
175405 .param_types = &.{ty.toIntern()},
175406 .sym = std.fmt.bufPrint(&sym_buf, "{s}{s}{s}", .{
175407 floatLibcAbiPrefix(ty),
175408 switch (tag) {
175409 .sin,
175410 .cos,
175411 .tan,
175412 .exp,
175413 .exp2,
175414 .log,
175415 .log2,
175416 .log10,
175417 .round,
175418 => @tagName(tag),
175419 else => unreachable,
175420 },
175421 floatLibcAbiSuffix(ty),
175422 }) catch unreachable,
175423 } }, &.{ty}, &.{.{ .air_ref = un_op }}, .{});
175424 return self.finishAir(inst, result, .{ un_op, .none, .none });
174375 return result;
175425174376}
175426174377
175427174378fn reuseOperand(
......@@ -175573,95 +174524,6 @@ fn store(
175573174524 }
175574174525}
175575174526
175576fn genUnOp(self: *CodeGen, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: Air.Inst.Ref) !MCValue {
175577 const pt = self.pt;
175578 const zcu = pt.zcu;
175579 const src_ty = self.typeOf(src_air);
175580 if (src_ty.zigTypeTag(zcu) == .vector)
175581 return self.fail("TODO implement genUnOp for {f}", .{src_ty.fmt(pt)});
175582
175583 var src_mcv = try self.resolveInst(src_air);
175584 switch (src_mcv) {
175585 .eflags => |cc| switch (tag) {
175586 .not => {
175587 if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv))
175588 return .{ .eflags = cc.negate() };
175589 try self.spillEflagsIfOccupied();
175590 src_mcv = try self.resolveInst(src_air);
175591 },
175592 else => {},
175593 },
175594 else => {},
175595 }
175596
175597 const src_lock = switch (src_mcv) {
175598 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
175599 else => null,
175600 };
175601 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
175602
175603 const dst_mcv: MCValue = dst: {
175604 if (maybe_inst) |inst| if (self.reuseOperand(inst, src_air, 0, src_mcv)) break :dst src_mcv;
175605
175606 const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, maybe_inst, true);
175607 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
175608 break :dst dst_mcv;
175609 };
175610 const dst_lock = switch (dst_mcv) {
175611 .register => |reg| self.register_manager.lockReg(reg),
175612 else => null,
175613 };
175614 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
175615
175616 const abi_size: u16 = @intCast(src_ty.abiSize(zcu));
175617 switch (tag) {
175618 .not => {
175619 const limb_abi_size: u16 = @min(abi_size, 8);
175620 const int_info: InternPool.Key.IntType = if (src_ty.ip_index == .bool_type)
175621 .{ .signedness = .unsigned, .bits = 1 }
175622 else
175623 src_ty.intInfo(zcu);
175624 var byte_off: i32 = 0;
175625 while (byte_off * 8 < int_info.bits) : (byte_off += limb_abi_size) {
175626 const limb_bits: u16 = @intCast(@min(switch (int_info.signedness) {
175627 .signed => abi_size * 8,
175628 .unsigned => int_info.bits,
175629 } - byte_off * 8, limb_abi_size * 8));
175630 const limb_ty = try pt.intType(int_info.signedness, limb_bits);
175631 const limb_mcv = switch (byte_off) {
175632 0 => dst_mcv,
175633 else => dst_mcv.address().offset(byte_off).deref(),
175634 };
175635
175636 if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) {
175637 const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits);
175638 try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask });
175639 } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv);
175640 }
175641 },
175642 .neg => {
175643 try self.genUnOpMir(.{ ._, .neg }, src_ty, dst_mcv);
175644 const bit_size = src_ty.intInfo(zcu).bits;
175645 if (abi_size * 8 > bit_size) {
175646 if (dst_mcv.isRegister()) {
175647 try self.truncateRegister(src_ty, dst_mcv.getReg().?);
175648 } else {
175649 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
175650 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
175651 defer self.register_manager.unlockReg(tmp_lock);
175652
175653 const hi_mcv = dst_mcv.address().offset(@intCast(bit_size / 64 * 8)).deref();
175654 try self.genSetReg(tmp_reg, .usize, hi_mcv, .{});
175655 try self.truncateRegister(src_ty, tmp_reg);
175656 try self.genCopy(.usize, hi_mcv, .{ .register = tmp_reg }, .{});
175657 }
175658 }
175659 },
175660 else => unreachable,
175661 }
175662 return dst_mcv;
175663}
175664
175665174527fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv: MCValue) !void {
175666174528 const pt = self.pt;
175667174529 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
......@@ -176346,1679 +175208,6 @@ fn genShiftBinOpMir(
176346175208 });
176347175209}
176348175210
176349fn genBinOp(
176350 self: *CodeGen,
176351 maybe_inst: ?Air.Inst.Index,
176352 air_tag: Air.Inst.Tag,
176353 lhs_air: Air.Inst.Ref,
176354 rhs_air: Air.Inst.Ref,
176355) !MCValue {
176356 const pt = self.pt;
176357 const zcu = pt.zcu;
176358 const lhs_ty = self.typeOf(lhs_air);
176359 const rhs_ty = self.typeOf(rhs_air);
176360 const abi_size: u32 = @intCast(lhs_ty.abiSize(zcu));
176361
176362 if (lhs_ty.isRuntimeFloat()) libcall: {
176363 const float_bits = lhs_ty.floatBits(self.target);
176364 const type_needs_libcall = switch (float_bits) {
176365 16 => !self.hasFeature(.f16c),
176366 32, 64 => false,
176367 80, 128 => true,
176368 else => unreachable,
176369 };
176370 switch (air_tag) {
176371 .rem, .mod => {},
176372 else => if (!type_needs_libcall) break :libcall,
176373 }
176374 var sym_buf: ["__mod?f3".len]u8 = undefined;
176375 const sym = switch (air_tag) {
176376 .add,
176377 .sub,
176378 .mul,
176379 .div_float,
176380 .div_trunc,
176381 .div_floor,
176382 .div_exact,
176383 => std.fmt.bufPrint(&sym_buf, "__{s}{c}f3", .{
176384 @tagName(air_tag)[0..3],
176385 floatCompilerRtAbiName(float_bits),
176386 }),
176387 .rem, .mod, .min, .max => std.fmt.bufPrint(&sym_buf, "{s}f{s}{s}", .{
176388 floatLibcAbiPrefix(lhs_ty),
176389 switch (air_tag) {
176390 .rem, .mod => "mod",
176391 .min => "min",
176392 .max => "max",
176393 else => unreachable,
176394 },
176395 floatLibcAbiSuffix(lhs_ty),
176396 }),
176397 else => return self.fail("TODO implement genBinOp for {s} {f}", .{
176398 @tagName(air_tag), lhs_ty.fmt(pt),
176399 }),
176400 } catch unreachable;
176401 const result = try self.genCall(.{ .extern_func = .{
176402 .return_type = lhs_ty.toIntern(),
176403 .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
176404 .sym = sym,
176405 } }, &.{ lhs_ty, rhs_ty }, &.{ .{ .air_ref = lhs_air }, .{ .air_ref = rhs_air } }, .{});
176406 return switch (air_tag) {
176407 .mod => result: {
176408 const adjusted: MCValue = if (type_needs_libcall) adjusted: {
176409 var add_sym_buf: ["__add?f3".len]u8 = undefined;
176410 break :adjusted try self.genCall(.{ .extern_func = .{
176411 .return_type = lhs_ty.toIntern(),
176412 .param_types = &.{
176413 lhs_ty.toIntern(),
176414 rhs_ty.toIntern(),
176415 },
176416 .sym = std.fmt.bufPrint(&add_sym_buf, "__add{c}f3", .{
176417 floatCompilerRtAbiName(float_bits),
176418 }) catch unreachable,
176419 } }, &.{ lhs_ty, rhs_ty }, &.{ result, .{ .air_ref = rhs_air } }, .{});
176420 } else switch (float_bits) {
176421 16, 32, 64 => adjusted: {
176422 const dst_reg = switch (result) {
176423 .register => |reg| reg,
176424 else => if (maybe_inst) |inst|
176425 (try self.copyToRegisterWithInstTracking(inst, lhs_ty, result)).register
176426 else
176427 try self.copyToTmpRegister(lhs_ty, result),
176428 };
176429 const dst_lock = self.register_manager.lockReg(dst_reg);
176430 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
176431
176432 const rhs_mcv = try self.resolveInst(rhs_air);
176433 const src_mcv: MCValue = if (float_bits == 16) src: {
176434 assert(self.hasFeature(.f16c));
176435 const tmp_reg = (try self.register_manager.allocReg(
176436 null,
176437 abi.RegisterClass.sse,
176438 )).to128();
176439 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
176440 defer self.register_manager.unlockReg(tmp_lock);
176441
176442 if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
176443 .{ .vp_w, .insr },
176444 dst_reg,
176445 dst_reg,
176446 try rhs_mcv.mem(self, .{ .size = .word }),
176447 .u(1),
176448 ) else try self.asmRegisterRegisterRegister(
176449 .{ .vp_, .unpcklwd },
176450 dst_reg,
176451 dst_reg,
176452 (if (rhs_mcv.isRegister())
176453 rhs_mcv.getReg().?
176454 else
176455 try self.copyToTmpRegister(rhs_ty, rhs_mcv)).to128(),
176456 );
176457 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
176458 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
176459 break :src .{ .register = tmp_reg };
176460 } else rhs_mcv;
176461
176462 if (self.hasFeature(.avx)) {
176463 const mir_tag: Mir.Inst.FixedTag = switch (float_bits) {
176464 16, 32 => .{ .v_ss, .add },
176465 64 => .{ .v_sd, .add },
176466 else => unreachable,
176467 };
176468 if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
176469 mir_tag,
176470 dst_reg,
176471 dst_reg,
176472 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
176473 ) else try self.asmRegisterRegisterRegister(
176474 mir_tag,
176475 dst_reg,
176476 dst_reg,
176477 (if (src_mcv.isRegister())
176478 src_mcv.getReg().?
176479 else
176480 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
176481 );
176482 } else {
176483 const mir_tag: Mir.Inst.FixedTag = switch (float_bits) {
176484 32 => .{ ._ss, .add },
176485 64 => .{ ._sd, .add },
176486 else => unreachable,
176487 };
176488 if (src_mcv.isBase()) try self.asmRegisterMemory(
176489 mir_tag,
176490 dst_reg,
176491 try src_mcv.mem(self, .{ .size = .fromBitSize(float_bits) }),
176492 ) else try self.asmRegisterRegister(
176493 mir_tag,
176494 dst_reg,
176495 (if (src_mcv.isRegister())
176496 src_mcv.getReg().?
176497 else
176498 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
176499 );
176500 }
176501
176502 if (float_bits == 16) try self.asmRegisterRegisterImmediate(
176503 .{ .v_, .cvtps2ph },
176504 dst_reg,
176505 dst_reg,
176506 bits.RoundMode.imm(.{}),
176507 );
176508 break :adjusted .{ .register = dst_reg };
176509 },
176510 80, 128 => return self.fail("TODO implement genBinOp for {s} of {f}", .{
176511 @tagName(air_tag), lhs_ty.fmt(pt),
176512 }),
176513 else => unreachable,
176514 };
176515 break :result try self.genCall(.{ .extern_func = .{
176516 .return_type = lhs_ty.toIntern(),
176517 .param_types = &.{ lhs_ty.toIntern(), rhs_ty.toIntern() },
176518 .sym = sym,
176519 } }, &.{ lhs_ty, rhs_ty }, &.{ adjusted, .{ .air_ref = rhs_air } }, .{});
176520 },
176521 .div_trunc, .div_floor => try self.genRoundLibcall(lhs_ty, result, .{
176522 .direction = switch (air_tag) {
176523 .div_trunc => .zero,
176524 .div_floor => .down,
176525 else => unreachable,
176526 },
176527 .precision = .inexact,
176528 }),
176529 else => result,
176530 };
176531 }
176532
176533 const sse_op = switch (lhs_ty.zigTypeTag(zcu)) {
176534 else => false,
176535 .float => true,
176536 .vector => switch (lhs_ty.childType(zcu).toIntern()) {
176537 .bool_type, .u1_type => false,
176538 else => true,
176539 },
176540 };
176541 if (sse_op and ((lhs_ty.scalarType(zcu).isRuntimeFloat() and
176542 lhs_ty.scalarType(zcu).floatBits(self.target) == 80) or
176543 lhs_ty.abiSize(zcu) > self.vectorSize(.float)))
176544 return self.fail("TODO implement genBinOp for {s} {f}", .{ @tagName(air_tag), lhs_ty.fmt(pt) });
176545
176546 const maybe_mask_reg = switch (air_tag) {
176547 else => null,
176548 .rem, .mod => unreachable,
176549 .max, .min => if (lhs_ty.scalarType(zcu).isRuntimeFloat()) registerAlias(
176550 if (!self.hasFeature(.avx) and self.hasFeature(.sse4_1)) mask: {
176551 try self.register_manager.getKnownReg(.xmm0, null);
176552 break :mask .xmm0;
176553 } else try self.register_manager.allocReg(null, abi.RegisterClass.sse),
176554 abi_size,
176555 ) else null,
176556 };
176557 const mask_lock =
176558 if (maybe_mask_reg) |mask_reg| self.register_manager.lockRegAssumeUnused(mask_reg) else null;
176559 defer if (mask_lock) |lock| self.register_manager.unlockReg(lock);
176560
176561 const ordered_air: [2]Air.Inst.Ref = if (lhs_ty.isVector(zcu) and
176562 switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
176563 .bool => false,
176564 .int => switch (air_tag) {
176565 .cmp_lt, .cmp_gte => true,
176566 else => false,
176567 },
176568 .float => switch (air_tag) {
176569 .cmp_gte, .cmp_gt => true,
176570 else => false,
176571 },
176572 else => unreachable,
176573 }) .{ rhs_air, lhs_air } else .{ lhs_air, rhs_air };
176574
176575 if (lhs_ty.isAbiInt(zcu)) for (ordered_air) |op_air| {
176576 switch (try self.resolveInst(op_air)) {
176577 .register => |op_reg| switch (op_reg.class()) {
176578 .sse => try self.register_manager.getReg(op_reg, null),
176579 else => {},
176580 },
176581 else => {},
176582 }
176583 };
176584
176585 const lhs_mcv = try self.resolveInst(ordered_air[0]);
176586 var rhs_mcv = try self.resolveInst(ordered_air[1]);
176587 switch (lhs_mcv) {
176588 .immediate => |imm| switch (imm) {
176589 0 => switch (air_tag) {
176590 .sub, .sub_wrap => return self.genUnOp(maybe_inst, .neg, ordered_air[1]),
176591 else => {},
176592 },
176593 else => {},
176594 },
176595 else => {},
176596 }
176597
176598 const is_commutative = switch (air_tag) {
176599 .add,
176600 .add_wrap,
176601 .mul,
176602 .bool_or,
176603 .bit_or,
176604 .bool_and,
176605 .bit_and,
176606 .xor,
176607 .min,
176608 .max,
176609 .cmp_eq,
176610 .cmp_neq,
176611 => true,
176612
176613 else => false,
176614 };
176615
176616 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
176617 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
176618 .register_pair => |lhs_regs| locks: {
176619 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
176620 break :locks .{ locks[0], locks[1] };
176621 },
176622 else => @splat(null),
176623 };
176624 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
176625
176626 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
176627 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
176628 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
176629 else => @splat(null),
176630 };
176631 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
176632
176633 var flipped = false;
176634 var copied_to_dst = true;
176635 const dst_mcv: MCValue = dst: {
176636 const tracked_inst = switch (air_tag) {
176637 else => maybe_inst,
176638 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => null,
176639 };
176640 if (maybe_inst) |inst| {
176641 if ((!sse_op or lhs_mcv.isRegister()) and
176642 self.reuseOperandAdvanced(inst, ordered_air[0], 0, lhs_mcv, tracked_inst))
176643 break :dst lhs_mcv;
176644 if (is_commutative and (!sse_op or rhs_mcv.isRegister()) and
176645 self.reuseOperandAdvanced(inst, ordered_air[1], 1, rhs_mcv, tracked_inst))
176646 {
176647 flipped = true;
176648 break :dst rhs_mcv;
176649 }
176650 }
176651 const dst_mcv = try self.allocRegOrMemAdvanced(lhs_ty, tracked_inst, true);
176652 if (sse_op and lhs_mcv.isRegister() and self.hasFeature(.avx))
176653 copied_to_dst = false
176654 else
176655 try self.genCopy(lhs_ty, dst_mcv, lhs_mcv, .{});
176656 rhs_mcv = try self.resolveInst(ordered_air[1]);
176657 break :dst dst_mcv;
176658 };
176659 const dst_locks: [2]?RegisterLock = switch (dst_mcv) {
176660 .register => |dst_reg| .{ self.register_manager.lockReg(dst_reg), null },
176661 .register_pair => |dst_regs| self.register_manager.lockRegs(2, dst_regs),
176662 else => @splat(null),
176663 };
176664 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
176665
176666 const unmat_src_mcv = if (flipped) lhs_mcv else rhs_mcv;
176667 const src_mcv: MCValue = if (maybe_mask_reg) |mask_reg|
176668 if (self.hasFeature(.avx) and unmat_src_mcv.isRegister() and maybe_inst != null and
176669 self.liveness.operandDies(maybe_inst.?, if (flipped) 0 else 1)) unmat_src_mcv else src: {
176670 try self.genSetReg(mask_reg, rhs_ty, unmat_src_mcv, .{});
176671 break :src .{ .register = mask_reg };
176672 }
176673 else
176674 unmat_src_mcv;
176675 const src_locks: [2]?RegisterLock = switch (src_mcv) {
176676 .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null },
176677 .register_pair => |src_regs| self.register_manager.lockRegs(2, src_regs),
176678 else => @splat(null),
176679 };
176680 defer for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock);
176681
176682 if (!sse_op) {
176683 switch (air_tag) {
176684 .add,
176685 .add_wrap,
176686 => try self.genBinOpMir(.{ ._, .add }, lhs_ty, dst_mcv, src_mcv),
176687
176688 .sub,
176689 .sub_wrap,
176690 => try self.genBinOpMir(.{ ._, .sub }, lhs_ty, dst_mcv, src_mcv),
176691
176692 .ptr_add,
176693 .ptr_sub,
176694 => {
176695 const tmp_reg = try self.copyToTmpRegister(rhs_ty, src_mcv);
176696 const tmp_mcv = MCValue{ .register = tmp_reg };
176697 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
176698 defer self.register_manager.unlockReg(tmp_lock);
176699
176700 const elem_size = lhs_ty.elemType2(zcu).abiSize(zcu);
176701 try self.genIntMulComplexOpMir(rhs_ty, tmp_mcv, .{ .immediate = elem_size });
176702 try self.genBinOpMir(
176703 switch (air_tag) {
176704 .ptr_add => .{ ._, .add },
176705 .ptr_sub => .{ ._, .sub },
176706 else => unreachable,
176707 },
176708 lhs_ty,
176709 dst_mcv,
176710 tmp_mcv,
176711 );
176712 },
176713
176714 .bool_or,
176715 .bit_or,
176716 => try self.genBinOpMir(.{ ._, .@"or" }, lhs_ty, dst_mcv, src_mcv),
176717
176718 .bool_and,
176719 .bit_and,
176720 => try self.genBinOpMir(.{ ._, .@"and" }, lhs_ty, dst_mcv, src_mcv),
176721
176722 .xor => try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv),
176723
176724 .min,
176725 .max,
176726 => {
176727 const resolved_src_mcv = switch (src_mcv) {
176728 else => src_mcv,
176729 .air_ref => |src_ref| try self.resolveInst(src_ref),
176730 };
176731
176732 if (abi_size > 8) {
176733 const dst_regs = switch (dst_mcv) {
176734 .register_pair => |dst_regs| dst_regs,
176735 else => dst: {
176736 const dst_regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
176737 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
176738 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
176739
176740 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{});
176741 break :dst dst_regs;
176742 },
176743 };
176744 const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs);
176745 defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|
176746 self.register_manager.unlockReg(lock);
176747
176748 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
176749 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
176750 defer self.register_manager.unlockReg(tmp_lock);
176751
176752 const signed = lhs_ty.isSignedInt(zcu);
176753 const cc: Condition = switch (air_tag) {
176754 .min => if (signed) .nl else .nb,
176755 .max => if (signed) .nge else .nae,
176756 else => unreachable,
176757 };
176758
176759 try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]);
176760 if (src_mcv.isBase()) {
176761 try self.asmRegisterMemory(
176762 .{ ._, .cmp },
176763 dst_regs[0],
176764 try src_mcv.mem(self, .{ .size = .qword }),
176765 );
176766 try self.asmRegisterMemory(
176767 .{ ._, .sbb },
176768 tmp_reg,
176769 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
176770 );
176771 try self.asmCmovccRegisterMemory(
176772 cc,
176773 dst_regs[0],
176774 try src_mcv.mem(self, .{ .size = .qword }),
176775 );
176776 try self.asmCmovccRegisterMemory(
176777 cc,
176778 dst_regs[1],
176779 try src_mcv.address().offset(8).deref().mem(self, .{ .size = .qword }),
176780 );
176781 } else {
176782 try self.asmRegisterRegister(
176783 .{ ._, .cmp },
176784 dst_regs[0],
176785 src_mcv.register_pair[0],
176786 );
176787 try self.asmRegisterRegister(
176788 .{ ._, .sbb },
176789 tmp_reg,
176790 src_mcv.register_pair[1],
176791 );
176792 try self.asmCmovccRegisterRegister(cc, dst_regs[0], src_mcv.register_pair[0]);
176793 try self.asmCmovccRegisterRegister(cc, dst_regs[1], src_mcv.register_pair[1]);
176794 }
176795 try self.genCopy(lhs_ty, dst_mcv, .{ .register_pair = dst_regs }, .{});
176796 } else {
176797 const mat_src_mcv: MCValue = if (switch (resolved_src_mcv) {
176798 .immediate,
176799 .eflags,
176800 .register_offset,
176801 .lea_frame,
176802 .load_nav,
176803 .lea_nav,
176804 .load_uav,
176805 .lea_uav,
176806 .load_lazy_sym,
176807 .lea_lazy_sym,
176808 .load_extern_func,
176809 .lea_extern_func,
176810 => true,
176811 .memory => |addr| std.math.cast(i32, @as(i64, @bitCast(addr))) == null,
176812 else => false,
176813 .register_pair,
176814 .register_overflow,
176815 => unreachable,
176816 })
176817 .{ .register = try self.copyToTmpRegister(rhs_ty, resolved_src_mcv) }
176818 else
176819 resolved_src_mcv;
176820 const mat_mcv_lock = switch (mat_src_mcv) {
176821 .register => |reg| self.register_manager.lockReg(reg),
176822 else => null,
176823 };
176824 defer if (mat_mcv_lock) |lock| self.register_manager.unlockReg(lock);
176825
176826 try self.genBinOpMir(.{ ._, .cmp }, lhs_ty, dst_mcv, mat_src_mcv);
176827
176828 const int_info = lhs_ty.intInfo(zcu);
176829 const cc: Condition = switch (int_info.signedness) {
176830 .unsigned => switch (air_tag) {
176831 .min => .a,
176832 .max => .b,
176833 else => unreachable,
176834 },
176835 .signed => switch (air_tag) {
176836 .min => .g,
176837 .max => .l,
176838 else => unreachable,
176839 },
176840 };
176841
176842 const cmov_abi_size = @max(@as(u32, @intCast(lhs_ty.abiSize(zcu))), 2);
176843 const tmp_reg = switch (dst_mcv) {
176844 .register => |reg| reg,
176845 else => try self.copyToTmpRegister(lhs_ty, dst_mcv),
176846 };
176847 const tmp_lock = self.register_manager.lockReg(tmp_reg);
176848 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
176849 switch (mat_src_mcv) {
176850 .none,
176851 .unreach,
176852 .dead,
176853 .undef,
176854 .immediate,
176855 .eflags,
176856 .register_pair,
176857 .register_triple,
176858 .register_quadruple,
176859 .register_offset,
176860 .register_overflow,
176861 .register_mask,
176862 .indirect_load_frame,
176863 .lea_frame,
176864 .load_nav,
176865 .lea_nav,
176866 .load_uav,
176867 .lea_uav,
176868 .load_lazy_sym,
176869 .lea_lazy_sym,
176870 .load_extern_func,
176871 .lea_extern_func,
176872 .elementwise_args,
176873 .reserved_frame,
176874 .air_ref,
176875 => unreachable,
176876 .register => |src_reg| try self.asmCmovccRegisterRegister(
176877 cc,
176878 registerAlias(tmp_reg, cmov_abi_size),
176879 registerAlias(src_reg, cmov_abi_size),
176880 ),
176881 .memory, .indirect, .load_frame => try self.asmCmovccRegisterMemory(
176882 cc,
176883 registerAlias(tmp_reg, cmov_abi_size),
176884 switch (mat_src_mcv) {
176885 .memory => |addr| .{
176886 .base = .{ .reg = .ds },
176887 .mod = .{ .rm = .{
176888 .size = .fromSize(cmov_abi_size),
176889 .disp = @intCast(@as(i64, @bitCast(addr))),
176890 } },
176891 },
176892 .indirect => |reg_off| .{
176893 .base = .{ .reg = reg_off.reg },
176894 .mod = .{ .rm = .{
176895 .size = .fromSize(cmov_abi_size),
176896 .disp = reg_off.off,
176897 } },
176898 },
176899 .load_frame => |frame_addr| .{
176900 .base = .{ .frame = frame_addr.index },
176901 .mod = .{ .rm = .{
176902 .size = .fromSize(cmov_abi_size),
176903 .disp = frame_addr.off,
176904 } },
176905 },
176906 else => unreachable,
176907 },
176908 ),
176909 }
176910 try self.genCopy(lhs_ty, dst_mcv, .{ .register = tmp_reg }, .{});
176911 }
176912 },
176913
176914 .cmp_eq, .cmp_neq => {
176915 assert(lhs_ty.isVector(zcu) and lhs_ty.childType(zcu).toIntern() == .bool_type);
176916 try self.genBinOpMir(.{ ._, .xor }, lhs_ty, dst_mcv, src_mcv);
176917 switch (air_tag) {
176918 .cmp_eq => try self.genUnOpMir(.{ ._, .not }, lhs_ty, dst_mcv),
176919 .cmp_neq => {},
176920 else => unreachable,
176921 }
176922 },
176923
176924 else => return self.fail("TODO implement genBinOp for {s} {f}", .{
176925 @tagName(air_tag), lhs_ty.fmt(pt),
176926 }),
176927 }
176928 return dst_mcv;
176929 }
176930
176931 const dst_reg = registerAlias(dst_mcv.getReg().?, abi_size);
176932 const mir_tag = @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
176933 else => unreachable,
176934 .float => switch (lhs_ty.floatBits(self.target)) {
176935 16 => {
176936 assert(self.hasFeature(.f16c));
176937 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
176938
176939 const tmp_reg = (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
176940 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
176941 defer self.register_manager.unlockReg(tmp_lock);
176942
176943 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
176944 .{ .vp_w, .insr },
176945 dst_reg,
176946 lhs_reg,
176947 try src_mcv.mem(self, .{ .size = .word }),
176948 .u(1),
176949 ) else try self.asmRegisterRegisterRegister(
176950 .{ .vp_, .unpcklwd },
176951 dst_reg,
176952 lhs_reg,
176953 (if (src_mcv.isRegister())
176954 src_mcv.getReg().?
176955 else
176956 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
176957 );
176958 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
176959 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
176960 try self.asmRegisterRegisterRegister(
176961 switch (air_tag) {
176962 .add => .{ .v_ss, .add },
176963 .sub => .{ .v_ss, .sub },
176964 .mul => .{ .v_ss, .mul },
176965 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
176966 .max => .{ .v_ss, .max },
176967 .min => .{ .v_ss, .min },
176968 else => unreachable,
176969 },
176970 dst_reg,
176971 dst_reg,
176972 tmp_reg,
176973 );
176974 switch (air_tag) {
176975 .div_trunc, .div_floor => try self.asmRegisterRegisterRegisterImmediate(
176976 .{ .v_ss, .round },
176977 dst_reg,
176978 dst_reg,
176979 dst_reg,
176980 bits.RoundMode.imm(.{
176981 .direction = switch (air_tag) {
176982 .div_trunc => .zero,
176983 .div_floor => .down,
176984 else => unreachable,
176985 },
176986 .precision = .inexact,
176987 }),
176988 ),
176989 else => {},
176990 }
176991 try self.asmRegisterRegisterImmediate(
176992 .{ .v_, .cvtps2ph },
176993 dst_reg,
176994 dst_reg,
176995 bits.RoundMode.imm(.{}),
176996 );
176997 return dst_mcv;
176998 },
176999 32 => switch (air_tag) {
177000 .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
177001 .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
177002 .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
177003 .div_float,
177004 .div_trunc,
177005 .div_floor,
177006 .div_exact,
177007 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
177008 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
177009 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
177010 else => unreachable,
177011 },
177012 64 => switch (air_tag) {
177013 .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
177014 .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
177015 .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
177016 .div_float,
177017 .div_trunc,
177018 .div_floor,
177019 .div_exact,
177020 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
177021 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
177022 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
177023 else => unreachable,
177024 },
177025 80, 128 => null,
177026 else => unreachable,
177027 },
177028 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177029 else => null,
177030 .int => switch (lhs_ty.childType(zcu).intInfo(zcu).bits) {
177031 8 => switch (lhs_ty.vectorLen(zcu)) {
177032 1...16 => switch (air_tag) {
177033 .add,
177034 .add_wrap,
177035 => if (self.hasFeature(.avx)) .{ .vp_b, .add } else .{ .p_b, .add },
177036 .sub,
177037 .sub_wrap,
177038 => if (self.hasFeature(.avx)) .{ .vp_b, .sub } else .{ .p_b, .sub },
177039 .bit_and => if (self.hasFeature(.avx))
177040 .{ .vp_, .@"and" }
177041 else
177042 .{ .p_, .@"and" },
177043 .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
177044 .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
177045 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177046 .signed => if (self.hasFeature(.avx))
177047 .{ .vp_b, .mins }
177048 else if (self.hasFeature(.sse4_1))
177049 .{ .p_b, .mins }
177050 else
177051 null,
177052 .unsigned => if (self.hasFeature(.avx))
177053 .{ .vp_b, .minu }
177054 else if (self.hasFeature(.sse4_1))
177055 .{ .p_b, .minu }
177056 else
177057 null,
177058 },
177059 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177060 .signed => if (self.hasFeature(.avx))
177061 .{ .vp_b, .maxs }
177062 else if (self.hasFeature(.sse4_1))
177063 .{ .p_b, .maxs }
177064 else
177065 null,
177066 .unsigned => if (self.hasFeature(.avx))
177067 .{ .vp_b, .maxu }
177068 else if (self.hasFeature(.sse4_1))
177069 .{ .p_b, .maxu }
177070 else
177071 null,
177072 },
177073 .cmp_lt,
177074 .cmp_lte,
177075 .cmp_gte,
177076 .cmp_gt,
177077 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177078 .signed => if (self.hasFeature(.avx))
177079 .{ .vp_b, .cmpgt }
177080 else
177081 .{ .p_b, .cmpgt },
177082 .unsigned => null,
177083 },
177084 .cmp_eq,
177085 .cmp_neq,
177086 => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else .{ .p_b, .cmpeq },
177087 else => null,
177088 },
177089 17...32 => switch (air_tag) {
177090 .add,
177091 .add_wrap,
177092 => if (self.hasFeature(.avx2)) .{ .vp_b, .add } else null,
177093 .sub,
177094 .sub_wrap,
177095 => if (self.hasFeature(.avx2)) .{ .vp_b, .sub } else null,
177096 .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
177097 .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
177098 .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
177099 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177100 .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .mins } else null,
177101 .unsigned => if (self.hasFeature(.avx)) .{ .vp_b, .minu } else null,
177102 },
177103 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177104 .signed => if (self.hasFeature(.avx2)) .{ .vp_b, .maxs } else null,
177105 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_b, .maxu } else null,
177106 },
177107 .cmp_lt,
177108 .cmp_lte,
177109 .cmp_gte,
177110 .cmp_gt,
177111 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177112 .signed => if (self.hasFeature(.avx)) .{ .vp_b, .cmpgt } else null,
177113 .unsigned => null,
177114 },
177115 .cmp_eq,
177116 .cmp_neq,
177117 => if (self.hasFeature(.avx)) .{ .vp_b, .cmpeq } else null,
177118 else => null,
177119 },
177120 else => null,
177121 },
177122 16 => switch (lhs_ty.vectorLen(zcu)) {
177123 1...8 => switch (air_tag) {
177124 .add,
177125 .add_wrap,
177126 => if (self.hasFeature(.avx)) .{ .vp_w, .add } else .{ .p_w, .add },
177127 .sub,
177128 .sub_wrap,
177129 => if (self.hasFeature(.avx)) .{ .vp_w, .sub } else .{ .p_w, .sub },
177130 .mul,
177131 .mul_wrap,
177132 => if (self.hasFeature(.avx)) .{ .vp_w, .mull } else .{ .p_d, .mull },
177133 .bit_and => if (self.hasFeature(.avx))
177134 .{ .vp_, .@"and" }
177135 else
177136 .{ .p_, .@"and" },
177137 .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
177138 .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
177139 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177140 .signed => if (self.hasFeature(.avx))
177141 .{ .vp_w, .mins }
177142 else
177143 .{ .p_w, .mins },
177144 .unsigned => if (self.hasFeature(.avx))
177145 .{ .vp_w, .minu }
177146 else
177147 .{ .p_w, .minu },
177148 },
177149 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177150 .signed => if (self.hasFeature(.avx))
177151 .{ .vp_w, .maxs }
177152 else
177153 .{ .p_w, .maxs },
177154 .unsigned => if (self.hasFeature(.avx))
177155 .{ .vp_w, .maxu }
177156 else
177157 .{ .p_w, .maxu },
177158 },
177159 .cmp_lt,
177160 .cmp_lte,
177161 .cmp_gte,
177162 .cmp_gt,
177163 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177164 .signed => if (self.hasFeature(.avx))
177165 .{ .vp_w, .cmpgt }
177166 else
177167 .{ .p_w, .cmpgt },
177168 .unsigned => null,
177169 },
177170 .cmp_eq,
177171 .cmp_neq,
177172 => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else .{ .p_w, .cmpeq },
177173 else => null,
177174 },
177175 9...16 => switch (air_tag) {
177176 .add,
177177 .add_wrap,
177178 => if (self.hasFeature(.avx2)) .{ .vp_w, .add } else null,
177179 .sub,
177180 .sub_wrap,
177181 => if (self.hasFeature(.avx2)) .{ .vp_w, .sub } else null,
177182 .mul,
177183 .mul_wrap,
177184 => if (self.hasFeature(.avx2)) .{ .vp_w, .mull } else null,
177185 .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
177186 .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
177187 .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
177188 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177189 .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .mins } else null,
177190 .unsigned => if (self.hasFeature(.avx)) .{ .vp_w, .minu } else null,
177191 },
177192 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177193 .signed => if (self.hasFeature(.avx2)) .{ .vp_w, .maxs } else null,
177194 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_w, .maxu } else null,
177195 },
177196 .cmp_lt,
177197 .cmp_lte,
177198 .cmp_gte,
177199 .cmp_gt,
177200 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177201 .signed => if (self.hasFeature(.avx)) .{ .vp_w, .cmpgt } else null,
177202 .unsigned => null,
177203 },
177204 .cmp_eq,
177205 .cmp_neq,
177206 => if (self.hasFeature(.avx)) .{ .vp_w, .cmpeq } else null,
177207 else => null,
177208 },
177209 else => null,
177210 },
177211 32 => switch (lhs_ty.vectorLen(zcu)) {
177212 1...4 => switch (air_tag) {
177213 .add,
177214 .add_wrap,
177215 => if (self.hasFeature(.avx)) .{ .vp_d, .add } else .{ .p_d, .add },
177216 .sub,
177217 .sub_wrap,
177218 => if (self.hasFeature(.avx)) .{ .vp_d, .sub } else .{ .p_d, .sub },
177219 .mul,
177220 .mul_wrap,
177221 => if (self.hasFeature(.avx))
177222 .{ .vp_d, .mull }
177223 else if (self.hasFeature(.sse4_1))
177224 .{ .p_d, .mull }
177225 else
177226 null,
177227 .bit_and => if (self.hasFeature(.avx))
177228 .{ .vp_, .@"and" }
177229 else
177230 .{ .p_, .@"and" },
177231 .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
177232 .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
177233 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177234 .signed => if (self.hasFeature(.avx))
177235 .{ .vp_d, .mins }
177236 else if (self.hasFeature(.sse4_1))
177237 .{ .p_d, .mins }
177238 else
177239 null,
177240 .unsigned => if (self.hasFeature(.avx))
177241 .{ .vp_d, .minu }
177242 else if (self.hasFeature(.sse4_1))
177243 .{ .p_d, .minu }
177244 else
177245 null,
177246 },
177247 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177248 .signed => if (self.hasFeature(.avx))
177249 .{ .vp_d, .maxs }
177250 else if (self.hasFeature(.sse4_1))
177251 .{ .p_d, .maxs }
177252 else
177253 null,
177254 .unsigned => if (self.hasFeature(.avx))
177255 .{ .vp_d, .maxu }
177256 else if (self.hasFeature(.sse4_1))
177257 .{ .p_d, .maxu }
177258 else
177259 null,
177260 },
177261 .cmp_lt,
177262 .cmp_lte,
177263 .cmp_gte,
177264 .cmp_gt,
177265 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177266 .signed => if (self.hasFeature(.avx))
177267 .{ .vp_d, .cmpgt }
177268 else
177269 .{ .p_d, .cmpgt },
177270 .unsigned => null,
177271 },
177272 .cmp_eq,
177273 .cmp_neq,
177274 => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else .{ .p_d, .cmpeq },
177275 else => null,
177276 },
177277 5...8 => switch (air_tag) {
177278 .add,
177279 .add_wrap,
177280 => if (self.hasFeature(.avx2)) .{ .vp_d, .add } else null,
177281 .sub,
177282 .sub_wrap,
177283 => if (self.hasFeature(.avx2)) .{ .vp_d, .sub } else null,
177284 .mul,
177285 .mul_wrap,
177286 => if (self.hasFeature(.avx2)) .{ .vp_d, .mull } else null,
177287 .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
177288 .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
177289 .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
177290 .min => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177291 .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .mins } else null,
177292 .unsigned => if (self.hasFeature(.avx)) .{ .vp_d, .minu } else null,
177293 },
177294 .max => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177295 .signed => if (self.hasFeature(.avx2)) .{ .vp_d, .maxs } else null,
177296 .unsigned => if (self.hasFeature(.avx2)) .{ .vp_d, .maxu } else null,
177297 },
177298 .cmp_lt,
177299 .cmp_lte,
177300 .cmp_gte,
177301 .cmp_gt,
177302 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177303 .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null,
177304 .unsigned => null,
177305 },
177306 .cmp_eq,
177307 .cmp_neq,
177308 => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null,
177309 else => null,
177310 },
177311 else => null,
177312 },
177313 64 => switch (lhs_ty.vectorLen(zcu)) {
177314 1...2 => switch (air_tag) {
177315 .add,
177316 .add_wrap,
177317 => if (self.hasFeature(.avx)) .{ .vp_q, .add } else .{ .p_q, .add },
177318 .sub,
177319 .sub_wrap,
177320 => if (self.hasFeature(.avx)) .{ .vp_q, .sub } else .{ .p_q, .sub },
177321 .bit_and => if (self.hasFeature(.avx))
177322 .{ .vp_, .@"and" }
177323 else
177324 .{ .p_, .@"and" },
177325 .bit_or => if (self.hasFeature(.avx)) .{ .vp_, .@"or" } else .{ .p_, .@"or" },
177326 .xor => if (self.hasFeature(.avx)) .{ .vp_, .xor } else .{ .p_, .xor },
177327 .cmp_lt,
177328 .cmp_lte,
177329 .cmp_gte,
177330 .cmp_gt,
177331 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177332 .signed => if (self.hasFeature(.avx))
177333 .{ .vp_q, .cmpgt }
177334 else if (self.hasFeature(.sse4_2))
177335 .{ .p_q, .cmpgt }
177336 else
177337 null,
177338 .unsigned => null,
177339 },
177340 .cmp_eq,
177341 .cmp_neq,
177342 => if (self.hasFeature(.avx))
177343 .{ .vp_q, .cmpeq }
177344 else if (self.hasFeature(.sse4_1))
177345 .{ .p_q, .cmpeq }
177346 else
177347 null,
177348 else => null,
177349 },
177350 3...4 => switch (air_tag) {
177351 .add,
177352 .add_wrap,
177353 => if (self.hasFeature(.avx2)) .{ .vp_q, .add } else null,
177354 .sub,
177355 .sub_wrap,
177356 => if (self.hasFeature(.avx2)) .{ .vp_q, .sub } else null,
177357 .bit_and => if (self.hasFeature(.avx2)) .{ .vp_, .@"and" } else null,
177358 .bit_or => if (self.hasFeature(.avx2)) .{ .vp_, .@"or" } else null,
177359 .xor => if (self.hasFeature(.avx2)) .{ .vp_, .xor } else null,
177360 .cmp_eq,
177361 .cmp_neq,
177362 => if (self.hasFeature(.avx)) .{ .vp_d, .cmpeq } else null,
177363 .cmp_lt,
177364 .cmp_lte,
177365 .cmp_gt,
177366 .cmp_gte,
177367 => switch (lhs_ty.childType(zcu).intInfo(zcu).signedness) {
177368 .signed => if (self.hasFeature(.avx)) .{ .vp_d, .cmpgt } else null,
177369 .unsigned => null,
177370 },
177371 else => null,
177372 },
177373 else => null,
177374 },
177375 else => null,
177376 },
177377 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177378 16 => tag: {
177379 assert(self.hasFeature(.f16c));
177380 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
177381 switch (lhs_ty.vectorLen(zcu)) {
177382 1 => {
177383 const tmp_reg =
177384 (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
177385 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
177386 defer self.register_manager.unlockReg(tmp_lock);
177387
177388 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
177389 .{ .vp_w, .insr },
177390 dst_reg,
177391 lhs_reg,
177392 try src_mcv.mem(self, .{ .size = .word }),
177393 .u(1),
177394 ) else try self.asmRegisterRegisterRegister(
177395 .{ .vp_, .unpcklwd },
177396 dst_reg,
177397 lhs_reg,
177398 (if (src_mcv.isRegister())
177399 src_mcv.getReg().?
177400 else
177401 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
177402 );
177403 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
177404 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp_reg, dst_reg);
177405 try self.asmRegisterRegisterRegister(
177406 switch (air_tag) {
177407 .add => .{ .v_ss, .add },
177408 .sub => .{ .v_ss, .sub },
177409 .mul => .{ .v_ss, .mul },
177410 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ss, .div },
177411 .max => .{ .v_ss, .max },
177412 .min => .{ .v_ss, .max },
177413 else => unreachable,
177414 },
177415 dst_reg,
177416 dst_reg,
177417 tmp_reg,
177418 );
177419 try self.asmRegisterRegisterImmediate(
177420 .{ .v_, .cvtps2ph },
177421 dst_reg,
177422 dst_reg,
177423 bits.RoundMode.imm(.{}),
177424 );
177425 return dst_mcv;
177426 },
177427 2 => {
177428 const tmp_reg = (try self.register_manager.allocReg(
177429 null,
177430 abi.RegisterClass.sse,
177431 )).to128();
177432 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
177433 defer self.register_manager.unlockReg(tmp_lock);
177434
177435 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
177436 .{ .vp_d, .insr },
177437 dst_reg,
177438 lhs_reg,
177439 try src_mcv.mem(self, .{ .size = .dword }),
177440 .u(1),
177441 ) else try self.asmRegisterRegisterRegister(
177442 .{ .v_ps, .unpckl },
177443 dst_reg,
177444 lhs_reg,
177445 (if (src_mcv.isRegister())
177446 src_mcv.getReg().?
177447 else
177448 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
177449 );
177450 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, dst_reg);
177451 try self.asmRegisterRegisterRegister(
177452 .{ .v_ps, .movhl },
177453 tmp_reg,
177454 dst_reg,
177455 dst_reg,
177456 );
177457 try self.asmRegisterRegisterRegister(
177458 switch (air_tag) {
177459 .add => .{ .v_ps, .add },
177460 .sub => .{ .v_ps, .sub },
177461 .mul => .{ .v_ps, .mul },
177462 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
177463 .max => .{ .v_ps, .max },
177464 .min => .{ .v_ps, .max },
177465 else => unreachable,
177466 },
177467 dst_reg,
177468 dst_reg,
177469 tmp_reg,
177470 );
177471 try self.asmRegisterRegisterImmediate(
177472 .{ .v_, .cvtps2ph },
177473 dst_reg,
177474 dst_reg,
177475 bits.RoundMode.imm(.{}),
177476 );
177477 return dst_mcv;
177478 },
177479 3...4 => {
177480 const tmp_reg = (try self.register_manager.allocReg(
177481 null,
177482 abi.RegisterClass.sse,
177483 )).to128();
177484 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
177485 defer self.register_manager.unlockReg(tmp_lock);
177486
177487 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg, lhs_reg);
177488 if (src_mcv.isBase()) try self.asmRegisterMemory(
177489 .{ .v_ps, .cvtph2 },
177490 tmp_reg,
177491 try src_mcv.mem(self, .{ .size = .qword }),
177492 ) else try self.asmRegisterRegister(
177493 .{ .v_ps, .cvtph2 },
177494 tmp_reg,
177495 (if (src_mcv.isRegister())
177496 src_mcv.getReg().?
177497 else
177498 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
177499 );
177500 try self.asmRegisterRegisterRegister(
177501 switch (air_tag) {
177502 .add => .{ .v_ps, .add },
177503 .sub => .{ .v_ps, .sub },
177504 .mul => .{ .v_ps, .mul },
177505 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
177506 .max => .{ .v_ps, .max },
177507 .min => .{ .v_ps, .max },
177508 else => unreachable,
177509 },
177510 dst_reg,
177511 dst_reg,
177512 tmp_reg,
177513 );
177514 try self.asmRegisterRegisterImmediate(
177515 .{ .v_, .cvtps2ph },
177516 dst_reg,
177517 dst_reg,
177518 bits.RoundMode.imm(.{}),
177519 );
177520 return dst_mcv;
177521 },
177522 5...8 => {
177523 const tmp_reg = (try self.register_manager.allocReg(
177524 null,
177525 abi.RegisterClass.sse,
177526 )).to256();
177527 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
177528 defer self.register_manager.unlockReg(tmp_lock);
177529
177530 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, dst_reg.to256(), lhs_reg);
177531 if (src_mcv.isBase()) try self.asmRegisterMemory(
177532 .{ .v_ps, .cvtph2 },
177533 tmp_reg,
177534 try src_mcv.mem(self, .{ .size = .xword }),
177535 ) else try self.asmRegisterRegister(
177536 .{ .v_ps, .cvtph2 },
177537 tmp_reg,
177538 (if (src_mcv.isRegister())
177539 src_mcv.getReg().?
177540 else
177541 try self.copyToTmpRegister(rhs_ty, src_mcv)).to128(),
177542 );
177543 try self.asmRegisterRegisterRegister(
177544 switch (air_tag) {
177545 .add => .{ .v_ps, .add },
177546 .sub => .{ .v_ps, .sub },
177547 .mul => .{ .v_ps, .mul },
177548 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
177549 .max => .{ .v_ps, .max },
177550 .min => .{ .v_ps, .max },
177551 else => unreachable,
177552 },
177553 dst_reg.to256(),
177554 dst_reg.to256(),
177555 tmp_reg,
177556 );
177557 try self.asmRegisterRegisterImmediate(
177558 .{ .v_, .cvtps2ph },
177559 dst_reg,
177560 dst_reg.to256(),
177561 bits.RoundMode.imm(.{}),
177562 );
177563 return dst_mcv;
177564 },
177565 else => break :tag null,
177566 }
177567 },
177568 32 => switch (lhs_ty.vectorLen(zcu)) {
177569 1 => switch (air_tag) {
177570 .add => if (self.hasFeature(.avx)) .{ .v_ss, .add } else .{ ._ss, .add },
177571 .sub => if (self.hasFeature(.avx)) .{ .v_ss, .sub } else .{ ._ss, .sub },
177572 .mul => if (self.hasFeature(.avx)) .{ .v_ss, .mul } else .{ ._ss, .mul },
177573 .div_float,
177574 .div_trunc,
177575 .div_floor,
177576 .div_exact,
177577 => if (self.hasFeature(.avx)) .{ .v_ss, .div } else .{ ._ss, .div },
177578 .max => if (self.hasFeature(.avx)) .{ .v_ss, .max } else .{ ._ss, .max },
177579 .min => if (self.hasFeature(.avx)) .{ .v_ss, .min } else .{ ._ss, .min },
177580 .cmp_lt,
177581 .cmp_lte,
177582 .cmp_eq,
177583 .cmp_gte,
177584 .cmp_gt,
177585 .cmp_neq,
177586 => if (self.hasFeature(.avx)) .{ .v_ss, .cmp } else .{ ._ss, .cmp },
177587 else => unreachable,
177588 },
177589 2...4 => switch (air_tag) {
177590 .add => if (self.hasFeature(.avx)) .{ .v_ps, .add } else .{ ._ps, .add },
177591 .sub => if (self.hasFeature(.avx)) .{ .v_ps, .sub } else .{ ._ps, .sub },
177592 .mul => if (self.hasFeature(.avx)) .{ .v_ps, .mul } else .{ ._ps, .mul },
177593 .div_float,
177594 .div_trunc,
177595 .div_floor,
177596 .div_exact,
177597 => if (self.hasFeature(.avx)) .{ .v_ps, .div } else .{ ._ps, .div },
177598 .max => if (self.hasFeature(.avx)) .{ .v_ps, .max } else .{ ._ps, .max },
177599 .min => if (self.hasFeature(.avx)) .{ .v_ps, .min } else .{ ._ps, .min },
177600 .cmp_lt,
177601 .cmp_lte,
177602 .cmp_eq,
177603 .cmp_gte,
177604 .cmp_gt,
177605 .cmp_neq,
177606 => if (self.hasFeature(.avx)) .{ .v_ps, .cmp } else .{ ._ps, .cmp },
177607 else => unreachable,
177608 },
177609 5...8 => if (self.hasFeature(.avx)) switch (air_tag) {
177610 .add => .{ .v_ps, .add },
177611 .sub => .{ .v_ps, .sub },
177612 .mul => .{ .v_ps, .mul },
177613 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_ps, .div },
177614 .max => .{ .v_ps, .max },
177615 .min => .{ .v_ps, .min },
177616 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_ps, .cmp },
177617 else => unreachable,
177618 } else null,
177619 else => null,
177620 },
177621 64 => switch (lhs_ty.vectorLen(zcu)) {
177622 1 => switch (air_tag) {
177623 .add => if (self.hasFeature(.avx)) .{ .v_sd, .add } else .{ ._sd, .add },
177624 .sub => if (self.hasFeature(.avx)) .{ .v_sd, .sub } else .{ ._sd, .sub },
177625 .mul => if (self.hasFeature(.avx)) .{ .v_sd, .mul } else .{ ._sd, .mul },
177626 .div_float,
177627 .div_trunc,
177628 .div_floor,
177629 .div_exact,
177630 => if (self.hasFeature(.avx)) .{ .v_sd, .div } else .{ ._sd, .div },
177631 .max => if (self.hasFeature(.avx)) .{ .v_sd, .max } else .{ ._sd, .max },
177632 .min => if (self.hasFeature(.avx)) .{ .v_sd, .min } else .{ ._sd, .min },
177633 .cmp_lt,
177634 .cmp_lte,
177635 .cmp_eq,
177636 .cmp_gte,
177637 .cmp_gt,
177638 .cmp_neq,
177639 => if (self.hasFeature(.avx)) .{ .v_sd, .cmp } else .{ ._sd, .cmp },
177640 else => unreachable,
177641 },
177642 2 => switch (air_tag) {
177643 .add => if (self.hasFeature(.avx)) .{ .v_pd, .add } else .{ ._pd, .add },
177644 .sub => if (self.hasFeature(.avx)) .{ .v_pd, .sub } else .{ ._pd, .sub },
177645 .mul => if (self.hasFeature(.avx)) .{ .v_pd, .mul } else .{ ._pd, .mul },
177646 .div_float,
177647 .div_trunc,
177648 .div_floor,
177649 .div_exact,
177650 => if (self.hasFeature(.avx)) .{ .v_pd, .div } else .{ ._pd, .div },
177651 .max => if (self.hasFeature(.avx)) .{ .v_pd, .max } else .{ ._pd, .max },
177652 .min => if (self.hasFeature(.avx)) .{ .v_pd, .min } else .{ ._pd, .min },
177653 .cmp_lt,
177654 .cmp_lte,
177655 .cmp_eq,
177656 .cmp_gte,
177657 .cmp_gt,
177658 .cmp_neq,
177659 => if (self.hasFeature(.avx)) .{ .v_pd, .cmp } else .{ ._pd, .cmp },
177660 else => unreachable,
177661 },
177662 3...4 => if (self.hasFeature(.avx)) switch (air_tag) {
177663 .add => .{ .v_pd, .add },
177664 .sub => .{ .v_pd, .sub },
177665 .mul => .{ .v_pd, .mul },
177666 .div_float, .div_trunc, .div_floor, .div_exact => .{ .v_pd, .div },
177667 .max => .{ .v_pd, .max },
177668 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => .{ .v_pd, .cmp },
177669 .min => .{ .v_pd, .min },
177670 else => unreachable,
177671 } else null,
177672 else => null,
177673 },
177674 80, 128 => null,
177675 else => unreachable,
177676 },
177677 },
177678 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177679 @tagName(air_tag), lhs_ty.fmt(pt),
177680 });
177681
177682 const lhs_copy_reg = if (maybe_mask_reg) |_| registerAlias(
177683 if (copied_to_dst) try self.copyToTmpRegister(lhs_ty, dst_mcv) else lhs_mcv.getReg().?,
177684 abi_size,
177685 ) else null;
177686 const lhs_copy_lock = if (lhs_copy_reg) |reg| self.register_manager.lockReg(reg) else null;
177687 defer if (lhs_copy_lock) |lock| self.register_manager.unlockReg(lock);
177688
177689 switch (mir_tag[1]) {
177690 else => if (self.hasFeature(.avx)) {
177691 const lhs_reg = if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
177692 if (src_mcv.isBase()) try self.asmRegisterRegisterMemory(
177693 mir_tag,
177694 dst_reg,
177695 lhs_reg,
177696 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
177697 else => .fromSize(abi_size),
177698 .vector => dst_reg.size(),
177699 } }),
177700 ) else try self.asmRegisterRegisterRegister(
177701 mir_tag,
177702 dst_reg,
177703 lhs_reg,
177704 registerAlias(if (src_mcv.isRegister())
177705 src_mcv.getReg().?
177706 else
177707 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
177708 );
177709 } else {
177710 assert(copied_to_dst);
177711 if (src_mcv.isBase()) try self.asmRegisterMemory(
177712 mir_tag,
177713 dst_reg,
177714 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
177715 else => .fromSize(abi_size),
177716 .vector => dst_reg.size(),
177717 } }),
177718 ) else try self.asmRegisterRegister(
177719 mir_tag,
177720 dst_reg,
177721 registerAlias(if (src_mcv.isRegister())
177722 src_mcv.getReg().?
177723 else
177724 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
177725 );
177726 },
177727 .cmp => {
177728 const imm: Immediate = .u(switch (air_tag) {
177729 .cmp_eq => 0,
177730 .cmp_lt, .cmp_gt => 1,
177731 .cmp_lte, .cmp_gte => 2,
177732 .cmp_neq => 4,
177733 else => unreachable,
177734 });
177735 if (self.hasFeature(.avx)) {
177736 const lhs_reg =
177737 if (copied_to_dst) dst_reg else registerAlias(lhs_mcv.getReg().?, abi_size);
177738 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
177739 mir_tag,
177740 dst_reg,
177741 lhs_reg,
177742 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
177743 else => .fromSize(abi_size),
177744 .vector => dst_reg.size(),
177745 } }),
177746 imm,
177747 ) else try self.asmRegisterRegisterRegisterImmediate(
177748 mir_tag,
177749 dst_reg,
177750 lhs_reg,
177751 registerAlias(if (src_mcv.isRegister())
177752 src_mcv.getReg().?
177753 else
177754 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
177755 imm,
177756 );
177757 } else {
177758 assert(copied_to_dst);
177759 if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate(
177760 mir_tag,
177761 dst_reg,
177762 try src_mcv.mem(self, .{ .size = switch (lhs_ty.zigTypeTag(zcu)) {
177763 else => .fromSize(abi_size),
177764 .vector => dst_reg.size(),
177765 } }),
177766 imm,
177767 ) else try self.asmRegisterRegisterImmediate(
177768 mir_tag,
177769 dst_reg,
177770 registerAlias(if (src_mcv.isRegister())
177771 src_mcv.getReg().?
177772 else
177773 try self.copyToTmpRegister(rhs_ty, src_mcv), abi_size),
177774 imm,
177775 );
177776 }
177777 },
177778 }
177779
177780 switch (air_tag) {
177781 .bit_and, .bit_or, .xor => {},
177782 .max, .min => if (maybe_mask_reg) |mask_reg| if (self.hasFeature(.avx)) {
177783 const rhs_copy_reg = registerAlias(src_mcv.getReg().?, abi_size);
177784
177785 try self.asmRegisterRegisterRegisterImmediate(
177786 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
177787 .float => switch (lhs_ty.floatBits(self.target)) {
177788 32 => .{ .v_ss, .cmp },
177789 64 => .{ .v_sd, .cmp },
177790 16, 80, 128 => null,
177791 else => unreachable,
177792 },
177793 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177794 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177795 32 => switch (lhs_ty.vectorLen(zcu)) {
177796 1 => .{ .v_ss, .cmp },
177797 2...8 => .{ .v_ps, .cmp },
177798 else => null,
177799 },
177800 64 => switch (lhs_ty.vectorLen(zcu)) {
177801 1 => .{ .v_sd, .cmp },
177802 2...4 => .{ .v_pd, .cmp },
177803 else => null,
177804 },
177805 16, 80, 128 => null,
177806 else => unreachable,
177807 },
177808 else => unreachable,
177809 },
177810 else => unreachable,
177811 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177812 @tagName(air_tag), lhs_ty.fmt(pt),
177813 }),
177814 mask_reg,
177815 rhs_copy_reg,
177816 rhs_copy_reg,
177817 bits.VexFloatPredicate.imm(.unord),
177818 );
177819 try self.asmRegisterRegisterRegisterRegister(
177820 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
177821 .float => switch (lhs_ty.floatBits(self.target)) {
177822 32 => .{ .v_ps, .blendv },
177823 64 => .{ .v_pd, .blendv },
177824 16, 80, 128 => null,
177825 else => unreachable,
177826 },
177827 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177828 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177829 32 => switch (lhs_ty.vectorLen(zcu)) {
177830 1...8 => .{ .v_ps, .blendv },
177831 else => null,
177832 },
177833 64 => switch (lhs_ty.vectorLen(zcu)) {
177834 1...4 => .{ .v_pd, .blendv },
177835 else => null,
177836 },
177837 16, 80, 128 => null,
177838 else => unreachable,
177839 },
177840 else => unreachable,
177841 },
177842 else => unreachable,
177843 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177844 @tagName(air_tag), lhs_ty.fmt(pt),
177845 }),
177846 dst_reg,
177847 dst_reg,
177848 lhs_copy_reg.?,
177849 mask_reg,
177850 );
177851 } else {
177852 const has_blend = self.hasFeature(.sse4_1);
177853 try self.asmRegisterRegisterImmediate(
177854 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
177855 .float => switch (lhs_ty.floatBits(self.target)) {
177856 32 => .{ ._ss, .cmp },
177857 64 => .{ ._sd, .cmp },
177858 16, 80, 128 => null,
177859 else => unreachable,
177860 },
177861 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177862 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177863 32 => switch (lhs_ty.vectorLen(zcu)) {
177864 1 => .{ ._ss, .cmp },
177865 2...4 => .{ ._ps, .cmp },
177866 else => null,
177867 },
177868 64 => switch (lhs_ty.vectorLen(zcu)) {
177869 1 => .{ ._sd, .cmp },
177870 2 => .{ ._pd, .cmp },
177871 else => null,
177872 },
177873 16, 80, 128 => null,
177874 else => unreachable,
177875 },
177876 else => unreachable,
177877 },
177878 else => unreachable,
177879 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177880 @tagName(air_tag), lhs_ty.fmt(pt),
177881 }),
177882 mask_reg,
177883 mask_reg,
177884 bits.SseFloatPredicate.imm(if (has_blend) .unord else .ord),
177885 );
177886 if (has_blend) try self.asmRegisterRegisterRegister(
177887 @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) {
177888 .float => switch (lhs_ty.floatBits(self.target)) {
177889 32 => .{ ._ps, .blendv },
177890 64 => .{ ._pd, .blendv },
177891 16, 80, 128 => null,
177892 else => unreachable,
177893 },
177894 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177895 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177896 32 => switch (lhs_ty.vectorLen(zcu)) {
177897 1...4 => .{ ._ps, .blendv },
177898 else => null,
177899 },
177900 64 => switch (lhs_ty.vectorLen(zcu)) {
177901 1...2 => .{ ._pd, .blendv },
177902 else => null,
177903 },
177904 16, 80, 128 => null,
177905 else => unreachable,
177906 },
177907 else => unreachable,
177908 },
177909 else => unreachable,
177910 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177911 @tagName(air_tag), lhs_ty.fmt(pt),
177912 }),
177913 dst_reg,
177914 lhs_copy_reg.?,
177915 mask_reg,
177916 ) else {
177917 const mir_fixes = @as(?Mir.Inst.Fixes, switch (lhs_ty.zigTypeTag(zcu)) {
177918 .float => switch (lhs_ty.floatBits(self.target)) {
177919 32 => ._ps,
177920 64 => ._pd,
177921 16, 80, 128 => null,
177922 else => unreachable,
177923 },
177924 .vector => switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177925 .float => switch (lhs_ty.childType(zcu).floatBits(self.target)) {
177926 32 => switch (lhs_ty.vectorLen(zcu)) {
177927 1...4 => ._ps,
177928 else => null,
177929 },
177930 64 => switch (lhs_ty.vectorLen(zcu)) {
177931 1...2 => ._pd,
177932 else => null,
177933 },
177934 16, 80, 128 => null,
177935 else => unreachable,
177936 },
177937 else => unreachable,
177938 },
177939 else => unreachable,
177940 }) orelse return self.fail("TODO implement genBinOp for {s} {f}", .{
177941 @tagName(air_tag), lhs_ty.fmt(pt),
177942 });
177943 try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_reg, mask_reg);
177944 try self.asmRegisterRegister(.{ mir_fixes, .andn }, mask_reg, lhs_copy_reg.?);
177945 try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_reg, mask_reg);
177946 }
177947 },
177948 .cmp_lt, .cmp_lte, .cmp_eq, .cmp_gte, .cmp_gt, .cmp_neq => {
177949 switch (lhs_ty.childType(zcu).zigTypeTag(zcu)) {
177950 .int => switch (air_tag) {
177951 .cmp_lt,
177952 .cmp_eq,
177953 .cmp_gt,
177954 => {},
177955 .cmp_lte,
177956 .cmp_gte,
177957 .cmp_neq,
177958 => {
177959 const unsigned_ty = try lhs_ty.toUnsigned(pt);
177960 const not_mcv = try self.lowerValue(try unsigned_ty.maxInt(pt, unsigned_ty));
177961 const not_mem: Memory = if (not_mcv.isBase())
177962 try not_mcv.mem(self, .{ .size = .fromSize(abi_size) })
177963 else
177964 .{ .base = .{
177965 .reg = try self.copyToTmpRegister(.usize, not_mcv.address()),
177966 }, .mod = .{ .rm = .{ .size = .fromSize(abi_size) } } };
177967 switch (mir_tag[0]) {
177968 .vp_b, .vp_d, .vp_q, .vp_w => try self.asmRegisterRegisterMemory(
177969 .{ .vp_, .xor },
177970 dst_reg,
177971 dst_reg,
177972 not_mem,
177973 ),
177974 .p_b, .p_d, .p_q, .p_w => try self.asmRegisterMemory(
177975 .{ .p_, .xor },
177976 dst_reg,
177977 not_mem,
177978 ),
177979 else => unreachable,
177980 }
177981 },
177982 else => unreachable,
177983 },
177984 .float => {},
177985 else => unreachable,
177986 }
177987
177988 const gp_reg = try self.register_manager.allocReg(maybe_inst, abi.RegisterClass.gp);
177989 const gp_lock = self.register_manager.lockRegAssumeUnused(gp_reg);
177990 defer self.register_manager.unlockReg(gp_lock);
177991
177992 try self.asmRegisterRegister(switch (mir_tag[0]) {
177993 ._pd, ._sd, .p_q => .{ ._pd, .movmsk },
177994 ._ps, ._ss, .p_d => .{ ._ps, .movmsk },
177995 .p_b => .{ .p_b, .movmsk },
177996 .p_w => movmsk: {
177997 try self.asmRegisterRegister(.{ .p_b, .ackssw }, dst_reg, dst_reg);
177998 break :movmsk .{ .p_b, .movmsk };
177999 },
178000 .v_pd, .v_sd, .vp_q => .{ .v_pd, .movmsk },
178001 .v_ps, .v_ss, .vp_d => .{ .v_ps, .movmsk },
178002 .vp_b => .{ .vp_b, .movmsk },
178003 .vp_w => movmsk: {
178004 try self.asmRegisterRegisterRegister(
178005 .{ .vp_b, .ackssw },
178006 dst_reg,
178007 dst_reg,
178008 dst_reg,
178009 );
178010 break :movmsk .{ .vp_b, .movmsk };
178011 },
178012 else => unreachable,
178013 }, gp_reg.to32(), dst_reg);
178014 return .{ .register = gp_reg };
178015 },
178016 else => unreachable,
178017 }
178018
178019 return dst_mcv;
178020}
178021
178022175211fn genBinOpMir(
178023175212 self: *CodeGen,
178024175213 mir_tag: Mir.Inst.FixedTag,
......@@ -178472,168 +175661,6 @@ fn genBinOpMir(
178472175661 }
178473175662}
178474175663
178475/// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv.
178476/// Does not support byte-size operands.
178477fn genIntMulComplexOpMir(self: *CodeGen, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void {
178478 const pt = self.pt;
178479 const abi_size: u32 = @intCast(dst_ty.abiSize(pt.zcu));
178480 try self.spillEflagsIfOccupied();
178481 switch (dst_mcv) {
178482 .none,
178483 .unreach,
178484 .dead,
178485 .undef,
178486 .immediate,
178487 .eflags,
178488 .register_offset,
178489 .register_overflow,
178490 .register_mask,
178491 .indirect_load_frame,
178492 .lea_frame,
178493 .lea_nav,
178494 .lea_uav,
178495 .lea_lazy_sym,
178496 .lea_extern_func,
178497 .elementwise_args,
178498 .reserved_frame,
178499 .air_ref,
178500 => unreachable, // unmodifiable destination
178501 .register => |dst_reg| {
178502 const alias_size = switch (abi_size) {
178503 1 => 4,
178504 else => abi_size,
178505 };
178506 const dst_alias = registerAlias(dst_reg, alias_size);
178507 const dst_lock = self.register_manager.lockReg(dst_reg);
178508 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
178509
178510 switch (abi_size) {
178511 1 => try self.asmRegisterRegister(.{ ._, .movzx }, dst_reg.to32(), dst_reg.to8()),
178512 else => {},
178513 }
178514
178515 const resolved_src_mcv = switch (src_mcv) {
178516 else => src_mcv,
178517 .air_ref => |src_ref| try self.resolveInst(src_ref),
178518 };
178519 switch (resolved_src_mcv) {
178520 .none,
178521 .unreach,
178522 .dead,
178523 .undef,
178524 .register_pair,
178525 .register_triple,
178526 .register_quadruple,
178527 .register_overflow,
178528 .register_mask,
178529 .indirect_load_frame,
178530 .elementwise_args,
178531 .reserved_frame,
178532 .air_ref,
178533 => unreachable,
178534 .register => |src_reg| {
178535 switch (abi_size) {
178536 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
178537 else => {},
178538 }
178539 try self.asmRegisterRegister(
178540 .{ .i_, .mul },
178541 dst_alias,
178542 registerAlias(src_reg, alias_size),
178543 );
178544 },
178545 .immediate => |imm| {
178546 if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| {
178547 try self.asmRegisterRegisterImmediate(.{ .i_, .mul }, dst_alias, dst_alias, .s(small));
178548 } else {
178549 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
178550 return self.genIntMulComplexOpMir(dst_ty, dst_mcv, MCValue{ .register = src_reg });
178551 }
178552 },
178553 .register_offset,
178554 .eflags,
178555 .lea_frame,
178556 .load_nav,
178557 .lea_nav,
178558 .load_uav,
178559 .lea_uav,
178560 .load_lazy_sym,
178561 .lea_lazy_sym,
178562 .load_extern_func,
178563 .lea_extern_func,
178564 => {
178565 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
178566 switch (abi_size) {
178567 1 => try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8()),
178568 else => {},
178569 }
178570 try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
178571 },
178572 .memory, .indirect, .load_frame => switch (abi_size) {
178573 1 => {
178574 const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv);
178575 try self.asmRegisterRegister(.{ ._, .movzx }, src_reg.to32(), src_reg.to8());
178576 try self.asmRegisterRegister(.{ .i_, .mul }, dst_alias, registerAlias(src_reg, alias_size));
178577 },
178578 else => try self.asmRegisterMemory(
178579 .{ .i_, .mul },
178580 dst_alias,
178581 switch (resolved_src_mcv) {
178582 .memory => |addr| .{
178583 .base = .{ .reg = .ds },
178584 .mod = .{ .rm = .{
178585 .size = .fromSize(abi_size),
178586 .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse
178587 return self.asmRegisterRegister(
178588 .{ .i_, .mul },
178589 dst_alias,
178590 registerAlias(
178591 try self.copyToTmpRegister(dst_ty, resolved_src_mcv),
178592 abi_size,
178593 ),
178594 ),
178595 } },
178596 },
178597 .indirect => |reg_off| .{
178598 .base = .{ .reg = reg_off.reg },
178599 .mod = .{ .rm = .{
178600 .size = .fromSize(abi_size),
178601 .disp = reg_off.off,
178602 } },
178603 },
178604 .load_frame => |frame_addr| .{
178605 .base = .{ .frame = frame_addr.index },
178606 .mod = .{ .rm = .{
178607 .size = .fromSize(abi_size),
178608 .disp = frame_addr.off,
178609 } },
178610 },
178611 else => unreachable,
178612 },
178613 ),
178614 },
178615 }
178616 },
178617 .register_pair, .register_triple, .register_quadruple => unreachable, // unimplemented
178618 .memory,
178619 .indirect,
178620 .load_frame,
178621 .load_nav,
178622 .load_uav,
178623 .load_lazy_sym,
178624 .load_extern_func,
178625 => {
178626 const tmp_reg = try self.copyToTmpRegister(dst_ty, dst_mcv);
178627 const tmp_mcv = MCValue{ .register = tmp_reg };
178628 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
178629 defer self.register_manager.unlockReg(tmp_lock);
178630
178631 try self.genIntMulComplexOpMir(dst_ty, tmp_mcv, src_mcv);
178632 try self.genCopy(dst_ty, dst_mcv, tmp_mcv, .{});
178633 },
178634 }
178635}
178636
178637175664fn airArg(self: *CodeGen, inst: Air.Inst.Index) !void {
178638175665 const zcu = self.pt.zcu;
178639175666 const arg_index = for (self.args, 0..) |arg, arg_index| {
......@@ -179247,475 +176274,6 @@ fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {
179247176274 try self.epilogue_relocs.append(self.gpa, jmp_reloc);
179248176275}
179249176276
179250fn airCmp(self: *CodeGen, inst: Air.Inst.Index, op: std.math.CompareOperator) !void {
179251 const pt = self.pt;
179252 const zcu = pt.zcu;
179253 const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op;
179254 var ty = self.typeOf(bin_op.lhs);
179255 var null_compare: ?Mir.Inst.Index = null;
179256
179257 const result: Condition = result: {
179258 try self.spillEflagsIfOccupied();
179259
179260 const lhs_mcv = try self.resolveInst(bin_op.lhs);
179261 const lhs_locks: [2]?RegisterLock = switch (lhs_mcv) {
179262 .register => |lhs_reg| .{ self.register_manager.lockRegAssumeUnused(lhs_reg), null },
179263 .register_pair => |lhs_regs| locks: {
179264 const locks = self.register_manager.lockRegsAssumeUnused(2, lhs_regs);
179265 break :locks .{ locks[0], locks[1] };
179266 },
179267 .register_offset => |lhs_ro| .{
179268 self.register_manager.lockRegAssumeUnused(lhs_ro.reg),
179269 null,
179270 },
179271 else => @splat(null),
179272 };
179273 defer for (lhs_locks) |lhs_lock| if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
179274
179275 const rhs_mcv = try self.resolveInst(bin_op.rhs);
179276 const rhs_locks: [2]?RegisterLock = switch (rhs_mcv) {
179277 .register => |rhs_reg| .{ self.register_manager.lockReg(rhs_reg), null },
179278 .register_pair => |rhs_regs| self.register_manager.lockRegs(2, rhs_regs),
179279 .register_offset => |rhs_ro| .{ self.register_manager.lockReg(rhs_ro.reg), null },
179280 else => @splat(null),
179281 };
179282 defer for (rhs_locks) |rhs_lock| if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
179283
179284 switch (ty.zigTypeTag(zcu)) {
179285 .float => {
179286 const float_bits = ty.floatBits(self.target);
179287 if (!switch (float_bits) {
179288 16 => self.hasFeature(.f16c),
179289 32 => self.hasFeature(.sse),
179290 64 => self.hasFeature(.sse2),
179291 80, 128 => false,
179292 else => unreachable,
179293 }) {
179294 var sym_buf: ["__???f2".len]u8 = undefined;
179295 const ret = try self.genCall(.{ .extern_func = .{
179296 .return_type = .i32_type,
179297 .param_types = &.{ ty.toIntern(), ty.toIntern() },
179298 .sym = std.fmt.bufPrint(&sym_buf, "__{s}{c}f2", .{
179299 switch (op) {
179300 .eq => "eq",
179301 .neq => "ne",
179302 .lt => "lt",
179303 .lte => "le",
179304 .gt => "gt",
179305 .gte => "ge",
179306 },
179307 floatCompilerRtAbiName(float_bits),
179308 }) catch unreachable,
179309 } }, &.{ ty, ty }, &.{ .{ .air_ref = bin_op.lhs }, .{ .air_ref = bin_op.rhs } }, .{});
179310 try self.genBinOpMir(.{ ._, .@"test" }, .i32, ret, ret);
179311 break :result switch (op) {
179312 .eq => .e,
179313 .neq => .ne,
179314 .lt => .l,
179315 .lte => .le,
179316 .gt => .g,
179317 .gte => .ge,
179318 };
179319 }
179320 },
179321 .optional => if (!ty.optionalReprIsPayload(zcu)) {
179322 const opt_ty = ty;
179323 const opt_abi_size: u31 = @intCast(opt_ty.abiSize(zcu));
179324 ty = opt_ty.optionalChild(zcu);
179325 const payload_abi_size: u31 = @intCast(ty.abiSize(zcu));
179326
179327 const temp_lhs_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
179328 const temp_lhs_lock = self.register_manager.lockRegAssumeUnused(temp_lhs_reg);
179329 defer self.register_manager.unlockReg(temp_lhs_lock);
179330
179331 if (lhs_mcv.isBase()) try self.asmRegisterMemory(
179332 .{ ._, .mov },
179333 temp_lhs_reg.to8(),
179334 try lhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte }),
179335 ) else {
179336 try self.genSetReg(temp_lhs_reg, opt_ty, lhs_mcv, .{});
179337 try self.asmRegisterImmediate(
179338 .{ ._r, .sh },
179339 registerAlias(temp_lhs_reg, opt_abi_size),
179340 .u(payload_abi_size * 8),
179341 );
179342 }
179343
179344 const payload_compare = payload_compare: {
179345 if (rhs_mcv.isBase()) {
179346 const rhs_mem =
179347 try rhs_mcv.address().offset(payload_abi_size).deref().mem(self, .{ .size = .byte });
179348 try self.asmMemoryRegister(.{ ._, .@"test" }, rhs_mem, temp_lhs_reg.to8());
179349 const payload_compare = try self.asmJccReloc(.nz, undefined);
179350 try self.asmRegisterMemory(.{ ._, .cmp }, temp_lhs_reg.to8(), rhs_mem);
179351 break :payload_compare payload_compare;
179352 }
179353
179354 const temp_rhs_reg = try self.copyToTmpRegister(opt_ty, rhs_mcv);
179355 const temp_rhs_lock = self.register_manager.lockRegAssumeUnused(temp_rhs_reg);
179356 defer self.register_manager.unlockReg(temp_rhs_lock);
179357
179358 try self.asmRegisterImmediate(
179359 .{ ._r, .sh },
179360 registerAlias(temp_rhs_reg, opt_abi_size),
179361 .u(payload_abi_size * 8),
179362 );
179363 try self.asmRegisterRegister(
179364 .{ ._, .@"test" },
179365 temp_lhs_reg.to8(),
179366 temp_rhs_reg.to8(),
179367 );
179368 const payload_compare = try self.asmJccReloc(.nz, undefined);
179369 try self.asmRegisterRegister(
179370 .{ ._, .cmp },
179371 temp_lhs_reg.to8(),
179372 temp_rhs_reg.to8(),
179373 );
179374 break :payload_compare payload_compare;
179375 };
179376 null_compare = try self.asmJmpReloc(undefined);
179377 self.performReloc(payload_compare);
179378 },
179379 else => {},
179380 }
179381
179382 switch (ty.zigTypeTag(zcu)) {
179383 else => {
179384 const abi_size: u16 = @intCast(ty.abiSize(zcu));
179385 const may_flip: enum {
179386 may_flip,
179387 must_flip,
179388 must_not_flip,
179389 } = if (abi_size > 8) switch (op) {
179390 .lt, .gte => .must_not_flip,
179391 .lte, .gt => .must_flip,
179392 .eq, .neq => .may_flip,
179393 } else .may_flip;
179394
179395 const flipped = switch (may_flip) {
179396 .may_flip => !lhs_mcv.isRegister() and !lhs_mcv.isBase(),
179397 .must_flip => true,
179398 .must_not_flip => false,
179399 };
179400 const unmat_dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
179401 const dst_mcv = if (unmat_dst_mcv.isRegister() or
179402 (abi_size <= 8 and unmat_dst_mcv.isBase())) unmat_dst_mcv else dst: {
179403 const dst_mcv = try self.allocTempRegOrMem(ty, true);
179404 try self.genCopy(ty, dst_mcv, unmat_dst_mcv, .{});
179405 break :dst dst_mcv;
179406 };
179407 const dst_lock =
179408 if (dst_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
179409 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
179410
179411 const src_mcv = try self.resolveInst(if (flipped) bin_op.lhs else bin_op.rhs);
179412 const src_lock =
179413 if (src_mcv.getReg()) |reg| self.register_manager.lockReg(reg) else null;
179414 defer if (src_lock) |lock| self.register_manager.unlockReg(lock);
179415
179416 break :result .fromCompareOperator(
179417 if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned,
179418 result_op: {
179419 const flipped_op = if (flipped) op.reverse() else op;
179420 if (abi_size > 8) switch (flipped_op) {
179421 .lt, .gte => {},
179422 .lte, .gt => unreachable,
179423 .eq, .neq => {
179424 const OpInfo = ?struct { addr_reg: Register, addr_lock: RegisterLock };
179425
179426 const resolved_dst_mcv = switch (dst_mcv) {
179427 else => dst_mcv,
179428 .air_ref => |dst_ref| try self.resolveInst(dst_ref),
179429 };
179430 const dst_info: OpInfo = switch (resolved_dst_mcv) {
179431 .none,
179432 .unreach,
179433 .dead,
179434 .undef,
179435 .immediate,
179436 .eflags,
179437 .register_offset,
179438 .register_overflow,
179439 .register_mask,
179440 .indirect,
179441 .lea_frame,
179442 .lea_nav,
179443 .lea_uav,
179444 .lea_lazy_sym,
179445 .lea_extern_func,
179446 .elementwise_args,
179447 .reserved_frame,
179448 .air_ref,
179449 => unreachable,
179450 .register,
179451 .register_pair,
179452 .register_triple,
179453 .register_quadruple,
179454 .load_frame,
179455 => null,
179456 .memory,
179457 .load_nav,
179458 .load_uav,
179459 .load_lazy_sym,
179460 .load_extern_func,
179461 => dst: {
179462 switch (resolved_dst_mcv) {
179463 .memory => |addr| if (std.math.cast(
179464 i32,
179465 @as(i64, @bitCast(addr)),
179466 ) != null and std.math.cast(
179467 i32,
179468 @as(i64, @bitCast(addr)) + abi_size - 8,
179469 ) != null) break :dst null,
179470 .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {},
179471 else => unreachable,
179472 }
179473
179474 const dst_addr_reg = (try self.register_manager.allocReg(
179475 null,
179476 abi.RegisterClass.gp,
179477 )).to64();
179478 const dst_addr_lock =
179479 self.register_manager.lockRegAssumeUnused(dst_addr_reg);
179480 errdefer self.register_manager.unlockReg(dst_addr_lock);
179481
179482 try self.genSetReg(dst_addr_reg, .usize, resolved_dst_mcv.address(), .{});
179483 break :dst .{
179484 .addr_reg = dst_addr_reg,
179485 .addr_lock = dst_addr_lock,
179486 };
179487 },
179488 };
179489 defer if (dst_info) |info| self.register_manager.unlockReg(info.addr_lock);
179490
179491 const resolved_src_mcv = switch (src_mcv) {
179492 else => src_mcv,
179493 .air_ref => |src_ref| try self.resolveInst(src_ref),
179494 };
179495 const src_info: OpInfo = switch (resolved_src_mcv) {
179496 .none,
179497 .unreach,
179498 .dead,
179499 .undef,
179500 .immediate,
179501 .eflags,
179502 .register,
179503 .register_offset,
179504 .register_overflow,
179505 .register_mask,
179506 .indirect,
179507 .lea_frame,
179508 .lea_nav,
179509 .lea_uav,
179510 .lea_lazy_sym,
179511 .lea_extern_func,
179512 .elementwise_args,
179513 .reserved_frame,
179514 .air_ref,
179515 => unreachable,
179516 .register_pair,
179517 .register_triple,
179518 .register_quadruple,
179519 .load_frame,
179520 => null,
179521 .memory,
179522 .load_nav,
179523 .load_uav,
179524 .load_lazy_sym,
179525 .load_extern_func,
179526 => src: {
179527 switch (resolved_src_mcv) {
179528 .memory => |addr| if (std.math.cast(
179529 i32,
179530 @as(i64, @bitCast(addr)),
179531 ) != null and std.math.cast(
179532 i32,
179533 @as(i64, @bitCast(addr)) + abi_size - 8,
179534 ) != null) break :src null,
179535 .load_nav, .load_uav, .load_lazy_sym, .load_extern_func => {},
179536 else => unreachable,
179537 }
179538
179539 const src_addr_reg = (try self.register_manager.allocReg(
179540 null,
179541 abi.RegisterClass.gp,
179542 )).to64();
179543 const src_addr_lock =
179544 self.register_manager.lockRegAssumeUnused(src_addr_reg);
179545 errdefer self.register_manager.unlockReg(src_addr_lock);
179546
179547 try self.genSetReg(src_addr_reg, .usize, resolved_src_mcv.address(), .{});
179548 break :src .{
179549 .addr_reg = src_addr_reg,
179550 .addr_lock = src_addr_lock,
179551 };
179552 },
179553 };
179554 defer if (src_info) |info|
179555 self.register_manager.unlockReg(info.addr_lock);
179556
179557 const regs = try self.register_manager.allocRegs(2, @splat(null), abi.RegisterClass.gp);
179558 const acc_reg = regs[0].to64();
179559 const locks = self.register_manager.lockRegsAssumeUnused(2, regs);
179560 defer for (locks) |lock| self.register_manager.unlockReg(lock);
179561
179562 const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable;
179563 var limb_i: u16 = 0;
179564 while (limb_i < limbs_len) : (limb_i += 1) {
179565 const off = limb_i * 8;
179566 const tmp_reg = regs[@min(limb_i, 1)].to64();
179567
179568 try self.genSetReg(tmp_reg, .usize, if (dst_info) |info| .{
179569 .indirect = .{ .reg = info.addr_reg, .off = off },
179570 } else switch (resolved_dst_mcv) {
179571 inline .register_pair,
179572 .register_triple,
179573 .register_quadruple,
179574 => |dst_regs| .{ .register = dst_regs[limb_i] },
179575 .memory => |dst_addr| .{
179576 .memory = @bitCast(@as(i64, @bitCast(dst_addr)) + off),
179577 },
179578 .indirect => |reg_off| .{ .indirect = .{
179579 .reg = reg_off.reg,
179580 .off = reg_off.off + off,
179581 } },
179582 .load_frame => |frame_addr| .{ .load_frame = .{
179583 .index = frame_addr.index,
179584 .off = frame_addr.off + off,
179585 } },
179586 else => unreachable,
179587 }, .{});
179588
179589 try self.genBinOpMir(
179590 .{ ._, .xor },
179591 .usize,
179592 .{ .register = tmp_reg },
179593 if (src_info) |info| .{
179594 .indirect = .{ .reg = info.addr_reg, .off = off },
179595 } else switch (resolved_src_mcv) {
179596 inline .register_pair,
179597 .register_triple,
179598 .register_quadruple,
179599 => |src_regs| .{ .register = src_regs[limb_i] },
179600 .memory => |src_addr| .{
179601 .memory = @bitCast(@as(i64, @bitCast(src_addr)) + off),
179602 },
179603 .indirect => |reg_off| .{ .indirect = .{
179604 .reg = reg_off.reg,
179605 .off = reg_off.off + off,
179606 } },
179607 .load_frame => |frame_addr| .{ .load_frame = .{
179608 .index = frame_addr.index,
179609 .off = frame_addr.off + off,
179610 } },
179611 else => unreachable,
179612 },
179613 );
179614
179615 if (limb_i > 0)
179616 try self.asmRegisterRegister(.{ ._, .@"or" }, acc_reg, tmp_reg);
179617 }
179618 assert(limbs_len >= 2); // use flags from or
179619 break :result_op flipped_op;
179620 },
179621 };
179622 try self.genBinOpMir(.{ ._, .cmp }, ty, dst_mcv, src_mcv);
179623 break :result_op flipped_op;
179624 },
179625 );
179626 },
179627 .float => {
179628 const flipped = switch (op) {
179629 .lt, .lte => true,
179630 .eq, .gte, .gt, .neq => false,
179631 };
179632
179633 const dst_mcv = if (flipped) rhs_mcv else lhs_mcv;
179634 const dst_reg = if (dst_mcv.isRegister())
179635 dst_mcv.getReg().?
179636 else
179637 try self.copyToTmpRegister(ty, dst_mcv);
179638 const dst_lock = self.register_manager.lockReg(dst_reg);
179639 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
179640 const src_mcv = if (flipped) lhs_mcv else rhs_mcv;
179641
179642 switch (ty.floatBits(self.target)) {
179643 16 => {
179644 assert(self.hasFeature(.f16c));
179645 const tmp1_reg =
179646 (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
179647 const tmp1_mcv = MCValue{ .register = tmp1_reg };
179648 const tmp1_lock = self.register_manager.lockRegAssumeUnused(tmp1_reg);
179649 defer self.register_manager.unlockReg(tmp1_lock);
179650
179651 const tmp2_reg =
179652 (try self.register_manager.allocReg(null, abi.RegisterClass.sse)).to128();
179653 const tmp2_mcv = MCValue{ .register = tmp2_reg };
179654 const tmp2_lock = self.register_manager.lockRegAssumeUnused(tmp2_reg);
179655 defer self.register_manager.unlockReg(tmp2_lock);
179656
179657 if (src_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate(
179658 .{ .vp_w, .insr },
179659 tmp1_reg,
179660 dst_reg.to128(),
179661 try src_mcv.mem(self, .{ .size = .word }),
179662 .u(1),
179663 ) else try self.asmRegisterRegisterRegister(
179664 .{ .vp_, .unpcklwd },
179665 tmp1_reg,
179666 dst_reg.to128(),
179667 (if (src_mcv.isRegister())
179668 src_mcv.getReg().?
179669 else
179670 try self.copyToTmpRegister(ty, src_mcv)).to128(),
179671 );
179672 try self.asmRegisterRegister(.{ .v_ps, .cvtph2 }, tmp1_reg, tmp1_reg);
179673 try self.asmRegisterRegister(.{ .v_, .movshdup }, tmp2_reg, tmp1_reg);
179674 try self.genBinOpMir(.{ ._ss, .ucomi }, ty, tmp1_mcv, tmp2_mcv);
179675 },
179676 32 => try self.genBinOpMir(
179677 .{ ._ss, .ucomi },
179678 ty,
179679 .{ .register = dst_reg },
179680 src_mcv,
179681 ),
179682 64 => try self.genBinOpMir(
179683 .{ ._sd, .ucomi },
179684 ty,
179685 .{ .register = dst_reg },
179686 src_mcv,
179687 ),
179688 else => unreachable,
179689 }
179690
179691 break :result switch (if (flipped) op.reverse() else op) {
179692 .lt, .lte => unreachable, // required to have been canonicalized to gt(e)
179693 .gt => .a,
179694 .gte => .ae,
179695 .eq => .z_and_np,
179696 .neq => .nz_or_p,
179697 };
179698 },
179699 }
179700 };
179701
179702 if (null_compare) |reloc| self.performReloc(reloc);
179703 self.eflags_inst = inst;
179704 return self.finishAir(inst, .{ .eflags = result }, .{ bin_op.lhs, bin_op.rhs, .none });
179705}
179706
179707fn airCmpVector(self: *CodeGen, inst: Air.Inst.Index) !void {
179708 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
179709 const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data;
179710 const dst_mcv = try self.genBinOp(
179711 inst,
179712 .fromCmpOp(extra.compareOperator(), false),
179713 extra.lhs,
179714 extra.rhs,
179715 );
179716 return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none });
179717}
179718
179719176277fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
179720176278 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
179721176279 const extra = self.air.extraData(Air.Try, pl_op.payload);
......@@ -181223,16 +177781,13 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
181223177781 .@".cfi_escape" => error.InvalidInstruction,
181224177782 else => unreachable,
181225177783 } else self.asmOps(mnem_fixed_tag, ops)) catch |err| switch (err) {
181226 error.InvalidInstruction => return self.fail(
181227 "invalid instruction: '{s} {s} {s} {s} {s}'",
181228 .{
181229 mnem_str,
181230 @tagName(ops[0]),
181231 @tagName(ops[1]),
181232 @tagName(ops[2]),
181233 @tagName(ops[3]),
181234 },
181235 ),
177784 error.InvalidInstruction => return self.fail("invalid instruction: '{s} {s} {s} {s} {s}'", .{
177785 mnem_str,
177786 @tagName(ops[0]),
177787 @tagName(ops[1]),
177788 @tagName(ops[2]),
177789 @tagName(ops[3]),
177790 }),
181236177791 else => |e| return e,
181237177792 };
181238177793 }
......@@ -182904,183 +179459,6 @@ fn airBitCast(self: *CodeGen, inst: Air.Inst.Index) !void {
182904179459 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
182905179460}
182906179461
182907fn airArrayToSlice(self: *CodeGen, inst: Air.Inst.Index) !void {
182908 const pt = self.pt;
182909 const zcu = pt.zcu;
182910 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
182911
182912 const slice_ty = self.typeOfIndex(inst);
182913 const ptr_ty = self.typeOf(ty_op.operand);
182914 const ptr = try self.resolveInst(ty_op.operand);
182915 const array_ty = ptr_ty.childType(zcu);
182916 const array_len = array_ty.arrayLen(zcu);
182917
182918 const frame_index = try self.allocFrameIndex(.initSpill(slice_ty, zcu));
182919 try self.genSetMem(.{ .frame = frame_index }, 0, ptr_ty, ptr, .{});
182920 try self.genSetMem(
182921 .{ .frame = frame_index },
182922 @intCast(ptr_ty.abiSize(zcu)),
182923 .usize,
182924 .{ .immediate = array_len },
182925 .{},
182926 );
182927
182928 const result = MCValue{ .load_frame = .{ .index = frame_index } };
182929 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
182930}
182931
182932fn airFloatFromInt(self: *CodeGen, inst: Air.Inst.Index) !void {
182933 const pt = self.pt;
182934 const zcu = pt.zcu;
182935 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
182936
182937 const dst_ty = self.typeOfIndex(inst);
182938 const dst_bits = dst_ty.floatBits(self.target);
182939
182940 const src_ty = self.typeOf(ty_op.operand);
182941 const src_bits: u32 = @intCast(src_ty.bitSize(zcu));
182942 const src_signedness =
182943 if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned;
182944 const src_size = std.math.divCeil(u32, @max(switch (src_signedness) {
182945 .signed => src_bits,
182946 .unsigned => src_bits + 1,
182947 }, 32), 8) catch unreachable;
182948
182949 const result = result: {
182950 if (switch (dst_bits) {
182951 16, 80, 128 => true,
182952 32, 64 => src_size > 8,
182953 else => unreachable,
182954 }) {
182955 if (src_bits > 128) return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{
182956 src_ty.fmt(pt), dst_ty.fmt(pt),
182957 });
182958
182959 var sym_buf: ["__floatun?i?f".len]u8 = undefined;
182960 break :result try self.genCall(.{ .extern_func = .{
182961 .return_type = dst_ty.toIntern(),
182962 .param_types = &.{src_ty.toIntern()},
182963 .sym = std.fmt.bufPrint(&sym_buf, "__float{s}{c}i{c}f", .{
182964 switch (src_signedness) {
182965 .signed => "",
182966 .unsigned => "un",
182967 },
182968 intCompilerRtAbiName(src_bits),
182969 floatCompilerRtAbiName(dst_bits),
182970 }) catch unreachable,
182971 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
182972 }
182973
182974 const src_mcv = try self.resolveInst(ty_op.operand);
182975 const src_reg = if (src_mcv.isRegister())
182976 src_mcv.getReg().?
182977 else
182978 try self.copyToTmpRegister(src_ty, src_mcv);
182979 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
182980 defer self.register_manager.unlockReg(src_lock);
182981
182982 if (src_bits < src_size * 8) try self.truncateRegister(src_ty, src_reg);
182983
182984 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
182985 const dst_mcv = MCValue{ .register = dst_reg };
182986 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
182987 defer self.register_manager.unlockReg(dst_lock);
182988
182989 const mir_tag = @as(?Mir.Inst.FixedTag, switch (dst_ty.zigTypeTag(zcu)) {
182990 .float => switch (dst_ty.floatBits(self.target)) {
182991 32 => if (self.hasFeature(.avx)) .{ .v_ss, .cvtsi2 } else .{ ._ss, .cvtsi2 },
182992 64 => if (self.hasFeature(.avx)) .{ .v_sd, .cvtsi2 } else .{ ._sd, .cvtsi2 },
182993 16, 80, 128 => null,
182994 else => unreachable,
182995 },
182996 else => null,
182997 }) orelse return self.fail("TODO implement airFloatFromInt from {f} to {f}", .{
182998 src_ty.fmt(pt), dst_ty.fmt(pt),
182999 });
183000 const dst_alias = dst_reg.to128();
183001 const src_alias = registerAlias(src_reg, src_size);
183002 switch (mir_tag[0]) {
183003 .v_ss, .v_sd => try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, src_alias),
183004 else => try self.asmRegisterRegister(mir_tag, dst_alias, src_alias),
183005 }
183006
183007 break :result dst_mcv;
183008 };
183009 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
183010}
183011
183012fn airIntFromFloat(self: *CodeGen, inst: Air.Inst.Index) !void {
183013 const pt = self.pt;
183014 const zcu = pt.zcu;
183015 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
183016
183017 const dst_ty = self.typeOfIndex(inst);
183018 const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu));
183019 const dst_signedness =
183020 if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned;
183021 const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) {
183022 .signed => dst_bits,
183023 .unsigned => dst_bits + 1,
183024 }, 32), 8) catch unreachable;
183025
183026 const src_ty = self.typeOf(ty_op.operand);
183027 const src_bits = src_ty.floatBits(self.target);
183028
183029 const result = result: {
183030 if (switch (src_bits) {
183031 16, 80, 128 => true,
183032 32, 64 => dst_size > 8,
183033 else => unreachable,
183034 }) {
183035 if (dst_bits > 128) return self.fail("TODO implement airIntFromFloat from {f} to {f}", .{
183036 src_ty.fmt(pt), dst_ty.fmt(pt),
183037 });
183038
183039 var sym_buf: ["__fixuns?f?i".len]u8 = undefined;
183040 break :result try self.genCall(.{ .extern_func = .{
183041 .return_type = dst_ty.toIntern(),
183042 .param_types = &.{src_ty.toIntern()},
183043 .sym = std.fmt.bufPrint(&sym_buf, "__fix{s}{c}f{c}i", .{
183044 switch (dst_signedness) {
183045 .signed => "",
183046 .unsigned => "uns",
183047 },
183048 floatCompilerRtAbiName(src_bits),
183049 intCompilerRtAbiName(dst_bits),
183050 }) catch unreachable,
183051 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }}, .{});
183052 }
183053
183054 const src_mcv = try self.resolveInst(ty_op.operand);
183055 const src_reg = if (src_mcv.isRegister())
183056 src_mcv.getReg().?
183057 else
183058 try self.copyToTmpRegister(src_ty, src_mcv);
183059 const src_lock = self.register_manager.lockRegAssumeUnused(src_reg);
183060 defer self.register_manager.unlockReg(src_lock);
183061
183062 const dst_reg = try self.register_manager.allocReg(inst, self.regSetForType(dst_ty));
183063 const dst_mcv = MCValue{ .register = dst_reg };
183064 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
183065 defer self.register_manager.unlockReg(dst_lock);
183066
183067 try self.asmRegisterRegister(
183068 switch (src_bits) {
183069 32 => if (self.hasFeature(.avx)) .{ .v_, .cvttss2si } else .{ ._, .cvttss2si },
183070 64 => if (self.hasFeature(.avx)) .{ .v_, .cvttsd2si } else .{ ._, .cvttsd2si },
183071 else => unreachable,
183072 },
183073 registerAlias(dst_reg, dst_size),
183074 src_reg.to128(),
183075 );
183076
183077 if (dst_bits < dst_size * 8) try self.truncateRegister(dst_ty, dst_reg);
183078
183079 break :result dst_mcv;
183080 };
183081 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
183082}
183083
183084179462fn airCmpxchg(self: *CodeGen, inst: Air.Inst.Index) !void {
183085179463 const pt = self.pt;
183086179464 const zcu = pt.zcu;
......@@ -183747,331 +180125,46 @@ fn airSplat(self: *CodeGen, inst: Air.Inst.Index) !void {
183747180125 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
183748180126 const vector_ty = self.typeOfIndex(inst);
183749180127 const vector_len = vector_ty.vectorLen(zcu);
183750 const dst_rc = self.regSetForType(vector_ty);
183751180128 const scalar_ty = self.typeOf(ty_op.operand);
183752180129
183753180130 const result: MCValue = result: {
183754 switch (scalar_ty.zigTypeTag(zcu)) {
183755 else => {},
183756 .bool => {
183757 const regs =
183758 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
183759 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
183760 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
183761
183762 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{});
183763 try self.genSetReg(
183764 regs[1],
183765 vector_ty,
183766 .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
183767 .{},
183768 );
183769 const src_mcv = try self.resolveInst(ty_op.operand);
183770 const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
183771 try self.asmCmovccRegisterRegister(
183772 switch (src_mcv) {
183773 .eflags => |cc| cc,
183774 .register => |src_reg| cc: {
183775 try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
183776 break :cc .nz;
183777 },
183778 else => cc: {
183779 try self.asmMemoryImmediate(
183780 .{ ._, .@"test" },
183781 try src_mcv.mem(self, .{ .size = .byte }),
183782 .u(1),
183783 );
183784 break :cc .nz;
183785 },
183786 },
183787 registerAlias(regs[0], abi_size),
183788 registerAlias(regs[1], abi_size),
183789 );
183790 break :result .{ .register = regs[0] };
183791 },
183792 .int => if (self.hasFeature(.avx2)) avx2: {
183793 const mir_tag = @as(?Mir.Inst.FixedTag, switch (scalar_ty.intInfo(zcu).bits) {
183794 else => null,
183795 1...8 => switch (vector_len) {
183796 else => null,
183797 1...32 => .{ .vp_b, .broadcast },
183798 },
183799 9...16 => switch (vector_len) {
183800 else => null,
183801 1...16 => .{ .vp_w, .broadcast },
183802 },
183803 17...32 => switch (vector_len) {
183804 else => null,
183805 1...8 => .{ .vp_d, .broadcast },
183806 },
183807 33...64 => switch (vector_len) {
183808 else => null,
183809 1...4 => .{ .vp_q, .broadcast },
183810 },
183811 65...128 => switch (vector_len) {
183812 else => null,
183813 1...2 => .{ .v_i128, .broadcast },
183814 },
183815 }) orelse break :avx2;
183816
183817 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
183818 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
183819 defer self.register_manager.unlockReg(dst_lock);
183820
183821 const src_mcv = try self.resolveInst(ty_op.operand);
183822 if (src_mcv.isBase()) try self.asmRegisterMemory(
183823 mir_tag,
183824 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
183825 try src_mcv.mem(self, .{ .size = self.memSize(scalar_ty) }),
183826 ) else {
183827 if (mir_tag[0] == .v_i128) break :avx2;
183828 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
183829 try self.asmRegisterRegister(
183830 mir_tag,
183831 registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu))),
183832 registerAlias(dst_reg, @intCast(scalar_ty.abiSize(zcu))),
183833 );
183834 }
183835 break :result .{ .register = dst_reg };
183836 } else {
183837 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);
183838 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
183839 defer self.register_manager.unlockReg(dst_lock);
183840
183841 try self.genSetReg(dst_reg, scalar_ty, .{ .air_ref = ty_op.operand }, .{});
183842 if (vector_len == 1) break :result .{ .register = dst_reg };
180131 if (scalar_ty.toIntern() != .bool_type) return self.fail("TODO implement airSplat for {f}", .{
180132 vector_ty.fmt(pt),
180133 });
180134 const regs =
180135 try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.gp);
180136 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, regs);
180137 defer for (reg_locks) |lock| self.register_manager.unlockReg(lock);
183843180138
183844 const dst_alias = registerAlias(dst_reg, @intCast(vector_ty.abiSize(zcu)));
183845 const scalar_bits = scalar_ty.intInfo(zcu).bits;
183846 if (switch (scalar_bits) {
183847 1...8 => true,
183848 9...128 => false,
183849 else => unreachable,
183850 }) if (self.hasFeature(.avx)) try self.asmRegisterRegisterRegister(
183851 .{ .vp_, .unpcklbw },
183852 dst_alias,
183853 dst_alias,
183854 dst_alias,
183855 ) else try self.asmRegisterRegister(
183856 .{ .p_, .unpcklbw },
183857 dst_alias,
183858 dst_alias,
183859 );
183860 if (switch (scalar_bits) {
183861 1...8 => vector_len > 2,
183862 9...16 => true,
183863 17...128 => false,
183864 else => unreachable,
183865 }) try self.asmRegisterRegisterImmediate(
183866 .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl },
183867 dst_alias,
183868 dst_alias,
183869 .u(0b00_00_00_00),
183870 );
183871 if (switch (scalar_bits) {
183872 1...8 => vector_len > 4,
183873 9...16 => vector_len > 2,
183874 17...64 => true,
183875 65...128 => false,
183876 else => unreachable,
183877 }) try self.asmRegisterRegisterImmediate(
183878 .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf },
183879 dst_alias,
183880 dst_alias,
183881 .u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00),
183882 );
183883 break :result .{ .register = dst_reg };
183884 },
183885 .float => switch (scalar_ty.floatBits(self.target)) {
183886 32 => switch (vector_len) {
183887 1 => {
183888 const src_mcv = try self.resolveInst(ty_op.operand);
183889 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
183890 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
183891 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
183892 break :result .{ .register = dst_reg };
183893 },
183894 2...4 => {
183895 const src_mcv = try self.resolveInst(ty_op.operand);
183896 if (self.hasFeature(.avx)) {
183897 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
183898 if (src_mcv.isBase()) try self.asmRegisterMemory(
183899 .{ .v_ss, .broadcast },
183900 dst_reg.to128(),
183901 try src_mcv.mem(self, .{ .size = .dword }),
183902 ) else {
183903 const src_reg = if (src_mcv.isRegister())
183904 src_mcv.getReg().?
183905 else
183906 try self.copyToTmpRegister(scalar_ty, src_mcv);
183907 try self.asmRegisterRegisterRegisterImmediate(
183908 .{ .v_ps, .shuf },
183909 dst_reg.to128(),
183910 src_reg.to128(),
183911 src_reg.to128(),
183912 .u(0),
183913 );
183914 }
183915 break :result .{ .register = dst_reg };
183916 } else {
183917 const dst_mcv = if (src_mcv.isRegister() and
183918 self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
183919 src_mcv
183920 else
183921 try self.copyToRegisterWithInstTracking(inst, scalar_ty, src_mcv);
183922 const dst_reg = dst_mcv.getReg().?;
183923 try self.asmRegisterRegisterImmediate(
183924 .{ ._ps, .shuf },
183925 dst_reg.to128(),
183926 dst_reg.to128(),
183927 .u(0),
183928 );
183929 break :result dst_mcv;
183930 }
183931 },
183932 5...8 => if (self.hasFeature(.avx)) {
183933 const src_mcv = try self.resolveInst(ty_op.operand);
183934 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
183935 if (src_mcv.isBase()) try self.asmRegisterMemory(
183936 .{ .v_ss, .broadcast },
183937 dst_reg.to256(),
183938 try src_mcv.mem(self, .{ .size = .dword }),
183939 ) else {
183940 const src_reg = if (src_mcv.isRegister())
183941 src_mcv.getReg().?
183942 else
183943 try self.copyToTmpRegister(scalar_ty, src_mcv);
183944 if (self.hasFeature(.avx2)) try self.asmRegisterRegister(
183945 .{ .v_ss, .broadcast },
183946 dst_reg.to256(),
183947 src_reg.to128(),
183948 ) else {
183949 try self.asmRegisterRegisterRegisterImmediate(
183950 .{ .v_ps, .shuf },
183951 dst_reg.to128(),
183952 src_reg.to128(),
183953 src_reg.to128(),
183954 .u(0),
183955 );
183956 try self.asmRegisterRegisterRegisterImmediate(
183957 .{ .v_f128, .insert },
183958 dst_reg.to256(),
183959 dst_reg.to256(),
183960 dst_reg.to128(),
183961 .u(1),
183962 );
183963 }
183964 }
183965 break :result .{ .register = dst_reg };
183966 },
183967 else => {},
183968 },
183969 64 => switch (vector_len) {
183970 1 => {
183971 const src_mcv = try self.resolveInst(ty_op.operand);
183972 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
183973 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
183974 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
183975 break :result .{ .register = dst_reg };
183976 },
183977 2 => {
183978 const src_mcv = try self.resolveInst(ty_op.operand);
183979 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
183980 if (self.hasFeature(.sse3)) {
183981 if (src_mcv.isBase()) try self.asmRegisterMemory(
183982 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
183983 dst_reg.to128(),
183984 try src_mcv.mem(self, .{ .size = .qword }),
183985 ) else try self.asmRegisterRegister(
183986 if (self.hasFeature(.avx)) .{ .v_, .movddup } else .{ ._, .movddup },
183987 dst_reg.to128(),
183988 (if (src_mcv.isRegister())
183989 src_mcv.getReg().?
183990 else
183991 try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(),
183992 );
183993 break :result .{ .register = dst_reg };
183994 } else try self.asmRegisterRegister(
183995 .{ ._ps, .movlh },
183996 dst_reg.to128(),
183997 (if (src_mcv.isRegister())
183998 src_mcv.getReg().?
183999 else
184000 try self.copyToTmpRegister(scalar_ty, src_mcv)).to128(),
184001 );
184002 },
184003 3...4 => if (self.hasFeature(.avx)) {
184004 const src_mcv = try self.resolveInst(ty_op.operand);
184005 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
184006 if (src_mcv.isBase()) try self.asmRegisterMemory(
184007 .{ .v_sd, .broadcast },
184008 dst_reg.to256(),
184009 try src_mcv.mem(self, .{ .size = .qword }),
184010 ) else {
184011 const src_reg = if (src_mcv.isRegister())
184012 src_mcv.getReg().?
184013 else
184014 try self.copyToTmpRegister(scalar_ty, src_mcv);
184015 if (self.hasFeature(.avx2)) try self.asmRegisterRegister(
184016 .{ .v_sd, .broadcast },
184017 dst_reg.to256(),
184018 src_reg.to128(),
184019 ) else {
184020 try self.asmRegisterRegister(
184021 .{ .v_, .movddup },
184022 dst_reg.to128(),
184023 src_reg.to128(),
184024 );
184025 try self.asmRegisterRegisterRegisterImmediate(
184026 .{ .v_f128, .insert },
184027 dst_reg.to256(),
184028 dst_reg.to256(),
184029 dst_reg.to128(),
184030 .u(1),
184031 );
184032 }
184033 }
184034 break :result .{ .register = dst_reg };
184035 },
184036 else => {},
180139 try self.genSetReg(regs[1], vector_ty, .{ .immediate = 0 }, .{});
180140 try self.genSetReg(
180141 regs[1],
180142 vector_ty,
180143 .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) },
180144 .{},
180145 );
180146 const src_mcv = try self.resolveInst(ty_op.operand);
180147 const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4);
180148 try self.asmCmovccRegisterRegister(
180149 switch (src_mcv) {
180150 .eflags => |cc| cc,
180151 .register => |src_reg| cc: {
180152 try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1));
180153 break :cc .nz;
184037180154 },
184038 128 => switch (vector_len) {
184039 1 => {
184040 const src_mcv = try self.resolveInst(ty_op.operand);
184041 if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv;
184042 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
184043 try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{});
184044 break :result .{ .register = dst_reg };
184045 },
184046 2 => if (self.hasFeature(.avx)) {
184047 const src_mcv = try self.resolveInst(ty_op.operand);
184048 const dst_reg = try self.register_manager.allocReg(inst, dst_rc);
184049 if (src_mcv.isBase()) try self.asmRegisterMemory(
184050 .{ .v_f128, .broadcast },
184051 dst_reg.to256(),
184052 try src_mcv.mem(self, .{ .size = .xword }),
184053 ) else {
184054 const src_reg = if (src_mcv.isRegister())
184055 src_mcv.getReg().?
184056 else
184057 try self.copyToTmpRegister(scalar_ty, src_mcv);
184058 try self.asmRegisterRegisterRegisterImmediate(
184059 .{ .v_f128, .insert },
184060 dst_reg.to256(),
184061 src_reg.to256(),
184062 src_reg.to128(),
184063 .u(1),
184064 );
184065 }
184066 break :result .{ .register = dst_reg };
184067 },
184068 else => {},
180155 else => cc: {
180156 try self.asmMemoryImmediate(
180157 .{ ._, .@"test" },
180158 try src_mcv.mem(self, .{ .size = .byte }),
180159 .u(1),
180160 );
180161 break :cc .nz;
184069180162 },
184070 16, 80 => {},
184071 else => unreachable,
184072180163 },
184073 }
184074 return self.fail("TODO implement airSplat for {f}", .{vector_ty.fmt(pt)});
180164 registerAlias(regs[0], abi_size),
180165 registerAlias(regs[1], abi_size),
180166 );
180167 break :result .{ .register = regs[0] };
184075180168 };
184076180169 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
184077180170}
......@@ -185349,161 +181442,135 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
185349181442 const result: MCValue = result: {
185350181443 switch (result_ty.zigTypeTag(zcu)) {
185351181444 .@"struct" => {
181445 if (result_ty.containerLayout(zcu) == .@"packed") return self.fail(
181446 "TODO implement airAggregateInit for {f}",
181447 .{result_ty.fmt(pt)},
181448 );
185352181449 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
185353 if (result_ty.containerLayout(zcu) == .@"packed") {
185354 const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
185355 try self.genInlineMemset(
185356 .{ .lea_frame = .{ .index = frame_index } },
185357 .{ .immediate = 0 },
185358 .{ .immediate = result_ty.abiSize(zcu) },
185359 .{},
185360 );
185361 for (elements, 0..) |elem, elem_i_usize| {
185362 const elem_i: u32 = @intCast(elem_i_usize);
185363 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
185364
185365 const elem_ty = result_ty.fieldType(elem_i, zcu);
185366 const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
185367 if (elem_bit_size > 64) {
185368 return self.fail(
185369 "TODO airAggregateInit implement packed structs with large fields",
185370 .{},
185371 );
185372 }
185373 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
185374 const elem_abi_bits = elem_abi_size * 8;
185375 const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i);
185376 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
185377 const elem_bit_off = elem_off % elem_abi_bits;
185378 const elem_mcv = try self.resolveInst(elem);
185379 const elem_lock = switch (elem_mcv) {
185380 .register => |reg| self.register_manager.lockReg(reg),
185381 .immediate => |imm| lock: {
185382 if (imm == 0) continue;
185383 break :lock null;
185384 },
185385 else => null,
185386 };
185387 defer if (elem_lock) |lock| self.register_manager.unlockReg(lock);
185388
185389 const elem_extra_bits = self.regExtraBits(elem_ty);
185390 {
185391 const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
185392 const temp_alias = registerAlias(temp_reg, elem_abi_size);
185393 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
185394 defer self.register_manager.unlockReg(temp_lock);
185395
185396 if (elem_bit_off < elem_extra_bits) {
185397 try self.truncateRegister(elem_ty, temp_alias);
185398 }
185399 if (elem_bit_off > 0) try self.genShiftBinOpMir(
185400 .{ ._l, .sh },
185401 elem_ty,
185402 .{ .register = temp_alias },
185403 .u8,
185404 .{ .immediate = elem_bit_off },
185405 );
185406 try self.genBinOpMir(
185407 .{ ._, .@"or" },
185408 elem_ty,
185409 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
185410 .{ .register = temp_alias },
185411 );
185412 }
185413 if (elem_bit_off > elem_extra_bits) {
185414 const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
185415 const temp_alias = registerAlias(temp_reg, elem_abi_size);
185416 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
185417 defer self.register_manager.unlockReg(temp_lock);
185418
185419 if (elem_extra_bits > 0) {
185420 try self.truncateRegister(elem_ty, temp_alias);
185421 }
185422 try self.genShiftBinOpMir(
185423 .{ ._r, .sh },
185424 elem_ty,
185425 .{ .register = temp_reg },
185426 .u8,
185427 .{ .immediate = elem_abi_bits - elem_bit_off },
185428 );
185429 try self.genBinOpMir(
185430 .{ ._, .@"or" },
185431 elem_ty,
185432 .{ .load_frame = .{
185433 .index = frame_index,
185434 .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
185435 } },
185436 .{ .register = temp_alias },
185437 );
185438 }
185439 }
185440 } else for (elements, 0..) |elem, elem_i| {
181450 const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern());
181451 try self.genInlineMemset(
181452 .{ .lea_frame = .{ .index = frame_index } },
181453 .{ .immediate = 0 },
181454 .{ .immediate = result_ty.abiSize(zcu) },
181455 .{},
181456 );
181457 for (elements, 0..) |elem, elem_i_usize| {
181458 const elem_i: u32 = @intCast(elem_i_usize);
185441181459 if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue;
185442181460
185443181461 const elem_ty = result_ty.fieldType(elem_i, zcu);
185444 const elem_off: i32 = @intCast(result_ty.structFieldOffset(elem_i, zcu));
181462 const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu));
181463 if (elem_bit_size > 64) {
181464 return self.fail(
181465 "TODO airAggregateInit implement packed structs with large fields",
181466 .{},
181467 );
181468 }
181469 const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu));
181470 const elem_abi_bits = elem_abi_size * 8;
181471 const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i);
181472 const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size);
181473 const elem_bit_off = elem_off % elem_abi_bits;
185445181474 const elem_mcv = try self.resolveInst(elem);
185446 try self.genSetMem(.{ .frame = frame_index }, elem_off, elem_ty, elem_mcv, .{});
185447 }
185448 break :result .{ .load_frame = .{ .index = frame_index } };
185449 },
185450 .array, .vector => {
185451 const elem_ty = result_ty.childType(zcu);
185452 if (result_ty.isVector(zcu) and elem_ty.toIntern() == .bool_type) {
185453 const result_size: u32 = @intCast(result_ty.abiSize(zcu));
185454 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
185455 try self.asmRegisterRegister(
185456 .{ ._, .xor },
185457 registerAlias(dst_reg, @min(result_size, 4)),
185458 registerAlias(dst_reg, @min(result_size, 4)),
185459 );
181475 const elem_lock = switch (elem_mcv) {
181476 .register => |reg| self.register_manager.lockReg(reg),
181477 .immediate => |imm| lock: {
181478 if (imm == 0) continue;
181479 break :lock null;
181480 },
181481 else => null,
181482 };
181483 defer if (elem_lock) |lock| self.register_manager.unlockReg(lock);
185460181484
185461 for (elements, 0..) |elem, elem_i| {
185462 const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
185463 const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
185464 defer self.register_manager.unlockReg(elem_lock);
181485 const elem_extra_bits = self.regExtraBits(elem_ty);
181486 {
181487 const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181488 const temp_alias = registerAlias(temp_reg, elem_abi_size);
181489 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181490 defer self.register_manager.unlockReg(temp_lock);
185465181491
185466 try self.asmRegisterImmediate(
185467 .{ ._, .@"and" },
185468 registerAlias(elem_reg, @min(result_size, 4)),
185469 .u(1),
185470 );
185471 if (elem_i > 0) try self.asmRegisterImmediate(
181492 if (elem_bit_off < elem_extra_bits) {
181493 try self.truncateRegister(elem_ty, temp_alias);
181494 }
181495 if (elem_bit_off > 0) try self.genShiftBinOpMir(
185472181496 .{ ._l, .sh },
185473 registerAlias(elem_reg, result_size),
185474 .u(@intCast(elem_i)),
181497 elem_ty,
181498 .{ .register = temp_alias },
181499 .u8,
181500 .{ .immediate = elem_bit_off },
185475181501 );
185476 try self.asmRegisterRegister(
181502 try self.genBinOpMir(
185477181503 .{ ._, .@"or" },
185478 registerAlias(dst_reg, result_size),
185479 registerAlias(elem_reg, result_size),
181504 elem_ty,
181505 .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } },
181506 .{ .register = temp_alias },
185480181507 );
185481181508 }
185482 break :result .{ .register = dst_reg };
185483 } else {
185484 const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu));
185485 const elem_size: u32 = @intCast(elem_ty.abiSize(zcu));
185486
185487 for (elements, 0..) |elem, elem_i| {
185488 const elem_mcv = try self.resolveInst(elem);
185489 const elem_off: i32 = @intCast(elem_size * elem_i);
185490 try self.genSetMem(
185491 .{ .frame = frame_index },
185492 elem_off,
181509 if (elem_bit_off > elem_extra_bits) {
181510 const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv);
181511 const temp_alias = registerAlias(temp_reg, elem_abi_size);
181512 const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg);
181513 defer self.register_manager.unlockReg(temp_lock);
181514
181515 if (elem_extra_bits > 0) {
181516 try self.truncateRegister(elem_ty, temp_alias);
181517 }
181518 try self.genShiftBinOpMir(
181519 .{ ._r, .sh },
185493181520 elem_ty,
185494 elem_mcv,
185495 .{},
181521 .{ .register = temp_reg },
181522 .u8,
181523 .{ .immediate = elem_abi_bits - elem_bit_off },
181524 );
181525 try self.genBinOpMir(
181526 .{ ._, .@"or" },
181527 elem_ty,
181528 .{ .load_frame = .{
181529 .index = frame_index,
181530 .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)),
181531 } },
181532 .{ .register = temp_alias },
185496181533 );
185497181534 }
185498 if (result_ty.sentinel(zcu)) |sentinel| try self.genSetMem(
185499 .{ .frame = frame_index },
185500 @intCast(elem_size * elements.len),
185501 elem_ty,
185502 try self.lowerValue(sentinel),
185503 .{},
181535 }
181536 break :result .{ .load_frame = .{ .index = frame_index } };
181537 },
181538 .vector => {
181539 const elem_ty = result_ty.childType(zcu);
181540 if (elem_ty.toIntern() != .bool_type) return self.fail(
181541 "TODO implement airAggregateInit for {f}",
181542 .{result_ty.fmt(pt)},
181543 );
181544 const result_size: u32 = @intCast(result_ty.abiSize(zcu));
181545 const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp);
181546 try self.asmRegisterRegister(
181547 .{ ._, .xor },
181548 registerAlias(dst_reg, @min(result_size, 4)),
181549 registerAlias(dst_reg, @min(result_size, 4)),
181550 );
181551
181552 for (elements, 0..) |elem, elem_i| {
181553 const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem });
181554 const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg);
181555 defer self.register_manager.unlockReg(elem_lock);
181556
181557 try self.asmRegisterImmediate(
181558 .{ ._, .@"and" },
181559 registerAlias(elem_reg, @min(result_size, 4)),
181560 .u(1),
181561 );
181562 if (elem_i > 0) try self.asmRegisterImmediate(
181563 .{ ._l, .sh },
181564 registerAlias(elem_reg, result_size),
181565 .u(@intCast(elem_i)),
181566 );
181567 try self.asmRegisterRegister(
181568 .{ ._, .@"or" },
181569 registerAlias(dst_reg, result_size),
181570 registerAlias(elem_reg, result_size),
185504181571 );
185505 break :result .{ .load_frame = .{ .index = frame_index } };
185506181572 }
181573 break :result .{ .register = dst_reg };
185507181574 },
185508181575 else => unreachable,
185509181576 }
......@@ -185519,220 +181586,6 @@ fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void {
185519181586 return self.finishAirResult(inst, result);
185520181587}
185521181588
185522fn airUnionInit(self: *CodeGen, inst: Air.Inst.Index) !void {
185523 const pt = self.pt;
185524 const zcu = pt.zcu;
185525 const ip = &zcu.intern_pool;
185526 const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl;
185527 const extra = self.air.extraData(Air.UnionInit, ty_pl.payload).data;
185528 const result: MCValue = result: {
185529 const union_ty = self.typeOfIndex(inst);
185530 const layout = union_ty.unionGetLayout(zcu);
185531
185532 const src_ty = self.typeOf(extra.init);
185533 const src_mcv = try self.resolveInst(extra.init);
185534 if (layout.tag_size == 0) {
185535 if (layout.abi_size <= src_ty.abiSize(zcu) and
185536 self.reuseOperand(inst, extra.init, 0, src_mcv)) break :result src_mcv;
185537
185538 const dst_mcv = try self.allocRegOrMem(inst, true);
185539 try self.genCopy(src_ty, dst_mcv, src_mcv, .{});
185540 break :result dst_mcv;
185541 }
185542
185543 const dst_mcv = try self.allocRegOrMem(inst, false);
185544
185545 const loaded_union = zcu.typeToUnion(union_ty).?;
185546 const field_name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index];
185547 const tag_ty: Type = .fromInterned(loaded_union.enum_tag_ty);
185548 const field_index = tag_ty.enumFieldIndex(field_name, zcu).?;
185549 const tag_val = try pt.enumValueFieldIndex(tag_ty, field_index);
185550 const tag_int_val = try tag_val.intFromEnum(tag_ty, pt);
185551 const tag_int = tag_int_val.toUnsignedInt(zcu);
185552 const tag_off: i32 = @intCast(layout.tagOffset());
185553 try self.genCopy(
185554 tag_ty,
185555 dst_mcv.address().offset(tag_off).deref(),
185556 .{ .immediate = tag_int },
185557 .{},
185558 );
185559
185560 const pl_off: i32 = @intCast(layout.payloadOffset());
185561 try self.genCopy(src_ty, dst_mcv.address().offset(pl_off).deref(), src_mcv, .{});
185562
185563 break :result dst_mcv;
185564 };
185565 return self.finishAir(inst, result, .{ extra.init, .none, .none });
185566}
185567
185568fn airMulAdd(self: *CodeGen, inst: Air.Inst.Index) !void {
185569 const pt = self.pt;
185570 const zcu = pt.zcu;
185571 const pl_op = self.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
185572 const extra = self.air.extraData(Air.Bin, pl_op.payload).data;
185573 const ty = self.typeOfIndex(inst);
185574
185575 const ops = [3]Air.Inst.Ref{ extra.lhs, extra.rhs, pl_op.operand };
185576 const result = result: {
185577 if (switch (ty.scalarType(zcu).floatBits(self.target)) {
185578 16, 80, 128 => true,
185579 32, 64 => !self.hasFeature(.fma),
185580 else => unreachable,
185581 }) {
185582 if (ty.zigTypeTag(zcu) != .float) return self.fail("TODO implement airMulAdd for {f}", .{
185583 ty.fmt(pt),
185584 });
185585
185586 var sym_buf: ["__fma?".len]u8 = undefined;
185587 break :result try self.genCall(.{ .extern_func = .{
185588 .return_type = ty.toIntern(),
185589 .param_types = &.{ ty.toIntern(), ty.toIntern(), ty.toIntern() },
185590 .sym = std.fmt.bufPrint(&sym_buf, "{s}fma{s}", .{
185591 floatLibcAbiPrefix(ty),
185592 floatLibcAbiSuffix(ty),
185593 }) catch unreachable,
185594 } }, &.{ ty, ty, ty }, &.{
185595 .{ .air_ref = extra.lhs }, .{ .air_ref = extra.rhs }, .{ .air_ref = pl_op.operand },
185596 }, .{});
185597 }
185598
185599 var mcvs: [3]MCValue = undefined;
185600 var locks: [3]?RegisterManager.RegisterLock = @splat(null);
185601 defer for (locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
185602 var order: [3]u2 = @splat(0);
185603 var unused: std.StaticBitSet(3) = .initFull();
185604 for (ops, &mcvs, &locks, 0..) |op, *mcv, *lock, op_i| {
185605 const op_index: u2 = @intCast(op_i);
185606 mcv.* = try self.resolveInst(op);
185607 if (unused.isSet(0) and mcv.isRegister() and self.reuseOperand(inst, op, op_index, mcv.*)) {
185608 order[op_index] = 1;
185609 unused.unset(0);
185610 } else if (unused.isSet(2) and mcv.isBase()) {
185611 order[op_index] = 3;
185612 unused.unset(2);
185613 }
185614 switch (mcv.*) {
185615 .register => |reg| lock.* = self.register_manager.lockReg(reg),
185616 else => {},
185617 }
185618 }
185619 for (&order, &mcvs, &locks) |*mop_index, *mcv, *lock| {
185620 if (mop_index.* != 0) continue;
185621 mop_index.* = 1 + @as(u2, @intCast(unused.toggleFirstSet().?));
185622 if (mop_index.* > 1 and mcv.isRegister()) continue;
185623 const reg = try self.copyToTmpRegister(ty, mcv.*);
185624 mcv.* = .{ .register = reg };
185625 if (lock.*) |old_lock| self.register_manager.unlockReg(old_lock);
185626 lock.* = self.register_manager.lockRegAssumeUnused(reg);
185627 }
185628
185629 const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or
185630 std.mem.eql(u2, &order, &.{ 3, 1, 2 }))
185631 switch (ty.zigTypeTag(zcu)) {
185632 .float => switch (ty.floatBits(self.target)) {
185633 32 => .{ .v_ss, .fmadd132 },
185634 64 => .{ .v_sd, .fmadd132 },
185635 16, 80, 128 => null,
185636 else => unreachable,
185637 },
185638 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
185639 .float => switch (ty.childType(zcu).floatBits(self.target)) {
185640 32 => switch (ty.vectorLen(zcu)) {
185641 1 => .{ .v_ss, .fmadd132 },
185642 2...8 => .{ .v_ps, .fmadd132 },
185643 else => null,
185644 },
185645 64 => switch (ty.vectorLen(zcu)) {
185646 1 => .{ .v_sd, .fmadd132 },
185647 2...4 => .{ .v_pd, .fmadd132 },
185648 else => null,
185649 },
185650 16, 80, 128 => null,
185651 else => unreachable,
185652 },
185653 else => unreachable,
185654 },
185655 else => unreachable,
185656 }
185657 else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 }))
185658 switch (ty.zigTypeTag(zcu)) {
185659 .float => switch (ty.floatBits(self.target)) {
185660 32 => .{ .v_ss, .fmadd213 },
185661 64 => .{ .v_sd, .fmadd213 },
185662 16, 80, 128 => null,
185663 else => unreachable,
185664 },
185665 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
185666 .float => switch (ty.childType(zcu).floatBits(self.target)) {
185667 32 => switch (ty.vectorLen(zcu)) {
185668 1 => .{ .v_ss, .fmadd213 },
185669 2...8 => .{ .v_ps, .fmadd213 },
185670 else => null,
185671 },
185672 64 => switch (ty.vectorLen(zcu)) {
185673 1 => .{ .v_sd, .fmadd213 },
185674 2...4 => .{ .v_pd, .fmadd213 },
185675 else => null,
185676 },
185677 16, 80, 128 => null,
185678 else => unreachable,
185679 },
185680 else => unreachable,
185681 },
185682 else => unreachable,
185683 }
185684 else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 }))
185685 switch (ty.zigTypeTag(zcu)) {
185686 .float => switch (ty.floatBits(self.target)) {
185687 32 => .{ .v_ss, .fmadd231 },
185688 64 => .{ .v_sd, .fmadd231 },
185689 16, 80, 128 => null,
185690 else => unreachable,
185691 },
185692 .vector => switch (ty.childType(zcu).zigTypeTag(zcu)) {
185693 .float => switch (ty.childType(zcu).floatBits(self.target)) {
185694 32 => switch (ty.vectorLen(zcu)) {
185695 1 => .{ .v_ss, .fmadd231 },
185696 2...8 => .{ .v_ps, .fmadd231 },
185697 else => null,
185698 },
185699 64 => switch (ty.vectorLen(zcu)) {
185700 1 => .{ .v_sd, .fmadd231 },
185701 2...4 => .{ .v_pd, .fmadd231 },
185702 else => null,
185703 },
185704 16, 80, 128 => null,
185705 else => unreachable,
185706 },
185707 else => unreachable,
185708 },
185709 else => unreachable,
185710 }
185711 else
185712 unreachable) orelse return self.fail("TODO implement airMulAdd for {f}", .{ty.fmt(pt)});
185713
185714 var mops: [3]MCValue = undefined;
185715 for (order, mcvs) |mop_index, mcv| mops[mop_index - 1] = mcv;
185716
185717 const abi_size: u32 = @intCast(ty.abiSize(zcu));
185718 const mop1_reg = registerAlias(mops[0].getReg().?, abi_size);
185719 const mop2_reg = registerAlias(mops[1].getReg().?, abi_size);
185720 if (mops[2].isRegister()) try self.asmRegisterRegisterRegister(
185721 mir_tag,
185722 mop1_reg,
185723 mop2_reg,
185724 registerAlias(mops[2].getReg().?, abi_size),
185725 ) else try self.asmRegisterRegisterMemory(
185726 mir_tag,
185727 mop1_reg,
185728 mop2_reg,
185729 try mops[2].mem(self, .{ .size = .fromSize(abi_size) }),
185730 );
185731 break :result mops[0];
185732 };
185733 return self.finishAir(inst, result, ops);
185734}
185735
185736181589fn airVaStart(self: *CodeGen, inst: Air.Inst.Index) !void {
185737181590 const pt = self.pt;
185738181591 const zcu = pt.zcu;
......@@ -186004,27 +181857,6 @@ fn getResolvedInstValue(self: *CodeGen, inst: Air.Inst.Index) *InstTracking {
186004181857 };
186005181858}
186006181859
186007/// If the MCValue is an immediate, and it does not fit within this type,
186008/// we put it in a register.
186009/// A potential opportunity for future optimization here would be keeping track
186010/// of the fact that the instruction is available both as an immediate
186011/// and as a register.
186012fn limitImmediateType(self: *CodeGen, operand: Air.Inst.Ref, comptime T: type) !MCValue {
186013 const mcv = try self.resolveInst(operand);
186014 const ti = @typeInfo(T).int;
186015 switch (mcv) {
186016 .immediate => |imm| {
186017 // This immediate is unsigned.
186018 const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed));
186019 if (imm >= std.math.maxInt(U)) {
186020 return MCValue{ .register = try self.copyToTmpRegister(.usize, mcv) };
186021 }
186022 },
186023 else => {},
186024 }
186025 return mcv;
186026}
186027
186028181860fn lowerValue(cg: *CodeGen, val: Value) Allocator.Error!MCValue {
186029181861 return switch (try codegen.lowerValue(cg.pt, val, cg.target)) {
186030181862 .none => .none,
......@@ -186134,7 +181966,7 @@ fn resolveCallingConventionValues(
186134181966
186135181967 const classes = switch (cc) {
186136181968 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, cg.target, .ret), .none),
186137 .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu, cg.target)},
181969 .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu, cg.target, .ret)},
186138181970 else => unreachable,
186139181971 };
186140181972 for (classes) |class| switch (class) {
......@@ -186215,7 +182047,7 @@ fn resolveCallingConventionValues(
186215182047
186216182048 const classes = switch (cc) {
186217182049 .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .arg), .none),
186218 .x86_64_win => &.{abi.classifyWindows(ty, zcu, cg.target)},
182050 .x86_64_win => &.{abi.classifyWindows(ty, zcu, cg.target, .arg)},
186219182051 else => unreachable,
186220182052 };
186221182053 classes: for (classes) |class| switch (class) {
......@@ -186678,53 +182510,6 @@ fn typeOfIndex(self: *CodeGen, inst: Air.Inst.Index) Type {
186678182510 return Temp.typeOf(.{ .index = inst }, self);
186679182511}
186680182512
186681fn intCompilerRtAbiName(int_bits: u32) u8 {
186682 return switch (int_bits) {
186683 1...32 => 's',
186684 33...64 => 'd',
186685 65...128 => 't',
186686 else => unreachable,
186687 };
186688}
186689
186690fn floatCompilerRtAbiName(float_bits: u32) u8 {
186691 return switch (float_bits) {
186692 16 => 'h',
186693 32 => 's',
186694 64 => 'd',
186695 80 => 'x',
186696 128 => 't',
186697 else => unreachable,
186698 };
186699}
186700
186701fn floatCompilerRtAbiType(self: *CodeGen, ty: Type, other_ty: Type) Type {
186702 if (ty.toIntern() == .f16_type and
186703 (other_ty.toIntern() == .f32_type or other_ty.toIntern() == .f64_type) and
186704 self.target.os.tag.isDarwin()) return .u16;
186705 return ty;
186706}
186707
186708fn floatLibcAbiPrefix(ty: Type) []const u8 {
186709 return switch (ty.toIntern()) {
186710 .f16_type, .f80_type => "__",
186711 .f32_type, .f64_type, .f128_type, .c_longdouble_type => "",
186712 else => unreachable,
186713 };
186714}
186715
186716fn floatLibcAbiSuffix(ty: Type) []const u8 {
186717 return switch (ty.toIntern()) {
186718 .f16_type => "h",
186719 .f32_type => "f",
186720 .f64_type => "",
186721 .f80_type => "x",
186722 .f128_type => "q",
186723 .c_longdouble_type => "l",
186724 else => unreachable,
186725 };
186726}
186727
186728182513fn promoteInt(self: *CodeGen, ty: Type) Type {
186729182514 const pt = self.pt;
186730182515 const zcu = pt.zcu;
src/codegen/x86_64/Emit.zig+32-71
......@@ -89,6 +89,7 @@ pub fn emitMir(emit: *Emit) Error!void {
8989 }
9090 var reloc_info_buf: [2]RelocInfo = undefined;
9191 var reloc_info_index: usize = 0;
92 const ip = &emit.pt.zcu.intern_pool;
9293 while (lowered_relocs.len > 0 and
9394 lowered_relocs[0].lowered_inst_index == lowered_index) : ({
9495 lowered_relocs = lowered_relocs[1..];
......@@ -114,7 +115,6 @@ pub fn emitMir(emit: *Emit) Error!void {
114115 return error.EmitFail;
115116 },
116117 };
117 const ip = &emit.pt.zcu.intern_pool;
118118 break :target switch (ip.getNav(nav).status) {
119119 .unresolved => unreachable,
120120 .type_resolved => |type_resolved| .{
......@@ -170,11 +170,8 @@ pub fn emitMir(emit: *Emit) Error!void {
170170 else if (emit.bin_file.cast(.macho)) |macho_file|
171171 macho_file.getZigObject().?.getOrCreateMetadataForLazySymbol(macho_file, emit.pt, lazy_sym) catch |err|
172172 return emit.fail("{s} creating lazy symbol", .{@errorName(err)})
173 else if (emit.bin_file.cast(.coff)) |coff_file|
174 if (coff_file.getOrCreateAtomForLazySymbol(emit.pt, lazy_sym)) |atom|
175 coff_file.getAtom(atom).getSymbolIndex().?
176 else |err|
177 return emit.fail("{s} creating lazy symbol", .{@errorName(err)})
173 else if (emit.bin_file.cast(.coff2)) |elf|
174 @intFromEnum(try elf.lazySymbol(lazy_sym))
178175 else
179176 return emit.fail("lazy symbols unimplemented for {s}", .{@tagName(emit.bin_file.tag)}),
180177 .is_extern = false,
......@@ -188,10 +185,13 @@ pub fn emitMir(emit: *Emit) Error!void {
188185 .type = .FUNC,
189186 })) else if (emit.bin_file.cast(.macho)) |macho_file|
190187 try macho_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, null)
191 else if (emit.bin_file.cast(.coff)) |coff_file|
192 try coff_file.getGlobalSymbol(extern_func.toSlice(&emit.lower.mir).?, "compiler_rt")
193 else
194 return emit.fail("external symbol unimplemented for {s}", .{@tagName(emit.bin_file.tag)}),
188 else if (emit.bin_file.cast(.coff2)) |coff| @intFromEnum(try coff.globalSymbol(
189 extern_func.toSlice(&emit.lower.mir).?,
190 switch (comp.compiler_rt_strat) {
191 .none, .lib, .obj, .zcu => null,
192 .dyn_lib => "compiler_rt",
193 },
194 )) else return emit.fail("external symbol unimplemented for {s}", .{@tagName(emit.bin_file.tag)}),
195195 .is_extern = true,
196196 .type = .symbol,
197197 },
......@@ -204,9 +204,7 @@ pub fn emitMir(emit: *Emit) Error!void {
204204 switch (lowered_inst.encoding.mnemonic) {
205205 .call => {
206206 reloc.target.type = .branch;
207 if (emit.bin_file.cast(.coff)) |_| try emit.encodeInst(try .new(.none, .call, &.{
208 .{ .mem = .initRip(.ptr, 0) },
209 }, emit.lower.target), reloc_info) else try emit.encodeInst(lowered_inst, reloc_info);
207 try emit.encodeInst(lowered_inst, reloc_info);
210208 continue :lowered_inst;
211209 },
212210 else => {},
......@@ -283,27 +281,8 @@ pub fn emitMir(emit: *Emit) Error!void {
283281 }, emit.lower.target), reloc_info),
284282 else => unreachable,
285283 }
286 } else if (emit.bin_file.cast(.coff)) |_| {
287 if (reloc.target.is_extern) switch (lowered_inst.encoding.mnemonic) {
288 .lea => try emit.encodeInst(try .new(.none, .mov, &.{
289 lowered_inst.ops[0],
290 .{ .mem = .initRip(.ptr, 0) },
291 }, emit.lower.target), reloc_info),
292 .mov => {
293 const dst_reg = lowered_inst.ops[0].reg.to64();
294 try emit.encodeInst(try .new(.none, .mov, &.{
295 .{ .reg = dst_reg },
296 .{ .mem = .initRip(.ptr, 0) },
297 }, emit.lower.target), reloc_info);
298 try emit.encodeInst(try .new(.none, .mov, &.{
299 lowered_inst.ops[0],
300 .{ .mem = .initSib(lowered_inst.ops[reloc.op_index].mem.sib.ptr_size, .{ .base = .{
301 .reg = dst_reg,
302 } }) },
303 }, emit.lower.target), &.{});
304 },
305 else => unreachable,
306 } else switch (lowered_inst.encoding.mnemonic) {
284 } else if (emit.bin_file.cast(.coff2)) |_| {
285 switch (lowered_inst.encoding.mnemonic) {
307286 .lea => try emit.encodeInst(try .new(.none, .lea, &.{
308287 lowered_inst.ops[0],
309288 .{ .mem = .initRip(.none, 0) },
......@@ -683,7 +662,7 @@ pub fn emitMir(emit: *Emit) Error!void {
683662 table_reloc.source_offset,
684663 @enumFromInt(emit.atom_index),
685664 @as(i64, table_offset) + table_reloc.target_offset,
686 .{ .x86_64 = .@"32" },
665 .{ .X86_64 = .@"32" },
687666 );
688667 for (emit.lower.mir.table) |entry| {
689668 try elf.addReloc(
......@@ -691,7 +670,7 @@ pub fn emitMir(emit: *Emit) Error!void {
691670 table_offset,
692671 @enumFromInt(emit.atom_index),
693672 emit.code_offset_mapping.items[entry],
694 .{ .x86_64 = .@"64" },
673 .{ .X86_64 = .@"64" },
695674 );
696675 table_offset += ptr_size;
697676 }
......@@ -800,23 +779,14 @@ fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocI
800779 end_offset - 4,
801780 @enumFromInt(reloc.target.index),
802781 reloc.off,
803 .{ .x86_64 = .@"32" },
804 ) else if (emit.bin_file.cast(.coff)) |coff_file| {
805 const atom_index = coff_file.getAtomIndexForSymbol(
806 .{ .sym_index = emit.atom_index, .file = null },
807 ).?;
808 try coff_file.addRelocation(atom_index, .{
809 .type = if (reloc.target.is_extern) .got else .direct,
810 .target = if (reloc.target.is_extern)
811 coff_file.getGlobalByIndex(reloc.target.index)
812 else
813 .{ .sym_index = reloc.target.index, .file = null },
814 .offset = end_offset - 4,
815 .addend = @intCast(reloc.off),
816 .pcrel = true,
817 .length = 2,
818 });
819 } else unreachable,
782 .{ .X86_64 = .@"32" },
783 ) else if (emit.bin_file.cast(.coff2)) |coff| try coff.addReloc(
784 @enumFromInt(emit.atom_index),
785 end_offset - 4,
786 @enumFromInt(reloc.target.index),
787 reloc.off,
788 .{ .AMD64 = .REL32 },
789 ) else unreachable,
820790 .branch => if (emit.bin_file.cast(.elf)) |elf_file| {
821791 const zo = elf_file.zigObjectPtr().?;
822792 const atom = zo.symbol(emit.atom_index).atom(elf_file).?;
......@@ -831,7 +801,7 @@ fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocI
831801 end_offset - 4,
832802 @enumFromInt(reloc.target.index),
833803 reloc.off - 4,
834 .{ .x86_64 = .PC32 },
804 .{ .X86_64 = .PC32 },
835805 ) else if (emit.bin_file.cast(.macho)) |macho_file| {
836806 const zo = macho_file.getZigObject().?;
837807 const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?;
......@@ -848,22 +818,13 @@ fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocI
848818 .symbolnum = @intCast(reloc.target.index),
849819 },
850820 });
851 } else if (emit.bin_file.cast(.coff)) |coff_file| {
852 const atom_index = coff_file.getAtomIndexForSymbol(
853 .{ .sym_index = emit.atom_index, .file = null },
854 ).?;
855 try coff_file.addRelocation(atom_index, .{
856 .type = if (reloc.target.is_extern) .import else .got,
857 .target = if (reloc.target.is_extern)
858 coff_file.getGlobalByIndex(reloc.target.index)
859 else
860 .{ .sym_index = reloc.target.index, .file = null },
861 .offset = end_offset - 4,
862 .addend = @intCast(reloc.off),
863 .pcrel = true,
864 .length = 2,
865 });
866 } else return emit.fail("TODO implement {s} reloc for {s}", .{
821 } else if (emit.bin_file.cast(.coff2)) |coff| try coff.addReloc(
822 @enumFromInt(emit.atom_index),
823 end_offset - 4,
824 @enumFromInt(reloc.target.index),
825 reloc.off,
826 .{ .AMD64 = .REL32 },
827 ) else return emit.fail("TODO implement {s} reloc for {s}", .{
867828 @tagName(reloc.target.type), @tagName(emit.bin_file.tag),
868829 }),
869830 .tls => if (emit.bin_file.cast(.elf)) |elf_file| {
......@@ -892,7 +853,7 @@ fn encodeInst(emit: *Emit, lowered_inst: Instruction, reloc_info: []const RelocI
892853 end_offset - 4,
893854 @enumFromInt(reloc.target.index),
894855 reloc.off,
895 .{ .x86_64 = .TPOFF32 },
856 .{ .X86_64 = .TPOFF32 },
896857 ) else if (emit.bin_file.cast(.macho)) |macho_file| {
897858 const zo = macho_file.getZigObject().?;
898859 const atom = zo.symbols.items[emit.atom_index].getAtom(macho_file).?;
src/codegen/x86_64/abi.zig+5-4
......@@ -110,7 +110,9 @@ pub const Class = enum {
110110 }
111111};
112112
113pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target) Class {
113pub const Context = enum { ret, arg, other };
114
115pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) Class {
114116 // https://docs.microsoft.com/en-gb/cpp/build/x64-calling-convention?view=vs-2017
115117 // "There's a strict one-to-one correspondence between a function call's arguments
116118 // and the registers used for those arguments. Any argument that doesn't fit in 8
......@@ -148,8 +150,9 @@ pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target) Class {
148150 },
149151
150152 .float => switch (ty.floatBits(target)) {
151 16, 32, 64, 128 => .sse,
153 16, 32, 64 => .sse,
152154 80 => .memory,
155 128 => if (ctx == .arg) .memory else .sse,
153156 else => unreachable,
154157 },
155158 .vector => .sse,
......@@ -166,8 +169,6 @@ pub fn classifyWindows(ty: Type, zcu: *Zcu, target: *const std.Target) Class {
166169 };
167170}
168171
169pub const Context = enum { ret, arg, other };
170
171172/// There are a maximum of 8 possible return slots. Returned values are in
172173/// the beginning of the array; unused slots are filled with .none.
173174pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) [8]Class {
src/dev.zig+2
......@@ -96,6 +96,7 @@ pub const Env = enum {
9696 .spirv_backend,
9797 .lld_linker,
9898 .coff_linker,
99 .coff2_linker,
99100 .elf_linker,
100101 .elf2_linker,
101102 .macho_linker,
......@@ -284,6 +285,7 @@ pub const Feature = enum {
284285
285286 lld_linker,
286287 coff_linker,
288 coff2_linker,
287289 elf_linker,
288290 elf2_linker,
289291 macho_linker,
src/link.zig+38-52
......@@ -574,16 +574,13 @@ pub const File = struct {
574574 const gpa = comp.gpa;
575575 switch (base.tag) {
576576 .lld => assert(base.file == null),
577 .coff, .elf, .macho, .wasm, .goff, .xcoff => {
577 .elf, .macho, .wasm, .goff, .xcoff => {
578578 if (base.file != null) return;
579579 dev.checkAny(&.{ .coff_linker, .elf_linker, .macho_linker, .plan9_linker, .wasm_linker, .goff_linker, .xcoff_linker });
580580 const emit = base.emit;
581581 if (base.child_pid) |pid| {
582582 if (builtin.os.tag == .windows) {
583 const coff_file = base.cast(.coff).?;
584 coff_file.ptraceAttach(pid) catch |err| {
585 log.warn("attaching failed with error: {s}", .{@errorName(err)});
586 };
583 return error.HotSwapUnavailableOnHostOperatingSystem;
587584 } else {
588585 // If we try to open the output file in write mode while it is running,
589586 // it will return ETXTBSY. So instead, we copy the file, atomically rename it
......@@ -610,27 +607,20 @@ pub const File = struct {
610607 }
611608 }
612609 }
613 const output_mode = comp.config.output_mode;
614 const link_mode = comp.config.link_mode;
615 base.file = try emit.root_dir.handle.createFile(emit.sub_path, .{
616 .truncate = false,
617 .read = true,
618 .mode = determineMode(output_mode, link_mode),
619 });
610 base.file = try emit.root_dir.handle.openFile(emit.sub_path, .{ .mode = .read_write });
620611 },
621 .elf2 => {
622 const elf = base.cast(.elf2).?;
623 if (base.file == null) {
624 elf.mf.file = try base.emit.root_dir.handle.createFile(base.emit.sub_path, .{
625 .truncate = false,
626 .read = true,
627 .mode = determineMode(comp.config.output_mode, comp.config.link_mode),
628 });
629 base.file = elf.mf.file;
630 try elf.mf.ensureTotalCapacity(
631 @intCast(elf.mf.nodes.items[0].location().resolve(&elf.mf)[1]),
632 );
633 }
612 .elf2, .coff2 => if (base.file == null) {
613 const mf = if (base.cast(.elf2)) |elf|
614 &elf.mf
615 else if (base.cast(.coff2)) |coff|
616 &coff.mf
617 else
618 unreachable;
619 mf.file = try base.emit.root_dir.handle.openFile(base.emit.sub_path, .{
620 .mode = .read_write,
621 });
622 base.file = mf.file;
623 try mf.ensureTotalCapacity(@intCast(mf.nodes.items[0].location().resolve(mf)[1]));
634624 },
635625 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),
636626 .plan9 => unreachable,
......@@ -654,12 +644,9 @@ pub const File = struct {
654644 pub fn makeExecutable(base: *File) !void {
655645 dev.check(.make_executable);
656646 const comp = base.comp;
657 const output_mode = comp.config.output_mode;
658 const link_mode = comp.config.link_mode;
659
660 switch (output_mode) {
647 switch (comp.config.output_mode) {
661648 .Obj => return,
662 .Lib => switch (link_mode) {
649 .Lib => switch (comp.config.link_mode) {
663650 .static => return,
664651 .dynamic => {},
665652 },
......@@ -681,7 +668,7 @@ pub const File = struct {
681668 }
682669 }
683670 },
684 .coff, .macho, .wasm, .goff, .xcoff => if (base.file) |f| {
671 .macho, .wasm, .goff, .xcoff => if (base.file) |f| {
685672 dev.checkAny(&.{ .coff_linker, .macho_linker, .plan9_linker, .wasm_linker, .goff_linker, .xcoff_linker });
686673 f.close();
687674 base.file = null;
......@@ -694,23 +681,22 @@ pub const File = struct {
694681 log.warn("detaching failed with error: {s}", .{@errorName(err)});
695682 };
696683 },
697 .windows => {
698 const coff_file = base.cast(.coff).?;
699 coff_file.ptraceDetach(pid);
700 },
701684 else => return error.HotSwapUnavailableOnHostOperatingSystem,
702685 }
703686 }
704687 },
705 .elf2 => {
706 const elf = base.cast(.elf2).?;
707 if (base.file) |f| {
708 elf.mf.unmap();
709 assert(elf.mf.file.handle == f.handle);
710 elf.mf.file = undefined;
711 f.close();
712 base.file = null;
713 }
688 .elf2, .coff2 => if (base.file) |f| {
689 const mf = if (base.cast(.elf2)) |elf|
690 &elf.mf
691 else if (base.cast(.coff2)) |coff|
692 &coff.mf
693 else
694 unreachable;
695 mf.unmap();
696 assert(mf.file.handle == f.handle);
697 mf.file = undefined;
698 f.close();
699 base.file = null;
714700 },
715701 .c, .spirv => dev.checkAny(&.{ .c_linker, .spirv_linker }),
716702 .plan9 => unreachable,
......@@ -828,7 +814,7 @@ pub const File = struct {
828814 .spirv => {},
829815 .goff, .xcoff => {},
830816 .plan9 => unreachable,
831 .elf2 => {},
817 .elf2, .coff2 => {},
832818 inline else => |tag| {
833819 dev.check(tag.devFeature());
834820 return @as(*tag.Type(), @fieldParentPtr("base", base)).updateLineNumber(pt, ti_id);
......@@ -864,7 +850,7 @@ pub const File = struct {
864850 pub fn idle(base: *File, tid: Zcu.PerThread.Id) !bool {
865851 switch (base.tag) {
866852 else => return false,
867 inline .elf2 => |tag| {
853 inline .elf2, .coff2 => |tag| {
868854 dev.check(tag.devFeature());
869855 return @as(*tag.Type(), @fieldParentPtr("base", base)).idle(tid);
870856 },
......@@ -874,7 +860,7 @@ pub const File = struct {
874860 pub fn updateErrorData(base: *File, pt: Zcu.PerThread) !void {
875861 switch (base.tag) {
876862 else => {},
877 inline .elf2 => |tag| {
863 inline .elf2, .coff2 => |tag| {
878864 dev.check(tag.devFeature());
879865 return @as(*tag.Type(), @fieldParentPtr("base", base)).updateErrorData(pt);
880866 },
......@@ -1155,7 +1141,7 @@ pub const File = struct {
11551141 if (base.zcu_object_basename != null) return;
11561142
11571143 switch (base.tag) {
1158 inline .elf2, .wasm => |tag| {
1144 inline .elf2, .coff2, .wasm => |tag| {
11591145 dev.check(tag.devFeature());
11601146 return @as(*tag.Type(), @fieldParentPtr("base", base)).prelink(base.comp.link_prog_node);
11611147 },
......@@ -1164,7 +1150,7 @@ pub const File = struct {
11641150 }
11651151
11661152 pub const Tag = enum {
1167 coff,
1153 coff2,
11681154 elf,
11691155 elf2,
11701156 macho,
......@@ -1178,7 +1164,7 @@ pub const File = struct {
11781164
11791165 pub fn Type(comptime tag: Tag) type {
11801166 return switch (tag) {
1181 .coff => Coff,
1167 .coff2 => Coff2,
11821168 .elf => Elf,
11831169 .elf2 => Elf2,
11841170 .macho => MachO,
......@@ -1194,7 +1180,7 @@ pub const File = struct {
11941180
11951181 fn fromObjectFormat(ofmt: std.Target.ObjectFormat, use_new_linker: bool) Tag {
11961182 return switch (ofmt) {
1197 .coff => .coff,
1183 .coff => .coff2,
11981184 .elf => if (use_new_linker) .elf2 else .elf,
11991185 .macho => .macho,
12001186 .wasm => .wasm,
......@@ -1279,7 +1265,7 @@ pub const File = struct {
12791265
12801266 pub const Lld = @import("link/Lld.zig");
12811267 pub const C = @import("link/C.zig");
1282 pub const Coff = @import("link/Coff.zig");
1268 pub const Coff2 = @import("link/Coff2.zig");
12831269 pub const Elf = @import("link/Elf.zig");
12841270 pub const Elf2 = @import("link/Elf2.zig");
12851271 pub const MachO = @import("link/MachO.zig");
src/link/Coff.zig deleted-3169
......@@ -1,3169 +0,0 @@
1//! The main driver of the self-hosted COFF linker.
2const Coff = @This();
3
4const std = @import("std");
5const build_options = @import("build_options");
6const builtin = @import("builtin");
7const assert = std.debug.assert;
8const coff_util = std.coff;
9const fmt = std.fmt;
10const fs = std.fs;
11const log = std.log.scoped(.link);
12const math = std.math;
13const mem = std.mem;
14
15const Allocator = std.mem.Allocator;
16const Path = std.Build.Cache.Path;
17const Directory = std.Build.Cache.Directory;
18const Cache = std.Build.Cache;
19
20const aarch64_util = link.aarch64;
21const allocPrint = std.fmt.allocPrint;
22const codegen = @import("../codegen.zig");
23const link = @import("../link.zig");
24const target_util = @import("../target.zig");
25const trace = @import("../tracy.zig").trace;
26
27const Compilation = @import("../Compilation.zig");
28const Zcu = @import("../Zcu.zig");
29const InternPool = @import("../InternPool.zig");
30const TableSection = @import("table_section.zig").TableSection;
31const StringTable = @import("StringTable.zig");
32const Type = @import("../Type.zig");
33const Value = @import("../Value.zig");
34const AnalUnit = InternPool.AnalUnit;
35const dev = @import("../dev.zig");
36
37base: link.File,
38image_base: u64,
39/// TODO this and minor_subsystem_version should be combined into one property and left as
40/// default or populated together. They should not be separate fields.
41major_subsystem_version: u16,
42minor_subsystem_version: u16,
43entry: link.File.OpenOptions.Entry,
44entry_addr: ?u32,
45module_definition_file: ?[]const u8,
46repro: bool,
47
48ptr_width: PtrWidth,
49page_size: u32,
50
51sections: std.MultiArrayList(Section) = .{},
52data_directories: [coff_util.IMAGE_NUMBEROF_DIRECTORY_ENTRIES]coff_util.ImageDataDirectory,
53
54text_section_index: ?u16 = null,
55got_section_index: ?u16 = null,
56rdata_section_index: ?u16 = null,
57data_section_index: ?u16 = null,
58reloc_section_index: ?u16 = null,
59idata_section_index: ?u16 = null,
60
61locals: std.ArrayListUnmanaged(coff_util.Symbol) = .empty,
62globals: std.ArrayListUnmanaged(SymbolWithLoc) = .empty,
63resolver: std.StringHashMapUnmanaged(u32) = .empty,
64unresolved: std.AutoArrayHashMapUnmanaged(u32, bool) = .empty,
65need_got_table: std.AutoHashMapUnmanaged(u32, void) = .empty,
66
67locals_free_list: std.ArrayListUnmanaged(u32) = .empty,
68globals_free_list: std.ArrayListUnmanaged(u32) = .empty,
69
70strtab: StringTable = .{},
71strtab_offset: ?u32 = null,
72
73temp_strtab: StringTable = .{},
74
75got_table: TableSection(SymbolWithLoc) = .{},
76
77/// A table of ImportTables partitioned by the library name.
78/// Key is an offset into the interning string table `temp_strtab`.
79import_tables: std.AutoArrayHashMapUnmanaged(u32, ImportTable) = .empty,
80
81got_table_count_dirty: bool = true,
82got_table_contents_dirty: bool = true,
83imports_count_dirty: bool = true,
84
85/// Table of tracked LazySymbols.
86lazy_syms: LazySymbolTable = .{},
87
88/// Table of tracked `Nav`s.
89navs: NavTable = .{},
90
91/// List of atoms that are either synthetic or map directly to the Zig source program.
92atoms: std.ArrayListUnmanaged(Atom) = .empty,
93
94/// Table of atoms indexed by the symbol index.
95atom_by_index_table: std.AutoHashMapUnmanaged(u32, Atom.Index) = .empty,
96
97uavs: UavTable = .{},
98
99/// A table of relocations indexed by the owning them `Atom`.
100/// Note that once we refactor `Atom`'s lifetime and ownership rules,
101/// this will be a table indexed by index into the list of Atoms.
102relocs: RelocTable = .{},
103
104/// A table of base relocations indexed by the owning them `Atom`.
105/// Note that once we refactor `Atom`'s lifetime and ownership rules,
106/// this will be a table indexed by index into the list of Atoms.
107base_relocs: BaseRelocationTable = .{},
108
109/// Hot-code swapping state.
110hot_state: if (is_hot_update_compatible) HotUpdateState else struct {} = .{},
111
112const is_hot_update_compatible = switch (builtin.target.os.tag) {
113 .windows => true,
114 else => false,
115};
116
117const HotUpdateState = struct {
118 /// Base address at which the process (image) got loaded.
119 /// We need this info to correctly slide pointers when relocating.
120 loaded_base_address: ?std.os.windows.HMODULE = null,
121};
122
123const NavTable = std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, AvMetadata);
124const UavTable = std.AutoHashMapUnmanaged(InternPool.Index, AvMetadata);
125const RelocTable = std.AutoArrayHashMapUnmanaged(Atom.Index, std.ArrayListUnmanaged(Relocation));
126const BaseRelocationTable = std.AutoArrayHashMapUnmanaged(Atom.Index, std.ArrayListUnmanaged(u32));
127
128const default_file_alignment: u16 = 0x200;
129const default_size_of_stack_reserve: u32 = 0x1000000;
130const default_size_of_stack_commit: u32 = 0x1000;
131const default_size_of_heap_reserve: u32 = 0x100000;
132const default_size_of_heap_commit: u32 = 0x1000;
133
134const Section = struct {
135 header: coff_util.SectionHeader,
136
137 last_atom_index: ?Atom.Index = null,
138
139 /// A list of atoms that have surplus capacity. This list can have false
140 /// positives, as functions grow and shrink over time, only sometimes being added
141 /// or removed from the freelist.
142 ///
143 /// An atom has surplus capacity when its overcapacity value is greater than
144 /// padToIdeal(minimum_atom_size). That is, when it has so
145 /// much extra capacity, that we could fit a small new symbol in it, itself with
146 /// ideal_capacity or more.
147 ///
148 /// Ideal capacity is defined by size + (size / ideal_factor).
149 ///
150 /// Overcapacity is measured by actual_capacity - ideal_capacity. Note that
151 /// overcapacity can be negative. A simple way to have negative overcapacity is to
152 /// allocate a fresh atom, which will have ideal capacity, and then grow it
153 /// by 1 byte. It will then have -1 overcapacity.
154 free_list: std.ArrayListUnmanaged(Atom.Index) = .empty,
155};
156
157const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata);
158
159const LazySymbolMetadata = struct {
160 const State = enum { unused, pending_flush, flushed };
161 text_atom: Atom.Index = undefined,
162 rdata_atom: Atom.Index = undefined,
163 text_state: State = .unused,
164 rdata_state: State = .unused,
165};
166
167const AvMetadata = struct {
168 atom: Atom.Index,
169 section: u16,
170 /// A list of all exports aliases of this Decl.
171 exports: std.ArrayListUnmanaged(u32) = .empty,
172
173 fn deinit(m: *AvMetadata, allocator: Allocator) void {
174 m.exports.deinit(allocator);
175 }
176
177 fn getExport(m: AvMetadata, coff: *const Coff, name: []const u8) ?u32 {
178 for (m.exports.items) |exp| {
179 if (mem.eql(u8, name, coff.getSymbolName(.{
180 .sym_index = exp,
181 .file = null,
182 }))) return exp;
183 }
184 return null;
185 }
186
187 fn getExportPtr(m: *AvMetadata, coff: *Coff, name: []const u8) ?*u32 {
188 for (m.exports.items) |*exp| {
189 if (mem.eql(u8, name, coff.getSymbolName(.{
190 .sym_index = exp.*,
191 .file = null,
192 }))) return exp;
193 }
194 return null;
195 }
196};
197
198pub const PtrWidth = enum {
199 p32,
200 p64,
201
202 /// Size in bytes.
203 pub fn size(pw: PtrWidth) u4 {
204 return switch (pw) {
205 .p32 => 4,
206 .p64 => 8,
207 };
208 }
209};
210
211pub const SymbolWithLoc = struct {
212 // Index into the respective symbol table.
213 sym_index: u32,
214
215 // null means it's a synthetic global or Zig source.
216 file: ?u32 = null,
217
218 pub fn eql(this: SymbolWithLoc, other: SymbolWithLoc) bool {
219 if (this.file == null and other.file == null) {
220 return this.sym_index == other.sym_index;
221 }
222 if (this.file != null and other.file != null) {
223 return this.sym_index == other.sym_index and this.file.? == other.file.?;
224 }
225 return false;
226 }
227};
228
229/// When allocating, the ideal_capacity is calculated by
230/// actual_capacity + (actual_capacity / ideal_factor)
231const ideal_factor = 3;
232
233/// In order for a slice of bytes to be considered eligible to keep metadata pointing at
234/// it as a possible place to put new symbols, it must have enough room for this many bytes
235/// (plus extra for reserved capacity).
236const minimum_text_block_size = 64;
237pub const min_text_capacity = padToIdeal(minimum_text_block_size);
238
239pub fn createEmpty(
240 arena: Allocator,
241 comp: *Compilation,
242 emit: Path,
243 options: link.File.OpenOptions,
244) !*Coff {
245 const target = &comp.root_mod.resolved_target.result;
246 assert(target.ofmt == .coff);
247 const optimize_mode = comp.root_mod.optimize_mode;
248 const output_mode = comp.config.output_mode;
249 const link_mode = comp.config.link_mode;
250 const use_llvm = comp.config.use_llvm;
251
252 const ptr_width: PtrWidth = switch (target.ptrBitWidth()) {
253 0...32 => .p32,
254 33...64 => .p64,
255 else => return error.UnsupportedCOFFArchitecture,
256 };
257 const page_size: u32 = switch (target.cpu.arch) {
258 else => 0x1000,
259 };
260
261 const coff = try arena.create(Coff);
262 coff.* = .{
263 .base = .{
264 .tag = .coff,
265 .comp = comp,
266 .emit = emit,
267 .zcu_object_basename = if (use_llvm)
268 try std.fmt.allocPrint(arena, "{s}_zcu.obj", .{fs.path.stem(emit.sub_path)})
269 else
270 null,
271 .stack_size = options.stack_size orelse 16777216,
272 .gc_sections = options.gc_sections orelse (optimize_mode != .Debug),
273 .print_gc_sections = options.print_gc_sections,
274 .allow_shlib_undefined = options.allow_shlib_undefined orelse false,
275 .file = null,
276 .build_id = options.build_id,
277 },
278 .ptr_width = ptr_width,
279 .page_size = page_size,
280
281 .data_directories = [1]coff_util.ImageDataDirectory{.{
282 .virtual_address = 0,
283 .size = 0,
284 }} ** coff_util.IMAGE_NUMBEROF_DIRECTORY_ENTRIES,
285
286 .image_base = options.image_base orelse switch (output_mode) {
287 .Exe => switch (target.cpu.arch) {
288 .aarch64, .x86_64 => 0x140000000,
289 .thumb, .x86 => 0x400000,
290 else => unreachable,
291 },
292 .Lib => switch (target.cpu.arch) {
293 .aarch64, .x86_64 => 0x180000000,
294 .thumb, .x86 => 0x10000000,
295 else => unreachable,
296 },
297 .Obj => 0,
298 },
299
300 .entry = options.entry,
301
302 .major_subsystem_version = options.major_subsystem_version orelse 6,
303 .minor_subsystem_version = options.minor_subsystem_version orelse 0,
304 .entry_addr = math.cast(u32, options.entry_addr orelse 0) orelse
305 return error.EntryAddressTooBig,
306 .module_definition_file = options.module_definition_file,
307 .repro = options.repro,
308 };
309 errdefer coff.base.destroy();
310
311 coff.base.file = try emit.root_dir.handle.createFile(emit.sub_path, .{
312 .truncate = true,
313 .read = true,
314 .mode = link.File.determineMode(output_mode, link_mode),
315 });
316
317 const gpa = comp.gpa;
318
319 try coff.strtab.buffer.ensureUnusedCapacity(gpa, @sizeOf(u32));
320 coff.strtab.buffer.appendNTimesAssumeCapacity(0, @sizeOf(u32));
321
322 try coff.temp_strtab.buffer.append(gpa, 0);
323
324 // Index 0 is always a null symbol.
325 try coff.locals.append(gpa, .{
326 .name = [_]u8{0} ** 8,
327 .value = 0,
328 .section_number = .UNDEFINED,
329 .type = .{ .base_type = .NULL, .complex_type = .NULL },
330 .storage_class = .NULL,
331 .number_of_aux_symbols = 0,
332 });
333
334 if (coff.text_section_index == null) {
335 const file_size: u32 = @intCast(options.program_code_size_hint);
336 coff.text_section_index = try coff.allocateSection(".text", file_size, .{
337 .CNT_CODE = 1,
338 .MEM_EXECUTE = 1,
339 .MEM_READ = 1,
340 });
341 }
342
343 if (coff.got_section_index == null) {
344 const file_size = @as(u32, @intCast(options.symbol_count_hint)) * coff.ptr_width.size();
345 coff.got_section_index = try coff.allocateSection(".got", file_size, .{
346 .CNT_INITIALIZED_DATA = 1,
347 .MEM_READ = 1,
348 });
349 }
350
351 if (coff.rdata_section_index == null) {
352 const file_size: u32 = coff.page_size;
353 coff.rdata_section_index = try coff.allocateSection(".rdata", file_size, .{
354 .CNT_INITIALIZED_DATA = 1,
355 .MEM_READ = 1,
356 });
357 }
358
359 if (coff.data_section_index == null) {
360 const file_size: u32 = coff.page_size;
361 coff.data_section_index = try coff.allocateSection(".data", file_size, .{
362 .CNT_INITIALIZED_DATA = 1,
363 .MEM_READ = 1,
364 .MEM_WRITE = 1,
365 });
366 }
367
368 if (coff.idata_section_index == null) {
369 const file_size = @as(u32, @intCast(options.symbol_count_hint)) * coff.ptr_width.size();
370 coff.idata_section_index = try coff.allocateSection(".idata", file_size, .{
371 .CNT_INITIALIZED_DATA = 1,
372 .MEM_READ = 1,
373 });
374 }
375
376 if (coff.reloc_section_index == null) {
377 const file_size = @as(u32, @intCast(options.symbol_count_hint)) * @sizeOf(coff_util.BaseRelocation);
378 coff.reloc_section_index = try coff.allocateSection(".reloc", file_size, .{
379 .CNT_INITIALIZED_DATA = 1,
380 .MEM_DISCARDABLE = 1,
381 .MEM_READ = 1,
382 });
383 }
384
385 if (coff.strtab_offset == null) {
386 const file_size = @as(u32, @intCast(coff.strtab.buffer.items.len));
387 coff.strtab_offset = coff.findFreeSpace(file_size, @alignOf(u32)); // 4bytes aligned seems like a good idea here
388 log.debug("found strtab free space 0x{x} to 0x{x}", .{ coff.strtab_offset.?, coff.strtab_offset.? + file_size });
389 }
390
391 {
392 // We need to find out what the max file offset is according to section headers.
393 // Otherwise, we may end up with an COFF binary with file size not matching the final section's
394 // offset + it's filesize.
395 // TODO I don't like this here one bit
396 var max_file_offset: u64 = 0;
397 for (coff.sections.items(.header)) |header| {
398 if (header.pointer_to_raw_data + header.size_of_raw_data > max_file_offset) {
399 max_file_offset = header.pointer_to_raw_data + header.size_of_raw_data;
400 }
401 }
402 try coff.pwriteAll(&[_]u8{0}, max_file_offset);
403 }
404
405 return coff;
406}
407
408pub fn open(
409 arena: Allocator,
410 comp: *Compilation,
411 emit: Path,
412 options: link.File.OpenOptions,
413) !*Coff {
414 // TODO: restore saved linker state, don't truncate the file, and
415 // participate in incremental compilation.
416 return createEmpty(arena, comp, emit, options);
417}
418
419pub fn deinit(coff: *Coff) void {
420 const gpa = coff.base.comp.gpa;
421
422 for (coff.sections.items(.free_list)) |*free_list| {
423 free_list.deinit(gpa);
424 }
425 coff.sections.deinit(gpa);
426
427 coff.atoms.deinit(gpa);
428 coff.locals.deinit(gpa);
429 coff.globals.deinit(gpa);
430
431 {
432 var it = coff.resolver.keyIterator();
433 while (it.next()) |key_ptr| {
434 gpa.free(key_ptr.*);
435 }
436 coff.resolver.deinit(gpa);
437 }
438
439 coff.unresolved.deinit(gpa);
440 coff.need_got_table.deinit(gpa);
441 coff.locals_free_list.deinit(gpa);
442 coff.globals_free_list.deinit(gpa);
443 coff.strtab.deinit(gpa);
444 coff.temp_strtab.deinit(gpa);
445 coff.got_table.deinit(gpa);
446
447 for (coff.import_tables.values()) |*itab| {
448 itab.deinit(gpa);
449 }
450 coff.import_tables.deinit(gpa);
451
452 coff.lazy_syms.deinit(gpa);
453
454 for (coff.navs.values()) |*metadata| {
455 metadata.deinit(gpa);
456 }
457 coff.navs.deinit(gpa);
458
459 coff.atom_by_index_table.deinit(gpa);
460
461 {
462 var it = coff.uavs.iterator();
463 while (it.next()) |entry| {
464 entry.value_ptr.exports.deinit(gpa);
465 }
466 coff.uavs.deinit(gpa);
467 }
468
469 for (coff.relocs.values()) |*relocs| {
470 relocs.deinit(gpa);
471 }
472 coff.relocs.deinit(gpa);
473
474 for (coff.base_relocs.values()) |*relocs| {
475 relocs.deinit(gpa);
476 }
477 coff.base_relocs.deinit(gpa);
478}
479
480fn allocateSection(coff: *Coff, name: []const u8, size: u32, flags: coff_util.SectionHeaderFlags) !u16 {
481 const index = @as(u16, @intCast(coff.sections.slice().len));
482 const off = coff.findFreeSpace(size, default_file_alignment);
483 // Memory is always allocated in sequence
484 // TODO: investigate if we can allocate .text last; this way it would never need to grow in memory!
485 const vaddr = blk: {
486 if (index == 0) break :blk coff.page_size;
487 const prev_header = coff.sections.items(.header)[index - 1];
488 break :blk mem.alignForward(u32, prev_header.virtual_address + prev_header.virtual_size, coff.page_size);
489 };
490 // We commit more memory than needed upfront so that we don't have to reallocate too soon.
491 const memsz = mem.alignForward(u32, size, coff.page_size) * 100;
492 log.debug("found {s} free space 0x{x} to 0x{x} (0x{x} - 0x{x})", .{
493 name,
494 off,
495 off + size,
496 vaddr,
497 vaddr + size,
498 });
499 var header = coff_util.SectionHeader{
500 .name = undefined,
501 .virtual_size = memsz,
502 .virtual_address = vaddr,
503 .size_of_raw_data = size,
504 .pointer_to_raw_data = off,
505 .pointer_to_relocations = 0,
506 .pointer_to_linenumbers = 0,
507 .number_of_relocations = 0,
508 .number_of_linenumbers = 0,
509 .flags = flags,
510 };
511 const gpa = coff.base.comp.gpa;
512 try coff.setSectionName(&header, name);
513 try coff.sections.append(gpa, .{ .header = header });
514 return index;
515}
516
517fn growSection(coff: *Coff, sect_id: u32, needed_size: u32) !void {
518 const header = &coff.sections.items(.header)[sect_id];
519 const maybe_last_atom_index = coff.sections.items(.last_atom_index)[sect_id];
520 const sect_capacity = coff.allocatedSize(header.pointer_to_raw_data);
521
522 if (needed_size > sect_capacity) {
523 const new_offset = coff.findFreeSpace(needed_size, default_file_alignment);
524 const current_size = if (maybe_last_atom_index) |last_atom_index| blk: {
525 const last_atom = coff.getAtom(last_atom_index);
526 const sym = last_atom.getSymbol(coff);
527 break :blk (sym.value + last_atom.size) - header.virtual_address;
528 } else 0;
529 log.debug("moving {s} from 0x{x} to 0x{x}", .{
530 coff.getSectionName(header),
531 header.pointer_to_raw_data,
532 new_offset,
533 });
534 const amt = try coff.base.file.?.copyRangeAll(
535 header.pointer_to_raw_data,
536 coff.base.file.?,
537 new_offset,
538 current_size,
539 );
540 if (amt != current_size) return error.InputOutput;
541 header.pointer_to_raw_data = new_offset;
542 }
543
544 const sect_vm_capacity = coff.allocatedVirtualSize(header.virtual_address);
545 if (needed_size > sect_vm_capacity) {
546 coff.markRelocsDirtyByAddress(header.virtual_address + header.virtual_size);
547 try coff.growSectionVirtualMemory(sect_id, needed_size);
548 }
549
550 header.virtual_size = @max(header.virtual_size, needed_size);
551 header.size_of_raw_data = needed_size;
552}
553
554fn growSectionVirtualMemory(coff: *Coff, sect_id: u32, needed_size: u32) !void {
555 const header = &coff.sections.items(.header)[sect_id];
556 const increased_size = padToIdeal(needed_size);
557 const old_aligned_end = header.virtual_address + mem.alignForward(u32, header.virtual_size, coff.page_size);
558 const new_aligned_end = header.virtual_address + mem.alignForward(u32, increased_size, coff.page_size);
559 const diff = new_aligned_end - old_aligned_end;
560 log.debug("growing {s} in virtual memory by {x}", .{ coff.getSectionName(header), diff });
561
562 // TODO: enforce order by increasing VM addresses in coff.sections container.
563 // This is required by the loader anyhow as far as I can tell.
564 for (coff.sections.items(.header)[sect_id + 1 ..], 0..) |*next_header, next_sect_id| {
565 const maybe_last_atom_index = coff.sections.items(.last_atom_index)[sect_id + 1 + next_sect_id];
566 next_header.virtual_address += diff;
567
568 if (maybe_last_atom_index) |last_atom_index| {
569 var atom_index = last_atom_index;
570 while (true) {
571 const atom = coff.getAtom(atom_index);
572 const sym = atom.getSymbolPtr(coff);
573 sym.value += diff;
574
575 if (atom.prev_index) |prev_index| {
576 atom_index = prev_index;
577 } else break;
578 }
579 }
580 }
581
582 header.virtual_size = increased_size;
583}
584
585fn allocateAtom(coff: *Coff, atom_index: Atom.Index, new_atom_size: u32, alignment: u32) !u32 {
586 const tracy = trace(@src());
587 defer tracy.end();
588
589 const atom = coff.getAtom(atom_index);
590 const sect_id = @intFromEnum(atom.getSymbol(coff).section_number) - 1;
591 const header = &coff.sections.items(.header)[sect_id];
592 const free_list = &coff.sections.items(.free_list)[sect_id];
593 const maybe_last_atom_index = &coff.sections.items(.last_atom_index)[sect_id];
594 const new_atom_ideal_capacity = if (header.isCode()) padToIdeal(new_atom_size) else new_atom_size;
595
596 // We use these to indicate our intention to update metadata, placing the new atom,
597 // and possibly removing a free list node.
598 // It would be simpler to do it inside the for loop below, but that would cause a
599 // problem if an error was returned later in the function. So this action
600 // is actually carried out at the end of the function, when errors are no longer possible.
601 var atom_placement: ?Atom.Index = null;
602 var free_list_removal: ?usize = null;
603
604 // First we look for an appropriately sized free list node.
605 // The list is unordered. We'll just take the first thing that works.
606 const vaddr = blk: {
607 var i: usize = 0;
608 while (i < free_list.items.len) {
609 const big_atom_index = free_list.items[i];
610 const big_atom = coff.getAtom(big_atom_index);
611 // We now have a pointer to a live atom that has too much capacity.
612 // Is it enough that we could fit this new atom?
613 const sym = big_atom.getSymbol(coff);
614 const capacity = big_atom.capacity(coff);
615 const ideal_capacity = if (header.isCode()) padToIdeal(capacity) else capacity;
616 const ideal_capacity_end_vaddr = math.add(u32, sym.value, ideal_capacity) catch ideal_capacity;
617 const capacity_end_vaddr = sym.value + capacity;
618 const new_start_vaddr_unaligned = capacity_end_vaddr - new_atom_ideal_capacity;
619 const new_start_vaddr = mem.alignBackward(u32, new_start_vaddr_unaligned, alignment);
620 if (new_start_vaddr < ideal_capacity_end_vaddr) {
621 // Additional bookkeeping here to notice if this free list node
622 // should be deleted because the atom that it points to has grown to take up
623 // more of the extra capacity.
624 if (!big_atom.freeListEligible(coff)) {
625 _ = free_list.swapRemove(i);
626 } else {
627 i += 1;
628 }
629 continue;
630 }
631 // At this point we know that we will place the new atom here. But the
632 // remaining question is whether there is still yet enough capacity left
633 // over for there to still be a free list node.
634 const remaining_capacity = new_start_vaddr - ideal_capacity_end_vaddr;
635 const keep_free_list_node = remaining_capacity >= min_text_capacity;
636
637 // Set up the metadata to be updated, after errors are no longer possible.
638 atom_placement = big_atom_index;
639 if (!keep_free_list_node) {
640 free_list_removal = i;
641 }
642 break :blk new_start_vaddr;
643 } else if (maybe_last_atom_index.*) |last_index| {
644 const last = coff.getAtom(last_index);
645 const last_symbol = last.getSymbol(coff);
646 const ideal_capacity = if (header.isCode()) padToIdeal(last.size) else last.size;
647 const ideal_capacity_end_vaddr = last_symbol.value + ideal_capacity;
648 const new_start_vaddr = mem.alignForward(u32, ideal_capacity_end_vaddr, alignment);
649 atom_placement = last_index;
650 break :blk new_start_vaddr;
651 } else {
652 break :blk mem.alignForward(u32, header.virtual_address, alignment);
653 }
654 };
655
656 const expand_section = if (atom_placement) |placement_index|
657 coff.getAtom(placement_index).next_index == null
658 else
659 true;
660 if (expand_section) {
661 const needed_size: u32 = (vaddr + new_atom_size) - header.virtual_address;
662 try coff.growSection(sect_id, needed_size);
663 maybe_last_atom_index.* = atom_index;
664 }
665 coff.getAtomPtr(atom_index).size = new_atom_size;
666
667 if (atom.prev_index) |prev_index| {
668 const prev = coff.getAtomPtr(prev_index);
669 prev.next_index = atom.next_index;
670 }
671 if (atom.next_index) |next_index| {
672 const next = coff.getAtomPtr(next_index);
673 next.prev_index = atom.prev_index;
674 }
675
676 if (atom_placement) |big_atom_index| {
677 const big_atom = coff.getAtomPtr(big_atom_index);
678 const atom_ptr = coff.getAtomPtr(atom_index);
679 atom_ptr.prev_index = big_atom_index;
680 atom_ptr.next_index = big_atom.next_index;
681 big_atom.next_index = atom_index;
682 } else {
683 const atom_ptr = coff.getAtomPtr(atom_index);
684 atom_ptr.prev_index = null;
685 atom_ptr.next_index = null;
686 }
687 if (free_list_removal) |i| {
688 _ = free_list.swapRemove(i);
689 }
690
691 return vaddr;
692}
693
694pub fn allocateSymbol(coff: *Coff) !u32 {
695 const gpa = coff.base.comp.gpa;
696 try coff.locals.ensureUnusedCapacity(gpa, 1);
697
698 const index = blk: {
699 if (coff.locals_free_list.pop()) |index| {
700 log.debug(" (reusing symbol index {d})", .{index});
701 break :blk index;
702 } else {
703 log.debug(" (allocating symbol index {d})", .{coff.locals.items.len});
704 const index = @as(u32, @intCast(coff.locals.items.len));
705 _ = coff.locals.addOneAssumeCapacity();
706 break :blk index;
707 }
708 };
709
710 coff.locals.items[index] = .{
711 .name = [_]u8{0} ** 8,
712 .value = 0,
713 .section_number = .UNDEFINED,
714 .type = .{ .base_type = .NULL, .complex_type = .NULL },
715 .storage_class = .NULL,
716 .number_of_aux_symbols = 0,
717 };
718
719 return index;
720}
721
722fn allocateGlobal(coff: *Coff) !u32 {
723 const gpa = coff.base.comp.gpa;
724 try coff.globals.ensureUnusedCapacity(gpa, 1);
725
726 const index = blk: {
727 if (coff.globals_free_list.pop()) |index| {
728 log.debug(" (reusing global index {d})", .{index});
729 break :blk index;
730 } else {
731 log.debug(" (allocating global index {d})", .{coff.globals.items.len});
732 const index = @as(u32, @intCast(coff.globals.items.len));
733 _ = coff.globals.addOneAssumeCapacity();
734 break :blk index;
735 }
736 };
737
738 coff.globals.items[index] = .{
739 .sym_index = 0,
740 .file = null,
741 };
742
743 return index;
744}
745
746fn addGotEntry(coff: *Coff, target: SymbolWithLoc) !void {
747 const gpa = coff.base.comp.gpa;
748 if (coff.got_table.lookup.contains(target)) return;
749 const got_index = try coff.got_table.allocateEntry(gpa, target);
750 try coff.writeOffsetTableEntry(got_index);
751 coff.got_table_count_dirty = true;
752 coff.markRelocsDirtyByTarget(target);
753}
754
755pub fn createAtom(coff: *Coff) !Atom.Index {
756 const gpa = coff.base.comp.gpa;
757 const atom_index = @as(Atom.Index, @intCast(coff.atoms.items.len));
758 const atom = try coff.atoms.addOne(gpa);
759 const sym_index = try coff.allocateSymbol();
760 try coff.atom_by_index_table.putNoClobber(gpa, sym_index, atom_index);
761 atom.* = .{
762 .sym_index = sym_index,
763 .file = null,
764 .size = 0,
765 .prev_index = null,
766 .next_index = null,
767 };
768 log.debug("creating ATOM(%{d}) at index {d}", .{ sym_index, atom_index });
769 return atom_index;
770}
771
772fn growAtom(coff: *Coff, atom_index: Atom.Index, new_atom_size: u32, alignment: u32) !u32 {
773 const atom = coff.getAtom(atom_index);
774 const sym = atom.getSymbol(coff);
775 const align_ok = mem.alignBackward(u32, sym.value, alignment) == sym.value;
776 const need_realloc = !align_ok or new_atom_size > atom.capacity(coff);
777 if (!need_realloc) return sym.value;
778 return coff.allocateAtom(atom_index, new_atom_size, alignment);
779}
780
781fn shrinkAtom(coff: *Coff, atom_index: Atom.Index, new_block_size: u32) void {
782 _ = coff;
783 _ = atom_index;
784 _ = new_block_size;
785 // TODO check the new capacity, and if it crosses the size threshold into a big enough
786 // capacity, insert a free list node for it.
787}
788
789fn writeAtom(coff: *Coff, atom_index: Atom.Index, code: []u8, resolve_relocs: bool) !void {
790 const atom = coff.getAtom(atom_index);
791 const sym = atom.getSymbol(coff);
792 const section = coff.sections.get(@intFromEnum(sym.section_number) - 1);
793 const file_offset = section.header.pointer_to_raw_data + sym.value - section.header.virtual_address;
794
795 log.debug("writing atom for symbol {s} at file offset 0x{x} to 0x{x}", .{
796 atom.getName(coff),
797 file_offset,
798 file_offset + code.len,
799 });
800
801 const gpa = coff.base.comp.gpa;
802
803 // Gather relocs which can be resolved.
804 // We need to do this as we will be applying different slide values depending
805 // if we are running in hot-code swapping mode or not.
806 // TODO: how crazy would it be to try and apply the actual image base of the loaded
807 // process for the in-file values rather than the Windows defaults?
808 var relocs = std.array_list.Managed(*Relocation).init(gpa);
809 defer relocs.deinit();
810
811 if (resolve_relocs) {
812 if (coff.relocs.getPtr(atom_index)) |rels| {
813 try relocs.ensureTotalCapacityPrecise(rels.items.len);
814 for (rels.items) |*reloc| {
815 if (reloc.isResolvable(coff) and reloc.dirty) {
816 relocs.appendAssumeCapacity(reloc);
817 }
818 }
819 }
820 }
821
822 if (is_hot_update_compatible) {
823 if (coff.base.child_pid) |handle| {
824 const slide = @intFromPtr(coff.hot_state.loaded_base_address.?);
825
826 const mem_code = try gpa.dupe(u8, code);
827 defer gpa.free(mem_code);
828 coff.resolveRelocs(atom_index, relocs.items, mem_code, slide);
829
830 const vaddr = sym.value + slide;
831 const pvaddr = @as(*anyopaque, @ptrFromInt(vaddr));
832
833 log.debug("writing to memory at address {x}", .{vaddr});
834
835 if (build_options.enable_logging) {
836 try debugMem(gpa, handle, pvaddr, mem_code);
837 }
838
839 if (section.header.flags.MEM_WRITE == 0) {
840 writeMemProtected(handle, pvaddr, mem_code) catch |err| {
841 log.warn("writing to protected memory failed with error: {s}", .{@errorName(err)});
842 };
843 } else {
844 writeMem(handle, pvaddr, mem_code) catch |err| {
845 log.warn("writing to protected memory failed with error: {s}", .{@errorName(err)});
846 };
847 }
848 }
849 }
850
851 if (resolve_relocs) {
852 coff.resolveRelocs(atom_index, relocs.items, code, coff.image_base);
853 }
854 try coff.pwriteAll(code, file_offset);
855 if (resolve_relocs) {
856 // Now we can mark the relocs as resolved.
857 while (relocs.pop()) |reloc| {
858 reloc.dirty = false;
859 }
860 }
861}
862
863fn debugMem(allocator: Allocator, handle: std.process.Child.Id, pvaddr: std.os.windows.LPVOID, code: []const u8) !void {
864 const buffer = try allocator.alloc(u8, code.len);
865 defer allocator.free(buffer);
866 const memread = try std.os.windows.ReadProcessMemory(handle, pvaddr, buffer);
867 log.debug("to write: {x}", .{code});
868 log.debug("in memory: {x}", .{memread});
869}
870
871fn writeMemProtected(handle: std.process.Child.Id, pvaddr: std.os.windows.LPVOID, code: []const u8) !void {
872 const old_prot = try std.os.windows.VirtualProtectEx(handle, pvaddr, code.len, std.os.windows.PAGE_EXECUTE_WRITECOPY);
873 try writeMem(handle, pvaddr, code);
874 // TODO: We can probably just set the pages writeable and leave it at that without having to restore the attributes.
875 // For that though, we want to track which page has already been modified.
876 _ = try std.os.windows.VirtualProtectEx(handle, pvaddr, code.len, old_prot);
877}
878
879fn writeMem(handle: std.process.Child.Id, pvaddr: std.os.windows.LPVOID, code: []const u8) !void {
880 const amt = try std.os.windows.WriteProcessMemory(handle, pvaddr, code);
881 if (amt != code.len) return error.InputOutput;
882}
883
884fn writeOffsetTableEntry(coff: *Coff, index: usize) !void {
885 const sect_id = coff.got_section_index.?;
886
887 if (coff.got_table_count_dirty) {
888 const needed_size: u32 = @intCast(coff.got_table.entries.items.len * coff.ptr_width.size());
889 try coff.growSection(sect_id, needed_size);
890 coff.got_table_count_dirty = false;
891 }
892
893 const header = &coff.sections.items(.header)[sect_id];
894 const entry = coff.got_table.entries.items[index];
895 const entry_value = coff.getSymbol(entry).value;
896 const entry_offset = index * coff.ptr_width.size();
897 const file_offset = header.pointer_to_raw_data + entry_offset;
898 const vmaddr = header.virtual_address + entry_offset;
899
900 log.debug("writing GOT entry {d}: @{x} => {x}", .{ index, vmaddr, entry_value + coff.image_base });
901
902 switch (coff.ptr_width) {
903 .p32 => {
904 var buf: [4]u8 = undefined;
905 mem.writeInt(u32, &buf, @intCast(entry_value + coff.image_base), .little);
906 try coff.base.file.?.pwriteAll(&buf, file_offset);
907 },
908 .p64 => {
909 var buf: [8]u8 = undefined;
910 mem.writeInt(u64, &buf, entry_value + coff.image_base, .little);
911 try coff.base.file.?.pwriteAll(&buf, file_offset);
912 },
913 }
914
915 if (is_hot_update_compatible) {
916 if (coff.base.child_pid) |handle| {
917 const gpa = coff.base.comp.gpa;
918 const slide = @intFromPtr(coff.hot_state.loaded_base_address.?);
919 const actual_vmaddr = vmaddr + slide;
920 const pvaddr = @as(*anyopaque, @ptrFromInt(actual_vmaddr));
921 log.debug("writing GOT entry to memory at address {x}", .{actual_vmaddr});
922 if (build_options.enable_logging) {
923 switch (coff.ptr_width) {
924 .p32 => {
925 var buf: [4]u8 = undefined;
926 try debugMem(gpa, handle, pvaddr, &buf);
927 },
928 .p64 => {
929 var buf: [8]u8 = undefined;
930 try debugMem(gpa, handle, pvaddr, &buf);
931 },
932 }
933 }
934
935 switch (coff.ptr_width) {
936 .p32 => {
937 var buf: [4]u8 = undefined;
938 mem.writeInt(u32, &buf, @as(u32, @intCast(entry_value + slide)), .little);
939 writeMem(handle, pvaddr, &buf) catch |err| {
940 log.warn("writing to protected memory failed with error: {s}", .{@errorName(err)});
941 };
942 },
943 .p64 => {
944 var buf: [8]u8 = undefined;
945 mem.writeInt(u64, &buf, entry_value + slide, .little);
946 writeMem(handle, pvaddr, &buf) catch |err| {
947 log.warn("writing to protected memory failed with error: {s}", .{@errorName(err)});
948 };
949 },
950 }
951 }
952 }
953}
954
955fn markRelocsDirtyByTarget(coff: *Coff, target: SymbolWithLoc) void {
956 if (!coff.base.comp.config.incremental) return;
957 // TODO: reverse-lookup might come in handy here
958 for (coff.relocs.values()) |*relocs| {
959 for (relocs.items) |*reloc| {
960 if (!reloc.target.eql(target)) continue;
961 reloc.dirty = true;
962 }
963 }
964}
965
966fn markRelocsDirtyByAddress(coff: *Coff, addr: u32) void {
967 if (!coff.base.comp.config.incremental) return;
968 const got_moved = blk: {
969 const sect_id = coff.got_section_index orelse break :blk false;
970 break :blk coff.sections.items(.header)[sect_id].virtual_address >= addr;
971 };
972
973 // TODO: dirty relocations targeting import table if that got moved in memory
974
975 for (coff.relocs.values()) |*relocs| {
976 for (relocs.items) |*reloc| {
977 if (reloc.isGotIndirection()) {
978 reloc.dirty = reloc.dirty or got_moved;
979 } else {
980 const target_vaddr = reloc.getTargetAddress(coff) orelse continue;
981 if (target_vaddr >= addr) reloc.dirty = true;
982 }
983 }
984 }
985
986 // TODO: dirty only really affected GOT cells
987 for (coff.got_table.entries.items) |entry| {
988 const target_addr = coff.getSymbol(entry).value;
989 if (target_addr >= addr) {
990 coff.got_table_contents_dirty = true;
991 break;
992 }
993 }
994}
995
996fn resolveRelocs(coff: *Coff, atom_index: Atom.Index, relocs: []const *const Relocation, code: []u8, image_base: u64) void {
997 log.debug("relocating '{s}'", .{coff.getAtom(atom_index).getName(coff)});
998 for (relocs) |reloc| {
999 reloc.resolve(atom_index, code, image_base, coff);
1000 }
1001}
1002
1003pub fn ptraceAttach(coff: *Coff, handle: std.process.Child.Id) !void {
1004 if (!is_hot_update_compatible) return;
1005
1006 log.debug("attaching to process with handle {*}", .{handle});
1007 coff.hot_state.loaded_base_address = std.os.windows.ProcessBaseAddress(handle) catch |err| {
1008 log.warn("failed to get base address for the process with error: {s}", .{@errorName(err)});
1009 return;
1010 };
1011}
1012
1013pub fn ptraceDetach(coff: *Coff, handle: std.process.Child.Id) void {
1014 if (!is_hot_update_compatible) return;
1015
1016 log.debug("detaching from process with handle {*}", .{handle});
1017 coff.hot_state.loaded_base_address = null;
1018}
1019
1020fn freeAtom(coff: *Coff, atom_index: Atom.Index) void {
1021 log.debug("freeAtom {d}", .{atom_index});
1022
1023 const gpa = coff.base.comp.gpa;
1024
1025 // Remove any relocs and base relocs associated with this Atom
1026 coff.freeRelocations(atom_index);
1027
1028 const atom = coff.getAtom(atom_index);
1029 const sym = atom.getSymbol(coff);
1030 const sect_id = @intFromEnum(sym.section_number) - 1;
1031 const free_list = &coff.sections.items(.free_list)[sect_id];
1032 var already_have_free_list_node = false;
1033 {
1034 var i: usize = 0;
1035 // TODO turn free_list into a hash map
1036 while (i < free_list.items.len) {
1037 if (free_list.items[i] == atom_index) {
1038 _ = free_list.swapRemove(i);
1039 continue;
1040 }
1041 if (free_list.items[i] == atom.prev_index) {
1042 already_have_free_list_node = true;
1043 }
1044 i += 1;
1045 }
1046 }
1047
1048 const maybe_last_atom_index = &coff.sections.items(.last_atom_index)[sect_id];
1049 if (maybe_last_atom_index.*) |last_atom_index| {
1050 if (last_atom_index == atom_index) {
1051 if (atom.prev_index) |prev_index| {
1052 // TODO shrink the section size here
1053 maybe_last_atom_index.* = prev_index;
1054 } else {
1055 maybe_last_atom_index.* = null;
1056 }
1057 }
1058 }
1059
1060 if (atom.prev_index) |prev_index| {
1061 const prev = coff.getAtomPtr(prev_index);
1062 prev.next_index = atom.next_index;
1063
1064 if (!already_have_free_list_node and prev.*.freeListEligible(coff)) {
1065 // The free list is heuristics, it doesn't have to be perfect, so we can
1066 // ignore the OOM here.
1067 free_list.append(gpa, prev_index) catch {};
1068 }
1069 } else {
1070 coff.getAtomPtr(atom_index).prev_index = null;
1071 }
1072
1073 if (atom.next_index) |next_index| {
1074 coff.getAtomPtr(next_index).prev_index = atom.prev_index;
1075 } else {
1076 coff.getAtomPtr(atom_index).next_index = null;
1077 }
1078
1079 // Appending to free lists is allowed to fail because the free lists are heuristics based anyway.
1080 const sym_index = atom.getSymbolIndex().?;
1081 coff.locals_free_list.append(gpa, sym_index) catch {};
1082
1083 // Try freeing GOT atom if this decl had one
1084 coff.got_table.freeEntry(gpa, .{ .sym_index = sym_index });
1085
1086 coff.locals.items[sym_index].section_number = .UNDEFINED;
1087 _ = coff.atom_by_index_table.remove(sym_index);
1088 log.debug(" adding local symbol index {d} to free list", .{sym_index});
1089 coff.getAtomPtr(atom_index).sym_index = 0;
1090}
1091
1092pub fn updateFunc(
1093 coff: *Coff,
1094 pt: Zcu.PerThread,
1095 func_index: InternPool.Index,
1096 mir: *const codegen.AnyMir,
1097) link.File.UpdateNavError!void {
1098 if (build_options.skip_non_native and builtin.object_format != .coff) {
1099 @panic("Attempted to compile for object format that was disabled by build configuration");
1100 }
1101 const tracy = trace(@src());
1102 defer tracy.end();
1103
1104 const zcu = pt.zcu;
1105 const gpa = zcu.gpa;
1106 const func = zcu.funcInfo(func_index);
1107 const nav_index = func.owner_nav;
1108
1109 const atom_index = try coff.getOrCreateAtomForNav(nav_index);
1110 coff.freeRelocations(atom_index);
1111
1112 coff.navs.getPtr(func.owner_nav).?.section = coff.text_section_index.?;
1113
1114 var aw: std.Io.Writer.Allocating = .init(gpa);
1115 defer aw.deinit();
1116
1117 codegen.emitFunction(
1118 &coff.base,
1119 pt,
1120 zcu.navSrcLoc(nav_index),
1121 func_index,
1122 coff.getAtom(atom_index).getSymbolIndex().?,
1123 mir,
1124 &aw.writer,
1125 .none,
1126 ) catch |err| switch (err) {
1127 error.WriteFailed => return error.OutOfMemory,
1128 else => |e| return e,
1129 };
1130
1131 try coff.updateNavCode(pt, nav_index, aw.written(), .FUNCTION);
1132
1133 // Exports will be updated by `Zcu.processExports` after the update.
1134}
1135
1136const LowerConstResult = union(enum) {
1137 ok: Atom.Index,
1138 fail: *Zcu.ErrorMsg,
1139};
1140
1141fn lowerConst(
1142 coff: *Coff,
1143 pt: Zcu.PerThread,
1144 name: []const u8,
1145 val: Value,
1146 required_alignment: InternPool.Alignment,
1147 sect_id: u16,
1148 src_loc: Zcu.LazySrcLoc,
1149) !LowerConstResult {
1150 const gpa = coff.base.comp.gpa;
1151
1152 var aw: std.Io.Writer.Allocating = .init(gpa);
1153 defer aw.deinit();
1154
1155 const atom_index = try coff.createAtom();
1156 const sym = coff.getAtom(atom_index).getSymbolPtr(coff);
1157 try coff.setSymbolName(sym, name);
1158 sym.section_number = @as(coff_util.SectionNumber, @enumFromInt(sect_id + 1));
1159
1160 try codegen.generateSymbol(&coff.base, pt, src_loc, val, &aw.writer, .{
1161 .atom_index = coff.getAtom(atom_index).getSymbolIndex().?,
1162 });
1163 const code = aw.written();
1164
1165 const atom = coff.getAtomPtr(atom_index);
1166 atom.size = @intCast(code.len);
1167 atom.getSymbolPtr(coff).value = try coff.allocateAtom(
1168 atom_index,
1169 atom.size,
1170 @intCast(required_alignment.toByteUnits().?),
1171 );
1172 errdefer coff.freeAtom(atom_index);
1173
1174 log.debug("allocated atom for {s} at 0x{x}", .{ name, atom.getSymbol(coff).value });
1175 log.debug(" (required alignment 0x{x})", .{required_alignment});
1176
1177 try coff.writeAtom(atom_index, code, coff.base.comp.config.incremental);
1178
1179 return .{ .ok = atom_index };
1180}
1181
1182pub fn updateNav(
1183 coff: *Coff,
1184 pt: Zcu.PerThread,
1185 nav_index: InternPool.Nav.Index,
1186) link.File.UpdateNavError!void {
1187 if (build_options.skip_non_native and builtin.object_format != .coff) {
1188 @panic("Attempted to compile for object format that was disabled by build configuration");
1189 }
1190 const tracy = trace(@src());
1191 defer tracy.end();
1192
1193 const zcu = pt.zcu;
1194 const gpa = zcu.gpa;
1195 const ip = &zcu.intern_pool;
1196 const nav = ip.getNav(nav_index);
1197
1198 const nav_val = zcu.navValue(nav_index);
1199 const nav_init = switch (ip.indexToKey(nav_val.toIntern())) {
1200 .func => return,
1201 .variable => |variable| Value.fromInterned(variable.init),
1202 .@"extern" => |@"extern"| {
1203 if (ip.isFunctionType(@"extern".ty)) return;
1204 // TODO make this part of getGlobalSymbol
1205 const name = nav.name.toSlice(ip);
1206 const lib_name = @"extern".lib_name.toSlice(ip);
1207 const global_index = try coff.getGlobalSymbol(name, lib_name);
1208 try coff.need_got_table.put(gpa, global_index, {});
1209 return;
1210 },
1211 else => nav_val,
1212 };
1213
1214 if (nav_init.typeOf(zcu).hasRuntimeBits(zcu)) {
1215 const atom_index = try coff.getOrCreateAtomForNav(nav_index);
1216 coff.freeRelocations(atom_index);
1217 const atom = coff.getAtom(atom_index);
1218
1219 coff.navs.getPtr(nav_index).?.section = coff.getNavOutputSection(nav_index);
1220
1221 var aw: std.Io.Writer.Allocating = .init(gpa);
1222 defer aw.deinit();
1223
1224 codegen.generateSymbol(
1225 &coff.base,
1226 pt,
1227 zcu.navSrcLoc(nav_index),
1228 nav_init,
1229 &aw.writer,
1230 .{ .atom_index = atom.getSymbolIndex().? },
1231 ) catch |err| switch (err) {
1232 error.WriteFailed => return error.OutOfMemory,
1233 else => |e| return e,
1234 };
1235
1236 try coff.updateNavCode(pt, nav_index, aw.written(), .NULL);
1237 }
1238
1239 // Exports will be updated by `Zcu.processExports` after the update.
1240}
1241
1242fn updateLazySymbolAtom(
1243 coff: *Coff,
1244 pt: Zcu.PerThread,
1245 sym: link.File.LazySymbol,
1246 atom_index: Atom.Index,
1247 section_index: u16,
1248) !void {
1249 const zcu = pt.zcu;
1250 const comp = coff.base.comp;
1251 const gpa = comp.gpa;
1252
1253 var required_alignment: InternPool.Alignment = .none;
1254 var aw: std.Io.Writer.Allocating = .init(gpa);
1255 defer aw.deinit();
1256
1257 const name = try allocPrint(gpa, "__lazy_{s}_{f}", .{
1258 @tagName(sym.kind),
1259 Type.fromInterned(sym.ty).fmt(pt),
1260 });
1261 defer gpa.free(name);
1262
1263 const local_sym_index = coff.getAtomPtr(atom_index).getSymbolIndex().?;
1264
1265 const src = Type.fromInterned(sym.ty).srcLocOrNull(zcu) orelse Zcu.LazySrcLoc.unneeded;
1266 try codegen.generateLazySymbol(
1267 &coff.base,
1268 pt,
1269 src,
1270 sym,
1271 &required_alignment,
1272 &aw.writer,
1273 .none,
1274 .{ .atom_index = local_sym_index },
1275 );
1276 const code = aw.written();
1277
1278 const atom = coff.getAtomPtr(atom_index);
1279 const symbol = atom.getSymbolPtr(coff);
1280 try coff.setSymbolName(symbol, name);
1281 symbol.section_number = @enumFromInt(section_index + 1);
1282 symbol.type = .{ .complex_type = .NULL, .base_type = .NULL };
1283
1284 const code_len: u32 = @intCast(code.len);
1285 const vaddr = try coff.allocateAtom(atom_index, code_len, @intCast(required_alignment.toByteUnits() orelse 0));
1286 errdefer coff.freeAtom(atom_index);
1287
1288 log.debug("allocated atom for {s} at 0x{x}", .{ name, vaddr });
1289 log.debug(" (required alignment 0x{x})", .{required_alignment});
1290
1291 atom.size = code_len;
1292 symbol.value = vaddr;
1293
1294 try coff.addGotEntry(.{ .sym_index = local_sym_index });
1295 try coff.writeAtom(atom_index, code, coff.base.comp.config.incremental);
1296}
1297
1298pub fn getOrCreateAtomForLazySymbol(
1299 coff: *Coff,
1300 pt: Zcu.PerThread,
1301 lazy_sym: link.File.LazySymbol,
1302) !Atom.Index {
1303 const gop = try coff.lazy_syms.getOrPut(pt.zcu.gpa, lazy_sym.ty);
1304 errdefer _ = if (!gop.found_existing) coff.lazy_syms.pop();
1305 if (!gop.found_existing) gop.value_ptr.* = .{};
1306 const atom_ptr, const state_ptr = switch (lazy_sym.kind) {
1307 .code => .{ &gop.value_ptr.text_atom, &gop.value_ptr.text_state },
1308 .const_data => .{ &gop.value_ptr.rdata_atom, &gop.value_ptr.rdata_state },
1309 };
1310 switch (state_ptr.*) {
1311 .unused => atom_ptr.* = try coff.createAtom(),
1312 .pending_flush => return atom_ptr.*,
1313 .flushed => {},
1314 }
1315 state_ptr.* = .pending_flush;
1316 const atom = atom_ptr.*;
1317 // anyerror needs to be deferred until flush
1318 if (lazy_sym.ty != .anyerror_type) try coff.updateLazySymbolAtom(pt, lazy_sym, atom, switch (lazy_sym.kind) {
1319 .code => coff.text_section_index.?,
1320 .const_data => coff.rdata_section_index.?,
1321 });
1322 return atom;
1323}
1324
1325pub fn getOrCreateAtomForNav(coff: *Coff, nav_index: InternPool.Nav.Index) !Atom.Index {
1326 const gpa = coff.base.comp.gpa;
1327 const gop = try coff.navs.getOrPut(gpa, nav_index);
1328 if (!gop.found_existing) {
1329 gop.value_ptr.* = .{
1330 .atom = try coff.createAtom(),
1331 // If necessary, this will be modified by `updateNav` or `updateFunc`.
1332 .section = coff.rdata_section_index.?,
1333 .exports = .{},
1334 };
1335 }
1336 return gop.value_ptr.atom;
1337}
1338
1339fn getNavOutputSection(coff: *Coff, nav_index: InternPool.Nav.Index) u16 {
1340 const zcu = coff.base.comp.zcu.?;
1341 const ip = &zcu.intern_pool;
1342 const nav = ip.getNav(nav_index);
1343 const ty = Type.fromInterned(nav.typeOf(ip));
1344 const zig_ty = ty.zigTypeTag(zcu);
1345 const val = Value.fromInterned(nav.status.fully_resolved.val);
1346 const index: u16 = blk: {
1347 if (val.isUndef(zcu)) {
1348 // TODO in release-fast and release-small, we should put undef in .bss
1349 break :blk coff.data_section_index.?;
1350 }
1351
1352 switch (zig_ty) {
1353 // TODO: what if this is a function pointer?
1354 .@"fn" => break :blk coff.text_section_index.?,
1355 else => {
1356 if (val.getVariable(zcu)) |_| {
1357 break :blk coff.data_section_index.?;
1358 }
1359 break :blk coff.rdata_section_index.?;
1360 },
1361 }
1362 };
1363 return index;
1364}
1365
1366fn updateNavCode(
1367 coff: *Coff,
1368 pt: Zcu.PerThread,
1369 nav_index: InternPool.Nav.Index,
1370 code: []u8,
1371 complex_type: coff_util.ComplexType,
1372) link.File.UpdateNavError!void {
1373 const zcu = pt.zcu;
1374 const ip = &zcu.intern_pool;
1375 const nav = ip.getNav(nav_index);
1376
1377 log.debug("updateNavCode {f} 0x{x}", .{ nav.fqn.fmt(ip), nav_index });
1378
1379 const mod = zcu.navFileScope(nav_index).mod.?;
1380 const target = &mod.resolved_target.result;
1381 const required_alignment = switch (nav.status.fully_resolved.alignment) {
1382 .none => switch (mod.optimize_mode) {
1383 .Debug, .ReleaseSafe, .ReleaseFast => target_util.defaultFunctionAlignment(target),
1384 .ReleaseSmall => target_util.minFunctionAlignment(target),
1385 },
1386 else => |a| a.maxStrict(target_util.minFunctionAlignment(target)),
1387 };
1388
1389 const nav_metadata = coff.navs.get(nav_index).?;
1390 const atom_index = nav_metadata.atom;
1391 const atom = coff.getAtom(atom_index);
1392 const sym_index = atom.getSymbolIndex().?;
1393 const sect_index = nav_metadata.section;
1394 const code_len: u32 = @intCast(code.len);
1395
1396 if (atom.size != 0) {
1397 const sym = atom.getSymbolPtr(coff);
1398 try coff.setSymbolName(sym, nav.fqn.toSlice(ip));
1399 sym.section_number = @enumFromInt(sect_index + 1);
1400 sym.type = .{ .complex_type = complex_type, .base_type = .NULL };
1401
1402 const capacity = atom.capacity(coff);
1403 const need_realloc = code.len > capacity or !required_alignment.check(sym.value);
1404 if (need_realloc) {
1405 const vaddr = coff.growAtom(atom_index, code_len, @intCast(required_alignment.toByteUnits() orelse 0)) catch |err| switch (err) {
1406 error.OutOfMemory => return error.OutOfMemory,
1407 else => |e| return coff.base.cgFail(nav_index, "failed to grow atom: {s}", .{@errorName(e)}),
1408 };
1409 log.debug("growing {f} from 0x{x} to 0x{x}", .{ nav.fqn.fmt(ip), sym.value, vaddr });
1410 log.debug(" (required alignment 0x{x}", .{required_alignment});
1411
1412 if (vaddr != sym.value) {
1413 sym.value = vaddr;
1414 log.debug(" (updating GOT entry)", .{});
1415 const got_entry_index = coff.got_table.lookup.get(.{ .sym_index = sym_index }).?;
1416 coff.writeOffsetTableEntry(got_entry_index) catch |err| switch (err) {
1417 error.OutOfMemory => return error.OutOfMemory,
1418 else => |e| return coff.base.cgFail(nav_index, "failed to write offset table entry: {s}", .{@errorName(e)}),
1419 };
1420 coff.markRelocsDirtyByTarget(.{ .sym_index = sym_index });
1421 }
1422 } else if (code_len < atom.size) {
1423 coff.shrinkAtom(atom_index, code_len);
1424 }
1425 coff.getAtomPtr(atom_index).size = code_len;
1426 } else {
1427 const sym = atom.getSymbolPtr(coff);
1428 try coff.setSymbolName(sym, nav.fqn.toSlice(ip));
1429 sym.section_number = @enumFromInt(sect_index + 1);
1430 sym.type = .{ .complex_type = complex_type, .base_type = .NULL };
1431
1432 const vaddr = coff.allocateAtom(atom_index, code_len, @intCast(required_alignment.toByteUnits() orelse 0)) catch |err| switch (err) {
1433 error.OutOfMemory => return error.OutOfMemory,
1434 else => |e| return coff.base.cgFail(nav_index, "failed to allocate atom: {s}", .{@errorName(e)}),
1435 };
1436 errdefer coff.freeAtom(atom_index);
1437 log.debug("allocated atom for {f} at 0x{x}", .{ nav.fqn.fmt(ip), vaddr });
1438 coff.getAtomPtr(atom_index).size = code_len;
1439 sym.value = vaddr;
1440
1441 coff.addGotEntry(.{ .sym_index = sym_index }) catch |err| switch (err) {
1442 error.OutOfMemory => return error.OutOfMemory,
1443 else => |e| return coff.base.cgFail(nav_index, "failed to add GOT entry: {s}", .{@errorName(e)}),
1444 };
1445 }
1446
1447 coff.writeAtom(atom_index, code, coff.base.comp.config.incremental) catch |err| switch (err) {
1448 error.OutOfMemory => return error.OutOfMemory,
1449 else => |e| return coff.base.cgFail(nav_index, "failed to write atom: {s}", .{@errorName(e)}),
1450 };
1451}
1452
1453pub fn freeNav(coff: *Coff, nav_index: InternPool.NavIndex) void {
1454 const gpa = coff.base.comp.gpa;
1455
1456 if (coff.decls.fetchOrderedRemove(nav_index)) |const_kv| {
1457 var kv = const_kv;
1458 coff.freeAtom(kv.value.atom);
1459 kv.value.exports.deinit(gpa);
1460 }
1461}
1462
1463pub fn updateExports(
1464 coff: *Coff,
1465 pt: Zcu.PerThread,
1466 exported: Zcu.Exported,
1467 export_indices: []const Zcu.Export.Index,
1468) link.File.UpdateExportsError!void {
1469 if (build_options.skip_non_native and builtin.object_format != .coff) {
1470 @panic("Attempted to compile for object format that was disabled by build configuration");
1471 }
1472
1473 const zcu = pt.zcu;
1474 const gpa = zcu.gpa;
1475
1476 const metadata = switch (exported) {
1477 .nav => |nav| blk: {
1478 _ = try coff.getOrCreateAtomForNav(nav);
1479 break :blk coff.navs.getPtr(nav).?;
1480 },
1481 .uav => |uav| coff.uavs.getPtr(uav) orelse blk: {
1482 const first_exp = export_indices[0].ptr(zcu);
1483 const res = try coff.lowerUav(pt, uav, .none, first_exp.src);
1484 switch (res) {
1485 .sym_index => {},
1486 .fail => |em| {
1487 // TODO maybe it's enough to return an error here and let Module.processExportsInner
1488 // handle the error?
1489 try zcu.failed_exports.ensureUnusedCapacity(zcu.gpa, 1);
1490 zcu.failed_exports.putAssumeCapacityNoClobber(export_indices[0], em);
1491 return;
1492 },
1493 }
1494 break :blk coff.uavs.getPtr(uav).?;
1495 },
1496 };
1497 const atom_index = metadata.atom;
1498 const atom = coff.getAtom(atom_index);
1499
1500 for (export_indices) |export_idx| {
1501 const exp = export_idx.ptr(zcu);
1502 log.debug("adding new export '{f}'", .{exp.opts.name.fmt(&zcu.intern_pool)});
1503
1504 if (exp.opts.section.toSlice(&zcu.intern_pool)) |section_name| {
1505 if (!mem.eql(u8, section_name, ".text")) {
1506 try zcu.failed_exports.putNoClobber(gpa, export_idx, try Zcu.ErrorMsg.create(
1507 gpa,
1508 exp.src,
1509 "Unimplemented: ExportOptions.section",
1510 .{},
1511 ));
1512 continue;
1513 }
1514 }
1515
1516 if (exp.opts.linkage == .link_once) {
1517 try zcu.failed_exports.putNoClobber(gpa, export_idx, try Zcu.ErrorMsg.create(
1518 gpa,
1519 exp.src,
1520 "Unimplemented: GlobalLinkage.link_once",
1521 .{},
1522 ));
1523 continue;
1524 }
1525
1526 const exp_name = exp.opts.name.toSlice(&zcu.intern_pool);
1527 const sym_index = metadata.getExport(coff, exp_name) orelse blk: {
1528 const sym_index = if (coff.getGlobalIndex(exp_name)) |global_index| ind: {
1529 const global = coff.globals.items[global_index];
1530 // TODO this is just plain wrong as it all should happen in a single `resolveSymbols`
1531 // pass. This will go away once we abstact away Zig's incremental compilation into
1532 // its own module.
1533 if (global.file == null and coff.getSymbol(global).section_number == .UNDEFINED) {
1534 _ = coff.unresolved.swapRemove(global_index);
1535 break :ind global.sym_index;
1536 }
1537 break :ind try coff.allocateSymbol();
1538 } else try coff.allocateSymbol();
1539 try metadata.exports.append(gpa, sym_index);
1540 break :blk sym_index;
1541 };
1542 const sym_loc = SymbolWithLoc{ .sym_index = sym_index, .file = null };
1543 const sym = coff.getSymbolPtr(sym_loc);
1544 try coff.setSymbolName(sym, exp_name);
1545 sym.value = atom.getSymbol(coff).value;
1546 sym.section_number = @as(coff_util.SectionNumber, @enumFromInt(metadata.section + 1));
1547 sym.type = atom.getSymbol(coff).type;
1548
1549 sym.storage_class = switch (exp.opts.linkage) {
1550 .internal => .EXTERNAL,
1551 .strong => .EXTERNAL,
1552 .weak => @panic("TODO WeakExternal"),
1553 else => unreachable,
1554 };
1555
1556 try coff.resolveGlobalSymbol(sym_loc);
1557 }
1558}
1559
1560pub fn deleteExport(
1561 coff: *Coff,
1562 exported: Zcu.Exported,
1563 name: InternPool.NullTerminatedString,
1564) void {
1565 const metadata = switch (exported) {
1566 .nav => |nav| coff.navs.getPtr(nav),
1567 .uav => |uav| coff.uavs.getPtr(uav),
1568 } orelse return;
1569 const zcu = coff.base.comp.zcu.?;
1570 const name_slice = name.toSlice(&zcu.intern_pool);
1571 const sym_index = metadata.getExportPtr(coff, name_slice) orelse return;
1572
1573 const gpa = coff.base.comp.gpa;
1574 const sym_loc = SymbolWithLoc{ .sym_index = sym_index.*, .file = null };
1575 const sym = coff.getSymbolPtr(sym_loc);
1576 log.debug("deleting export '{f}'", .{name.fmt(&zcu.intern_pool)});
1577 assert(sym.storage_class == .EXTERNAL and sym.section_number != .UNDEFINED);
1578 sym.* = .{
1579 .name = [_]u8{0} ** 8,
1580 .value = 0,
1581 .section_number = .UNDEFINED,
1582 .type = .{ .base_type = .NULL, .complex_type = .NULL },
1583 .storage_class = .NULL,
1584 .number_of_aux_symbols = 0,
1585 };
1586 coff.locals_free_list.append(gpa, sym_index.*) catch {};
1587
1588 if (coff.resolver.fetchRemove(name_slice)) |entry| {
1589 defer gpa.free(entry.key);
1590 coff.globals_free_list.append(gpa, entry.value) catch {};
1591 coff.globals.items[entry.value] = .{
1592 .sym_index = 0,
1593 .file = null,
1594 };
1595 }
1596
1597 sym_index.* = 0;
1598}
1599
1600fn resolveGlobalSymbol(coff: *Coff, current: SymbolWithLoc) !void {
1601 const gpa = coff.base.comp.gpa;
1602 const sym = coff.getSymbol(current);
1603 const sym_name = coff.getSymbolName(current);
1604
1605 const gop = try coff.getOrPutGlobalPtr(sym_name);
1606 if (!gop.found_existing) {
1607 gop.value_ptr.* = current;
1608 if (sym.section_number == .UNDEFINED) {
1609 try coff.unresolved.putNoClobber(gpa, coff.getGlobalIndex(sym_name).?, false);
1610 }
1611 return;
1612 }
1613
1614 log.debug("TODO finish resolveGlobalSymbols implementation", .{});
1615
1616 if (sym.section_number == .UNDEFINED) return;
1617
1618 _ = coff.unresolved.swapRemove(coff.getGlobalIndex(sym_name).?);
1619
1620 gop.value_ptr.* = current;
1621}
1622
1623pub fn flush(
1624 coff: *Coff,
1625 arena: Allocator,
1626 tid: Zcu.PerThread.Id,
1627 prog_node: std.Progress.Node,
1628) link.File.FlushError!void {
1629 const tracy = trace(@src());
1630 defer tracy.end();
1631
1632 const comp = coff.base.comp;
1633 const diags = &comp.link_diags;
1634
1635 switch (coff.base.comp.config.output_mode) {
1636 .Exe, .Obj => {},
1637 .Lib => return diags.fail("writing lib files not yet implemented for COFF", .{}),
1638 }
1639
1640 const sub_prog_node = prog_node.start("COFF Flush", 0);
1641 defer sub_prog_node.end();
1642
1643 return flushInner(coff, arena, tid) catch |err| switch (err) {
1644 error.OutOfMemory => return error.OutOfMemory,
1645 error.LinkFailure => return error.LinkFailure,
1646 else => |e| return diags.fail("COFF flush failed: {s}", .{@errorName(e)}),
1647 };
1648}
1649
1650fn flushInner(coff: *Coff, arena: Allocator, tid: Zcu.PerThread.Id) !void {
1651 _ = arena;
1652
1653 const comp = coff.base.comp;
1654 const gpa = comp.gpa;
1655 const diags = &comp.link_diags;
1656
1657 const pt: Zcu.PerThread = .activate(
1658 comp.zcu orelse return diags.fail("linking without zig source is not yet implemented", .{}),
1659 tid,
1660 );
1661 defer pt.deactivate();
1662
1663 if (coff.lazy_syms.getPtr(.anyerror_type)) |metadata| {
1664 // Most lazy symbols can be updated on first use, but
1665 // anyerror needs to wait for everything to be flushed.
1666 if (metadata.text_state != .unused) try coff.updateLazySymbolAtom(
1667 pt,
1668 .{ .kind = .code, .ty = .anyerror_type },
1669 metadata.text_atom,
1670 coff.text_section_index.?,
1671 );
1672 if (metadata.rdata_state != .unused) try coff.updateLazySymbolAtom(
1673 pt,
1674 .{ .kind = .const_data, .ty = .anyerror_type },
1675 metadata.rdata_atom,
1676 coff.rdata_section_index.?,
1677 );
1678 }
1679 for (coff.lazy_syms.values()) |*metadata| {
1680 if (metadata.text_state != .unused) metadata.text_state = .flushed;
1681 if (metadata.rdata_state != .unused) metadata.rdata_state = .flushed;
1682 }
1683
1684 {
1685 var it = coff.need_got_table.iterator();
1686 while (it.next()) |entry| {
1687 const global = coff.globals.items[entry.key_ptr.*];
1688 try coff.addGotEntry(global);
1689 }
1690 }
1691
1692 while (coff.unresolved.pop()) |entry| {
1693 assert(entry.value);
1694 const global = coff.globals.items[entry.key];
1695 const sym = coff.getSymbol(global);
1696 const res = try coff.import_tables.getOrPut(gpa, sym.value);
1697 const itable = res.value_ptr;
1698 if (!res.found_existing) {
1699 itable.* = .{};
1700 }
1701 if (itable.lookup.contains(global)) continue;
1702 // TODO: we could technically write the pointer placeholder for to-be-bound import here,
1703 // but since this happens in flush, there is currently no point.
1704 _ = try itable.addImport(gpa, global);
1705 coff.imports_count_dirty = true;
1706 }
1707
1708 try coff.writeImportTables();
1709
1710 for (coff.relocs.keys(), coff.relocs.values()) |atom_index, relocs| {
1711 const needs_update = for (relocs.items) |reloc| {
1712 if (reloc.dirty) break true;
1713 } else false;
1714
1715 if (!needs_update) continue;
1716
1717 const atom = coff.getAtom(atom_index);
1718 const sym = atom.getSymbol(coff);
1719 const section = coff.sections.get(@intFromEnum(sym.section_number) - 1).header;
1720 const file_offset = section.pointer_to_raw_data + sym.value - section.virtual_address;
1721
1722 var code = std.array_list.Managed(u8).init(gpa);
1723 defer code.deinit();
1724 try code.resize(math.cast(usize, atom.size) orelse return error.Overflow);
1725 assert(atom.size > 0);
1726
1727 const amt = try coff.base.file.?.preadAll(code.items, file_offset);
1728 if (amt != code.items.len) return error.InputOutput;
1729
1730 try coff.writeAtom(atom_index, code.items, true);
1731 }
1732
1733 // Update GOT if it got moved in memory.
1734 if (coff.got_table_contents_dirty) {
1735 for (coff.got_table.entries.items, 0..) |entry, i| {
1736 if (!coff.got_table.lookup.contains(entry)) continue;
1737 // TODO: write all in one go rather than incrementally.
1738 try coff.writeOffsetTableEntry(i);
1739 }
1740 coff.got_table_contents_dirty = false;
1741 }
1742
1743 try coff.writeBaseRelocations();
1744
1745 if (coff.getEntryPoint()) |entry_sym_loc| {
1746 coff.entry_addr = coff.getSymbol(entry_sym_loc).value;
1747 }
1748
1749 if (build_options.enable_logging) {
1750 coff.logSymtab();
1751 coff.logImportTables();
1752 }
1753
1754 try coff.writeStrtab();
1755 try coff.writeDataDirectoriesHeaders();
1756 try coff.writeSectionHeaders();
1757
1758 if (coff.entry_addr == null and comp.config.output_mode == .Exe) {
1759 log.debug("flushing. no_entry_point_found = true\n", .{});
1760 diags.flags.no_entry_point_found = true;
1761 } else {
1762 log.debug("flushing. no_entry_point_found = false\n", .{});
1763 diags.flags.no_entry_point_found = false;
1764 try coff.writeHeader();
1765 }
1766
1767 assert(!coff.imports_count_dirty);
1768
1769 // hack for stage2_x86_64 + coff
1770 if (comp.compiler_rt_dyn_lib) |crt_file| {
1771 const compiler_rt_sub_path = try std.fs.path.join(gpa, &.{
1772 std.fs.path.dirname(coff.base.emit.sub_path) orelse "",
1773 std.fs.path.basename(crt_file.full_object_path.sub_path),
1774 });
1775 defer gpa.free(compiler_rt_sub_path);
1776 try crt_file.full_object_path.root_dir.handle.copyFile(
1777 crt_file.full_object_path.sub_path,
1778 coff.base.emit.root_dir.handle,
1779 compiler_rt_sub_path,
1780 .{},
1781 );
1782 }
1783}
1784
1785pub fn getNavVAddr(
1786 coff: *Coff,
1787 pt: Zcu.PerThread,
1788 nav_index: InternPool.Nav.Index,
1789 reloc_info: link.File.RelocInfo,
1790) !u64 {
1791 const zcu = pt.zcu;
1792 const ip = &zcu.intern_pool;
1793 const nav = ip.getNav(nav_index);
1794 log.debug("getNavVAddr {f}({d})", .{ nav.fqn.fmt(ip), nav_index });
1795 const sym_index = if (nav.getExtern(ip)) |e|
1796 try coff.getGlobalSymbol(nav.name.toSlice(ip), e.lib_name.toSlice(ip))
1797 else
1798 coff.getAtom(try coff.getOrCreateAtomForNav(nav_index)).getSymbolIndex().?;
1799 const atom_index = coff.getAtomIndexForSymbol(.{
1800 .sym_index = reloc_info.parent.atom_index,
1801 .file = null,
1802 }).?;
1803 const target = SymbolWithLoc{ .sym_index = sym_index, .file = null };
1804 try coff.addRelocation(atom_index, .{
1805 .type = .direct,
1806 .target = target,
1807 .offset = @as(u32, @intCast(reloc_info.offset)),
1808 .addend = reloc_info.addend,
1809 .pcrel = false,
1810 .length = 3,
1811 });
1812 try coff.addBaseRelocation(atom_index, @as(u32, @intCast(reloc_info.offset)));
1813
1814 return 0;
1815}
1816
1817pub fn lowerUav(
1818 coff: *Coff,
1819 pt: Zcu.PerThread,
1820 uav: InternPool.Index,
1821 explicit_alignment: InternPool.Alignment,
1822 src_loc: Zcu.LazySrcLoc,
1823) !codegen.SymbolResult {
1824 const zcu = pt.zcu;
1825 const gpa = zcu.gpa;
1826 const val = Value.fromInterned(uav);
1827 const uav_alignment = switch (explicit_alignment) {
1828 .none => val.typeOf(zcu).abiAlignment(zcu),
1829 else => explicit_alignment,
1830 };
1831 if (coff.uavs.get(uav)) |metadata| {
1832 const atom = coff.getAtom(metadata.atom);
1833 const existing_addr = atom.getSymbol(coff).value;
1834 if (uav_alignment.check(existing_addr))
1835 return .{ .sym_index = atom.getSymbolIndex().? };
1836 }
1837
1838 var name_buf: [32]u8 = undefined;
1839 const name = std.fmt.bufPrint(&name_buf, "__anon_{d}", .{
1840 @intFromEnum(uav),
1841 }) catch unreachable;
1842 const res = coff.lowerConst(
1843 pt,
1844 name,
1845 val,
1846 uav_alignment,
1847 coff.rdata_section_index.?,
1848 src_loc,
1849 ) catch |err| switch (err) {
1850 error.OutOfMemory => return error.OutOfMemory,
1851 else => |e| return .{ .fail = try Zcu.ErrorMsg.create(
1852 gpa,
1853 src_loc,
1854 "lowerAnonDecl failed with error: {s}",
1855 .{@errorName(e)},
1856 ) },
1857 };
1858 const atom_index = switch (res) {
1859 .ok => |atom_index| atom_index,
1860 .fail => |em| return .{ .fail = em },
1861 };
1862 try coff.uavs.put(gpa, uav, .{
1863 .atom = atom_index,
1864 .section = coff.rdata_section_index.?,
1865 });
1866 return .{ .sym_index = coff.getAtom(atom_index).getSymbolIndex().? };
1867}
1868
1869pub fn getUavVAddr(
1870 coff: *Coff,
1871 uav: InternPool.Index,
1872 reloc_info: link.File.RelocInfo,
1873) !u64 {
1874 const this_atom_index = coff.uavs.get(uav).?.atom;
1875 const sym_index = coff.getAtom(this_atom_index).getSymbolIndex().?;
1876 const atom_index = coff.getAtomIndexForSymbol(.{
1877 .sym_index = reloc_info.parent.atom_index,
1878 .file = null,
1879 }).?;
1880 const target = SymbolWithLoc{ .sym_index = sym_index, .file = null };
1881 try coff.addRelocation(atom_index, .{
1882 .type = .direct,
1883 .target = target,
1884 .offset = @as(u32, @intCast(reloc_info.offset)),
1885 .addend = reloc_info.addend,
1886 .pcrel = false,
1887 .length = 3,
1888 });
1889 try coff.addBaseRelocation(atom_index, @as(u32, @intCast(reloc_info.offset)));
1890
1891 return 0;
1892}
1893
1894pub fn getGlobalSymbol(coff: *Coff, name: []const u8, lib_name_name: ?[]const u8) !u32 {
1895 const gop = try coff.getOrPutGlobalPtr(name);
1896 const global_index = coff.getGlobalIndex(name).?;
1897
1898 if (gop.found_existing) {
1899 return global_index;
1900 }
1901
1902 const sym_index = try coff.allocateSymbol();
1903 const sym_loc = SymbolWithLoc{ .sym_index = sym_index, .file = null };
1904 gop.value_ptr.* = sym_loc;
1905
1906 const gpa = coff.base.comp.gpa;
1907 const sym = coff.getSymbolPtr(sym_loc);
1908 try coff.setSymbolName(sym, name);
1909 sym.storage_class = .EXTERNAL;
1910
1911 if (lib_name_name) |lib_name| {
1912 // We repurpose the 'value' of the Symbol struct to store an offset into
1913 // temporary string table where we will store the library name hint.
1914 sym.value = try coff.temp_strtab.insert(gpa, lib_name);
1915 }
1916
1917 try coff.unresolved.putNoClobber(gpa, global_index, true);
1918
1919 return global_index;
1920}
1921
1922pub fn updateLineNumber(coff: *Coff, pt: Zcu.PerThread, ti_id: InternPool.TrackedInst.Index) !void {
1923 _ = coff;
1924 _ = pt;
1925 _ = ti_id;
1926 log.debug("TODO implement updateLineNumber", .{});
1927}
1928
1929/// TODO: note if we need to rewrite base relocations by dirtying any of the entries in the global table
1930/// TODO: note that .ABSOLUTE is used as padding within each block; we could use this fact to do
1931/// incremental updates and writes into the table instead of doing it all at once
1932fn writeBaseRelocations(coff: *Coff) !void {
1933 const gpa = coff.base.comp.gpa;
1934
1935 var page_table = std.AutoHashMap(u32, std.array_list.Managed(coff_util.BaseRelocation)).init(gpa);
1936 defer {
1937 var it = page_table.valueIterator();
1938 while (it.next()) |inner| {
1939 inner.deinit();
1940 }
1941 page_table.deinit();
1942 }
1943
1944 {
1945 var it = coff.base_relocs.iterator();
1946 while (it.next()) |entry| {
1947 const atom_index = entry.key_ptr.*;
1948 const atom = coff.getAtom(atom_index);
1949 const sym = atom.getSymbol(coff);
1950 const offsets = entry.value_ptr.*;
1951
1952 for (offsets.items) |offset| {
1953 const rva = sym.value + offset;
1954 const page = mem.alignBackward(u32, rva, coff.page_size);
1955 const gop = try page_table.getOrPut(page);
1956 if (!gop.found_existing) {
1957 gop.value_ptr.* = std.array_list.Managed(coff_util.BaseRelocation).init(gpa);
1958 }
1959 try gop.value_ptr.append(.{
1960 .offset = @as(u12, @intCast(rva - page)),
1961 .type = .DIR64,
1962 });
1963 }
1964 }
1965
1966 {
1967 const header = &coff.sections.items(.header)[coff.got_section_index.?];
1968 for (coff.got_table.entries.items, 0..) |entry, index| {
1969 if (!coff.got_table.lookup.contains(entry)) continue;
1970
1971 const sym = coff.getSymbol(entry);
1972 if (sym.section_number == .UNDEFINED) continue;
1973
1974 const rva = @as(u32, @intCast(header.virtual_address + index * coff.ptr_width.size()));
1975 const page = mem.alignBackward(u32, rva, coff.page_size);
1976 const gop = try page_table.getOrPut(page);
1977 if (!gop.found_existing) {
1978 gop.value_ptr.* = std.array_list.Managed(coff_util.BaseRelocation).init(gpa);
1979 }
1980 try gop.value_ptr.append(.{
1981 .offset = @as(u12, @intCast(rva - page)),
1982 .type = .DIR64,
1983 });
1984 }
1985 }
1986 }
1987
1988 // Sort pages by address.
1989 var pages = try std.array_list.Managed(u32).initCapacity(gpa, page_table.count());
1990 defer pages.deinit();
1991 {
1992 var it = page_table.keyIterator();
1993 while (it.next()) |page| {
1994 pages.appendAssumeCapacity(page.*);
1995 }
1996 }
1997 mem.sort(u32, pages.items, {}, std.sort.asc(u32));
1998
1999 var buffer = std.array_list.Managed(u8).init(gpa);
2000 defer buffer.deinit();
2001
2002 for (pages.items) |page| {
2003 const entries = page_table.getPtr(page).?;
2004 // Pad to required 4byte alignment
2005 if (!mem.isAlignedGeneric(
2006 usize,
2007 entries.items.len * @sizeOf(coff_util.BaseRelocation),
2008 @sizeOf(u32),
2009 )) {
2010 try entries.append(.{
2011 .offset = 0,
2012 .type = .ABSOLUTE,
2013 });
2014 }
2015
2016 const block_size = @as(
2017 u32,
2018 @intCast(entries.items.len * @sizeOf(coff_util.BaseRelocation) + @sizeOf(coff_util.BaseRelocationDirectoryEntry)),
2019 );
2020 try buffer.ensureUnusedCapacity(block_size);
2021 buffer.appendSliceAssumeCapacity(mem.asBytes(&coff_util.BaseRelocationDirectoryEntry{
2022 .page_rva = page,
2023 .block_size = block_size,
2024 }));
2025 buffer.appendSliceAssumeCapacity(mem.sliceAsBytes(entries.items));
2026 }
2027
2028 const header = &coff.sections.items(.header)[coff.reloc_section_index.?];
2029 const needed_size = @as(u32, @intCast(buffer.items.len));
2030 try coff.growSection(coff.reloc_section_index.?, needed_size);
2031
2032 try coff.pwriteAll(buffer.items, header.pointer_to_raw_data);
2033
2034 coff.data_directories[@intFromEnum(coff_util.DirectoryEntry.BASERELOC)] = .{
2035 .virtual_address = header.virtual_address,
2036 .size = needed_size,
2037 };
2038}
2039
2040fn writeImportTables(coff: *Coff) !void {
2041 if (coff.idata_section_index == null) return;
2042 if (!coff.imports_count_dirty) return;
2043
2044 const gpa = coff.base.comp.gpa;
2045
2046 const ext = ".dll";
2047 const header = &coff.sections.items(.header)[coff.idata_section_index.?];
2048
2049 // Calculate needed size
2050 var iat_size: u32 = 0;
2051 var dir_table_size: u32 = @sizeOf(coff_util.ImportDirectoryEntry); // sentinel
2052 var lookup_table_size: u32 = 0;
2053 var names_table_size: u32 = 0;
2054 var dll_names_size: u32 = 0;
2055 for (coff.import_tables.keys(), 0..) |off, i| {
2056 const lib_name = coff.temp_strtab.getAssumeExists(off);
2057 const itable = coff.import_tables.values()[i];
2058 iat_size += itable.size() + 8;
2059 dir_table_size += @sizeOf(coff_util.ImportDirectoryEntry);
2060 lookup_table_size += @as(u32, @intCast(itable.entries.items.len + 1)) * @sizeOf(coff_util.ImportLookupEntry64.ByName);
2061 for (itable.entries.items) |entry| {
2062 const sym_name = coff.getSymbolName(entry);
2063 names_table_size += 2 + mem.alignForward(u32, @as(u32, @intCast(sym_name.len + 1)), 2);
2064 }
2065 dll_names_size += @as(u32, @intCast(lib_name.len + ext.len + 1));
2066 }
2067
2068 const needed_size = iat_size + dir_table_size + lookup_table_size + names_table_size + dll_names_size;
2069 try coff.growSection(coff.idata_section_index.?, needed_size);
2070
2071 // Do the actual writes
2072 var buffer = std.array_list.Managed(u8).init(gpa);
2073 defer buffer.deinit();
2074 try buffer.ensureTotalCapacityPrecise(needed_size);
2075 buffer.resize(needed_size) catch unreachable;
2076
2077 const dir_header_size = @sizeOf(coff_util.ImportDirectoryEntry);
2078 const lookup_entry_size = @sizeOf(coff_util.ImportLookupEntry64.ByName);
2079
2080 var iat_offset: u32 = 0;
2081 var dir_table_offset = iat_size;
2082 var lookup_table_offset = dir_table_offset + dir_table_size;
2083 var names_table_offset = lookup_table_offset + lookup_table_size;
2084 var dll_names_offset = names_table_offset + names_table_size;
2085 for (coff.import_tables.keys(), 0..) |off, i| {
2086 const lib_name = coff.temp_strtab.getAssumeExists(off);
2087 const itable = coff.import_tables.values()[i];
2088
2089 // Lookup table header
2090 const lookup_header = coff_util.ImportDirectoryEntry{
2091 .import_lookup_table_rva = header.virtual_address + lookup_table_offset,
2092 .time_date_stamp = 0,
2093 .forwarder_chain = 0,
2094 .name_rva = header.virtual_address + dll_names_offset,
2095 .import_address_table_rva = header.virtual_address + iat_offset,
2096 };
2097 @memcpy(buffer.items[dir_table_offset..][0..@sizeOf(coff_util.ImportDirectoryEntry)], mem.asBytes(&lookup_header));
2098 dir_table_offset += dir_header_size;
2099
2100 for (itable.entries.items) |entry| {
2101 const import_name = coff.getSymbolName(entry);
2102
2103 // IAT and lookup table entry
2104 const lookup = coff_util.ImportLookupEntry64.ByName{ .name_table_rva = @as(u31, @intCast(header.virtual_address + names_table_offset)) };
2105 @memcpy(
2106 buffer.items[iat_offset..][0..@sizeOf(coff_util.ImportLookupEntry64.ByName)],
2107 mem.asBytes(&lookup),
2108 );
2109 iat_offset += lookup_entry_size;
2110 @memcpy(
2111 buffer.items[lookup_table_offset..][0..@sizeOf(coff_util.ImportLookupEntry64.ByName)],
2112 mem.asBytes(&lookup),
2113 );
2114 lookup_table_offset += lookup_entry_size;
2115
2116 // Names table entry
2117 mem.writeInt(u16, buffer.items[names_table_offset..][0..2], 0, .little); // Hint set to 0 until we learn how to parse DLLs
2118 names_table_offset += 2;
2119 @memcpy(buffer.items[names_table_offset..][0..import_name.len], import_name);
2120 names_table_offset += @as(u32, @intCast(import_name.len));
2121 buffer.items[names_table_offset] = 0;
2122 names_table_offset += 1;
2123 if (!mem.isAlignedGeneric(usize, names_table_offset, @sizeOf(u16))) {
2124 buffer.items[names_table_offset] = 0;
2125 names_table_offset += 1;
2126 }
2127 }
2128
2129 // IAT sentinel
2130 mem.writeInt(u64, buffer.items[iat_offset..][0..lookup_entry_size], 0, .little);
2131 iat_offset += 8;
2132
2133 // Lookup table sentinel
2134 @memcpy(
2135 buffer.items[lookup_table_offset..][0..@sizeOf(coff_util.ImportLookupEntry64.ByName)],
2136 mem.asBytes(&coff_util.ImportLookupEntry64.ByName{ .name_table_rva = 0 }),
2137 );
2138 lookup_table_offset += lookup_entry_size;
2139
2140 // DLL name
2141 @memcpy(buffer.items[dll_names_offset..][0..lib_name.len], lib_name);
2142 dll_names_offset += @as(u32, @intCast(lib_name.len));
2143 @memcpy(buffer.items[dll_names_offset..][0..ext.len], ext);
2144 dll_names_offset += @as(u32, @intCast(ext.len));
2145 buffer.items[dll_names_offset] = 0;
2146 dll_names_offset += 1;
2147 }
2148
2149 // Sentinel
2150 const lookup_header = coff_util.ImportDirectoryEntry{
2151 .import_lookup_table_rva = 0,
2152 .time_date_stamp = 0,
2153 .forwarder_chain = 0,
2154 .name_rva = 0,
2155 .import_address_table_rva = 0,
2156 };
2157 @memcpy(
2158 buffer.items[dir_table_offset..][0..@sizeOf(coff_util.ImportDirectoryEntry)],
2159 mem.asBytes(&lookup_header),
2160 );
2161 dir_table_offset += dir_header_size;
2162
2163 assert(dll_names_offset == needed_size);
2164
2165 try coff.pwriteAll(buffer.items, header.pointer_to_raw_data);
2166
2167 coff.data_directories[@intFromEnum(coff_util.DirectoryEntry.IMPORT)] = .{
2168 .virtual_address = header.virtual_address + iat_size,
2169 .size = dir_table_size,
2170 };
2171 coff.data_directories[@intFromEnum(coff_util.DirectoryEntry.IAT)] = .{
2172 .virtual_address = header.virtual_address,
2173 .size = iat_size,
2174 };
2175
2176 coff.imports_count_dirty = false;
2177}
2178
2179fn writeStrtab(coff: *Coff) !void {
2180 if (coff.strtab_offset == null) return;
2181
2182 const comp = coff.base.comp;
2183 const gpa = comp.gpa;
2184 const diags = &comp.link_diags;
2185 const allocated_size = coff.allocatedSize(coff.strtab_offset.?);
2186 const needed_size: u32 = @intCast(coff.strtab.buffer.items.len);
2187
2188 if (needed_size > allocated_size) {
2189 coff.strtab_offset = null;
2190 coff.strtab_offset = @intCast(coff.findFreeSpace(needed_size, @alignOf(u32)));
2191 }
2192
2193 log.debug("writing strtab from 0x{x} to 0x{x}", .{ coff.strtab_offset.?, coff.strtab_offset.? + needed_size });
2194
2195 var buffer = std.array_list.Managed(u8).init(gpa);
2196 defer buffer.deinit();
2197 try buffer.ensureTotalCapacityPrecise(needed_size);
2198 buffer.appendSliceAssumeCapacity(coff.strtab.buffer.items);
2199 // Here, we do a trick in that we do not commit the size of the strtab to strtab buffer, instead
2200 // we write the length of the strtab to a temporary buffer that goes to file.
2201 mem.writeInt(u32, buffer.items[0..4], @as(u32, @intCast(coff.strtab.buffer.items.len)), .little);
2202
2203 coff.pwriteAll(buffer.items, coff.strtab_offset.?) catch |err| {
2204 return diags.fail("failed to write: {s}", .{@errorName(err)});
2205 };
2206}
2207
2208fn writeSectionHeaders(coff: *Coff) !void {
2209 const offset = coff.getSectionHeadersOffset();
2210 try coff.pwriteAll(@ptrCast(coff.sections.items(.header)), offset);
2211}
2212
2213fn writeDataDirectoriesHeaders(coff: *Coff) !void {
2214 const offset = coff.getDataDirectoryHeadersOffset();
2215 try coff.pwriteAll(@ptrCast(&coff.data_directories), offset);
2216}
2217
2218fn writeHeader(coff: *Coff) !void {
2219 const target = &coff.base.comp.root_mod.resolved_target.result;
2220 const gpa = coff.base.comp.gpa;
2221 var buffer: std.Io.Writer.Allocating = .init(gpa);
2222 defer buffer.deinit();
2223 const writer = &buffer.writer;
2224
2225 try buffer.ensureTotalCapacity(coff.getSizeOfHeaders());
2226 writer.writeAll(&msdos_stub) catch unreachable;
2227 mem.writeInt(u32, buffer.writer.buffer[0x3c..][0..4], msdos_stub.len, .little);
2228
2229 writer.writeAll("PE\x00\x00") catch unreachable;
2230 var flags = coff_util.CoffHeaderFlags{
2231 .EXECUTABLE_IMAGE = 1,
2232 .DEBUG_STRIPPED = 1, // TODO
2233 };
2234 switch (coff.ptr_width) {
2235 .p32 => flags.@"32BIT_MACHINE" = 1,
2236 .p64 => flags.LARGE_ADDRESS_AWARE = 1,
2237 }
2238 if (coff.base.comp.config.output_mode == .Lib and coff.base.comp.config.link_mode == .dynamic) {
2239 flags.DLL = 1;
2240 }
2241
2242 const timestamp = if (coff.repro) 0 else std.time.timestamp();
2243 const size_of_optional_header = @as(u16, @intCast(coff.getOptionalHeaderSize() + coff.getDataDirectoryHeadersSize()));
2244 var coff_header = coff_util.CoffHeader{
2245 .machine = target.toCoffMachine(),
2246 .number_of_sections = @as(u16, @intCast(coff.sections.slice().len)), // TODO what if we prune a section
2247 .time_date_stamp = @as(u32, @truncate(@as(u64, @bitCast(timestamp)))),
2248 .pointer_to_symbol_table = coff.strtab_offset orelse 0,
2249 .number_of_symbols = 0,
2250 .size_of_optional_header = size_of_optional_header,
2251 .flags = flags,
2252 };
2253
2254 writer.writeAll(mem.asBytes(&coff_header)) catch unreachable;
2255
2256 const dll_flags: coff_util.DllFlags = .{
2257 .HIGH_ENTROPY_VA = 1, // TODO do we want to permit non-PIE builds at all?
2258 .DYNAMIC_BASE = 1,
2259 .TERMINAL_SERVER_AWARE = 1, // We are not a legacy app
2260 .NX_COMPAT = 1, // We are compatible with Data Execution Prevention
2261 };
2262 const subsystem: coff_util.Subsystem = .WINDOWS_CUI;
2263 const size_of_image: u32 = coff.getSizeOfImage();
2264 const size_of_headers: u32 = mem.alignForward(u32, coff.getSizeOfHeaders(), default_file_alignment);
2265 const base_of_code = coff.sections.get(coff.text_section_index.?).header.virtual_address;
2266 const base_of_data = coff.sections.get(coff.data_section_index.?).header.virtual_address;
2267
2268 var size_of_code: u32 = 0;
2269 var size_of_initialized_data: u32 = 0;
2270 var size_of_uninitialized_data: u32 = 0;
2271 for (coff.sections.items(.header)) |header| {
2272 if (header.flags.CNT_CODE == 1) {
2273 size_of_code += header.size_of_raw_data;
2274 }
2275 if (header.flags.CNT_INITIALIZED_DATA == 1) {
2276 size_of_initialized_data += header.size_of_raw_data;
2277 }
2278 if (header.flags.CNT_UNINITIALIZED_DATA == 1) {
2279 size_of_uninitialized_data += header.size_of_raw_data;
2280 }
2281 }
2282
2283 switch (coff.ptr_width) {
2284 .p32 => {
2285 var opt_header = coff_util.OptionalHeaderPE32{
2286 .magic = coff_util.IMAGE_NT_OPTIONAL_HDR32_MAGIC,
2287 .major_linker_version = 0,
2288 .minor_linker_version = 0,
2289 .size_of_code = size_of_code,
2290 .size_of_initialized_data = size_of_initialized_data,
2291 .size_of_uninitialized_data = size_of_uninitialized_data,
2292 .address_of_entry_point = coff.entry_addr orelse 0,
2293 .base_of_code = base_of_code,
2294 .base_of_data = base_of_data,
2295 .image_base = @intCast(coff.image_base),
2296 .section_alignment = coff.page_size,
2297 .file_alignment = default_file_alignment,
2298 .major_operating_system_version = 6,
2299 .minor_operating_system_version = 0,
2300 .major_image_version = 0,
2301 .minor_image_version = 0,
2302 .major_subsystem_version = @intCast(coff.major_subsystem_version),
2303 .minor_subsystem_version = @intCast(coff.minor_subsystem_version),
2304 .win32_version_value = 0,
2305 .size_of_image = size_of_image,
2306 .size_of_headers = size_of_headers,
2307 .checksum = 0,
2308 .subsystem = subsystem,
2309 .dll_flags = dll_flags,
2310 .size_of_stack_reserve = default_size_of_stack_reserve,
2311 .size_of_stack_commit = default_size_of_stack_commit,
2312 .size_of_heap_reserve = default_size_of_heap_reserve,
2313 .size_of_heap_commit = default_size_of_heap_commit,
2314 .loader_flags = 0,
2315 .number_of_rva_and_sizes = @intCast(coff.data_directories.len),
2316 };
2317 writer.writeAll(mem.asBytes(&opt_header)) catch unreachable;
2318 },
2319 .p64 => {
2320 var opt_header = coff_util.OptionalHeaderPE64{
2321 .magic = coff_util.IMAGE_NT_OPTIONAL_HDR64_MAGIC,
2322 .major_linker_version = 0,
2323 .minor_linker_version = 0,
2324 .size_of_code = size_of_code,
2325 .size_of_initialized_data = size_of_initialized_data,
2326 .size_of_uninitialized_data = size_of_uninitialized_data,
2327 .address_of_entry_point = coff.entry_addr orelse 0,
2328 .base_of_code = base_of_code,
2329 .image_base = coff.image_base,
2330 .section_alignment = coff.page_size,
2331 .file_alignment = default_file_alignment,
2332 .major_operating_system_version = 6,
2333 .minor_operating_system_version = 0,
2334 .major_image_version = 0,
2335 .minor_image_version = 0,
2336 .major_subsystem_version = coff.major_subsystem_version,
2337 .minor_subsystem_version = coff.minor_subsystem_version,
2338 .win32_version_value = 0,
2339 .size_of_image = size_of_image,
2340 .size_of_headers = size_of_headers,
2341 .checksum = 0,
2342 .subsystem = subsystem,
2343 .dll_flags = dll_flags,
2344 .size_of_stack_reserve = default_size_of_stack_reserve,
2345 .size_of_stack_commit = default_size_of_stack_commit,
2346 .size_of_heap_reserve = default_size_of_heap_reserve,
2347 .size_of_heap_commit = default_size_of_heap_commit,
2348 .loader_flags = 0,
2349 .number_of_rva_and_sizes = @intCast(coff.data_directories.len),
2350 };
2351 writer.writeAll(mem.asBytes(&opt_header)) catch unreachable;
2352 },
2353 }
2354
2355 try coff.pwriteAll(buffer.written(), 0);
2356}
2357
2358pub fn padToIdeal(actual_size: anytype) @TypeOf(actual_size) {
2359 return actual_size +| (actual_size / ideal_factor);
2360}
2361
2362fn detectAllocCollision(coff: *Coff, start: u32, size: u32) ?u32 {
2363 const headers_size = @max(coff.getSizeOfHeaders(), coff.page_size);
2364 if (start < headers_size)
2365 return headers_size;
2366
2367 const end = start + padToIdeal(size);
2368
2369 if (coff.strtab_offset) |off| {
2370 const tight_size = @as(u32, @intCast(coff.strtab.buffer.items.len));
2371 const increased_size = padToIdeal(tight_size);
2372 const test_end = off + increased_size;
2373 if (end > off and start < test_end) {
2374 return test_end;
2375 }
2376 }
2377
2378 for (coff.sections.items(.header)) |header| {
2379 const tight_size = header.size_of_raw_data;
2380 const increased_size = padToIdeal(tight_size);
2381 const test_end = header.pointer_to_raw_data + increased_size;
2382 if (end > header.pointer_to_raw_data and start < test_end) {
2383 return test_end;
2384 }
2385 }
2386
2387 return null;
2388}
2389
2390fn allocatedSize(coff: *Coff, start: u32) u32 {
2391 if (start == 0)
2392 return 0;
2393 var min_pos: u32 = std.math.maxInt(u32);
2394 if (coff.strtab_offset) |off| {
2395 if (off > start and off < min_pos) min_pos = off;
2396 }
2397 for (coff.sections.items(.header)) |header| {
2398 if (header.pointer_to_raw_data <= start) continue;
2399 if (header.pointer_to_raw_data < min_pos) min_pos = header.pointer_to_raw_data;
2400 }
2401 return min_pos - start;
2402}
2403
2404fn findFreeSpace(coff: *Coff, object_size: u32, min_alignment: u32) u32 {
2405 var start: u32 = 0;
2406 while (coff.detectAllocCollision(start, object_size)) |item_end| {
2407 start = mem.alignForward(u32, item_end, min_alignment);
2408 }
2409 return start;
2410}
2411
2412fn allocatedVirtualSize(coff: *Coff, start: u32) u32 {
2413 if (start == 0)
2414 return 0;
2415 var min_pos: u32 = std.math.maxInt(u32);
2416 for (coff.sections.items(.header)) |header| {
2417 if (header.virtual_address <= start) continue;
2418 if (header.virtual_address < min_pos) min_pos = header.virtual_address;
2419 }
2420 return min_pos - start;
2421}
2422
2423fn getSizeOfHeaders(coff: Coff) u32 {
2424 const msdos_hdr_size = msdos_stub.len + 4;
2425 return @as(u32, @intCast(msdos_hdr_size + @sizeOf(coff_util.CoffHeader) + coff.getOptionalHeaderSize() +
2426 coff.getDataDirectoryHeadersSize() + coff.getSectionHeadersSize()));
2427}
2428
2429fn getOptionalHeaderSize(coff: Coff) u32 {
2430 return switch (coff.ptr_width) {
2431 .p32 => @as(u32, @intCast(@sizeOf(coff_util.OptionalHeaderPE32))),
2432 .p64 => @as(u32, @intCast(@sizeOf(coff_util.OptionalHeaderPE64))),
2433 };
2434}
2435
2436fn getDataDirectoryHeadersSize(coff: Coff) u32 {
2437 return @as(u32, @intCast(coff.data_directories.len * @sizeOf(coff_util.ImageDataDirectory)));
2438}
2439
2440fn getSectionHeadersSize(coff: Coff) u32 {
2441 return @as(u32, @intCast(coff.sections.slice().len * @sizeOf(coff_util.SectionHeader)));
2442}
2443
2444fn getDataDirectoryHeadersOffset(coff: Coff) u32 {
2445 const msdos_hdr_size = msdos_stub.len + 4;
2446 return @as(u32, @intCast(msdos_hdr_size + @sizeOf(coff_util.CoffHeader) + coff.getOptionalHeaderSize()));
2447}
2448
2449fn getSectionHeadersOffset(coff: Coff) u32 {
2450 return coff.getDataDirectoryHeadersOffset() + coff.getDataDirectoryHeadersSize();
2451}
2452
2453fn getSizeOfImage(coff: Coff) u32 {
2454 var image_size: u32 = mem.alignForward(u32, coff.getSizeOfHeaders(), coff.page_size);
2455 for (coff.sections.items(.header)) |header| {
2456 image_size += mem.alignForward(u32, header.virtual_size, coff.page_size);
2457 }
2458 return image_size;
2459}
2460
2461/// Returns symbol location corresponding to the set entrypoint (if any).
2462pub fn getEntryPoint(coff: Coff) ?SymbolWithLoc {
2463 const comp = coff.base.comp;
2464
2465 // TODO This is incomplete.
2466 // The entry symbol name depends on the subsystem as well as the set of
2467 // public symbol names from linked objects.
2468 // See LinkerDriver::findDefaultEntry from the LLD project for the flow chart.
2469 const entry_name = switch (coff.entry) {
2470 .disabled => return null,
2471 .default => switch (comp.config.output_mode) {
2472 .Exe => "wWinMainCRTStartup",
2473 .Obj, .Lib => return null,
2474 },
2475 .enabled => "wWinMainCRTStartup",
2476 .named => |name| name,
2477 };
2478 const global_index = coff.resolver.get(entry_name) orelse return null;
2479 return coff.globals.items[global_index];
2480}
2481
2482/// Returns pointer-to-symbol described by `sym_loc` descriptor.
2483pub fn getSymbolPtr(coff: *Coff, sym_loc: SymbolWithLoc) *coff_util.Symbol {
2484 assert(sym_loc.file == null); // TODO linking object files
2485 return &coff.locals.items[sym_loc.sym_index];
2486}
2487
2488/// Returns symbol described by `sym_loc` descriptor.
2489pub fn getSymbol(coff: *const Coff, sym_loc: SymbolWithLoc) *const coff_util.Symbol {
2490 assert(sym_loc.file == null); // TODO linking object files
2491 return &coff.locals.items[sym_loc.sym_index];
2492}
2493
2494/// Returns name of the symbol described by `sym_loc` descriptor.
2495pub fn getSymbolName(coff: *const Coff, sym_loc: SymbolWithLoc) []const u8 {
2496 assert(sym_loc.file == null); // TODO linking object files
2497 const sym = coff.getSymbol(sym_loc);
2498 const offset = sym.getNameOffset() orelse return sym.getName().?;
2499 return coff.strtab.get(offset).?;
2500}
2501
2502/// Returns pointer to the global entry for `name` if one exists.
2503pub fn getGlobalPtr(coff: *Coff, name: []const u8) ?*SymbolWithLoc {
2504 const global_index = coff.resolver.get(name) orelse return null;
2505 return &coff.globals.items[global_index];
2506}
2507
2508/// Returns the global entry for `name` if one exists.
2509pub fn getGlobal(coff: *const Coff, name: []const u8) ?SymbolWithLoc {
2510 const global_index = coff.resolver.get(name) orelse return null;
2511 return coff.globals.items[global_index];
2512}
2513
2514/// Returns the index of the global entry for `name` if one exists.
2515pub fn getGlobalIndex(coff: *const Coff, name: []const u8) ?u32 {
2516 return coff.resolver.get(name);
2517}
2518
2519/// Returns global entry at `index`.
2520pub fn getGlobalByIndex(coff: *const Coff, index: u32) SymbolWithLoc {
2521 assert(index < coff.globals.items.len);
2522 return coff.globals.items[index];
2523}
2524
2525const GetOrPutGlobalPtrResult = struct {
2526 found_existing: bool,
2527 value_ptr: *SymbolWithLoc,
2528};
2529
2530/// Return pointer to the global entry for `name` if one exists.
2531/// Puts a new global entry for `name` if one doesn't exist, and
2532/// returns a pointer to it.
2533pub fn getOrPutGlobalPtr(coff: *Coff, name: []const u8) !GetOrPutGlobalPtrResult {
2534 if (coff.getGlobalPtr(name)) |ptr| {
2535 return GetOrPutGlobalPtrResult{ .found_existing = true, .value_ptr = ptr };
2536 }
2537 const gpa = coff.base.comp.gpa;
2538 const global_index = try coff.allocateGlobal();
2539 const global_name = try gpa.dupe(u8, name);
2540 _ = try coff.resolver.put(gpa, global_name, global_index);
2541 const ptr = &coff.globals.items[global_index];
2542 return GetOrPutGlobalPtrResult{ .found_existing = false, .value_ptr = ptr };
2543}
2544
2545pub fn getAtom(coff: *const Coff, atom_index: Atom.Index) Atom {
2546 assert(atom_index < coff.atoms.items.len);
2547 return coff.atoms.items[atom_index];
2548}
2549
2550pub fn getAtomPtr(coff: *Coff, atom_index: Atom.Index) *Atom {
2551 assert(atom_index < coff.atoms.items.len);
2552 return &coff.atoms.items[atom_index];
2553}
2554
2555/// Returns atom if there is an atom referenced by the symbol described by `sym_loc` descriptor.
2556/// Returns null on failure.
2557pub fn getAtomIndexForSymbol(coff: *const Coff, sym_loc: SymbolWithLoc) ?Atom.Index {
2558 assert(sym_loc.file == null); // TODO linking with object files
2559 return coff.atom_by_index_table.get(sym_loc.sym_index);
2560}
2561
2562fn setSectionName(coff: *Coff, header: *coff_util.SectionHeader, name: []const u8) !void {
2563 if (name.len <= 8) {
2564 @memcpy(header.name[0..name.len], name);
2565 @memset(header.name[name.len..], 0);
2566 return;
2567 }
2568 const gpa = coff.base.comp.gpa;
2569 const offset = try coff.strtab.insert(gpa, name);
2570 const name_offset = fmt.bufPrint(&header.name, "/{d}", .{offset}) catch unreachable;
2571 @memset(header.name[name_offset.len..], 0);
2572}
2573
2574fn getSectionName(coff: *const Coff, header: *const coff_util.SectionHeader) []const u8 {
2575 if (header.getName()) |name| {
2576 return name;
2577 }
2578 const offset = header.getNameOffset().?;
2579 return coff.strtab.get(offset).?;
2580}
2581
2582fn setSymbolName(coff: *Coff, symbol: *coff_util.Symbol, name: []const u8) !void {
2583 if (name.len <= 8) {
2584 @memcpy(symbol.name[0..name.len], name);
2585 @memset(symbol.name[name.len..], 0);
2586 return;
2587 }
2588 const gpa = coff.base.comp.gpa;
2589 const offset = try coff.strtab.insert(gpa, name);
2590 @memset(symbol.name[0..4], 0);
2591 mem.writeInt(u32, symbol.name[4..8], offset, .little);
2592}
2593
2594fn logSymAttributes(sym: *const coff_util.Symbol, buf: *[4]u8) []const u8 {
2595 @memset(buf[0..4], '_');
2596 switch (sym.section_number) {
2597 .UNDEFINED => {
2598 buf[3] = 'u';
2599 switch (sym.storage_class) {
2600 .EXTERNAL => buf[1] = 'e',
2601 .WEAK_EXTERNAL => buf[1] = 'w',
2602 .NULL => {},
2603 else => unreachable,
2604 }
2605 },
2606 .ABSOLUTE => unreachable, // handle ABSOLUTE
2607 .DEBUG => unreachable,
2608 else => {
2609 buf[0] = 's';
2610 switch (sym.storage_class) {
2611 .EXTERNAL => buf[1] = 'e',
2612 .WEAK_EXTERNAL => buf[1] = 'w',
2613 .NULL => {},
2614 else => unreachable,
2615 }
2616 },
2617 }
2618 return buf[0..];
2619}
2620
2621fn logSymtab(coff: *Coff) void {
2622 var buf: [4]u8 = undefined;
2623
2624 log.debug("symtab:", .{});
2625 log.debug(" object(null)", .{});
2626 for (coff.locals.items, 0..) |*sym, sym_id| {
2627 const where = if (sym.section_number == .UNDEFINED) "ord" else "sect";
2628 const def_index: u16 = switch (sym.section_number) {
2629 .UNDEFINED => 0, // TODO
2630 .ABSOLUTE => unreachable, // TODO
2631 .DEBUG => unreachable, // TODO
2632 else => @intFromEnum(sym.section_number),
2633 };
2634 log.debug(" %{d}: {s} @{x} in {s}({d}), {s}", .{
2635 sym_id,
2636 coff.getSymbolName(.{ .sym_index = @as(u32, @intCast(sym_id)), .file = null }),
2637 sym.value,
2638 where,
2639 def_index,
2640 logSymAttributes(sym, &buf),
2641 });
2642 }
2643
2644 log.debug("globals table:", .{});
2645 for (coff.globals.items) |sym_loc| {
2646 const sym_name = coff.getSymbolName(sym_loc);
2647 log.debug(" {s} => %{d} in object({?d})", .{ sym_name, sym_loc.sym_index, sym_loc.file });
2648 }
2649
2650 log.debug("GOT entries:", .{});
2651 log.debug("{f}", .{coff.got_table});
2652}
2653
2654fn logSections(coff: *Coff) void {
2655 log.debug("sections:", .{});
2656 for (coff.sections.items(.header)) |*header| {
2657 log.debug(" {s}: VM({x}, {x}) FILE({x}, {x})", .{
2658 coff.getSectionName(header),
2659 header.virtual_address,
2660 header.virtual_address + header.virtual_size,
2661 header.pointer_to_raw_data,
2662 header.pointer_to_raw_data + header.size_of_raw_data,
2663 });
2664 }
2665}
2666
2667fn logImportTables(coff: *const Coff) void {
2668 log.debug("import tables:", .{});
2669 for (coff.import_tables.keys(), 0..) |off, i| {
2670 const itable = coff.import_tables.values()[i];
2671 log.debug("{f}", .{itable.fmtDebug(.{
2672 .coff = coff,
2673 .index = i,
2674 .name_off = off,
2675 })});
2676 }
2677}
2678
2679pub const Atom = struct {
2680 /// Each decl always gets a local symbol with the fully qualified name.
2681 /// The vaddr and size are found here directly.
2682 /// The file offset is found by computing the vaddr offset from the section vaddr
2683 /// the symbol references, and adding that to the file offset of the section.
2684 /// If this field is 0, it means the codegen size = 0 and there is no symbol or
2685 /// offset table entry.
2686 sym_index: u32,
2687
2688 /// null means symbol defined by Zig source.
2689 file: ?u32,
2690
2691 /// Size of the atom
2692 size: u32,
2693
2694 /// Points to the previous and next neighbors, based on the `text_offset`.
2695 /// This can be used to find, for example, the capacity of this `Atom`.
2696 prev_index: ?Index,
2697 next_index: ?Index,
2698
2699 const Index = u32;
2700
2701 pub fn getSymbolIndex(atom: Atom) ?u32 {
2702 if (atom.sym_index == 0) return null;
2703 return atom.sym_index;
2704 }
2705
2706 /// Returns symbol referencing this atom.
2707 fn getSymbol(atom: Atom, coff: *const Coff) *const coff_util.Symbol {
2708 const sym_index = atom.getSymbolIndex().?;
2709 return coff.getSymbol(.{
2710 .sym_index = sym_index,
2711 .file = atom.file,
2712 });
2713 }
2714
2715 /// Returns pointer-to-symbol referencing this atom.
2716 fn getSymbolPtr(atom: Atom, coff: *Coff) *coff_util.Symbol {
2717 const sym_index = atom.getSymbolIndex().?;
2718 return coff.getSymbolPtr(.{
2719 .sym_index = sym_index,
2720 .file = atom.file,
2721 });
2722 }
2723
2724 fn getSymbolWithLoc(atom: Atom) SymbolWithLoc {
2725 const sym_index = atom.getSymbolIndex().?;
2726 return .{ .sym_index = sym_index, .file = atom.file };
2727 }
2728
2729 /// Returns the name of this atom.
2730 fn getName(atom: Atom, coff: *const Coff) []const u8 {
2731 const sym_index = atom.getSymbolIndex().?;
2732 return coff.getSymbolName(.{
2733 .sym_index = sym_index,
2734 .file = atom.file,
2735 });
2736 }
2737
2738 /// Returns how much room there is to grow in virtual address space.
2739 fn capacity(atom: Atom, coff: *const Coff) u32 {
2740 const atom_sym = atom.getSymbol(coff);
2741 if (atom.next_index) |next_index| {
2742 const next = coff.getAtom(next_index);
2743 const next_sym = next.getSymbol(coff);
2744 return next_sym.value - atom_sym.value;
2745 } else {
2746 // We are the last atom.
2747 // The capacity is limited only by virtual address space.
2748 return std.math.maxInt(u32) - atom_sym.value;
2749 }
2750 }
2751
2752 fn freeListEligible(atom: Atom, coff: *const Coff) bool {
2753 // No need to keep a free list node for the last atom.
2754 const next_index = atom.next_index orelse return false;
2755 const next = coff.getAtom(next_index);
2756 const atom_sym = atom.getSymbol(coff);
2757 const next_sym = next.getSymbol(coff);
2758 const cap = next_sym.value - atom_sym.value;
2759 const ideal_cap = padToIdeal(atom.size);
2760 if (cap <= ideal_cap) return false;
2761 const surplus = cap - ideal_cap;
2762 return surplus >= min_text_capacity;
2763 }
2764};
2765
2766pub const Relocation = struct {
2767 type: enum {
2768 // x86, x86_64
2769 /// RIP-relative displacement to a GOT pointer
2770 got,
2771 /// RIP-relative displacement to an import pointer
2772 import,
2773
2774 // aarch64
2775 /// PC-relative distance to target page in GOT section
2776 got_page,
2777 /// Offset to a GOT pointer relative to the start of a page in GOT section
2778 got_pageoff,
2779 /// PC-relative distance to target page in a section (e.g., .rdata)
2780 page,
2781 /// Offset to a pointer relative to the start of a page in a section (e.g., .rdata)
2782 pageoff,
2783 /// PC-relative distance to target page in a import section
2784 import_page,
2785 /// Offset to a pointer relative to the start of a page in an import section (e.g., .rdata)
2786 import_pageoff,
2787
2788 // common
2789 /// Absolute pointer value
2790 direct,
2791 },
2792 target: SymbolWithLoc,
2793 offset: u32,
2794 addend: u32,
2795 pcrel: bool,
2796 length: u2,
2797 dirty: bool = true,
2798
2799 /// Returns true if and only if the reloc can be resolved.
2800 fn isResolvable(reloc: Relocation, coff: *Coff) bool {
2801 _ = reloc.getTargetAddress(coff) orelse return false;
2802 return true;
2803 }
2804
2805 fn isGotIndirection(reloc: Relocation) bool {
2806 return switch (reloc.type) {
2807 .got, .got_page, .got_pageoff => true,
2808 else => false,
2809 };
2810 }
2811
2812 /// Returns address of the target if any.
2813 fn getTargetAddress(reloc: Relocation, coff: *const Coff) ?u32 {
2814 switch (reloc.type) {
2815 .got, .got_page, .got_pageoff => {
2816 const got_index = coff.got_table.lookup.get(reloc.target) orelse return null;
2817 const header = coff.sections.items(.header)[coff.got_section_index.?];
2818 return header.virtual_address + got_index * coff.ptr_width.size();
2819 },
2820 .import, .import_page, .import_pageoff => {
2821 const sym = coff.getSymbol(reloc.target);
2822 const index = coff.import_tables.getIndex(sym.value) orelse return null;
2823 const itab = coff.import_tables.values()[index];
2824 return itab.getImportAddress(reloc.target, .{
2825 .coff = coff,
2826 .index = index,
2827 .name_off = sym.value,
2828 });
2829 },
2830 else => {
2831 const target_atom_index = coff.getAtomIndexForSymbol(reloc.target) orelse return null;
2832 const target_atom = coff.getAtom(target_atom_index);
2833 return target_atom.getSymbol(coff).value;
2834 },
2835 }
2836 }
2837
2838 fn resolve(reloc: Relocation, atom_index: Atom.Index, code: []u8, image_base: u64, coff: *Coff) void {
2839 const atom = coff.getAtom(atom_index);
2840 const source_sym = atom.getSymbol(coff);
2841 const source_vaddr = source_sym.value + reloc.offset;
2842
2843 const target_vaddr = reloc.getTargetAddress(coff).?; // Oops, you didn't check if the relocation can be resolved with isResolvable().
2844 const target_vaddr_with_addend = target_vaddr + reloc.addend;
2845
2846 log.debug(" ({x}: [() => 0x{x} ({s})) ({s}) ", .{
2847 source_vaddr,
2848 target_vaddr_with_addend,
2849 coff.getSymbolName(reloc.target),
2850 @tagName(reloc.type),
2851 });
2852
2853 const ctx: Context = .{
2854 .source_vaddr = source_vaddr,
2855 .target_vaddr = target_vaddr_with_addend,
2856 .image_base = image_base,
2857 .code = code,
2858 .ptr_width = coff.ptr_width,
2859 };
2860
2861 const target = &coff.base.comp.root_mod.resolved_target.result;
2862 switch (target.cpu.arch) {
2863 .aarch64 => reloc.resolveAarch64(ctx),
2864 .x86, .x86_64 => reloc.resolveX86(ctx),
2865 else => unreachable, // unhandled target architecture
2866 }
2867 }
2868
2869 const Context = struct {
2870 source_vaddr: u32,
2871 target_vaddr: u32,
2872 image_base: u64,
2873 code: []u8,
2874 ptr_width: PtrWidth,
2875 };
2876
2877 fn resolveAarch64(reloc: Relocation, ctx: Context) void {
2878 const Instruction = aarch64_util.encoding.Instruction;
2879 var buffer = ctx.code[reloc.offset..];
2880 switch (reloc.type) {
2881 .got_page, .import_page, .page => {
2882 const source_page = @as(i32, @intCast(ctx.source_vaddr >> 12));
2883 const target_page = @as(i32, @intCast(ctx.target_vaddr >> 12));
2884 const pages: i21 = @intCast(target_page - source_page);
2885 var inst: Instruction = .read(buffer[0..Instruction.size]);
2886 inst.data_processing_immediate.pc_relative_addressing.group.immhi = @intCast(pages >> 2);
2887 inst.data_processing_immediate.pc_relative_addressing.group.immlo = @truncate(@as(u21, @bitCast(pages)));
2888 inst.write(buffer[0..Instruction.size]);
2889 },
2890 .got_pageoff, .import_pageoff, .pageoff => {
2891 assert(!reloc.pcrel);
2892
2893 const narrowed: u12 = @truncate(@as(u64, @intCast(ctx.target_vaddr)));
2894 var inst: Instruction = .read(buffer[0..Instruction.size]);
2895 switch (inst.decode()) {
2896 else => unreachable,
2897 .data_processing_immediate => inst.data_processing_immediate.add_subtract_immediate.group.imm12 = narrowed,
2898 .load_store => |load_store| inst.load_store.register_unsigned_immediate.group.imm12 =
2899 switch (load_store.register_unsigned_immediate.decode()) {
2900 .integer => |integer| @shrExact(narrowed, @intFromEnum(integer.group.size)),
2901 .vector => |vector| @shrExact(narrowed, @intFromEnum(vector.group.opc1.decode(vector.group.size))),
2902 },
2903 }
2904 inst.write(buffer[0..Instruction.size]);
2905 },
2906 .direct => {
2907 assert(!reloc.pcrel);
2908 switch (reloc.length) {
2909 2 => mem.writeInt(
2910 u32,
2911 buffer[0..4],
2912 @as(u32, @truncate(ctx.target_vaddr + ctx.image_base)),
2913 .little,
2914 ),
2915 3 => mem.writeInt(u64, buffer[0..8], ctx.target_vaddr + ctx.image_base, .little),
2916 else => unreachable,
2917 }
2918 },
2919
2920 .got => unreachable,
2921 .import => unreachable,
2922 }
2923 }
2924
2925 fn resolveX86(reloc: Relocation, ctx: Context) void {
2926 var buffer = ctx.code[reloc.offset..];
2927 switch (reloc.type) {
2928 .got_page => unreachable,
2929 .got_pageoff => unreachable,
2930 .page => unreachable,
2931 .pageoff => unreachable,
2932 .import_page => unreachable,
2933 .import_pageoff => unreachable,
2934
2935 .got, .import => {
2936 assert(reloc.pcrel);
2937 const disp = @as(i32, @intCast(ctx.target_vaddr)) - @as(i32, @intCast(ctx.source_vaddr)) - 4;
2938 mem.writeInt(i32, buffer[0..4], disp, .little);
2939 },
2940 .direct => {
2941 if (reloc.pcrel) {
2942 const disp = @as(i32, @intCast(ctx.target_vaddr)) - @as(i32, @intCast(ctx.source_vaddr)) - 4;
2943 mem.writeInt(i32, buffer[0..4], disp, .little);
2944 } else switch (ctx.ptr_width) {
2945 .p32 => mem.writeInt(u32, buffer[0..4], @as(u32, @intCast(ctx.target_vaddr + ctx.image_base)), .little),
2946 .p64 => switch (reloc.length) {
2947 2 => mem.writeInt(u32, buffer[0..4], @as(u32, @truncate(ctx.target_vaddr + ctx.image_base)), .little),
2948 3 => mem.writeInt(u64, buffer[0..8], ctx.target_vaddr + ctx.image_base, .little),
2949 else => unreachable,
2950 },
2951 }
2952 },
2953 }
2954 }
2955};
2956
2957pub fn addRelocation(coff: *Coff, atom_index: Atom.Index, reloc: Relocation) !void {
2958 const comp = coff.base.comp;
2959 const gpa = comp.gpa;
2960 log.debug(" (adding reloc of type {s} to target %{d})", .{ @tagName(reloc.type), reloc.target.sym_index });
2961 const gop = try coff.relocs.getOrPut(gpa, atom_index);
2962 if (!gop.found_existing) {
2963 gop.value_ptr.* = .{};
2964 }
2965 try gop.value_ptr.append(gpa, reloc);
2966}
2967
2968fn addBaseRelocation(coff: *Coff, atom_index: Atom.Index, offset: u32) !void {
2969 const comp = coff.base.comp;
2970 const gpa = comp.gpa;
2971 log.debug(" (adding base relocation at offset 0x{x} in %{d})", .{
2972 offset,
2973 coff.getAtom(atom_index).getSymbolIndex().?,
2974 });
2975 const gop = try coff.base_relocs.getOrPut(gpa, atom_index);
2976 if (!gop.found_existing) {
2977 gop.value_ptr.* = .{};
2978 }
2979 try gop.value_ptr.append(gpa, offset);
2980}
2981
2982fn freeRelocations(coff: *Coff, atom_index: Atom.Index) void {
2983 const comp = coff.base.comp;
2984 const gpa = comp.gpa;
2985 var removed_relocs = coff.relocs.fetchOrderedRemove(atom_index);
2986 if (removed_relocs) |*relocs| relocs.value.deinit(gpa);
2987 var removed_base_relocs = coff.base_relocs.fetchOrderedRemove(atom_index);
2988 if (removed_base_relocs) |*base_relocs| base_relocs.value.deinit(gpa);
2989}
2990
2991/// Represents an import table in the .idata section where each contained pointer
2992/// is to a symbol from the same DLL.
2993///
2994/// The layout of .idata section is as follows:
2995///
2996/// --- ADDR1 : IAT (all import tables concatenated together)
2997/// ptr
2998/// ptr
2999/// 0 sentinel
3000/// ptr
3001/// 0 sentinel
3002/// --- ADDR2: headers
3003/// ImportDirectoryEntry header
3004/// ImportDirectoryEntry header
3005/// sentinel
3006/// --- ADDR2: lookup tables
3007/// Lookup table
3008/// 0 sentinel
3009/// Lookup table
3010/// 0 sentinel
3011/// --- ADDR3: name hint tables
3012/// hint-symname
3013/// hint-symname
3014/// --- ADDR4: DLL names
3015/// DLL#1 name
3016/// DLL#2 name
3017/// --- END
3018const ImportTable = struct {
3019 entries: std.ArrayListUnmanaged(SymbolWithLoc) = .empty,
3020 free_list: std.ArrayListUnmanaged(u32) = .empty,
3021 lookup: std.AutoHashMapUnmanaged(SymbolWithLoc, u32) = .empty,
3022
3023 fn deinit(itab: *ImportTable, allocator: Allocator) void {
3024 itab.entries.deinit(allocator);
3025 itab.free_list.deinit(allocator);
3026 itab.lookup.deinit(allocator);
3027 }
3028
3029 /// Size of the import table does not include the sentinel.
3030 fn size(itab: ImportTable) u32 {
3031 return @as(u32, @intCast(itab.entries.items.len)) * @sizeOf(u64);
3032 }
3033
3034 fn addImport(itab: *ImportTable, allocator: Allocator, target: SymbolWithLoc) !ImportIndex {
3035 try itab.entries.ensureUnusedCapacity(allocator, 1);
3036 const index: u32 = blk: {
3037 if (itab.free_list.pop()) |index| {
3038 log.debug(" (reusing import entry index {d})", .{index});
3039 break :blk index;
3040 } else {
3041 log.debug(" (allocating import entry at index {d})", .{itab.entries.items.len});
3042 const index = @as(u32, @intCast(itab.entries.items.len));
3043 _ = itab.entries.addOneAssumeCapacity();
3044 break :blk index;
3045 }
3046 };
3047 itab.entries.items[index] = target;
3048 try itab.lookup.putNoClobber(allocator, target, index);
3049 return index;
3050 }
3051
3052 const Context = struct {
3053 coff: *const Coff,
3054 /// Index of this ImportTable in a global list of all tables.
3055 /// This is required in order to calculate the base vaddr of this ImportTable.
3056 index: usize,
3057 /// Offset into the string interning table of the DLL this ImportTable corresponds to.
3058 name_off: u32,
3059 };
3060
3061 fn getBaseAddress(ctx: Context) u32 {
3062 const header = ctx.coff.sections.items(.header)[ctx.coff.idata_section_index.?];
3063 var addr = header.virtual_address;
3064 for (ctx.coff.import_tables.values(), 0..) |other_itab, i| {
3065 if (ctx.index == i) break;
3066 addr += @as(u32, @intCast(other_itab.entries.items.len * @sizeOf(u64))) + 8;
3067 }
3068 return addr;
3069 }
3070
3071 fn getImportAddress(itab: *const ImportTable, target: SymbolWithLoc, ctx: Context) ?u32 {
3072 const index = itab.lookup.get(target) orelse return null;
3073 const base_vaddr = getBaseAddress(ctx);
3074 return base_vaddr + index * @sizeOf(u64);
3075 }
3076
3077 const Format = struct {
3078 itab: ImportTable,
3079 ctx: Context,
3080
3081 fn default(f: Format, writer: *std.Io.Writer) std.Io.Writer.Error!void {
3082 const lib_name = f.ctx.coff.temp_strtab.getAssumeExists(f.ctx.name_off);
3083 const base_vaddr = getBaseAddress(f.ctx);
3084 try writer.print("IAT({s}.dll) @{x}:", .{ lib_name, base_vaddr });
3085 for (f.itab.entries.items, 0..) |entry, i| {
3086 try writer.print("\n {d}@{?x} => {s}", .{
3087 i,
3088 f.itab.getImportAddress(entry, f.ctx),
3089 f.ctx.coff.getSymbolName(entry),
3090 });
3091 }
3092 }
3093 };
3094
3095 fn fmtDebug(itab: ImportTable, ctx: Context) fmt.Alt(Format, Format.default) {
3096 return .{ .data = .{ .itab = itab, .ctx = ctx } };
3097 }
3098
3099 const ImportIndex = u32;
3100};
3101
3102fn pwriteAll(coff: *Coff, bytes: []const u8, offset: u64) error{LinkFailure}!void {
3103 const comp = coff.base.comp;
3104 const diags = &comp.link_diags;
3105 coff.base.file.?.pwriteAll(bytes, offset) catch |err| {
3106 return diags.fail("failed to write: {s}", .{@errorName(err)});
3107 };
3108}
3109
3110/// This is the start of a Portable Executable (PE) file.
3111/// It starts with a MS-DOS header followed by a MS-DOS stub program.
3112/// This data does not change so we include it as follows in all binaries.
3113///
3114/// In this context,
3115/// A "paragraph" is 16 bytes.
3116/// A "page" is 512 bytes.
3117/// A "long" is 4 bytes.
3118/// A "word" is 2 bytes.
3119const msdos_stub: [120]u8 = .{
3120 'M', 'Z', // Magic number. Stands for Mark Zbikowski (designer of the MS-DOS executable format).
3121 0x78, 0x00, // Number of bytes in the last page. This matches the size of this entire MS-DOS stub.
3122 0x01, 0x00, // Number of pages.
3123 0x00, 0x00, // Number of entries in the relocation table.
3124 0x04, 0x00, // The number of paragraphs taken up by the header. 4 * 16 = 64, which matches the header size (all bytes before the MS-DOS stub program).
3125 0x00, 0x00, // The number of paragraphs required by the program.
3126 0x00, 0x00, // The number of paragraphs requested by the program.
3127 0x00, 0x00, // Initial value for SS (relocatable segment address).
3128 0x00, 0x00, // Initial value for SP.
3129 0x00, 0x00, // Checksum.
3130 0x00, 0x00, // Initial value for IP.
3131 0x00, 0x00, // Initial value for CS (relocatable segment address).
3132 0x40, 0x00, // Absolute offset to relocation table. 64 matches the header size (all bytes before the MS-DOS stub program).
3133 0x00, 0x00, // Overlay number. Zero means this is the main executable.
3134}
3135 // Reserved words.
3136 ++ .{ 0x00, 0x00 } ** 4
3137 // OEM-related fields.
3138 ++ .{
3139 0x00, 0x00, // OEM identifier.
3140 0x00, 0x00, // OEM information.
3141 }
3142 // Reserved words.
3143 ++ .{ 0x00, 0x00 } ** 10
3144 // Address of the PE header (a long). This matches the size of this entire MS-DOS stub, so that's the address of what's after this MS-DOS stub.
3145 ++ .{ 0x78, 0x00, 0x00, 0x00 }
3146 // What follows is a 16-bit x86 MS-DOS program of 7 instructions that prints the bytes after these instructions and then exits.
3147 ++ .{
3148 // Set the value of the data segment to the same value as the code segment.
3149 0x0e, // push cs
3150 0x1f, // pop ds
3151 // Set the DX register to the address of the message.
3152 // If you count all bytes of these 7 instructions you get 14, so that's the address of what's after these instructions.
3153 0xba, 14, 0x00, // mov dx, 14
3154 // Set AH to the system call code for printing a message.
3155 0xb4, 0x09, // mov ah, 0x09
3156 // Perform the system call to print the message.
3157 0xcd, 0x21, // int 0x21
3158 // Set AH to 0x4c which is the system call code for exiting, and set AL to 0x01 which is the exit code.
3159 0xb8, 0x01, 0x4c, // mov ax, 0x4c01
3160 // Peform the system call to exit the program with exit code 1.
3161 0xcd, 0x21, // int 0x21
3162 }
3163 // Message to print.
3164 ++ "This program cannot be run in DOS mode.".*
3165 // Message terminators.
3166 ++ .{
3167 '$', // We do not pass a length to the print system call; the string is terminated by this character.
3168 0x00, 0x00, // Terminating zero bytes.
3169 };
src/link/Coff2.zig created+2193
......@@ -0,0 +1,2193 @@
1base: link.File,
2endian: std.builtin.Endian,
3mf: MappedFile,
4nodes: std.MultiArrayList(Node),
5import_table: ImportTable,
6strings: std.HashMapUnmanaged(
7 u32,
8 void,
9 std.hash_map.StringIndexContext,
10 std.hash_map.default_max_load_percentage,
11),
12string_bytes: std.ArrayList(u8),
13section_table: std.ArrayList(Symbol.Index),
14symbol_table: std.ArrayList(Symbol),
15globals: std.AutoArrayHashMapUnmanaged(GlobalName, Symbol.Index),
16global_pending_index: u32,
17navs: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, Symbol.Index),
18uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Symbol.Index),
19lazy: std.EnumArray(link.File.LazySymbol.Kind, struct {
20 map: std.AutoArrayHashMapUnmanaged(InternPool.Index, Symbol.Index),
21 pending_index: u32,
22}),
23pending_uavs: std.AutoArrayHashMapUnmanaged(Node.UavMapIndex, struct {
24 alignment: InternPool.Alignment,
25 src_loc: Zcu.LazySrcLoc,
26}),
27relocs: std.ArrayList(Reloc),
28/// This is hiding actual bugs with global symbols! Reconsider once they are implemented correctly.
29entry_hack: Symbol.Index,
30
31pub const default_file_alignment: u16 = 0x200;
32pub const default_size_of_stack_reserve: u32 = 0x1000000;
33pub const default_size_of_stack_commit: u32 = 0x1000;
34pub const default_size_of_heap_reserve: u32 = 0x100000;
35pub const default_size_of_heap_commit: u32 = 0x1000;
36
37/// This is the start of a Portable Executable (PE) file.
38/// It starts with a MS-DOS header followed by a MS-DOS stub program.
39/// This data does not change so we include it as follows in all binaries.
40///
41/// In this context,
42/// A "paragraph" is 16 bytes.
43/// A "page" is 512 bytes.
44/// A "long" is 4 bytes.
45/// A "word" is 2 bytes.
46pub const msdos_stub: [120]u8 = .{
47 'M', 'Z', // Magic number. Stands for Mark Zbikowski (designer of the MS-DOS executable format).
48 0x78, 0x00, // Number of bytes in the last page. This matches the size of this entire MS-DOS stub.
49 0x01, 0x00, // Number of pages.
50 0x00, 0x00, // Number of entries in the relocation table.
51 0x04, 0x00, // The number of paragraphs taken up by the header. 4 * 16 = 64, which matches the header size (all bytes before the MS-DOS stub program).
52 0x00, 0x00, // The number of paragraphs required by the program.
53 0x00, 0x00, // The number of paragraphs requested by the program.
54 0x00, 0x00, // Initial value for SS (relocatable segment address).
55 0x00, 0x00, // Initial value for SP.
56 0x00, 0x00, // Checksum.
57 0x00, 0x00, // Initial value for IP.
58 0x00, 0x00, // Initial value for CS (relocatable segment address).
59 0x40, 0x00, // Absolute offset to relocation table. 64 matches the header size (all bytes before the MS-DOS stub program).
60 0x00, 0x00, // Overlay number. Zero means this is the main executable.
61}
62 // Reserved words.
63 ++ .{ 0x00, 0x00 } ** 4
64 // OEM-related fields.
65 ++ .{
66 0x00, 0x00, // OEM identifier.
67 0x00, 0x00, // OEM information.
68 }
69 // Reserved words.
70 ++ .{ 0x00, 0x00 } ** 10
71 // Address of the PE header (a long). This matches the size of this entire MS-DOS stub, so that's the address of what's after this MS-DOS stub.
72 ++ .{ 0x78, 0x00, 0x00, 0x00 }
73 // What follows is a 16-bit x86 MS-DOS program of 7 instructions that prints the bytes after these instructions and then exits.
74 ++ .{
75 // Set the value of the data segment to the same value as the code segment.
76 0x0e, // push cs
77 0x1f, // pop ds
78 // Set the DX register to the address of the message.
79 // If you count all bytes of these 7 instructions you get 14, so that's the address of what's after these instructions.
80 0xba, 14, 0x00, // mov dx, 14
81 // Set AH to the system call code for printing a message.
82 0xb4, 0x09, // mov ah, 0x09
83 // Perform the system call to print the message.
84 0xcd, 0x21, // int 0x21
85 // Set AH to 0x4c which is the system call code for exiting, and set AL to 0x01 which is the exit code.
86 0xb8, 0x01, 0x4c, // mov ax, 0x4c01
87 // Peform the system call to exit the program with exit code 1.
88 0xcd, 0x21, // int 0x21
89 }
90 // Message to print.
91 ++ "This program cannot be run in DOS mode.".*
92 // Message terminators.
93 ++ .{
94 '$', // We do not pass a length to the print system call; the string is terminated by this character.
95 0x00, 0x00, // Terminating zero bytes.
96 };
97
98pub const Node = union(enum) {
99 file,
100 header,
101 signature,
102 coff_header,
103 optional_header,
104 data_directories,
105 section_table,
106 section: Symbol.Index,
107 import_directory_table,
108 import_lookup_table: u32,
109 import_address_table: u32,
110 import_hint_name_table: u32,
111 global: GlobalMapIndex,
112 nav: NavMapIndex,
113 uav: UavMapIndex,
114 lazy_code: LazyMapRef.Index(.code),
115 lazy_const_data: LazyMapRef.Index(.const_data),
116
117 pub const GlobalMapIndex = enum(u32) {
118 _,
119
120 pub fn globalName(gmi: GlobalMapIndex, coff: *const Coff) GlobalName {
121 return coff.globals.keys()[@intFromEnum(gmi)];
122 }
123
124 pub fn symbol(gmi: GlobalMapIndex, coff: *const Coff) Symbol.Index {
125 return coff.globals.values()[@intFromEnum(gmi)];
126 }
127 };
128
129 pub const NavMapIndex = enum(u32) {
130 _,
131
132 pub fn navIndex(nmi: NavMapIndex, coff: *const Coff) InternPool.Nav.Index {
133 return coff.navs.keys()[@intFromEnum(nmi)];
134 }
135
136 pub fn symbol(nmi: NavMapIndex, coff: *const Coff) Symbol.Index {
137 return coff.navs.values()[@intFromEnum(nmi)];
138 }
139 };
140
141 pub const UavMapIndex = enum(u32) {
142 _,
143
144 pub fn uavValue(umi: UavMapIndex, coff: *const Coff) InternPool.Index {
145 return coff.uavs.keys()[@intFromEnum(umi)];
146 }
147
148 pub fn symbol(umi: UavMapIndex, coff: *const Coff) Symbol.Index {
149 return coff.uavs.values()[@intFromEnum(umi)];
150 }
151 };
152
153 pub const LazyMapRef = struct {
154 kind: link.File.LazySymbol.Kind,
155 index: u32,
156
157 pub fn Index(comptime kind: link.File.LazySymbol.Kind) type {
158 return enum(u32) {
159 _,
160
161 pub fn ref(lmi: @This()) LazyMapRef {
162 return .{ .kind = kind, .index = @intFromEnum(lmi) };
163 }
164
165 pub fn lazySymbol(lmi: @This(), coff: *const Coff) link.File.LazySymbol {
166 return lmi.ref().lazySymbol(coff);
167 }
168
169 pub fn symbol(lmi: @This(), coff: *const Coff) Symbol.Index {
170 return lmi.ref().symbol(coff);
171 }
172 };
173 }
174
175 pub fn lazySymbol(lmr: LazyMapRef, coff: *const Coff) link.File.LazySymbol {
176 return .{ .kind = lmr.kind, .ty = coff.lazy.getPtrConst(lmr.kind).map.keys()[lmr.index] };
177 }
178
179 pub fn symbol(lmr: LazyMapRef, coff: *const Coff) Symbol.Index {
180 return coff.lazy.getPtrConst(lmr.kind).map.values()[lmr.index];
181 }
182 };
183
184 pub const Tag = @typeInfo(Node).@"union".tag_type.?;
185
186 const known_count = @typeInfo(@TypeOf(known)).@"struct".fields.len;
187 const known = known: {
188 const Known = enum {
189 file,
190 header,
191 signature,
192 coff_header,
193 optional_header,
194 data_directories,
195 section_table,
196 };
197 var mut_known: std.enums.EnumFieldStruct(Known, MappedFile.Node.Index, null) = undefined;
198 for (@typeInfo(Known).@"enum".fields) |field|
199 @field(mut_known, field.name) = @enumFromInt(field.value);
200 break :known mut_known;
201 };
202
203 comptime {
204 if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 8);
205 }
206};
207
208pub const DataDirectory = enum {
209 export_table,
210 import_table,
211 resorce_table,
212 exception_table,
213 certificate_table,
214 base_relocation_table,
215 debug,
216 architecture,
217 global_ptr,
218 tls_table,
219 load_config_table,
220 bound_import,
221 import_address_table,
222 delay_import_descriptor,
223 clr_runtime_header,
224 reserved,
225};
226
227pub const ImportTable = struct {
228 directory_table_ni: MappedFile.Node.Index,
229 dlls: std.AutoArrayHashMapUnmanaged(void, Dll),
230
231 pub const Dll = struct {
232 import_lookup_table_ni: MappedFile.Node.Index,
233 import_address_table_si: Symbol.Index,
234 import_hint_name_table_ni: MappedFile.Node.Index,
235 len: u32,
236 hint_name_len: u32,
237 };
238
239 const Adapter = struct {
240 coff: *Coff,
241
242 pub fn eql(adapter: Adapter, lhs_key: []const u8, _: void, rhs_index: usize) bool {
243 const coff = adapter.coff;
244 const dll_name = coff.import_table.dlls.values()[rhs_index]
245 .import_hint_name_table_ni.sliceConst(&coff.mf);
246 return std.mem.startsWith(u8, dll_name, lhs_key) and
247 std.mem.startsWith(u8, dll_name[lhs_key.len..], ".dll\x00");
248 }
249
250 pub fn hash(_: Adapter, key: []const u8) u32 {
251 assert(std.mem.indexOfScalar(u8, key, 0) == null);
252 return std.array_hash_map.hashString(key);
253 }
254 };
255};
256
257pub const String = enum(u32) {
258 _,
259
260 pub const Optional = enum(u32) {
261 none = std.math.maxInt(u32),
262 _,
263
264 pub fn unwrap(os: String.Optional) ?String {
265 return switch (os) {
266 else => |s| @enumFromInt(@intFromEnum(s)),
267 .none => null,
268 };
269 }
270
271 pub fn toSlice(os: String.Optional, coff: *Coff) ?[:0]const u8 {
272 return (os.unwrap() orelse return null).toSlice(coff);
273 }
274 };
275
276 pub fn toSlice(s: String, coff: *Coff) [:0]const u8 {
277 const slice = coff.string_bytes.items[@intFromEnum(s)..];
278 return slice[0..std.mem.indexOfScalar(u8, slice, 0).? :0];
279 }
280
281 pub fn toOptional(s: String) String.Optional {
282 return @enumFromInt(@intFromEnum(s));
283 }
284};
285
286pub const GlobalName = struct { name: String, lib_name: String.Optional };
287
288pub const Symbol = struct {
289 ni: MappedFile.Node.Index,
290 rva: u32,
291 size: u32,
292 /// Relocations contained within this symbol
293 loc_relocs: Reloc.Index,
294 /// Relocations targeting this symbol
295 target_relocs: Reloc.Index,
296 section_number: SectionNumber,
297 data_directory: ?DataDirectory,
298 unused0: u32 = 0,
299 unused1: u32 = 0,
300
301 pub const SectionNumber = enum(i16) {
302 UNDEFINED = 0,
303 ABSOLUTE = -1,
304 DEBUG = -2,
305 _,
306
307 fn toIndex(sn: SectionNumber) u15 {
308 return @intCast(@intFromEnum(sn) - 1);
309 }
310
311 pub fn symbol(sn: SectionNumber, coff: *const Coff) Symbol.Index {
312 return coff.section_table.items[sn.toIndex()];
313 }
314
315 pub fn header(sn: SectionNumber, coff: *Coff) *std.coff.SectionHeader {
316 return &coff.sectionTableSlice()[sn.toIndex()];
317 }
318 };
319
320 pub const Index = enum(u32) {
321 null,
322 data,
323 idata,
324 rdata,
325 text,
326 _,
327
328 const known_count = @typeInfo(Index).@"enum".fields.len;
329
330 pub fn get(si: Symbol.Index, coff: *Coff) *Symbol {
331 return &coff.symbol_table.items[@intFromEnum(si)];
332 }
333
334 pub fn node(si: Symbol.Index, coff: *Coff) MappedFile.Node.Index {
335 const ni = si.get(coff).ni;
336 assert(ni != .none);
337 return ni;
338 }
339
340 pub fn flushMoved(si: Symbol.Index, coff: *Coff) void {
341 const sym = si.get(coff);
342 sym.rva = coff.computeNodeRva(sym.ni);
343 if (si == coff.entry_hack)
344 coff.targetStore(&coff.optionalHeaderStandardPtr().address_of_entry_point, sym.rva);
345 si.applyLocationRelocs(coff);
346 si.applyTargetRelocs(coff);
347 }
348
349 pub fn applyLocationRelocs(si: Symbol.Index, coff: *Coff) void {
350 for (coff.relocs.items[@intFromEnum(si.get(coff).loc_relocs)..]) |*reloc| {
351 if (reloc.loc != si) break;
352 reloc.apply(coff);
353 }
354 }
355
356 pub fn applyTargetRelocs(si: Symbol.Index, coff: *Coff) void {
357 var ri = si.get(coff).target_relocs;
358 while (ri != .none) {
359 const reloc = ri.get(coff);
360 assert(reloc.target == si);
361 reloc.apply(coff);
362 ri = reloc.next;
363 }
364 }
365
366 pub fn deleteLocationRelocs(si: Symbol.Index, coff: *Coff) void {
367 const sym = si.get(coff);
368 for (coff.relocs.items[@intFromEnum(sym.loc_relocs)..]) |*reloc| {
369 if (reloc.loc != si) break;
370 reloc.delete(coff);
371 }
372 sym.loc_relocs = .none;
373 }
374 };
375
376 comptime {
377 if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Symbol) == 32);
378 }
379};
380
381pub const Reloc = extern struct {
382 type: Reloc.Type,
383 prev: Reloc.Index,
384 next: Reloc.Index,
385 loc: Symbol.Index,
386 target: Symbol.Index,
387 unused: u32,
388 offset: u64,
389 addend: i64,
390
391 pub const Type = extern union {
392 AMD64: std.coff.IMAGE.REL.AMD64,
393 ARM: std.coff.IMAGE.REL.ARM,
394 ARM64: std.coff.IMAGE.REL.ARM64,
395 SH: std.coff.IMAGE.REL.SH,
396 PPC: std.coff.IMAGE.REL.PPC,
397 I386: std.coff.IMAGE.REL.I386,
398 IA64: std.coff.IMAGE.REL.IA64,
399 MIPS: std.coff.IMAGE.REL.MIPS,
400 M32R: std.coff.IMAGE.REL.M32R,
401 };
402
403 pub const Index = enum(u32) {
404 none = std.math.maxInt(u32),
405 _,
406
407 pub fn get(si: Reloc.Index, coff: *Coff) *Reloc {
408 return &coff.relocs.items[@intFromEnum(si)];
409 }
410 };
411
412 pub fn apply(reloc: *const Reloc, coff: *Coff) void {
413 const loc_sym = reloc.loc.get(coff);
414 switch (loc_sym.ni) {
415 .none => return,
416 else => |ni| if (ni.hasMoved(&coff.mf)) return,
417 }
418 const target_sym = reloc.target.get(coff);
419 switch (target_sym.ni) {
420 .none => return,
421 else => |ni| if (ni.hasMoved(&coff.mf)) return,
422 }
423 const loc_slice = loc_sym.ni.slice(&coff.mf)[@intCast(reloc.offset)..];
424 const target_rva = target_sym.rva +% @as(u64, @bitCast(reloc.addend));
425 const target_endian = coff.targetEndian();
426 switch (coff.targetLoad(&coff.headerPtr().machine)) {
427 else => |machine| @panic(@tagName(machine)),
428 .AMD64 => switch (reloc.type.AMD64) {
429 else => |kind| @panic(@tagName(kind)),
430 .ABSOLUTE => {},
431 .ADDR64 => std.mem.writeInt(
432 u64,
433 loc_slice[0..8],
434 coff.optionalHeaderField(.image_base) + target_rva,
435 target_endian,
436 ),
437 .ADDR32 => std.mem.writeInt(
438 u32,
439 loc_slice[0..4],
440 @intCast(coff.optionalHeaderField(.image_base) + target_rva),
441 target_endian,
442 ),
443 .ADDR32NB => std.mem.writeInt(
444 u32,
445 loc_slice[0..4],
446 @intCast(target_rva),
447 target_endian,
448 ),
449 .REL32 => std.mem.writeInt(
450 i32,
451 loc_slice[0..4],
452 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 4)))),
453 target_endian,
454 ),
455 .REL32_1 => std.mem.writeInt(
456 i32,
457 loc_slice[0..4],
458 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 5)))),
459 target_endian,
460 ),
461 .REL32_2 => std.mem.writeInt(
462 i32,
463 loc_slice[0..4],
464 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 6)))),
465 target_endian,
466 ),
467 .REL32_3 => std.mem.writeInt(
468 i32,
469 loc_slice[0..4],
470 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 7)))),
471 target_endian,
472 ),
473 .REL32_4 => std.mem.writeInt(
474 i32,
475 loc_slice[0..4],
476 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 8)))),
477 target_endian,
478 ),
479 .REL32_5 => std.mem.writeInt(
480 i32,
481 loc_slice[0..4],
482 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 9)))),
483 target_endian,
484 ),
485 },
486 .I386 => switch (reloc.type.I386) {
487 else => |kind| @panic(@tagName(kind)),
488 .ABSOLUTE => {},
489 .DIR16 => std.mem.writeInt(
490 u16,
491 loc_slice[0..2],
492 @intCast(coff.optionalHeaderField(.image_base) + target_rva),
493 target_endian,
494 ),
495 .REL16 => std.mem.writeInt(
496 i16,
497 loc_slice[0..2],
498 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 2)))),
499 target_endian,
500 ),
501 .DIR32 => std.mem.writeInt(
502 u32,
503 loc_slice[0..4],
504 @intCast(coff.optionalHeaderField(.image_base) + target_rva),
505 target_endian,
506 ),
507 .DIR32NB => std.mem.writeInt(
508 u32,
509 loc_slice[0..4],
510 @intCast(target_rva),
511 target_endian,
512 ),
513 .REL32 => std.mem.writeInt(
514 i32,
515 loc_slice[0..4],
516 @intCast(@as(i64, @bitCast(target_rva -% (loc_sym.rva + reloc.offset + 4)))),
517 target_endian,
518 ),
519 },
520 }
521 }
522
523 pub fn delete(reloc: *Reloc, coff: *Coff) void {
524 switch (reloc.prev) {
525 .none => {
526 const target = reloc.target.get(coff);
527 assert(target.target_relocs.get(coff) == reloc);
528 target.target_relocs = reloc.next;
529 },
530 else => |prev| prev.get(coff).next = reloc.next,
531 }
532 switch (reloc.next) {
533 .none => {},
534 else => |next| next.get(coff).prev = reloc.prev,
535 }
536 reloc.* = undefined;
537 }
538
539 comptime {
540 if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Reloc) == 40);
541 }
542};
543
544pub fn open(
545 arena: std.mem.Allocator,
546 comp: *Compilation,
547 path: std.Build.Cache.Path,
548 options: link.File.OpenOptions,
549) !*Coff {
550 return create(arena, comp, path, options);
551}
552pub fn createEmpty(
553 arena: std.mem.Allocator,
554 comp: *Compilation,
555 path: std.Build.Cache.Path,
556 options: link.File.OpenOptions,
557) !*Coff {
558 return create(arena, comp, path, options);
559}
560fn create(
561 arena: std.mem.Allocator,
562 comp: *Compilation,
563 path: std.Build.Cache.Path,
564 options: link.File.OpenOptions,
565) !*Coff {
566 const target = &comp.root_mod.resolved_target.result;
567 assert(target.ofmt == .coff);
568 const is_image = switch (comp.config.output_mode) {
569 .Exe => true,
570 .Lib => switch (comp.config.link_mode) {
571 .static => false,
572 .dynamic => true,
573 },
574 .Obj => false,
575 };
576 const machine = target.toCoffMachine();
577 const timestamp: u32 = if (options.repro) 0 else @truncate(@as(u64, @bitCast(std.time.timestamp())));
578 const major_subsystem_version = options.major_subsystem_version orelse 6;
579 const minor_subsystem_version = options.minor_subsystem_version orelse 0;
580 const magic: std.coff.OptionalHeader.Magic = switch (target.ptrBitWidth()) {
581 0...32 => .PE32,
582 33...64 => .@"PE32+",
583 else => return error.UnsupportedCOFFArchitecture,
584 };
585 const section_align: std.mem.Alignment = switch (machine) {
586 .AMD64, .I386 => @enumFromInt(12),
587 .SH3, .SH3DSP, .SH4, .SH5 => @enumFromInt(12),
588 .MIPS16, .MIPSFPU, .MIPSFPU16, .WCEMIPSV2 => @enumFromInt(12),
589 .POWERPC, .POWERPCFP => @enumFromInt(12),
590 .ALPHA, .ALPHA64 => @enumFromInt(13),
591 .IA64 => @enumFromInt(13),
592 .ARM => @enumFromInt(12),
593 else => return error.UnsupportedCOFFArchitecture,
594 };
595
596 const coff = try arena.create(Coff);
597 const file = try path.root_dir.handle.createFile(path.sub_path, .{
598 .read = true,
599 .mode = link.File.determineMode(comp.config.output_mode, comp.config.link_mode),
600 });
601 errdefer file.close();
602 coff.* = .{
603 .base = .{
604 .tag = .coff2,
605
606 .comp = comp,
607 .emit = path,
608
609 .file = file,
610 .gc_sections = false,
611 .print_gc_sections = false,
612 .build_id = .none,
613 .allow_shlib_undefined = false,
614 .stack_size = 0,
615 },
616 .endian = target.cpu.arch.endian(),
617 .mf = try .init(file, comp.gpa),
618 .nodes = .empty,
619 .import_table = .{
620 .directory_table_ni = .none,
621 .dlls = .empty,
622 },
623 .strings = .empty,
624 .string_bytes = .empty,
625 .section_table = .empty,
626 .symbol_table = .empty,
627 .globals = .empty,
628 .global_pending_index = 0,
629 .navs = .empty,
630 .uavs = .empty,
631 .lazy = .initFill(.{
632 .map = .empty,
633 .pending_index = 0,
634 }),
635 .pending_uavs = .empty,
636 .relocs = .empty,
637 .entry_hack = .null,
638 };
639 errdefer coff.deinit();
640
641 try coff.initHeaders(
642 is_image,
643 machine,
644 timestamp,
645 major_subsystem_version,
646 minor_subsystem_version,
647 magic,
648 section_align,
649 );
650 return coff;
651}
652
653pub fn deinit(coff: *Coff) void {
654 const gpa = coff.base.comp.gpa;
655 coff.mf.deinit(gpa);
656 coff.nodes.deinit(gpa);
657 coff.import_table.dlls.deinit(gpa);
658 coff.strings.deinit(gpa);
659 coff.string_bytes.deinit(gpa);
660 coff.section_table.deinit(gpa);
661 coff.symbol_table.deinit(gpa);
662 coff.globals.deinit(gpa);
663 coff.navs.deinit(gpa);
664 coff.uavs.deinit(gpa);
665 for (&coff.lazy.values) |*lazy| lazy.map.deinit(gpa);
666 coff.pending_uavs.deinit(gpa);
667 coff.relocs.deinit(gpa);
668 coff.* = undefined;
669}
670
671fn initHeaders(
672 coff: *Coff,
673 is_image: bool,
674 machine: std.coff.IMAGE.FILE.MACHINE,
675 timestamp: u32,
676 major_subsystem_version: u16,
677 minor_subsystem_version: u16,
678 magic: std.coff.OptionalHeader.Magic,
679 section_align: std.mem.Alignment,
680) !void {
681 const comp = coff.base.comp;
682 const gpa = comp.gpa;
683 const file_align: std.mem.Alignment = comptime .fromByteUnits(default_file_alignment);
684 const target_endian = coff.targetEndian();
685
686 const optional_header_size: u16 = if (is_image) switch (magic) {
687 _ => unreachable,
688 inline else => |ct_magic| @sizeOf(@field(std.coff.OptionalHeader, @tagName(ct_magic))),
689 } else 0;
690 const data_directories_len = @typeInfo(DataDirectory).@"enum".fields.len;
691 const data_directories_size: u16 = if (is_image)
692 @sizeOf(std.coff.ImageDataDirectory) * data_directories_len
693 else
694 0;
695
696 try coff.nodes.ensureTotalCapacity(gpa, Node.known_count);
697 coff.nodes.appendAssumeCapacity(.file);
698
699 const header_ni = Node.known.header;
700 assert(header_ni == try coff.mf.addOnlyChildNode(gpa, .root, .{
701 .alignment = coff.mf.flags.block_size,
702 .fixed = true,
703 }));
704 coff.nodes.appendAssumeCapacity(.header);
705
706 const signature_ni = Node.known.signature;
707 assert(signature_ni == try coff.mf.addOnlyChildNode(gpa, header_ni, .{
708 .size = (if (is_image) msdos_stub.len else 0) + "PE\x00\x00".len,
709 .alignment = .@"4",
710 .fixed = true,
711 }));
712 coff.nodes.appendAssumeCapacity(.signature);
713 {
714 const signature_slice = signature_ni.slice(&coff.mf);
715 if (is_image) @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub);
716 @memcpy(signature_slice[signature_slice.len - 4 ..], "PE\x00\x00");
717 }
718
719 const coff_header_ni = Node.known.coff_header;
720 assert(coff_header_ni == try coff.mf.addLastChildNode(gpa, header_ni, .{
721 .size = @sizeOf(std.coff.Header),
722 .alignment = .@"4",
723 .fixed = true,
724 }));
725 coff.nodes.appendAssumeCapacity(.coff_header);
726 {
727 const coff_header: *std.coff.Header = @ptrCast(@alignCast(coff_header_ni.slice(&coff.mf)));
728 coff_header.* = .{
729 .machine = machine,
730 .number_of_sections = 0,
731 .time_date_stamp = timestamp,
732 .pointer_to_symbol_table = 0,
733 .number_of_symbols = 0,
734 .size_of_optional_header = optional_header_size + data_directories_size,
735 .flags = .{
736 .RELOCS_STRIPPED = is_image,
737 .EXECUTABLE_IMAGE = is_image,
738 .DEBUG_STRIPPED = true,
739 .@"32BIT_MACHINE" = magic == .PE32,
740 .LARGE_ADDRESS_AWARE = magic == .@"PE32+",
741 .DLL = comp.config.output_mode == .Lib and comp.config.link_mode == .dynamic,
742 },
743 };
744 if (target_endian != native_endian) std.mem.byteSwapAllFields(std.coff.Header, coff_header);
745 }
746
747 const optional_header_ni = Node.known.optional_header;
748 assert(optional_header_ni == try coff.mf.addLastChildNode(gpa, header_ni, .{
749 .size = optional_header_size,
750 .alignment = .@"4",
751 .fixed = true,
752 }));
753 coff.nodes.appendAssumeCapacity(.optional_header);
754 if (is_image) switch (magic) {
755 _ => unreachable,
756 .PE32 => {
757 const optional_header: *std.coff.OptionalHeader.PE32 =
758 @ptrCast(@alignCast(optional_header_ni.slice(&coff.mf)));
759 optional_header.* = .{
760 .standard = .{
761 .magic = .PE32,
762 .major_linker_version = 0,
763 .minor_linker_version = 0,
764 .size_of_code = 0,
765 .size_of_initialized_data = 0,
766 .size_of_uninitialized_data = 0,
767 .address_of_entry_point = 0,
768 .base_of_code = 0,
769 },
770 .base_of_data = 0,
771 .image_base = switch (coff.base.comp.config.output_mode) {
772 .Exe => 0x400000,
773 .Lib => switch (coff.base.comp.config.link_mode) {
774 .static => 0,
775 .dynamic => 0x10000000,
776 },
777 .Obj => 0,
778 },
779 .section_alignment = @intCast(section_align.toByteUnits()),
780 .file_alignment = @intCast(file_align.toByteUnits()),
781 .major_operating_system_version = 6,
782 .minor_operating_system_version = 0,
783 .major_image_version = 0,
784 .minor_image_version = 0,
785 .major_subsystem_version = major_subsystem_version,
786 .minor_subsystem_version = minor_subsystem_version,
787 .win32_version_value = 0,
788 .size_of_image = 0,
789 .size_of_headers = 0,
790 .checksum = 0,
791 .subsystem = .WINDOWS_CUI,
792 .dll_flags = .{
793 .HIGH_ENTROPY_VA = true,
794 .DYNAMIC_BASE = true,
795 .TERMINAL_SERVER_AWARE = true,
796 .NX_COMPAT = true,
797 },
798 .size_of_stack_reserve = default_size_of_stack_reserve,
799 .size_of_stack_commit = default_size_of_stack_commit,
800 .size_of_heap_reserve = default_size_of_heap_reserve,
801 .size_of_heap_commit = default_size_of_heap_commit,
802 .loader_flags = 0,
803 .number_of_rva_and_sizes = data_directories_len,
804 };
805 if (target_endian != native_endian)
806 std.mem.byteSwapAllFields(std.coff.OptionalHeader.PE32, optional_header);
807 },
808 .@"PE32+" => {
809 const header: *std.coff.OptionalHeader.@"PE32+" =
810 @ptrCast(@alignCast(optional_header_ni.slice(&coff.mf)));
811 header.* = .{
812 .standard = .{
813 .magic = .@"PE32+",
814 .major_linker_version = 0,
815 .minor_linker_version = 0,
816 .size_of_code = 0,
817 .size_of_initialized_data = 0,
818 .size_of_uninitialized_data = 0,
819 .address_of_entry_point = 0,
820 .base_of_code = 0,
821 },
822 .image_base = switch (coff.base.comp.config.output_mode) {
823 .Exe => 0x140000000,
824 .Lib => switch (coff.base.comp.config.link_mode) {
825 .static => 0,
826 .dynamic => 0x180000000,
827 },
828 .Obj => 0,
829 },
830 .section_alignment = @intCast(section_align.toByteUnits()),
831 .file_alignment = @intCast(file_align.toByteUnits()),
832 .major_operating_system_version = 6,
833 .minor_operating_system_version = 0,
834 .major_image_version = 0,
835 .minor_image_version = 0,
836 .major_subsystem_version = major_subsystem_version,
837 .minor_subsystem_version = minor_subsystem_version,
838 .win32_version_value = 0,
839 .size_of_image = 0,
840 .size_of_headers = 0,
841 .checksum = 0,
842 .subsystem = .WINDOWS_CUI,
843 .dll_flags = .{
844 .HIGH_ENTROPY_VA = true,
845 .DYNAMIC_BASE = true,
846 .TERMINAL_SERVER_AWARE = true,
847 .NX_COMPAT = true,
848 },
849 .size_of_stack_reserve = default_size_of_stack_reserve,
850 .size_of_stack_commit = default_size_of_stack_commit,
851 .size_of_heap_reserve = default_size_of_heap_reserve,
852 .size_of_heap_commit = default_size_of_heap_commit,
853 .loader_flags = 0,
854 .number_of_rva_and_sizes = data_directories_len,
855 };
856 if (target_endian != native_endian)
857 std.mem.byteSwapAllFields(std.coff.OptionalHeader.@"PE32+", header);
858 },
859 };
860
861 const data_directories_ni = Node.known.data_directories;
862 assert(data_directories_ni == try coff.mf.addLastChildNode(gpa, header_ni, .{
863 .size = data_directories_size,
864 .alignment = .@"4",
865 .fixed = true,
866 }));
867 coff.nodes.appendAssumeCapacity(.data_directories);
868 {
869 const data_directories: *[data_directories_len]std.coff.ImageDataDirectory =
870 @ptrCast(@alignCast(data_directories_ni.slice(&coff.mf)));
871 @memset(data_directories, .{ .virtual_address = 0, .size = 0 });
872 if (target_endian != native_endian) for (data_directories) |*data_directory|
873 std.mem.byteSwapAllFields(std.coff.ImageDataDirectory, data_directory);
874 }
875
876 const section_table_ni = Node.known.section_table;
877 assert(section_table_ni == try coff.mf.addLastChildNode(gpa, header_ni, .{
878 .alignment = .@"4",
879 .fixed = true,
880 }));
881 coff.nodes.appendAssumeCapacity(.section_table);
882
883 assert(coff.nodes.len == Node.known_count);
884
885 try coff.symbol_table.ensureTotalCapacity(gpa, Symbol.Index.known_count);
886 coff.symbol_table.addOneAssumeCapacity().* = .{
887 .ni = .none,
888 .rva = 0,
889 .size = 0,
890 .loc_relocs = .none,
891 .target_relocs = .none,
892 .section_number = .UNDEFINED,
893 .data_directory = null,
894 };
895 assert(try coff.addSection(".data", null, .{
896 .CNT_INITIALIZED_DATA = true,
897 .MEM_READ = true,
898 .MEM_WRITE = true,
899 }) == .data);
900 assert(try coff.addSection(".idata", .import_table, .{
901 .CNT_INITIALIZED_DATA = true,
902 .MEM_READ = true,
903 }) == .idata);
904 assert(try coff.addSection(".rdata", null, .{
905 .CNT_INITIALIZED_DATA = true,
906 .MEM_READ = true,
907 }) == .rdata);
908 assert(try coff.addSection(".text", null, .{
909 .CNT_CODE = true,
910 .MEM_EXECUTE = true,
911 .MEM_READ = true,
912 }) == .text);
913 coff.import_table.directory_table_ni = try coff.mf.addLastChildNode(
914 gpa,
915 Symbol.Index.idata.node(coff),
916 .{
917 .alignment = .@"4",
918 .fixed = true,
919 },
920 );
921 coff.nodes.appendAssumeCapacity(.import_directory_table);
922 assert(coff.symbol_table.items.len == Symbol.Index.known_count);
923}
924
925fn getNode(coff: *const Coff, ni: MappedFile.Node.Index) Node {
926 return coff.nodes.get(@intFromEnum(ni));
927}
928fn computeNodeRva(coff: *Coff, ni: MappedFile.Node.Index) u32 {
929 var section_offset: u32 = 0;
930 var parent_ni = ni;
931 while (true) {
932 assert(parent_ni != .none);
933 switch (coff.getNode(parent_ni)) {
934 else => {},
935 .section => |si| return si.get(coff).rva + section_offset,
936 }
937 const parent_offset, _ = parent_ni.location(&coff.mf).resolve(&coff.mf);
938 section_offset += @intCast(parent_offset);
939 parent_ni = parent_ni.parent(&coff.mf);
940 }
941}
942
943pub inline fn targetEndian(coff: *const Coff) std.builtin.Endian {
944 return coff.endian;
945}
946fn targetLoad(coff: *const Coff, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
947 const Child = @typeInfo(@TypeOf(ptr)).pointer.child;
948 return switch (@typeInfo(Child)) {
949 else => @compileError(@typeName(Child)),
950 .int => std.mem.toNative(Child, ptr.*, coff.targetEndian()),
951 .@"enum" => |@"enum"| @enumFromInt(coff.targetLoad(@as(*@"enum".tag_type, @ptrCast(ptr)))),
952 .@"struct" => |@"struct"| @bitCast(
953 coff.targetLoad(@as(*@"struct".backing_integer.?, @ptrCast(ptr))),
954 ),
955 };
956}
957fn targetStore(coff: *const Coff, ptr: anytype, val: @typeInfo(@TypeOf(ptr)).pointer.child) void {
958 const Child = @typeInfo(@TypeOf(ptr)).pointer.child;
959 return switch (@typeInfo(Child)) {
960 else => @compileError(@typeName(Child)),
961 .int => ptr.* = std.mem.nativeTo(Child, val, coff.targetEndian()),
962 .@"enum" => |@"enum"| coff.targetStore(
963 @as(*@"enum".tag_type, @ptrCast(ptr)),
964 @intFromEnum(val),
965 ),
966 .@"struct" => |@"struct"| coff.targetStore(
967 @as(*@"struct".backing_integer.?, @ptrCast(ptr)),
968 @bitCast(val),
969 ),
970 };
971}
972
973pub fn headerPtr(coff: *Coff) *std.coff.Header {
974 return @ptrCast(@alignCast(Node.known.coff_header.slice(&coff.mf)));
975}
976
977pub fn optionalHeaderStandardPtr(coff: *Coff) *std.coff.OptionalHeader {
978 return @ptrCast(@alignCast(
979 Node.known.optional_header.slice(&coff.mf)[0..@sizeOf(std.coff.OptionalHeader)],
980 ));
981}
982
983pub const OptionalHeaderPtr = union(std.coff.OptionalHeader.Magic) {
984 PE32: *std.coff.OptionalHeader.PE32,
985 @"PE32+": *std.coff.OptionalHeader.@"PE32+",
986};
987pub fn optionalHeaderPtr(coff: *Coff) OptionalHeaderPtr {
988 const slice = Node.known.optional_header.slice(&coff.mf);
989 return switch (coff.targetLoad(&coff.optionalHeaderStandardPtr().magic)) {
990 _ => unreachable,
991 inline else => |magic| @unionInit(
992 OptionalHeaderPtr,
993 @tagName(magic),
994 @ptrCast(@alignCast(slice)),
995 ),
996 };
997}
998pub fn optionalHeaderField(
999 coff: *Coff,
1000 comptime field: std.meta.FieldEnum(std.coff.OptionalHeader.@"PE32+"),
1001) @FieldType(std.coff.OptionalHeader.@"PE32+", @tagName(field)) {
1002 return switch (coff.optionalHeaderPtr()) {
1003 inline else => |optional_header| coff.targetLoad(&@field(optional_header, @tagName(field))),
1004 };
1005}
1006
1007pub fn dataDirectoriesSlice(coff: *Coff) []std.coff.ImageDataDirectory {
1008 return @ptrCast(@alignCast(Node.known.data_directories.slice(&coff.mf)));
1009}
1010
1011pub fn sectionTableSlice(coff: *Coff) []std.coff.SectionHeader {
1012 return @ptrCast(@alignCast(Node.known.section_table.slice(&coff.mf)));
1013}
1014
1015fn addSymbolAssumeCapacity(coff: *Coff) Symbol.Index {
1016 defer coff.symbol_table.addOneAssumeCapacity().* = .{
1017 .ni = .none,
1018 .rva = 0,
1019 .size = 0,
1020 .loc_relocs = .none,
1021 .target_relocs = .none,
1022 .section_number = .UNDEFINED,
1023 .data_directory = null,
1024 };
1025 return @enumFromInt(coff.symbol_table.items.len);
1026}
1027
1028fn initSymbolAssumeCapacity(coff: *Coff) !Symbol.Index {
1029 const si = coff.addSymbolAssumeCapacity();
1030 return si;
1031}
1032
1033fn getOrPutString(coff: *Coff, string: []const u8) !String {
1034 const gpa = coff.base.comp.gpa;
1035 try coff.string_bytes.ensureUnusedCapacity(gpa, string.len + 1);
1036 const gop = try coff.strings.getOrPutContextAdapted(
1037 gpa,
1038 string,
1039 std.hash_map.StringIndexAdapter{ .bytes = &coff.string_bytes },
1040 .{ .bytes = &coff.string_bytes },
1041 );
1042 if (!gop.found_existing) {
1043 gop.key_ptr.* = @intCast(coff.string_bytes.items.len);
1044 gop.value_ptr.* = {};
1045 coff.string_bytes.appendSliceAssumeCapacity(string);
1046 coff.string_bytes.appendAssumeCapacity(0);
1047 }
1048 return @enumFromInt(gop.key_ptr.*);
1049}
1050
1051fn getOrPutOptionalString(coff: *Coff, string: ?[]const u8) !String.Optional {
1052 return (try coff.getOrPutString(string orelse return .none)).toOptional();
1053}
1054
1055pub fn globalSymbol(coff: *Coff, name: []const u8, lib_name: ?[]const u8) !Symbol.Index {
1056 const gpa = coff.base.comp.gpa;
1057 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1058 const sym_gop = try coff.globals.getOrPut(gpa, .{
1059 .name = try coff.getOrPutString(name),
1060 .lib_name = try coff.getOrPutOptionalString(lib_name),
1061 });
1062 if (!sym_gop.found_existing) {
1063 sym_gop.value_ptr.* = coff.addSymbolAssumeCapacity();
1064 coff.base.comp.link_synth_prog_node.increaseEstimatedTotalItems(1);
1065 }
1066 return sym_gop.value_ptr.*;
1067}
1068
1069fn navMapIndex(coff: *Coff, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Node.NavMapIndex {
1070 const gpa = zcu.gpa;
1071 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1072 const sym_gop = try coff.navs.getOrPut(gpa, nav_index);
1073 if (!sym_gop.found_existing) sym_gop.value_ptr.* = coff.addSymbolAssumeCapacity();
1074 return @enumFromInt(sym_gop.index);
1075}
1076pub fn navSymbol(coff: *Coff, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Symbol.Index {
1077 const ip = &zcu.intern_pool;
1078 const nav = ip.getNav(nav_index);
1079 if (nav.getExtern(ip)) |@"extern"| return coff.globalSymbol(
1080 @"extern".name.toSlice(ip),
1081 @"extern".lib_name.toSlice(ip),
1082 );
1083 const nmi = try coff.navMapIndex(zcu, nav_index);
1084 return nmi.symbol(coff);
1085}
1086
1087fn uavMapIndex(coff: *Coff, uav_val: InternPool.Index) !Node.UavMapIndex {
1088 const gpa = coff.base.comp.gpa;
1089 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1090 const sym_gop = try coff.uavs.getOrPut(gpa, uav_val);
1091 if (!sym_gop.found_existing) sym_gop.value_ptr.* = coff.addSymbolAssumeCapacity();
1092 return @enumFromInt(sym_gop.index);
1093}
1094pub fn uavSymbol(coff: *Coff, uav_val: InternPool.Index) !Symbol.Index {
1095 const umi = try coff.uavMapIndex(uav_val);
1096 return umi.symbol(coff);
1097}
1098
1099pub fn lazySymbol(coff: *Coff, lazy: link.File.LazySymbol) !Symbol.Index {
1100 const gpa = coff.base.comp.gpa;
1101 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1102 const sym_gop = try coff.lazy.getPtr(lazy.kind).map.getOrPut(gpa, lazy.ty);
1103 if (!sym_gop.found_existing) {
1104 sym_gop.value_ptr.* = try coff.initSymbolAssumeCapacity();
1105 coff.base.comp.link_synth_prog_node.increaseEstimatedTotalItems(1);
1106 }
1107 return sym_gop.value_ptr.*;
1108}
1109
1110pub fn getNavVAddr(
1111 coff: *Coff,
1112 pt: Zcu.PerThread,
1113 nav: InternPool.Nav.Index,
1114 reloc_info: link.File.RelocInfo,
1115) !u64 {
1116 return coff.getVAddr(reloc_info, try coff.navSymbol(pt.zcu, nav));
1117}
1118
1119pub fn getUavVAddr(
1120 coff: *Coff,
1121 uav: InternPool.Index,
1122 reloc_info: link.File.RelocInfo,
1123) !u64 {
1124 return coff.getVAddr(reloc_info, try coff.uavSymbol(uav));
1125}
1126
1127pub fn getVAddr(coff: *Coff, reloc_info: link.File.RelocInfo, target_si: Symbol.Index) !u64 {
1128 try coff.addReloc(
1129 @enumFromInt(reloc_info.parent.atom_index),
1130 reloc_info.offset,
1131 target_si,
1132 reloc_info.addend,
1133 switch (coff.targetLoad(&coff.headerPtr().machine)) {
1134 else => unreachable,
1135 .AMD64 => .{ .AMD64 = .ADDR64 },
1136 .I386 => .{ .I386 = .DIR32 },
1137 },
1138 );
1139 return coff.optionalHeaderField(.image_base) + target_si.get(coff).rva;
1140}
1141
1142fn addSection(
1143 coff: *Coff,
1144 name: []const u8,
1145 maybe_data_directory: ?DataDirectory,
1146 flags: std.coff.SectionHeader.Flags,
1147) !Symbol.Index {
1148 const gpa = coff.base.comp.gpa;
1149 try coff.nodes.ensureUnusedCapacity(gpa, 1);
1150 try coff.section_table.ensureUnusedCapacity(gpa, 1);
1151 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1152
1153 const coff_header = coff.headerPtr();
1154 const section_index = coff.targetLoad(&coff_header.number_of_sections);
1155 const section_table_len = section_index + 1;
1156 coff.targetStore(&coff_header.number_of_sections, section_table_len);
1157 try Node.known.section_table.resize(
1158 &coff.mf,
1159 gpa,
1160 @sizeOf(std.coff.SectionHeader) * section_table_len,
1161 );
1162 const ni = try coff.mf.addLastChildNode(gpa, .root, .{
1163 .alignment = coff.mf.flags.block_size,
1164 .moved = true,
1165 .bubbles_moved = false,
1166 });
1167 const si = coff.addSymbolAssumeCapacity();
1168 coff.section_table.appendAssumeCapacity(si);
1169 coff.nodes.appendAssumeCapacity(.{ .section = si });
1170 const section_table = coff.sectionTableSlice();
1171 const virtual_size = coff.optionalHeaderField(.section_alignment);
1172 const rva: u32 = switch (section_index) {
1173 0 => @intCast(Node.known.header.location(&coff.mf).resolve(&coff.mf)[1]),
1174 else => coff.section_table.items[section_index - 1].get(coff).rva +
1175 coff.targetLoad(&section_table[section_index - 1].virtual_size),
1176 };
1177 {
1178 const sym = si.get(coff);
1179 sym.ni = ni;
1180 sym.rva = rva;
1181 sym.section_number = @enumFromInt(section_table_len);
1182 sym.data_directory = maybe_data_directory;
1183 }
1184 const section = &section_table[section_index];
1185 section.* = .{
1186 .name = undefined,
1187 .virtual_size = virtual_size,
1188 .virtual_address = rva,
1189 .size_of_raw_data = 0,
1190 .pointer_to_raw_data = 0,
1191 .pointer_to_relocations = 0,
1192 .pointer_to_linenumbers = 0,
1193 .number_of_relocations = 0,
1194 .number_of_linenumbers = 0,
1195 .flags = flags,
1196 };
1197 @memcpy(section.name[0..name.len], name);
1198 @memset(section.name[name.len..], 0);
1199 if (coff.targetEndian() != native_endian)
1200 std.mem.byteSwapAllFields(std.coff.SectionHeader, section);
1201 if (maybe_data_directory) |data_directory|
1202 coff.dataDirectoriesSlice()[@intFromEnum(data_directory)] = .{
1203 .virtual_address = section.virtual_address,
1204 .size = section.virtual_size,
1205 };
1206 switch (coff.optionalHeaderPtr()) {
1207 inline else => |optional_header| coff.targetStore(
1208 &optional_header.size_of_image,
1209 @intCast(rva + virtual_size),
1210 ),
1211 }
1212 return si;
1213}
1214
1215pub fn addReloc(
1216 coff: *Coff,
1217 loc_si: Symbol.Index,
1218 offset: u64,
1219 target_si: Symbol.Index,
1220 addend: i64,
1221 @"type": Reloc.Type,
1222) !void {
1223 const gpa = coff.base.comp.gpa;
1224 const target = target_si.get(coff);
1225 const ri: Reloc.Index = @enumFromInt(coff.relocs.items.len);
1226 (try coff.relocs.addOne(gpa)).* = .{
1227 .type = @"type",
1228 .prev = .none,
1229 .next = target.target_relocs,
1230 .loc = loc_si,
1231 .target = target_si,
1232 .unused = 0,
1233 .offset = offset,
1234 .addend = addend,
1235 };
1236 switch (target.target_relocs) {
1237 .none => {},
1238 else => |target_ri| target_ri.get(coff).prev = ri,
1239 }
1240 target.target_relocs = ri;
1241}
1242
1243pub fn prelink(coff: *Coff, prog_node: std.Progress.Node) void {
1244 _ = coff;
1245 _ = prog_node;
1246}
1247
1248pub fn updateNav(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {
1249 coff.updateNavInner(pt, nav_index) catch |err| switch (err) {
1250 error.OutOfMemory,
1251 error.Overflow,
1252 error.RelocationNotByteAligned,
1253 => |e| return e,
1254 else => |e| return coff.base.cgFail(nav_index, "linker failed to update variable: {t}", .{e}),
1255 };
1256}
1257fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) !void {
1258 const zcu = pt.zcu;
1259 const gpa = zcu.gpa;
1260 const ip = &zcu.intern_pool;
1261
1262 const nav = ip.getNav(nav_index);
1263 const nav_val = nav.status.fully_resolved.val;
1264 const nav_init, const is_threadlocal = switch (ip.indexToKey(nav_val)) {
1265 else => .{ nav_val, false },
1266 .variable => |variable| .{ variable.init, variable.is_threadlocal },
1267 .@"extern" => return,
1268 .func => .{ .none, false },
1269 };
1270 if (nav_init == .none or !Type.fromInterned(ip.typeOf(nav_init)).hasRuntimeBits(zcu)) return;
1271
1272 const nmi = try coff.navMapIndex(zcu, nav_index);
1273 const si = nmi.symbol(coff);
1274 const ni = ni: {
1275 const sym = si.get(coff);
1276 switch (sym.ni) {
1277 .none => {
1278 try coff.nodes.ensureUnusedCapacity(gpa, 1);
1279 _ = is_threadlocal;
1280 const ni = try coff.mf.addLastChildNode(gpa, Symbol.Index.data.node(coff), .{
1281 .alignment = pt.navAlignment(nav_index).toStdMem(),
1282 .moved = true,
1283 });
1284 coff.nodes.appendAssumeCapacity(.{ .nav = nmi });
1285 sym.ni = ni;
1286 sym.section_number = Symbol.Index.data.get(coff).section_number;
1287 },
1288 else => si.deleteLocationRelocs(coff),
1289 }
1290 assert(sym.loc_relocs == .none);
1291 sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1292 break :ni sym.ni;
1293 };
1294
1295 var nw: MappedFile.Node.Writer = undefined;
1296 ni.writer(&coff.mf, gpa, &nw);
1297 defer nw.deinit();
1298 codegen.generateSymbol(
1299 &coff.base,
1300 pt,
1301 zcu.navSrcLoc(nav_index),
1302 .fromInterned(nav_init),
1303 &nw.interface,
1304 .{ .atom_index = @intFromEnum(si) },
1305 ) catch |err| switch (err) {
1306 error.WriteFailed => return error.OutOfMemory,
1307 else => |e| return e,
1308 };
1309 si.get(coff).size = @intCast(nw.interface.end);
1310 si.applyLocationRelocs(coff);
1311}
1312
1313pub fn lowerUav(
1314 coff: *Coff,
1315 pt: Zcu.PerThread,
1316 uav_val: InternPool.Index,
1317 uav_align: InternPool.Alignment,
1318 src_loc: Zcu.LazySrcLoc,
1319) !codegen.SymbolResult {
1320 const zcu = pt.zcu;
1321 const gpa = zcu.gpa;
1322
1323 try coff.pending_uavs.ensureUnusedCapacity(gpa, 1);
1324 const umi = try coff.uavMapIndex(uav_val);
1325 const si = umi.symbol(coff);
1326 if (switch (si.get(coff).ni) {
1327 .none => true,
1328 else => |ni| uav_align.toStdMem().order(ni.alignment(&coff.mf)).compare(.gt),
1329 }) {
1330 const gop = coff.pending_uavs.getOrPutAssumeCapacity(umi);
1331 if (gop.found_existing) {
1332 gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align);
1333 } else {
1334 gop.value_ptr.* = .{
1335 .alignment = uav_align,
1336 .src_loc = src_loc,
1337 };
1338 coff.base.comp.link_const_prog_node.increaseEstimatedTotalItems(1);
1339 }
1340 }
1341 return .{ .sym_index = @intFromEnum(si) };
1342}
1343
1344pub fn updateFunc(
1345 coff: *Coff,
1346 pt: Zcu.PerThread,
1347 func_index: InternPool.Index,
1348 mir: *const codegen.AnyMir,
1349) !void {
1350 coff.updateFuncInner(pt, func_index, mir) catch |err| switch (err) {
1351 error.OutOfMemory,
1352 error.Overflow,
1353 error.RelocationNotByteAligned,
1354 error.CodegenFail,
1355 => |e| return e,
1356 else => |e| return coff.base.cgFail(
1357 pt.zcu.funcInfo(func_index).owner_nav,
1358 "linker failed to update function: {s}",
1359 .{@errorName(e)},
1360 ),
1361 };
1362}
1363fn updateFuncInner(
1364 coff: *Coff,
1365 pt: Zcu.PerThread,
1366 func_index: InternPool.Index,
1367 mir: *const codegen.AnyMir,
1368) !void {
1369 const zcu = pt.zcu;
1370 const gpa = zcu.gpa;
1371 const ip = &zcu.intern_pool;
1372 const func = zcu.funcInfo(func_index);
1373 const nav = ip.getNav(func.owner_nav);
1374
1375 const nmi = try coff.navMapIndex(zcu, func.owner_nav);
1376 const si = nmi.symbol(coff);
1377 log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), si });
1378 const ni = ni: {
1379 const sym = si.get(coff);
1380 switch (sym.ni) {
1381 .none => {
1382 try coff.nodes.ensureUnusedCapacity(gpa, 1);
1383 const mod = zcu.navFileScope(func.owner_nav).mod.?;
1384 const target = &mod.resolved_target.result;
1385 const ni = try coff.mf.addLastChildNode(gpa, Symbol.Index.text.node(coff), .{
1386 .alignment = switch (nav.status.fully_resolved.alignment) {
1387 .none => switch (mod.optimize_mode) {
1388 .Debug,
1389 .ReleaseSafe,
1390 .ReleaseFast,
1391 => target_util.defaultFunctionAlignment(target),
1392 .ReleaseSmall => target_util.minFunctionAlignment(target),
1393 },
1394 else => |a| a.maxStrict(target_util.minFunctionAlignment(target)),
1395 }.toStdMem(),
1396 .moved = true,
1397 });
1398 coff.nodes.appendAssumeCapacity(.{ .nav = nmi });
1399 sym.ni = ni;
1400 sym.section_number = Symbol.Index.text.get(coff).section_number;
1401 },
1402 else => si.deleteLocationRelocs(coff),
1403 }
1404 assert(sym.loc_relocs == .none);
1405 sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1406 break :ni sym.ni;
1407 };
1408
1409 var nw: MappedFile.Node.Writer = undefined;
1410 ni.writer(&coff.mf, gpa, &nw);
1411 defer nw.deinit();
1412 codegen.emitFunction(
1413 &coff.base,
1414 pt,
1415 zcu.navSrcLoc(func.owner_nav),
1416 func_index,
1417 @intFromEnum(si),
1418 mir,
1419 &nw.interface,
1420 .none,
1421 ) catch |err| switch (err) {
1422 error.WriteFailed => return nw.err.?,
1423 else => |e| return e,
1424 };
1425 si.get(coff).size = @intCast(nw.interface.end);
1426 si.applyLocationRelocs(coff);
1427}
1428
1429pub fn updateErrorData(coff: *Coff, pt: Zcu.PerThread) !void {
1430 coff.flushLazy(pt, .{
1431 .kind = .const_data,
1432 .index = @intCast(coff.lazy.getPtr(.const_data).map.getIndex(.anyerror_type) orelse return),
1433 }) catch |err| switch (err) {
1434 error.OutOfMemory => return error.OutOfMemory,
1435 error.CodegenFail => return error.LinkFailure,
1436 else => |e| return coff.base.comp.link_diags.fail("updateErrorData failed {t}", .{e}),
1437 };
1438}
1439
1440pub fn flush(
1441 coff: *Coff,
1442 arena: std.mem.Allocator,
1443 tid: Zcu.PerThread.Id,
1444 prog_node: std.Progress.Node,
1445) !void {
1446 _ = arena;
1447 _ = prog_node;
1448 while (try coff.idle(tid)) {}
1449
1450 // hack for stage2_x86_64 + coff
1451 const comp = coff.base.comp;
1452 if (comp.compiler_rt_dyn_lib) |crt_file| {
1453 const gpa = comp.gpa;
1454 const compiler_rt_sub_path = try std.fs.path.join(gpa, &.{
1455 std.fs.path.dirname(coff.base.emit.sub_path) orelse "",
1456 std.fs.path.basename(crt_file.full_object_path.sub_path),
1457 });
1458 defer gpa.free(compiler_rt_sub_path);
1459 crt_file.full_object_path.root_dir.handle.copyFile(
1460 crt_file.full_object_path.sub_path,
1461 coff.base.emit.root_dir.handle,
1462 compiler_rt_sub_path,
1463 .{},
1464 ) catch |err| switch (err) {
1465 else => |e| return comp.link_diags.fail("Copy '{s}' failed: {s}", .{
1466 compiler_rt_sub_path,
1467 @errorName(e),
1468 }),
1469 };
1470 }
1471}
1472
1473pub fn idle(coff: *Coff, tid: Zcu.PerThread.Id) !bool {
1474 const comp = coff.base.comp;
1475 task: {
1476 while (coff.pending_uavs.pop()) |pending_uav| {
1477 const sub_prog_node = coff.idleProgNode(
1478 tid,
1479 comp.link_const_prog_node,
1480 .{ .uav = pending_uav.key },
1481 );
1482 defer sub_prog_node.end();
1483 coff.flushUav(
1484 .{ .zcu = coff.base.comp.zcu.?, .tid = tid },
1485 pending_uav.key,
1486 pending_uav.value.alignment,
1487 pending_uav.value.src_loc,
1488 ) catch |err| switch (err) {
1489 error.OutOfMemory => return error.OutOfMemory,
1490 else => |e| return coff.base.comp.link_diags.fail(
1491 "linker failed to lower constant: {t}",
1492 .{e},
1493 ),
1494 };
1495 break :task;
1496 }
1497 if (coff.global_pending_index < coff.globals.count()) {
1498 const pt: Zcu.PerThread = .{ .zcu = coff.base.comp.zcu.?, .tid = tid };
1499 const gmi: Node.GlobalMapIndex = @enumFromInt(coff.global_pending_index);
1500 coff.global_pending_index += 1;
1501 const sub_prog_node = comp.link_synth_prog_node.start(
1502 gmi.globalName(coff).name.toSlice(coff),
1503 0,
1504 );
1505 defer sub_prog_node.end();
1506 coff.flushGlobal(pt, gmi) catch |err| switch (err) {
1507 error.OutOfMemory => return error.OutOfMemory,
1508 else => |e| return coff.base.comp.link_diags.fail(
1509 "linker failed to lower constant: {t}",
1510 .{e},
1511 ),
1512 };
1513 break :task;
1514 }
1515 var lazy_it = coff.lazy.iterator();
1516 while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) {
1517 const pt: Zcu.PerThread = .{ .zcu = coff.base.comp.zcu.?, .tid = tid };
1518 const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index };
1519 lazy.value.pending_index += 1;
1520 const kind = switch (lmr.kind) {
1521 .code => "code",
1522 .const_data => "data",
1523 };
1524 var name: [std.Progress.Node.max_name_len]u8 = undefined;
1525 const sub_prog_node = comp.link_synth_prog_node.start(
1526 std.fmt.bufPrint(&name, "lazy {s} for {f}", .{
1527 kind,
1528 Type.fromInterned(lmr.lazySymbol(coff).ty).fmt(pt),
1529 }) catch &name,
1530 0,
1531 );
1532 defer sub_prog_node.end();
1533 coff.flushLazy(pt, lmr) catch |err| switch (err) {
1534 error.OutOfMemory => return error.OutOfMemory,
1535 else => |e| return coff.base.comp.link_diags.fail(
1536 "linker failed to lower lazy {s}: {t}",
1537 .{ kind, e },
1538 ),
1539 };
1540 break :task;
1541 };
1542 while (coff.mf.updates.pop()) |ni| {
1543 const clean_moved = ni.cleanMoved(&coff.mf);
1544 const clean_resized = ni.cleanResized(&coff.mf);
1545 if (clean_moved or clean_resized) {
1546 const sub_prog_node = coff.idleProgNode(tid, coff.mf.update_prog_node, coff.getNode(ni));
1547 defer sub_prog_node.end();
1548 if (clean_moved) try coff.flushMoved(ni);
1549 if (clean_resized) try coff.flushResized(ni);
1550 break :task;
1551 } else coff.mf.update_prog_node.completeOne();
1552 }
1553 }
1554 if (coff.pending_uavs.count() > 0) return true;
1555 for (&coff.lazy.values) |lazy| if (lazy.map.count() > lazy.pending_index) return true;
1556 if (coff.mf.updates.items.len > 0) return true;
1557 return false;
1558}
1559
1560fn idleProgNode(
1561 coff: *Coff,
1562 tid: Zcu.PerThread.Id,
1563 prog_node: std.Progress.Node,
1564 node: Node,
1565) std.Progress.Node {
1566 var name: [std.Progress.Node.max_name_len]u8 = undefined;
1567 return prog_node.start(name: switch (node) {
1568 else => |tag| @tagName(tag),
1569 .section => |si| std.mem.sliceTo(&si.get(coff).section_number.header(coff).name, 0),
1570 .nav => |nmi| {
1571 const ip = &coff.base.comp.zcu.?.intern_pool;
1572 break :name ip.getNav(nmi.navIndex(coff)).fqn.toSlice(ip);
1573 },
1574 .uav => |umi| std.fmt.bufPrint(&name, "{f}", .{
1575 Value.fromInterned(umi.uavValue(coff)).fmtValue(.{
1576 .zcu = coff.base.comp.zcu.?,
1577 .tid = tid,
1578 }),
1579 }) catch &name,
1580 }, 0);
1581}
1582
1583fn flushUav(
1584 coff: *Coff,
1585 pt: Zcu.PerThread,
1586 umi: Node.UavMapIndex,
1587 uav_align: InternPool.Alignment,
1588 src_loc: Zcu.LazySrcLoc,
1589) !void {
1590 const zcu = pt.zcu;
1591 const gpa = zcu.gpa;
1592
1593 const uav_val = umi.uavValue(coff);
1594 const si = umi.symbol(coff);
1595 const ni = ni: {
1596 const sym = si.get(coff);
1597 switch (sym.ni) {
1598 .none => {
1599 try coff.nodes.ensureUnusedCapacity(gpa, 1);
1600 const ni = try coff.mf.addLastChildNode(gpa, Symbol.Index.data.node(coff), .{
1601 .alignment = uav_align.toStdMem(),
1602 .moved = true,
1603 });
1604 coff.nodes.appendAssumeCapacity(.{ .uav = umi });
1605 sym.ni = ni;
1606 sym.section_number = Symbol.Index.data.get(coff).section_number;
1607 },
1608 else => {
1609 if (sym.ni.alignment(&coff.mf).order(uav_align.toStdMem()).compare(.gte)) return;
1610 si.deleteLocationRelocs(coff);
1611 },
1612 }
1613 assert(sym.loc_relocs == .none);
1614 sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1615 break :ni sym.ni;
1616 };
1617
1618 var nw: MappedFile.Node.Writer = undefined;
1619 ni.writer(&coff.mf, gpa, &nw);
1620 defer nw.deinit();
1621 codegen.generateSymbol(
1622 &coff.base,
1623 pt,
1624 src_loc,
1625 .fromInterned(uav_val),
1626 &nw.interface,
1627 .{ .atom_index = @intFromEnum(si) },
1628 ) catch |err| switch (err) {
1629 error.WriteFailed => return error.OutOfMemory,
1630 else => |e| return e,
1631 };
1632 si.get(coff).size = @intCast(nw.interface.end);
1633 si.applyLocationRelocs(coff);
1634}
1635
1636fn flushGlobal(coff: *Coff, pt: Zcu.PerThread, gmi: Node.GlobalMapIndex) !void {
1637 const zcu = pt.zcu;
1638 const comp = zcu.comp;
1639 const gpa = zcu.gpa;
1640 const gn = gmi.globalName(coff);
1641 if (gn.lib_name.toSlice(coff)) |lib_name| {
1642 const name = gn.name.toSlice(coff);
1643 try coff.nodes.ensureUnusedCapacity(gpa, 4);
1644 try coff.symbol_table.ensureUnusedCapacity(gpa, 1);
1645
1646 const target_endian = coff.targetEndian();
1647 const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic);
1648 const addr_size: u64, const addr_align: std.mem.Alignment = switch (magic) {
1649 _ => unreachable,
1650 .PE32 => .{ 4, .@"4" },
1651 .@"PE32+" => .{ 8, .@"8" },
1652 };
1653
1654 const gop = try coff.import_table.dlls.getOrPutAdapted(
1655 gpa,
1656 lib_name,
1657 ImportTable.Adapter{ .coff = coff },
1658 );
1659 const import_hint_name_align: std.mem.Alignment = .@"2";
1660 if (!gop.found_existing) {
1661 errdefer _ = coff.import_table.dlls.pop();
1662 try coff.import_table.directory_table_ni.resize(
1663 &coff.mf,
1664 gpa,
1665 @sizeOf(std.coff.ImportDirectoryEntry) * (gop.index + 2),
1666 );
1667 const import_hint_name_table_len =
1668 import_hint_name_align.forward(lib_name.len + ".dll".len + 1);
1669 const idata_section_ni = Symbol.Index.idata.node(coff);
1670 const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
1671 .size = addr_size * 2,
1672 .alignment = addr_align,
1673 .moved = true,
1674 });
1675 const import_address_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
1676 .size = addr_size * 2,
1677 .alignment = addr_align,
1678 .moved = true,
1679 });
1680 const import_address_table_si = coff.addSymbolAssumeCapacity();
1681 {
1682 const import_address_table_sym = import_address_table_si.get(coff);
1683 import_address_table_sym.ni = import_address_table_ni;
1684 assert(import_address_table_sym.loc_relocs == .none);
1685 import_address_table_sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1686 import_address_table_sym.section_number = Symbol.Index.idata.get(coff).section_number;
1687 }
1688 const import_hint_name_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
1689 .size = import_hint_name_table_len,
1690 .alignment = import_hint_name_align,
1691 .moved = true,
1692 });
1693 gop.value_ptr.* = .{
1694 .import_lookup_table_ni = import_lookup_table_ni,
1695 .import_address_table_si = import_address_table_si,
1696 .import_hint_name_table_ni = import_hint_name_table_ni,
1697 .len = 0,
1698 .hint_name_len = @intCast(import_hint_name_table_len),
1699 };
1700 const import_hint_name_slice = import_hint_name_table_ni.slice(&coff.mf);
1701 @memcpy(import_hint_name_slice[0..lib_name.len], lib_name);
1702 @memcpy(import_hint_name_slice[lib_name.len..][0..".dll".len], ".dll");
1703 @memset(import_hint_name_slice[lib_name.len + ".dll".len ..], 0);
1704 coff.nodes.appendAssumeCapacity(.{ .import_lookup_table = @intCast(gop.index) });
1705 coff.nodes.appendAssumeCapacity(.{ .import_address_table = @intCast(gop.index) });
1706 coff.nodes.appendAssumeCapacity(.{ .import_hint_name_table = @intCast(gop.index) });
1707
1708 const import_directory_table: []std.coff.ImportDirectoryEntry =
1709 @ptrCast(@alignCast(coff.import_table.directory_table_ni.slice(&coff.mf)));
1710 import_directory_table[gop.index..][0..2].* = .{ .{
1711 .import_lookup_table_rva = coff.computeNodeRva(import_lookup_table_ni),
1712 .time_date_stamp = 0,
1713 .forwarder_chain = 0,
1714 .name_rva = coff.computeNodeRva(import_hint_name_table_ni),
1715 .import_address_table_rva = coff.computeNodeRva(import_address_table_ni),
1716 }, .{
1717 .import_lookup_table_rva = 0,
1718 .time_date_stamp = 0,
1719 .forwarder_chain = 0,
1720 .name_rva = 0,
1721 .import_address_table_rva = 0,
1722 } };
1723 }
1724 const import_symbol_index = gop.value_ptr.len;
1725 gop.value_ptr.len = import_symbol_index + 1;
1726 const new_symbol_table_size = addr_size * (import_symbol_index + 2);
1727 const import_hint_name_index = gop.value_ptr.hint_name_len;
1728 gop.value_ptr.hint_name_len = @intCast(
1729 import_hint_name_align.forward(import_hint_name_index + 2 + name.len + 1),
1730 );
1731 try gop.value_ptr.import_lookup_table_ni.resize(&coff.mf, gpa, new_symbol_table_size);
1732 const import_address_table_ni = gop.value_ptr.import_address_table_si.node(coff);
1733 try import_address_table_ni.resize(&coff.mf, gpa, new_symbol_table_size);
1734 try gop.value_ptr.import_hint_name_table_ni.resize(&coff.mf, gpa, gop.value_ptr.hint_name_len);
1735 const import_lookup_slice = gop.value_ptr.import_lookup_table_ni.slice(&coff.mf);
1736 const import_address_slice = import_address_table_ni.slice(&coff.mf);
1737 const import_hint_name_slice = gop.value_ptr.import_hint_name_table_ni.slice(&coff.mf);
1738 @memset(import_hint_name_slice[import_hint_name_index..][0..2], 0);
1739 @memcpy(import_hint_name_slice[import_hint_name_index + 2 ..][0..name.len], name);
1740 @memset(import_hint_name_slice[import_hint_name_index + 2 + name.len ..], 0);
1741 const import_hint_name_rva =
1742 coff.computeNodeRva(gop.value_ptr.import_hint_name_table_ni) + import_hint_name_index;
1743 switch (magic) {
1744 _ => unreachable,
1745 inline .PE32, .@"PE32+" => |ct_magic| {
1746 const Addr = switch (ct_magic) {
1747 _ => comptime unreachable,
1748 .PE32 => u32,
1749 .@"PE32+" => u64,
1750 };
1751 const import_lookup_table: []Addr = @ptrCast(@alignCast(import_lookup_slice));
1752 const import_address_table: []Addr = @ptrCast(@alignCast(import_address_slice));
1753 const import_hint_name_rvas: [2]Addr = .{
1754 std.mem.nativeTo(Addr, @intCast(import_hint_name_rva), target_endian),
1755 std.mem.nativeTo(Addr, 0, target_endian),
1756 };
1757 import_lookup_table[import_symbol_index..][0..2].* = import_hint_name_rvas;
1758 import_address_table[import_symbol_index..][0..2].* = import_hint_name_rvas;
1759 },
1760 }
1761 const si = gmi.symbol(coff);
1762 const sym = si.get(coff);
1763 sym.section_number = Symbol.Index.text.get(coff).section_number;
1764 assert(sym.loc_relocs == .none);
1765 sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1766 switch (coff.targetLoad(&coff.headerPtr().machine)) {
1767 else => |tag| @panic(@tagName(tag)),
1768 .AMD64 => {
1769 const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 };
1770 const target = &comp.root_mod.resolved_target.result;
1771 const ni = try coff.mf.addLastChildNode(gpa, Symbol.Index.text.node(coff), .{
1772 .alignment = switch (comp.root_mod.optimize_mode) {
1773 .Debug,
1774 .ReleaseSafe,
1775 .ReleaseFast,
1776 => target_util.defaultFunctionAlignment(target),
1777 .ReleaseSmall => target_util.minFunctionAlignment(target),
1778 }.toStdMem(),
1779 .size = init.len,
1780 });
1781 @memcpy(ni.slice(&coff.mf)[0..init.len], &init);
1782 sym.ni = ni;
1783 sym.size = init.len;
1784 try coff.addReloc(
1785 si,
1786 init.len - 4,
1787 gop.value_ptr.import_address_table_si,
1788 @intCast(addr_size * import_symbol_index),
1789 .{ .AMD64 = .REL32 },
1790 );
1791 },
1792 }
1793 coff.nodes.appendAssumeCapacity(.{ .global = gmi });
1794 sym.rva = coff.computeNodeRva(sym.ni);
1795 si.applyLocationRelocs(coff);
1796 }
1797}
1798
1799fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void {
1800 const zcu = pt.zcu;
1801 const gpa = zcu.gpa;
1802
1803 const lazy = lmr.lazySymbol(coff);
1804 const si = lmr.symbol(coff);
1805 const ni = ni: {
1806 const sym = si.get(coff);
1807 switch (sym.ni) {
1808 .none => {
1809 try coff.nodes.ensureUnusedCapacity(gpa, 1);
1810 const sec_si: Symbol.Index = switch (lazy.kind) {
1811 .code => .text,
1812 .const_data => .rdata,
1813 };
1814 const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ .moved = true });
1815 coff.nodes.appendAssumeCapacity(switch (lazy.kind) {
1816 .code => .{ .lazy_code = @enumFromInt(lmr.index) },
1817 .const_data => .{ .lazy_const_data = @enumFromInt(lmr.index) },
1818 });
1819 sym.ni = ni;
1820 sym.section_number = sec_si.get(coff).section_number;
1821 },
1822 else => si.deleteLocationRelocs(coff),
1823 }
1824 assert(sym.loc_relocs == .none);
1825 sym.loc_relocs = @enumFromInt(coff.relocs.items.len);
1826 break :ni sym.ni;
1827 };
1828
1829 var required_alignment: InternPool.Alignment = .none;
1830 var nw: MappedFile.Node.Writer = undefined;
1831 ni.writer(&coff.mf, gpa, &nw);
1832 defer nw.deinit();
1833 try codegen.generateLazySymbol(
1834 &coff.base,
1835 pt,
1836 Type.fromInterned(lazy.ty).srcLocOrNull(pt.zcu) orelse .unneeded,
1837 lazy,
1838 &required_alignment,
1839 &nw.interface,
1840 .none,
1841 .{ .atom_index = @intFromEnum(si) },
1842 );
1843 si.get(coff).size = @intCast(nw.interface.end);
1844 si.applyLocationRelocs(coff);
1845}
1846
1847fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void {
1848 const node = coff.getNode(ni);
1849 switch (node) {
1850 else => |tag| @panic(@tagName(tag)),
1851 .section => |si| return coff.targetStore(
1852 &si.get(coff).section_number.header(coff).pointer_to_raw_data,
1853 @intCast(ni.fileLocation(&coff.mf, false).offset),
1854 ),
1855 .import_directory_table => {},
1856 .import_lookup_table => |import_directory_table_index| {
1857 const import_directory_table: []std.coff.ImportDirectoryEntry =
1858 @ptrCast(@alignCast(coff.import_table.directory_table_ni.slice(&coff.mf)));
1859 const import_directory_entry = &import_directory_table[import_directory_table_index];
1860 coff.targetStore(&import_directory_entry.import_lookup_table_rva, coff.computeNodeRva(ni));
1861 },
1862 .import_address_table => |import_directory_table_index| {
1863 const import_directory_table: []std.coff.ImportDirectoryEntry =
1864 @ptrCast(@alignCast(coff.import_table.directory_table_ni.slice(&coff.mf)));
1865 const import_directory_entry = &import_directory_table[import_directory_table_index];
1866 coff.targetStore(&import_directory_entry.import_lookup_table_rva, coff.computeNodeRva(ni));
1867 const import_address_table_si =
1868 coff.import_table.dlls.values()[import_directory_table_index].import_address_table_si;
1869 import_address_table_si.flushMoved(coff);
1870 coff.targetStore(
1871 &import_directory_entry.import_address_table_rva,
1872 import_address_table_si.get(coff).rva,
1873 );
1874 },
1875 .import_hint_name_table => |import_directory_table_index| {
1876 const target_endian = coff.targetEndian();
1877 const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic);
1878 const import_directory_table: []std.coff.ImportDirectoryEntry =
1879 @ptrCast(@alignCast(coff.import_table.directory_table_ni.slice(&coff.mf)));
1880 const import_directory_entry = &import_directory_table[import_directory_table_index];
1881 const import_hint_name_rva = coff.computeNodeRva(ni);
1882 coff.targetStore(&import_directory_entry.name_rva, import_hint_name_rva);
1883 const import_entry = &coff.import_table.dlls.values()[import_directory_table_index];
1884 const import_lookup_slice = import_entry.import_lookup_table_ni.slice(&coff.mf);
1885 const import_address_slice =
1886 import_entry.import_address_table_si.node(coff).slice(&coff.mf);
1887 const import_hint_name_slice = ni.slice(&coff.mf);
1888 const import_hint_name_align = ni.alignment(&coff.mf);
1889 var import_hint_name_index: u32 = 0;
1890 for (0..import_entry.len) |import_symbol_index| {
1891 import_hint_name_index = @intCast(import_hint_name_align.forward(
1892 std.mem.indexOfScalarPos(
1893 u8,
1894 import_hint_name_slice,
1895 import_hint_name_index,
1896 0,
1897 ).? + 1,
1898 ));
1899 switch (magic) {
1900 _ => unreachable,
1901 inline .PE32, .@"PE32+" => |ct_magic| {
1902 const Addr = switch (ct_magic) {
1903 _ => comptime unreachable,
1904 .PE32 => u32,
1905 .@"PE32+" => u64,
1906 };
1907 const import_lookup_table: []Addr = @ptrCast(@alignCast(import_lookup_slice));
1908 const import_address_table: []Addr = @ptrCast(@alignCast(import_address_slice));
1909 const rva = std.mem.nativeTo(
1910 Addr,
1911 import_hint_name_rva + import_hint_name_index,
1912 target_endian,
1913 );
1914 import_lookup_table[import_symbol_index] = rva;
1915 import_address_table[import_symbol_index] = rva;
1916 },
1917 }
1918 import_hint_name_index += 2;
1919 }
1920 },
1921 inline .global,
1922 .nav,
1923 .uav,
1924 .lazy_code,
1925 .lazy_const_data,
1926 => |mi| mi.symbol(coff).flushMoved(coff),
1927 }
1928 try ni.childrenMoved(coff.base.comp.gpa, &coff.mf);
1929}
1930
1931fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void {
1932 _, const size = ni.location(&coff.mf).resolve(&coff.mf);
1933 const node = coff.getNode(ni);
1934 switch (node) {
1935 else => |tag| @panic(@tagName(tag)),
1936 .file => {},
1937 .header => {
1938 switch (coff.optionalHeaderPtr()) {
1939 inline else => |optional_header| coff.targetStore(
1940 &optional_header.size_of_headers,
1941 @intCast(size),
1942 ),
1943 }
1944 if (size > coff.section_table.items[0].get(coff).rva) try coff.virtualSlide(
1945 0,
1946 std.mem.alignForward(
1947 u32,
1948 @intCast(size * 4),
1949 coff.optionalHeaderField(.section_alignment),
1950 ),
1951 );
1952 },
1953 .section_table => {},
1954 .section => |si| {
1955 const sym = si.get(coff);
1956 const section_table = coff.sectionTableSlice();
1957 const section_index = sym.section_number.toIndex();
1958 const section = &section_table[section_index];
1959 coff.targetStore(&section.size_of_raw_data, @intCast(size));
1960 if (size > coff.targetLoad(&section.virtual_size)) {
1961 const virtual_size = std.mem.alignForward(
1962 u32,
1963 @intCast(size * 4),
1964 coff.optionalHeaderField(.section_alignment),
1965 );
1966 coff.targetStore(&section.virtual_size, virtual_size);
1967 if (sym.data_directory) |data_directory|
1968 coff.dataDirectoriesSlice()[@intFromEnum(data_directory)].size =
1969 section.virtual_size;
1970 try coff.virtualSlide(section_index + 1, sym.rva + virtual_size);
1971 }
1972 },
1973 .import_directory_table,
1974 .import_lookup_table,
1975 .import_address_table,
1976 .import_hint_name_table,
1977 .global,
1978 .nav,
1979 .uav,
1980 .lazy_code,
1981 .lazy_const_data,
1982 => {},
1983 }
1984}
1985
1986fn virtualSlide(coff: *Coff, start_section_index: usize, start_rva: u32) !void {
1987 const section_table = coff.sectionTableSlice();
1988 var rva = start_rva;
1989 for (
1990 coff.section_table.items[start_section_index..],
1991 section_table[start_section_index..],
1992 ) |section_si, *section| {
1993 const section_sym = section_si.get(coff);
1994 section_sym.rva = rva;
1995 coff.targetStore(&section.virtual_address, rva);
1996 if (section_sym.data_directory) |data_directory|
1997 coff.dataDirectoriesSlice()[@intFromEnum(data_directory)].virtual_address =
1998 section.virtual_address;
1999 try section_sym.ni.childrenMoved(coff.base.comp.gpa, &coff.mf);
2000 rva += coff.targetLoad(&section.virtual_size);
2001 }
2002 switch (coff.optionalHeaderPtr()) {
2003 inline else => |optional_header| coff.targetStore(
2004 &optional_header.size_of_image,
2005 @intCast(rva),
2006 ),
2007 }
2008}
2009
2010pub fn updateExports(
2011 coff: *Coff,
2012 pt: Zcu.PerThread,
2013 exported: Zcu.Exported,
2014 export_indices: []const Zcu.Export.Index,
2015) !void {
2016 return coff.updateExportsInner(pt, exported, export_indices) catch |err| switch (err) {
2017 error.OutOfMemory => error.OutOfMemory,
2018 error.LinkFailure => error.AnalysisFail,
2019 };
2020}
2021fn updateExportsInner(
2022 coff: *Coff,
2023 pt: Zcu.PerThread,
2024 exported: Zcu.Exported,
2025 export_indices: []const Zcu.Export.Index,
2026) !void {
2027 const zcu = pt.zcu;
2028 const gpa = zcu.gpa;
2029 const ip = &zcu.intern_pool;
2030
2031 switch (exported) {
2032 .nav => |nav| log.debug("updateExports({f})", .{ip.getNav(nav).fqn.fmt(ip)}),
2033 .uav => |uav| log.debug("updateExports(@as({f}, {f}))", .{
2034 Type.fromInterned(ip.typeOf(uav)).fmt(pt),
2035 Value.fromInterned(uav).fmtValue(pt),
2036 }),
2037 }
2038 try coff.symbol_table.ensureUnusedCapacity(gpa, export_indices.len);
2039 const exported_si: Symbol.Index = switch (exported) {
2040 .nav => |nav| try coff.navSymbol(zcu, nav),
2041 .uav => |uav| @enumFromInt(switch (try coff.lowerUav(
2042 pt,
2043 uav,
2044 Type.fromInterned(ip.typeOf(uav)).abiAlignment(zcu),
2045 export_indices[0].ptr(zcu).src,
2046 )) {
2047 .sym_index => |si| si,
2048 .fail => |em| {
2049 defer em.destroy(gpa);
2050 return coff.base.comp.link_diags.fail("{s}", .{em.msg});
2051 },
2052 }),
2053 };
2054 while (try coff.idle(pt.tid)) {}
2055 const exported_ni = exported_si.node(coff);
2056 const exported_sym = exported_si.get(coff);
2057 for (export_indices) |export_index| {
2058 const @"export" = export_index.ptr(zcu);
2059 const export_si = try coff.globalSymbol(@"export".opts.name.toSlice(ip), null);
2060 const export_sym = export_si.get(coff);
2061 export_sym.ni = exported_ni;
2062 export_sym.rva = exported_sym.rva;
2063 export_sym.size = exported_sym.size;
2064 export_sym.section_number = exported_sym.section_number;
2065 export_si.applyTargetRelocs(coff);
2066 if (@"export".opts.name.eqlSlice("wWinMainCRTStartup", ip)) {
2067 coff.entry_hack = exported_si;
2068 coff.optionalHeaderStandardPtr().address_of_entry_point = exported_sym.rva;
2069 }
2070 }
2071}
2072
2073pub fn deleteExport(coff: *Coff, exported: Zcu.Exported, name: InternPool.NullTerminatedString) void {
2074 _ = coff;
2075 _ = exported;
2076 _ = name;
2077}
2078
2079pub fn dump(coff: *Coff, tid: Zcu.PerThread.Id) void {
2080 const w = std.debug.lockStderrWriter(&.{});
2081 defer std.debug.unlockStderrWriter();
2082 coff.printNode(tid, w, .root, 0) catch {};
2083}
2084
2085pub fn printNode(
2086 coff: *Coff,
2087 tid: Zcu.PerThread.Id,
2088 w: *std.Io.Writer,
2089 ni: MappedFile.Node.Index,
2090 indent: usize,
2091) !void {
2092 const node = coff.getNode(ni);
2093 try w.splatByteAll(' ', indent);
2094 try w.writeAll(@tagName(node));
2095 switch (node) {
2096 else => {},
2097 .section => |si| try w.print("({s})", .{
2098 std.mem.sliceTo(&si.get(coff).section_number.header(coff).name, 0),
2099 }),
2100 .import_lookup_table,
2101 .import_address_table,
2102 .import_hint_name_table,
2103 => |import_directory_table_index| try w.print("({s})", .{
2104 std.mem.sliceTo(coff.import_table.dlls.values()[import_directory_table_index]
2105 .import_hint_name_table_ni.sliceConst(&coff.mf), 0),
2106 }),
2107 .global => |gmi| {
2108 const gn = gmi.globalName(coff);
2109 try w.writeByte('(');
2110 if (gn.lib_name.toSlice(coff)) |lib_name| try w.print("{s}.dll, ", .{lib_name});
2111 try w.print("{s})", .{gn.name.toSlice(coff)});
2112 },
2113 .nav => |nmi| {
2114 const zcu = coff.base.comp.zcu.?;
2115 const ip = &zcu.intern_pool;
2116 const nav = ip.getNav(nmi.navIndex(coff));
2117 try w.print("({f}, {f})", .{
2118 Type.fromInterned(nav.typeOf(ip)).fmt(.{ .zcu = zcu, .tid = tid }),
2119 nav.fqn.fmt(ip),
2120 });
2121 },
2122 .uav => |umi| {
2123 const zcu = coff.base.comp.zcu.?;
2124 const val: Value = .fromInterned(umi.uavValue(coff));
2125 try w.print("({f}, {f})", .{
2126 val.typeOf(zcu).fmt(.{ .zcu = zcu, .tid = tid }),
2127 val.fmtValue(.{ .zcu = zcu, .tid = tid }),
2128 });
2129 },
2130 inline .lazy_code, .lazy_const_data => |lmi| try w.print("({f})", .{
2131 Type.fromInterned(lmi.lazySymbol(coff).ty).fmt(.{
2132 .zcu = coff.base.comp.zcu.?,
2133 .tid = tid,
2134 }),
2135 }),
2136 }
2137 {
2138 const mf_node = &coff.mf.nodes.items[@intFromEnum(ni)];
2139 const off, const size = mf_node.location().resolve(&coff.mf);
2140 try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{
2141 @intFromEnum(ni),
2142 off,
2143 size,
2144 mf_node.flags.alignment.toByteUnits(),
2145 if (mf_node.flags.fixed) " fixed" else "",
2146 if (mf_node.flags.moved) " moved" else "",
2147 if (mf_node.flags.resized) " resized" else "",
2148 if (mf_node.flags.has_content) " has_content" else "",
2149 });
2150 }
2151 var leaf = true;
2152 var child_it = ni.children(&coff.mf);
2153 while (child_it.next()) |child_ni| {
2154 leaf = false;
2155 try coff.printNode(tid, w, child_ni, indent + 1);
2156 }
2157 if (leaf) {
2158 const file_loc = ni.fileLocation(&coff.mf, false);
2159 if (file_loc.size == 0) return;
2160 var address = file_loc.offset;
2161 const line_len = 0x10;
2162 var line_it = std.mem.window(
2163 u8,
2164 coff.mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
2165 line_len,
2166 line_len,
2167 );
2168 while (line_it.next()) |line_bytes| : (address += line_len) {
2169 try w.splatByteAll(' ', indent + 1);
2170 try w.print("{x:0>8} ", .{address});
2171 for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte});
2172 try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1);
2173 for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.');
2174 try w.writeByte('\n');
2175 }
2176 }
2177}
2178
2179const assert = std.debug.assert;
2180const builtin = @import("builtin");
2181const codegen = @import("../codegen.zig");
2182const Compilation = @import("../Compilation.zig");
2183const Coff = @This();
2184const InternPool = @import("../InternPool.zig");
2185const link = @import("../link.zig");
2186const log = std.log.scoped(.link);
2187const MappedFile = @import("MappedFile.zig");
2188const native_endian = builtin.cpu.arch.endian();
2189const std = @import("std");
2190const target_util = @import("../target.zig");
2191const Type = @import("../Type.zig");
2192const Value = @import("../Value.zig");
2193const Zcu = @import("../Zcu.zig");
src/link/Elf2.zig+325-303
......@@ -11,7 +11,7 @@ lazy: std.EnumArray(link.File.LazySymbol.Kind, struct {
1111 map: std.AutoArrayHashMapUnmanaged(InternPool.Index, Symbol.Index),
1212 pending_index: u32,
1313}),
14pending_uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, struct {
14pending_uavs: std.AutoArrayHashMapUnmanaged(Node.UavMapIndex, struct {
1515 alignment: InternPool.Alignment,
1616 src_loc: Zcu.LazySrcLoc,
1717}),
......@@ -25,10 +25,65 @@ pub const Node = union(enum) {
2525 shdr,
2626 segment: u32,
2727 section: Symbol.Index,
28 nav: InternPool.Nav.Index,
29 uav: InternPool.Index,
30 lazy_code: InternPool.Index,
31 lazy_const_data: InternPool.Index,
28 nav: NavMapIndex,
29 uav: UavMapIndex,
30 lazy_code: LazyMapRef.Index(.code),
31 lazy_const_data: LazyMapRef.Index(.const_data),
32
33 pub const NavMapIndex = enum(u32) {
34 _,
35
36 pub fn navIndex(nmi: NavMapIndex, elf: *const Elf) InternPool.Nav.Index {
37 return elf.navs.keys()[@intFromEnum(nmi)];
38 }
39
40 pub fn symbol(nmi: NavMapIndex, elf: *const Elf) Symbol.Index {
41 return elf.navs.values()[@intFromEnum(nmi)];
42 }
43 };
44
45 pub const UavMapIndex = enum(u32) {
46 _,
47
48 pub fn uavValue(umi: UavMapIndex, elf: *const Elf) InternPool.Index {
49 return elf.uavs.keys()[@intFromEnum(umi)];
50 }
51
52 pub fn symbol(umi: UavMapIndex, elf: *const Elf) Symbol.Index {
53 return elf.uavs.values()[@intFromEnum(umi)];
54 }
55 };
56
57 pub const LazyMapRef = struct {
58 kind: link.File.LazySymbol.Kind,
59 index: u32,
60
61 pub fn Index(comptime kind: link.File.LazySymbol.Kind) type {
62 return enum(u32) {
63 _,
64
65 pub fn ref(lmi: @This()) LazyMapRef {
66 return .{ .kind = kind, .index = @intFromEnum(lmi) };
67 }
68
69 pub fn lazySymbol(lmi: @This(), elf: *const Elf) link.File.LazySymbol {
70 return lmi.ref().lazySymbol(elf);
71 }
72
73 pub fn symbol(lmi: @This(), elf: *const Elf) Symbol.Index {
74 return lmi.ref().symbol(elf);
75 }
76 };
77 }
78
79 pub fn lazySymbol(lmr: LazyMapRef, elf: *const Elf) link.File.LazySymbol {
80 return .{ .kind = lmr.kind, .ty = elf.lazy.getPtrConst(lmr.kind).map.keys()[lmr.index] };
81 }
82
83 pub fn symbol(lmr: LazyMapRef, elf: *const Elf) Symbol.Index {
84 return elf.lazy.getPtrConst(lmr.kind).map.values()[lmr.index];
85 }
86 };
3287
3388 pub const Tag = @typeInfo(Node).@"union".tag_type.?;
3489
......@@ -43,11 +98,7 @@ pub const Node = union(enum) {
4398 seg_text,
4499 seg_data,
45100 };
46 var mut_known: std.enums.EnumFieldStruct(
47 Known,
48 MappedFile.Node.Index,
49 null,
50 ) = undefined;
101 var mut_known: std.enums.EnumFieldStruct(Known, MappedFile.Node.Index, null) = undefined;
51102 for (@typeInfo(Known).@"enum".fields) |field|
52103 @field(mut_known, field.name) = @enumFromInt(field.value);
53104 break :known mut_known;
......@@ -223,10 +274,10 @@ pub const Reloc = extern struct {
223274 addend: i64,
224275
225276 pub const Type = extern union {
226 x86_64: std.elf.R_X86_64,
227 aarch64: std.elf.R_AARCH64,
228 riscv: std.elf.R_RISCV,
229 ppc64: std.elf.R_PPC64,
277 X86_64: std.elf.R_X86_64,
278 AARCH64: std.elf.R_AARCH64,
279 RISCV: std.elf.R_RISCV,
280 PPC64: std.elf.R_PPC64,
230281 };
231282
232283 pub const Index = enum(u32) {
......@@ -239,7 +290,7 @@ pub const Reloc = extern struct {
239290 };
240291
241292 pub fn apply(reloc: *const Reloc, elf: *Elf) void {
242 const target_endian = elf.endian();
293 const target_endian = elf.targetEndian();
243294 switch (reloc.loc.get(elf).ni) {
244295 .none => return,
245296 else => |ni| if (ni.hasMoved(&elf.mf)) return,
......@@ -274,7 +325,7 @@ pub const Reloc = extern struct {
274325 ) +% @as(u64, @bitCast(reloc.addend));
275326 switch (elf.ehdrField(.machine)) {
276327 else => |machine| @panic(@tagName(machine)),
277 .X86_64 => switch (reloc.type.x86_64) {
328 .X86_64 => switch (reloc.type.X86_64) {
278329 else => |kind| @panic(@tagName(kind)),
279330 .@"64" => std.mem.writeInt(
280331 u64,
......@@ -394,37 +445,7 @@ fn create(
394445 },
395446 .Obj => .REL,
396447 };
397 const machine: std.elf.EM = switch (target.cpu.arch) {
398 .spirv32, .spirv64, .wasm32, .wasm64 => .NONE,
399 .sparc => .SPARC,
400 .x86 => .@"386",
401 .m68k => .@"68K",
402 .mips, .mipsel, .mips64, .mips64el => .MIPS,
403 .powerpc, .powerpcle => .PPC,
404 .powerpc64, .powerpc64le => .PPC64,
405 .s390x => .S390,
406 .arm, .armeb, .thumb, .thumbeb => .ARM,
407 .hexagon => .SH,
408 .sparc64 => .SPARCV9,
409 .arc => .ARC,
410 .x86_64 => .X86_64,
411 .or1k => .OR1K,
412 .xtensa => .XTENSA,
413 .msp430 => .MSP430,
414 .avr => .AVR,
415 .nvptx, .nvptx64 => .CUDA,
416 .kalimba => .CSR_KALIMBA,
417 .aarch64, .aarch64_be => .AARCH64,
418 .xcore => .XCORE,
419 .amdgcn => .AMDGPU,
420 .riscv32, .riscv32be, .riscv64, .riscv64be => .RISCV,
421 .lanai => .LANAI,
422 .bpfel, .bpfeb => .BPF,
423 .ve => .VE,
424 .csky => .CSKY,
425 .loongarch32, .loongarch64 => .LOONGARCH,
426 .propeller => if (target.cpu.has(.propeller, .p2)) .PROPELLER2 else .PROPELLER,
427 };
448 const machine = target.toElfMachine();
428449 const maybe_interp = switch (comp.config.output_mode) {
429450 .Exe, .Lib => switch (comp.config.link_mode) {
430451 .static => null,
......@@ -479,7 +500,7 @@ fn create(
479500
480501 switch (class) {
481502 .NONE, _ => unreachable,
482 inline .@"32", .@"64" => |ct_class| try elf.initHeaders(
503 inline else => |ct_class| try elf.initHeaders(
483504 ct_class,
484505 data,
485506 osabi,
......@@ -567,30 +588,31 @@ fn initHeaders(
567588 .fixed = true,
568589 }));
569590 elf.nodes.appendAssumeCapacity(.ehdr);
570
571 const ehdr: *ElfN.Ehdr = @ptrCast(@alignCast(ehdr_ni.slice(&elf.mf)));
572 const EI = std.elf.EI;
573 @memcpy(ehdr.ident[0..std.elf.MAGIC.len], std.elf.MAGIC);
574 ehdr.ident[EI.CLASS] = @intFromEnum(class);
575 ehdr.ident[EI.DATA] = @intFromEnum(data);
576 ehdr.ident[EI.VERSION] = 1;
577 ehdr.ident[EI.OSABI] = @intFromEnum(osabi);
578 ehdr.ident[EI.ABIVERSION] = 0;
579 @memset(ehdr.ident[EI.PAD..], 0);
580 ehdr.type = @"type";
581 ehdr.machine = machine;
582 ehdr.version = 1;
583 ehdr.entry = 0;
584 ehdr.phoff = 0;
585 ehdr.shoff = 0;
586 ehdr.flags = 0;
587 ehdr.ehsize = @sizeOf(ElfN.Ehdr);
588 ehdr.phentsize = @sizeOf(ElfN.Phdr);
589 ehdr.phnum = @min(phnum, std.elf.PN_XNUM);
590 ehdr.shentsize = @sizeOf(ElfN.Shdr);
591 ehdr.shnum = 1;
592 ehdr.shstrndx = 0;
593 if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr);
591 {
592 const ehdr: *ElfN.Ehdr = @ptrCast(@alignCast(ehdr_ni.slice(&elf.mf)));
593 const EI = std.elf.EI;
594 @memcpy(ehdr.ident[0..std.elf.MAGIC.len], std.elf.MAGIC);
595 ehdr.ident[EI.CLASS] = @intFromEnum(class);
596 ehdr.ident[EI.DATA] = @intFromEnum(data);
597 ehdr.ident[EI.VERSION] = 1;
598 ehdr.ident[EI.OSABI] = @intFromEnum(osabi);
599 ehdr.ident[EI.ABIVERSION] = 0;
600 @memset(ehdr.ident[EI.PAD..], 0);
601 ehdr.type = @"type";
602 ehdr.machine = machine;
603 ehdr.version = 1;
604 ehdr.entry = 0;
605 ehdr.phoff = 0;
606 ehdr.shoff = 0;
607 ehdr.flags = 0;
608 ehdr.ehsize = @sizeOf(ElfN.Ehdr);
609 ehdr.phentsize = @sizeOf(ElfN.Phdr);
610 ehdr.phnum = @min(phnum, std.elf.PN_XNUM);
611 ehdr.shentsize = @sizeOf(ElfN.Shdr);
612 ehdr.shnum = 1;
613 ehdr.shstrndx = 0;
614 if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr);
615 }
594616
595617 const phdr_ni = Node.known.phdr;
596618 assert(phdr_ni == try elf.mf.addLastChildNode(gpa, seg_rodata_ni, .{
......@@ -750,7 +772,10 @@ fn initHeaders(
750772 },
751773 .shndx = std.elf.SHN_UNDEF,
752774 };
753 ehdr.shstrndx = ehdr.shnum;
775 {
776 const ehdr = @field(elf.ehdrPtr(), @tagName(class));
777 ehdr.shstrndx = ehdr.shnum;
778 }
754779 assert(try elf.addSection(seg_rodata_ni, .{
755780 .type = std.elf.SHT_STRTAB,
756781 .addralign = elf.mf.flags.block_size,
......@@ -821,6 +846,24 @@ fn getNode(elf: *Elf, ni: MappedFile.Node.Index) Node {
821846 return elf.nodes.get(@intFromEnum(ni));
822847}
823848
849pub fn identClass(elf: *Elf) std.elf.CLASS {
850 return @enumFromInt(elf.mf.contents[std.elf.EI.CLASS]);
851}
852
853pub fn identData(elf: *Elf) std.elf.DATA {
854 return @enumFromInt(elf.mf.contents[std.elf.EI.DATA]);
855}
856fn endianForData(data: std.elf.DATA) std.builtin.Endian {
857 return switch (data) {
858 .NONE, _ => unreachable,
859 .@"2LSB" => .little,
860 .@"2MSB" => .big,
861 };
862}
863pub fn targetEndian(elf: *Elf) std.builtin.Endian {
864 return endianForData(elf.identData());
865}
866
824867pub const EhdrPtr = union(std.elf.CLASS) {
825868 NONE: noreturn,
826869 @"32": *std.elf.Elf32.Ehdr,
......@@ -830,7 +873,7 @@ pub fn ehdrPtr(elf: *Elf) EhdrPtr {
830873 const slice = Node.known.ehdr.slice(&elf.mf);
831874 return switch (elf.identClass()) {
832875 .NONE, _ => unreachable,
833 inline .@"32", .@"64" => |class| @unionInit(
876 inline else => |class| @unionInit(
834877 EhdrPtr,
835878 @tagName(class),
836879 @ptrCast(@alignCast(slice)),
......@@ -841,35 +884,15 @@ pub fn ehdrField(
841884 elf: *Elf,
842885 comptime field: enum { type, machine },
843886) @FieldType(std.elf.Elf32.Ehdr, @tagName(field)) {
844 const Field = @FieldType(std.elf.Elf32.Ehdr, @tagName(field));
845 comptime assert(@FieldType(std.elf.Elf64.Ehdr, @tagName(field)) == Field);
846887 return @enumFromInt(std.mem.toNative(
847 @typeInfo(Field).@"enum".tag_type,
888 @typeInfo(@FieldType(std.elf.Elf32.Ehdr, @tagName(field))).@"enum".tag_type,
848889 @intFromEnum(switch (elf.ehdrPtr()) {
849890 inline else => |ehdr| @field(ehdr, @tagName(field)),
850891 }),
851 elf.endian(),
892 elf.targetEndian(),
852893 ));
853894}
854895
855pub fn identClass(elf: *Elf) std.elf.CLASS {
856 return @enumFromInt(elf.mf.contents[std.elf.EI.CLASS]);
857}
858
859pub fn identData(elf: *Elf) std.elf.DATA {
860 return @enumFromInt(elf.mf.contents[std.elf.EI.DATA]);
861}
862fn endianForData(data: std.elf.DATA) std.builtin.Endian {
863 return switch (data) {
864 .NONE, _ => unreachable,
865 .@"2LSB" => .little,
866 .@"2MSB" => .big,
867 };
868}
869pub fn endian(elf: *Elf) std.builtin.Endian {
870 return endianForData(elf.identData());
871}
872
873896fn baseAddrForType(@"type": std.elf.ET) u64 {
874897 return switch (@"type") {
875898 else => 0,
......@@ -889,7 +912,7 @@ pub fn phdrSlice(elf: *Elf) PhdrSlice {
889912 const slice = Node.known.phdr.slice(&elf.mf);
890913 return switch (elf.identClass()) {
891914 .NONE, _ => unreachable,
892 inline .@"32", .@"64" => |class| @unionInit(
915 inline else => |class| @unionInit(
893916 PhdrSlice,
894917 @tagName(class),
895918 @ptrCast(@alignCast(slice)),
......@@ -906,7 +929,7 @@ pub fn shdrSlice(elf: *Elf) ShdrSlice {
906929 const slice = Node.known.shdr.slice(&elf.mf);
907930 return switch (elf.identClass()) {
908931 .NONE, _ => unreachable,
909 inline .@"32", .@"64" => |class| @unionInit(
932 inline else => |class| @unionInit(
910933 ShdrSlice,
911934 @tagName(class),
912935 @ptrCast(@alignCast(slice)),
......@@ -923,7 +946,7 @@ pub fn symSlice(elf: *Elf) SymSlice {
923946 const slice = Symbol.Index.symtab.node(elf).slice(&elf.mf);
924947 return switch (elf.identClass()) {
925948 .NONE, _ => unreachable,
926 inline .@"32", .@"64" => |class| @unionInit(
949 inline else => |class| @unionInit(
927950 SymSlice,
928951 @tagName(class),
929952 @ptrCast(@alignCast(slice)),
......@@ -942,7 +965,7 @@ pub fn symPtr(elf: *Elf, si: Symbol.Index) SymPtr {
942965 };
943966}
944967
945fn addSymbolAssumeCapacity(elf: *Elf) !Symbol.Index {
968fn addSymbolAssumeCapacity(elf: *Elf) Symbol.Index {
946969 defer elf.symtab.addOneAssumeCapacity().* = .{
947970 .ni = .none,
948971 .loc_relocs = .none,
......@@ -953,30 +976,27 @@ fn addSymbolAssumeCapacity(elf: *Elf) !Symbol.Index {
953976}
954977
955978fn initSymbolAssumeCapacity(elf: *Elf, opts: Symbol.Index.InitOptions) !Symbol.Index {
956 const si = try elf.addSymbolAssumeCapacity();
979 const si = elf.addSymbolAssumeCapacity();
957980 try si.init(elf, opts);
958981 return si;
959982}
960983
961pub fn globalSymbol(
962 elf: *Elf,
963 opts: struct {
964 name: []const u8,
965 type: std.elf.STT,
966 bind: std.elf.STB = .GLOBAL,
967 visibility: std.elf.STV = .DEFAULT,
968 },
969) !Symbol.Index {
984pub fn globalSymbol(elf: *Elf, opts: struct {
985 name: []const u8,
986 type: std.elf.STT,
987 bind: std.elf.STB = .GLOBAL,
988 visibility: std.elf.STV = .DEFAULT,
989}) !Symbol.Index {
970990 const gpa = elf.base.comp.gpa;
971991 try elf.symtab.ensureUnusedCapacity(gpa, 1);
972 const sym_gop = try elf.globals.getOrPut(gpa, try elf.string(.strtab, opts.name));
973 if (!sym_gop.found_existing) sym_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
992 const global_gop = try elf.globals.getOrPut(gpa, try elf.string(.strtab, opts.name));
993 if (!global_gop.found_existing) global_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
974994 .name = opts.name,
975995 .type = opts.type,
976996 .bind = opts.bind,
977997 .visibility = opts.visibility,
978998 });
979 return sym_gop.value_ptr.*;
999 return global_gop.value_ptr.*;
9801000}
9811001
9821002fn navType(
......@@ -1008,8 +1028,19 @@ fn navType(
10081028 },
10091029 };
10101030}
1011pub fn navSymbol(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Symbol.Index {
1031fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Node.NavMapIndex {
10121032 const gpa = zcu.gpa;
1033 const ip = &zcu.intern_pool;
1034 const nav = ip.getNav(nav_index);
1035 try elf.symtab.ensureUnusedCapacity(gpa, 1);
1036 const nav_gop = try elf.navs.getOrPut(gpa, nav_index);
1037 if (!nav_gop.found_existing) nav_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
1038 .name = nav.fqn.toSlice(ip),
1039 .type = navType(ip, nav.status, elf.base.comp.config.any_non_single_threaded),
1040 });
1041 return @enumFromInt(nav_gop.index);
1042}
1043pub fn navSymbol(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Symbol.Index {
10131044 const ip = &zcu.intern_pool;
10141045 const nav = ip.getNav(nav_index);
10151046 if (nav.getExtern(ip)) |@"extern"| return elf.globalSymbol(.{
......@@ -1027,40 +1058,37 @@ pub fn navSymbol(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) !Symbol.
10271058 .protected => .PROTECTED,
10281059 },
10291060 });
1030 try elf.symtab.ensureUnusedCapacity(gpa, 1);
1031 const sym_gop = try elf.navs.getOrPut(gpa, nav_index);
1032 if (!sym_gop.found_existing) {
1033 sym_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
1034 .name = nav.fqn.toSlice(ip),
1035 .type = navType(ip, nav.status, elf.base.comp.config.any_non_single_threaded),
1036 });
1037 }
1038 return sym_gop.value_ptr.*;
1061 const nmi = try elf.navMapIndex(zcu, nav_index);
1062 return nmi.symbol(elf);
10391063}
10401064
1041pub fn uavSymbol(elf: *Elf, uav_val: InternPool.Index) !Symbol.Index {
1065fn uavMapIndex(elf: *Elf, uav_val: InternPool.Index) !Node.UavMapIndex {
10421066 const gpa = elf.base.comp.gpa;
10431067 try elf.symtab.ensureUnusedCapacity(gpa, 1);
1044 const sym_gop = try elf.uavs.getOrPut(gpa, uav_val);
1045 if (!sym_gop.found_existing)
1046 sym_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{ .type = .OBJECT });
1047 return sym_gop.value_ptr.*;
1068 const uav_gop = try elf.uavs.getOrPut(gpa, uav_val);
1069 if (!uav_gop.found_existing)
1070 uav_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{ .type = .OBJECT });
1071 return @enumFromInt(uav_gop.index);
1072}
1073pub fn uavSymbol(elf: *Elf, uav_val: InternPool.Index) !Symbol.Index {
1074 const umi = try elf.uavMapIndex(uav_val);
1075 return umi.symbol(elf);
10481076}
10491077
10501078pub fn lazySymbol(elf: *Elf, lazy: link.File.LazySymbol) !Symbol.Index {
10511079 const gpa = elf.base.comp.gpa;
10521080 try elf.symtab.ensureUnusedCapacity(gpa, 1);
1053 const sym_gop = try elf.lazy.getPtr(lazy.kind).map.getOrPut(gpa, lazy.ty);
1054 if (!sym_gop.found_existing) {
1055 sym_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
1081 const lazy_gop = try elf.lazy.getPtr(lazy.kind).map.getOrPut(gpa, lazy.ty);
1082 if (!lazy_gop.found_existing) {
1083 lazy_gop.value_ptr.* = try elf.initSymbolAssumeCapacity(.{
10561084 .type = switch (lazy.kind) {
10571085 .code => .FUNC,
10581086 .const_data => .OBJECT,
10591087 },
10601088 });
1061 elf.base.comp.link_lazy_prog_node.increaseEstimatedTotalItems(1);
1089 elf.base.comp.link_synth_prog_node.increaseEstimatedTotalItems(1);
10621090 }
1063 return sym_gop.value_ptr.*;
1091 return lazy_gop.value_ptr.*;
10641092}
10651093
10661094pub fn getNavVAddr(
......@@ -1088,7 +1116,7 @@ pub fn getVAddr(elf: *Elf, reloc_info: link.File.RelocInfo, target_si: Symbol.In
10881116 reloc_info.addend,
10891117 switch (elf.ehdrField(.machine)) {
10901118 else => unreachable,
1091 .X86_64 => .{ .x86_64 = switch (elf.identClass()) {
1119 .X86_64 => .{ .X86_64 = switch (elf.identClass()) {
10921120 .NONE, _ => unreachable,
10931121 .@"32" => .@"32",
10941122 .@"64" => .@"64",
......@@ -1107,7 +1135,7 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct {
11071135 entsize: std.elf.Word = 0,
11081136}) !Symbol.Index {
11091137 const gpa = elf.base.comp.gpa;
1110 const target_endian = elf.endian();
1138 const target_endian = elf.targetEndian();
11111139 try elf.nodes.ensureUnusedCapacity(gpa, 1);
11121140 try elf.symtab.ensureUnusedCapacity(gpa, 1);
11131141
......@@ -1127,7 +1155,7 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct {
11271155 .size = opts.size,
11281156 .moved = true,
11291157 });
1130 const si = try elf.addSymbolAssumeCapacity();
1158 const si = elf.addSymbolAssumeCapacity();
11311159 elf.nodes.appendAssumeCapacity(.{ .section = si });
11321160 si.get(elf).ni = ni;
11331161 try si.init(elf, .{
......@@ -1160,7 +1188,7 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct {
11601188fn renameSection(elf: *Elf, si: Symbol.Index, name: []const u8) !void {
11611189 const strtab_entry = try elf.string(.strtab, name);
11621190 const shstrtab_entry = try elf.string(.shstrtab, name);
1163 const target_endian = elf.endian();
1191 const target_endian = elf.targetEndian();
11641192 switch (elf.shdrSlice()) {
11651193 inline else => |shdr, class| {
11661194 const sym = @field(elf.symPtr(si), @tagName(class));
......@@ -1173,7 +1201,7 @@ fn renameSection(elf: *Elf, si: Symbol.Index, name: []const u8) !void {
11731201}
11741202
11751203fn linkSections(elf: *Elf, si: Symbol.Index, link_si: Symbol.Index) !void {
1176 const target_endian = elf.endian();
1204 const target_endian = elf.targetEndian();
11771205 switch (elf.shdrSlice()) {
11781206 inline else => |shdr, class| {
11791207 const sym = @field(elf.symPtr(si), @tagName(class));
......@@ -1184,7 +1212,7 @@ fn linkSections(elf: *Elf, si: Symbol.Index, link_si: Symbol.Index) !void {
11841212}
11851213
11861214fn sectionName(elf: *Elf, si: Symbol.Index) [:0]const u8 {
1187 const target_endian = elf.endian();
1215 const target_endian = elf.targetEndian();
11881216 const name = Symbol.Index.shstrtab.node(elf).slice(&elf.mf)[name: switch (elf.shdrSlice()) {
11891217 inline else => |shndx, class| {
11901218 const sym = @field(elf.symPtr(si), @tagName(class));
......@@ -1263,7 +1291,8 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index)
12631291 };
12641292 if (nav_init == .none or !Type.fromInterned(ip.typeOf(nav_init)).hasRuntimeBits(zcu)) return;
12651293
1266 const si = try elf.navSymbol(zcu, nav_index);
1294 const nmi = try elf.navMapIndex(zcu, nav_index);
1295 const si = nmi.symbol(elf);
12671296 const ni = ni: {
12681297 const sym = si.get(elf);
12691298 switch (sym.ni) {
......@@ -1275,7 +1304,7 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index)
12751304 .alignment = pt.navAlignment(nav_index).toStdMem(),
12761305 .moved = true,
12771306 });
1278 elf.nodes.appendAssumeCapacity(.{ .nav = nav_index });
1307 elf.nodes.appendAssumeCapacity(.{ .nav = nmi });
12791308 sym.ni = ni;
12801309 switch (elf.symPtr(si)) {
12811310 inline else => |sym_ptr, class| sym_ptr.shndx =
......@@ -1289,28 +1318,24 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index)
12891318 break :ni sym.ni;
12901319 };
12911320
1292 const size = size: {
1293 var nw: MappedFile.Node.Writer = undefined;
1294 ni.writer(&elf.mf, gpa, &nw);
1295 defer nw.deinit();
1296 codegen.generateSymbol(
1297 &elf.base,
1298 pt,
1299 zcu.navSrcLoc(nav_index),
1300 .fromInterned(nav_init),
1301 &nw.interface,
1302 .{ .atom_index = @intFromEnum(si) },
1303 ) catch |err| switch (err) {
1304 error.WriteFailed => return error.OutOfMemory,
1305 else => |e| return e,
1306 };
1307 break :size nw.interface.end;
1321 var nw: MappedFile.Node.Writer = undefined;
1322 ni.writer(&elf.mf, gpa, &nw);
1323 defer nw.deinit();
1324 codegen.generateSymbol(
1325 &elf.base,
1326 pt,
1327 zcu.navSrcLoc(nav_index),
1328 .fromInterned(nav_init),
1329 &nw.interface,
1330 .{ .atom_index = @intFromEnum(si) },
1331 ) catch |err| switch (err) {
1332 error.WriteFailed => return error.OutOfMemory,
1333 else => |e| return e,
13081334 };
1309
1310 const target_endian = elf.endian();
1335 const target_endian = elf.targetEndian();
13111336 switch (elf.symPtr(si)) {
13121337 inline else => |sym| sym.size =
1313 std.mem.nativeTo(@TypeOf(sym.size), @intCast(size), target_endian),
1338 std.mem.nativeTo(@TypeOf(sym.size), @intCast(nw.interface.end), target_endian),
13141339 }
13151340 si.applyLocationRelocs(elf);
13161341}
......@@ -1326,7 +1351,7 @@ pub fn lowerUav(
13261351 const gpa = zcu.gpa;
13271352
13281353 try elf.pending_uavs.ensureUnusedCapacity(gpa, 1);
1329 const si = elf.uavSymbol(uav_val) catch |err| switch (err) {
1354 const umi = elf.uavMapIndex(uav_val) catch |err| switch (err) {
13301355 error.OutOfMemory => return error.OutOfMemory,
13311356 else => |e| return .{ .fail = try Zcu.ErrorMsg.create(
13321357 gpa,
......@@ -1335,11 +1360,12 @@ pub fn lowerUav(
13351360 .{@errorName(e)},
13361361 ) },
13371362 };
1363 const si = umi.symbol(elf);
13381364 if (switch (si.get(elf).ni) {
13391365 .none => true,
13401366 else => |ni| uav_align.toStdMem().order(ni.alignment(&elf.mf)).compare(.gt),
13411367 }) {
1342 const gop = elf.pending_uavs.getOrPutAssumeCapacity(uav_val);
1368 const gop = elf.pending_uavs.getOrPutAssumeCapacity(umi);
13431369 if (gop.found_existing) {
13441370 gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align);
13451371 } else {
......@@ -1347,7 +1373,7 @@ pub fn lowerUav(
13471373 .alignment = uav_align,
13481374 .src_loc = src_loc,
13491375 };
1350 elf.base.comp.link_uav_prog_node.increaseEstimatedTotalItems(1);
1376 elf.base.comp.link_const_prog_node.increaseEstimatedTotalItems(1);
13511377 }
13521378 }
13531379 return .{ .sym_index = @intFromEnum(si) };
......@@ -1384,7 +1410,8 @@ fn updateFuncInner(
13841410 const func = zcu.funcInfo(func_index);
13851411 const nav = ip.getNav(func.owner_nav);
13861412
1387 const si = try elf.navSymbol(zcu, func.owner_nav);
1413 const nmi = try elf.navMapIndex(zcu, func.owner_nav);
1414 const si = nmi.symbol(elf);
13881415 log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), si });
13891416 const ni = ni: {
13901417 const sym = si.get(elf);
......@@ -1406,7 +1433,7 @@ fn updateFuncInner(
14061433 }.toStdMem(),
14071434 .moved = true,
14081435 });
1409 elf.nodes.appendAssumeCapacity(.{ .nav = func.owner_nav });
1436 elf.nodes.appendAssumeCapacity(.{ .nav = nmi });
14101437 sym.ni = ni;
14111438 switch (elf.symPtr(si)) {
14121439 inline else => |sym_ptr, class| sym_ptr.shndx =
......@@ -1420,37 +1447,35 @@ fn updateFuncInner(
14201447 break :ni sym.ni;
14211448 };
14221449
1423 const size = size: {
1424 var nw: MappedFile.Node.Writer = undefined;
1425 ni.writer(&elf.mf, gpa, &nw);
1426 defer nw.deinit();
1427 codegen.emitFunction(
1428 &elf.base,
1429 pt,
1430 zcu.navSrcLoc(func.owner_nav),
1431 func_index,
1432 @intFromEnum(si),
1433 mir,
1434 &nw.interface,
1435 .none,
1436 ) catch |err| switch (err) {
1437 error.WriteFailed => return nw.err.?,
1438 else => |e| return e,
1439 };
1440 break :size nw.interface.end;
1450 var nw: MappedFile.Node.Writer = undefined;
1451 ni.writer(&elf.mf, gpa, &nw);
1452 defer nw.deinit();
1453 codegen.emitFunction(
1454 &elf.base,
1455 pt,
1456 zcu.navSrcLoc(func.owner_nav),
1457 func_index,
1458 @intFromEnum(si),
1459 mir,
1460 &nw.interface,
1461 .none,
1462 ) catch |err| switch (err) {
1463 error.WriteFailed => return nw.err.?,
1464 else => |e| return e,
14411465 };
1442
1443 const target_endian = elf.endian();
1466 const target_endian = elf.targetEndian();
14441467 switch (elf.symPtr(si)) {
14451468 inline else => |sym| sym.size =
1446 std.mem.nativeTo(@TypeOf(sym.size), @intCast(size), target_endian),
1469 std.mem.nativeTo(@TypeOf(sym.size), @intCast(nw.interface.end), target_endian),
14471470 }
14481471 si.applyLocationRelocs(elf);
14491472}
14501473
14511474pub fn updateErrorData(elf: *Elf, pt: Zcu.PerThread) !void {
1452 const si = elf.lazy.getPtr(.const_data).map.get(.anyerror_type) orelse return;
1453 elf.flushLazy(pt, .{ .kind = .const_data, .ty = .anyerror_type }, si) catch |err| switch (err) {
1475 elf.flushLazy(pt, .{
1476 .kind = .const_data,
1477 .index = @intCast(elf.lazy.getPtr(.const_data).map.getIndex(.anyerror_type) orelse return),
1478 }) catch |err| switch (err) {
14541479 error.OutOfMemory => return error.OutOfMemory,
14551480 error.CodegenFail => return error.LinkFailure,
14561481 else => |e| return elf.base.comp.link_diags.fail("updateErrorData failed {t}", .{e}),
......@@ -1472,14 +1497,13 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) !bool {
14721497 const comp = elf.base.comp;
14731498 task: {
14741499 while (elf.pending_uavs.pop()) |pending_uav| {
1475 const sub_prog_node =
1476 elf.idleProgNode(
1477 tid,
1478 comp.link_uav_prog_node,
1479 .{ .uav = pending_uav.key },
1480 );
1500 const sub_prog_node = elf.idleProgNode(
1501 tid,
1502 comp.link_const_prog_node,
1503 .{ .uav = pending_uav.key },
1504 );
14811505 defer sub_prog_node.end();
1482 break :task elf.flushUav(
1506 elf.flushUav(
14831507 .{ .zcu = elf.base.comp.zcu.?, .tid = tid },
14841508 pending_uav.key,
14851509 pending_uav.value.alignment,
......@@ -1491,37 +1515,34 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) !bool {
14911515 .{e},
14921516 ),
14931517 };
1518 break :task;
14941519 }
14951520 var lazy_it = elf.lazy.iterator();
1496 while (lazy_it.next()) |lazy| for (
1497 lazy.value.map.keys()[lazy.value.pending_index..],
1498 lazy.value.map.values()[lazy.value.pending_index..],
1499 ) |ty, si| {
1500 lazy.value.pending_index += 1;
1521 while (lazy_it.next()) |lazy| if (lazy.value.pending_index < lazy.value.map.count()) {
15011522 const pt: Zcu.PerThread = .{ .zcu = elf.base.comp.zcu.?, .tid = tid };
1502 const kind = switch (lazy.key) {
1523 const lmr: Node.LazyMapRef = .{ .kind = lazy.key, .index = lazy.value.pending_index };
1524 lazy.value.pending_index += 1;
1525 const kind = switch (lmr.kind) {
15031526 .code => "code",
15041527 .const_data => "data",
15051528 };
15061529 var name: [std.Progress.Node.max_name_len]u8 = undefined;
1507 const sub_prog_node = comp.link_lazy_prog_node.start(
1530 const sub_prog_node = comp.link_synth_prog_node.start(
15081531 std.fmt.bufPrint(&name, "lazy {s} for {f}", .{
15091532 kind,
1510 Type.fromInterned(ty).fmt(pt),
1533 Type.fromInterned(lmr.lazySymbol(elf).ty).fmt(pt),
15111534 }) catch &name,
15121535 0,
15131536 );
15141537 defer sub_prog_node.end();
1515 break :task elf.flushLazy(pt, .{
1516 .kind = lazy.key,
1517 .ty = ty,
1518 }, si) catch |err| switch (err) {
1538 elf.flushLazy(pt, lmr) catch |err| switch (err) {
15191539 error.OutOfMemory => return error.OutOfMemory,
15201540 else => |e| return elf.base.comp.link_diags.fail(
15211541 "linker failed to lower lazy {s}: {t}",
15221542 .{ kind, e },
15231543 ),
15241544 };
1545 break :task;
15251546 };
15261547 while (elf.mf.updates.pop()) |ni| {
15271548 const clean_moved = ni.cleanMoved(&elf.mf);
......@@ -1551,12 +1572,12 @@ fn idleProgNode(
15511572 return prog_node.start(name: switch (node) {
15521573 else => |tag| @tagName(tag),
15531574 .section => |si| elf.sectionName(si),
1554 .nav => |nav| {
1575 .nav => |nmi| {
15551576 const ip = &elf.base.comp.zcu.?.intern_pool;
1556 break :name ip.getNav(nav).fqn.toSlice(ip);
1577 break :name ip.getNav(nmi.navIndex(elf)).fqn.toSlice(ip);
15571578 },
1558 .uav => |uav| std.fmt.bufPrint(&name, "{f}", .{
1559 Value.fromInterned(uav).fmtValue(.{ .zcu = elf.base.comp.zcu.?, .tid = tid }),
1579 .uav => |umi| std.fmt.bufPrint(&name, "{f}", .{
1580 Value.fromInterned(umi.uavValue(elf)).fmtValue(.{ .zcu = elf.base.comp.zcu.?, .tid = tid }),
15601581 }) catch &name,
15611582 }, 0);
15621583}
......@@ -1564,14 +1585,15 @@ fn idleProgNode(
15641585fn flushUav(
15651586 elf: *Elf,
15661587 pt: Zcu.PerThread,
1567 uav_val: InternPool.Index,
1588 umi: Node.UavMapIndex,
15681589 uav_align: InternPool.Alignment,
15691590 src_loc: Zcu.LazySrcLoc,
15701591) !void {
15711592 const zcu = pt.zcu;
15721593 const gpa = zcu.gpa;
15731594
1574 const si = try elf.uavSymbol(uav_val);
1595 const uav_val = umi.uavValue(elf);
1596 const si = umi.symbol(elf);
15751597 const ni = ni: {
15761598 const sym = si.get(elf);
15771599 switch (sym.ni) {
......@@ -1581,7 +1603,7 @@ fn flushUav(
15811603 .alignment = uav_align.toStdMem(),
15821604 .moved = true,
15831605 });
1584 elf.nodes.appendAssumeCapacity(.{ .uav = uav_val });
1606 elf.nodes.appendAssumeCapacity(.{ .uav = umi });
15851607 sym.ni = ni;
15861608 switch (elf.symPtr(si)) {
15871609 inline else => |sym_ptr, class| sym_ptr.shndx =
......@@ -1598,36 +1620,34 @@ fn flushUav(
15981620 break :ni sym.ni;
15991621 };
16001622
1601 const size = size: {
1602 var nw: MappedFile.Node.Writer = undefined;
1603 ni.writer(&elf.mf, gpa, &nw);
1604 defer nw.deinit();
1605 codegen.generateSymbol(
1606 &elf.base,
1607 pt,
1608 src_loc,
1609 .fromInterned(uav_val),
1610 &nw.interface,
1611 .{ .atom_index = @intFromEnum(si) },
1612 ) catch |err| switch (err) {
1613 error.WriteFailed => return error.OutOfMemory,
1614 else => |e| return e,
1615 };
1616 break :size nw.interface.end;
1623 var nw: MappedFile.Node.Writer = undefined;
1624 ni.writer(&elf.mf, gpa, &nw);
1625 defer nw.deinit();
1626 codegen.generateSymbol(
1627 &elf.base,
1628 pt,
1629 src_loc,
1630 .fromInterned(uav_val),
1631 &nw.interface,
1632 .{ .atom_index = @intFromEnum(si) },
1633 ) catch |err| switch (err) {
1634 error.WriteFailed => return error.OutOfMemory,
1635 else => |e| return e,
16171636 };
1618
1619 const target_endian = elf.endian();
1637 const target_endian = elf.targetEndian();
16201638 switch (elf.symPtr(si)) {
16211639 inline else => |sym| sym.size =
1622 std.mem.nativeTo(@TypeOf(sym.size), @intCast(size), target_endian),
1640 std.mem.nativeTo(@TypeOf(sym.size), @intCast(nw.interface.end), target_endian),
16231641 }
16241642 si.applyLocationRelocs(elf);
16251643}
16261644
1627fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lazy: link.File.LazySymbol, si: Symbol.Index) !void {
1645fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void {
16281646 const zcu = pt.zcu;
16291647 const gpa = zcu.gpa;
16301648
1649 const lazy = lmr.lazySymbol(elf);
1650 const si = lmr.symbol(elf);
16311651 const ni = ni: {
16321652 const sym = si.get(elf);
16331653 switch (sym.ni) {
......@@ -1639,8 +1659,8 @@ fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lazy: link.File.LazySymbol, si: Symbo
16391659 };
16401660 const ni = try elf.mf.addLastChildNode(gpa, sec_si.node(elf), .{ .moved = true });
16411661 elf.nodes.appendAssumeCapacity(switch (lazy.kind) {
1642 .code => .{ .lazy_code = lazy.ty },
1643 .const_data => .{ .lazy_const_data = lazy.ty },
1662 .code => .{ .lazy_code = @enumFromInt(lmr.index) },
1663 .const_data => .{ .lazy_const_data = @enumFromInt(lmr.index) },
16441664 });
16451665 sym.ni = ni;
16461666 switch (elf.symPtr(si)) {
......@@ -1655,34 +1675,30 @@ fn flushLazy(elf: *Elf, pt: Zcu.PerThread, lazy: link.File.LazySymbol, si: Symbo
16551675 break :ni sym.ni;
16561676 };
16571677
1658 const size = size: {
1659 var required_alignment: InternPool.Alignment = .none;
1660 var nw: MappedFile.Node.Writer = undefined;
1661 ni.writer(&elf.mf, gpa, &nw);
1662 defer nw.deinit();
1663 try codegen.generateLazySymbol(
1664 &elf.base,
1665 pt,
1666 Type.fromInterned(lazy.ty).srcLocOrNull(pt.zcu) orelse .unneeded,
1667 lazy,
1668 &required_alignment,
1669 &nw.interface,
1670 .none,
1671 .{ .atom_index = @intFromEnum(si) },
1672 );
1673 break :size nw.interface.end;
1674 };
1675
1676 const target_endian = elf.endian();
1678 var required_alignment: InternPool.Alignment = .none;
1679 var nw: MappedFile.Node.Writer = undefined;
1680 ni.writer(&elf.mf, gpa, &nw);
1681 defer nw.deinit();
1682 try codegen.generateLazySymbol(
1683 &elf.base,
1684 pt,
1685 Type.fromInterned(lazy.ty).srcLocOrNull(pt.zcu) orelse .unneeded,
1686 lazy,
1687 &required_alignment,
1688 &nw.interface,
1689 .none,
1690 .{ .atom_index = @intFromEnum(si) },
1691 );
1692 const target_endian = elf.targetEndian();
16771693 switch (elf.symPtr(si)) {
16781694 inline else => |sym| sym.size =
1679 std.mem.nativeTo(@TypeOf(sym.size), @intCast(size), target_endian),
1695 std.mem.nativeTo(@TypeOf(sym.size), @intCast(nw.interface.end), target_endian),
16801696 }
16811697 si.applyLocationRelocs(elf);
16821698}
16831699
16841700fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) !void {
1685 const target_endian = elf.endian();
1701 const target_endian = elf.targetEndian();
16861702 const file_offset = ni.fileLocation(&elf.mf, false).offset;
16871703 const node = elf.getNode(ni);
16881704 switch (node) {
......@@ -1738,11 +1754,8 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) !void {
17381754 .nav, .uav, .lazy_code, .lazy_const_data => {
17391755 const si = switch (node) {
17401756 else => unreachable,
1741 .nav => |nav| elf.navs.get(nav),
1742 .uav => |uav| elf.uavs.get(uav),
1743 .lazy_code => |ty| elf.lazy.getPtr(.code).map.get(ty),
1744 .lazy_const_data => |ty| elf.lazy.getPtr(.const_data).map.get(ty),
1745 }.?;
1757 inline .nav, .uav, .lazy_code, .lazy_const_data => |mi| mi.symbol(elf),
1758 };
17461759 switch (elf.shdrSlice()) {
17471760 inline else => |shdr, class| {
17481761 const sym = @field(elf.symPtr(si), @tagName(class));
......@@ -1773,7 +1786,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) !void {
17731786}
17741787
17751788fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) !void {
1776 const target_endian = elf.endian();
1789 const target_endian = elf.targetEndian();
17771790 _, const size = ni.location(&elf.mf).resolve(&elf.mf);
17781791 const node = elf.getNode(ni);
17791792 switch (node) {
......@@ -1957,65 +1970,74 @@ pub fn printNode(
19571970 indent: usize,
19581971) !void {
19591972 const node = elf.getNode(ni);
1960 const mf_node = &elf.mf.nodes.items[@intFromEnum(ni)];
1961 const off, const size = mf_node.location().resolve(&elf.mf);
19621973 try w.splatByteAll(' ', indent);
19631974 try w.writeAll(@tagName(node));
19641975 switch (node) {
19651976 else => {},
19661977 .section => |si| try w.print("({s})", .{elf.sectionName(si)}),
1967 .nav => |nav_index| {
1978 .nav => |nmi| {
19681979 const zcu = elf.base.comp.zcu.?;
19691980 const ip = &zcu.intern_pool;
1970 const nav = ip.getNav(nav_index);
1981 const nav = ip.getNav(nmi.navIndex(elf));
19711982 try w.print("({f}, {f})", .{
19721983 Type.fromInterned(nav.typeOf(ip)).fmt(.{ .zcu = zcu, .tid = tid }),
19731984 nav.fqn.fmt(ip),
19741985 });
19751986 },
1976 .uav => |uav| {
1987 .uav => |umi| {
19771988 const zcu = elf.base.comp.zcu.?;
1978 const val: Value = .fromInterned(uav);
1989 const val: Value = .fromInterned(umi.uavValue(elf));
19791990 try w.print("({f}, {f})", .{
19801991 val.typeOf(zcu).fmt(.{ .zcu = zcu, .tid = tid }),
19811992 val.fmtValue(.{ .zcu = zcu, .tid = tid }),
19821993 });
19831994 },
1995 inline .lazy_code, .lazy_const_data => |lmi| try w.print("({f})", .{
1996 Type.fromInterned(lmi.lazySymbol(elf).ty).fmt(.{
1997 .zcu = elf.base.comp.zcu.?,
1998 .tid = tid,
1999 }),
2000 }),
19842001 }
1985 try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{
1986 @intFromEnum(ni),
1987 off,
1988 size,
1989 mf_node.flags.alignment.toByteUnits(),
1990 if (mf_node.flags.fixed) " fixed" else "",
1991 if (mf_node.flags.moved) " moved" else "",
1992 if (mf_node.flags.resized) " resized" else "",
1993 if (mf_node.flags.has_content) " has_content" else "",
1994 });
1995 var child_ni = mf_node.first;
1996 switch (child_ni) {
1997 .none => {
1998 const file_loc = ni.fileLocation(&elf.mf, false);
1999 if (file_loc.size == 0) return;
2000 var address = file_loc.offset;
2001 const line_len = 0x10;
2002 var line_it = std.mem.window(
2003 u8,
2004 elf.mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
2005 line_len,
2006 line_len,
2007 );
2008 while (line_it.next()) |line_bytes| : (address += line_len) {
2009 try w.splatByteAll(' ', indent + 1);
2010 try w.print("{x:0>8}", .{address});
2011 for (line_bytes) |byte| try w.print(" {x:0>2}", .{byte});
2012 try w.writeByte('\n');
2013 }
2014 },
2015 else => while (child_ni != .none) {
2016 try elf.printNode(tid, w, child_ni, indent + 1);
2017 child_ni = elf.mf.nodes.items[@intFromEnum(child_ni)].next;
2018 },
2002 {
2003 const mf_node = &elf.mf.nodes.items[@intFromEnum(ni)];
2004 const off, const size = mf_node.location().resolve(&elf.mf);
2005 try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{
2006 @intFromEnum(ni),
2007 off,
2008 size,
2009 mf_node.flags.alignment.toByteUnits(),
2010 if (mf_node.flags.fixed) " fixed" else "",
2011 if (mf_node.flags.moved) " moved" else "",
2012 if (mf_node.flags.resized) " resized" else "",
2013 if (mf_node.flags.has_content) " has_content" else "",
2014 });
2015 }
2016 var leaf = true;
2017 var child_it = ni.children(&elf.mf);
2018 while (child_it.next()) |child_ni| {
2019 leaf = false;
2020 try elf.printNode(tid, w, child_ni, indent + 1);
2021 }
2022 if (leaf) {
2023 const file_loc = ni.fileLocation(&elf.mf, false);
2024 if (file_loc.size == 0) return;
2025 var address = file_loc.offset;
2026 const line_len = 0x10;
2027 var line_it = std.mem.window(
2028 u8,
2029 elf.mf.contents[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
2030 line_len,
2031 line_len,
2032 );
2033 while (line_it.next()) |line_bytes| : (address += line_len) {
2034 try w.splatByteAll(' ', indent + 1);
2035 try w.print("{x:0>8} ", .{address});
2036 for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte});
2037 try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1);
2038 for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.');
2039 try w.writeByte('\n');
2040 }
20192041 }
20202042}
20212043
src/link/MappedFile.zig+50-20
......@@ -34,17 +34,28 @@ pub fn init(file: std.fs.File, gpa: std.mem.Allocator) !MappedFile {
3434 .writers = .{},
3535 };
3636 errdefer mf.deinit(gpa);
37 const size: u64, const blksize = if (is_windows)
38 .{ try windows.GetFileSizeEx(file.handle), 1 }
39 else stat: {
37 const size: u64, const block_size = stat: {
38 if (is_windows) {
39 var sbi: windows.SYSTEM_BASIC_INFORMATION = undefined;
40 break :stat .{
41 try windows.GetFileSizeEx(file.handle),
42 switch (windows.ntdll.NtQuerySystemInformation(
43 .SystemBasicInformation,
44 &sbi,
45 @sizeOf(windows.SYSTEM_BASIC_INFORMATION),
46 null,
47 )) {
48 .SUCCESS => @max(sbi.PageSize, sbi.AllocationGranularity),
49 else => std.heap.page_size_max,
50 },
51 };
52 }
4053 const stat = try std.posix.fstat(mf.file.handle);
4154 if (!std.posix.S.ISREG(stat.mode)) return error.PathAlreadyExists;
42 break :stat .{ @bitCast(stat.size), stat.blksize };
55 break :stat .{ @bitCast(stat.size), @max(std.heap.pageSize(), stat.blksize) };
4356 };
4457 mf.flags = .{
45 .block_size = .fromByteUnits(
46 std.math.ceilPowerOfTwoAssert(usize, @max(std.heap.pageSize(), blksize)),
47 ),
58 .block_size = .fromByteUnits(std.math.ceilPowerOfTwoAssert(usize, block_size)),
4859 .copy_file_range_unsupported = false,
4960 .fallocate_insert_range_unsupported = false,
5061 .fallocate_punch_hole_unsupported = false,
......@@ -90,9 +101,11 @@ pub const Node = extern struct {
90101 resized: bool,
91102 /// Whether this node might contain non-zero bytes.
92103 has_content: bool,
104 /// Whether a moved event on this node bubbles down to children.
105 bubbles_moved: bool,
93106 unused: @Type(.{ .int = .{
94107 .signedness = .unsigned,
95 .bits = 32 - @bitSizeOf(std.mem.Alignment) - 5,
108 .bits = 32 - @bitSizeOf(std.mem.Alignment) - 6,
96109 } }) = 0,
97110 };
98111
......@@ -136,6 +149,25 @@ pub const Node = extern struct {
136149 return &mf.nodes.items[@intFromEnum(ni)];
137150 }
138151
152 pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index {
153 return ni.get(mf).parent;
154 }
155
156 pub const ChildIterator = struct {
157 mf: *const MappedFile,
158 ni: Node.Index,
159
160 pub fn next(it: *ChildIterator) ?Node.Index {
161 const ni = it.ni;
162 if (ni == .none) return null;
163 it.ni = ni.get(it.mf).next;
164 return ni;
165 }
166 };
167 pub fn children(ni: Node.Index, mf: *const MappedFile) ChildIterator {
168 return .{ .mf = mf, .ni = ni.get(mf).first };
169 }
170
139171 pub fn childrenMoved(ni: Node.Index, gpa: std.mem.Allocator, mf: *MappedFile) !void {
140172 var child_ni = ni.get(mf).last;
141173 while (child_ni != .none) {
......@@ -147,9 +179,10 @@ pub const Node = extern struct {
147179 pub fn hasMoved(ni: Node.Index, mf: *const MappedFile) bool {
148180 var parent_ni = ni;
149181 while (parent_ni != Node.Index.root) {
150 const parent = parent_ni.get(mf);
151 if (parent.flags.moved) return true;
152 parent_ni = parent.parent;
182 const parent_node = parent_ni.get(mf);
183 if (!parent_node.flags.bubbles_moved) break;
184 if (parent_node.flags.moved) return true;
185 parent_ni = parent_node.parent;
153186 }
154187 return false;
155188 }
......@@ -163,12 +196,7 @@ pub const Node = extern struct {
163196 return node_moved.*;
164197 }
165198 fn movedAssumeCapacity(ni: Node.Index, mf: *MappedFile) void {
166 var parent_ni = ni;
167 while (parent_ni != Node.Index.root) {
168 const parent_node = parent_ni.get(mf);
169 if (parent_node.flags.moved) return;
170 parent_ni = parent_node.parent;
171 }
199 if (ni.hasMoved(mf)) return;
172200 const node = ni.get(mf);
173201 node.flags.moved = true;
174202 if (node.flags.resized) return;
......@@ -242,10 +270,10 @@ pub const Node = extern struct {
242270 var offset, const size = ni.location(mf).resolve(mf);
243271 var parent_ni = ni;
244272 while (true) {
245 const parent = parent_ni.get(mf);
246 if (set_has_content) parent.flags.has_content = true;
273 const parent_node = parent_ni.get(mf);
274 if (set_has_content) parent_node.flags.has_content = true;
247275 if (parent_ni == .none) break;
248 parent_ni = parent.parent;
276 parent_ni = parent_node.parent;
249277 offset += parent_ni.location(mf).resolve(mf)[0];
250278 }
251279 return .{ .offset = offset, .size = size };
......@@ -449,6 +477,7 @@ fn addNode(mf: *MappedFile, gpa: std.mem.Allocator, opts: struct {
449477 .moved = true,
450478 .resized = true,
451479 .has_content = false,
480 .bubbles_moved = opts.add_node.bubbles_moved,
452481 },
453482 .location_payload = location_payload,
454483 };
......@@ -471,6 +500,7 @@ pub const AddNodeOptions = struct {
471500 fixed: bool = false,
472501 moved: bool = false,
473502 resized: bool = false,
503 bubbles_moved: bool = true,
474504};
475505
476506pub fn addOnlyChildNode(
src/target.zig+1-1
......@@ -233,7 +233,7 @@ pub fn hasLldSupport(ofmt: std.Target.ObjectFormat) bool {
233233
234234pub fn hasNewLinkerSupport(ofmt: std.Target.ObjectFormat, backend: std.builtin.CompilerBackend) bool {
235235 return switch (ofmt) {
236 .elf => switch (backend) {
236 .elf, .coff => switch (backend) {
237237 .stage2_x86_64 => true,
238238 else => false,
239239 },
test/behavior/cast.zig-2
......@@ -1650,7 +1650,6 @@ test "coerce between pointers of compatible differently-named floats" {
16501650 if (builtin.zig_backend == .stage2_c and builtin.os.tag == .windows and !builtin.link_libc) return error.SkipZigTest;
16511651 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
16521652 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1653 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
16541653 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
16551654
16561655 if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
......@@ -2883,7 +2882,6 @@ test "@intFromFloat vector boundary cases" {
28832882 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest;
28842883 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
28852884 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
2886 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
28872885
28882886 const S = struct {
28892887 fn case(comptime I: type, unshifted_inputs: [2]f32, expected: [2]I) !void {
test/behavior/export_keyword.zig-1
......@@ -43,7 +43,6 @@ export fn testPackedStuff(a: *const PackedStruct, b: *const PackedUnion) void {
4343}
4444
4545test "export function alias" {
46 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
4746 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
4847
4948 _ = struct {
test/behavior/extern.zig-2
......@@ -16,7 +16,6 @@ export var a_mystery_symbol: i32 = 1234;
1616
1717test "function extern symbol" {
1818 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
19 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
2019 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
2120
2221 const a = @extern(*const fn () callconv(.c) i32, .{ .name = "a_mystery_function" });
......@@ -29,7 +28,6 @@ export fn a_mystery_function() i32 {
2928
3029test "function extern symbol matches extern decl" {
3130 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
32 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
3331 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
3432
3533 const S = struct {
test/behavior/floatop.zig-17
......@@ -158,7 +158,6 @@ test "cmp f80/c_longdouble" {
158158 if (builtin.zig_backend == .stage2_llvm) return error.SkipZigTest;
159159 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
160160 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
161 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
162161
163162 try testCmp(f80);
164163 try comptime testCmp(f80);
......@@ -283,7 +282,6 @@ test "vector cmp f80/c_longdouble" {
283282 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
284283 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
285284 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
286 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
287285
288286 try testCmpVector(f80);
289287 try comptime testCmpVector(f80);
......@@ -396,7 +394,6 @@ test "@sqrt f80/f128/c_longdouble" {
396394 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
397395 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
398396 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
399 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
400397
401398 if (builtin.os.tag == .freebsd) {
402399 // TODO https://github.com/ziglang/zig/issues/10875
......@@ -526,7 +523,6 @@ test "@sin f80/f128/c_longdouble" {
526523 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
527524 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
528525 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
529 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
530526
531527 try testSin(f80);
532528 comptime try testSin(f80);
......@@ -596,7 +592,6 @@ test "@cos f80/f128/c_longdouble" {
596592 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
597593 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
598594 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
599 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
600595
601596 try testCos(f80);
602597 try comptime testCos(f80);
......@@ -666,7 +661,6 @@ test "@tan f80/f128/c_longdouble" {
666661 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
667662 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
668663 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
669 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
670664
671665 try testTan(f80);
672666 try comptime testTan(f80);
......@@ -736,7 +730,6 @@ test "@exp f80/f128/c_longdouble" {
736730 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
737731 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
738732 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
739 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
740733
741734 try testExp(f80);
742735 try comptime testExp(f80);
......@@ -810,7 +803,6 @@ test "@exp2 f80/f128/c_longdouble" {
810803 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
811804 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
812805 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
813 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
814806
815807 try testExp2(f80);
816808 try comptime testExp2(f80);
......@@ -879,7 +871,6 @@ test "@log f80/f128/c_longdouble" {
879871 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
880872 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
881873 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
882 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
883874
884875 try testLog(f80);
885876 try comptime testLog(f80);
......@@ -946,7 +937,6 @@ test "@log2 f80/f128/c_longdouble" {
946937 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
947938 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
948939 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
949 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
950940
951941 try testLog2(f80);
952942 try comptime testLog2(f80);
......@@ -1019,7 +1009,6 @@ test "@log10 f80/f128/c_longdouble" {
10191009 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
10201010 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
10211011 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1022 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
10231012
10241013 try testLog10(f80);
10251014 try comptime testLog10(f80);
......@@ -1086,7 +1075,6 @@ test "@abs f80/f128/c_longdouble" {
10861075 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
10871076 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
10881077 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1089 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
10901078
10911079 try testFabs(f80);
10921080 try comptime testFabs(f80);
......@@ -1204,7 +1192,6 @@ test "@floor f80/f128/c_longdouble" {
12041192 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
12051193 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
12061194 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1207 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
12081195 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
12091196 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
12101197
......@@ -1295,7 +1282,6 @@ test "@ceil f80/f128/c_longdouble" {
12951282 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
12961283 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
12971284 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1298 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
12991285 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
13001286 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
13011287
......@@ -1388,7 +1374,6 @@ test "@trunc f80/f128/c_longdouble" {
13881374 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
13891375 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
13901376 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1391 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
13921377
13931378 if (builtin.zig_backend == .stage2_llvm and builtin.os.tag == .windows) {
13941379 // https://github.com/ziglang/zig/issues/12602
......@@ -1485,7 +1470,6 @@ test "neg f80/f128/c_longdouble" {
14851470 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
14861471 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
14871472 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1488 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
14891473
14901474 try testNeg(f80);
14911475 try comptime testNeg(f80);
......@@ -1741,7 +1725,6 @@ test "comptime calls are only memoized when float arguments are bit-for-bit equa
17411725test "result location forwarded through unary float builtins" {
17421726 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
17431727 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1744 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
17451728 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
17461729 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
17471730
test/behavior/import_c_keywords.zig-1
......@@ -31,7 +31,6 @@ test "import c keywords" {
3131 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest;
3232 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
3333 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest;
34 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
3534 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
3635
3736 try std.testing.expect(int == .c_keyword_variable);
test/behavior/math.zig+6-4
......@@ -1416,7 +1416,6 @@ test "remainder division" {
14161416 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
14171417 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
14181418 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1419 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
14201419 if (builtin.zig_backend == .stage2_c and builtin.cpu.arch.isArm()) return error.SkipZigTest;
14211420 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
14221421
......@@ -1425,6 +1424,8 @@ test "remainder division" {
14251424 return error.SkipZigTest;
14261425 }
14271426
1427 if (builtin.zig_backend == .stage2_x86_64 and builtin.object_format == .coff and builtin.abi != .gnu) return error.SkipZigTest;
1428
14281429 try comptime remdiv(f16);
14291430 try comptime remdiv(f32);
14301431 try comptime remdiv(f64);
......@@ -1496,9 +1497,10 @@ test "float modulo division using @mod" {
14961497 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
14971498 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
14981499 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1499 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
15001500 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
15011501
1502 if (builtin.zig_backend == .stage2_x86_64 and builtin.object_format == .coff and builtin.abi != .gnu) return error.SkipZigTest;
1503
15021504 try comptime fmod(f16);
15031505 try comptime fmod(f32);
15041506 try comptime fmod(f64);
......@@ -1686,7 +1688,6 @@ test "signed zeros are represented properly" {
16861688 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
16871689 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
16881690 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1689 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
16901691
16911692 const S = struct {
16921693 fn doTheTest() !void {
......@@ -1824,7 +1825,8 @@ test "float divide by zero" {
18241825 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
18251826 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
18261827 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
1827 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest;
1828
1829 if (builtin.zig_backend == .stage2_x86_64 and builtin.object_format == .coff and builtin.abi != .gnu) return error.SkipZigTest;
18281830
18291831 const S = struct {
18301832 fn doTheTest(comptime F: type, zero: F, one: F) !void {
test/behavior/multiple_externs_with_conflicting_types.zig-1
......@@ -14,7 +14,6 @@ test "call extern function defined with conflicting type" {
1414 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
1515 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
1616 if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest;
17 if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt == .coff) return error.SkipZigTest;
1817 if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest;
1918
2019 @import("conflicting_externs/a.zig").issue529(null);
test/behavior/x86_64/binary.zig+2-10
......@@ -5172,15 +5172,6 @@ test mulSaturate {
51725172 try test_mul_saturate.testIntVectors();
51735173}
51745174
5175inline fn multiply(comptime Type: type, lhs: Type, rhs: Type) Type {
5176 return lhs * rhs;
5177}
5178test multiply {
5179 const test_multiply = binary(multiply, .{});
5180 try test_multiply.testFloats();
5181 try test_multiply.testFloatVectors();
5182}
5183
51845175inline fn divide(comptime Type: type, lhs: Type, rhs: Type) Type {
51855176 return lhs / rhs;
51865177}
......@@ -5264,7 +5255,8 @@ inline fn mod(comptime Type: type, lhs: Type, rhs: Type) Type {
52645255 return @mod(lhs, rhs);
52655256}
52665257test mod {
5267 if (@import("builtin").object_format == .coff) return error.SkipZigTest;
5258 const builtin = @import("builtin");
5259 if (builtin.object_format == .coff and builtin.abi != .gnu) return error.SkipZigTest;
52685260 const test_mod = binary(mod, .{});
52695261 try test_mod.testInts();
52705262 try test_mod.testIntVectors();
test/incremental/add_decl+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45//#target=wasm32-wasi-selfhosted
test/incremental/add_decl_namespaced+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45//#target=wasm32-wasi-selfhosted
test/incremental/analysis_error_and_syntax_error+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/bad_import+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_embed_file+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_enum_tag_type+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_exports+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45
test/incremental/change_fn_type+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#update=initial version
test/incremental/change_generic_line_number+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=wasm32-wasi-selfhosted
34#update=initial version
45#file=main.zig
test/incremental/change_line_number+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=wasm32-wasi-selfhosted
34#update=initial version
45#file=main.zig
test/incremental/change_module+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_panic_handler+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#update=initial version
test/incremental/change_panic_handler_explicit+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#update=initial version
test/incremental/change_shift_op+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_struct_same_fields+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/change_zon_file+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45//#target=wasm32-wasi-selfhosted
test/incremental/change_zon_file_no_result_type+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45//#target=wasm32-wasi-selfhosted
test/incremental/compile_error_then_log+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/compile_log+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/delete_comptime_decls+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/dependency_on_type_of_inferred_global+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/fix_astgen_failure+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/function_becomes_inline+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#update=non-inline version
test/incremental/hello+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/make_decl_pub+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/modify_inline_fn+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/move_src+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/no_change_preserves_tag_names+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45//#target=wasm32-wasi-selfhosted
test/incremental/recursive_function_becomes_non_recursive+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/remove_enum_field+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/remove_invalid_union_backing_enum+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/temporary_parse_error+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/type_becomes_comptime_only+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted
test/incremental/unreferenced_error+1
......@@ -1,4 +1,5 @@
11#target=x86_64-linux-selfhosted
2#target=x86_64-windows-selfhosted
23#target=x86_64-linux-cbe
34#target=x86_64-windows-cbe
45#target=wasm32-wasi-selfhosted